etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

git-svn-id: svn://192.168.0.12/source@174 8346c931-da38-4b9b-9d4c-e48b93cbd075

Browse Source
This commit is contained in:
admin
2015-08-17 08:36:18 +00:00
parent f4a0718962
commit af5294d607
2 changed files with 194 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

275
ProxyUI/widget.cpp
View File

@@ -1,14 +1,11 @@
#include "widget.h"
#include "ui_widget.h"
/*
#include <QSet>
#include <QRegExp>
#define UPDATE_TIMER 43200000
#define CHECK_TIMER 180000
#define MIN_PROXY 200
*/
#define UPDATE_TIMER 43200
#define CHECK_TIMER 1800
#define MIN_PROXY 200
#define CRAWLING_TIMER 10800000
Widget::Widget(QWidget *parent) :
QWidget(parent),
@@ -103,18 +100,23 @@ Widget::Widget(QWidget *parent) :
glayout->addWidget(p_labelTimerOnOff, 2, 4);
m_pResultList = new QListWidget;
m_pCrawlingResultList = new QListWidget;
//glayout->addWidget(m_pResultList, 5, 0, 1, 5);
glayout->addWidget(m_pResultList, 5, 0, 1, 5);
glayout->addWidget(m_pCrawlingResultList, 7, 0, 1, 5);
}
p_timer = new QTimer(this);
p_checkTimer = new QTimer(this);
connect(p_timer,SIGNAL(timeout()),this,SLOT(updateProxy()));
p_crawlingTimer = new QTimer(this);
connect(p_timer,SIGNAL(timeout()),this,SLOT(updateDB()));
connect(p_checkTimer,SIGNAL(timeout()),this,SLOT(checkProxy()));
connect(p_crawlingTimer, SIGNAL(timeout()), this, SLOT(crawlingProxy()));
connect(p_btnStart, SIGNAL(clicked()), this, SLOT(btnStart()));
connect(p_btnStop, SIGNAL(clicked()), this, SLOT(btnStop()));
connect(p_btnChoose, SIGNAL(clicked()), this, SLOT(btnChoose()));
@@ -127,7 +129,7 @@ Widget::Widget(QWidget *parent) :
hlayoutDefault->addLayout(vlayoutButton);
*/
setLayout(glayout);
resize(700, 400);
resize(700, 500);
}
Widget::~Widget()
@@ -149,11 +151,12 @@ void Widget::btnChoose()
void Widget::btnStart()
{
qDebug("btnStart()");
if(p_timer->isActive() && p_checkTimer->isActive())
if(p_timer->isActive() && p_checkTimer->isActive() && p_crawlingTimer->isActive())
return;
InsertLog("Proxy Update Executed");
updateProxy();
//updateProxy();
crawlingProxy();
if(!p_timer->isActive())
{
@@ -168,6 +171,11 @@ void Widget::btnStart()
p_checkTimer->start(CHECK_TIMER);
InsertLog("Proxy DB Check Timer Start");
}
if(!p_crawlingTimer->isActive())
{
p_crawlingTimer->start(CRAWLING_TIMER);
InsertCrawlingLog("Crawling Timer Start");
}
p_labelTimerOnOff->setText("Timer ON");
}
@@ -186,6 +194,11 @@ void Widget::btnStop()
p_checkTimer->stop();
InsertLog("Proxy DB Check Timer Timer Stop");
}
if(p_crawlingTimer->isActive())
{
p_crawlingTimer->stop();
InsertCrawlingLog("CrawlerTimer stop");
}
p_labelTimerOnOff->setText("Timer OFF");
p_labelNextTime->setText("");
m_index = 0;
@@ -193,23 +206,14 @@ void Widget::btnStop()
void Widget::update()
{
InsertLog("Update Executed");
if(p_checkTimer->isActive())
InsertCrawlingLog("Update Executed");
if(p_crawlingTimer->isActive())
{
p_checkTimer->stop();
InsertLog("Proxy DB Check Timer Timer Stop");
p_crawlingTimer->stop();
InsertCrawlingLog("Crawling Timer Stop");
}
QString strCheck;
if(p_chkDb->isChecked())
strCheck = "db";
if(p_chkLocal->isChecked())
strCheck = "local";
if(p_chkDb->isChecked() && p_chkLocal->isChecked())
strCheck = "both";
InsertLog(m_slIpUrl.at(m_index));
v_pro.start("ProxyProcess",QStringList() << m_slIpUrl.at(m_index++) << p_lineProxyFile->text() << strCheck);
InsertCrawlingLog(m_slIpUrl.at(m_index));
v_pro.start("ProxyProcess",QStringList() << m_slIpUrl.at(m_index) << p_lineProxyFile->text());
//p_labelStatus->setText("Updating");
}
@@ -218,86 +222,83 @@ void Widget::processFinished(int exitCode, QProcess::ExitStatus exitStatus)
//qDebug("process finished");
QProcess *pPro = (QProcess*)sender();
QString str = pPro->readAllStandardOutput();
QString strerr = pPro->readAllStandardError();
QString message;
if(str.right(2).compare("ok") == 0)
{
m_slIpPortList << str.split("\n");
m_slIpPortList.removeAt(m_slIpPortList.size() - 1);
InsertLog("Successfully Get Proxy : " + QString::number(str.split("\n").length() - 1));
if(str.right(3).compare("uok") == 0)
InsertLog("Success to update Ip list to DB");
else if(str.right(3).compare("fok") == 0)
InsertLog("Fail to update Ip list to DB");
m_slIpPortListTemp << str.split("\n");
m_slIpPortListTemp.removeAt(m_slIpPortListTemp.size() - 1);
InsertCrawlingLog("Successfully Get Proxy : " + QString::number(str.split("\n").length() - 1));
if(m_slIpPortListTemp.length() > 1)
{
mutex.lock();
m_slIpPortList << m_slIpPortListTemp;
m_slIpPortList.removeDuplicates();
mutex.unlock();
InsertCrawlingLog("Number of IP : " + QString::number(m_slIpPortList.length()));
}
/*
QStringList slIpPortList = str.split("\n");
slIpPortList.removeAt(slIpPortList.size() - 1);
QString site;
if(slIpPortList.length() > 0)
{
QStringList sltemp = slIpPortList.at(0).split(",");
if(sltemp.length() >= 2)
site = sltemp.at(2).trimmed();
}
if(strerr.isEmpty())
InsertLog("no error");
if(site.isEmpty())
{
}
else
InsertLog(strerr);
{
QRegExp re(QString("\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3},\\d{1,3},(?!" + site + ")"));
mutex.lock();
QStringList temp = m_slIpPortList.filter(re);
m_slIpPortList.clear();
m_slIpPortList = temp;
m_slIpPortList << slIpPortList;
InsertCrawlingLog("Number of Ip List : " + QString::number(m_slIpPortList.length()));
mutex.unlock();
}
InsertCrawlingLog("Successfully Get Proxy : " + QString::number(slIpPortList.length() - 1));
*/
}
else if(str.right(7).compare("dbcfail") == 0)
{
InsertLog("DB Connection Fail");
InsertCrawlingLog("DB Connection Fail");
}
else if(str.right(7).compare("dbufail") == 0)
{
InsertLog("DB Update Fail");
InsertCrawlingLog("DB Update Fail");
}
else if(str.right(8).compare("savefail") == 0)
{
InsertLog("File Save Failed");
InsertCrawlingLog("File Save Failed");
}
else if(str.right(7).compare("timeout") == 0)
{
InsertLog("Timeout");
InsertCrawlingLog("Timeout, skip this site");
skipSite();
}
else if(str.right(8).compare("sitedown") == 0)
{
InsertLog("The site may have some problem");
InsertCrawlingLog("the web site many have problem");
}
else
InsertLog("Unknown error occurs");
{
InsertCrawlingLog("Unknown error occurs");
}
pPro->kill();
m_index++;
if(m_index < m_slIpUrl.length())
{
update();
}
else
{
m_index = 0;
int nDuplicate = m_slIpPortList.removeDuplicates();
InsertLog("Remove Duplications : " + QString::number(nDuplicate));
/*
if(p_chkDb->isChecked())
{
if(SendIpList(m_slIpPortList))
{
InsertLog("Proxy DB Update Complete Successfully : " + QString::number(m_slIpPortList.length()));
}
else
{
InsertLog("Proxy DB Update Failed");
}
}
*/
if(m_slIpPortList.size() < 300)
{
m_slIpBackup << m_slIpPortList;
m_slIpBackup.removeDuplicates();
SendIpList(m_slIpBackup);
InsertLog("Insert IP Backup List to DB : " + QString::number(m_slIpBackup.length()));
}
else
{
m_slIpBackup.clear();
m_slIpBackup = m_slIpPortList;
InsertLog("Insert IP List to Backup List : " + QString::number(m_slIpBackup.length()));
}
if(p_chkLocal->isChecked())
{
@@ -310,12 +311,24 @@ void Widget::processFinished(int exitCode, QProcess::ExitStatus exitStatus)
InsertLog("proxy.txt Failed");
}
}
p_labelRecentTime->setText(QDateTime::currentDateTime().toLocalTime().toString("yyyy-MM-dd hh:mm:ss"));
if(!p_checkTimer->isActive())
{
p_checkTimer->start();
InsertLog("Proxy DB Check Timer Start");
mutex.lock();
if(m_slIpPortListTemp.length() > 500)
{
m_slIpPortList.clear();
m_slIpPortList = m_slIpPortListTemp;
}
mutex.unlock();
InsertCrawlingLog("Number of IP: " + QString::number(m_slIpPortList.length()));
}
p_labelRecentTime->setText(QDateTime::currentDateTime().toLocalTime().toString("yyyy-MM-dd hh:mm:ss"));
if(!p_crawlingTimer->isActive())
{
p_crawlingTimer->start(CRAWLING_TIMER);
InsertCrawlingLog("Crawler timer start");
}
InsertCrawlingLog("Reading proxylist.txt file is complete");
}
}
@@ -374,7 +387,7 @@ bool Widget::SendIpList(QStringList _slIpList)
strUtf8 = strQuery.toUtf8();
if (sql.exec(strUtf8) == false)
{
InsertLog(sql.lastQuery() + " is Failed");
InsertLog(sql.lastQuery() + "is Failed");
//return false;
}
}
@@ -400,7 +413,7 @@ bool Widget::Debug(QString _strFilename,QString _strData)
return true;
}
QStringList Widget::readProxyList(QString strPath)
QStringList Widget::readList(QString strPath)
{
QFile file(strPath);
QStringList slProxyList;
@@ -440,7 +453,6 @@ void Widget::checkProxy()
return;
}
QSqlQuery query;
QString strQuery;
@@ -460,6 +472,7 @@ void Widget::checkProxy()
if(result < MIN_PROXY)
{
/*
if(p_timer->isActive())
{
p_timer->stop();
@@ -467,6 +480,8 @@ void Widget::checkProxy()
}
p_timer->start(UPDATE_TIMER);
updateProxy();
*/
updateDB();
}
db.close();
}
@@ -495,32 +510,108 @@ void Widget::InsertLog(QString str)
QListWidgetItem* item = m_pResultList->takeItem(0);
delete item;
}
m_pResultList->setCurrentRow( m_pResultList->count() - 1 );
m_pResultList->setCurrentRow(m_pResultList->count() - 1 );
m_pResultList->repaint();
}
void Widget::updateProxy()
void Widget::InsertCrawlingLog(QString str)
{
QTime time = QTime::currentTime();
QString strOut = time.toString("[hh:mm:ss] ") + str;
m_pCrawlingResultList->addItem(strOut);
QDate date = QDate::currentDate();
//QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
QFile file(date.toString(Qt::ISODate)+ "_" + "proxyget" + ".log");
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
return;
QTextStream out(&file);
out << strOut << "\n";
file.close();
if (m_pCrawlingResultList->count() > 1024)
{
m_pCrawlingResultList->removeItemWidget(m_pResultList->item(0));
QListWidgetItem* item = m_pCrawlingResultList->takeItem(0);
delete item;
}
m_pCrawlingResultList->setCurrentRow( m_pCrawlingResultList->count() - 1 );
m_pCrawlingResultList->repaint();
}
void Widget::updateDB()
{
if(p_timer->isActive())
{
p_timer->stop();
InsertLog("update timer stop");
}
if(p_checkTimer->isActive())
{
p_checkTimer->stop();
InsertLog("check timer stop");
}
mutex.lock();
QStringList iplist = m_slIpPortList;
mutex.unlock();
if(SendIpList(iplist))
{
InsertLog("Success update ip list to ProxyDB : " + QString::number(iplist.length()));
}
else
{
InsertLog("Fail to update ip list to proxyDB");
}
if(!p_timer->isActive())
{
p_timer->start(UPDATE_TIMER);
InsertLog("update timer start");
}
if(!p_checkTimer->isActive())
{
p_checkTimer->start(CHECK_TIMER);
InsertLog("check timer start");
}
}
void Widget::crawlingProxy()
{
p_labelNextTime->setText(QDateTime::currentDateTime().toLocalTime().addSecs(p_lineTime->text().toInt()).toString("yyyy-MM-dd hh:mm:ss"));
InsertLog("update Proxy Timer Executed");
m_slIpPortList.clear();
InsertLog("Ip List Clear()");
m_slIpPortList.clear();
InsertCrawlingLog("crawlingProxy Executed");
//m_slIpPortList.clear();
m_slIpPortListTemp.clear();
m_slIpUrl.clear();
m_slIpUrl = getProxyList();
InsertLog("get Ip list from proxylist.txt");
m_index = 0;
InsertCrawlingLog("Ip URL List Clear()");
update();
}
QStringList Widget::getProxyList()
{
QStringList slIpUrl = readProxyList("proxylist.txt");
QStringList slIpUrl = readList("proxylist.txt");
if(slIpUrl.size() < 1)
{
slIpUrl << "http://proxylist.hidemyass.com/search-1305249#listable"
m_slIpUrl << "http://proxylist.hidemyass.com/search-1305249#listable"
<< "http://www.cybersyndrome.net/pla.html"
<< "https://nordvpn.com/free-proxy-list/?country=&ports=&speed%5B1%5D=on&proto%5BHTTP%5D=on&anon%5BHigh%5D=on&by=l&order=ASC&perpage=500";
}
return slIpUrl;
}
void Widget::skipSite()
{
QStringList temp = m_slIpUrl.at(m_index).split("/");
if(temp.size() > 3)
{
int nindex = m_slIpUrl.lastIndexOf(QRegExp(QString("http[s]?://" + temp.at(2) + "[\\S]*")));
if(nindex != -1)
m_index = nindex;
}
}

15
ProxyUI/widget.h
View File

@@ -17,6 +17,7 @@
#include <QTextStream>
#include <QFile>
#include <QListWidget>
#include <QMutex>
namespace Ui {
class Widget;
}
@@ -31,8 +32,11 @@ public:
bool SendIpList(QStringList _str);
bool Debug(QString _strFilename,QString _strData);
void InsertLog(QString str);
QStringList readProxyList(QString strPath);
void InsertCrawlingLog(QString str);
void skipSite();
QStringList getProxyList();
QStringList readList(QString strPath);
private:
Ui::Widget *ui;
@@ -44,7 +48,7 @@ private:
QLineEdit *p_lineUrl;
QLineEdit *p_lineTime;
QLineEdit *p_lineProxyFile;
QMutex mutex;
QLabel *p_labelUrl;
QLabel *p_labelTime;
@@ -60,15 +64,17 @@ private:
QFileDialog *p_FileDialog;
QListWidget *m_pResultList;
QListWidget *m_pCrawlingResultList;
QPushButton *p_btnStart;
QPushButton *p_btnStop;
QPushButton *p_btnChoose;
QProcess v_pro;
QTimer *p_timer;
QTimer *p_checkTimer;
QTimer *p_crawlingTimer;
QStringList m_slIpPortList;
QStringList m_slIpPortListTemp;
QStringList m_slIpUrl;
QStringList m_slIpBackup;
int m_index;
public slots:
void btnStart();
@@ -80,7 +86,8 @@ public slots:
void chkDb();
void chkLocal();
void checkProxy();
void updateProxy();
void crawlingProxy();
void updateDB();
};
#endif // WIDGET_H