diff --git a/ProxyProcess/ProxyProcess.pro b/ProxyProcess/ProxyProcess.pro new file mode 100644 index 0000000..4daeffb --- /dev/null +++ b/ProxyProcess/ProxyProcess.pro @@ -0,0 +1,19 @@ +#------------------------------------------------- +# +# Project created by QtCreator 2014-10-13T04:17:40 +# +#------------------------------------------------- +QT += webkitwidgets network widgets sql + +TARGET = ProxyProcess +CONFIG += console +CONFIG -= app_bundle + +TEMPLATE = app + +SOURCES += main.cpp \ + scrawler.cpp \ + + +HEADERS += \ + scrawler.h diff --git a/ProxyProcess/ProxyProcess.pro.user b/ProxyProcess/ProxyProcess.pro.user new file mode 100644 index 0000000..907868a --- /dev/null +++ b/ProxyProcess/ProxyProcess.pro.user @@ -0,0 +1,257 @@ + + + + + + EnvironmentId + {a05fc9db-6c77-45e7-9770-7cd31ad1131c} + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + true + false + true + + Cpp + + CppGlobal + + + + QmlJS + + QmlJSGlobal + + + 2 + UTF-8 + false + 4 + false + 80 + true + true + 1 + true + false + 0 + true + 0 + 8 + true + 1 + true + true + true + false + + + + ProjectExplorer.Project.PluginSettings + + + + ProjectExplorer.Project.Target.0 + + Desktop Qt 5.4.0 MinGW 32bit + Desktop Qt 5.4.0 MinGW 32bit + qt.54.win32_mingw491_kit + 1 + 0 + 0 + + C:/source/build-ProxyProcess-Desktop_Qt_5_4_0_MinGW_32bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-ProxyProcess-Desktop_Qt_5_4_0_MinGW_32bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + ProxyProcess + + Qt4ProjectManager.Qt4RunConfiguration:C:/source/ProxyProcess/ProxyProcess.pro + "http://proxylist.hidemyass.com/search-1305249#listable" + ProxyProcess.pro + false + true + + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.FileVersion + 18 + + + Version + 18 + + diff --git a/ProxyProcess/main.cpp b/ProxyProcess/main.cpp new file mode 100644 index 0000000..4114f80 --- /dev/null +++ b/ProxyProcess/main.cpp @@ -0,0 +1,38 @@ +#include "scrawler.h" + +#include +#include +#include +#include +#include +#include +using namespace std; + +int main(int argc, char *argv[]) +{ + //srand(time(0)); + QApplication a(argc, argv); + QDateTime start = QDateTime::currentDateTime(); + + a.setApplicationName(QString("Chrome")); + a.setApplicationVersion(QString("39.0.2171.95")); + + SCrawler *process = new SCrawler; + QObject::connect(process, SIGNAL(finished()), QApplication::instance(), SLOT(quit())); + + QWebSettings::setObjectCacheCapacities(0,0,0); + QWebSettings::clearMemoryCaches(); + + QStringList strArgv; + for (int i = 1; i < argc ; i++) + strArgv.push_back(argv[i]); + + process->load(strArgv); + + a.exec(); + QDateTime end = QDateTime::currentDateTime(); + //QDateTime interval = end - start; + cout << start.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; + cout << end.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; + return 0; +} diff --git a/ProxyProcess/scrawler.cpp b/ProxyProcess/scrawler.cpp new file mode 100644 index 0000000..fc7fe97 --- /dev/null +++ b/ProxyProcess/scrawler.cpp @@ -0,0 +1,267 @@ +#include "scrawler.h" +#include +#include +#include +#include +#include + +using namespace std; + +struct SProxyList +{ + QString m_strAddress; + int m_nPort; +}; + +SCrawler::SCrawler():QObject() +{ + m_page = new QWebPage; + connect(m_page, SIGNAL(loadFinished(bool)), this, SLOT(saveResult(bool))); +} + +SCrawler::~SCrawler() +{ +} + +void SCrawler::load(QStringList _strlistArgv) +{ + QUrl url(_strlistArgv.at(0)); + + m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, false); + m_page->settings()->setAttribute(QWebSettings::JavaEnabled, false); + + + if(_strlistArgv.length() > 1) + m_strFolder = _strlistArgv.at(1) + "\\"; + + m_page->currentFrame()->load(url); +} + +void SCrawler::UpdateError(QString _strError) +{ + + m_bError = true; +} + +void SCrawler::saveResult(bool ok) +{ + if (!ok) + { + std::cerr << "Failed loading " << qPrintable(m_page->mainFrame()->url().toString()) << std::endl; + emit finished(); + return; + } + + QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner"); + + //Debug("test.html",m_page->currentFrame()->toHtml()); + //Debug("filtered.html", p_parse.toInnerXml()); + + QWebElementCollection trs = p_parse.findAll("tr"); + QString DISPLAYINLINE = "{display:inline}"; + QString DISPLAYNONE = "{display:none}"; + QString totalResult; + + for(int i = 1; i < trs.count(); i++) + { + QWebElementCollection tds = trs.at(i).findAll("td"); + + QWebElement firstStyle = trs.at(i).findFirst("style"); + + QString strFirstStyle = firstStyle.toInnerXml().trimmed(); + + QStringList disinline; + QStringList disnone; + // {display:inline} {display:none} + QStringList strlist = strFirstStyle.split(".", QString::SkipEmptyParts); + + foreach(QString str, strlist) + { + if(str.trimmed().right(DISPLAYINLINE.length()).compare(DISPLAYINLINE) == 0) + { + disinline.append(str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length())); + // cout << DISPLAYINLINE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length()).toStdString() << endl; + } + else + { + disnone.append(str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length())); +// cout << DISPLAYNONE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length()).toStdString() << endl; + } + } + + QString strIpContents = tds.at(1).toInnerXml(); + foreach(QString str, disinline) + { + strIpContents.replace(str.trimmed(),"display: inline"); + } + foreach(QString str, disnone) + { + strIpContents.replace(str.trimmed(),"display:none"); + } + +// cout << strIpContents.toStdString() << endl; + QStringList strIpContentsList = strIpContents.split(QRegExp("[<>]"),QString::SkipEmptyParts); + QString result; + for(int j = 0; j < strIpContentsList.count(); j++) + { + QString str1 = strIpContentsList.at(j).trimmed(); + + if(str1.at(0) == '/') + continue; + if(str1.left(4).compare("span") == 0) + continue; + if(str1.left(3).compare("div") == 0) + continue; + if(str1.left(5).compare("style") == 0) + continue; + if(str1.mid(1,3).compare("dis") == 0) + continue; + + if(j > 0) + { + QString str2 = strIpContentsList.at(j-1).trimmed(); + if(str2.right(5).compare("none\"") == 0) + continue; + } + + result += str1; + } + + //cout << strIpContentsList.length() << endl; + cout << result.toStdString() << endl; + //Debug("proxy.txt",result); + QString port = tds.at(2).toInnerXml().trimmed(); + result += ("," + port); + totalResult += result; + if(i < trs.count() - 1) + totalResult += "\n"; + } + + Debug(m_strFolder + "proxy.txt",totalResult); + + emit finished(); +} + +int SCrawler::GetNumber(QString _str) +{ + QString strNumber; + for (int i = 0; i < _str.size();i++) + { + if (_str.at(i).isNumber()) + strNumber += _str.at(i); + } + return strNumber.toInt(); +} + +void SCrawler::Debug(QString _strFilename,QString _strData) +{ + QFile file(_strFilename); + QFile::remove(_strFilename); + if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) +// if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) + return; + QTextStream out(&file); + out << _strData; + file.close(); +} + +QString SCrawler::SqlString(QString _str) +{ + _str = _str.replace("'","\\'"); + _str = _str.replace("\"","\\\""); + return _str; +} + + +QString SCrawler::GetSafeUtf(QString _strData) +{ + QString str; + QChar *pch = _strData.data(); + + for (int i = 0; i < _strData.length(); i++) + { + if (pch[i].unicode() >= 12593 && pch[i].unicode() <= 12622) + str += pch[i]; + if (pch[i].unicode() >= 44032 && pch[i].unicode() <= 55203) + str += pch[i]; + if (pch[i].isDigit() || pch[i].isNumber() || pch[i].isSpace() || pch[i].isLower() || pch[i].isUpper() || pch[i].isSymbol() ) + str += pch[i]; + } + return str; +} + +QWebElement SCrawler::Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind) +{ + QWebElementCollection elements = _FindElement.findAll(_strElement); + foreach (QWebElement element, elements) + { + if (element.attribute(_strAttrib) == _strFind) + { + return element; + } + } + QWebElement element; + return element; +} + +QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength) +{ + QWebElementCollection elements = _FindElement.findAll(_strElement); + foreach (QWebElement element, elements) + { + QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength); + if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0) + { + return element; + } + } + QWebElement element; + return element; +} + +QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart) +{ + int _strLength = _strFind.length(); + return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart, _strLength); +} + +QWebElement SCrawler::FindRight(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind) +{ + QWebElementCollection elements = _FindElement.findAll(_strElement); + foreach (QWebElement element, elements) + { + QString str = element.attribute(_strAttrib).trimmed().right(_strFind.length()); + cout << "FindRight : " << str.toStdString() << endl; + cout << "FindRight right : " << _strFind.toStdString() << endl; + if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0) + { + return element; + } + } + QWebElement element; + return element; +} + +QWebElement SCrawler::FindLeft(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind) +{ + int _strStart = 0; + return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart); +} + + + +QList SCrawler::FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength) +{ + QWebElementCollection elements = _FindElement.findAll(_strElement); + QList returnElements = QList(); + + foreach (QWebElement element, elements) + { + QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength); + if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0) + { + returnElements.append(element); + } + } + return returnElements; +} diff --git a/ProxyProcess/scrawler.h b/ProxyProcess/scrawler.h new file mode 100644 index 0000000..c924103 --- /dev/null +++ b/ProxyProcess/scrawler.h @@ -0,0 +1,51 @@ +#ifndef SCRAWLER_H +#define SCRAWLER_H + +#include + +class SCrawler : public QObject +{ + Q_OBJECT +public: + SCrawler(); + ~SCrawler(); + void load(QStringList _strlistArgv); + void saveFile(); +// static void Debug(QString _strFilename,QString _strData); + void Debug(QString _strFilename,QString _strData); +signals: + void finished(); +private slots: + void saveResult(bool ok); +private: + int m_nSelect; + //QString m_strTableReply; + QString m_strReper; + QString m_strKeywordID; + + QWebPage *m_page; + QString m_strFile; + QString m_strUrl; + QString m_strTable; + QString m_strFolder; + + bool m_bUse; + bool m_bLast; + bool m_bError; + + QString SqlString(QString _str); + QString GetSafeUtf(QString _strData); + int GetNumber(QString _str); + + + QWebElement Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind); + QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength); + QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart); + QWebElement FindRight(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind); + QWebElement FindLeft(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind); + QList FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength); + QWebElementCollection Finds(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind); + void UpdateError(QString _strError); +}; + +#endif // SCRAWLER_H diff --git a/ProxyUI/ProxyUI.pro b/ProxyUI/ProxyUI.pro new file mode 100644 index 0000000..f2e4582 --- /dev/null +++ b/ProxyUI/ProxyUI.pro @@ -0,0 +1,20 @@ +#------------------------------------------------- +# +# Project created by QtCreator 2015-03-12T11:40:22 +# +#------------------------------------------------- + +QT += core gui + +greaterThan(QT_MAJOR_VERSION, 4): QT += widgets + +TARGET = ProxyUI +TEMPLATE = app + + +SOURCES += main.cpp\ + widget.cpp + +HEADERS += widget.h + +FORMS += widget.ui diff --git a/ProxyUI/main.cpp b/ProxyUI/main.cpp new file mode 100644 index 0000000..4d6c97b --- /dev/null +++ b/ProxyUI/main.cpp @@ -0,0 +1,11 @@ +#include "widget.h" +#include + +int main(int argc, char *argv[]) +{ + QApplication a(argc, argv); + Widget w; + w.show(); + + return a.exec(); +} diff --git a/ProxyUI/widget.cpp b/ProxyUI/widget.cpp new file mode 100644 index 0000000..07d1b76 --- /dev/null +++ b/ProxyUI/widget.cpp @@ -0,0 +1,160 @@ +#include "widget.h" +#include "ui_widget.h" + +Widget::Widget(QWidget *parent) : + QWidget(parent), + ui(new Ui::Widget) +{ + ui->setupUi(this); + + { + p_labelUrl = new QLabel(tr("URL: ")); + p_labelTime = new QLabel(tr("Update Time Period(sec): ")); + p_labelProxyFile = new QLabel(tr("Save Location: ")); + p_labelRecentTime = new QLabel(""); + p_labelNextTime = new QLabel(""); + + p_labelRecent = new QLabel(tr("Recent Updated Time: ")); + p_labelNext = new QLabel("Next Update Time: "); + + p_lineUrl = new QLineEdit("http://proxylist.hidemyass.com/search-1305249#listable"); + p_lineTime = new QLineEdit("1800"); + p_lineProxyFile = new QLineEdit(qApp->applicationDirPath()); + p_btnChoose = new QPushButton("Choose"); + + p_FileDialog = new QFileDialog(); + glayout = new QGridLayout(); + + p_lineUrl->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred); + p_lineTime->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred); + p_lineProxyFile->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred); + + glayout->addWidget(p_labelUrl, 0, 0); + glayout->addWidget(p_lineUrl, 0, 1, 1, 3); + glayout->addWidget(p_labelTime, 1, 0); + glayout->addWidget(p_lineTime, 1, 1, 1, 3); + glayout->addWidget(p_labelProxyFile, 2, 0); + glayout->addWidget(p_lineProxyFile, 2, 1, 1, 2); + glayout->addWidget(p_btnChoose, 2, 3); + glayout->addWidget(p_labelRecent , 3, 0); + glayout->addWidget(p_labelRecentTime, 3, 1); + glayout->addWidget(p_labelNext, 3, 2); + glayout->addWidget(p_labelNextTime, 3, 3); + /* + for(int i = 0; i < 4; i++) + { + hlayoutParam[i] = new QHBoxLayout(); + } + + hlayoutParam[0]->addWidget(p_labelUrl); + hlayoutParam[0]->addWidget(p_lineUrl); + hlayoutParam[1]->addWidget(p_labelTime); + hlayoutParam[1]->addWidget(p_lineTime); + hlayoutParam[2]->addWidget(p_labelProxyFile); + hlayoutParam[2]->addWidget(p_lineProxyFile); + hlayoutParam[2]->addWidget(p_btnChoose); + hlayoutParam[3]->addWidget(p_labelRecent); + hlayoutParam[3]->addWidget(p_labelRecentTime); + hlayoutParam[3]->addWidget(p_labelNext); + hlayoutParam[3]->addWidget(p_labelNextTime); + + vlayoutParam = new QVBoxLayout(); + + for(int i = 0; i < 4; i++) + { + vlayoutParam->addLayout(hlayoutParam[i]); + } + */ + } + + { + p_btnStart = new QPushButton(tr("Timer Start")); + p_btnStop = new QPushButton(tr("Timer Stop")); + p_labelTimerOnOff = new QLabel(tr("Timer OFF")); + p_labelStatus = new QLabel(tr("Waiting")); +// vlayoutButton = new QVBoxLayout(); + glayout->addWidget(p_btnStart,0, 4); + glayout->addWidget(p_btnStop, 1, 4); + glayout->addWidget(p_labelTimerOnOff, 2, 4); + glayout->addWidget(p_labelStatus,3 , 4); + QLabel *blank = new QLabel; + glayout->addWidget(blank, 4, 0, 1, 4); + } + + p_timer = new QTimer(this); + + connect(p_timer,SIGNAL(timeout()),this,SLOT(update())); + connect(p_btnStart, SIGNAL(clicked()), this, SLOT(btnStart())); + connect(p_btnStop, SIGNAL(clicked()), this, SLOT(btnStop())); + connect(p_btnChoose, SIGNAL(clicked()), this, SLOT(btnChoose())); + connect(&v_pro,SIGNAL(finished(int,QProcess::ExitStatus)),this,SLOT(processFinished())); + +/* + hlayoutDefault = new QHBoxLayout(this); + hlayoutDefault->addLayout(vlayoutParam); + hlayoutDefault->addLayout(glayout); + hlayoutDefault->addLayout(vlayoutButton); +*/ + setLayout(glayout); + resize(700, 200); +} + +Widget::~Widget() +{ + delete ui; +} + +void Widget::btnChoose() +{ + qDebug("btnChoose()"); + QString directory = QFileDialog::getExistingDirectory(this, + tr("Find Files"), QDir::currentPath()); + + if (!directory.isEmpty()) { + this->p_lineProxyFile->setText(directory); + } +} + +void Widget::btnStart() +{ + qDebug("btnStart()"); + if(p_timer->isActive()) + return; + + update(); + + p_timer->start(p_lineTime->text().toInt()*1000); + p_labelTimerOnOff->setText("Timer ON"); + +} + +void Widget::btnStop() +{ + qDebug("btnStop()"); + + if(p_timer->isActive()) + p_timer->stop(); + + p_labelTimerOnOff->setText("Timer OFF"); + p_labelNextTime->setText(""); + p_labelStatus->setText("Waiting"); +} + +void Widget::update() +{ + qDebug("update()"); + //QDateTime time = QDateTime::currentDateTime(); + + v_pro.start("ProxyProcess",QStringList() << p_lineUrl->text() << p_lineProxyFile->text() ); + //p_labelRecentTime->setText(time.toLocalTime().toString("yyyy-MM-dd hh:mm:ss")); + p_labelNextTime->setText(QDateTime::currentDateTime().toLocalTime().addSecs(p_lineTime->text().toInt()).toString("yyyy-MM-dd hh:mm:ss")); + p_labelStatus->setText("Updating"); + +} + +void Widget::processFinished() +{ + qDebug("process finished"); + p_labelRecentTime->setText(QDateTime::currentDateTime().toLocalTime().toString("yyyy-MM-dd hh:mm:ss")); + p_labelStatus->setText("Update Done"); +} diff --git a/ProxyUI/widget.h b/ProxyUI/widget.h new file mode 100644 index 0000000..27e5d0c --- /dev/null +++ b/ProxyUI/widget.h @@ -0,0 +1,65 @@ +#ifndef WIDGET_H +#define WIDGET_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ui { +class Widget; +} + +class Widget : public QWidget +{ + Q_OBJECT + +public: + explicit Widget(QWidget *parent = 0); + ~Widget(); + +private: + Ui::Widget *ui; + + QHBoxLayout *hlayoutParam[4]; + QHBoxLayout *hlayoutDefault; + QVBoxLayout *vlayoutButton; + QVBoxLayout *vlayoutParam; + QGridLayout *glayout; + QLineEdit *p_lineUrl; + QLineEdit *p_lineTime; + QLineEdit *p_lineProxyFile; + + + QLabel *p_labelUrl; + QLabel *p_labelTime; + QLabel *p_labelProxyFile; + QLabel *p_labelRecentTime; + QLabel *p_labelNextTime; + QLabel *p_labelTimerOnOff; + QLabel *p_labelRecent; + QLabel *p_labelNext; + QLabel *p_labelStatus; + + QFileDialog *p_FileDialog; + + QPushButton *p_btnStart; + QPushButton *p_btnStop; + QPushButton *p_btnChoose; + QProcess v_pro; + QTimer *p_timer; + +public slots: + void btnStart(); + void btnStop(); + void btnChoose(); + void update(); + void processFinished(); +}; + +#endif // WIDGET_H diff --git a/ProxyUI/widget.ui b/ProxyUI/widget.ui new file mode 100644 index 0000000..8c91283 --- /dev/null +++ b/ProxyUI/widget.ui @@ -0,0 +1,20 @@ + + Widget + + + + 0 + 0 + 400 + 300 + + + + Widget + + + + + + +