diff --git a/ProxyProcess/ProxyProcess.pro.user b/ProxyProcess/ProxyProcess.pro.user index 907868a..eeba9c5 100644 --- a/ProxyProcess/ProxyProcess.pro.user +++ b/ProxyProcess/ProxyProcess.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId diff --git a/ProxyProcess/main.cpp b/ProxyProcess/main.cpp index c695a42..d60bc04 100644 --- a/ProxyProcess/main.cpp +++ b/ProxyProcess/main.cpp @@ -12,23 +12,11 @@ int main(int argc, char *argv[]) { //srand(time(0)); QApplication a(argc, argv); - QDateTime start = QDateTime::currentDateTime(); + //QDateTime start = QDateTime::currentDateTime(); a.setApplicationName(QString("Chrome")); a.setApplicationVersion(QString("39.0.2171.95")); - QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL"); - db.setHostName("bigbird.iptime.org"); - db.setUserName("admin"); - db.setPassword("admin123"); - db.setDatabaseName("concepters"); - - if (db.open() == false) - { - cout << "error : db open fail..."; - return 0; - } - SCrawler *process = new SCrawler; QObject::connect(process, SIGNAL(finished()), QApplication::instance(), SLOT(quit())); @@ -42,9 +30,9 @@ int main(int argc, char *argv[]) process->load(strArgv); a.exec(); - QDateTime end = QDateTime::currentDateTime(); + //QDateTime end = QDateTime::currentDateTime(); //QDateTime interval = end - start; - cout << start.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; - cout << end.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; + //cout << start.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; + //cout << end.toString("yyyy-MM-dd hh:mm:ss").toStdString() << endl; return 0; } diff --git a/ProxyProcess/scrawler.cpp b/ProxyProcess/scrawler.cpp index e777e09..deb051f 100644 --- a/ProxyProcess/scrawler.cpp +++ b/ProxyProcess/scrawler.cpp @@ -17,6 +17,9 @@ SCrawler::SCrawler():QObject() { m_page = new QWebPage; connect(m_page, SIGNAL(loadFinished(bool)), this, SLOT(saveResult(bool))); +// p_timer = new QTimer(this); +// connect(p_timer,SIGNAL(timeout()), this, SLOT(killProcess())); + QTimer::singleShot(90000, this, SLOT(killProcess())); } SCrawler::~SCrawler() @@ -32,9 +35,15 @@ void SCrawler::load(QStringList _strlistArgv) if(_strlistArgv.length() > 1) - m_strFolder = _strlistArgv.at(1) + "\\"; + m_strFolder = _strlistArgv.at(1).trimmed() + "\\"; + + if(_strlistArgv.length() > 2) + m_strLocation = _strlistArgv.at(2).trimmed(); + else + m_strLocation = "local"; m_page->currentFrame()->load(url); + } void SCrawler::UpdateError(QString _strError) @@ -60,9 +69,19 @@ void SCrawler::saveResult(bool ok) //SendIpList(strIpList); - Debug(m_strFolder + "proxy.txt",strIpList); + if((m_strLocation.compare("local") == 0) || (m_strLocation.compare("both") == 0)) + { + if(Debug(m_strFolder + "proxy.txt",strIpList) == false) + return; + } + if((m_strLocation.compare("db") == 0) || (m_strLocation.compare("both") == 0)) + { + if(SendIpList(strIpList) == false) + return; + } emit finished(); + cout << endl << "ok"; } int SCrawler::GetNumber(QString _str) @@ -76,16 +95,21 @@ int SCrawler::GetNumber(QString _str) return strNumber.toInt(); } -void SCrawler::Debug(QString _strFilename,QString _strData) +bool SCrawler::Debug(QString _strFilename,QString _strData) { QFile file(_strFilename); QFile::remove(_strFilename); if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) + { // if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) - return; + cout << "savefail" << endl; + emit finished(); + return false; + } QTextStream out(&file); out << _strData; file.close(); + return true; } QString SCrawler::SqlString(QString _str) @@ -267,11 +291,50 @@ QString SCrawler::getIpList(const QWebElement _FindElement) bool SCrawler::SendIpList(QString _strIpList) { - QStringList strListIp = _strIpList.split("\n"); + QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL"); + db.setHostName("bigbird.iptime.org"); + db.setUserName("admin"); + db.setPassword("admin123"); + db.setDatabaseName("concepters"); + if (db.open() == false) + { + cout << "error : db open fail..." << endl; + cout << "dbcfail"; + emit finished(); + return false; + } - return false; + QSqlQuery sql; + //QString strQuery = "truncate table Proxy"; + QString strQuery = "delete from Proxy"; + QString strUtf8(strQuery.toUtf8()); + if (sql.exec(strUtf8) == false) + { + cout << "error : " << sql.lastError().text().toStdString(); + cout << endl << "dbdfail"; + //emit finished(); + //return; + } + + QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts); + + foreach(QString str, strListIpList) + { + strQuery = "insert into Proxy set Proxy='"; + strQuery += str.split(',').at(0).trimmed(); + strQuery += "', Port="; + strQuery += str.split(',').at(1).trimmed(); + strUtf8 = strQuery.toUtf8(); + if (sql.exec(strUtf8) == false) + { + cout << "error : " << sql.lastError().text().toStdString(); + cout << endl << "dbufail"; + return false; + } + } + return true; } QList SCrawler::FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength) @@ -289,3 +352,9 @@ QList SCrawler::FindAllMid(const QWebElement _FindElement,const QSt } return returnElements; } + +void SCrawler::killProcess() +{ + cout << endl << "timeout"; + emit finished(); +} diff --git a/ProxyProcess/scrawler.h b/ProxyProcess/scrawler.h index 11abeed..0366602 100644 --- a/ProxyProcess/scrawler.h +++ b/ProxyProcess/scrawler.h @@ -2,7 +2,8 @@ #define SCRAWLER_H #include - +#include +#include class SCrawler : public QObject { Q_OBJECT @@ -12,33 +13,36 @@ public: void load(QStringList _strlistArgv); void saveFile(); // static void Debug(QString _strFilename,QString _strData); - void Debug(QString _strFilename,QString _strData); + bool Debug(QString _strFilename,QString _strData); signals: void finished(); private slots: void saveResult(bool ok); + void killProcess(); private: int m_nSelect; //QString m_strTableReply; QString m_strReper; QString m_strKeywordID; - + QTimer *p_timer; QWebPage *m_page; QString m_strFile; QString m_strUrl; QString m_strTable; QString m_strFolder; + QString m_strLocation; bool m_bUse; bool m_bLast; bool m_bError; + +public: QString SqlString(QString _str); QString GetSafeUtf(QString _strData); int GetNumber(QString _str); bool SendIpList(QString _str); - QString getIpList(const QWebElement _FindElement); QWebElement Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind); QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength);