diff --git a/ProxyProcess/ProxyProcess.pro.user b/ProxyProcess/ProxyProcess.pro.user index 41fc35e..8753660 100644 --- a/ProxyProcess/ProxyProcess.pro.user +++ b/ProxyProcess/ProxyProcess.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -227,7 +227,7 @@ ProxyProcess Qt4ProjectManager.Qt4RunConfiguration:C:/source/ProxyProcess/ProxyProcess.pro - "http://www.proxylists.net/us_102_ext.html" + "http://free-proxy.cz/en/proxylist/country/all/http/speed/level2" ProxyProcess.pro false true diff --git a/ProxyProcess/scrawler.cpp b/ProxyProcess/scrawler.cpp index 5456f0f..aaa8ef8 100644 --- a/ProxyProcess/scrawler.cpp +++ b/ProxyProcess/scrawler.cpp @@ -61,6 +61,9 @@ void SCrawler::saveResult(bool ok) return; } + Debug("c:/data/test3.html", m_page->currentFrame()->toHtml()); + + QString strIpList; if(m_strUrl.contains("hidemyass")) { @@ -108,11 +111,30 @@ void SCrawler::saveResult(bool ok) if(strIpList.trimmed().size() > 0) cout << strIpList.trimmed().toStdString(); - + // success to crawling if(strIpList.size() > 8) { - cout << endl << "ok"; + // in case sending iplist to db + if(m_strLocation.compare("local") != 0) + { + // success to send ip list to db + if(SendIpList(strIpList)) + { + cout << endl << "uok"; + } + // fail to sen ip list to db + else + { + cout << endl << "fok"; + } + } + // in case not sending iplist to db + else + { + cout << endl << "ok"; + } } + // fail to crawling else { cout << "sitedown"; @@ -386,7 +408,7 @@ QString SCrawler::getIpListFromProxylists(const QWebElement _FindElement) return totalResult.trimmed(); } - +/* bool SCrawler::SendIpList(QString _strIpList) { QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL"); @@ -442,6 +464,65 @@ bool SCrawler::SendIpList(QString _strIpList) } return true; } +*/ + +bool SCrawler::SendIpList(QString _strIpList) +{ + + QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL"); + db.setHostName("bigbird.iptime.org"); + db.setUserName("admin"); + db.setPassword("admin123"); + db.setDatabaseName("concepters"); + + if (db.open() == false) + { + qDebug() << "DB open Failed in SendIpList()"; + return false; + } + + QSqlQuery sql; + //QString strQuery = "truncate table Proxy"; + /* + QString strQuery = "delete from Proxy"; + QString strUtf8(strQuery.toUtf8()); + + if (sql.exec(strUtf8) == false) + { + p_labelStatus->setText("Delete Query\n Fail"); + return false; + } + */ + QString strQuery; + QString strUtf8; + QStringList _slIpList = _strIpList.split("\n"); + foreach(QString str, _slIpList) + { + strQuery = "insert into Proxy set Proxy='"; + strQuery += str.split(',').at(0).trimmed(); + strQuery += "', Port="; + strQuery += str.split(',').at(1).trimmed(); + if(str.split(',').size() > 2) + { + strQuery += ", Source='"; + strQuery += str.split(',').at(2).trimmed(); + strQuery += "'"; + } + strUtf8 = strQuery.toUtf8(); + if (sql.exec(strUtf8) == false) + { + //InsertLog(sql.lastQuery() + "is Failed"); + //return false; + cerr << sql.lastQuery().toStdString() << endl; + //cout << "PP send ip list failed" << endl; + } + } + //cout << "PP send ip list : " << _slIpList.size() << endl; + db.close(); + return true; +} + + QList SCrawler::FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength) {