etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

최신

Browse Source 
git-svn-id: svn://192.168.0.12/source@128 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-05-28 09:53:26 +00:00
parent dc37b20426
commit 5bbf1143e5
4 changed files with 51 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
ProxyProcess/scrawler.cpp
View File

@@ -20,6 +20,7 @@ SCrawler::SCrawler():QObject()
// p_timer = new QTimer(this);
// connect(p_timer,SIGNAL(timeout()), this, SLOT(killProcess()));
QTimer::singleShot(90000, this, SLOT(killProcess()));
m_bCrawled = false;
}
SCrawler::~SCrawler()
@@ -29,10 +30,11 @@ SCrawler::~SCrawler()
void SCrawler::load(QStringList _strlistArgv)
{
QUrl url(_strlistArgv.at(0));
QNetworkRequest *request = new QNetworkRequest;
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, false);
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, false);
m_strUrl = _strlistArgv.at(0);
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, true);
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, true);
if(_strlistArgv.length() > 1)
m_strFolder = _strlistArgv.at(1).trimmed() + "\\";
@@ -42,13 +44,11 @@ void SCrawler::load(QStringList _strlistArgv)
else
m_strLocation = "local";
m_page->currentFrame()->load(url);
m_page->mainFrame()->load(url);
}
void SCrawler::UpdateError(QString _strError)
{
m_bError = true;
}
@@ -56,14 +56,15 @@ void SCrawler::saveResult(bool ok)
{
if (!ok)
{
std::cerr << "Failed loading " << qPrintable(m_page->mainFrame()->url().toString()) << std::endl;
cout << "Failed loading " << std::endl;
emit finished();
return;
}
QString strIpList;
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
strIpList = getIpListFromAss(p_parse);
QString strIpList = getIpList(p_parse);
//Debug("test.html",m_page->currentFrame()->toHtml());
//Debug("filtered.html", p_parse.toInnerXml());
@@ -194,7 +195,7 @@ QWebElement SCrawler::FindLeft(const QWebElement _FindElement,const QString _str
}
QString SCrawler::getIpList(const QWebElement _FindElement)
QString SCrawler::getIpListFromAss(const QWebElement _FindElement)
{
QString DISPLAYINLINE = "{display:inline}";
@@ -289,6 +290,29 @@ QString SCrawler::getIpList(const QWebElement _FindElement)
}
QString SCrawler::getIpListFromFreeProxy(const QWebElement _FindElement)
{
QWebElementCollection trs = _FindElement.findAll("tr");
QString totalResult;
for(int i = 1; i < trs.count(); i++)
{
QWebElementCollection tds = trs.at(i).findAll("td");
if(tds.count() < 2)
continue;
QString strip = tds.at(0).findFirst("a").toPlainText().trimmed();
QString strport = tds.at(1).toPlainText().trimmed();
totalResult += strip;
totalResult += ",";
totalResult += strport;
if(i < trs.count() - 1)
totalResult += "\n";
}
return totalResult;
}
bool SCrawler::SendIpList(QString _strIpList)
{
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
@@ -305,6 +329,15 @@ bool SCrawler::SendIpList(QString _strIpList)
return false;
}
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
if(strListIpList.size() < 1)
{
cout << "sitedown";
return false;
}
QSqlQuery sql;
//QString strQuery = "truncate table Proxy";
QString strQuery = "delete from Proxy";
@@ -318,7 +351,6 @@ bool SCrawler::SendIpList(QString _strIpList)
//return;
}
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
foreach(QString str, strListIpList)
{
@@ -358,3 +390,4 @@ void SCrawler::killProcess()
cout << endl << "timeout";
emit finished();
}