etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

최신

Browse Source 
git-svn-id: svn://192.168.0.12/source@128 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-05-28 09:53:26 +00:00
parent dc37b20426
commit 5bbf1143e5
4 changed files with 51 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ProxyProcess/ProxyProcess.pro.user
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2015-03-16T15:39:40. -->
<!-- Written by QtCreator 3.3.0, 2015-05-28T18:52:51. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>
@@ -61,7 +61,7 @@
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Desktop Qt 5.4.0 MinGW 32bit</value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Desktop Qt 5.4.0 MinGW 32bit</value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">qt.54.win32_mingw491_kit</value>
<value type="int" key="ProjectExplorer.Target.ActiveBuildConfiguration">1</value>
<value type="int" key="ProjectExplorer.Target.ActiveBuildConfiguration">0</value>
<value type="int" key="ProjectExplorer.Target.ActiveDeployConfiguration">0</value>
<value type="int" key="ProjectExplorer.Target.ActiveRunConfiguration">0</value>
<valuemap type="QVariantMap" key="ProjectExplorer.Target.BuildConfiguration.0">

4
ProxyProcess/main.cpp
View File

@@ -14,8 +14,8 @@ int main(int argc, char *argv[])
QApplication a(argc, argv);
//QDateTime start = QDateTime::currentDateTime();
a.setApplicationName(QString("Chrome"));
a.setApplicationVersion(QString("39.0.2171.95"));
a.setApplicationName(QString("Google Chrome"));
a.setApplicationVersion(QString("43.0.2357.81"));
SCrawler *process = new SCrawler;
QObject::connect(process, SIGNAL(finished()), QApplication::instance(), SLOT(quit()));

53
ProxyProcess/scrawler.cpp
View File

@@ -20,6 +20,7 @@ SCrawler::SCrawler():QObject()
// p_timer = new QTimer(this);
// connect(p_timer,SIGNAL(timeout()), this, SLOT(killProcess()));
QTimer::singleShot(90000, this, SLOT(killProcess()));
m_bCrawled = false;
}
SCrawler::~SCrawler()
@@ -29,10 +30,11 @@ SCrawler::~SCrawler()
void SCrawler::load(QStringList _strlistArgv)
{
QUrl url(_strlistArgv.at(0));
QNetworkRequest *request = new QNetworkRequest;
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, false);
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, false);
m_strUrl = _strlistArgv.at(0);
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, true);
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, true);
if(_strlistArgv.length() > 1)
m_strFolder = _strlistArgv.at(1).trimmed() + "\\";
@@ -42,13 +44,11 @@ void SCrawler::load(QStringList _strlistArgv)
else
m_strLocation = "local";
m_page->currentFrame()->load(url);
m_page->mainFrame()->load(url);
}
void SCrawler::UpdateError(QString _strError)
{
m_bError = true;
}
@@ -56,14 +56,15 @@ void SCrawler::saveResult(bool ok)
{
if (!ok)
{
std::cerr << "Failed loading " << qPrintable(m_page->mainFrame()->url().toString()) << std::endl;
cout << "Failed loading " << std::endl;
emit finished();
return;
}
QString strIpList;
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
strIpList = getIpListFromAss(p_parse);
QString strIpList = getIpList(p_parse);
//Debug("test.html",m_page->currentFrame()->toHtml());
//Debug("filtered.html", p_parse.toInnerXml());
@@ -194,7 +195,7 @@ QWebElement SCrawler::FindLeft(const QWebElement _FindElement,const QString _str
}
QString SCrawler::getIpList(const QWebElement _FindElement)
QString SCrawler::getIpListFromAss(const QWebElement _FindElement)
{
QString DISPLAYINLINE = "{display:inline}";
@@ -289,6 +290,29 @@ QString SCrawler::getIpList(const QWebElement _FindElement)
}
QString SCrawler::getIpListFromFreeProxy(const QWebElement _FindElement)
{
QWebElementCollection trs = _FindElement.findAll("tr");
QString totalResult;
for(int i = 1; i < trs.count(); i++)
{
QWebElementCollection tds = trs.at(i).findAll("td");
if(tds.count() < 2)
continue;
QString strip = tds.at(0).findFirst("a").toPlainText().trimmed();
QString strport = tds.at(1).toPlainText().trimmed();
totalResult += strip;
totalResult += ",";
totalResult += strport;
if(i < trs.count() - 1)
totalResult += "\n";
}
return totalResult;
}
bool SCrawler::SendIpList(QString _strIpList)
{
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
@@ -305,6 +329,15 @@ bool SCrawler::SendIpList(QString _strIpList)
return false;
}
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
if(strListIpList.size() < 1)
{
cout << "sitedown";
return false;
}
QSqlQuery sql;
//QString strQuery = "truncate table Proxy";
QString strQuery = "delete from Proxy";
@@ -318,7 +351,6 @@ bool SCrawler::SendIpList(QString _strIpList)
//return;
}
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
foreach(QString str, strListIpList)
{
@@ -358,3 +390,4 @@ void SCrawler::killProcess()
cout << endl << "timeout";
emit finished();
}

7
ProxyProcess/scrawler.h
View File

@@ -35,15 +35,16 @@ private:
bool m_bUse;
bool m_bLast;
bool m_bError;
bool m_bCrawled;
public:
QString SqlString(QString _str);
QString GetSafeUtf(QString _strData);
int GetNumber(QString _str);
bool SendIpList(QString _str);
QString getIpList(const QWebElement _FindElement);
void SearchChildFrame(QWebFrame *frame);
QString getIpListFromAss(const QWebElement _FindElement);
QString getIpListFromFreeProxy(const QWebElement _FindElement);
QWebElement Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind);
QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength);
QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart);