etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

PROXY 접속 추가

Browse Source 
git-svn-id: svn://192.168.0.12/source@18 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-02-03 08:15:33 +00:00
parent bd3da63bde
commit 035ccf60d7
2 changed files with 52 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CrawlerProcess/main.cpp
View File

@@ -3,11 +3,14 @@
#include <QCoreApplication>
#include <iostream>
#include <QSqlDatabase>
#include <QNetworkProxy>
#include <time.h>
using namespace std;
int main(int argc, char *argv[])
{
srand(time(0));
QApplication a(argc, argv);
a.setApplicationName(QString("Chrome"));
a.setApplicationVersion(QString("39.0.2171.95"));

55
CrawlerProcess/scrawler.cpp
View File

@@ -7,6 +7,12 @@
using namespace std;
struct SProxyList
{
QString m_strAddress;
int m_nPort;
};
SCrawler::SCrawler():QObject()
{
m_page = new QWebPage;
@@ -28,6 +34,33 @@ void SCrawler::load(QStringList _strlistArgv)
m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_CAFE_LIST;
m_strKeywordID = _strlistArgv[4];
QFile file("proxy.txt");
if (file.open(QIODevice::ReadOnly | QIODevice::Text))
{
QVector <QStringList> vecProxy;
while (!file.atEnd())
{
QString str = QString(file.readLine());
if (str.isEmpty()) continue;
vecProxy.push_back(str.split(","));
}
if (vecProxy.size() > 0)
{
QStringList strList = vecProxy.at(rand()%vecProxy.size());
switch(strList.size())
{
case 1:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0))));
break;
case 2:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0),strList.at(1).toInt())));
break;
}
}
}
}
if (_strlistArgv[1] == "cafe_data")
@@ -69,10 +102,14 @@ void SCrawler::load(QStringList _strlistArgv)
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages,false);
QNetworkRequest *request = new QNetworkRequest;
request->setUrl(url);
request->setRawHeader("Cache-Control","max-age=0");
/*
request->setRawHeader("Cache-Control","max-age=0, no-cache");
request->setRawHeader("Pragma","no-cache");
request->setRawHeader("Expires","Thu, 01 Jan 1970 16:00:00 GMT");
*/
if (m_strReper.isEmpty() == false && m_nSelect == E_NAVER_CAFE_DATA)
request->setRawHeader("Referer",m_strReper.toLocal8Bit());
request->setRawHeader("Accept-Language","ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4,zh-CN;q=0.2,zh;q=0.2");
request->setRawHeader("Accept-Language","ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4,zh-CN;q=0.2,zh;q=0.2");
m_page->mainFrame()->load(*request);
m_bLast = false;
m_bError = false;
@@ -115,7 +152,7 @@ void SCrawler::saveResult(bool ok)
{
case E_NAVER_CAFE_LIST:
case E_NAVER_BLOG_LIST:
if (m_bError) cout << "last";// block
if (m_bError) cout << "block";// block
if (m_bLast) cout << "last";
break;
case E_NAVER_BLOG_REPLY:
@@ -259,8 +296,10 @@ enum E_DATA
void SCrawler::saveFrameUrl(QWebFrame *frame)
{
//static int cz = 0;
//Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
/*
static int cz = 0;
Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
*/
QSqlQuery sql;
if (frame->frameName().compare(QString("BuddyConnectIframe")) == 0)
@@ -477,6 +516,10 @@ void SCrawler::saveFrameComment(QWebFrame *frame)
void SCrawler::saveFrameCafeList(QWebFrame *frame)
{
if (m_bUse == true) return;
static int cz = 0;
Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
QWebElement eleMain = Find(frame->documentElement(),"div","class","cafe_article section _cafeBase");
foreach(QWebElement eleSub,eleMain.findAll("li"))
{
@@ -531,7 +574,7 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
void SCrawler::saveFrameCafeUrl(QWebFrame *frame)
{
if (m_bUse) return;
if (m_bUse) return;
QWebElement other = Find(frame->documentElement(),"h1","class","d-none");
if (other.toPlainText().isEmpty() == false)