etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

실시간 중간 버젼

Browse Source 
git-svn-id: svn://192.168.0.12/source@78 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-04-17 05:22:20 +00:00
parent b20c30a93b
commit f553261f78
7 changed files with 272 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
CrawlerProcess/scrawler.cpp
View File

@@ -34,37 +34,7 @@ void SCrawler::load(QStringList _strlistArgv)
m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_CAFE_LIST;
m_strKeywordID = _strlistArgv[4];
/*
QFile file("proxy.txt");
if (file.open(QIODevice::ReadOnly | QIODevice::Text))
{
QVector <QStringList> vecProxy;
while (!file.atEnd())
{
QString str = QString(file.readLine());
if (str.isEmpty()) continue;
vecProxy.push_back(str.split(","));
}
if (vecProxy.size() > 0)
{
QStringList strList = vecProxy.at(rand()%vecProxy.size());
switch(strList.size())
{
case 1:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0))));
break;
case 2:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0),strList.at(1).toInt())));
break;
}
}
}*/
setProxy();
}
if (_strlistArgv[1] == "cafe_data")
@@ -79,15 +49,13 @@ void SCrawler::load(QStringList _strlistArgv)
m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_BLOG_LIST;
m_strKeywordID = _strlistArgv[4];
setProxy();
}
if (_strlistArgv[1] == "blog_url")
{
m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_BLOG_BODY;
//m_strReper = _strlistArgv[4];
m_nSelect = E_NAVER_BLOG_BODY;
m_bUse = true;
}
@@ -310,7 +278,10 @@ void SCrawler::saveFrameList(QWebFrame *frame)
QString strBlogMe = "blog.me";
if ((strList.at(0).compare("blog.naver.com") != 0 ) && (strList.at(0).right(strBlogMe.length()).compare(strBlogMe) != 0)) { cout << "x http://" << strUrl.toStdString() <<endl; continue; };
if ((strList.at(0).compare("blog.naver.com") != 0 ) && (strList.at(0).right(strBlogMe.length()).compare(strBlogMe) != 0))
{
cout << "x http://" << strUrl.toStdString() <<endl; continue;
}
if((strList.at(0).right(strBlogMe.length()).compare(strBlogMe) == 0))
{
@@ -321,12 +292,20 @@ void SCrawler::saveFrameList(QWebFrame *frame)
strUrl += strList.at(1);
}
/*
QString strQuery = "select article_url from ";
strQuery += m_strTable;
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery);
*/
if (sql.size() == 0 || sql.size() == -1)
QString strQuery = "delete from ";
strQuery += m_strTable;
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery);
//if (sql.size() == 0 || sql.size() == -1)
{
QString str = Find(sub,"a","class","txt84").toPlainText();
str = GetSafeUtf(str);
@@ -349,8 +328,8 @@ void SCrawler::saveFrameList(QWebFrame *frame)
else
cout << "o ";
}
else
cout << "v ";
//else
// cout << "v ";
cout << "http://" << strUrl.toStdString() << endl;
m_bUse = true;
}
@@ -741,11 +720,11 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
{
QSqlQuery sql;
QString strQuery = "select article_url from ";
QString strQuery = "delete from ";
strQuery += m_strTable;
strQuery += QString(" where article_url = '%1'").arg(strUrl);
sql.exec(strQuery);
if (sql.size() == 0 || sql.size() == -1)
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery);
//if (sql.size() == 0 || sql.size() == -1)
{
QString strQuery = QString("insert into ");
strQuery += m_strTable;
@@ -758,8 +737,8 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
cout << "o " << strUrl.toStdString() << endl;
}
}
else
cout << "v " << strUrl.toStdString() << endl;
//else
// cout << "v " << strUrl.toStdString() << endl;
}
m_bUse = true;
}
@@ -962,11 +941,10 @@ void SCrawler::saveFrameDaumCafeList(QWebFrame *frame)
{
QSqlQuery sql;
QString strQuery = "select article_url from ";
QString strQuery = "delete from ";
strQuery += m_strTable;
strQuery += QString(" where article_url = '%1'").arg(strUrl);
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery);
if (sql.size() == 0 || sql.size() == -1)
{
QString strQuery = QString("insert into ");
strQuery += m_strTable;
@@ -977,8 +955,8 @@ void SCrawler::saveFrameDaumCafeList(QWebFrame *frame)
else
cout << "o " << strUrl.toStdString() << endl;
}
else
cout << "v " << strUrl.toStdString() << endl;
//else
// cout << "v " << strUrl.toStdString() << endl;
}
m_bUse = true;
}