etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

block, kill process시 list 재수집

Browse Source 
git-svn-id: svn://192.168.0.12/source@54 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-03-16 10:13:07 +00:00
parent caa5d670b7
commit a95853ed61
1 changed files with 19 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
CrawlerProcess/scrawler.cpp
View File

@@ -360,6 +360,11 @@ QString SCrawler::GetSafeUtf(QString _strData)
void SCrawler::saveFrameList(QWebFrame *frame) void SCrawler::saveFrameList(QWebFrame *frame)
{ {
if (m_bUse == true) return; if (m_bUse == true) return;
QWebElement notFound = Find(frame->documentElement(),"div","id","notfound");
if(notFound.isNull() == false)
m_bLast = true;
QWebElement eleMain = Find(frame->documentElement(),"div","class","blog section _blogBase"); QWebElement eleMain = Find(frame->documentElement(),"div","class","blog section _blogBase");
QSqlQuery sql; QSqlQuery sql;
for (int i = 0; i < 10 ; i++) for (int i = 0; i < 10 ; i++)
@@ -369,9 +374,10 @@ void SCrawler::saveFrameList(QWebFrame *frame)
QString strUrl = Find(sub,"a","class","url").toPlainText(); QString strUrl = Find(sub,"a","class","url").toPlainText();
if (strUrl.isEmpty()) if (strUrl.isEmpty())
{ {
m_bLast = true;
//m_bLast = true;
m_bUse = true; m_bUse = true;
return; break;
} }
QStringList strList = strUrl.split('/'); QStringList strList = strUrl.split('/');
@@ -745,8 +751,11 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
if (m_bUse == true) return; if (m_bUse == true) return;
static int cz = 0; static int cz = 0;
// //Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
QWebElement notFound = Find(frame->documentElement(),"div","id","notfound");
if(notFound.isNull() == false)
m_bLast = true;
QWebElement eleMain = Find(frame->documentElement(),"div","class","cafe_article section _cafeBase"); QWebElement eleMain = Find(frame->documentElement(),"div","class","cafe_article section _cafeBase");
foreach(QWebElement eleSub,eleMain.findAll("li")) foreach(QWebElement eleSub,eleMain.findAll("li"))
@@ -1004,13 +1013,18 @@ void SCrawler::saveFrameDaumCafeList(QWebFrame *frame)
} }
{ {
bool b_last = false;
QWebElement paging_comm = Find(frame->documentElement(), "div", "class", "paging_comm");
QWebElement paging_inner = Find(paging_comm, "span", "class", "paging_inner");
b_last = Find(paging_inner, "a", "class", "ico_comm1 btn_page btn_next").isNull();
QWebElement total = Find(eleMain,"span","class","f_nb f_l"); QWebElement total = Find(eleMain,"span","class","f_nb f_l");
if (total.toPlainText().isEmpty()) {m_bError = true; return;} if (total.toPlainText().isEmpty()) {m_bError = true; return;}
total.toPlainText().split("/").size(); total.toPlainText().split("/").size();
QStringList strList = total.toPlainText().split("/").at(0).trimmed().split("-"); QStringList strList = total.toPlainText().split("/").at(0).trimmed().split("-");
int nNow = GetNumber(strList.at(strList.size() - 1)); int nNow = GetNumber(strList.at(strList.size() - 1));
int nNowFirst = GetNumber(strList.at(strList.size() - 2)); int nNowFirst = GetNumber(strList.at(strList.size() - 2));
if (nNow >= 1000 || (nNow - nNowFirst) < 9) if (nNow >= 1000 || (nNow - nNowFirst) < 9 || b_last)
m_bLast = true; m_bLast = true;
} }
} }