etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

git-svn-id: svn://192.168.0.12/source@331 8346c931-da38-4b9b-9d4c-e48b93cbd075

Browse Source
This commit is contained in:
admin
2016-12-26 10:26:49 +00:00
parent bcecd771c6
commit ea4a3c8f11
4 changed files with 45 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
CrawlerProcess/scrawler.cpp
View File

@@ -196,7 +196,7 @@ void SCrawler::UpdateError(QString _strError)
void SCrawler::saveResult(bool ok)
{
//qDebug() << "saveResult";
// qDebug() << "saveResult";
if (!ok)
{
@@ -215,7 +215,12 @@ void SCrawler::saveResult(bool ok)
static bool loaded = false;
if(!loaded)
{
saveFrameNewsUrl(m_page->mainFrame());
loaded = true;
if(!saveFrameNewsUrl(m_page->mainFrame()))
{
loaded = false;
return;
}
bodydata.sendDB();
}
loaded = true;
@@ -1665,9 +1670,9 @@ void SCrawler::saveFrameNewsList(QWebFrame *frame)
}
void SCrawler::saveFrameNewsUrl(QWebFrame *frame)
bool SCrawler::saveFrameNewsUrl(QWebFrame *frame)
{
if (m_bUse) return;
if (m_bUse) return true;
{
QString strQuery = "delete from ";
@@ -1683,7 +1688,7 @@ void SCrawler::saveFrameNewsUrl(QWebFrame *frame)
QString strTitle,strDate,strData,strPlatID,strPlatTitle,strlike;
QString strTitle,strDate,strData,strPlatID,strPlatTitle,strlike,strReply,strSympathy;
{
QWebElement element = Find(frame->documentElement(),"div","class","article_info");
{
@@ -1692,6 +1697,34 @@ void SCrawler::saveFrameNewsUrl(QWebFrame *frame)
}
strData = Find(frame->documentElement(),"div","id","articleBodyContents").toPlainText();
strlike = Find(frame->documentElement(),"div","class","u_likeit_module").toPlainText();
strReply = Find(frame->documentElement(), "span", "class", "lo_txt").toPlainText().replace(QRegExp("[\\D]"), "").trimmed(); // normal
strSympathy = frame->documentElement().findFirst("em.u_cnt").toPlainText().replace(QRegExp("[\\D]"), "").trimmed();
if (strReply.isEmpty())
{
strReply = Find(frame->documentElement(), "a", "class", "reply_count").toPlainText().replace(QRegExp("[\\D]"), "").trimmed(); // entertain
}
if (strReply.isEmpty())
{
strReply = Find(frame->documentElement(), "span", "id", "newsCommentCount").toPlainText().replace(QRegExp("[\\D]"), "").trimmed(); // sports
}
if (strReply.isEmpty())
{
strReply = Find(frame->documentElement(), "span", "class", "u_cbox_count").toPlainText().replace(QRegExp("[\\D]"), "").trimmed();
}
qDebug() << strReply << ", " << strSympathy;
if ((strReply.isEmpty() || (strReply == "0") || strSympathy.isEmpty() || (strSympathy == "0")) && (++m_nRetryCount < RETRY_MAX))
{
//qDebug() << "singleshot";
QTimer::singleShot(1000, this, SLOT(reloadPage()));
return false;
}
if (strSympathy.isEmpty())
strSympathy = "0";
//e​ntertainment
if (strTitle.isEmpty())
{
@@ -1709,6 +1742,7 @@ void SCrawler::saveFrameNewsUrl(QWebFrame *frame)
}
//sport
{
if(strTitle.isEmpty())
@@ -1789,9 +1823,10 @@ void SCrawler::saveFrameNewsUrl(QWebFrame *frame)
bodydata.setData("body", SCrawlerData::ARTICLE_FORM);
bodydata.setData(m_strUrl, SCrawlerData::ARTICLE_URL);
bodydata.setData(m_strKeywordID, SCrawlerData::KEYWORD_ID);
bodydata.setData(strReply, SCrawlerData::ARTICLE_ORDER);
bodydata.setData(strSympathy, SCrawlerData::REPLY_URL);
m_bUse = true;
return true;
}
bool SCrawler::saveFrameNewsComment(QWebFrame *frame)

2
CrawlerProcess/scrawler.h
View File

@@ -69,7 +69,7 @@ private:
void saveFrameDaumBlogComment(QWebFrame *frame);
void saveFrameDaumCafeUrl(QWebFrame *frame);
void saveFrameNewsList(QWebFrame *frame);
void saveFrameNewsUrl(QWebFrame *frame);
bool saveFrameNewsUrl(QWebFrame *frame);
bool saveFrameNewsComment(QWebFrame *frame);
int GetNumber(QString _str);

2
ProxyProcess/ProxyProcess.pro.user
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2016-10-25T11:19:44. -->
<!-- Written by QtCreator 3.3.0, 2016-12-23T19:09:12. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>

2
ProxyProcess/scrawler.cpp
View File

@@ -778,7 +778,7 @@ void SCrawler::getIpListFromGatherProxy()
if (webPageNavi.isNull())
{
m_bDone = true;
QTimer::singleShot(1000, this, SLOT(saveResultManual()));\
QTimer::singleShot(1000, this, SLOT(saveResultManual()));
//cout << "no webpageNavi";
return;
}