From a296920e7419a73f8d280e4d0fff8d6550f293f5 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 6 Jan 2017 07:20:57 +0000 Subject: [PATCH] =?UTF-8?q?effect=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: svn://192.168.0.12/source@334 8346c931-da38-4b9b-9d4c-e48b93cbd075 --- EffectProcess/EffectProcess.pro.user | 25 ++- EffectProcess/seffectprocess.cpp | 2 - EffectProcess/seffectstatistics.cpp | 16 +- EffectProcess/snaverblog.cpp | 99 +++++---- EffectProcess/snaverblog.h | 1 + EffectProcess/sresultsender.cpp | 6 +- EffectUI/EffectUI.pro.user | 294 ++++++++++++++++++++++++++- EffectUI/sparentprocess.cpp | 4 +- EffectUI/srunningprocess.cpp | 2 +- EffectUI/stimer.cpp | 4 +- 10 files changed, 381 insertions(+), 72 deletions(-) diff --git a/EffectProcess/EffectProcess.pro.user b/EffectProcess/EffectProcess.pro.user index df685a6..8e32998 100644 --- a/EffectProcess/EffectProcess.pro.user +++ b/EffectProcess/EffectProcess.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -228,15 +228,18 @@ 13 14 - -1 + 2 - - - false - %{buildDir} - Custom Executable + EffectProcess - ProjectExplorer.CustomExecutableRunConfiguration + Qt4ProjectManager.Qt4RunConfiguration:C:/source/EffectProcess/EffectProcess.pro + true + + EffectProcess.pro + false + true + + C:/source/build-EffectProcess-Desktop_Qt_5_4_0_MSVC2013_32bit-Debug 3768 false true @@ -430,7 +433,7 @@ false true - C:/source/build-EffectProcess-Desktop_Qt_5_4_1_MSVC2013_64bit-Debug + 3768 false true @@ -624,7 +627,7 @@ false true - C:/source/build-EffectProcess-Desktop_Qt_5_5_1_MSVC2013_64bit3-Debug + 3768 false true @@ -1061,7 +1064,7 @@ Qt4ProjectManager.Qt4RunConfiguration:C:/source/EffectProcess/EffectProcess.pro true - "naver" "blog" "5" "http://blog.naver.com/nososo1/220702021045" + "naver" "blog" "22" "http://blog.naver.com/27056/220820109447" EffectProcess.pro false diff --git a/EffectProcess/seffectprocess.cpp b/EffectProcess/seffectprocess.cpp index 3e93144..377e204 100644 --- a/EffectProcess/seffectprocess.cpp +++ b/EffectProcess/seffectprocess.cpp @@ -65,5 +65,3 @@ bool SEffectProcess::send(const State_s1_effect& _result) { return m_pResultSender->send(_result); } - - diff --git a/EffectProcess/seffectstatistics.cpp b/EffectProcess/seffectstatistics.cpp index b060364..26b707b 100644 --- a/EffectProcess/seffectstatistics.cpp +++ b/EffectProcess/seffectstatistics.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include int getInteractionCount(const EffectData& _data) @@ -47,11 +48,12 @@ QString getReplyBuzz(const EffectData& _data) QDate firstDate = QDateTime::fromString(_data.body.data[ARTICLE_DATE].trimmed().replace("T", " ").replace(".","-").replace("- "," "), "yyyy-MM-dd hh:mm:ss").date(); QDate lastDate = QDate::currentDate(); - /* + + qDebug() << _data.body.data[ARTICLE_DATE]; qDebug() << "firstDate:" << firstDate.toString("yyyy-MM-dd"); qDebug() << "lastDate:" << lastDate.toString("yyyy-MM-dd"); - */ + for (QDate date = firstDate; date <= lastDate; date = date.addDays(1)) { @@ -67,12 +69,16 @@ QString getReplyBuzz(const EffectData& _data) continue; mapDate[date]++; } - QJsonObject jObj; + QJsonArray jArray; + for (auto iter = mapDate.cbegin(); iter != mapDate.cend(); ++iter) { - jObj[iter.key().toString("yyyyMMdd")] = iter.value(); + QJsonObject jObj; + jObj["date"] = iter.key().toString("MM-dd-yyyy"); + jObj["value"] = iter.value(); + jArray.append(jObj); } - QJsonDocument jDoc(jObj); + QJsonDocument jDoc(jArray); QString result = jDoc.toJson(QJsonDocument::Compact); return result; } diff --git a/EffectProcess/snaverblog.cpp b/EffectProcess/snaverblog.cpp index 4c60127..5518592 100644 --- a/EffectProcess/snaverblog.cpp +++ b/EffectProcess/snaverblog.cpp @@ -14,14 +14,14 @@ const int TIME = 3000;//3 second using std::placeholders::_1; -const QString SNaverBlog:: -scriptArticleDate = "function findArticledate() {" - "var tt = frames['mainFrame'].document.querySelector('p.date.fil5.pcol2._postAddDate');" - "if(!tt)" - " tt = frames['mainFrame'].document.querySelector('span.se_publishDate.pcol2.fil5');" - "return tt.textContent;" - "}" - "findArticledate();"; +const QString SNaverBlog::scriptArticleDate = + "function findArticledate() {" + "var tt = frames['mainFrame'].document.querySelector('p.date.fil5.pcol2._postAddDate'); " + "if(!tt) " + " tt = frames['mainFrame'].document.querySelector('span.se_publishDate.pcol2.fil5'); " + "return tt.innerText;" + "} " + "findArticledate();"; const QString SNaverBlog::scriptArticleTitle = "function findArticletitle() " @@ -356,6 +356,23 @@ void SNaverBlog::initConnect() m_pWebPage->settings()->setAttribute(QWebEngineSettings::AutoLoadImages, false); } +QString SNaverBlog::changeUrl(const QString& _url) +{ + QStringList strList = _url.split("/"); + QString strOut = ""; + if ((strList.size() > 3) && strList.at(2).contains("blog.me")) //id.blog.me + { + strOut = "http://blog.naver.com/"; + strOut += strList.at(2).split(".").at(0); + strOut += "/"; + strOut += strList.at(3); + return strOut; + } + else + return _url; +} + + void SNaverBlog::slotAlert(const QString& msg) { emit signalError(E_ERROR_CODE::DELETED_URL_ERROR, msg.toUtf8()); @@ -365,7 +382,8 @@ void SNaverBlog::go(const QString &_url, E_CRAWL_MODE _mode) { //qDebug() << _url; m_eMode = _mode; - QUrl url(_url.trimmed()); + + QUrl url(changeUrl(_url)); if (_mode == E_CRAWL_MODE::BODY) m_strUrl = _url.trimmed(); /* @@ -423,6 +441,7 @@ void SNaverBlog::slotLoadFinished(bool ok) qDebug() << "reply loadfinished"; m_pWebPage->runJavaScript(scriptReply, std::bind(&SNaverBlog::cbReply, this, _1)); + /* m_pWebPage->toHtml([](const QString &str){ QFile file("reply.html"); @@ -434,6 +453,7 @@ void SNaverBlog::slotLoadFinished(bool ok) file.close(); }); + */ break; } } @@ -496,7 +516,7 @@ DataForm SNaverBlog::getBody() bool SNaverBlog::check(const DataForm &_data) { if (_data.data[ARTICLE_NICKNAME].isEmpty() && - (_data.data[ARTICLE_DATE] == "1990-01-01 00:00" || + (_data.data[ARTICLE_DATE] == "1990-01-01 00:00:00" || _data.data[ARTICLE_DATE].isEmpty())) return false; else @@ -512,35 +532,30 @@ QString SNaverBlog::makeReplyUrl(const QString& _url) { QStringList strList = _url.split("/"); QString strOut = ""; - try + if((strList.size() > 4) && (strList.at(2).compare("blog.naver.com") == 0)) { - if(strList.at(2).compare("blog.naver.com") == 0) - { - //strOut = _strUrl; - strOut = "http://blog.naver.com"; - strOut += "/CommentList.nhn?blogId="; - strOut += strList.at(3); - strOut += "&logNo="; - strOut += strList.at(4); - strOut += "¤tPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false"; - } - else //id.blog.me - { - strOut = "http://blog.naver.com/"; - //strOut += strList.at(2).split(".").at(0); - //strOut += "/"; - //strOut += strList.at(3); - strOut += "/CommentList.nhn?blogId="; - strOut += strList.at(2).split(".").at(0); - strOut += "&logNo="; - strOut += strList.at(3); - strOut += "¤tPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false"; - } + //strOut = _strUrl; + strOut = "http://blog.naver.com"; + strOut += "/CommentList.nhn?blogId="; + strOut += strList.at(3); + strOut += "&logNo="; + strOut += strList.at(4); + strOut += "¤tPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false"; } - catch (...) + else if ((strList.size() > 3) && strList.at(2).contains("blog.me")) //id.blog.me { - emit signalError(E_ERROR_CODE::OUT_DATED_CRAWLER, "Check ReplyUrl\n" + strOut); + strOut = "http://blog.naver.com/"; + //strOut += strList.at(2).split(".").at(0); + //strOut += "/"; + //strOut += strList.at(3); + strOut += "/CommentList.nhn?blogId="; + strOut += strList.at(2).split(".").at(0); + strOut += "&logNo="; + strOut += strList.at(3); + strOut += "¤tPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false"; } + else + emit signalError(E_ERROR_CODE::OUT_DATED_CRAWLER, "Check Body and ReplyUrl\n" + strOut); return strOut; } @@ -557,6 +572,20 @@ void SNaverBlog::cbBodyArticleDate(const QVariant& _result) strDate += ":00"; else strDate = "1990-01-01 00:00:00"; + + /* + qDebug() << "Body Article Date : " << strDate; + + if (strDate.trimmed().isEmpty() && (Retry-- > 0)) + { + QTimer::singleShot(TIME, this, [this](){ + this->m_pWebPage->runJavaScript(scriptArticleDate, std::bind(&SNaverBlog::cbBodyArticleDate, this, _1)); + }); + return; + } + */ + + m_efData.body.data[ARTICLE_DATE] = strDate; ok = doneBodyCrawler(E_FUNC_ARTICLE_DATE); //qDebug() << "done articledate"; diff --git a/EffectProcess/snaverblog.h b/EffectProcess/snaverblog.h index e92e406..8620c3d 100644 --- a/EffectProcess/snaverblog.h +++ b/EffectProcess/snaverblog.h @@ -42,6 +42,7 @@ public: private: QString makeReplyUrl(const QString& _url); + QString changeUrl(const QString& _url); void initConnect(); bool check(const DataForm& _data); bool check(const QVector& _data); diff --git a/EffectProcess/sresultsender.cpp b/EffectProcess/sresultsender.cpp index 5130c12..489513d 100644 --- a/EffectProcess/sresultsender.cpp +++ b/EffectProcess/sresultsender.cpp @@ -85,12 +85,12 @@ bool SResultSender::send(const State_s1_effect& _result) QString strQuery = "insert into stats_s1_effect (" - "event_num,replybuzz,replycount,likecount,interactioncount,engagementcount,lastupdate_filter) " + "event_num,replybuzz,replycount,likecount,interactioncount,engagementcount,lastupdate_filter,status) " "values " - "(:event_num,:replybuzz,:replycount,:likecount,:interactioncount,:engagementcount,:lastupdate_filter) " + "(:event_num,:replybuzz,:replycount,:likecount,:interactioncount,:engagementcount,:lastupdate_filter,'OK') " "on duplicate key update replybuzz = :replybuzz, replycount = :replycount," "likecount = :likecount, interactioncount = :interactioncount," - "engagementcount = :engagementcount, lastupdate_filter = :lastupdate_filter"; + "engagementcount = :engagementcount, lastupdate_filter = :lastupdate_filter, status = 'OK'"; QSqlQuery query(db); query.prepare(strQuery); diff --git a/EffectUI/EffectUI.pro.user b/EffectUI/EffectUI.pro.user index 377a220..e7be7ea 100644 --- a/EffectUI/EffectUI.pro.user +++ b/EffectUI/EffectUI.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -8,7 +8,7 @@ ProjectExplorer.Project.ActiveTarget - 0 + 4 ProjectExplorer.Project.EditorSettings @@ -40,6 +40,7 @@ true false 0 + true true 0 8 @@ -73,9 +74,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -127,9 +128,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -188,6 +189,11 @@ 1 + false + false + 0 + + true false false @@ -227,11 +233,13 @@ EffectUI Qt4ProjectManager.Qt4RunConfiguration:C:/source/EffectUI/EffectUI.pro + true EffectUI.pro false false + C:/source/build-EffectUI-Desktop_Qt_5_4_0_MSVC2013_32bit-Debug 3768 false true @@ -260,9 +268,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -314,9 +322,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -375,6 +383,11 @@ 1 + false + false + 0 + + true false false @@ -446,9 +459,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -500,9 +513,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -561,6 +574,11 @@ 1 + false + false + 0 + + true false false @@ -600,11 +618,13 @@ EffectUI Qt4ProjectManager.Qt4RunConfiguration:C:/source/EffectUI/EffectUI.pro + true EffectUI.pro false false + 3768 false true @@ -633,9 +653,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -687,9 +707,9 @@ QtProjectManager.QMakeBuildStep false - false false + false false @@ -748,6 +768,11 @@ 1 + false + false + 0 + + true false false @@ -801,9 +826,256 @@ 1 + + ProjectExplorer.Project.Target.4 + + Desktop Qt 5.5.1 MSVC2013 64bit3 + Desktop Qt 5.5.1 MSVC2013 64bit3 + qt.55.win64_msvc2013_64_kit + 1 + 0 + 0 + + C:/source/build-EffectUI-Desktop_Qt_5_5_1_MSVC2013_64bit3-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + true + + false + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-EffectUI-Desktop_Qt_5_5_1_MSVC2013_64bit3-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + + false + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + + C:/source/build-EffectUI-Desktop_Qt_5_5_1_MSVC2013_64bit3-Profile + + + true + qmake + + QtProjectManager.QMakeBuildStep + true + + false + true + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Profile + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 3 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + false + false + 1000 + + true + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + EffectUI + + Qt4ProjectManager.Qt4RunConfiguration:C:/source/EffectUI/EffectUI.pro + true + + EffectUI.pro + false + + C:/source/build-EffectUI-Desktop_Qt_5_5_1_MSVC2013_64bit3-Release + 3768 + false + true + false + false + true + + 1 + + ProjectExplorer.Project.TargetCount - 4 + 5 ProjectExplorer.Project.Updater.FileVersion diff --git a/EffectUI/sparentprocess.cpp b/EffectUI/sparentprocess.cpp index 26c907b..2fc79a4 100644 --- a/EffectUI/sparentprocess.cpp +++ b/EffectUI/sparentprocess.cpp @@ -17,7 +17,7 @@ const QString PYTHON = "python3"; #endif -const int TIMEOUT = 18; // 3 minutes +const int TIMEOUT = 9; // 3 minutes const QString SPLIT_KEYWORD = "!@#"; const QString PYTAPPNAME = "effectprocess.py"; const QString NAVER_BLOG_1 = "blog.naver.com/"; @@ -206,6 +206,7 @@ void SParentProcess::update() if (++nTime >= TIMEOUT) { setInnerMode(E_INNER_RUN_MODE::MODE_WAIT_HANGING); + m_pProcess->kill(); } @@ -421,7 +422,6 @@ SError SParentProcess::analyzeStdOut(const QString& _str) errorResult.str_error = _str.trimmed().left(200); return errorResult; - } } } diff --git a/EffectUI/srunningprocess.cpp b/EffectUI/srunningprocess.cpp index c72525e..c8466ba 100644 --- a/EffectUI/srunningprocess.cpp +++ b/EffectUI/srunningprocess.cpp @@ -3,7 +3,7 @@ #include namespace{ -const int TIMER_SECOND = 15000; // 10 second +const int TIMER_SECOND = 20000; // 20 second const int HALF_SECOND = 500; // 0.5 second } diff --git a/EffectUI/stimer.cpp b/EffectUI/stimer.cpp index 01dadfc..a6b9be3 100644 --- a/EffectUI/stimer.cpp +++ b/EffectUI/stimer.cpp @@ -22,10 +22,10 @@ void STimer::init() settings.setIniCodec("UTF-8"); settings.beginGroup("timer"); - QString strTimes = settings.value("time", "1").toString(); + QString strTimes = settings.value("time", "3").toString(); if (strTimes.trimmed().isEmpty()) - strTimes = "1"; + strTimes = "3"; qDebug() << strTimes; settings.endGroup();