diff --git a/sfilterprocess/sconsumer.cpp b/sfilterprocess/sconsumer.cpp index de4da9b..d5b64b3 100644 --- a/sfilterprocess/sconsumer.cpp +++ b/sfilterprocess/sconsumer.cpp @@ -350,6 +350,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMapgetColumnIntArray(); QDate dateStart(2100,1,1),dateEnd(2000,1,1); + QDate dateStartBody(2100,1,1),dateEndBody(2000,1,1); + QDate dateStartReply(2100,1,1),dateEndReply(2000,1,1); + QList listBuzzy; // filtered { @@ -367,6 +370,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap dateEnd ) ? date : dateEnd; + dateStartBody = (date < dateStartBody ) ? date : dateStartBody; + dateEndBody = (date > dateEndBody ) ? date : dateEndBody; + if (stbuzzy.m_mapBodyBuzz.contains(date)) stbuzzy.m_mapBodyBuzz[date]++; @@ -382,6 +388,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap dateEnd ) ? date : dateEnd; + dateStartReply = (date < dateStartReply ) ? date : dateStartReply; + dateEndReply = (date > dateEndReply ) ? date : dateEndReply; + if (stbuzzy.m_mapReplyBuzz.contains(date)) stbuzzy.m_mapReplyBuzz[date]++; else @@ -424,6 +433,19 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMapgetAllCategoryNum(2))) + { + foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(2)].m_setRealAllUrl) + { + nTotalBodyCount++; + nTotalReplyCount += (*mapReply)[strUrl].size(); + } + nTotalCount = nTotalBodyCount + nTotalReplyCount; + } + + { QJsonArray jsArrayAll; QJsonArray jsArrayReply; @@ -477,14 +499,33 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMapgetPlatformForm(strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed()); stcommunity.platformname_num = m_pSInitializer->getPlatformName(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed()); stcommunity.title = strListBody[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + if (stcommunity.title.trimmed().isEmpty()) + stcommunity.title = strListBody[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); stcommunity.id = strListBody[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); stcommunity.url = m_SCommunity.getCommunityKey(strListBody).trimmed(); if(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed() == QString("daum") && strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == QString("cafe")) @@ -1230,7 +1232,7 @@ SDatagroup::stCommunity* SDatagroup::SCommunity::get(const QString &_strKey) if (m_mapCommunity.contains(_strKey.trimmed())) return &(m_mapCommunity[_strKey.trimmed()]); else - return 0; + return &(m_mapCommunity[" "]); } bool SDatagroup::SCommunity::changeTable(QSqlDatabase &_db, const int &_nCompany) diff --git a/sfilterprocess/sfilterprocess.cpp b/sfilterprocess/sfilterprocess.cpp index e0c8fae..854eea6 100644 --- a/sfilterprocess/sfilterprocess.cpp +++ b/sfilterprocess/sfilterprocess.cpp @@ -75,7 +75,7 @@ bool SFilterProcess::run() m_SDatagroup.makeTable(); m_SConsumer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_CONSUMER)); - m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER)); + //m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER)); m_SSpammer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_SPAMMER)); m_SPowercafe.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_POWERCAFE)); diff --git a/sfilterprocess/sfilterprocess.pro.user b/sfilterprocess/sfilterprocess.pro.user index 50ade45..6a71ade 100644 --- a/sfilterprocess/sfilterprocess.pro.user +++ b/sfilterprocess/sfilterprocess.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -787,7 +787,7 @@ sfilterprocess Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro - "469" "test2y" + "510" "testall" sfilterprocess.pro false true diff --git a/sfilterprocess/sinfluencer.cpp b/sfilterprocess/sinfluencer.cpp index 706247d..f51670a 100644 --- a/sfilterprocess/sinfluencer.cpp +++ b/sfilterprocess/sinfluencer.cpp @@ -6,6 +6,10 @@ #include #include #include +#include +#include +#include +#include #include "sinfluencer.h" void SInfluencer::setInitiaizer(SInitializer &_initializer) @@ -13,9 +17,10 @@ void SInfluencer::setInitiaizer(SInitializer &_initializer) m_pSInitializer = &_initializer; m_SBodyInfluencerMap.setInitiaizer(_initializer); m_SReplyInfluencerMap.setInitiaizer(_initializer); - m_SStatsInfBloggerBuzzbyday.setInitiaizer(_initializer); - m_SStatsInfBloggerRank.setInitiaizer(_initializer); + //m_SStatsInfBloggerBuzzbyday.setInitiaizer(_initializer); + //m_SStatsInfBloggerRank.setInitiaizer(_initializer); m_STInfluencer.setInitiaizer(_initializer); + m_SStatsJson.setInitiaizer(_initializer); } bool SInfluencer::uploadData(int _companyNum) @@ -34,6 +39,7 @@ bool SInfluencer::uploadData(int _companyNum) m_pSInitializer->insertLog("Fail to upload influencer reply map"); return false; } + /* if (!m_SStatsInfBloggerBuzzbyday.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_buzzbyday"); @@ -44,11 +50,18 @@ bool SInfluencer::uploadData(int _companyNum) m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_rank"); return false; } + */ if (!m_STInfluencer.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload influencer"); return false; } + if (!m_SStatsJson.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_json"); + return false; + } + m_pSInitializer->insertLog("End upload influencer"); m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Influence (Finish)"); @@ -170,19 +183,24 @@ bool SInfluencer::makeOverallCategory(int _categoryNum) bool SInfluencer::makeTable(SDatagroup &_datagroup) { - if (m_mapCategory.size() > 1) + //if (m_mapCategory.size() > 1) + if (m_mapCategory.size() > 0) { m_pSInitializer->insertLog("Make influencer tables"); m_pSInitializer->insertLog("Make influencer body tables"); m_SBodyInfluencerMap.makeTable(_datagroup, m_mapCategory); m_pSInitializer->insertLog("Make influencer reply tables"); m_SReplyInfluencerMap.makeTable(_datagroup, m_mapCategory); + /* m_pSInitializer->insertLog("Make influencer stats_inf_blogger_buzzy tables"); m_SStatsInfBloggerBuzzbyday.makeTable(_datagroup, m_mapCategory); m_pSInitializer->insertLog("Make influencer stats_inf_blogger_rank table"); m_SStatsInfBloggerRank.makeTable(_datagroup, m_mapCategory); + */ m_pSInitializer->insertLog("Make influencer influencer"); m_STInfluencer.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make influencer stats_json"); + m_SStatsJson.makeTable(_datagroup, m_mapCategory); } return true; } @@ -216,13 +234,19 @@ void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMaptitle; - mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->url; - mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->id; + //mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->title; + //mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->url; + //mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->id; + mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity((*mapBody)[_url])->title; + mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity((*mapBody)[_url])->url; + mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity((*mapBody)[_url])->id; mapRank[stprofileurlkey].nickname_nickname = _datagroup.getstBody(_url)->nickname_nickname; mapRank[stprofileurlkey].nickname_exnickname = _datagroup.getstBody(_url)->nickname_exnickname; mapRank[stprofileurlkey].platformname_num = _datagroup.getstBody(_url)->platformname_num; mapRank[stprofileurlkey].platformname_name = _datagroup.getstBody(_url)->platformname_name; + //m_pSInitializer->insertLog(_url); + //m_pSInitializer->insertLog(stprofileurlkey.m_strId); + //m_pSInitializer->insertLog(mapRank[stprofileurlkey].community_url); } mapRank[stprofileurlkey].bodycount++; @@ -1041,4 +1065,207 @@ bool SInfluencer::SReplyInfluencerMap::changeTable(QSqlDatabase &_db, const int return true; } +namespace +{ + QString strSeparator = "!@#"; +} + +struct stCount +{ + unsigned int body; + unsigned int reply; + QSet id; + stCount():body(0), reply(0){} +}; + +struct stDate +{ + QDate dStart; + QDate dEnd; + stDate():dStart(2020, 1, 1), dEnd(1990, 1, 1){} +}; + + +void SInfluencer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + const QStringList *slOrderedUrl = _datagroup.getOrderedUrl(); + + + QMap mapCount; + QMap mapBodyDate; + + foreach (const QString& strUrl, *slOrderedUrl) + { + if (mapBody->contains(strUrl)) + { + QString strKey = (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapCount.contains(strKey)) + mapCount.insert(strKey, stCount()); + + mapCount[strKey].body += 1; + mapCount[strKey].id.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + + if (mapReply->contains(strUrl)) + mapCount[strKey].reply += (*mapReply)[strUrl].size(); + + if (!mapBodyDate.contains(strKey)) + mapBodyDate.insert(strKey, stDate()); + //m_pSInitializer->insertLog((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_date]].trimmed()); + QDate dBodyDate = QDateTime::fromString((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + //m_pSInitializer->insertLog(dBodyDate.toString("yyyy-MM-dd")); + mapBodyDate[strKey].dStart = (mapBodyDate.value(strKey).dStart < dBodyDate) ? mapBodyDate.value(strKey).dStart : dBodyDate; + mapBodyDate[strKey].dEnd = (mapBodyDate.value(strKey).dEnd > dBodyDate) ? mapBodyDate.value(strKey).dEnd : dBodyDate; + } + } + + QSet setRealAllUrl; + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + stStatsJson ststatsjson; + ststatsjson.category_num = iterPos.key(); + ststatsjson.platformname_num = 1; + ststatsjson.servicenum = 1; + ststatsjson.title = "info"; + ststatsjson.subtitle = ""; + + const stReal& streal = iterPos.value(); + QSet setPlatformType; + + foreach (const QString& strUrl, streal.m_setRealAllUrl) + { + if (mapBody->contains(strUrl)) + setPlatformType.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed()); + setRealAllUrl.insert(strUrl); + } + + unsigned int nBody = 0; +// unsigned int nReply = 0; + unsigned int nUser = 0; + QJsonObject jsonObj; + + foreach (const QString& strKey, setPlatformType) + { + if (strKey == "naver!@#blog") + { +// m_pSInitializer->insertLog(mapBodyDate.value(strKey).dStart.toString("yyyy-MM-dd")); +// m_pSInitializer->insertLog(mapBodyDate.value(strKey).dEnd.toString("yyyy-MM-dd")); +// m_pSInitializer->insertLog(QString::number(mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd))); + nBody += (mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd) * 1000); + //nReply += (mapCount[strKey].reply * 1000); + if (setPlatformType.size() == 1) + jsonObj["pagenum"] = m_pSInitializer->getBlogPageNum(); + } + else + { + nBody += mapCount[strKey].body; +// nReply += mapCount[strKey].reply; + } + nUser += mapCount[strKey].id.size(); + } + + /* + unsigned int nRealBody = streal.m_setRealAllUrl.size(); + unsigned int nRealReply = 0; + + foreach (const QString& strUrl, streal.m_setRealAllUrl) + { + if (mapReply->contains(strUrl)) + nRealReply += mapReply->value(strUrl).size(); + } + */ + + //unsigned int nInfluencerValue = (nRealBody == 0) ? 0 : nRealReply / nRealBody; + unsigned int nTotalCount = nBody; + jsonObj["totalcount"] = static_cast(nTotalCount); + jsonObj["totalusercount"] = static_cast(nUser); + //jsonObj["value"] = static_cast(nInfluencerValue); + if (!jsonObj.contains("pagenum")) + jsonObj["pagenum"] = -1; + QJsonDocument jsonDoc(jsonObj); + ststatsjson.jdata = jsonDoc.toJson(); + m_listMap.append(ststatsjson); + } + + { + stStatsJson ststatsjson; + ststatsjson.category_num = -1; + ststatsjson.platformname_num = 1; + ststatsjson.servicenum = 1; + ststatsjson.title = "info"; + ststatsjson.subtitle = ""; + + QSet setPlatformType; + + foreach (const QString& strUrl, setRealAllUrl) + { + if (mapBody->contains(strUrl)) + setPlatformType.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed()); + } + + unsigned int nBody = 0; + unsigned int nUser = 0; + QJsonObject jsonObj; + + foreach (const QString& strKey, setPlatformType) + { + if (strKey == "naver!@#blog") + { + nBody += (mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd) * 1000); + if (setPlatformType.size() == 1) + jsonObj["pagenum"] = m_pSInitializer->getBlogPageNum(); + } + else + { + nBody += mapCount[strKey].body; + } + nUser += mapCount[strKey].id.size(); + } + unsigned int nTotalCount = nBody; + jsonObj["totalcount"] = static_cast(nTotalCount); + jsonObj["totalusercount"] = static_cast(nUser); + if (!jsonObj.contains("pagenum")) + jsonObj["pagenum"] = -1; + QJsonDocument jsonDoc(jsonObj); + ststatsjson.jdata = jsonDoc.toJson(); + m_listMap.append(ststatsjson); + } +} + +bool SInfluencer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QSqlQuery query(_db); + { + QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 1"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + foreach (const stStatsJson& ststats, m_listMap) + { + QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values ("; + strQuery += QString::number(_nCompany) + ","; + strQuery += QString::number(ststats.servicenum) + ","; + strQuery += "'" + ststats.title + "',"; + strQuery += "'" + ststats.subtitle + "',"; + strQuery += QString::number(ststats.category_num) + ","; + strQuery += QString::number(ststats.platformname_num) + ","; + strQuery += "'" + ststats.jdata +"')"; + + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} diff --git a/sfilterprocess/sinfluencer.h b/sfilterprocess/sinfluencer.h index d400d72..abc66c7 100644 --- a/sfilterprocess/sinfluencer.h +++ b/sfilterprocess/sinfluencer.h @@ -88,6 +88,20 @@ public: QString body_date; stReplyInfluencerMap():num(0),reply_num(0),body_num(0),id_num(0),category_num(0),realin(0),community_num(0){} }; + + class SReplyInfluencerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + class STInfluencer { private: @@ -138,17 +152,29 @@ public: bool changeTable(QSqlDatabase &_db, const int &_nCompany); }; - class SReplyInfluencerMap + + struct stStatsJson + { + int company_num; + int servicenum; + QString title; + QString subtitle; + int platformname_num; + QString jdata; + int category_num; + stStatsJson():company_num(0), servicenum(2), platformname_num(1), category_num(0){} + }; + + class SStatsJson { private: SInitializer *m_pSInitializer; - QList m_listMap; + QList m_listMap; public: - void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } - void clear(){ m_listMap.clear(); } + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); - bool uploadData(QSqlDatabase &_db, const int &_nCompany); - bool changeTable(QSqlDatabase &_db, const int &_nCompany); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear(){ m_listMap.clear(); } }; struct isGreaterInfluencerValue @@ -211,9 +237,10 @@ private: QMap m_mapCategory; SBodyInfluencerMap m_SBodyInfluencerMap; SReplyInfluencerMap m_SReplyInfluencerMap; - SStatsInfBloggerBuzzbyday m_SStatsInfBloggerBuzzbyday; - SStatsInfBloggerRank m_SStatsInfBloggerRank; + //SStatsInfBloggerBuzzbyday m_SStatsInfBloggerBuzzbyday; + //SStatsInfBloggerRank m_SStatsInfBloggerRank; STInfluencer m_STInfluencer; + SStatsJson m_SStatsJson; public: virtual void setInitiaizer(SInitializer &_initializer); diff --git a/sfilterprocess/sinitializer.cpp b/sfilterprocess/sinitializer.cpp index 9cf0e74..db4e7ad 100644 --- a/sfilterprocess/sinitializer.cpp +++ b/sfilterprocess/sinitializer.cpp @@ -166,6 +166,25 @@ bool SInitializer::initSpammer() return true; } + + +bool SInitializer::initBlogPageNum() +{ + QSettings settings(QString("setup.ini"), QSettings::IniFormat); + settings.setIniCodec("UTF-8"); + settings.beginGroup(QString("pagenumber")); + + if (settings.childKeys().size() == 0) + m_nBlogPageNum = 10; + else + m_nBlogPageNum = settings.value("pagenumber").toInt(); + + settings.endGroup(); + return true; +} + + + bool SInitializer::initDBConnection() { QSettings settings(QString("setup.ini"), QSettings::IniFormat); @@ -212,11 +231,13 @@ bool SInitializer::init(const QString& _strFilterProcessGroupID, const QString& b_ok &= initDays(); b_ok &= initInfluencer(); b_ok &= initSpammer(); + b_ok &= initBlogPageNum(); b_ok &= initFilterProcess(); b_ok &= initCompanyNum(); b_ok &= initPlatform(); //b_ok &= initConsumerCategory(); b_ok &= initCategory(); + /* if (!b_ok) cout << "FALSE" << endl; @@ -472,6 +493,7 @@ bool SInitializer::initColumn() return true; } + bool SInitializer::initCompanyNum() { return true; @@ -523,6 +545,13 @@ const int* SInitializer::getColumnIntArray() const return m_stColumn.m_anColumn; } + +int SInitializer::getBlogPageNum() +{ + return m_nBlogPageNum; +} + + QList SInitializer::getFilterProcess() { return m_listFilterProcess; diff --git a/sfilterprocess/sinitializer.h b/sfilterprocess/sinitializer.h index ad803ac..d8ff0a6 100644 --- a/sfilterprocess/sinitializer.h +++ b/sfilterprocess/sinitializer.h @@ -28,6 +28,7 @@ private: bool initPlatform(); //bool initConsumerCategory(); bool initCategory(); + bool initBlogPageNum(); public: @@ -177,6 +178,7 @@ private: int m_nConsumerAllCategory; int m_anAllCategory[E_SERVICE_MAX]; QMap m_mapCategory; + int m_nBlogPageNum; public: bool init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName); @@ -205,6 +207,7 @@ public: void updateWebDBInfoComplete(QSqlDatabase _db, QString _mindate, QString _maxdate, QString _lastupdate, int _bodycount, int _replycount, int _nCompany); int getAllCategoryNum(int _nService); QString getCategoryName(int _nCategory) const; + int getBlogPageNum(); //bool disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); //bool enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); //bool disableSettings(QSqlDatabase& db, int _nMode = MODE); diff --git a/sfilterprocess/spowercafe.cpp b/sfilterprocess/spowercafe.cpp index 2849577..dfc6f45 100644 --- a/sfilterprocess/spowercafe.cpp +++ b/sfilterprocess/spowercafe.cpp @@ -4,6 +4,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "spowercafe.h" void SPowercafe::setInitiaizer(SInitializer &_initializer) @@ -12,7 +18,8 @@ void SPowercafe::setInitiaizer(SInitializer &_initializer) m_SBodyPowercafeMap.setInitiaizer(_initializer); m_SReplyPowercafeMap.setInitiaizer(_initializer); m_STPowercafe.setInitiaizer(_initializer); - m_SStatsPowCafeRank.setInitiaizer(_initializer); + //m_SStatsPowCafeRank.setInitiaizer(_initializer); + m_SStatsJson.setInitiaizer(_initializer); } bool SPowercafe::uploadData(int _companyNum) @@ -31,16 +38,23 @@ bool SPowercafe::uploadData(int _companyNum) m_pSInitializer->insertLog("Fail to upload Powercafe reply map"); return false; } + /* if (!m_SStatsPowCafeRank.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload stats_pow_cafe_rank"); return false; } + */ if (!m_STPowercafe.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload Powercafe"); return false; } + if (!m_SStatsJson.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload sstatsJson"); + return false; + } m_pSInitializer->insertLog("End upload Powercafe"); m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Powercafe (Finish)"); @@ -176,10 +190,14 @@ bool SPowercafe::makeTable(SDatagroup &_datagroup) m_SBodyPowercafeMap.makeTable(_datagroup, m_mapCategory); m_pSInitializer->insertLog("Make powercafe reply tables"); m_SReplyPowercafeMap.makeTable(_datagroup, m_mapCategory); + /* m_pSInitializer->insertLog("Make powercafe stats_pow_cafe_rank tables"); m_SStatsPowCafeRank.makeTable(_datagroup, m_mapCategory); + */ m_pSInitializer->insertLog("Make powercafe powercafe table"); m_STPowercafe.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make powercafe sstatsjson"); + m_SStatsJson.makeTable(_datagroup, m_mapCategory); } return true; @@ -833,3 +851,226 @@ bool SPowercafe::SReplyPowercafeMap::changeTable(QSqlDatabase &_db, const int &_ } return true; } + +void SPowercafe::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + /* + QMap mapPowercafeAll; + for (QMap::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++) + { + if (iterPos.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stStatsPowCafeRank()); + mapPowercafeAll[strKey].community_id = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + mapPowercafeAll[strKey].community_title = iterPos.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + mapPowercafeAll[strKey].subject = "articlecountall"; + } + mapPowercafeAll[strKey].articlecountall++; + } + + for (QMap >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++) + { + for (QMap::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++) + { + + if (iterPos1.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stStatsPowCafeRank()); + mapPowercafeAll[strKey].community_id = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + mapPowercafeAll[strKey].community_title = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + mapPowercafeAll[strKey].subject = "articlecountall"; + } + mapPowercafeAll[strKey].articlecountall++; + } + } + + { + QList listPowercafe = mapPowercafeAll.values(); + + qSort(listPowercafe.begin(), listPowercafe.end(), isGreaterCountAll()); + int rank = 1; + foreach (stStatsPowCafeRank ststats, listPowercafe) + { + ststats.rank = rank++; + m_listMap.push_back(ststats); + if (rank > 10) + break; + } + } + */ + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapScore; + const stReal& streal = iterPos.value(); + QDate dateStart(2030, 1, 1); + QDate dateEnd(2000, 1, 1); + + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + if (((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")|| + ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" )) + continue; + + QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ","; + strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stPowercafeRank()); + mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[_url])->url; + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->title ; + if (mapScore[strKey].community.trimmed().isEmpty()) + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->id; + } + mapScore[strKey].count++; + QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + if (((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")|| + ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" )) + continue; + QString strUrl = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_url]].trimmed(); + QString strKey = (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ","; + strKey += ((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stPowercafeRank()); + mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[strUrl])->url; + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->title; + if (mapScore[strKey].community.trimmed().isEmpty()) + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->id; + } + mapScore[strKey].count++; + QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + //QList listSpammer; + QMap > mapRank; + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + QString strPlatformName = iterPos1.key().split(",").at(1); + if (!mapRank.contains(m_pSInitializer->getPlatformName(strPlatformName))) + mapRank.insert(m_pSInitializer->getPlatformName(strPlatformName), QList()); + + mapRank[m_pSInitializer->getPlatformName(strPlatformName)].append(iterPos1.value()); + } + + for (QMap >::iterator iterPos1 = mapRank.begin(); iterPos1 != mapRank.end(); iterPos1++) + qSort(iterPos1.value().begin(), iterPos1.value().end(), isGreaterCountReal()); + + QList keys = mapRank.keys().toSet().toList(); + + QJsonArray jsonArray; + stStatsJson stjsonRank; + { + stjsonRank.category_num = iterPos.key(); + stjsonRank.servicenum = 4; + stjsonRank.title = "rank"; + stjsonRank.subtitle = ""; + stjsonRank.platformname_num = 1; + } + int nTotalCount = 0; + for (QList::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++) + { + QList values = mapRank.value(*iterPos1); + int nRankLimit = (values.size() <= 10) ? values.size() : 10; + QJsonObject jsonObj; + jsonObj["platformname_num"] = *iterPos1; + QJsonArray jsonSubArray; + for (int i = 0; i < nRankLimit; i++) + { + QJsonObject jsonSubObj; + jsonSubObj["rank"] = i + 1; + jsonSubObj["realcount"] = values.at(i).count; + jsonSubObj["community"] = values.at(i).community; + if (values.at(i).url.contains("instagram.com")) + jsonSubObj["url"] = "https://www." + values.at(i).url; + else if (values.at(i).url.contains("cafe.naver.com")) + jsonSubObj["url"] = "http://" + values.at(i).url; + else if (values.at(i).url.contains("facebook.com")) + jsonSubObj["url"] = "https://www." + values.at(i).url; + else if (values.at(i).url.contains("kakaostory.com")) + jsonSubObj["url"] = "https://" + values.at(i).url; + else if (values.at(i).url.contains("cafe.daum.net")) + jsonSubObj["url"] = "http://" + values.at(i).url; + else + jsonSubObj["url"] = "http://" + values.at(i).url; + jsonSubArray.append(jsonSubObj); + } + jsonObj["data"] = jsonSubArray; + jsonArray.append(jsonObj); + nTotalCount += values.size(); + } + QJsonObject jObj; + jObj["data"] = jsonArray; + jObj["mindate"] = dateStart.toString("yyyy.MM.dd"); + jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd"); + jObj["totalcount"] = nTotalCount; + QJsonDocument jsonDoc(jObj); + stjsonRank.jdata = jsonDoc.toJson(); + m_listMap.append(stjsonRank); + } +} + + +bool SPowercafe::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + + QSqlQuery query(_db); + { + QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 4"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + foreach (const stStatsJson& ststats, m_listMap) + { + QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values ("; + strQuery += QString::number(_nCompany) + ","; + strQuery += QString::number(ststats.servicenum) + ","; + strQuery += "'" + ststats.title + "',"; + strQuery += "'" + ststats.subtitle + "',"; + strQuery += QString::number(ststats.category_num) + ","; + strQuery += QString::number(ststats.platformname_num) + ","; + strQuery += "'" + ststats.jdata +"')"; + + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + diff --git a/sfilterprocess/spowercafe.h b/sfilterprocess/spowercafe.h index 331135e..2082004 100644 --- a/sfilterprocess/spowercafe.h +++ b/sfilterprocess/spowercafe.h @@ -132,6 +132,39 @@ public: bool changeTable(QSqlDatabase &_db, const int &_nCompany); }; + struct stStatsJson + { + int company_num; + int servicenum; + QString title; + QString subtitle; + int platformname_num; + QString jdata; + int category_num; + stStatsJson():company_num(0), servicenum(2), platformname_num(1), category_num(0){} + }; + + + class SStatsJson + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear(){ m_listMap.clear(); } + }; + + struct stPowercafeRank + { + int count; + QString url; + QString community; + stPowercafeRank(): count(0){} + }; + struct isGreaterCountReal { bool operator()(const stPowercafe& first, const stPowercafe& second) @@ -142,6 +175,10 @@ public: { return first.articlecountreal > second.articlecountreal; } + bool operator()(const stPowercafeRank& first, const stPowercafeRank& second) + { + return first.count > second.count; + } }; struct isGreaterCountAll @@ -160,7 +197,8 @@ private: SBodyPowercafeMap m_SBodyPowercafeMap; SReplyPowercafeMap m_SReplyPowercafeMap; STPowercafe m_STPowercafe; - SStatsPowCafeRank m_SStatsPowCafeRank; + //SStatsPowCafeRank m_SStatsPowCafeRank; + SStatsJson m_SStatsJson; public: virtual void setInitiaizer(SInitializer &_initializer); diff --git a/sfilterprocess/sspammer.cpp b/sfilterprocess/sspammer.cpp index 6b8c7cc..9edae5b 100644 --- a/sfilterprocess/sspammer.cpp +++ b/sfilterprocess/sspammer.cpp @@ -5,6 +5,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "sspammer.h" @@ -13,8 +19,9 @@ void SSpammer::setInitiaizer(SInitializer &_initializer) m_pSInitializer = &_initializer; m_SBodySpammerMap.setInitiaizer(_initializer); m_SReplySpammerMap.setInitiaizer(_initializer); - m_SStatsSpaSpammerRank.setInitiaizer(_initializer); + //m_SStatsSpaSpammerRank.setInitiaizer(_initializer); m_STSpammer.setInitiaizer(_initializer); + m_SStatsJson.setInitiaizer(_initializer); } bool SSpammer::uploadData(int _companyNum) @@ -33,16 +40,23 @@ bool SSpammer::uploadData(int _companyNum) m_pSInitializer->insertLog("Fail to upload Spammer reply map"); return false; } + /* if (!m_SStatsSpaSpammerRank.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload stats_spa_spammer_rank"); return false; } + */ if (!m_STSpammer.uploadData(dbWeb, _companyNum)) { m_pSInitializer->insertLog("Fail to upload spammer"); return false; } + if (!m_SStatsJson.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload statsjson"); + return false; + } m_pSInitializer->insertLog("End upload Spammer"); m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Spammer (Finish)"); @@ -168,13 +182,18 @@ bool SSpammer::makeTable(SDatagroup &_datagroup) m_pSInitializer->insertLog("Make spammer tables"); m_pSInitializer->insertLog("Make spammer spammer"); m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer); + /* m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables"); m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory); + */ m_pSInitializer->insertLog("Make spammer body tables"); m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer); m_pSInitializer->insertLog("Make spammer reply tables"); m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer); + m_pSInitializer->insertLog("Make sstatsjson table"); + m_SStatsJson.makeTable(_datagroup, m_mapCategory); } + return true; } @@ -824,3 +843,390 @@ bool SSpammer::SReplySpammerMap::changeTable(QSqlDatabase &_db, const int &_nCom } return true; } + +void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + QDate dateStart(2100,1,1),dateEnd(2000,1,1); + QDate dateStartBody(2100,1,1),dateEndBody(2000,1,1); + QDate dateStartReply(2100,1,1),dateEndReply(2000,1,1); + + QList listBuzzy; + // filtered + { + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + stBuzzy stbuzzy; + stbuzzy.m_nCategory = iterPos.key(); + const stReal &streal = iterPos.value(); + foreach (const QString& strUrl, streal.m_setRealBodyUrl) + { + QDate date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + //if (date.isNull()) + // date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed(), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + dateStartBody = (date < dateStartBody ) ? date : dateStartBody; + dateEndBody = (date > dateEndBody ) ? date : dateEndBody; + + + if (stbuzzy.m_mapBodyBuzz.contains(date)) + stbuzzy.m_mapBodyBuzz[date]++; + else + stbuzzy.m_mapBodyBuzz.insert(date, 1); + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + dateStartReply = (date < dateStartReply ) ? date : dateStartReply; + dateEndReply = (date > dateEndReply ) ? date : dateEndReply; + + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + + listBuzzy.append(stbuzzy); + } + } + //all buzzy + { + stBuzzy stbuzzy; + stbuzzy.m_nCategory = 0; + for (QMap::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++) + { + QDate date = QDateTime::fromString(iterPos.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + + if (stbuzzy.m_mapBodyBuzz.contains(date)) + stbuzzy.m_mapBodyBuzz[date]++; + else + stbuzzy.m_mapBodyBuzz.insert(date, 1); + } + + for (QMap >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++) + { + for (QMap::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++) + { + QDate date = QDateTime::fromString(iterPos1.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + } + listBuzzy.append(stbuzzy); + } + + int nTotalCount = 0, nTotalBodyCount = 0, nTotalReplyCount = 0; + + if (_mapCategory.contains(m_pSInitializer->getAllCategoryNum(3))) + { + foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(3)].m_setRealAllUrl) + { + nTotalBodyCount++; + nTotalReplyCount += (*mapReply)[strUrl].size(); + } + nTotalCount = nTotalBodyCount + nTotalReplyCount; + } + + + { + QJsonArray jsArrayAll; + QJsonArray jsArrayReply; + QJsonArray jsArrayBody; + + foreach (const stBuzzy &stbuzzy, listBuzzy) + { + QJsonArray jsValueAll; + QJsonArray jsValueBody; + QJsonArray jsValueReply; + + for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1)) + { + QJsonObject jsoBody; + QJsonObject jsoAll; + QJsonObject jsoReply; + + jsoAll["date"] = jsoReply["date"] = jsoBody["date"] = i.toString("yyyyMMdd"); + jsoBody["count"] = stbuzzy.m_mapBodyBuzz[i]; + jsoReply["count"] = stbuzzy.m_mapReplyBuzz[i]; + jsoAll["count"] = stbuzzy.m_mapBodyBuzz[i] + stbuzzy.m_mapReplyBuzz[i]; + + jsValueAll.append(jsoAll); + jsValueBody.append(jsoBody); + jsValueReply.append(jsoReply); + } + + QJsonObject jsNameReply; + QJsonObject jsNameAll; + QJsonObject jsNameBody; + + if (stbuzzy.m_nCategory != 0) + jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = m_pSInitializer->getCategoryName(stbuzzy.m_nCategory); + else + jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALL"; + + jsNameAll["value"] = jsValueAll; + jsNameBody["value"] = jsValueBody; + jsNameReply["value"] = jsValueReply; + + jsArrayAll.append(jsNameAll); + jsArrayBody.append(jsNameBody); + jsArrayReply.append(jsNameReply); + } + stStatsJson stbodybuzz; + stStatsJson streplybuzz; + stStatsJson stallbuzz; + + stallbuzz.title = streplybuzz.title = stbodybuzz.title = "buzz"; + + stbodybuzz.subtitle = "body"; + streplybuzz.subtitle = "reply"; + stallbuzz.subtitle = "all"; + + QJsonObject jobjAll; + QJsonObject jobjBody; + QJsonObject jobjReply; + + jobjAll["data"] = jsArrayAll; + jobjBody["data"] = jsArrayBody; + jobjReply["data"] = jsArrayReply; + + jobjAll["mindate"] = dateStart.toString("yyyy.MM.dd"); + jobjBody["mindate"] = dateStartBody.toString("yyyy.MM.dd"); + jobjReply["mindate"] = dateStartReply.toString("yyyy.MM.dd"); + + jobjAll["maxdate"] = dateEnd.toString("yyyy.MM.dd"); + jobjBody["maxdate"] = dateEndBody.toString("yyyy.MM.dd"); + jobjReply["maxdate"] = dateEndReply.toString("yyyy.MM.dd"); + + jobjAll["totalcount"] = nTotalCount; + jobjBody["totalcount"] = nTotalBodyCount; + jobjReply["totalcount"] = nTotalReplyCount; + + QJsonDocument jdAll(jobjAll); + QJsonDocument jdBody(jobjBody); + QJsonDocument jdReply(jobjReply); + + stbodybuzz.jdata = jdBody.toJson(QJsonDocument::Compact); + streplybuzz.jdata = jdReply.toJson(QJsonDocument::Compact); + stallbuzz.jdata = jdAll.toJson(QJsonDocument::Compact); + + stallbuzz.platformname_num = stbodybuzz.platformname_num = streplybuzz.platformname_num = 1; + m_listMap.append(stbodybuzz); + m_listMap.append(streplybuzz); + m_listMap.append(stallbuzz); + } + + /* + + { + for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1)) + { + //body + stStatsJson stbodybuzz; + stStatsJson streplybuzz; + stStatsJson stallbuzz; + foreach (const stBuzzy& stbuzzy, listBuzzy) + { + if (stbuzzy.m_mapBodyBuzz.contains(i)) + { + stbodybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapBodyBuzz[i]; + stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapBodyBuzz[i]; + } + + if (stbuzzy.m_mapReplyBuzz.contains(i)) + { + streplybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapReplyBuzz[i]; + stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapReplyBuzz[i]; + } + } + stallbuzz.date = streplybuzz.date = stbodybuzz.date = i; + stbodybuzz.subtitle = "body"; + streplybuzz.subtitle = "reply"; + stallbuzz.subtitle = "all"; + m_listMap.append(stallbuzz); + m_listMap.append(stbodybuzz); + m_listMap.append(streplybuzz); + } + } + */ +} + +void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapScore; + const stReal& streal = iterPos.value(); + QDate dateStart(2030, 1, 1); + QDate dateEnd(2000, 1, 1); + + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(_url)->id_num; + } + mapScore[strKey].body++; + mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(iterPos1.key())->id_num; + } + mapScore[strKey].reply++; + mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + //QList listSpammer; + QMap mapSpammer; + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + if (iterPos1->body >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] || + iterPos1->reply >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] || + iterPos1->nickname.size() >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT]) + { + double value = iterPos1->body * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO] + + iterPos1->reply * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO] + + iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO]; + stSpammerRank rank; + rank.value = value; + rank.id = iterPos1.key().split(",").at(0); + rank.platformform_num = m_pSInitializer->getPlatformName(iterPos1.key().split(",").at(1)); + rank.id_num = iterPos1.value().id_num; + mapSpammer.insertMulti(rank.platformform_num, rank); + } + } + + QList keys = mapSpammer.keys().toSet().toList(); + + QJsonArray jsonArray; + stStatsJson stjsonRank; + { + stjsonRank.category_num = iterPos.key(); + stjsonRank.servicenum = 3; + stjsonRank.title = "rank"; + stjsonRank.subtitle = ""; + stjsonRank.platformname_num = 1; + } + int nTotalCount = 0; + for (QList::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++) + { + QList values = mapSpammer.values(*iterPos1); + qSort(values.begin(), values.end(), isGreaterSpammerValue()); + int nRankLimit = (values.size() <= 10) ? values.size() : 10; + QJsonObject jsonObj; + jsonObj["platformname_num"] = *iterPos1; + QJsonArray jsonSubArray; + for (int i = 0; i < nRankLimit; i++) + { + QJsonObject jsonSubObj; + jsonSubObj["rank"] = i + 1; + jsonSubObj["value"] = values.at(i).value; + jsonSubObj["id"] = values.at(i).id; + jsonSubObj["id_num"] = values.at(i).id_num; + jsonSubArray.append(jsonSubObj); + } + jsonObj["data"] = jsonSubArray; + jsonArray.append(jsonObj); + + nTotalCount += values.size(); + } + QJsonObject jObj; + jObj["data"] = jsonArray; + jObj["mindate"] = dateStart.toString("yyyy.MM.dd"); + jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd"); + jObj["totalcount"] = nTotalCount; + QJsonDocument jsonDoc(jObj); + stjsonRank.jdata = jsonDoc.toJson(); + m_listMap.append(stjsonRank); + } +} + + +void SSpammer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + makeTableBuzz(_datagroup, _mapCategory); + makeTableRank(_datagroup, _mapCategory); +} + + + +bool SSpammer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QSqlQuery query(_db); + { + QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 3"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + foreach (const stStatsJson& ststats, m_listMap) + { + QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values ("; + strQuery += QString::number(_nCompany) + ","; + strQuery += QString::number(ststats.servicenum) + ","; + strQuery += "'" + ststats.title + "',"; + strQuery += "'" + ststats.subtitle + "',"; + strQuery += QString::number(ststats.category_num) + ","; + strQuery += QString::number(ststats.platformname_num) + ","; + strQuery += "'" + ststats.jdata +"')"; + + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} diff --git a/sfilterprocess/sspammer.h b/sfilterprocess/sspammer.h index c8c24cf..8598380 100644 --- a/sfilterprocess/sspammer.h +++ b/sfilterprocess/sspammer.h @@ -77,7 +77,35 @@ public: int reply; int id_num; QSet nickname; - stSpammerScore():body(0),reply(0){} + stSpammerScore():body(0),reply(0), id_num(0){} + }; + + struct stSpammerRank + { + double value; + int platformform_num; + QString id; + int id_num; + stSpammerRank():value(0.0), platformform_num(1), id_num(0){} + }; + + struct stBuzzy + { + int m_nCategory; + QMap m_mapBodyBuzz; + QMap m_mapReplyBuzz; + }; + + struct stStatsJson + { + int company_num; + int servicenum; + QString title; + QString subtitle; + int platformname_num; + QString jdata; + int category_num; + stStatsJson():company_num(0), servicenum(3), platformname_num(1), category_num(0){} }; class STSpammer @@ -143,16 +171,36 @@ public: { return first.spammervalue > second.spammervalue; } + bool operator()(const stSpammerRank& first, const stSpammerRank& second) const + { + return first.value > second.value; + } }; + class SStatsJson + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + void makeTableBuzz(SDatagroup &_datagroup, const QMap& _mapCategory); + void makeTableRank(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear(){ m_listMap.clear(); } + }; + + private: SFilterAlgorithm m_SFilterAlgorithm; QMap m_mapCategory; STSpammer m_STSpammer; - SStatsSpaSpammerRank m_SStatsSpaSpammerRank; + //SStatsSpaSpammerRank m_SStatsSpaSpammerRank; SBodySpammerMap m_SBodySpammerMap; SReplySpammerMap m_SReplySpammerMap; QSet m_setSpammer; + SStatsJson m_SStatsJson; public: virtual void setInitiaizer(SInitializer &_initializer);