diff --git a/sfilterprocess/sconsumer.cpp b/sfilterprocess/sconsumer.cpp index d5b64b3..84bd618 100644 --- a/sfilterprocess/sconsumer.cpp +++ b/sfilterprocess/sconsumer.cpp @@ -183,10 +183,12 @@ void SConsumer::SBodyConsumerMap::makeTable(SDatagroup &_datagroup, const QMapplatformname_num; stbody.platformform_num = _datagroup.getstBody(strUrl)->platformform_num; - stbody.body_date = _datagroup.getstBody(strUrl)->date; - stbody.body_replycount = _datagroup.getstBody(strUrl)->replycount; - stbody.body_view = _datagroup.getstBody(strUrl)->view; - stbody.body_id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.date = _datagroup.getstBody(strUrl)->date; + stbody.replycount = _datagroup.getstBody(strUrl)->replycount; + stbody.view = _datagroup.getstBody(strUrl)->view; + stbody.id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.community_num = _datagroup.getstBody(strUrl)->community_num; + stbody.nickname_num = _datagroup.getstBody(strUrl)->nickname_num; if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) { @@ -210,10 +212,12 @@ void SConsumer::SBodyConsumerMap::makeTable(SDatagroup &_datagroup, const QMapplatformname_num; stbody.platformform_num = _datagroup.getstBody(strUrl)->platformform_num; - stbody.body_date = _datagroup.getstBody(strUrl)->date; - stbody.body_replycount = _datagroup.getstBody(strUrl)->replycount; - stbody.body_view = _datagroup.getstBody(strUrl)->view; - stbody.body_id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.date = _datagroup.getstBody(strUrl)->date; + stbody.replycount = _datagroup.getstBody(strUrl)->replycount; + stbody.view = _datagroup.getstBody(strUrl)->view; + stbody.id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.community_num = _datagroup.getstBody(strUrl)->community_num; + stbody.nickname_num = _datagroup.getstBody(strUrl)->nickname_num; foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) { @@ -288,7 +292,7 @@ bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCom } strQuery = QString("insert into body_consumer_map_" + QString::number(_nCompany) + - "_update (num,body_num,category_num,realin,platformname_num,platformform_num,body_date,body_replycount,body_view,body_id_num) VALUES ("); + "_update (num,body_num,category_num,realin,platformname_num,platformform_num,date,replycount,view,id_num,community_num,nickname_num) VALUES ("); query.clear(); @@ -301,10 +305,12 @@ bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCom str += QString::number(stbody.realin) + ","; str += QString::number(stbody.platformname_num) + ","; str += QString::number(stbody.platformform_num) + ","; - str += "'" + stbody.body_date + "',"; - str += QString::number(stbody.body_replycount) + ","; - str += QString::number(stbody.body_view) + ","; - str += QString::number(stbody.body_id_num) + ")"; + str += "'" + stbody.date + "',"; + str += QString::number(stbody.replycount) + ","; + str += QString::number(stbody.view) + ","; + str += QString::number(stbody.id_num) + ","; + str += QString::number(stbody.community_num) + ","; + str += QString::number(stbody.nickname_num) + ")"; if (query.exec(str.toUtf8()) == false) { @@ -354,6 +360,7 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap listBuzzy; + stReal allCategory; // filtered { for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) @@ -363,6 +370,8 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) { const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + if (!allCategory.m_mapRealReplyUniqueUrl.contains(iterPos1.key()) || !allCategory.m_mapRealReplyUniqueUrl.values(iterPos1.key()).contains(iterPos1.value())) + allCategory.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value()); + allCategory.m_setRealAllUrl.insert(iterPos1.key()); QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); if (date.year() == 1990) continue; @@ -400,6 +412,37 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap::const_iterator iterPos1 = allCategory.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != allCategory.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + listBuzzy.append(stbuzzy); + } + //all buzzy { stBuzzy stbuzzy; @@ -435,9 +478,8 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMapgetAllCategoryNum(2))) { - foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(2)].m_setRealAllUrl) + foreach (const QString& strUrl, allCategory.m_setRealAllUrl) { nTotalBodyCount++; nTotalReplyCount += (*mapReply)[strUrl].size(); @@ -477,10 +519,12 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap 0) jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = m_pSInitializer->getCategoryName(stbuzzy.m_nCategory); - else + else if (stbuzzy.m_nCategory == 0) jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALL"; + else + jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALLCATEGORY"; jsNameAll["value"] = jsValueAll; jsNameBody["value"] = jsValueBody; @@ -523,6 +567,7 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + QMap mapConsumerMarketer = m_pSInitializer->getConsumerMaketer(); + + for (QMap::iterator iterPos = mapConsumerMarketer.begin(); iterPos != mapConsumerMarketer.end(); ++iterPos) + { + if (iterPos.value().trimmed().isEmpty()) + continue; + + + for (QMap::const_iterator iterPos1 = mapBody->cbegin(); iterPos1 != mapBody->cend(); ++iterPos1) + { + if ((m_pSInitializer->getPlatformName(iterPos1.value()[anColumn[SInitializer::E_DATA_platform_name]].trimmed()) == iterPos.key()) + && (iterPos.value() == iterPos1.value()[anColumn[SInitializer::E_DATA_article_id]].trimmed())) + { + _datagroup.addRealUrl(iterPos1.value()[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + for (QMap >::const_iterator iterPos1 = mapReply->cbegin(); iterPos1 != mapReply->cend(); ++iterPos1) + { + for(QMap::const_iterator iterPos2 = iterPos1->cbegin(); iterPos2 != iterPos1->cend(); ++iterPos2) + { + if ((m_pSInitializer->getPlatformName(iterPos2.value()[anColumn[SInitializer::E_DATA_platform_name]].trimmed()) == iterPos.key()) + && (iterPos.value() == iterPos2.value()[anColumn[SInitializer::E_DATA_article_id]].trimmed())) + { + _datagroup.addRealUrl(iterPos2.value()[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + //_datagroup.addRealUrl(iterPos1.key().trimmed()); + } + } + } + } + return true; + +} + + +bool SConsumerCheckList::makeTable(SDatagroup &_datagroup) +{ + + QStringList slUrl = m_pSInitializer->getConsumerCheckList(); + QMap mapUsers = m_pSInitializer->getConsumerCheckListUser(); + if (slUrl.isEmpty() && mapUsers.isEmpty()) + return true; + + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + const QStringList* pslorderedUrl = _datagroup.getOrderedUrl(); + + foreach (const QString& strUrl, *pslorderedUrl) + { + foreach (const QString& url, slUrl) + { + if (!url.trimmed().isEmpty() && strUrl.contains(url.trimmed())) + _datagroup.addRealUrl(strUrl); + } + + for(QMap::iterator iterPos = mapUsers.begin(); iterPos != mapUsers.end(); ++iterPos) + { + if (iterPos.value().trimmed().isEmpty()) + continue; + + if (m_pSInitializer->getPlatformName((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed()) == iterPos.key() + && (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_id]].trimmed() == iterPos.value().trimmed()) + { + _datagroup.addRealUrl((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_url]].trimmed()); + //_datagroup.addRealUrl(strUrl); + } + + if (mapReply->contains(strUrl)) + { + for (QMap::const_iterator iterPos1 = (*mapReply)[strUrl].cbegin(); iterPos1 != (*mapReply)[strUrl].cend(); ++iterPos1) + { + if (m_pSInitializer->getPlatformName(iterPos1.value()[anColumn[SInitializer::E_DATA_platform_name]].trimmed()) == iterPos.key() + && iterPos1.value()[anColumn[SInitializer::E_DATA_article_id]].trimmed() == iterPos.value().trimmed()) + { + //_datagroup.addRealUrl(strUrl); + _datagroup.addRealUrl(iterPos1.value()[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + } + } + } + + return true; +} + diff --git a/sfilterprocess/sconsumer.h b/sfilterprocess/sconsumer.h index 42f5d2f..98ae416 100644 --- a/sfilterprocess/sconsumer.h +++ b/sfilterprocess/sconsumer.h @@ -21,11 +21,13 @@ public: int realin; int platformname_num; int platformform_num; - QString body_date; - int body_replycount; - int body_view; - int body_id_num; - stBodyConsumerMap():num(0), body_num(0), category_num(0), realin(0), platformname_num(0), platformform_num(0), body_replycount(0), body_view(0), body_id_num(0){} + QString date; + int replycount; + int view; + int id_num; + int community_num; + int nickname_num; + stBodyConsumerMap():num(0), body_num(0), category_num(0), realin(0), platformname_num(0), platformform_num(0), replycount(0), view(0), id_num(0), community_num(0), nickname_num(0){} }; struct stReplyConsumerMap { @@ -139,6 +141,34 @@ public: }; +class SConsumerAssist:public SService +{ +public: + virtual void setInitiaizer(SInitializer &_initializer) + { + m_pSInitializer = &_initializer; + } + virtual bool uploadData(int _companyNum){ return true; } + virtual bool changeTable(int _companyNum){ return true; } + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter){ return true; } + virtual bool makeOverallCategory(int _nCategory){ return true; } + virtual bool makeTable(SDatagroup &_datagroup){return true;} +}; + +class SConsumerMarketer:public SConsumerAssist +{ +public: + + virtual bool makeTable(SDatagroup &_datagroup); +}; + + +class SConsumerCheckList:public SConsumerAssist +{ +public: + + virtual bool makeTable(SDatagroup &_datagroup); +}; diff --git a/sfilterprocess/sdatagroup.cpp b/sfilterprocess/sdatagroup.cpp index c2bec7e..dea46d9 100644 --- a/sfilterprocess/sdatagroup.cpp +++ b/sfilterprocess/sdatagroup.cpp @@ -114,6 +114,8 @@ bool SDatagroup::loadData(const int& _nDatagroupNumber) if (strList[anColumn[SInitializer::E_DATA_article_nickname]].trimmed().isEmpty()) continue; if (strList[anColumn[SInitializer::E_DATA_platform_id]].trimmed().isEmpty()) continue; if (strList[anColumn[SInitializer::E_DATA_platform_id]].trimmed().isEmpty()) continue; + if (!strList[anColumn[SInitializer::E_DATA_article_url]].trimmed().startsWith("http")) continue; + setCheckDupUrl.insert(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()); m_slOrderedBodyUrl.append(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()); m_mapBody.insert(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed(), strList); @@ -205,7 +207,8 @@ void SDatagroup::addRealReplyCategory(const QString& _strUrl, const int& _order, void SDatagroup::addRealUrl(const QString& _strUrl) { - m_setFilteredUrl.insert(_strUrl.trimmed()); + if (_strUrl.trimmed().startsWith("http")) + m_setFilteredUrl.insert(_strUrl.trimmed()); } diff --git a/sfilterprocess/sfilterprocess.cpp b/sfilterprocess/sfilterprocess.cpp index 854eea6..b6d458c 100644 --- a/sfilterprocess/sfilterprocess.cpp +++ b/sfilterprocess/sfilterprocess.cpp @@ -20,6 +20,8 @@ bool SFilterProcess::init(QString _strFilterProcessGroupID, QString _strFilterPr m_SInfluencer.setInitiaizer(m_SInitializer); m_SSpammer.setInitiaizer(m_SInitializer); m_SPowercafe.setInitiaizer(m_SInitializer); + m_SConsumerMarketer.setInitiaizer(m_SInitializer); + m_SConsumerCheckList.setInitiaizer(m_SInitializer); m_SInitializer.closeConnection(SDBManager::E_DATABASE_WEB); return true; } @@ -72,12 +74,14 @@ bool SFilterProcess::run() } case E_PROCSS_TYPE_SEND: { + m_SConsumerMarketer.makeTable(m_SDatagroup); + m_SConsumerCheckList.makeTable(m_SDatagroup); m_SDatagroup.makeTable(); - m_SConsumer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_CONSUMER)); + //m_SConsumer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_CONSUMER)); //m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER)); - m_SSpammer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_SPAMMER)); - m_SPowercafe.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_POWERCAFE)); + //m_SSpammer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_SPAMMER)); + //m_SPowercafe.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_POWERCAFE)); m_SConsumer.makeTable(m_SDatagroup); m_SInfluencer.makeTable(m_SDatagroup); diff --git a/sfilterprocess/sfilterprocess.h b/sfilterprocess/sfilterprocess.h index 33af32f..a6dd6d0 100644 --- a/sfilterprocess/sfilterprocess.h +++ b/sfilterprocess/sfilterprocess.h @@ -17,6 +17,8 @@ private: SInfluencer m_SInfluencer; SSpammer m_SSpammer; SPowercafe m_SPowercafe; + SConsumerMarketer m_SConsumerMarketer; + SConsumerCheckList m_SConsumerCheckList; public: enum E_SERVICE diff --git a/sfilterprocess/sfilterprocess.pro.user b/sfilterprocess/sfilterprocess.pro.user index 8c19990..952da53 100644 --- a/sfilterprocess/sfilterprocess.pro.user +++ b/sfilterprocess/sfilterprocess.pro.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -787,7 +787,7 @@ sfilterprocess Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro - "514" "testall" + "527" "testall2" sfilterprocess.pro false true diff --git a/sfilterprocess/sinfluencer.cpp b/sfilterprocess/sinfluencer.cpp index f51670a..05b629b 100644 --- a/sfilterprocess/sinfluencer.cpp +++ b/sfilterprocess/sinfluencer.cpp @@ -722,7 +722,7 @@ void SInfluencer::SBodyInfluencerMap::makeTable(SDatagroup &_datagroup, const QM stbody.category_num = iterPos.key(); stbody.realin |= REAL_BODY; stbody.community_num = _datagroup.getstBody(strUrl)->community_num; - stbody.body_date = _datagroup.getstBody(strUrl)->date; + stbody.date = _datagroup.getstBody(strUrl)->date; m_listMap.append(stbody); _datagroup.addRealBodyCategory(strUrl, iterPos.key()); if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) @@ -784,7 +784,7 @@ bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_ m_pSInitializer->insertLog(query.lastError().text()); return false; } - strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,community_num,body_date)" + strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,community_num,date)" " values ("; query.clear(); @@ -798,7 +798,7 @@ bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_ str += QString::number(stmap.body_num) + ","; str += QString::number(stmap.realin) + ","; str += QString::number(stmap.community_num) + ","; - str += "'" + stmap.body_date + "')"; + str += "'" + stmap.date + "')"; if (query.exec(str.toUtf8()) == false) { diff --git a/sfilterprocess/sinfluencer.h b/sfilterprocess/sinfluencer.h index abc66c7..01c5c55 100644 --- a/sfilterprocess/sinfluencer.h +++ b/sfilterprocess/sinfluencer.h @@ -73,7 +73,7 @@ public: int category_num; int realin; int community_num; - QString body_date; + QString date; stBodyInfluencerMap():num(0),body_num(0), id_num(0), category_num(0), realin(0), community_num(0){} }; struct stReplyInfluencerMap diff --git a/sfilterprocess/sinitializer.cpp b/sfilterprocess/sinitializer.cpp index db4e7ad..22b509b 100644 --- a/sfilterprocess/sinitializer.cpp +++ b/sfilterprocess/sinitializer.cpp @@ -237,6 +237,8 @@ bool SInitializer::init(const QString& _strFilterProcessGroupID, const QString& b_ok &= initPlatform(); //b_ok &= initConsumerCategory(); b_ok &= initCategory(); + b_ok &= initConsumerMarketer(); + b_ok &= initConsumerCheckList(); /* if (!b_ok) @@ -385,6 +387,74 @@ bool SInitializer::initCategory() } return true; } + + +bool SInitializer::initConsumerMarketer() +{ + if (m_listFilterProcess.isEmpty()) + return false; + + QString strQuery = QString("select id_id, platformname_num from marketer where company_num = ") + + QString::number(m_listFilterProcess.last().m_nCompanyNum); + + QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]); + + if(query.exec(strQuery.toUtf8()) == false) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + + while(query.next()) + m_mapConsumerMaketers.insertMulti(query.value(1).toInt(), + query.value(0).toString().trimmed()); + + return true; +} + +bool SInitializer::initConsumerCheckList() +{ + if (m_listFilterProcess.isEmpty()) + return false; + + QString strQuery = QString("select body_url, id_id, platformname_num from checklist_consumer where company_num = ") + + QString::number(m_listFilterProcess.last().m_nCompanyNum); + + QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]); + + if(query.exec(strQuery.toUtf8()) == false) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + + while(query.next()) + { + if (!query.value(0).toString().trimmed().isEmpty()) + m_slConsumerCheckList.append(query.value(0).toString().trimmed()); + if (!query.value(1).toString().trimmed().isEmpty()) + m_mapConsumerCheckListUser.insertMulti(query.value(2).toInt(), query.value(1).toString().trimmed()); + } + return true; +} + +QMap SInitializer::getConsumerMaketer() +{ + return m_mapConsumerMaketers; +} + +QStringList SInitializer::getConsumerCheckList() +{ + return m_slConsumerCheckList; +} + +QMap SInitializer::getConsumerCheckListUser() +{ + return m_mapConsumerCheckListUser; +} + QString SInitializer::getCategoryName(int _nCategory) const { return m_mapCategory.value(_nCategory); diff --git a/sfilterprocess/sinitializer.h b/sfilterprocess/sinitializer.h index d8ff0a6..35ec7b2 100644 --- a/sfilterprocess/sinitializer.h +++ b/sfilterprocess/sinitializer.h @@ -29,6 +29,8 @@ private: //bool initConsumerCategory(); bool initCategory(); bool initBlogPageNum(); + bool initConsumerMarketer(); + bool initConsumerCheckList(); public: @@ -179,6 +181,9 @@ private: int m_anAllCategory[E_SERVICE_MAX]; QMap m_mapCategory; int m_nBlogPageNum; + QMap m_mapConsumerMaketers; + QMap m_mapConsumerCheckListUser; + QStringList m_slConsumerCheckList; public: bool init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName); @@ -208,6 +213,9 @@ public: int getAllCategoryNum(int _nService); QString getCategoryName(int _nCategory) const; int getBlogPageNum(); + QMap getConsumerMaketer(); + QStringList getConsumerCheckList(); + QMap getConsumerCheckListUser(); //bool disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); //bool enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); //bool disableSettings(QSqlDatabase& db, int _nMode = MODE); diff --git a/sfilterprocess/spowercafe.cpp b/sfilterprocess/spowercafe.cpp index dfc6f45..ffa393b 100644 --- a/sfilterprocess/spowercafe.cpp +++ b/sfilterprocess/spowercafe.cpp @@ -183,7 +183,7 @@ bool SPowercafe::makeOverallCategory(int _categoryNum) bool SPowercafe::makeTable(SDatagroup &_datagroup) { - if (m_mapCategory.size() > 1) + if (m_mapCategory.size() > 0) { m_pSInitializer->insertLog("Make powercafe tables"); m_pSInitializer->insertLog("Make powercafe body tables"); @@ -628,7 +628,7 @@ void SPowercafe::SBodyPowercafeMap::makeTable(SDatagroup &_datagroup, const QMap stbody.community_num = _datagroup.getstBody(strUrl)->community_num; stbody.category_num = iterPos.key(); stbody.realin |= REAL_BODY; - stbody.body_date = _datagroup.getstBody(strUrl)->date; + stbody.date = _datagroup.getstBody(strUrl)->date; stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num; m_listMap.append(stbody); _datagroup.addRealBodyCategory(strUrl, iterPos.key()); @@ -717,7 +717,7 @@ bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nC return false; } strQuery = "insert into body_powercafe_map_" + QString::number(_nCompany) - + "_update (num,category_num,community_num,body_num,realin,body_date,platformname_num)" + + "_update (num,category_num,community_num,body_num,realin,date,platformname_num)" " values ("; query.clear(); @@ -729,7 +729,7 @@ bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nC str += QString::number(stmap.community_num) + ","; str += QString::number(stmap.body_num) + ","; str += QString::number(stmap.realin) + ","; - str += "'" + stmap.body_date + "',"; + str += "'" + stmap.date + "',"; str += QString::number(stmap.platformname_num) + ")"; @@ -911,6 +911,8 @@ void SPowercafe::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) { QMap mapScore; @@ -923,7 +925,8 @@ void SPowercafe::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap mapScore; + const stReal& streal = allCategory; + QDate dateStart(2030, 1, 1); + QDate dateEnd(2000, 1, 1); + + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + if (((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")|| + ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" )) + continue; + QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ","; + strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stPowercafeRank()); + mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[_url])->url; + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->title ; + if (mapScore[strKey].community.trimmed().isEmpty()) + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->id; + } + mapScore[strKey].count++; + QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + if (((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")|| + ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" )) + continue; + QString strUrl = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_url]].trimmed(); + QString strKey = (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ","; + strKey += ((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stPowercafeRank()); + mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[strUrl])->url; + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->title; + if (mapScore[strKey].community.trimmed().isEmpty()) + mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->id; + } + mapScore[strKey].count++; + QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + //QList listSpammer; + QMap > mapRank; + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + QString strPlatformName = iterPos1.key().split(",").at(1); + if (!mapRank.contains(m_pSInitializer->getPlatformName(strPlatformName))) + mapRank.insert(m_pSInitializer->getPlatformName(strPlatformName), QList()); + + mapRank[m_pSInitializer->getPlatformName(strPlatformName)].append(iterPos1.value()); + } + + for (QMap >::iterator iterPos1 = mapRank.begin(); iterPos1 != mapRank.end(); iterPos1++) + qSort(iterPos1.value().begin(), iterPos1.value().end(), isGreaterCountReal()); + + QList keys = mapRank.keys().toSet().toList(); + + QJsonArray jsonArray; + stStatsJson stjsonRank; + { + stjsonRank.category_num = -1; + stjsonRank.servicenum = 4; + stjsonRank.title = "rank"; + stjsonRank.subtitle = ""; + stjsonRank.platformname_num = 1; + } + int nTotalCount = 0; + for (QList::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++) + { + QList values = mapRank.value(*iterPos1); + int nRankLimit = (values.size() <= 10) ? values.size() : 10; + QJsonObject jsonObj; + jsonObj["platformname_num"] = *iterPos1; + QJsonArray jsonSubArray; + for (int i = 0; i < nRankLimit; i++) + { + QJsonObject jsonSubObj; + jsonSubObj["rank"] = i + 1; + jsonSubObj["realcount"] = values.at(i).count; + jsonSubObj["community"] = values.at(i).community; + if (values.at(i).url.contains("instagram.com")) + jsonSubObj["url"] = "https://www." + values.at(i).url; + else if (values.at(i).url.contains("cafe.naver.com")) + jsonSubObj["url"] = "http://" + values.at(i).url; + else if (values.at(i).url.contains("facebook.com")) + jsonSubObj["url"] = "https://www." + values.at(i).url; + else if (values.at(i).url.contains("kakaostory.com")) + jsonSubObj["url"] = "https://" + values.at(i).url; + else if (values.at(i).url.contains("cafe.daum.net")) + jsonSubObj["url"] = "http://" + values.at(i).url; + else + jsonSubObj["url"] = "http://" + values.at(i).url; + jsonSubArray.append(jsonSubObj); + } + jsonObj["data"] = jsonSubArray; + jsonArray.append(jsonObj); + nTotalCount += values.size(); + } + QJsonObject jObj; + jObj["data"] = jsonArray; + jObj["mindate"] = dateStart.toString("yyyy.MM.dd"); + jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd"); + jObj["totalcount"] = nTotalCount; + QJsonDocument jsonDoc(jObj); + stjsonRank.jdata = jsonDoc.toJson(); + m_listMap.append(stjsonRank); + } } bool SPowercafe::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany) { - QSqlQuery query(_db); { QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 4"; diff --git a/sfilterprocess/spowercafe.h b/sfilterprocess/spowercafe.h index 2082004..48b1dfa 100644 --- a/sfilterprocess/spowercafe.h +++ b/sfilterprocess/spowercafe.h @@ -59,7 +59,7 @@ public: int community_num; int category_num; int realin; - QString body_date; + QString date; int platformname_num; stBodyPowercafeMap():num(0), body_num(0), community_num(0), category_num(0), realin(0), platformname_num(0){} }; diff --git a/sfilterprocess/sspammer.cpp b/sfilterprocess/sspammer.cpp index 9edae5b..8417828 100644 --- a/sfilterprocess/sspammer.cpp +++ b/sfilterprocess/sspammer.cpp @@ -582,7 +582,7 @@ void SSpammer::SBodySpammerMap::makeTable(SDatagroup &_datagroup, const QMapplatformname_num; - stbody.body_date = _datagroup.getstBody(strUrl)->date; + stbody.date = _datagroup.getstBody(strUrl)->date; if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) { stbody.realin |= REAL_REPLY; @@ -606,7 +606,7 @@ void SSpammer::SBodySpammerMap::makeTable(SDatagroup &_datagroup, const QMapid_num; stbody.realin |= REAL_REPLY; stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num; - stbody.body_date = _datagroup.getstBody(strUrl)->date; + stbody.date = _datagroup.getstBody(strUrl)->date; foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) { @@ -665,7 +665,7 @@ bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompa m_pSInitializer->insertLog(query.lastError().text()); return false; } - strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,body_date,platformname_num)" + strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,date,platformname_num)" " values ("; query.clear(); @@ -677,7 +677,7 @@ bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompa str += QString::number(stmap.id_num) + ","; str += QString::number(stmap.body_num) + ","; str += QString::number(stmap.realin) + ","; - str += "'" + stmap.body_date + "',"; + str += "'" + stmap.date + "',"; str += QString::number(stmap.platformname_num) + ")"; if (query.exec(str.toUtf8()) == false) { @@ -855,6 +855,7 @@ void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap listBuzzy; + stReal allCategory; // filtered { for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) @@ -864,6 +865,8 @@ void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) { const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + if (!allCategory.m_mapRealReplyUniqueUrl.contains(iterPos1.key()) || !allCategory.m_mapRealReplyUniqueUrl.values(iterPos1.key()).contains(iterPos1.value())) + allCategory.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value()); + allCategory.m_setRealAllUrl.insert(iterPos1.key()); + QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); if (date.year() == 1990) continue; @@ -901,6 +908,38 @@ void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap::const_iterator iterPos1 = allCategory.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != allCategory.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (date.year() == 1990) + continue; + + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + listBuzzy.append(stbuzzy); + } + + //all buzzy { stBuzzy stbuzzy; @@ -936,9 +975,8 @@ void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMapgetAllCategoryNum(3))) { - foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(3)].m_setRealAllUrl) + foreach (const QString& strUrl, allCategory.m_setRealAllUrl) { nTotalBodyCount++; nTotalReplyCount += (*mapReply)[strUrl].size(); @@ -978,10 +1016,12 @@ void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap 0) jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = m_pSInitializer->getCategoryName(stbuzzy.m_nCategory); - else + else if (stbuzzy.m_nCategory == 0) jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALL"; + else + jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALLCATEGORY"; jsNameAll["value"] = jsValueAll; jsNameBody["value"] = jsValueBody; @@ -1076,6 +1116,8 @@ void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMap >* mapReply = _datagroup.getmapReply(); const int* anColumn = m_pSInitializer->getColumnIntArray(); + stReal allCategory; + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) { QMap mapScore; @@ -1083,6 +1125,125 @@ void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMapid_num; + } + mapScore[strKey].body++; + mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + allCategory.m_setRealAllUrl.insert(iterPos1.key()); + if (!allCategory.m_mapRealReplyUniqueUrl.contains(iterPos1.key()) || !allCategory.m_mapRealReplyUniqueUrl.values(iterPos1.key()).contains(iterPos1.value())) + allCategory.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value()); + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(iterPos1.key())->id_num; + } + mapScore[strKey].reply++; + mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date(); + if (dateThis.isValid() && dateThis.year() > 1990) + { + dateStart = (dateStart < dateThis) ? dateStart : dateThis; + dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis; + } + } + + //QList listSpammer; + QMap mapSpammer; + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + if (iterPos1->body >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] || + iterPos1->reply >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] || + iterPos1->nickname.size() >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT]) + { + double value = iterPos1->body * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO] + + iterPos1->reply * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO] + + iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO]; + stSpammerRank rank; + rank.value = value; + rank.id = iterPos1.key().split(",").at(0); + rank.platformform_num = m_pSInitializer->getPlatformName(iterPos1.key().split(",").at(1)); + rank.id_num = iterPos1.value().id_num; + mapSpammer.insertMulti(rank.platformform_num, rank); + } + } + + QList keys = mapSpammer.keys().toSet().toList(); + + QJsonArray jsonArray; + stStatsJson stjsonRank; + { + stjsonRank.category_num = iterPos.key(); + stjsonRank.servicenum = 3; + stjsonRank.title = "rank"; + stjsonRank.subtitle = ""; + stjsonRank.platformname_num = 1; + } + int nTotalCount = 0; + + for (QList::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++) + { + QList values = mapSpammer.values(*iterPos1); + qSort(values.begin(), values.end(), isGreaterSpammerValue()); + int nRankLimit = (values.size() <= 10) ? values.size() : 10; + QJsonObject jsonObj; + jsonObj["platformname_num"] = *iterPos1; + QJsonArray jsonSubArray; + for (int i = 0; i < nRankLimit; i++) + { + QJsonObject jsonSubObj; + jsonSubObj["rank"] = i + 1; + jsonSubObj["value"] = values.at(i).value; + jsonSubObj["id"] = values.at(i).id; + jsonSubObj["id_num"] = values.at(i).id_num; + jsonSubArray.append(jsonSubObj); + } + jsonObj["data"] = jsonSubArray; + jsonArray.append(jsonObj); + + nTotalCount += values.size(); + } + QJsonObject jObj; + jObj["data"] = jsonArray; + jObj["mindate"] = dateStart.toString("yyyy.MM.dd"); + jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd"); + jObj["totalcount"] = nTotalCount; + QJsonDocument jsonDoc(jObj); + stjsonRank.jdata = jsonDoc.toJson(); + m_listMap.append(stjsonRank); + } + + + //all filtered Category + { + QMap mapScore; + const stReal& streal = allCategory; + QDate dateStart(2030, 1, 1); + QDate dateEnd(2000, 1, 1); + foreach (const QString& _url, streal.m_setRealBodyUrl) { QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; @@ -1148,13 +1309,14 @@ void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMap::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++) { QList values = mapSpammer.values(*iterPos1); diff --git a/sfilterprocess/sspammer.h b/sfilterprocess/sspammer.h index 8598380..724a1dd 100644 --- a/sfilterprocess/sspammer.h +++ b/sfilterprocess/sspammer.h @@ -54,7 +54,7 @@ public: int id_num; int category_num; int realin; - QString body_date; + QString date; int platformname_num; stBodySpammerMap():num(0), body_num(0), id_num(0), category_num(0), realin(0), platformname_num(0){}