diff --git a/sfilterprocess/main.cpp b/sfilterprocess/main.cpp new file mode 100644 index 0000000..00cc0fb --- /dev/null +++ b/sfilterprocess/main.cpp @@ -0,0 +1,37 @@ +#include +#include "sfilterprocess.h" +#include +#include +#include + +using namespace std; +int main(int argc, char *argv[]) +{ + cout << "build date : "; + cout << QLocale("en_US").toDate(QString(__DATE__).simplified(), "MMM d yyyy").toString("yyyy-MM-dd").toStdString(); + cout << " "; + cout << QString(__TIME__).toStdString() << endl; + SFilterProcess filter; + + if (argc != 3) + { + cout << "check argument " << endl; + return -1; + } + + + if (!filter.init(argv[1], argv[2])) + //if (!filter.init(QString::number(370), "Check")) + { + cout << "filter init error" << endl; + exit(1); + } + + if (!filter.run()) + { + cout << "filter run error" << endl; + exit(1); + } + + return 0; +} diff --git a/sfilterprocess/sconsumer.cpp b/sfilterprocess/sconsumer.cpp new file mode 100644 index 0000000..a2fe3fb --- /dev/null +++ b/sfilterprocess/sconsumer.cpp @@ -0,0 +1,514 @@ +#include "sconsumer.h" +#include +#include +#include +#include +#include +#include + +bool SConsumer::uploadData(int _companyNum) +{ + + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + m_pSInitializer->insertLog("Start Consumer upload"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Consumer (Start)"); + if (!m_SBodyConsumerMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload body_consumer_map"); + return false; + } + if (!m_SStatsConUserBuzzybyday.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_con_user_buzzbyday"); + return false; + } + m_pSInitializer->insertLog("End Consumer upload"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Consumer (Finish)"); + return true; +} + +bool SConsumer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter) +{ + QVector vecData[SInitializer::E_ARTICLE_MAX]; + bool bFiltered[SInitializer::E_ARTICLE_MAX]; + + for (int i = 0; i < SInitializer::E_ARTICLE_MAX; i++) + bFiltered[i] = false; + _datagroup.copyData(vecData); + foreach (const SInitializer::stFilter& filter, _listFilter) + { + switch(filter.m_nType) + { + case E_FILTER_TYPE_DATE://Date + m_SFilterAlgorithm.FilterDate(vecData, bFiltered, filter.m_vData.toString(), + m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_article_date]); + + break; + case E_FILTER_TYPE_SEARCH: + m_SFilterAlgorithm.FilterSearch(vecData, bFiltered, filter.m_vData.toString()); + + break; + case E_FILTER_TYPE_LENGTH: + m_SFilterAlgorithm.FilterLength(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_REPLACE: + m_SFilterAlgorithm.FilterReplace(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_COUNTER: + m_SFilterAlgorithm.FilterCounter(vecData, bFiltered, filter.m_vData.toString()); + break; + } + } + stReal streal; + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size(); + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size(); + const int *anColumn = m_pSInitializer->getColumnIntArray(); + if (bFiltered[SInitializer::E_ARTICLE_BODY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_BODY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_setRealBodyUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + if (bFiltered[SInitializer::E_ARTICLE_REPLY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_REPLY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_mapRealReplyUniqueUrl.insertMulti(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed(), + strlist[anColumn[SInitializer::E_DATA_article_order]].trimmed().toInt()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + m_mapCategory.insert(_nCategory, streal); + return true; +} + +bool SConsumer::changeTable(int _companyNum) +{ + m_pSInitializer->insertLog("Start change consumer tables"); + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + if (!m_SBodyConsumerMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to body_consumer_change"); + return false; + } + + return true; +} + +void SConsumer::setInitiaizer(SInitializer &_initializer) +{ + m_pSInitializer = &_initializer; + m_SBodyConsumerMap.setInitiaizer(_initializer); + m_SStatsConUserBuzzybyday.setInitiaizer(_initializer); +} + +bool SConsumer::makeOverallCategory(int _nCategory) +{ + if (m_mapCategory.size() < 1) + return true; + + stReal streal; + for (QMap::iterator iterPos = m_mapCategory.begin(); iterPos != m_mapCategory.end(); iterPos++ ) + { + stReal& val = iterPos.value(); + foreach (const QString& _strurl, val.m_setRealBodyUrl ) + streal.m_setRealBodyUrl.insert(_strurl); + foreach (const QString& _strurl, val.m_setRealAllUrl ) + streal.m_setRealAllUrl.insert(_strurl); + + for (QMap::iterator iterPos1 = val.m_mapRealReplyUniqueUrl.begin(); iterPos1 != val.m_mapRealReplyUniqueUrl.end(); + iterPos1++) + { + streal.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value()); + } + } + + m_mapCategory.insert(_nCategory, streal); + return true; +} + +bool SConsumer::makeTable(SDatagroup &_datagroup) +{ + m_pSInitializer->insertLog("Make Consumer Tables"); + m_SBodyConsumerMap.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make Body Tables"); + m_SStatsConUserBuzzybyday.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make Reply Tables"); + return true; +} + +void SConsumer::SBodyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& strUrl, *orderedUrl) + { + if (streal.m_setRealBodyUrl.contains(strUrl)) + { + stBodyConsumerMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.category_num = iterPos.key(); + stbody.realin |= REAL_BODY; + if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stbody.realin |= REAL_REPLY; + + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + } + m_listMap.append(stbody); + _datagroup.addRealBodyCategory(strUrl, iterPos.key()); + } + else if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stBodyConsumerMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.category_num = iterPos.key(); + stbody.realin |= REAL_REPLY; + + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + m_listMap.append(stbody); + } + } + } +} + +bool SConsumer::SBodyConsumerMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "body_consumer_map_" + QString::number(_nCompany) + " TO " + "body_consumer_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "body_consumer_map_" + QString::number(_nCompany) + "_update" + " TO " + "body_consumer_map_" + QString::number(_nCompany) + ","; + strQuery += "body_consumer_map_" + QString::number(_nCompany) + "_temp" + " TO " + "body_consumer_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} +/* +bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_consumer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into body_consumer_map_" + QString::number(_nCompany) + + "_update (num,body_num,category_num,realin) VALUES (:NUM,:BODY_NUM,:CATEGORY_NUM,:REALIN)"); + + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stBodyConsumerMap& stbody, m_listMap) + { + query.bindValue(":NUM", stbody.num); + query.bindValue(":BODY_NUM", stbody.body_num); + query.bindValue(":CATEGORY_NUM", stbody.category_num); + query.bindValue(":REALIN", stbody.realin); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + +bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_consumer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into body_consumer_map_" + QString::number(_nCompany) + + "_update (num,body_num,category_num,realin) VALUES ("); + + query.clear(); + + foreach (const stBodyConsumerMap& stbody, m_listMap) + { + QString str = strQuery; + str += QString::number(stbody.num) + ","; + str += QString::number(stbody.body_num) + ","; + str += QString::number(stbody.category_num) + ","; + str += QString::number(stbody.realin) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + +void SConsumer::SReplyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& strUrl, *orderedUrl) + { + if (streal.m_setRealAllUrl.contains(strUrl)) + { + if (_datagroup.getstBody(strUrl)->reply_count == 0) + continue; + + const QMap* mapReply = _datagroup.getstReply(strUrl); + + for(QMap::const_iterator iterPos1 = mapReply->constBegin(); iterPos1 != mapReply->constEnd(); iterPos1++) + { + stReplyConsumerMap streply; + streply.num = num++; + streply.category_num = iterPos.key(); + } + } + } + } +} + +void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + QDate dateStart(2100,1,1),dateEnd(2000,1,1); + + QMap* mapConsumerCategory = m_pSInitializer->getConsumerCategory(); + QList listBuzzy; + //filtered buzzy + { + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + stBuzzy stbuzzy; + stbuzzy.m_nCategory = (*mapConsumerCategory)[iterPos.key()]; + const stReal &streal = iterPos.value(); + foreach (const QString& strUrl, streal.m_setRealBodyUrl) + { + QDate date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + //if (date.isNull()) + // date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed(), "yyyy-MM-dd HH:mm:ss").date(); + + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + + if (stbuzzy.m_mapBodyBuzz.contains(date)) + stbuzzy.m_mapBodyBuzz[date]++; + else + stbuzzy.m_mapBodyBuzz.insert(date, 1); + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()]; + QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + + listBuzzy.append(stbuzzy); + } + } + + //all buzzy + { + stBuzzy stbuzzy; + stbuzzy.m_nCategory = 0; + for (QMap::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++) + { + QDate date = QDateTime::fromString(iterPos.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (stbuzzy.m_mapBodyBuzz.contains(date)) + stbuzzy.m_mapBodyBuzz[date]++; + else + stbuzzy.m_mapBodyBuzz.insert(date, 1); + } + + for (QMap >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++) + { + for (QMap::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++) + { + QDate date = QDateTime::fromString(iterPos1.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + if (stbuzzy.m_mapReplyBuzz.contains(date)) + stbuzzy.m_mapReplyBuzz[date]++; + else + stbuzzy.m_mapReplyBuzz.insert(date, 1); + } + } + listBuzzy.append(stbuzzy); + } + + { + for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1)) + { + //body + stStatsConUserBuzzybyday stbodybuzz; + stStatsConUserBuzzybyday streplybuzz; + stStatsConUserBuzzybyday stallbuzz; + foreach (const stBuzzy& stbuzzy, listBuzzy) + { + if (stbuzzy.m_mapBodyBuzz.contains(i)) + { + stbodybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapBodyBuzz[i]; + stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapBodyBuzz[i]; + } + + if (stbuzzy.m_mapReplyBuzz.contains(i)) + { + streplybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapReplyBuzz[i]; + stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapReplyBuzz[i]; + } + } + stallbuzz.date = streplybuzz.date = stbodybuzz.date = i; + stbodybuzz.subject = "body"; + streplybuzz.subject = "reply"; + stallbuzz.subject = "all"; + m_listMap.append(stallbuzz); + m_listMap.append(stbodybuzz); + m_listMap.append(streplybuzz); + } + } +} + +/* +bool SConsumer::SStatsConUserBuzzybyday::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + +} +*/ + +/* +bool SConsumer::SStatsConUserBuzzybyday::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QSqlQuery query(_db); + { + QString strQuery = "delete from stats_con_user_buzzbyday where company_num = " + QString::number(_nCompany); + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + //_db.transaction(); + QString strQuery = "insert into stats_con_user_buzzbyday ("; + for (int i = 0; i < 11; i++) + { + strQuery += ("category_" + QString::number(i) + ","); + } + + strQuery += "date,subject,company_num) values ("; + for (int i = 0; i < 11; i++) + { + strQuery += (":category_" + QString::number(i) + ","); + } + + strQuery += ":date,:subject,:company_num)"; + + query.prepare(strQuery); + + foreach (const stStatsConUserBuzzybyday& ststats, m_listMap) + { + for (int i = 0; i < 11; i++) + { + query.bindValue(":category_" + QString::number(i), ststats.category[i]); + } + query.bindValue(":date", ststats.date.toString("yyyy-MM-dd")); + query.bindValue(":subject", ststats.subject); + query.bindValue(":company_num", _nCompany); + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} +*/ + + +bool SConsumer::SStatsConUserBuzzybyday::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QSqlQuery query(_db); + { + QString strQuery = "delete from stats_con_user_buzzbyday where company_num = " + QString::number(_nCompany); + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + //_db.transaction(); + QString strQuery = "insert into stats_con_user_buzzbyday ("; + for (int i = 0; i < 11; i++) + { + strQuery += ("category_" + QString::number(i) + ","); + } + + strQuery += "date,subject,company_num) values ("; + + foreach (const stStatsConUserBuzzybyday& ststats, m_listMap) + { + QString str = strQuery; + for (int i = 0; i < 11; i++) + { + str += QString::number(ststats.category[i]) + ","; + } + str += "'" + ststats.date.toString("yyyy-MM-dd") + "',"; + str += "'" + ststats.subject + "',"; + str += QString::number(_nCompany) + ")"; + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} diff --git a/sfilterprocess/sconsumer.h b/sfilterprocess/sconsumer.h new file mode 100644 index 0000000..2f596c0 --- /dev/null +++ b/sfilterprocess/sconsumer.h @@ -0,0 +1,110 @@ +#ifndef SCONSUMER +#define SCONSUMER +#include "sservice.h" +#include "sfilteralgorithm.h" +#include +class SConsumer:public SService +{ +public: + struct stReal + { + QSet m_setRealAllUrl; + QSet m_setRealBodyUrl; + QMap m_mapRealReplyUniqueUrl; //multiple + }; + struct stBodyConsumerMap + { + int num; + int body_num; + int category_num; + int realin; + stBodyConsumerMap():num(0), body_num(0), category_num(0), realin(0){} + }; + struct stReplyConsumerMap + { + int num; + int reply_num; + int body_num; + int id_num; + int category_num; + int realin; + stReplyConsumerMap():num(0), reply_num(0), body_num(0), id_num(0), category_num(0), realin(0) {} + }; + struct stStatsConUserBuzzybyday + { + int category[11]; + QDate date; + QString subject; + int company_num; + stStatsConUserBuzzybyday():company_num(0) + { + for (int i = 0; i < 11; i++) + category[i] = 0; + } + }; + struct stBuzzy + { + int m_nCategory; + QMap m_mapBodyBuzz; + QMap m_mapReplyBuzz; + }; + + class SBodyConsumerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool changeTable(QSqlDatabase& _db, const int &_nCompany); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear() { m_listMap.clear(); } + }; + class SReplyConsumerMap // Why did you define this? + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool changeTable(QSqlDatabase& _db, const int &_nCompany); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear(){ m_listMap.clear(); } + + }; + class SStatsConUserBuzzybyday + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + //bool changeTable(QSqlDatabase& _db, const int &_nCompany); + bool uploadData(QSqlDatabase& _db, const int &_nCompany); + void clear(){ m_listMap.clear(); } + }; + +private: + SFilterAlgorithm m_SFilterAlgorithm; + QMap m_mapCategory; + SBodyConsumerMap m_SBodyConsumerMap; + SStatsConUserBuzzybyday m_SStatsConUserBuzzybyday; + +public: + virtual void setInitiaizer(SInitializer &_initializer); + virtual bool uploadData(int _companyNum); + virtual bool changeTable(int _companyNum); + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter); + virtual bool makeOverallCategory(int _nCategory); + virtual bool makeTable(SDatagroup &_datagroup); +}; + + + + + +#endif // SCONSUMER + diff --git a/sfilterprocess/sdatagroup.cpp b/sfilterprocess/sdatagroup.cpp new file mode 100644 index 0000000..43fab14 --- /dev/null +++ b/sfilterprocess/sdatagroup.cpp @@ -0,0 +1,1254 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sdatagroup.h" +#include "sinfluencer.h" +#include "sinitializer.h" + +QString sqlString(QString _str) +{ + _str.replace("\\","\\\\"); + _str.replace("'","\\'"); + _str.replace("\"","\\\""); + _str.replace(",","\\,"); + return _str; +} + +void SDatagroup::setInitializer(SInitializer &initializer) +{ + m_pSInitializer = &initializer; + m_SBody.setInitializer(initializer); + m_SCommunity.setInitializer(initializer); + m_SReply.setInitializer(initializer); + m_SId.setInitializer(initializer); + m_SNickname.setInitializer(initializer); +} + +void SDatagroup::init() +{ + m_mapBody.clear(); + m_mapReply.clear(); + m_setFilteredUrl.clear(); + m_slOrderedBodyUrl.clear(); +} + +void SDatagroup::setloadQuery(QString _astrQuery[], int _nDatagroupNumber) +{ + QStringList slColumn = m_pSInitializer->getColumnStringList(); + QString strQuery("select "); + foreach (const QString& str, slColumn) + strQuery += (str + ","); + strQuery = strQuery.left(strQuery.length() - 1); + strQuery += " from data_"; + strQuery += QString::number(_nDatagroupNumber); + _astrQuery[E_ARTICLE_BODY] = _astrQuery[E_ARTICLE_REPLY] = strQuery; + + _astrQuery[E_ARTICLE_REPLY] += " where article_form = 'reply'"; + _astrQuery[E_ARTICLE_BODY] += " where article_form = 'body'"; + _astrQuery[E_ARTICLE_BODY] += " and date(article_date) >= date('" + m_pSInitializer->getStartDay() + "')"; + _astrQuery[E_ARTICLE_BODY] += " and date(article_date) <= date('" + m_pSInitializer->getEndDay() + "')"; + _astrQuery[E_ARTICLE_BODY] += " Order by article_date DESC"; +} + + +bool SDatagroup::loadData(const int& _nDatagroupNumber) +{ + init(); + QSqlQuery query(m_pSInitializer->getDatabase(SDBManager::E_DATABASE_DATA)); + const int *anColumn = m_pSInitializer->getColumnIntArray(); + + QString astrWhere[E_ARTICLE_MAX]; + setloadQuery(astrWhere, _nDatagroupNumber); + + m_pSInitializer->insertLog("Start Loding data_" + QString::number(_nDatagroupNumber)); + + QSet setCheckDupUrl; + for (int i = 0; i < E_ARTICLE_MAX; i++) + { + if (query.exec(astrWhere[i].toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + while (query.next()) + { + QStringList strList; + for (int j = 0; j < m_pSInitializer->getColumnStringList().size(); j++) + strList.append(" " + query.value(j).toString() + " "); + + if(strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "post" || + strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "story" || + strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "channel") + strList[anColumn[SInitializer::E_DATA_platform_form]] = " sns "; + + if(strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "group") + strList[anColumn[SInitializer::E_DATA_platform_form]] = " community "; + + switch(i) + { + case E_ARTICLE_BODY: + { + if (setCheckDupUrl.contains(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed())) + { + continue; + } + if (strList[anColumn[SInitializer::E_DATA_article_id]].trimmed().isEmpty() && strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news") + { + strList[anColumn[SInitializer::E_DATA_article_id]] = strList[anColumn[SInitializer::E_DATA_platform_title]]; + strList[anColumn[SInitializer::E_DATA_article_nickname]] = strList[anColumn[SInitializer::E_DATA_platform_title]]; + } + if (strList[anColumn[SInitializer::E_DATA_article_date]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_article_date]].trimmed() == "0000-00-00 00:00:00") + strList[anColumn[SInitializer::E_DATA_article_date]] = "1990-01-01 00:00:00"; + if (strList[anColumn[SInitializer::E_DATA_article_id]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_article_nickname]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_platform_id]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_platform_id]].trimmed().isEmpty()) continue; + setCheckDupUrl.insert(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + m_slOrderedBodyUrl.append(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + m_mapBody.insert(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed(), strList); + //profiles + makeProfile(strList); + + }break; + + case E_ARTICLE_REPLY: + { + if (!setCheckDupUrl.contains(strList[anColumn[SInitializer::E_DATA_article_url]].trimmed())) + continue; + if (strList[anColumn[SInitializer::E_DATA_article_id]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_article_nickname]].trimmed().isEmpty()) continue; + if (strList[anColumn[SInitializer::E_DATA_article_date]].trimmed() == "0000-00-00 00:00:00" || strList[anColumn[SInitializer::E_DATA_article_date]].trimmed().isEmpty()) + strList[anColumn[SInitializer::E_DATA_article_date]] = "1990-01-01 00:00:00"; + QString strKey = strList[anColumn[SInitializer::E_DATA_article_url]].trimmed(); + int nKey = strList[anColumn[SInitializer::E_DATA_article_order]].trimmed().toInt(); + + if (m_mapReply.contains(strKey)) + { + if (!m_mapReply[strKey].contains(nKey)) + { + m_mapReply[strKey].insert(nKey, strList); + } + } + else + { + QMap reply; + reply.insert(nKey, strList); + m_mapReply.insert(strKey, reply); + } + }break; + } + } + query.clear(); + } + + int ncReply = 0; + for (QMap >::iterator iterPos1 = m_mapReply.begin(); iterPos1 != m_mapReply.end(); iterPos1++) + { + for (QMap::iterator iterPos2 = iterPos1.value().begin(); iterPos2 != iterPos1.value().end(); iterPos2++) + { + ncReply++; + } + } + qDebug() << "body size = " << m_mapBody.size(); + qDebug() << "reply size = " << ncReply; + + m_stBodyReplyCount.m_nAllBodyCount = m_mapBody.size(); + m_stBodyReplyCount.m_nAllReplyCount = ncReply; + m_pSInitializer->insertLog("End Loading"); + return true; +} + + +void SDatagroup::makeProfile(const QStringList& _strList) +{ + const int *anColumn = m_pSInitializer->getColumnIntArray(); + stProfileurlKey stprofileurlkey(m_pSInitializer->getPlatformForm( + _strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed()), + _strList[anColumn[SInitializer::E_DATA_article_id]].trimmed()); + if (m_mapProfile.contains(stprofileurlkey)) + { + if (!(*(m_mapProfile[stprofileurlkey]))[SInitializer::E_DATA_article_profileurl].trimmed().isEmpty()) + { + if ((*(m_mapProfile[stprofileurlkey]))[SInitializer::E_DATA_article_date].trimmed() < _strList[SInitializer::E_DATA_article_date].trimmed()) + { + m_mapProfile[stprofileurlkey] = &(m_mapBody[_strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()]); + } + } + } + else + { + m_mapProfile.insert(stprofileurlkey, &(m_mapBody[_strList[anColumn[SInitializer::E_DATA_article_url]].trimmed()])); + } +} + +void SDatagroup::addRealBodyCategory(const QString& _strUrl, const int _nCategory) +{ + m_SBody.putCategory(_strUrl, _nCategory); +} + +void SDatagroup::addRealReplyCategory(const QString& _strUrl, const int& _order, int _nCategory) +{ + m_SReply.putCategory(_strUrl, _order, _nCategory); +} + + +void SDatagroup::addRealUrl(const QString& _strUrl) +{ + m_setFilteredUrl.insert(_strUrl.trimmed()); +} + + +bool SDatagroup::uploadData(const int &_nCompany) +{ + m_pSInitializer->insertLog("Start upload data"); + m_pSInitializer->insertLog("Start upload body data"); + + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + m_pSInitializer->updateWebDBInfoState(dbWeb, _nCompany, "Datagoup (Start)"); + + if (!m_SBody.uploadData(dbWeb, _nCompany, m_slOrderedBodyUrl)) + { + m_pSInitializer->insertLog("Change body_" + QString::number(_nCompany) + " fail"); + return false; + } + m_pSInitializer->insertLog("Start upload reply data"); + if (!m_SReply.uploadData(dbWeb, _nCompany, m_slOrderedBodyUrl)) + { + m_pSInitializer->insertLog("upload reply_" + QString::number(_nCompany) + " fail"); + return false; + } + + m_pSInitializer->insertLog("Start upload id data"); + + if (!m_SId.uploadData(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("upload id_" + QString::number(_nCompany) + " fail"); + return false; + } + m_pSInitializer->insertLog("Start upload nickname data"); + + if (!m_SNickname.uploadData(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("upload nickname_" + QString::number(_nCompany) + " fail"); + return false; + } + m_pSInitializer->insertLog("Start upload community data"); + + if (!m_SCommunity.uploadData(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("upload community_" + QString::number(_nCompany) + " fail"); + return false; + } + m_pSInitializer->insertLog("Success upload data"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _nCompany, "Datagoup (End)"); + return true; +} + +bool SDatagroup::makeTable() +{ + QRegExp rxHttp("http[s]?://"); + m_pSInitializer->insertLog("Start makeTable"); + m_pSInitializer->insertLog("#ofALL/#ofFiltered :" + QString::number(m_mapBody.size()) + "/" + QString::number(m_setFilteredUrl.size()) ); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + int nBody = 0; + int nId = 0; + int nNickname = 0; + int nCommunity = 0; + int nReply = 0; + + { + stCommunity stcommunity; + stcommunity.num = nCommunity++; + m_SCommunity.put(" ", stcommunity); + + stId stid; + stid.num = nId++; + m_SId.put("(none)", stid); + } + + foreach (const QString& strUrl, m_slOrderedBodyUrl) + { + if (m_setFilteredUrl.contains(strUrl)) + { + QStringList &strListBody = m_mapBody[strUrl]; + QMap &mapReply = m_mapReply[strUrl]; + + // community + if (!m_SCommunity.contains(m_SCommunity.getCommunityKey(strListBody).trimmed())) + { + stCommunity stcommunity; + stcommunity.num = nCommunity++; + stcommunity.platformform_num = m_pSInitializer->getPlatformForm(strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed()); + stcommunity.platformname_num = m_pSInitializer->getPlatformName(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed()); + stcommunity.title = strListBody[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + stcommunity.id = strListBody[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + stcommunity.url = m_SCommunity.getCommunityKey(strListBody).trimmed(); + if(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed() == QString("daum") && strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == QString("cafe")) + stcommunity.exid = strListBody[anColumn[SInitializer::E_DATA_etc]].trimmed(); + m_SCommunity.put(m_SCommunity.getCommunityKey(strListBody).trimmed(), stcommunity); + } + + //m_SReply + + if (!m_SId.contains(m_SId.getIdKey(strListBody))) + { + stId stid; + stid.num = nId++; + stid.community_num = 0;//m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody).trimmed())->num; + stid.platformname_num = m_pSInitializer->getPlatformName(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed()); + stid.id = strListBody[anColumn[SInitializer::E_DATA_article_id]].trimmed(); + m_SId.put(strListBody[anColumn[SInitializer::E_DATA_article_id]].trimmed(), stid); + } + + if (!m_SNickname.contains(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())) + { + stNickname stnickname; + stnickname.nickname = strListBody[anColumn[SInitializer::E_DATA_article_nickname]]; + stnickname.num = nNickname++; + m_SNickname.put(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed(), stnickname); + } + + QString lastupdate = strListBody[anColumn[SInitializer::E_DATA_article_date]].trimmed(); + if (mapReply.size() > 0) + { + for(QMap::iterator iterPos = mapReply.begin(); iterPos != mapReply.end(); iterPos++) + { + QStringList &strListReply = iterPos.value(); + + lastupdate = ((strListReply[anColumn[SInitializer::E_DATA_article_date]].trimmed() > lastupdate)? + strListReply[anColumn[SInitializer::E_DATA_article_date]].trimmed() : lastupdate); + + m_stDate.m_strLastupdate = (( m_stDate.m_strLastupdate > lastupdate ) ? m_stDate.m_strLastupdate : lastupdate); + + if (!m_SId.contains(strListReply[anColumn[SInitializer::E_DATA_article_id]].trimmed())) + { + stId stid; + stid.num = nId++; + stid.community_num = 0;//m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody).trimmed())->num; + stid.platformname_num = m_pSInitializer->getPlatformName(strListReply[anColumn[SInitializer::E_DATA_platform_name]].trimmed()); + stid.id = strListReply[anColumn[SInitializer::E_DATA_article_id]].trimmed(); + m_SId.put(strListReply[anColumn[SInitializer::E_DATA_article_id]].trimmed(), stid); + } + + if (!m_SNickname.contains(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())) + { + stNickname stnickname; + stnickname.nickname = strListReply[anColumn[SInitializer::E_DATA_article_nickname]]; + stnickname.num = nNickname++; + m_SNickname.put(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed(), stnickname); + } + } + } + { + if (m_SId.contains(m_SId.getIdKey(strListBody)) && m_SNickname.contains(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())) + m_SNickname.get(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->id_num = + m_SId.get(m_SId.getIdKey(strListBody))->num; + } + + // m_SBody + { + stBody stbody; + stbody.num = nBody++; + stbody.data = &(strListBody[anColumn[SInitializer::E_DATA_article_data]]); + stbody.title = &(strListBody[anColumn[SInitializer::E_DATA_article_title]]); + stbody.reply_count = mapReply.size(); + stbody.id_id = strListBody[anColumn[SInitializer::E_DATA_article_id]].trimmed(); + stbody.date = strListBody[anColumn[SInitializer::E_DATA_article_date]].trimmed(); + stbody.lastupdate = lastupdate; + stbody.view = ( (strListBody[anColumn[SInitializer::E_DATA_article_hit]].trimmed().isEmpty()) ? -1 : + strListBody[anColumn[SInitializer::E_DATA_article_hit]].trimmed().toInt()); + stbody.platformform_num = m_pSInitializer->getPlatformForm(strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed()); + stbody.platformname_num = m_pSInitializer->getPlatformName(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed()); + stbody.url = strListBody[anColumn[SInitializer::E_DATA_article_url]].trimmed().replace(rxHttp, ""); + stbody.id_num = m_SId.get(m_SId.getIdKey(strListBody))->num; + stbody.community_num = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->num; + stbody.nickname_num = m_SNickname.get(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->num; + stbody.reply_startnum = ((mapReply.size() > 0) ? nReply : -1); + stbody.reply_endnum = ((mapReply.size() > 0) ? nReply + mapReply.size() - 1 : -1); + m_SBody.put(strUrl, stbody); + + m_stDate.m_strMaxdate = (( m_stDate.m_strMaxdate > stbody.date ) ? m_stDate.m_strMaxdate : stbody.date ); + m_stDate.m_strMindate = (( m_stDate.m_strMindate < stbody.date ) ? m_stDate.m_strMindate : stbody.date ); + } + // m_SReply + if (mapReply.size() > 0) + { + for(QMap::iterator iterPos = mapReply.begin(); iterPos != mapReply.end(); iterPos++) + { + QStringList &strListReply = iterPos.value(); + stReply streply; + streply.num = nReply++; + streply.body_num = m_SBody.getBodyNum(strUrl); + streply.id_id = strListReply[anColumn[SInitializer::E_DATA_article_id]].trimmed(); + streply.id_num = m_SId.get(m_SId.getIdKey(strListReply))->num; + streply.nickname_num = m_SNickname.get(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->num; + streply.community_num = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->num; + streply.data = &(strListReply[anColumn[SInitializer::E_DATA_article_data]]); + streply.date = strListReply[anColumn[SInitializer::E_DATA_article_date]].trimmed(); + streply.parent = strListReply[anColumn[SInitializer::E_DATA_article_parent]].trimmed(); + if (streply.parent.isEmpty()) + streply.parent = " "; + if (m_SId.contains(m_SId.getIdKey(strListReply)) && m_SNickname.contains(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())) + m_SNickname.get(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->id_num = + m_SId.get(m_SId.getIdKey(strListReply))->num; + m_SReply.put(strUrl, iterPos.key(), streply); + } + } + } + } + m_pSInitializer->insertLog("End makeTable"); + m_stBodyReplyCount.m_nFilterBodyCount = nBody; + m_stBodyReplyCount.m_nFilterReplyCount = nReply; + return true; +} + +const SDatagroup::stDate* SDatagroup::getstDate() +{ + return &m_stDate; +} + +const SDatagroup::stBodyReplyCount* SDatagroup::getCounts() +{ + return &m_stBodyReplyCount; +} + +const QStringList* SDatagroup::getmapBody(const stProfileurlKey& _stprofileurlkey) +{ + return m_mapProfile[_stprofileurlkey]; +} + +bool SDatagroup::changeTable(const int &_nCompany) +{ + m_pSInitializer->insertLog("Start change tables"); + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + if (!m_SBody.changeTable(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("Change body_" + QString::number(_nCompany) + " fail"); + return false; + } + + if (!m_SReply.changeTable(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("Change reply_" + QString::number(_nCompany) + " fail"); + return false; + } + + if (!m_SId.changeTable(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("Change id_" + QString::number(_nCompany) + " fail"); + return false; + } + + if (!m_SNickname.changeTable(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("Change nickname_" + QString::number(_nCompany) + " fail"); + return false; + } + + if (!m_SCommunity.changeTable(dbWeb, _nCompany)) + { + m_pSInitializer->insertLog("Change community_" + QString::number(_nCompany) + " fail"); + return false; + } + m_pSInitializer->insertLog("Success change tables"); + return true; +} + +void SDatagroup::copyData(QVector *_data) +{ + for(QMap::const_iterator iterPos = m_mapBody.begin(); iterPos != m_mapBody.end(); iterPos++) + _data[SInitializer::E_ARTICLE_BODY].append(iterPos.value()); + + for(QMap >::const_iterator iterPos1 = m_mapReply.begin(); iterPos1 != m_mapReply.end(); iterPos1++) + { + for(QMap::const_iterator iterPos2 = iterPos1.value().begin(); iterPos2 != iterPos1.value().end(); iterPos2++) + { + _data[SInitializer::E_ARTICLE_REPLY].append(iterPos2.value()); + } + } + +} + +const SDatagroup::stCommunity* SDatagroup::getstCommunity(const QStringList& _sl) +{ + return m_SCommunity.get(m_SCommunity.getCommunityKey(_sl)); +} + +const QStringList* SDatagroup::getOrderedUrl() +{ + return &m_slOrderedBodyUrl; +} + + +SDatagroup::stBody* SDatagroup::SBody::get(const QString &_url) +{ + if (m_mapBody.contains(_url)) + return &(m_mapBody[_url]); + else + return 0; +} + +const SDatagroup::stBody* SDatagroup::getstBody(const QString& _strUrl) +{ + return m_SBody.get(_strUrl); +} + +const SDatagroup::stReply* SDatagroup::getstReply(const QString& _strUrl, const int &_order) +{ + return m_SReply.get(_strUrl, _order); +} + +const QMap* SDatagroup::getstReply(const QString& _strUrl) +{ + return m_SReply.get(_strUrl); +} + +int SDatagroup::getstBodyNum(const QString& _strUrl) +{ + return m_SBody.getBodyNum(_strUrl); +} +int SDatagroup::getstReplyNum(const QString& _strUrl, const int &_order) +{ + return m_SReply.getReplyNum(_strUrl, _order); +} + +const QMap* SDatagroup::getmapBody() +{ + return &m_mapBody; +} + +const QMap >* SDatagroup::getmapReply() +{ + return &m_mapReply; +} + + +bool SDatagroup::SBody::putCategory(const QString &_url, const int &_nCategory) +{ + if (m_mapBody.contains(_url.trimmed())) + { + m_mapBody[_url.trimmed()].realbycategorynum.append(_nCategory); + return true; + } + else + return false; +} + +void SDatagroup::SBody::put(const QString &_url, const stBody &_stbody) +{ + m_mapBody.insert(_url.trimmed(), _stbody); +} + +void SDatagroup::SBody::clear() +{ + m_mapBody.clear(); +} + +QString SDatagroup::SBody::vecToString(const QVector& _realbycategorynum) +{ + QString str = ""; + foreach (int real, _realbycategorynum) + str += (QString::number(real) + " "); + return str.trimmed(); +} + +/* +bool SDatagroup::SBody::uploadData(QSqlDatabase &_db, const int& _nCompany, const QStringList &_order) +{ + QString strQuery = "truncate body_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if(query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into body_" + QString::number(_nCompany) + + "_update(num,id_num,nickname_num,community_num,id_id,platformname_num,platformform_num, title, data, url, view, date, reply_startnum, reply_endnum, reply_count, lastupdate, realbycategorynum)" + " VALUES (:NUM,:ID_NUM,:NICKNAME_NUM,:COMMUNITY_NUM,:ID_ID,:PLATFORMNAME_NUM,:PLATFORMFORM_NUM,:TITLE,:DATA,:URL,:VIEW,:DATE,:REPLY_STARTNUM,:REPLY_ENDNUM,:REPLY_COUNT,:LASTUPDATE,:REALBYCATEGORYNUM)").toUtf8(); + query.prepare(strQuery); + + foreach (const QString& url, _order) + { + if (m_mapBody.contains(url.trimmed())) + { + stBody &body = m_mapBody[url.trimmed()]; + query.bindValue(":NUM", body.num); + query.bindValue(":ID_NUM", body.id_num); + query.bindValue(":NICKNAME_NUM", body.nickname_num); + query.bindValue(":COMMUNITY_NUM", body.community_num); + query.bindValue(":ID_ID", body.id_id); + query.bindValue(":PLATFORMNAME_NUM", body.platformname_num); + query.bindValue(":PLATFORMFORM_NUM", body.platformform_num); + query.bindValue(":TITLE", sqlString(*(body.title)).replace(QChar(160),"")); + query.bindValue(":DATA", sqlString(*(body.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"").left(65535)); + query.bindValue(":URL", body.url); + query.bindValue(":VIEW", body.view); + query.bindValue(":DATE", body.date); + query.bindValue(":REPLY_STARTNUM", body.reply_startnum); + query.bindValue(":REPLY_ENDNUM", body.reply_endnum); + query.bindValue(":REPLY_COUNT", body.reply_count); + query.bindValue(":LASTUPDATE", body.lastupdate); + query.bindValue(":REALBYCATEGORYNUM", vecToString(body.realbycategorynum)); + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + + return false; + } + } + } + return true; +} +*/ + +bool SDatagroup::SBody::uploadData(QSqlDatabase &_db, const int& _nCompany, const QStringList &_order) +{ + QString strQuery = "truncate body_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if(query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = "insert into body_" + QString::number(_nCompany) + + "_update(num,id_num,nickname_num,community_num,id_id,platformname_num,platformform_num, title, data, url, view, date, reply_startnum, reply_endnum, reply_count, lastupdate, realbycategorynum) VALUES("; + + foreach (const QString& url, _order) + { + if (m_mapBody.contains(url.trimmed())) + { + stBody &body = m_mapBody[url.trimmed()]; + QString str = strQuery; + str += QString::number(body.num) + ","; + str += QString::number(body.id_num) + ","; + str += QString::number(body.nickname_num) + ","; + str += QString::number(body.community_num) + ","; + str += "'" + sqlString(body.id_id) + "',"; + str += QString::number(body.platformname_num) + ","; + str += QString::number(body.platformname_num) + ","; + str += "'" + sqlString(*(body.title)).replace(QChar(160),"") + "',"; + str += "'" + sqlString(*(body.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"").left(65535) + "',"; + str += "'" + body.url + "',"; + str += QString::number(body.view) + ","; + str += "'" + body.date + "',"; + str += QString::number(body.reply_startnum) + ","; + str += QString::number(body.reply_endnum) + ","; + str += QString::number(body.reply_count) + ","; + str += "'" + body.lastupdate + "',"; + str += "'" + vecToString(body.realbycategorynum) + "')"; + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + } + return true; +} + + +bool SDatagroup::SBody::changeTable(QSqlDatabase &_db, const int &nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "body_" + QString::number(nCompany) + " TO " + "body_" + QString::number(nCompany) + "_temp" + ","; + strQuery += "body_" + QString::number(nCompany) + "_update" + " TO " + "body_" + QString::number(nCompany) + ","; + strQuery += "body_" + QString::number(nCompany) + "_temp" + " TO " + "body_" + QString::number(nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + +int SDatagroup::SBody::getBodyNum(const QString &_url) +{ + if (m_mapBody.contains(_url)) + return m_mapBody[_url].num; + else + return -1; +} + + +void SDatagroup::SReply::put(const QString &_url, const int &_order, const stReply &_streply) +{ + + if (m_mapReply.contains(_url.trimmed())) + { + if (!m_mapReply[_url.trimmed()].contains(_order)) + { + m_mapReply[_url.trimmed()].insert(_order, _streply); + } + } + else + { + QMap streply; + streply.insert(_order, _streply); + m_mapReply.insert(_url.trimmed(), streply); + } +} + +bool SDatagroup::SReply::putCategory(const QString &_url, const int &_order, int &_nCategory) +{ + if (m_mapReply.contains(_url.trimmed()) && m_mapReply[_url.trimmed()].contains(_order)) + { + m_mapReply[_url.trimmed()][_order].realbycategorynum.push_back(_nCategory); + return true; + } + else + { + return false; + } +} + +QMap* SDatagroup::SReply::get(const QString& _url) +{ + if (m_mapReply.contains(_url.trimmed())) + return &(m_mapReply[_url.trimmed()]); + else + return 0; +} + +void SDatagroup::SReply::clear() +{ + m_mapReply.clear(); +} + +int SDatagroup::SReply::getReplyNum(const QString &_url, const int &_order) +{ + if (m_mapReply.contains(_url.trimmed()) && m_mapReply[_url.trimmed()].contains(_order)) + return m_mapReply[_url.trimmed()][_order].num; + else + return -1; +} + +SDatagroup::stReply* SDatagroup::SReply::get(const QString &_url, const int &_order) +{ + if (m_mapReply.contains(_url.trimmed()) && m_mapReply[_url.trimmed()].contains(_order)) + return &(m_mapReply[_url.trimmed()][_order]); + else + return 0; +} + +/* +bool SDatagroup::SReply::uploadData(QSqlDatabase &_db, const int &_nCompany, QStringList &_order) +{ + QString strQuery = "truncate reply_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if(query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into reply_" + QString::number(_nCompany) + "_update (num,id_num,nickname_num,community_num,id_id,parent,data,date,body_num,realbycategorynum) VALUES " + "(:NUM,:ID_NUM,:NICKNAME_NUM,:COMMUNITY_NUM,:ID_ID,:PARENT,:DATA,:DATE,:BODY_NUM,:REALBYCATEGORYNUM)").toUtf8(); + query.prepare(strQuery); + + foreach (const QString& url, _order) + { + if (m_mapReply.contains(url)) + { + foreach (const stReply& reply, m_mapReply[url].values()) + { + query.bindValue(":NUM", reply.num); + query.bindValue(":ID_NUM", reply.id_num); + query.bindValue(":NICKNAME_NUM", reply.nickname_num); + query.bindValue(":COMMUNITY_NUM", reply.community_num); + query.bindValue(":ID_ID", reply.id_id); + query.bindValue(":PARENT", reply.parent); + query.bindValue(":DATA", sqlString(*(reply.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"")); + query.bindValue(":DATE", reply.date); + query.bindValue(":BODY_NUM", reply.body_num); + query.bindValue(":REALBYCATEGORYNUM", vecToString(reply.realbycategorynum)); + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + + return false; + } + } + } + } + return true; +} +*/ + +bool SDatagroup::SReply::uploadData(QSqlDatabase &_db, const int &_nCompany, QStringList &_order) +{ + QString strQuery = "truncate reply_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if(query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = "insert into reply_" + QString::number(_nCompany) + "_update (num,id_num,nickname_num,community_num,id_id,parent,data,date,body_num,realbycategorynum) VALUES ("; + + foreach (const QString& url, _order) + { + if (m_mapReply.contains(url)) + { + foreach (const stReply& reply, m_mapReply[url].values()) + { + QString str = strQuery; + str += QString::number(reply.num) + ","; + str += QString::number(reply.id_num) + ","; + str += QString::number(reply.nickname_num) + ","; + str += QString::number(reply.community_num) + ","; + str += "'" + sqlString(reply.id_id) + "',"; + str += "'" + sqlString(reply.parent) + "',"; + str += "'" + sqlString(*(reply.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"") + "',"; + str += "'" + reply.date + "',"; + str += QString::number(reply.body_num) + ","; + str += "'" + vecToString(reply.realbycategorynum) + "')"; + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + + return false; + } + } + } + } + return true; +} + + +QString SDatagroup::SReply::vecToString(const QVector& _realbycategorynum) +{ + QString str = ""; + foreach (int real, _realbycategorynum) + str += (QString::number(real) + " "); + return str.trimmed(); +} + + +bool SDatagroup::SReply::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "reply_" + QString::number(_nCompany) + " TO " + "reply_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "reply_" + QString::number(_nCompany) + "_update" + " TO " + "reply_" + QString::number(_nCompany) + ","; + strQuery += "reply_" + QString::number(_nCompany) + "_temp" + " TO " + "reply_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + +bool SDatagroup::SId::contains(const QString &_strKey) +{ + return m_mapId.contains(_strKey.trimmed()); +} + +void SDatagroup::SId::put(const QString &_strKey, const stId &_stid) +{ + m_mapId.insert(_strKey.trimmed(), _stid); +} + +void SDatagroup::SId::put(const stId &_stid) +{ + m_mapId.insert(_stid.id, _stid); +} + +QString SDatagroup::SId::getIdKey(const QStringList &_strlist) +{ + return _strlist[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_article_id]].trimmed(); +} + +SDatagroup::stId* SDatagroup::SId::get(const QString &_strKey) +{ + if (m_mapId.contains(_strKey.trimmed())) + return &(m_mapId[_strKey.trimmed()]); + else + return 0; +} + +bool SDatagroup::SId::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "id_" + QString::number(_nCompany) + " TO " + "id_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "id_" + QString::number(_nCompany) + "_update" + " TO " + "id_" + QString::number(_nCompany) + ","; + strQuery += "id_" + QString::number(_nCompany) + "_temp" + " TO " + "id_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + + +/* +bool SDatagroup::SId::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + + strQuery = "delete from id_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = QString("insert into id_" + QString::number(_nCompany) + "_update (num,id,platformname_num,community_num) values (:NUM,:ID,:PLATFORMNAME_NUM,:COMMUNITY_NUM)").toUtf8(); + query.prepare(strQuery); + + foreach (const stId& _ids, m_mapId.values()) + { + query.bindValue(":NUM", _ids.num); + query.bindValue(":ID", _ids.id); + query.bindValue(":PLATFORMNAME_NUM", _ids.platformname_num); + query.bindValue(":COMMUNITY_NUM", _ids.platformname_num); + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + return true; +} +*/ + + +bool SDatagroup::SId::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + + strQuery = "delete from id_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = QString("insert into id_" + QString::number(_nCompany) + "_update (num,id,platformname_num,community_num) values ("); + + foreach (const stId& _ids, m_mapId.values()) + { + QString str = strQuery; + str += QString::number(_ids.num) + ","; + str += "'" + _ids.id + "',"; + str += QString::number(_ids.platformname_num) + ","; + str += QString::number(_ids.community_num) + ")"; + + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + return true; +} + + + +void SDatagroup::SId::clear() +{ + m_mapId.clear(); +} + +SDatagroup::stNickname* SDatagroup::SNickname::get(const QString &_strKey) +{ + if (m_mapNickname.contains(_strKey.trimmed())) + return &(m_mapNickname[_strKey.trimmed()]); + else + return 0; +} + + +void SDatagroup::SNickname::put(const QString &_strKey, const stNickname &_stnickname) +{ + m_mapNickname.insert(_strKey.trimmed(), _stnickname); +} + +void SDatagroup::SNickname::put(const stNickname &_nickname) +{ + m_mapNickname.insert(_nickname.nickname, _nickname); +} + +bool SDatagroup::SNickname::contains(const QString &_strKey) +{ + return m_mapNickname.contains(_strKey.trimmed()); +} + +bool SDatagroup::SNickname::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "nickname_" + QString::number(_nCompany) + " TO " + "nickname_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "nickname_" + QString::number(_nCompany) + "_update" + " TO " + "nickname_" + QString::number(_nCompany) + ","; + strQuery += "nickname_" + QString::number(_nCompany) + "_temp" + " TO " + "nickname_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} +/* +bool SDatagroup::SNickname::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery; + strQuery = "truncate nickname_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into nickname_" + QString::number(_nCompany) + "_update (num,nickname,id_num,exnickname) values (:NUM,:NICKNAME,:ID_NUM,:EXNICKNAME)").toUtf8(); + query.prepare(strQuery); + + foreach (const stNickname& stnickname, m_mapNickname.values()) + { + query.bindValue(":NUM", stnickname.num); + query.bindValue(":NICKNAME", sqlString(stnickname.nickname).replace(QChar(160),"")); + query.bindValue(":ID_NUM", stnickname.id_num); + query.bindValue(":EXNICKNAME", UniCodeToEucKr(stnickname.nickname)); + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + + +bool SDatagroup::SNickname::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + strQuery = "truncate nickname_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + + strQuery = QString("insert into nickname_" + QString::number(_nCompany) + "_update (num,nickname,id_num,exnickname) values ("); + + foreach (const stNickname& stnickname, m_mapNickname.values()) + { + QString str = strQuery; + str += QString::number(stnickname.num) + ","; + str += "'" + sqlString(stnickname.nickname).replace(QChar(160),"") + "',"; + str += QString::number(stnickname.id_num) + ","; + str += "'" + sqlString(UniCodeToEucKr(stnickname.nickname)) + "')"; + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +QString SDatagroup::SNickname::UniCodeToEucKr(const QString& _strData) +{ + { + QString strOut; + const QChar *pch = _strData.data(); + QTextCodec *codec = QTextCodec::codecForName("EUC-KR"); + for (int i = 0; i < _strData.length(); i++) + { + if (pch[i].isDigit() || pch[i].isLower() || pch[i].isUpper()) + strOut += pch[i]; + else if (pch[i].isSpace()) + strOut += "%" + codec->fromUnicode(pch[i]).toHex().toUpper(); + else + { + QString str; + str = codec->fromUnicode(pch[i]).toHex().toUpper(); + if (str.length()%2) continue; + if (str.length() == 2) strOut += pch[i]; + else + { + int i = 0; + while (i < str.length()) + { + strOut += "%" + str[i++]; + strOut += str[i++]; + } + } + } + } + return strOut; + } +} + +QString SDatagroup::SCommunity::getCommunityKey(const QStringList &_strList) +{ + QString str; + str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_form]].trimmed()); + str.append("."); + str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed()); + str.append(".com/"); + str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed()); + return str.trimmed(); +} + +SDatagroup::stCommunity* SDatagroup::SCommunity::get(const QString &_strKey) +{ + if (m_mapCommunity.contains(_strKey.trimmed())) + return &(m_mapCommunity[_strKey.trimmed()]); + else + return 0; +} + +bool SDatagroup::SCommunity::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "community_" + QString::number(_nCompany) + " TO " + "community_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "community_" + QString::number(_nCompany) + "_update" + " TO " + "community_" + QString::number(_nCompany) + ","; + strQuery += "community_" + QString::number(_nCompany) + "_temp" + " TO " + "community_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} +/* +bool SDatagroup::SCommunity::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery; + strQuery = "delete from community_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + //m_pSInitializer->insertLog(query.lastQuery()); + //m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = QString("insert into community_" + QString::number(_nCompany) + + "_update (num,id,title,url,exid,platformname_num,platformform_num) values (:NUM,:ID,:TITLE,:URL,:EXID,:PLATFORMNAME_NUM,:PLATFORMFORM_NUM)"); + query.prepare(strQuery); + + foreach (const stCommunity& stcommunity, m_mapCommunity.values()) + { + query.bindValue(":NUM", stcommunity.num); + query.bindValue(":ID", stcommunity.id); + query.bindValue(":TITLE", sqlString(stcommunity.title).replace(QChar(160),"")); + query.bindValue(":URL", stcommunity.url); + query.bindValue(":EXID", stcommunity.exid); + query.bindValue(":PLATFORMNAME_NUM", stcommunity.platformname_num); + query.bindValue(":PLATFORMFORM_NUM", stcommunity.platformform_num); + + if (!query.exec()) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + + +bool SDatagroup::SCommunity::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery; + strQuery = "delete from community_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + //m_pSInitializer->insertLog(query.lastQuery()); + //m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = QString("insert into community_" + QString::number(_nCompany) + + "_update (num,id,title,url,exid,platformname_num,platformform_num) values ("); + + foreach (const stCommunity& stcommunity, m_mapCommunity.values()) + { + QString str = strQuery; + str += QString::number(stcommunity.num) + ","; + str += "'" + sqlString(stcommunity.id) + "',"; + str += "'" + sqlString(stcommunity.title).replace(QChar(160),"") + "',"; + str += "'" + stcommunity.url + "',"; + str += "'" + stcommunity.exid + "',"; + str += QString::number(stcommunity.platformname_num) + ","; + str += QString::number(stcommunity.platformform_num) + ")"; + + if (!query.exec(str.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +void SDatagroup::SCommunity::put(const QString &_strKey, const stCommunity &_stcommunity) +{ + m_mapCommunity.insert(_strKey.trimmed(), _stcommunity); +} + +bool SDatagroup::SCommunity::contains(const QString &_strKey) +{ + return m_mapCommunity.contains(_strKey.trimmed()); +} + +void SDatagroup::SCommunity::clear() +{ + m_mapCommunity.clear(); +} + + diff --git a/sfilterprocess/sdatagroup.h b/sfilterprocess/sdatagroup.h new file mode 100644 index 0000000..f76e266 --- /dev/null +++ b/sfilterprocess/sdatagroup.h @@ -0,0 +1,263 @@ +#ifndef SDATAGROUP +#define SDATAGROUP +#include +#include +#include +#include +#include "../common.h" +class SInitializer; +class QString; +class QStringList; +class QSqlDatabase; + +QString sqlString(QString _str); + +class SDatagroup +{ +public: + enum E_ARTICLE + { + E_ARTICLE_BODY = 0, + E_ARTICLE_REPLY, + E_ARTICLE_MAX, + }; + struct stBody + { + int num; + int id_num; + QString id_id; + int nickname_num; + int community_num; + int platformname_num; + int platformform_num; + QString* title; + QString* data; + QString url; + int view; + QString date; + int reply_startnum; + int reply_endnum; + int reply_count; + QString lastupdate; + QVector realbycategorynum; + stBody():reply_endnum(-1), reply_startnum(-1), reply_count(0) {} + }; + + struct stReply + { + int num; + int id_num; + QString id_id; + int nickname_num; + int community_num; + int body_num; + QString parent; + QString* data; + QString date; + QVector realbycategorynum; + }; + struct stCommunity + { + int num; + QString id; + QString exid; + QString title; + QString url; + int platformname_num; + int platformform_num; + stCommunity():num(0), platformform_num(1), platformname_num(1), title("(none)"), url(" "), id(" "), exid(" ") {} + }; + struct stId + { + int num; + QString id; + int community_num; + int platformname_num; + stId():num(0), id("(none)"), community_num(0), platformname_num(1) {} + }; + struct stNickname + { + int num; + QString nickname; + int id_num; + stNickname():num(0) {} + }; + struct stProfileurlKey + { + int m_nPlatformForm; + QString m_strId; + stProfileurlKey(int _nPlatform, const QString& _strId):m_nPlatformForm(_nPlatform), m_strId(_strId) {} + bool operator<(const stProfileurlKey& other) const + { + if (m_nPlatformForm < other.m_nPlatformForm) + { + return true; + } + else if ((m_nPlatformForm == other.m_nPlatformForm) && (m_strId < other.m_strId)) + { + return true; + } + else + { + return false; + } + } + bool operator==(const stProfileurlKey &other) const + { + if ((m_nPlatformForm == other.m_nPlatformForm) && (m_strId == other.m_strId)) + return true; + else + return false; + } + }; + + class SBody + { + private: + //QList m_listBody; + SInitializer *m_pSInitializer; + QMap m_mapBody; + public: + void put(const QString& _url, const stBody& _stbody); + bool putCategory(const QString& _url, const int& _nCategory); + bool uploadData(QSqlDatabase &_db, const int& _nCompany, const QStringList& _order); + bool changeTable(QSqlDatabase &_db, const int& nCompany); + int getBodyNum(const QString& _url); + QString vecToString(const QVector& _realbycategorynum); + void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; } + stBody* get(const QString& _url); + void clear(); + }; + + class SReply + { + private: + //QList m_listReply; + SInitializer *m_pSInitializer; + QMap > m_mapReply; + public: + void put(const QString& _url, const int& _order, const stReply& _streply); + bool putCategory(const QString &_url, const int &_order, int& _nCategory); + bool uploadData(QSqlDatabase &_db, const int& _nCompany, QStringList& _order); + bool changeTable(QSqlDatabase &_db, const int& _nCompany); + int getReplyNum(const QString &_url, const int& _order); + stReply* get(const QString& _url, const int& _order); + QMap* get(const QString& _url); + QString vecToString(const QVector& _realbycategorynum); + void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; } + void clear(); + }; + class SId + { + private: + QMap m_mapId; + SInitializer *m_pSInitializer; + public: + bool contains(const QString& _strKey); + void put(const QString& _strKey, const stId& _stid); + void put(const stId& _stid); + QString getIdKey(const QStringList& _strlist); + stId* get(const QString &_strKey); + bool changeTable(QSqlDatabase &_db, const int& _nCompany); + bool uploadData(QSqlDatabase &_db, const int& _nCompany); + void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; } + void clear(); + }; + class SNickname + { + private: + QMap m_mapNickname; + SInitializer *m_pSInitializer; + public: + //QString getNicknameKey(const QStringList& _strlist); + stNickname* get(const QString &_strKey); + bool changeTable(QSqlDatabase &_db, const int& _nCompany); + bool uploadData(QSqlDatabase &_db, const int& _nCompany); + void put(const QString& _strKey, const stNickname& _stnickname); + void put(const stNickname& _nickname); + bool contains(const QString& _strKey); + QString UniCodeToEucKr(const QString& _strData); + void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; } + void clear(); + }; + class SCommunity + { + private: + QMap m_mapCommunity; + SInitializer *m_pSInitializer; + public: + QString getCommunityKey(const QStringList& _strlist); + stCommunity* get(const QString &_strKey); + bool changeTable(QSqlDatabase &_db, const int& _nCompany); + bool uploadData(QSqlDatabase &_db, const int& _nCompany); + void put(const QString& _strKey, const stCommunity& _stcommunity); + bool contains(const QString& _strKey); + void clear(); + void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; } + }; + struct stBodyReplyCount + { + int m_nAllBodyCount; + int m_nAllReplyCount; + int m_nFilterBodyCount; + int m_nFilterReplyCount; + stBodyReplyCount():m_nAllBodyCount(0), m_nAllReplyCount(0), m_nFilterBodyCount(0), m_nFilterReplyCount(0){} + }; + struct stDate + { + QString m_strMindate; + QString m_strMaxdate; + QString m_strLastupdate; + stDate():m_strMindate("2100-12-31 23:59:59"), m_strMaxdate("1900-01-01 00:00:00"), m_strLastupdate("1900-01-01 00:00:00"){} + }; + +private: + SInitializer* m_pSInitializer; + QSet m_setFilteredUrl; + + QStringList m_slOrderedBodyUrl; + QMap > m_mapReply; + QMap m_mapBody; + QMap m_mapProfile; + + + SBody m_SBody; + SReply m_SReply; + SCommunity m_SCommunity; + SId m_SId; + SNickname m_SNickname; + stBodyReplyCount m_stBodyReplyCount; + stDate m_stDate; + +public: + void setInitializer(SInitializer &initializer); + bool loadData(const int& _nDatagroupNumber); + bool uploadData(const int& _nCompany); + bool changeTable(const int& _nCompany); + bool makeTable(); + void addRealUrl(const QString& _strUrl); + void copyData(QVector *_data); + const QMap* getmapBody(); + const QStringList* getmapBody(const stProfileurlKey& _stprofileurlkey); + const QMap >* getmapReply(); + const stBody* getstBody(const QString& _strUrl); + const stReply* getstReply(const QString& _strUrl, const int &_order); + const QMap* getstReply(const QString& _strUrl); + int getstBodyNum(const QString& _strUrl); + int getstReplyNum(const QString& _strUrl, const int &_order); + const QStringList* getOrderedUrl(); + void addRealBodyCategory(const QString& _strUrl, const int _nCategory); + void addRealReplyCategory(const QString& _strUrl, const int& _order, int _nCategory); + const stCommunity* getstCommunity(const QStringList& _sl); + const stBodyReplyCount* getCounts(); + const stDate* getstDate(); + +private: + void init(); + void setloadQuery(QString _astrQuery[], int _nDatagroupNumber); + void makeProfile(const QStringList& _strList); +}; + + +#endif // SDATAGROUP + diff --git a/sfilterprocess/sdbmanager.cpp b/sfilterprocess/sdbmanager.cpp new file mode 100644 index 0000000..d58d1e2 --- /dev/null +++ b/sfilterprocess/sdbmanager.cpp @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include +#include "sdbmanager.h" + +void SDBManager::addDatabase(const QString& _dbName,const QString& _host,const QString& _id, const QString& _pw, const QString& _db, const int& _port) +{ + QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL", _dbName); + db.setHostName(_host); + db.setUserName(_id); + db.setPassword(_pw); + db.setDatabaseName(_db); + db.setPort(_port); +} + + +void SDBManager::addDatabase(const QString& _dbName, const E_DATABASE& _eDatabase, const QString& _host, const QString& _id, const QString& _pw, const QString& _db, const int& _port) +{ + QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL", _dbName); + db.setHostName(_host); + db.setUserName(_id); + db.setPassword(_pw); + db.setDatabaseName(_db); + db.setPort(_port); + m_mapDBname.insert(_eDatabase, _dbName); +} + +QSqlDatabase SDBManager::getDBConnection(const QString& _dbName) const +{ + return QSqlDatabase::database(_dbName); +} + +QSqlDatabase SDBManager::getDBConnection(const E_DATABASE& _eDatabase) const +{ + return QSqlDatabase::database(m_mapDBname.value(_eDatabase)); +} + +bool SDBManager::open(const QString& _dbName) +{ + QSqlDatabase db = QSqlDatabase::database(_dbName); + return db.open(); +} + +bool SDBManager::open(const E_DATABASE& _eDatabase) +{ + QSqlDatabase db = QSqlDatabase::database(m_mapDBname.value(_eDatabase)); + return db.open(); +} + +void SDBManager::close(const E_DATABASE& _eDatabase) +{ + QSqlDatabase db = QSqlDatabase::database(m_mapDBname.value(_eDatabase)); + db.close(); +} + +void SDBManager::close(const QString& _dbName) +{ + QSqlDatabase db = QSqlDatabase::database(_dbName); + db.close(); +} + +void SDBManager::close() +{ + QStringList strlist = QSqlDatabase::connectionNames(); + foreach (const QString& str, strlist) + QSqlDatabase::database(str).close(); +} + +QSqlQuery SDBManager::sqlQuery(const E_DATABASE& _eDatabase) +{ + QSqlQuery query(QSqlDatabase::database(m_mapDBname.value(_eDatabase))); + return query; +} + +QSqlQuery SDBManager::sqlQuery(const QString& _dbName) +{ + QSqlQuery query(QSqlDatabase::database(_dbName)); + return query; +} + +bool SDBManager::reOpen(const QString& _dbName) +{ + close(_dbName); + return open(_dbName); +} + +bool SDBManager::reOpen(const E_DATABASE& _eDatabase) +{ + close(_eDatabase); + return open(_eDatabase); +} + +bool SDBManager::isConnectionError(const QSqlError& _error) +{ + if(_error.number() == 2013 || _error.number() == 2006 || _error.number() == 10054) + return true; + return false; +} + +SDBManager::~SDBManager() +{ + QStringList strlist = QSqlDatabase::connectionNames(); + foreach (const QString& str, strlist) + { + QSqlDatabase db = QSqlDatabase::database(str); + if(db.isOpen()) + db.close(); + } + foreach (const QString& str, strlist) + QSqlDatabase::removeDatabase(str); +} + +QSqlDatabase SDBManager::operator[](const QString& _dbName) const +{ + return QSqlDatabase::database(_dbName); +} + +QSqlDatabase SDBManager::operator[](const E_DATABASE& _eDatabase) const +{ + return QSqlDatabase::database(m_mapDBname.value(_eDatabase)); +} diff --git a/sfilterprocess/sdbmanager.h b/sfilterprocess/sdbmanager.h new file mode 100644 index 0000000..2a41bdd --- /dev/null +++ b/sfilterprocess/sdbmanager.h @@ -0,0 +1,37 @@ +#ifndef SDBMANAGER +#define SDBMANAGER +#include +class QSqlDatabase; +class QSqlQuery; +class QString; +class QStringList; +class QSqlError; + +class SDBManager +{ +public: + enum E_DATABASE{ E_DATABASE_DATA, E_DATABASE_WEB }; + QSqlDatabase getDBConnection(const QString& _dbName) const; + QSqlDatabase getDBConnection(const E_DATABASE& _eDatabase) const; + bool open(const QString& _dbName); + bool open(const E_DATABASE& _eDatabase); + void close(const QString& _dbName); + void close(const E_DATABASE& _eDatabase); + void close(); + void addDatabase(const QString& _dbName, const QString& _host, const QString& _id, const QString& _pw, const QString& _db, const int& _port = 3306); + void addDatabase(const QString& _dbName, const E_DATABASE& _eDatabase, const QString& _host, const QString& _id, const QString& _pw, const QString& _db, const int& _port = 3306); + QSqlQuery sqlQuery(const QString& _dbName); + QSqlQuery sqlQuery(const E_DATABASE& _eDatabase); + bool reOpen(const QString& _dbName); + bool reOpen(const E_DATABASE& _eDatabase); + bool isConnectionError(const QSqlError& _error); + QSqlDatabase operator[](const QString& _dbName) const; + QSqlDatabase operator[](const E_DATABASE& _eDatabase) const; + virtual ~SDBManager(); +private: + QMap m_mapDBname; +}; + + +#endif // SDBMANAGER + diff --git a/sfilterprocess/sfilteralgorithm.cpp b/sfilterprocess/sfilteralgorithm.cpp new file mode 100644 index 0000000..2d1b148 --- /dev/null +++ b/sfilterprocess/sfilteralgorithm.cpp @@ -0,0 +1,243 @@ +#include +#include +#include +#include "sfilteralgorithm.h" +#include "../Json/sjson.h" +#include + +void FilterSearch(QVector *m_data, bool *m_bDataFlag, QString _strJson); +void FilterCounter(QVector *m_data, bool *m_bDataFlag, QString _strJson); +void FilterReplace(QVector *m_data, bool *m_bDataFlag, QString _strJson); +void FilterDate(QVector *m_data, bool *m_bDataFlag, QString _strJson, int _nDateColumn); +void FilterLength(QVector *m_data, bool *m_bDataFlag, QString _strJson); + +void SFilterAlgorithm::FilterDate(QVector *m_data, bool *m_bDataFlag, QString _strJson, int _nDateColumn) +{ + SJson json; + int nSelect = json.Get(_strJson,"Article").toInt() - 1; + if (nSelect < 0) return; + m_bDataFlag[nSelect] = true; + QVector copyData; + QDate dateStart = QDate::fromString(json.Get(_strJson,"Start"),"yyyy-MM-dd"); + QDate dateEnd = QDate::fromString(json.Get(_strJson,"End"),"yyyy-MM-dd"); + foreach(const QStringList& strList , m_data[nSelect]) + { + bool bFlag = false; + QString strTime = strList.at(_nDateColumn).trimmed(); + if (strTime.size() >= 10) + { + QChar ch = strTime.at(4); + QString strFormat = QString("yyyy")+ch+QString("MM")+ch+QString("dd"); + QDate date = QDate::fromString(strTime.left(10),strFormat); + if (dateStart <= date && dateEnd >= date) + bFlag = true; + } + if (bFlag) + copyData.push_back(strList); + } + m_data[nSelect].clear(); + m_data[nSelect] = copyData; +} + +void SFilterAlgorithm::FilterLength(QVector *m_data, bool *m_bDataFlag, QString _strJson) +{ + SJson json; + int nSelect = json.Get(_strJson,"Article").toInt() - 1; + if (nSelect < 0) return; + m_bDataFlag[nSelect] = true; + + int nCatalog = json.GetNumber(_strJson,"Category"); + int nComp = json.GetNumber(_strJson,"Comp"); + int nFlag = json.GetNumber(_strJson,"InsDel"); + int nLength = json.Get(_strJson,"String").toInt(); + + QVector copyData; + foreach(const QStringList& strList , m_data[nSelect]) + { + bool bFlag = false; + int nCurrentLength = strList.at(nCatalog).trimmed().length(); + switch(nComp) + { + case E_LENGTH_COMP_GREATER: + if (nCurrentLength > nLength) + bFlag = true; + break; + case E_LENGTH_COMP_LESS: + if (nCurrentLength < nLength) + bFlag = true; + break; + case E_LENGTH_COMP_EQUAL: + if (nCurrentLength == nLength) + bFlag = true; + break; + } + if (nFlag == 1) bFlag = !bFlag; + if (bFlag) + copyData.push_back(strList); + } + m_data[nSelect].clear(); + m_data[nSelect] = copyData; +} + +void SFilterAlgorithm::FilterCounter(QVector *m_data, bool *m_bDataFlag, QString _strJson) +{ + SJson json; + int nSelect = json.Get(_strJson,"Article").toInt() - 1; + if (nSelect < 0) return; + m_bDataFlag[nSelect] = true; + + int nCatalog = json.GetNumber(_strJson,"Category"); + int nComp = json.GetNumber(_strJson,"Comp"); + int nFlag = json.GetNumber(_strJson,"InsDel"); + int nCounter = json.Get(_strJson,"String").toInt(); + + QMap mapData, mapDataresult; + foreach(const QStringList& strList , m_data[nSelect]) + { + QString str = strList.at(nCatalog).trimmed(); + if(mapData.contains(str)) + mapData[str]++; + else + mapData.insert(str, 1); + } + + for(QMap ::iterator iterPos = mapData.begin(); iterPos != mapData.end(); iterPos++) + { + bool bFlag = false; + switch(nComp) + { + case E_LENGTH_COMP_GREATER: + if (iterPos.value() > nCounter) + bFlag = true; + break; + case E_LENGTH_COMP_LESS: + if (iterPos.value() < nCounter) + bFlag = true; + break; + case E_LENGTH_COMP_EQUAL: + if (iterPos.value() == nCounter) + bFlag = true; + break; + } + if (nFlag == 1) bFlag = !bFlag; + if (bFlag) + mapDataresult.insert(iterPos.key(), iterPos.value()); + } + + mapData.clear(); + + QVector copyData; + foreach(const QStringList& strList , m_data[nSelect]) + { + QString str = strList.at(nCatalog).trimmed(); + if(mapDataresult.contains(str)) + copyData.push_back(strList); + } + m_data[nSelect].clear(); + m_data[nSelect] = copyData; +} + +void SFilterAlgorithm::FilterReplace(QVector *m_data, bool *m_bDataFlag, QString _strJson) +{ + SJson json; + int nSelect = json.Get(_strJson,"Article").toInt() - 1; + if (nSelect < 0) return; + m_bDataFlag[nSelect] = true; + + int nCatalog = json.GetNumber(_strJson,"Category"); + QString strFind = json.Get(_strJson,"String_Find"); + QString strReplace = json.Get(_strJson,"String_Replace"); + + bool bFlag = false; + QStringList strListKeyword; + if (json.GetNumber(_strJson,"Find") == E_REPLACE_SPACE) + { + bFlag = true; + strListKeyword = strFind.split(" "); + } + + for (int i = 0 ; i < m_data[nSelect].size(); i++) + { + if (bFlag) + { + foreach(const QString& str,strListKeyword) + { + m_data[nSelect][i][nCatalog] = m_data[nSelect][i][nCatalog].replace(str,strReplace); + } + } + else + m_data[nSelect][i][nCatalog] = m_data[nSelect][i][nCatalog].replace(strFind,strReplace); + } +} + +void SFilterAlgorithm::FilterSearch(QVector *m_data, bool *m_bDataFlag, QString _strJson) +{ + SJson json; + int nSelect = json.Get(_strJson,"Article").toInt() - 1; + if (nSelect < 0) return; + m_bDataFlag[nSelect] = true; + { + SKeyword stKeyword; + stKeyword.m_nCatalog = json.GetNumber(_strJson,"Category"); + stKeyword.m_nKeyword = json.GetNumber(_strJson,"Keyword"); + stKeyword.m_strListKeyword = json.Get(_strJson,"String").split(" "); + if (json.GetNumber(_strJson,"Method") == 1) + { + for (int i = 0 ; i < stKeyword.m_strListKeyword.size(); i++ ) + stKeyword.m_strListKeyword[i] = " " + stKeyword.m_strListKeyword[i] + " "; + } + m_vecKeyword.push_back(stKeyword); + } + if (json.GetBool(_strJson,"Last")) // + { + QVector copyData; + + foreach(const QStringList& strList , m_data[nSelect]) + { + bool bFlag = false; + foreach(const SKeyword& stKeyword,m_vecKeyword) + { + QString strData = strList.at(stKeyword.m_nCatalog); + switch(stKeyword.m_nKeyword) + { + case E_SEARCH_KEYWORD_OR: + { + foreach(const QString& strKey , stKeyword.m_strListKeyword) + if (strData.contains(strKey)){bFlag = true;break;} + break; + } + case E_SEARCH_KEYWORD_AND: + { + int nKeyCount = 0; + foreach(const QString& strKey , stKeyword.m_strListKeyword) + if (strData.contains(strKey)) nKeyCount++; + if (nKeyCount == stKeyword.m_strListKeyword.size()) + bFlag = true; + break; + } + case E_SEARCH_KEYWORD_CELL_DELETE_OR: + { + bFlag = true; + foreach(const QString& strKey , stKeyword.m_strListKeyword) + if (strData.contains(strKey)){bFlag = false;break;} + break; + } + case E_SEARCH_KEYWORD_CELL_DELETE_AND: + { + bFlag = true; + int nCount = 0; + foreach(const QString& strKey , stKeyword.m_strListKeyword) + if (strData.contains(strKey)){nCount++;} + if (nCount == stKeyword.m_strListKeyword.size()) + bFlag = false; + break; + } + } + } + if (bFlag) copyData.push_back(strList); + } + m_data[nSelect].clear(); + m_data[nSelect] = copyData; + m_vecKeyword.clear(); + } +} diff --git a/sfilterprocess/sfilteralgorithm.h b/sfilterprocess/sfilteralgorithm.h new file mode 100644 index 0000000..04cd1a8 --- /dev/null +++ b/sfilterprocess/sfilteralgorithm.h @@ -0,0 +1,23 @@ +#ifndef SFILTERALGORITHM +#define SFILTERALGORITHM +#include +#include "../common.h" +class QString; +class QStringList; +class SFilterAlgorithm +{ +private: + QVector m_vecKeyword; +public: + void FilterSearch(QVector *m_data, bool *m_bDataFlag, QString _strJson); + void FilterCounter(QVector *m_data, bool *m_bDataFlag, QString _strJson); + void FilterReplace(QVector *m_data, bool *m_bDataFlag, QString _strJson); + void FilterDate(QVector *m_data, bool *m_bDataFlag, QString _strJson, int _nDateColumn); + void FilterLength(QVector *m_data, bool *m_bDataFlag, QString _strJson); + +}; + + + +#endif // SFILTERALGORITHM + diff --git a/sfilterprocess/sfilterprocess.cpp b/sfilterprocess/sfilterprocess.cpp new file mode 100644 index 0000000..02a2cb9 --- /dev/null +++ b/sfilterprocess/sfilterprocess.cpp @@ -0,0 +1,96 @@ +#include +#include +#include "sfilteralgorithm.h" +#include "sfilterprocess.h" +#include "../Json/sjson.h" +#include +#include +using namespace std; + +bool SFilterProcess::init(QString _strFilterProcessGroupID, QString _strFilterProcessGroupName) +{ + if (!m_SInitializer.init(_strFilterProcessGroupID, _strFilterProcessGroupName)) + { + cout << "initializer init error" << endl; + return false; + } + + m_SDatagroup.setInitializer(m_SInitializer); + m_SConsumer.setInitiaizer(m_SInitializer); + m_SInfluencer.setInitiaizer(m_SInitializer); + m_SSpammer.setInitiaizer(m_SInitializer); + m_SPowercafe.setInitiaizer(m_SInitializer); + m_SInitializer.closeConnection(SDBManager::E_DATABASE_WEB); + return true; +} + +bool SFilterProcess::run() +{ + m_SInitializer.insertLog("New Filter start"); + QList listFilterProcess = m_SInitializer.getFilterProcess(); + + foreach (SInitializer::stFilterProcess process, listFilterProcess) + { + switch(process.m_nFilterProcessType) + { + case E_PROCSS_TYPE_LOAD: + { + m_SDatagroup.loadData(process.m_nDatagroupID); + break; + } + case E_PROCSS_TYPE_CATEGORY_MAKE: + { + m_SConsumer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID)); + break; + } + case E_PROCSS_TYPE_SEND: + { + m_SDatagroup.makeTable(); + + m_SConsumer.makeOverallCategory(m_SInitializer.getConsumerAllCategory()); + m_SConsumer.makeTable(m_SDatagroup); + m_SInfluencer.makeTable(m_SDatagroup); + m_SSpammer.makeTable(m_SDatagroup); + m_SPowercafe.makeTable(m_SDatagroup); + + m_SInitializer.openConnection(SDBManager::E_DATABASE_WEB); + m_SDatagroup.uploadData(process.m_nCompanyNum); + m_SConsumer.uploadData(process.m_nCompanyNum); + m_SInfluencer.uploadData(process.m_nCompanyNum); + m_SSpammer.uploadData(process.m_nCompanyNum); + m_SPowercafe.uploadData(process.m_nCompanyNum); + + m_SDatagroup.changeTable(process.m_nCompanyNum); + m_SConsumer.changeTable(process.m_nCompanyNum); + m_SInfluencer.changeTable(process.m_nCompanyNum); + m_SSpammer.changeTable(process.m_nCompanyNum); + m_SPowercafe.changeTable(process.m_nCompanyNum); + + m_SInitializer.updateWebDBInfoComplete(m_SInitializer.getDatabase(SDBManager::E_DATABASE_WEB), m_SDatagroup.getstDate()->m_strMindate, + m_SDatagroup.getstDate()->m_strMaxdate, m_SDatagroup.getstDate()->m_strLastupdate, + m_SDatagroup.getCounts()->m_nFilterBodyCount, m_SDatagroup.getCounts()->m_nFilterReplyCount, + process.m_nCompanyNum); + break; + } + case E_PROCSS_TYPE_INFLUENCE_COUNT: + { + m_SInfluencer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID)); + break; + } + case E_PROCSS_TYPE_SPAMMERS_COUNT: + { + m_SSpammer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID)); + break; + } + case E_PROCSS_TYPE_POWER_CAFE_COUNT: + { + m_SPowercafe.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID)); + break; + } + } + } + m_SInitializer.insertLog("New Filter END"); + + return true; +} + diff --git a/sfilterprocess/sfilterprocess.h b/sfilterprocess/sfilterprocess.h new file mode 100644 index 0000000..0f53579 --- /dev/null +++ b/sfilterprocess/sfilterprocess.h @@ -0,0 +1,28 @@ +#ifndef SFILTERPROCESS +#define SFILTERPROCESS +#include "sinitializer.h" +#include "sdatagroup.h" +#include "sconsumer.h" +#include "sinfluencer.h" +#include "sspammer.h" +#include "spowercafe.h" +class QString; + +class SFilterProcess +{ +private: + SInitializer m_SInitializer; + SDatagroup m_SDatagroup; + SConsumer m_SConsumer; + SInfluencer m_SInfluencer; + SSpammer m_SSpammer; + SPowercafe m_SPowercafe; + +public: + bool init(QString _strFilterProcessGroupID, QString _strFilterProcessGroupName); + bool run(); + +}; + +#endif // SFILTERPROCESS + diff --git a/sfilterprocess/sfilterprocess.pro b/sfilterprocess/sfilterprocess.pro new file mode 100644 index 0000000..8b6eb25 --- /dev/null +++ b/sfilterprocess/sfilterprocess.pro @@ -0,0 +1,42 @@ +#------------------------------------------------- +# +# Project created by QtCreator 2016-02-29T16:47:52 +# +#------------------------------------------------- + +QT += core sql + +QT -= gui + +TARGET = sfilterprocess +CONFIG += console +CONFIG -= app_bundle + +TEMPLATE = app + + +SOURCES += main.cpp \ + sdbmanager.cpp \ + sinitializer.cpp \ + sfilterprocess.cpp \ + sfilteralgorithm.cpp \ + ../Json/sjson.cpp \ + sdatagroup.cpp \ + sconsumer.cpp \ + sinfluencer.cpp \ + sspammer.cpp \ + spowercafe.cpp + +HEADERS += \ + sdbmanager.h \ + sinitializer.h \ + sfilterprocess.h \ + sfilteralgorithm.h \ + ../common.h \ + ../Json/sjson.h \ + sdatagroup.h \ + sservice.h \ + sinfluencer.h \ + sconsumer.h \ + sspammer.h \ + spowercafe.h diff --git a/sfilterprocess/sfilterprocess.pro.user b/sfilterprocess/sfilterprocess.pro.user new file mode 100644 index 0000000..2c3fbf4 --- /dev/null +++ b/sfilterprocess/sfilterprocess.pro.user @@ -0,0 +1,1003 @@ + + + + + + EnvironmentId + {a05fc9db-6c77-45e7-9770-7cd31ad1131c} + + + ProjectExplorer.Project.ActiveTarget + 3 + + + ProjectExplorer.Project.EditorSettings + + true + false + true + + Cpp + + CppGlobal + + + + QmlJS + + QmlJSGlobal + + + 2 + UTF-8 + false + 4 + false + 80 + true + true + 1 + true + false + 0 + true + 0 + 8 + true + 1 + true + true + true + false + + + + ProjectExplorer.Project.PluginSettings + + + + ProjectExplorer.Project.Target.0 + + Desktop Qt 5.4.0 MinGW 32bit + Desktop Qt 5.4.0 MinGW 32bit + qt.54.win32_mingw491_kit + 1 + 0 + 0 + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MinGW_32bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MinGW_32bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + sfilterprocess + + Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro + + sfilterprocess.pro + false + true + + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.Target.1 + + Desktop Qt 5.4.0 MSVC2013 32bit + Desktop Qt 5.4.0 MSVC2013 32bit + {41ee5de3-e0c6-4932-89be-818ac7411cf5} + 0 + 0 + 0 + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MSVC2013_32bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MSVC2013_32bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + + + false + %{buildDir} + Custom Executable + + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.Target.2 + + Desktop Qt 5.4.1 MSVC2013 64bit + Desktop Qt 5.4.1 MSVC2013 64bit + {481b3484-407d-4ae0-ba41-458f5e9d2bf2} + 1 + 0 + 0 + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_1_MSVC2013_64bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_1_MSVC2013_64bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + sfilterprocess + + Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro + + sfilterprocess.pro + false + true + + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.Target.3 + + Desktop Qt 5.4.0 MSVC2013 64bit + Desktop Qt 5.4.0 MSVC2013 64bit + {4a73e1a8-867f-4a8f-a70a-a3bb186185e4} + 1 + 0 + 0 + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MSVC2013_64bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_0_MSVC2013_64bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + 2 + + sfilterprocess + + Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro + "370" "test" + sfilterprocess.pro + false + true + + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.Target.4 + + Desktop Qt 5.4.1 MSVC2013 32bit + Desktop Qt 5.4.1 MSVC2013 32bit + {9ae54b69-b1a2-4c5f-aded-871471d7c65f} + 0 + 0 + 0 + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_1_MSVC2013_32bit-Debug + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + true + + + C:/source/build-sfilterprocess-Desktop_Qt_5_4_1_MSVC2013_32bit-Release + + + true + qmake + + QtProjectManager.QMakeBuildStep + false + true + + false + false + + + true + Make + + Qt4ProjectManager.MakeStep + + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + true + Make + + Qt4ProjectManager.MakeStep + + true + clean + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + true + + 2 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + Deploy locally + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + + + false + false + false + false + true + 0.01 + 10 + true + 1 + 25 + + 1 + true + false + true + valgrind + + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + + -1 + + + + false + %{buildDir} + Custom Executable + + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + false + true + false + false + true + + 1 + + + + ProjectExplorer.Project.TargetCount + 5 + + + ProjectExplorer.Project.Updater.FileVersion + 18 + + + Version + 18 + + diff --git a/sfilterprocess/sinfluencer.cpp b/sfilterprocess/sinfluencer.cpp new file mode 100644 index 0000000..46366a9 --- /dev/null +++ b/sfilterprocess/sinfluencer.cpp @@ -0,0 +1,873 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "sinfluencer.h" + +void SInfluencer::setInitiaizer(SInitializer &_initializer) +{ + m_pSInitializer = &_initializer; + m_SBodyInfluencerMap.setInitiaizer(_initializer); + m_SReplyInfluencerMap.setInitiaizer(_initializer); + m_SStatsInfBloggerBuzzbyday.setInitiaizer(_initializer); + m_SStatsInfBloggerRank.setInitiaizer(_initializer); + m_STInfluencer.setInitiaizer(_initializer); +} + +bool SInfluencer::uploadData(int _companyNum) +{ + m_pSInitializer->insertLog("Start upload influencer"); + + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Influence (Start)"); + if (!m_SBodyInfluencerMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload influencer body map"); + return false; + } + if (!m_SReplyInfluencerMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload influencer reply map"); + return false; + } + if (!m_SStatsInfBloggerBuzzbyday.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_buzzbyday"); + return false; + } + if (!m_SStatsInfBloggerRank.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_rank"); + return false; + } + if (!m_STInfluencer.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload influencer"); + return false; + } + m_pSInitializer->insertLog("End upload influencer"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Influence (Finish)"); + + return true; +} + +bool SInfluencer::changeTable(int _companyNum) +{ + m_pSInitializer->insertLog("Start table change influencer"); + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + if (!m_SBodyInfluencerMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload influencer body map"); + return false; + } + if (!m_SReplyInfluencerMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload influencer reply map"); + return false; + } + return true; +} + +bool SInfluencer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter) +{ + QVector vecData[SInitializer::E_ARTICLE_MAX]; + bool bFiltered[SInitializer::E_ARTICLE_MAX]; + + for (int i = 0; i < SInitializer::E_ARTICLE_MAX; i++) + bFiltered[i] = false; + _datagroup.copyData(vecData); + foreach (SInitializer::stFilter filter, _listFilter) + { + switch(filter.m_nType) + { + case E_FILTER_TYPE_DATE://Date + m_SFilterAlgorithm.FilterDate(vecData, bFiltered, filter.m_vData.toString(), + m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_article_date]); + + break; + case E_FILTER_TYPE_SEARCH: + m_SFilterAlgorithm.FilterSearch(vecData, bFiltered, filter.m_vData.toString()); + + break; + case E_FILTER_TYPE_LENGTH: + m_SFilterAlgorithm.FilterLength(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_REPLACE: + m_SFilterAlgorithm.FilterReplace(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_COUNTER: + m_SFilterAlgorithm.FilterCounter(vecData, bFiltered, filter.m_vData.toString()); + break; + } + } + stReal streal; + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size(); + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size(); + const int *anColumn = m_pSInitializer->getColumnIntArray(); + if (bFiltered[SInitializer::E_ARTICLE_BODY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_BODY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_setRealBodyUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + if (bFiltered[SInitializer::E_ARTICLE_REPLY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_REPLY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_mapRealReplyUniqueUrl.insertMulti(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed(), + strlist[anColumn[SInitializer::E_DATA_article_order]].trimmed().toInt()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + m_mapCategory.insert(_nCategory, streal); + return true; +} + +bool SInfluencer::makeOverallCategory(int _categoryNum) +{ + return true; +} + +bool SInfluencer::makeTable(SDatagroup &_datagroup) +{ + m_pSInitializer->insertLog("Make influencer tables"); + m_pSInitializer->insertLog("Make influencer body tables"); + m_SBodyInfluencerMap.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make influencer reply tables"); + m_SReplyInfluencerMap.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make influencer stats_inf_blogger_buzzy tables"); + m_SStatsInfBloggerBuzzbyday.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make influencer stats_inf_blogger_rank table"); + m_SStatsInfBloggerRank.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make influencer influencer"); + m_STInfluencer.makeTable(_datagroup, m_mapCategory); + return true; +} + +void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMap& _mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + QMap mapRank; + + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& _url, *orderedUrl) + { + if (streal.m_setRealBodyUrl.contains(_url)) + { + SDatagroup::stProfileurlKey stprofileurlkey(m_pSInitializer->getPlatformForm((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed()), + (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + + if (!mapRank.contains(stprofileurlkey)) + { + mapRank.insert(stprofileurlkey, stInfluencer()); + mapRank[stprofileurlkey].id_id = _datagroup.getstBody(_url)->id_id; + mapRank[stprofileurlkey].id_num = _datagroup.getstBody(_url)->id_num; + mapRank[stprofileurlkey].nickname_num = _datagroup.getstBody(_url)->nickname_num; + mapRank[stprofileurlkey].community_num = _datagroup.getstBody(_url)->community_num; + mapRank[stprofileurlkey].profile = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profile]].trimmed(); + mapRank[stprofileurlkey].profileurl = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profileurl]].trimmed(); + } + mapRank[stprofileurlkey].bodycount++; + + if ((*mapReply).contains(_url)) + { + for (QMap::const_iterator iterPos1 = (*mapReply)[_url].constBegin(); iterPos1 != (*mapReply)[_url].constEnd(); iterPos1++) + { + //if ((iterPos1.value()[anColumn[SInitializer::E_DATA_article_id]].trimmed() != (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed())) + mapRank[stprofileurlkey].replycount++; + } + } + } + } + } + + for (QMap::iterator iterPos = mapRank.begin(); iterPos != mapRank.end(); iterPos++) + { + stInfluencer &val = iterPos.value(); + val.influencervalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1); + } + + m_listMap = mapRank.values(); + + qSort(m_listMap.begin(), m_listMap.end(), isGreaterInfluencerValue()); + + int rank = 1; + for (QList::iterator iterPos = m_listMap.begin(); iterPos != m_listMap.end(); iterPos++) + { + iterPos->rank = rank++; + } +} + +/* +bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from influencer where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into influencer (rank,company_num,id_num,nickname_num,community_num,bodycount,replycount,influencervalue,profile,profileurl,id_id)" + " VALUES (:RANK,:COMPANY_NUM,:ID_NUM,:NICKNAME_NUM,:COMMUNITY_NUM,:BODY_COUNT,:REPLY_COUNT,:INPL,:PROFILE,:PROFILE_URL,:ID_ID)"; + + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stInfluencer& stinfluencer, m_listMap) + { + query.bindValue(":RANK", stinfluencer.rank); + query.bindValue(":COMPANY_NUM", _nCompany); + query.bindValue(":ID_NUM", stinfluencer.id_num); + query.bindValue(":NICKNAME_NUM", stinfluencer.nickname_num); + query.bindValue(":COMMUNITY_NUM", stinfluencer.community_num); + query.bindValue(":BODY_COUNT", stinfluencer.bodycount); + query.bindValue(":REPLY_COUNT", stinfluencer.replycount); + query.bindValue(":ID_ID", stinfluencer.id_id); + query.bindValue(":INPL", QString::number(stinfluencer.influencervalue, 'g', 3)); + //query.bindValue(":PROFILE", ((stinfluencer.profile.trimmed().isEmpty())? " " : sqlString(stinfluencer.profile).trimmed().replace(QChar(160), ""))); + //query.bindValue(":PROFILE_URL", ((stinfluencer.profileurl.trimmed().isEmpty())? " ", stinfluencer.profileurl.trimmed())); + query.bindValue(":PROFILE", sqlString(stinfluencer.profile).trimmed().replace(QChar(160), "")); + query.bindValue(":PROFILE_URL", stinfluencer.profileurl.trimmed()); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + strQuery = "update dbinfo set influencercount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + + return true; +} +*/ + +bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from influencer where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into influencer (rank,company_num,id_num,nickname_num,community_num,bodycount,replycount,influencervalue,profile,profileurl,id_id)" + " VALUES ("; + + query.clear(); + + foreach (const stInfluencer& stinfluencer, m_listMap) + { + QString str = strQuery; + str += QString::number(stinfluencer.rank) + ","; + str += QString::number(_nCompany) + ","; + str += QString::number(stinfluencer.id_num) + ","; + str += QString::number(stinfluencer.nickname_num) + ","; + str += QString::number(stinfluencer.community_num) + ","; + str += QString::number(stinfluencer.bodycount) + ","; + str += QString::number(stinfluencer.replycount) + ","; + str += QString::number(stinfluencer.influencervalue, 'g', 3) + ","; + str += "'" + sqlString(stinfluencer.profile).trimmed().replace(QChar(160), "") + "',"; + str += "'" + stinfluencer.profileurl.trimmed() + "',"; + str += "'" + sqlString(stinfluencer.id_id) + "')"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + + strQuery = "update dbinfo set influencercount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + + return true; +} + +void SInfluencer::SStatsInfBloggerRank::makeTable(SDatagroup &_datagroup, const QMap& _mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + const int RANK = 10; + + QMap mapRank; + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + SDatagroup::stProfileurlKey stprofileurlkey(m_pSInitializer->getPlatformForm((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed()), + (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + + if (!mapRank.contains(stprofileurlkey)) + { + mapRank.insert(stprofileurlkey, stStatsInfBloggerRank()); + mapRank[stprofileurlkey].id_id = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed(); + mapRank[stprofileurlkey].nickname_nickname = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed(); + } + + mapRank[stprofileurlkey].bodycount++; + + if ((*mapReply).contains(_url)) + { + for (QMap::const_iterator iterPos1 = (*mapReply)[_url].constBegin(); iterPos1 != (*mapReply)[_url].constEnd(); iterPos1++) + { + //if ((iterPos1.value()[anColumn[SInitializer::E_DATA_article_id]].trimmed() != (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed())) + mapRank[stprofileurlkey].replycount++; + } + } + } + } + + for (QMap::iterator iterPos = mapRank.begin(); iterPos != mapRank.end(); iterPos++) + { + stStatsInfBloggerRank &val = iterPos.value(); + val.influencevalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1); + } + + QList listMap = mapRank.values(); + + qSort(listMap.begin(), listMap.end(), isGreaterInfluencerValue()); + // influencevalue + for(int i = 0; i < RANK; i++) + { + listMap[i].rank = i + 1; + listMap[i].subject = "influencevalue"; + m_listMap.append(listMap.at(i)); + //qDebug() << listMap[i].id_id; + } + + qSort(listMap.begin(), listMap.end(), isGreaterBodyCount()); + // influencevalue + for(int i = 0; i < RANK; i++) + { + listMap[i].rank = i + 1; + listMap[i].subject = "bodycount"; + m_listMap.append(listMap.at(i)); + //qDebug() << listMap[i].id_id; + } + + qSort(listMap.begin(), listMap.end(), isGreaterReplyCount()); + // influencevalue + for(int i = 0; i < RANK; i++) + { + listMap[i].rank = i + 1; + listMap[i].subject = "replycount"; + m_listMap.append(listMap.at(i)); + //qDebug() << listMap[i].id_id; + } + listMap.clear(); +} + +/* +bool SInfluencer::SStatsInfBloggerRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from stats_inf_blogger_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into stats_inf_blogger_rank(rank,id_id,nickname_nickname,bodycount,replycount,influencevalue,subject,company_num)" + " VALUES (:rank,:id_id,:nickname_nickname,:bodycount,:replycount,:influencevalue,:subject,:company_num)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stStatsInfBloggerRank& ststats, m_listMap) + { + query.bindValue(":rank", ststats.rank); + query.bindValue(":id_id", ststats.id_id); + query.bindValue(":company_num", _nCompany); + query.bindValue(":nickname_nickname", ststats.nickname_nickname); + query.bindValue(":bodycount", ststats.bodycount); + query.bindValue(":replycount", ststats.replycount); + query.bindValue(":influencevalue", QString::number(ststats.influencevalue, 'g', 3)); + query.bindValue(":subject", ststats.subject); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} +*/ + +bool SInfluencer::SStatsInfBloggerRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from stats_inf_blogger_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into stats_inf_blogger_rank(rank,id_id,nickname_nickname,bodycount,replycount,influencevalue,subject,company_num)" + " VALUES ("; + query.clear(); + + foreach (const stStatsInfBloggerRank& ststats, m_listMap) + { + QString str = strQuery; + str += QString::number(ststats.rank) + ","; + str += "'" + sqlString(ststats.id_id) + "',"; + str += "'" + sqlString(ststats.nickname_nickname) + "',"; + str += QString::number(ststats.bodycount) + ","; + str += QString::number(ststats.replycount) + ","; + str += QString::number(ststats.influencevalue, 'g', 3) + ","; + str += "'" + ststats.subject + "',"; + str += QString::number(_nCompany) + ")"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} + + +void SInfluencer::SStatsInfBloggerBuzzbyday::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + QDate dateStart(2100,1,1),dateEnd(2000,1,1); + + QMap map; + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + QDate date = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date(); + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + if (!map.contains(date)) + map.insert(date, stInfluenceBuzz()); + map[date].bodycount++; + map[date].idcount.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + } + if (streal.m_mapRealReplyUniqueUrl.size() > 0) + { + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QDate date = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), + "yyyy-MM-dd HH:mm:ss").date(); + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + if (!map.contains(date)) + map.insert(date, stInfluenceBuzz()); + map[date].replycount++; + } + } + else + { + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + for (QMap::const_iterator iterPos1 = mapReply->value(_url).constBegin(); iterPos1 != mapReply->value(_url).constEnd(); iterPos1++) + { + QDate date = QDateTime::fromString((*iterPos1)[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), + "yyyy-MM-dd HH:mm:ss").date(); + dateStart = (date < dateStart ) ? date : dateStart; + dateEnd = (date > dateEnd ) ? date : dateEnd; + if (!map.contains(date)) + map.insert(date, stInfluenceBuzz()); + map[date].replycount++; + } + } + } + } + + for (QDate date = dateStart; date <= dateEnd; date = date.addDays(1)) + { + if (!map.contains(date)) + map.insert(date, stInfluenceBuzz()); + stStatsInfBloggerBuzzbyday ststats; + + ststats.date = date.toString("yyyy-MM-dd"); + ststats.formateddate = date.toString("yyyyMMdd").toInt(); + ststats.bodycount = map[date].bodycount; + ststats.idcount = map[date].idcount.size(); + ststats.replycount = map[date].replycount; + m_listMap.append(ststats); + } +} +/* +bool SInfluencer::SStatsInfBloggerBuzzbyday::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from stats_inf_blogger_buzzbyday where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into stats_inf_blogger_buzzbyday (company_num,date,formateddate,idcount,bodycount,replycount) " + "VALUES (:company_num,:date,:formateddate,:idcount,:bodycount,:replycount)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stStatsInfBloggerBuzzbyday& ststats, m_listMap) + { + query.bindValue(":company_num", _nCompany); + query.bindValue(":date", ststats.date); + query.bindValue(":formateddate", ststats.formateddate); + query.bindValue(":idcount", ststats.idcount); + query.bindValue(":bodycount", ststats.bodycount); + query.bindValue(":replycount", ststats.replycount); + + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} +*/ + + +bool SInfluencer::SStatsInfBloggerBuzzbyday::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from stats_inf_blogger_buzzbyday where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into stats_inf_blogger_buzzbyday (company_num,date,formateddate,idcount,bodycount,replycount) " + "VALUES ("; + query.clear(); + + foreach (const stStatsInfBloggerBuzzbyday& ststats, m_listMap) + { + QString str = strQuery; + str += QString::number(_nCompany) + ","; + str += "'" + ststats.date + "',"; + str += QString::number(ststats.formateddate) + ","; + str += QString::number(ststats.idcount) + ","; + str += QString::number(ststats.bodycount) + ","; + str += QString::number(ststats.replycount) + ")"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +void SInfluencer::SBodyInfluencerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& strUrl, *orderedUrl) + { + if (streal.m_setRealBodyUrl.contains(strUrl)) + { + stBodyInfluencerMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.category_num = iterPos.key(); + stbody.realin |= REAL_BODY; + m_listMap.append(stbody); + //_datagroup.addRealBodyCategory(strUrl, iterPos.key()); + if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stbody.realin |= REAL_REPLY; + /* + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + */ + } + } + } + } +} +/* +bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_influencer_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)" + " values (:num,:category_num,:id_num,:body_num,:realin)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stBodyInfluencerMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":id_num", stmap.id_num); + query.bindValue(":body_num", stmap.body_num); + query.bindValue(":realin", stmap.realin); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + + +bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_influencer_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)" + " values ("; + query.clear(); + + + foreach (const stBodyInfluencerMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.id_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.realin) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + + + +void SInfluencer::SReplyInfluencerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + const QMap >* mapReply = _datagroup.getmapReply(); + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + + if (streal.m_mapRealReplyUniqueUrl.size() > 0) + { + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.end(); iterPos1++) + { + stReplyInfluencerMap streply; + streply.num = num++; + streply.body_num = _datagroup.getstBodyNum(iterPos1.key()); + streply.category_num = iterPos.key(); + streply.realin |= REAL_REPLY; + streply.id_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num; + streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num; + m_listMap.append(streply); + + /* + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + */ + } + } + else + { + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + for (QMap::const_iterator iterPos1 = mapReply->value(_url).constBegin(); iterPos1 != mapReply->value(_url).constEnd(); iterPos1++) + { + stReplyInfluencerMap streply; + streply.num = num++; + streply.body_num = _datagroup.getstBodyNum(_url); + streply.category_num = iterPos.key(); + streply.realin |= REAL_REPLY; + streply.id_num = _datagroup.getstReply(_url, iterPos1.key())->id_num; + streply.reply_num = _datagroup.getstReply(_url, iterPos1.key())->num; + m_listMap.append(streply); + } + } + } + } +} + + +/* +bool SInfluencer::SReplyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate reply_influencer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,reply_num,realin)" + " values (:num,:category_num,:id_num,:body_num,:reply_num,:realin)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stReplyInfluencerMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":id_num", stmap.id_num); + query.bindValue(":body_num", stmap.body_num); + query.bindValue(":realin", stmap.realin); + query.bindValue(":reply_num", stmap.reply_num); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + +bool SInfluencer::SReplyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate reply_influencer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_influencer_map_" + QString::number(_nCompany) + + "_update (num,category_num,id_num,body_num,reply_num,realin)" + " values ("; + query.clear(); + + foreach (const stReplyInfluencerMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.id_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.reply_num) + ","; + str += QString::number(stmap.realin) + ")"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +bool SInfluencer::SBodyInfluencerMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "body_influencer_map_" + QString::number(_nCompany) + " TO " + "body_influencer_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "body_influencer_map_" + QString::number(_nCompany) + "_update" + " TO " + "body_influencer_map_" + QString::number(_nCompany) + ","; + strQuery += "body_influencer_map_" + QString::number(_nCompany) + "_temp" + " TO " + "body_influencer_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + + +bool SInfluencer::SReplyInfluencerMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "reply_influencer_map_" + QString::number(_nCompany) + " TO " + "reply_influencer_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "reply_influencer_map_" + QString::number(_nCompany) + "_update" + " TO " + "reply_influencer_map_" + QString::number(_nCompany) + ","; + strQuery += "reply_influencer_map_" + QString::number(_nCompany) + "_temp" + " TO " + "reply_influencer_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + + diff --git a/sfilterprocess/sinfluencer.h b/sfilterprocess/sinfluencer.h new file mode 100644 index 0000000..88df563 --- /dev/null +++ b/sfilterprocess/sinfluencer.h @@ -0,0 +1,216 @@ +#ifndef SINFLUENCER +#define SINFLUENCER +#include "sservice.h" +#include "sfilteralgorithm.h" +#include "sinitializer.h" +#include +class QString; +class QStringList; +class SInfluencer:public SService +{ +public: + struct stReal + { + QSet m_setRealAllUrl; + QSet m_setRealBodyUrl; + QMap m_mapRealReplyUniqueUrl; //multiple + }; + struct stInfluencer + { + int id_num; + int nickname_num; + int community_num; + int company_num; + QString id_id; + QString profile; + QString profileurl; + int bodycount; + int replycount; + double influencervalue; + int rank; + stInfluencer() + { + id_num = nickname_num = community_num = company_num = bodycount = replycount = rank = 0; + influencervalue = 0.0; + } + + }; + struct stStatsInfBloggerRank + { + QString id_id; + QString nickname_nickname; + int bodycount; + int replycount; + double influencevalue; + QString subject; + int company_num; + int rank; + stStatsInfBloggerRank() + { + bodycount = replycount = rank = 0; + influencevalue = 0.0; + } + stStatsInfBloggerRank(const QString &_id_id, const QString &_nickname_nickname):id_id(_id_id),nickname_nickname(_nickname_nickname), + bodycount(0), replycount(0), rank(0), influencevalue(0.0) {} + }; + + struct stStatsInfBloggerBuzzbyday + { + QString date; + int formateddate; + int idcount; + int bodycount; + int replycount; + int company_num; + stStatsInfBloggerBuzzbyday():formateddate(0), idcount(0), bodycount(0), replycount(0), company_num(0){} + }; + + struct stBodyInfluencerMap + { + int num; + int body_num; + int id_num; + int category_num; + int realin; + stBodyInfluencerMap():num(0),body_num(0), id_num(0), category_num(0), realin(0) {} + }; + struct stReplyInfluencerMap + { + int num; + int reply_num; + int body_num; + int id_num; + int category_num; + int realin; + stReplyInfluencerMap() + { + num = reply_num = body_num = id_num = category_num = realin = 0; + } + }; + class STInfluencer + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + void clear(){ m_listMap.clear(); } + }; + + class SStatsInfBloggerRank + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + }; + + class SStatsInfBloggerBuzzbyday + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + }; + + class SBodyInfluencerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + class SReplyInfluencerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + struct isGreaterInfluencerValue + { + bool operator()(const stInfluencer& first, const stInfluencer& second) + { + return first.influencervalue > second.influencervalue; + } + bool operator()(const stStatsInfBloggerRank& first, const stStatsInfBloggerRank& second) + { + return first.influencevalue > second.influencevalue; + } + }; + + struct isGreaterBodyCount + { + bool operator()(const stStatsInfBloggerRank& first, const stStatsInfBloggerRank& second) + { + return first.bodycount > second.bodycount; + } + bool operator()(const stInfluencer& first, const stInfluencer& second) + { + return first.bodycount > second.bodycount; + } + }; + + struct isGreaterReplyCount + { + bool operator()(const stStatsInfBloggerRank& first, const stStatsInfBloggerRank& second) + { + return first.replycount > second.replycount; + } + bool operator()(const stInfluencer& first, const stInfluencer& second) + { + return first.replycount > second.replycount; + } + }; + struct stInfluenceBuzz + { + int bodycount; + int replycount; + QSet idcount; + stInfluenceBuzz():bodycount(0), replycount(0) {} + }; + +private: + SFilterAlgorithm m_SFilterAlgorithm; + QMap m_mapCategory; + SBodyInfluencerMap m_SBodyInfluencerMap; + SReplyInfluencerMap m_SReplyInfluencerMap; + SStatsInfBloggerBuzzbyday m_SStatsInfBloggerBuzzbyday; + SStatsInfBloggerRank m_SStatsInfBloggerRank; + STInfluencer m_STInfluencer; + +public: + virtual void setInitiaizer(SInitializer &_initializer); + virtual bool uploadData(int _companyNum); + virtual bool changeTable(int _companyNum); + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter); + virtual bool makeOverallCategory(int _categoryNum); + virtual bool makeTable(SDatagroup &_datagroup); + +}; + + +#endif // SINFLUENCER + diff --git a/sfilterprocess/sinitializer.cpp b/sfilterprocess/sinitializer.cpp new file mode 100644 index 0000000..0c4c36a --- /dev/null +++ b/sfilterprocess/sinitializer.cpp @@ -0,0 +1,527 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sinitializer.h" +#include "../common.h" +using namespace std; +/* +bool SInitializer::disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode) +{ + if (_nMode == DEBUG) + { + return true; + } + + QSqlQuery query(db); + + foreach (const QString& strIndex, _strIndexNames) + { + QString strQuery = "alter table " + _tableName + " drop index " + strIndex; + + if(!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + } + return true; +} + +bool SInitializer::enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode) +{ + if (_nMode == DEBUG) + { + return true; + } + + QSqlQuery query(db); + foreach (const QString& strIndex, _strIndexNames) + { + QString strQuery = "alter table " + _tableName + " add index (" +strIndex + ")"; + if(!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + } + return true; +} + +bool SInitializer::disableSettings(QSqlDatabase& db, int _nMode) +{ + if (_nMode == DEBUG) + { + return true; + } + QSqlQuery query(db); + QString strQuery = "SET AUTOCOMMIT = 0; SET FOREIGN_KEY_CHECKS = 0; SET UNIQUE_CHECKS = 0"; + if(!query.exec(strQuery)) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + return true; +} + +bool SInitializer::enableSettings(QSqlDatabase& db, int _nMode) +{ + if (_nMode == DEBUG) + { + return true; + } + QSqlQuery query(db); + QString strQuery = "SET AUTOCOMMIT = 1; SET FOREIGN_KEY_CHECKS = 1; SET UNIQUE_CHECKS = 1"; + if(!query.exec(strQuery)) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + return true; +} +*/ + +void SInitializer::updateWebDBInfoState(QSqlDatabase &_db, int _nCompany, QString _strStatus) +{ + QString strQuery = "update dbinfo set " + "status = '" + _strStatus + "' " + "where company_num = " + QString::number(_nCompany); + _db.exec(strQuery.toUtf8()); +} + +void SInitializer::updateWebDBInfoComplete(QSqlDatabase _db, QString _mindate, QString _maxdate, QString _lastupdate, int _bodycount, int _replycount, int _nCompany) +{ + QString strQuery = "update dbinfo set " + "bodycount = " + QString::number(_bodycount) + " , " + "replycount = " + QString::number(_replycount)+" , " + "mindate = '" + _mindate + "'," + "maxdate = '" + _maxdate + "'," + "lastupdate = '" + _lastupdate + "' " + "where company_num = " + QString::number(_nCompany); + _db.exec(strQuery.toUtf8()); +} + + +bool SInitializer::initDays() +{ + QSettings settings(QString("setup.ini"), QSettings::IniFormat); + settings.setIniCodec("UTF-8"); + settings.beginGroup(QString("settings")); + if (settings.childKeys().size() == 0) return false; + int nCutDays = settings.value("days").toInt(); + + QString strFrom = settings.value("from").toString().trimmed(); + QString strTo = settings.value("to").toString().trimmed(); + + if(!strFrom.isEmpty()) + { + m_strStartDay = strFrom; + } + else + { + m_strStartDay = QDate::currentDate().addDays(-abs(nCutDays)).toString("yyyy-MM-dd"); + } + if(!strTo.isEmpty()) + { + m_strEndDay = strTo; + } + else + { + m_strEndDay = QDate::currentDate().toString("yyyy-MM-dd"); + } + settings.endGroup(); + return true; +} + +bool SInitializer::initSpammer() +{ + QSettings settings(QString("setup.ini"), QSettings::IniFormat); + settings.setIniCodec("UTF-8"); + settings.beginGroup(QString("spammers")); + + if (settings.childKeys().size() == 0) return false; + m_adSpammerParam[E_SPAMER_BODY_COUNT_CUT] = settings.value("body_cut").toDouble(); + m_adSpammerParam[E_SPAMER_BODY_COUNT_RATIO] = settings.value("body_ratio").toDouble(); + m_adSpammerParam[E_SPAMER_REPLY_COUNT_CUT] = settings.value("reply_cut").toDouble(); + m_adSpammerParam[E_SPAMER_REPLY_COUNT_RATIO] = settings.value("reply_ratio").toDouble(); + m_adSpammerParam[E_SPAMER_NICK_COUNT_CUT] = settings.value("nick_cut").toDouble(); + m_adSpammerParam[E_SPAMER_NICK_COUNT_RATIO] = settings.value("nick_ratio").toDouble(); + settings.endGroup(); + + return true; +} + +bool SInitializer::initDBConnection() +{ + QSettings settings(QString("setup.ini"), QSettings::IniFormat); + settings.setIniCodec("UTF-8"); + settings.beginGroup(QString("database")); + if (settings.childKeys().size() == 0) return false; + m_stWebDBInfo.m_strHostName = settings.value("host").toString(); + m_stWebDBInfo.m_strUserName = settings.value("user").toString(); + m_stWebDBInfo.m_strPassword = settings.value("pass").toString(); + m_stWebDBInfo.m_strDBName = settings.value("name").toString(); + settings.endGroup(); + + + m_sDBManager.addDatabase("DATA", SDBManager::E_DATABASE_DATA, QString("bigbird.iptime.org"), + QString("admin"), QString("admin123"), QString("concepters")); + + m_sDBManager.addDatabase("WEB", SDBManager::E_DATABASE_WEB, + m_stWebDBInfo.m_strHostName, m_stWebDBInfo.m_strUserName, + m_stWebDBInfo.m_strPassword, m_stWebDBInfo.m_strDBName); + + + if(!m_sDBManager.open("DATA")) + { + insertLog("DATA DB open failed"); + return false; + } + if(!m_sDBManager.open("WEB")) + { + insertLog("WEB DB open failed"); + return false; + } + + return true; +} + +bool SInitializer::init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName) +{ + m_stFilterProcessGroup.m_nFilterProcessGroupID = _strFilterProcessGroupID.toInt(); + m_stFilterProcessGroup.m_strFilterProcessGroupName = _strFilterProcessGroupName; + + bool b_ok = true; + b_ok &= initDBConnection(); + b_ok &= initColumn(); + b_ok &= initDays(); + b_ok &= initInfluencer(); + b_ok &= initSpammer(); + b_ok &= initFilterProcess(); + b_ok &= initCompanyNum(); + b_ok &= initPlatform(); + b_ok &= initConsumerCategory(); +/* + if (!b_ok) + cout << "FALSE" << endl; + else + cout << "TRUE" << endl; +*/ + return b_ok; +} + +bool SInitializer::initInfluencer() +{ + m_stInfluencerParam.m_dBody = 0.4; + m_stInfluencerParam.m_dReply = 0.6; + return true; +} + + +bool SInitializer::initFilterProcess() +{ + QString strQuery = QString("select filterprocess_type,datagroup_id,filtergroup_id,company_num,category_num,filterprocess_order " + "from filterprocess where filterprocessgroup_id = ") + QString::number(m_stFilterProcessGroup.m_nFilterProcessGroupID); + + QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_DATA]); + + if(query.exec(strQuery.toUtf8()) == false) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + + while(query.next()) + { + stFilterProcess stfilterprocess; + stfilterprocess.m_nFilterProcessType = query.value(0).toInt(); + stfilterprocess.m_nDatagroupID = query.value(1).toInt(); + stfilterprocess.m_nFilterGroupID = query.value(2).toInt(); + stfilterprocess.m_nCompanyNum = query.value(3).toInt(); + stfilterprocess.m_nCategoryNum = query.value(4).toInt(); + stfilterprocess.m_nFilterProcessOrder = query.value(5).toInt(); + m_listFilterProcess.append(stfilterprocess); + } + + query.clear(); + strQuery = "select type, data from filter where filtergroup_id = "; + foreach (const stFilterProcess& stfilterprocess, m_listFilterProcess) + { + if (stfilterprocess.m_nFilterGroupID == 0) + continue; + + if (!query.exec(QString(strQuery + QString::number(stfilterprocess.m_nFilterGroupID)).toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + QList listFilter; + + while (query.next()) + { + stFilter stfilter; + stfilter.m_nFilterGroupID = stfilterprocess.m_nFilterGroupID; + stfilter.m_vData = query.value(1); + stfilter.m_nType = query.value(0).toInt(); + listFilter.append(stfilter); + } + if (!m_mapFilterGroup.contains(stfilterprocess.m_nFilterGroupID)) + m_mapFilterGroup.insert(stfilterprocess.m_nFilterGroupID, listFilter); + + } + return true; +} + + +bool SInitializer::initConsumerCategory() +{ + QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]); + int company_num = -1; + m_nConsumerAllCategory = -1; + foreach (const stFilterProcess& stfilterprocess, m_listFilterProcess) + { + if (stfilterprocess.m_nFilterProcessType == 1) + { + QString strQuery = "select statsColumnNum from category where company_num = " + + QString::number(stfilterprocess.m_nCompanyNum) + " and num = " + QString::number(stfilterprocess.m_nCategoryNum); + if (!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + while (query.next()) + { + m_mapConsumerCategory.insert(stfilterprocess.m_nCategoryNum, query.value(0).toInt()); + } + company_num = stfilterprocess.m_nCompanyNum; + } + } + if (company_num != -1) + { + QString strQuery = "select num from category where company_num = " + QString::number(company_num) + " and statsColumnNum = 1"; + if (!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + while (query.next()) + { + m_nConsumerAllCategory = query.value(0).toInt(); + m_mapConsumerCategory.insert(m_nConsumerAllCategory ,1); + } + } + return true; +} + +int SInitializer::getConsumerAllCategory() +{ + return m_nConsumerAllCategory; +} + + +const double* SInitializer::getSpammerParam() const +{ + return m_adSpammerParam; +} + +bool SInitializer::initColumn() +{ + m_stColumn.m_slColumn.clear(); + for (int i = 0; i < E_DATA_MAX; i++) + m_stColumn.m_anColumn[i] = -1; + + QVector vecColumn; + QFile file("column.txt"); + if (file.open(QIODevice::ReadOnly | QIODevice::Text) == false) return false; + { + while (!file.atEnd()) + { + QString str = QString(file.readLine()); + if (str.at(0) == QChar('#')) continue; + if (str.trimmed().isEmpty()) continue; + vecColumn.push_back(str.split(",")); + } + } + if (vecColumn.size() <= 0) return false; + + foreach (const QStringList& strList,vecColumn) + m_stColumn.m_slColumn.push_back(strList.at(E_COLUMN_DATABASE)); + + for (QStringList::size_type i = 0; i < m_stColumn.m_slColumn.size(); i++) + { + if (m_stColumn.m_slColumn[i].trimmed() == QString("platform_title")) m_stColumn.m_anColumn[E_DATA_platform_title] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_form")) m_stColumn.m_anColumn[E_DATA_article_form] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_parent")) m_stColumn.m_anColumn[E_DATA_article_parent] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_id")) m_stColumn.m_anColumn[E_DATA_article_id] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_nickname")) m_stColumn.m_anColumn[E_DATA_article_nickname] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_title")) m_stColumn.m_anColumn[E_DATA_article_title] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_data")) m_stColumn.m_anColumn[E_DATA_article_data] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_url")) m_stColumn.m_anColumn[E_DATA_article_url] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_hit")) m_stColumn.m_anColumn[E_DATA_article_hit] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_date")) m_stColumn.m_anColumn[E_DATA_article_date] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_order")) m_stColumn.m_anColumn[E_DATA_article_order] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("platform_id")) m_stColumn.m_anColumn[E_DATA_platform_id] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("platform_name")) m_stColumn.m_anColumn[E_DATA_platform_name] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("platform_form")) m_stColumn.m_anColumn[E_DATA_platform_form] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("reply_url")) m_stColumn.m_anColumn[E_DATA_reply_url] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("keyword_id")) m_stColumn.m_anColumn[E_DATA_keyword_id] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_profileurl")) m_stColumn.m_anColumn[E_DATA_article_profileurl] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("article_profile")) m_stColumn.m_anColumn[E_DATA_article_profile] = i; + if (m_stColumn.m_slColumn[i].trimmed() == QString("etc")) m_stColumn.m_anColumn[E_DATA_etc] = i; + } + return true; +} + +bool SInitializer::initCompanyNum() +{ + return true; +} + +void SInitializer::insertLog(const QString& _strMsg) const +{ + cout << "["; + cout << m_stFilterProcessGroup.m_strFilterProcessGroupName.toStdString(); + cout << "] "; + cout << _strMsg.toStdString() << endl; + + QFile file(m_stFilterProcessGroup.m_strFilterProcessGroupName + "." + QDate::currentDate().toString(Qt::ISODate)+".log"); + if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) + return; + + QTextStream out(&file); + out << QTime::currentTime().toString("[hh:mm:ss] ") << _strMsg << "\n"; + file.close(); +} + +QSqlDatabase SInitializer::getDatabase(const QString& _dbName) const +{ + return m_sDBManager[_dbName]; +} + +QSqlDatabase SInitializer::getDatabase(const SDBManager::E_DATABASE& _eDatabase) const +{ + return m_sDBManager[_eDatabase]; +} + +QStringList SInitializer::getColumnStringList() const +{ + return m_stColumn.m_slColumn; +} + +QString SInitializer::getStartDay() const +{ + return m_strStartDay; +} + +QString SInitializer::getEndDay() const +{ + return m_strEndDay; +} + +const int* SInitializer::getColumnIntArray() const +{ + return m_stColumn.m_anColumn; +} + +QList SInitializer::getFilterProcess() +{ + return m_listFilterProcess; +} + +QMap > SInitializer::getFilterGroup() +{ + return m_mapFilterGroup; +} + +bool SInitializer::initPlatform() +{ + QString strQuery; + strQuery = "select num,form FROM platformform"; + QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]); + + if (!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + while(query.next()) + m_stPlatform.m_mapPlatformForm.insert(query.value(1).toString(),query.value(0).toInt()); + + + query.clear(); + strQuery = "select num,name FROM platformname"; + + if (!query.exec(strQuery.toUtf8())) + { + insertLog(query.lastQuery()); + insertLog(query.lastError().text()); + return false; + } + + while(query.next()) + m_stPlatform.m_mapPlatformName.insert(query.value(1).toString(),query.value(0).toInt()); + + return true; +} +QList SInitializer::getFilterGroup(int _nFilterGroupId) +{ + return m_mapFilterGroup[_nFilterGroupId]; +} + +int SInitializer::getPlatformForm(const QString &_form) +{ + return m_stPlatform.m_mapPlatformForm.value(_form.trimmed(), 1); +} + +int SInitializer::getPlatformName(const QString &_name) +{ + return m_stPlatform.m_mapPlatformName.value(_name.trimmed(), 1); +} + +QMap* SInitializer::getConsumerCategory() +{ + return &m_mapConsumerCategory; +} +double SInitializer::getInfluencerParam(int _BodyorReply) +{ + return m_stInfluencerParam[_BodyorReply]; +} + +void SInitializer::closeConnection(const SDBManager::E_DATABASE _eDatabase) +{ + m_sDBManager.close(_eDatabase); +} + +void SInitializer::closeConnection(const QString& _str) +{ + m_sDBManager.close(_str); +} + +bool SInitializer::openConnection(const SDBManager::E_DATABASE _eDatabase) +{ + return m_sDBManager.open(_eDatabase); +} + +bool SInitializer::openConnection(const QString& _str) +{ + return m_sDBManager.open(_str); +} diff --git a/sfilterprocess/sinitializer.h b/sfilterprocess/sinitializer.h new file mode 100644 index 0000000..37cd235 --- /dev/null +++ b/sfilterprocess/sinitializer.h @@ -0,0 +1,202 @@ +#ifndef SINITIALIZER +#define SINITIALIZER +#include "sdbmanager.h" +#include +#include +#include +class QString; +class QStringList; +class QSqlDatabase; + +const int REAL_BODY = 0x01; +const int REAL_REPLY = 0x02; +//const int DEBUG = 0; +//const int RELEASE = 1; +//const int MODE = RELEASE; + +class SInitializer +{ + +private: + bool initDays(); + bool initDBConnection(); + bool initFilterProcess(); + bool initCompanyNum(); + bool initColumn(); + bool initInfluencer(); + bool initSpammer(); + bool initPlatform(); + bool initConsumerCategory(); + + +public: + enum E_SPAMER + { + E_SPAMER_BODY_COUNT_CUT = 0, + E_SPAMER_BODY_COUNT_RATIO, + E_SPAMER_REPLY_COUNT_CUT, + E_SPAMER_REPLY_COUNT_RATIO, + E_SPAMER_NICK_COUNT_CUT, + E_SPAMER_NICK_COUNT_RATIO, + E_SPAMER_MAX + }; + + enum E_COLUMN + { + E_COLUMN_DATABASE=0, + E_COLUMN_NAME, + E_COLUMN_DATE, + E_COLUMN_COUNT, + }; + + enum E_ARTICLE + { + E_ARTICLE_BODY = 0, + E_ARTICLE_REPLY, + E_ARTICLE_MAX, + }; + + enum E_DATA + { + E_DATA_platform_title = 0, + E_DATA_article_form, + E_DATA_article_parent, + E_DATA_article_id, + E_DATA_article_nickname, + E_DATA_article_title, + E_DATA_article_data, + E_DATA_article_url, + E_DATA_article_hit, + E_DATA_article_date, + E_DATA_article_order, + E_DATA_platform_id, + E_DATA_platform_name, + E_DATA_platform_form, + E_DATA_reply_url, + E_DATA_keyword_id, + E_DATA_article_profileurl, + E_DATA_article_profile, + E_DATA_etc, + E_DATA_MAX, + }; + + struct stDataBaseInfo + { + QString m_strHostName; + QString m_strUserName; + QString m_strPassword; + QString m_strDBName; + }; + struct stFilterProcessGroup + { + int m_nFilterProcessGroupID; + QString m_strFilterProcessGroupName; + }; + struct stInfluencerParam + { + double m_dBody; + double m_dReply; + stInfluencerParam() : m_dBody(0.4), m_dReply(0.6) {} + double operator[](const int nSelect) const + { + switch(nSelect) + { + case 0: + return m_dBody; + case 1: + return m_dReply; + default: + return 0.0; + } + } + }; + struct stColumn + { + QStringList m_slColumn; + int m_anColumn[E_DATA_MAX]; + stColumn() + { + for (int i = 0; i < E_DATA_MAX; i++) + m_anColumn[i] = -1; + } + }; + struct stFilterProcess + { + int m_nDatagroupID; + int m_nFilterGroupID; + int m_nCategoryNum; + int m_nFilterProcessOrder; + int m_nFilterProcessType; + int m_nCompanyNum; + }; + + struct stFilter + { + int m_nType; + int m_nFilterGroupID; + QVariant m_vData; + stFilter() + { + m_nType = -1; + m_nFilterGroupID = -1; + } + }; + struct stPlatform + { + QMap m_mapPlatformName; + QMap m_mapPlatformForm; + }; + + + +private: + SDBManager m_sDBManager; + QString m_strStartDay; + QString m_strEndDay; + double m_adSpammerParam[E_SPAMER_MAX]; + stDataBaseInfo m_stWebDBInfo; + int m_nCompanyNum; + stFilterProcessGroup m_stFilterProcessGroup; + stInfluencerParam m_stInfluencerParam; + stColumn m_stColumn; + QList m_listFilterProcess; + QMap > m_mapFilterGroup; // key : filtergroupid + int m_nDataGroupNumber; + stPlatform m_stPlatform; + QMap m_mapConsumerCategory; + int m_nConsumerAllCategory; + +public: + bool init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName); + void insertLog(const QString& _strMsg) const; + QSqlDatabase getDatabase(const QString& _dbName) const; + QSqlDatabase getDatabase(const SDBManager::E_DATABASE& _eDatabase) const; + int getDataGroupNumber() const; + QStringList getColumnStringList() const; + QString getStartDay() const; + QString getEndDay() const; + const int* getColumnIntArray() const; + QList getFilterProcess(); + QMap > getFilterGroup(); + const double* getSpammerParam() const; + QList getFilterGroup(int _nFilterGroupId); + int getPlatformForm(const QString &_form); + int getPlatformName(const QString &_name); + QMap* getConsumerCategory(); + int getConsumerAllCategory(); + void closeConnection(const SDBManager::E_DATABASE _eDatabase); + void closeConnection(const QString& _str); + bool openConnection(const SDBManager::E_DATABASE _eDatabase); + bool openConnection(const QString& _str); + double getInfluencerParam(int _BodyorReply); + void updateWebDBInfoState(QSqlDatabase &_db, int _nCompany, QString _strStatus); + void updateWebDBInfoComplete(QSqlDatabase _db, QString _mindate, QString _maxdate, QString _lastupdate, int _bodycount, int _replycount, int _nCompany); + //bool disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); + //bool enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE); + //bool disableSettings(QSqlDatabase& db, int _nMode = MODE); + //bool enableSettings(QSqlDatabase& db, int _nMode = MODE); +}; + + +#endif // SINITIALIZER + diff --git a/sfilterprocess/spowercafe.cpp b/sfilterprocess/spowercafe.cpp new file mode 100644 index 0000000..469eeaa --- /dev/null +++ b/sfilterprocess/spowercafe.cpp @@ -0,0 +1,758 @@ +#include +#include +#include +#include +#include +#include +#include "spowercafe.h" + +void SPowercafe::setInitiaizer(SInitializer &_initializer) +{ + m_pSInitializer = &_initializer; + m_SBodyPowercafeMap.setInitiaizer(_initializer); + m_SReplyPowercafeMap.setInitiaizer(_initializer); + m_STPowercafe.setInitiaizer(_initializer); + m_SStatsPowCafeRank.setInitiaizer(_initializer); +} + +bool SPowercafe::uploadData(int _companyNum) +{ + m_pSInitializer->insertLog("Start upload Powercafe"); + + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Powercafe (Start)"); + if (!m_SBodyPowercafeMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload Powercafe body map"); + return false; + } + if (!m_SReplyPowercafeMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload Powercafe reply map"); + return false; + } + if (!m_SStatsPowCafeRank.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_pow_cafe_rank"); + return false; + } + if (!m_STPowercafe.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload Powercafe"); + return false; + } + m_pSInitializer->insertLog("End upload Powercafe"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Powercafe (Finish)"); + + return true; +} + +bool SPowercafe::changeTable(int _companyNum) +{ + m_pSInitializer->insertLog("Start table change powercafe"); + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + if (!m_SBodyPowercafeMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload powercafe body map"); + return false; + } + if (!m_SReplyPowercafeMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload powercafe reply map"); + return false; + } + return true; +} + +bool SPowercafe::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter) +{ + QVector vecData[SInitializer::E_ARTICLE_MAX]; + bool bFiltered[SInitializer::E_ARTICLE_MAX]; + + for (int i = 0; i < SInitializer::E_ARTICLE_MAX; i++) + bFiltered[i] = false; + _datagroup.copyData(vecData); + foreach (const SInitializer::stFilter& filter, _listFilter) + { + switch(filter.m_nType) + { + case E_FILTER_TYPE_DATE://Date + m_SFilterAlgorithm.FilterDate(vecData, bFiltered, filter.m_vData.toString(), + m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_article_date]); + + break; + case E_FILTER_TYPE_SEARCH: + m_SFilterAlgorithm.FilterSearch(vecData, bFiltered, filter.m_vData.toString()); + + break; + case E_FILTER_TYPE_LENGTH: + m_SFilterAlgorithm.FilterLength(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_REPLACE: + m_SFilterAlgorithm.FilterReplace(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_COUNTER: + m_SFilterAlgorithm.FilterCounter(vecData, bFiltered, filter.m_vData.toString()); + break; + } + } + stReal streal; + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size(); + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size(); + const int *anColumn = m_pSInitializer->getColumnIntArray(); + if (bFiltered[SInitializer::E_ARTICLE_BODY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_BODY]) + { + if (strlist[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_setRealBodyUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + if (bFiltered[SInitializer::E_ARTICLE_REPLY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_REPLY]) + { + if (strlist[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_mapRealReplyUniqueUrl.insertMulti(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed(), + strlist[anColumn[SInitializer::E_DATA_article_order]].trimmed().toInt()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + m_mapCategory.insert(_nCategory, streal); + return true; +} + +bool SPowercafe::makeOverallCategory(int _categoryNum) +{ + return true; +} + +bool SPowercafe::makeTable(SDatagroup &_datagroup) +{ + m_pSInitializer->insertLog("Make powercafe tables"); + m_pSInitializer->insertLog("Make powercafe body tables"); + m_SBodyPowercafeMap.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make powercafe reply tables"); + m_SReplyPowercafeMap.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make powercafe stats_pow_cafe_rank tables"); + m_SStatsPowCafeRank.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make powercafe powercafe table"); + m_STPowercafe.makeTable(_datagroup, m_mapCategory); + return true; + +} + +void SPowercafe::STPowercafe::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + QMap mapPowercafeAll; + + QMap > mapUserAll; + QMap > mapUserReal; + + + for (QMap::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++) + { + if (iterPos.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stPowercafe()); + mapUserAll.insert(strKey, QSet()); + } + mapPowercafeAll[strKey].bodycountall++; + mapUserAll[strKey].insert(iterPos.value().at(anColumn[SInitializer::E_DATA_article_id]).trimmed()); + } + + for (QMap >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++) + { + for (QMap::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++) + { + if (iterPos1.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos1.value().at(anColumn[SInitializer::E_DATA_platform_id]).trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stPowercafe()); + mapUserAll.insert(strKey, QSet()); + } + mapPowercafeAll[strKey].replycountall++; + mapUserAll[strKey].insert(iterPos1.value().at(anColumn[SInitializer::E_DATA_article_id]).trimmed()); + } + } + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapPowercafeReal; + const stReal& streal = iterPos.value(); + + foreach (const QString& url, streal.m_setRealBodyUrl) + { + QString strKey = (*mapBody)[url][anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeReal.contains(strKey)) + { + mapPowercafeReal.insert(strKey, stPowercafe()); + mapPowercafeReal[strKey].community_id = _datagroup.getstCommunity((*mapBody)[url])->id; + mapPowercafeReal[strKey].community_num = _datagroup.getstCommunity((*mapBody)[url])->num; + mapPowercafeReal[strKey].bodycountall = mapPowercafeAll[strKey].bodycountall; + mapPowercafeReal[strKey].replycountall = mapPowercafeAll[strKey].replycountall; + mapPowercafeReal[strKey].allcountall = mapPowercafeAll[strKey].bodycountall + mapPowercafeAll[strKey].replycountall; + mapPowercafeReal[strKey].usercountall = mapUserAll[strKey].size(); + mapUserReal.insert(strKey, QSet()); + } + mapPowercafeReal[strKey].bodycountreal++; + mapUserReal[strKey].insert((*mapBody)[url][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.begin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeReal.contains(strKey)) + { + mapPowercafeReal.insert(strKey, stPowercafe()); + mapPowercafeReal[strKey].community_id = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->id; + mapPowercafeReal[strKey].community_num = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->num; + mapPowercafeReal[strKey].bodycountall = mapPowercafeAll[strKey].bodycountall; + mapPowercafeReal[strKey].replycountall = mapPowercafeAll[strKey].replycountall; + mapPowercafeReal[strKey].allcountall = mapPowercafeAll[strKey].bodycountall + mapPowercafeAll[strKey].replycountall; + mapPowercafeReal[strKey].usercountall = mapUserAll[strKey].size(); + mapUserReal.insert(strKey, QSet()); + } + mapPowercafeReal[strKey].replycountreal++; + mapUserReal[strKey].insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed()); + } + + for (QMap::iterator iterPos1 = mapPowercafeReal.begin(); iterPos1 != mapPowercafeReal.end(); iterPos1++) + { + iterPos1.value().usercountreal = mapUserReal[iterPos1.key()].size(); + iterPos1.value().allcountreal = iterPos1.value().bodycountreal + iterPos1.value().replycountreal; + } + QList listPowercafe = mapPowercafeReal.values(); + + qSort(listPowercafe.begin(), listPowercafe.end(), isGreaterCountReal()); + int rank = 1; + foreach (stPowercafe stpowercafe, listPowercafe) + { + stpowercafe.rank = rank++; + m_listMap.push_back(stpowercafe); + } + } +} + +/* +bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from powercafe where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into powercafe (company_num,rank,community_num,community_id,bodycountall,replycountall,allcountall,usercountall,bodycountreal,replycountreal,allcountreal,usercountreal)" + " VALUES (:COMPANY_NUM,:RANK,:COMMUNITY_NUM,:COMMUNITY_ID,:BODY_ALL,:REPLY_ALL,:COUNT_ALL,:USER_ALL,:BODY_REAL,:REPLY_REAL,:ALL_REAL,:USER_REAL)"; + + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stPowercafe& stpowercafe, m_listMap) + { + query.bindValue(":COMPANY_NUM", _nCompany); + query.bindValue(":COMMUNITY_NUM", stpowercafe.community_num); + query.bindValue(":RANK", stpowercafe.rank); + query.bindValue(":COMMUNITY_ID", stpowercafe.community_id); + query.bindValue(":BODY_ALL", stpowercafe.bodycountall); + query.bindValue(":REPLY_ALL", stpowercafe.replycountall); + query.bindValue(":COUNT_ALL", stpowercafe.allcountall); + query.bindValue(":USER_ALL", stpowercafe.usercountall); + query.bindValue(":BODY_REAL", stpowercafe.bodycountreal); + query.bindValue(":REPLY_REAL", stpowercafe.replycountreal); + query.bindValue(":ALL_REAL", stpowercafe.allcountreal); + query.bindValue(":USER_REAL", stpowercafe.usercountreal); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + strQuery = "update dbinfo set powercafecount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + return true; +} +*/ + +bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from powercafe where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into powercafe (company_num,rank,community_num,community_id,bodycountall,replycountall,allcountall,usercountall," + "bodycountreal,replycountreal,allcountreal,usercountreal)" + " VALUES ("; + + query.clear(); + + foreach (const stPowercafe& stpowercafe, m_listMap) + { + QString str = strQuery; + str += QString::number(_nCompany) + ","; + str += QString::number(stpowercafe.rank) + ","; + str += QString::number(stpowercafe.community_num) + ","; + str += "'" + sqlString(stpowercafe.community_id) + "',"; + str += QString::number(stpowercafe.bodycountall) + ","; + str += QString::number(stpowercafe.replycountall) + ","; + str += QString::number(stpowercafe.allcountall) + ","; + str += QString::number(stpowercafe.usercountall) + ","; + str += QString::number(stpowercafe.bodycountreal) + ","; + str += QString::number(stpowercafe.replycountreal) + ","; + str += QString::number(stpowercafe.allcountreal) + ","; + str += QString::number(stpowercafe.usercountreal) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + strQuery = "update dbinfo set powercafecount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + return true; +} + + +void SPowercafe::SStatsPowCafeRank::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + QMap mapPowercafeAll; + for (QMap::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++) + { + if (iterPos.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stStatsPowCafeRank()); + mapPowercafeAll[strKey].community_id = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + mapPowercafeAll[strKey].community_title = iterPos.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + mapPowercafeAll[strKey].subject = "articlecountall"; + } + mapPowercafeAll[strKey].articlecountall++; + } + + for (QMap >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++) + { + for (QMap::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++) + { + if (iterPos1.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe") + continue; + + QString strKey = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeAll.contains(strKey)) + { + mapPowercafeAll.insert(strKey, stStatsPowCafeRank()); + mapPowercafeAll[strKey].community_id = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + mapPowercafeAll[strKey].community_title = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed(); + mapPowercafeAll[strKey].subject = "articlecountall"; + } + mapPowercafeAll[strKey].articlecountall++; + } + } + + { + QList listPowercafe = mapPowercafeAll.values(); + + qSort(listPowercafe.begin(), listPowercafe.end(), isGreaterCountAll()); + int rank = 1; + foreach (stStatsPowCafeRank ststats, listPowercafe) + { + ststats.rank = rank++; + m_listMap.push_back(ststats); + if (rank > 10) + break; + } + } + + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapPowercafeReal; + const stReal& streal = iterPos.value(); + + foreach (const QString& url, streal.m_setRealBodyUrl) + { + QString strKey = (*mapBody)[url][anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeReal.contains(strKey)) + { + mapPowercafeReal.insert(strKey, stStatsPowCafeRank()); + mapPowercafeReal[strKey].community_id = _datagroup.getstCommunity((*mapBody)[url])->id; + mapPowercafeReal[strKey].community_title = _datagroup.getstCommunity((*mapBody)[url])->title; + mapPowercafeReal[strKey].subject = "articlecountreal"; + } + mapPowercafeReal[strKey].articlecountreal++; + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.begin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_id]].trimmed(); + if (!mapPowercafeReal.contains(strKey)) + { + mapPowercafeReal.insert(strKey, stStatsPowCafeRank()); + mapPowercafeReal[strKey].community_id = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->id; + mapPowercafeReal[strKey].community_title = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->title; + mapPowercafeReal[strKey].subject = "aritclecountreal"; + } + mapPowercafeReal[strKey].articlecountreal++; + } + + QList listPowercafe = mapPowercafeReal.values(); + qSort(listPowercafe.begin(), listPowercafe.end(), isGreaterCountReal()); + int rank = 1; + foreach (stStatsPowCafeRank ststats, listPowercafe) + { + ststats.rank = rank++; + m_listMap.push_back(ststats); + if (rank > 10) + break; + } + } +} + +/* +bool SPowercafe::SStatsPowCafeRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from stats_pow_cafe_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into stats_pow_cafe_rank (community_id,community_title,articlecountall,articlecountreal,usercountreal,subject,rank,company_num)" + "VALUES (:community_id,:community_title,:articlecountall,:articlecountreal,:usercountreal,:subject,:rank,:company_num)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + foreach (const stStatsPowCafeRank& ststats, m_listMap) + { + query.bindValue(":community_id", ststats.community_id); + query.bindValue(":community_title", sqlString(ststats.community_title)); + query.bindValue(":articlecountall", ststats.articlecountall); + query.bindValue(":articlecountreal", ststats.articlecountreal); + query.bindValue(":usercountreal", ststats.usercountreal); + query.bindValue(":subject", ststats.subject); + query.bindValue(":rank", ststats.rank); + query.bindValue(":company_num", _nCompany); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + + +bool SPowercafe::SStatsPowCafeRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "delete from stats_pow_cafe_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into stats_pow_cafe_rank (community_id,community_title,articlecountall,articlecountreal," + "usercountreal,subject,rank,company_num)" + "VALUES ("; + query.clear(); + + foreach (const stStatsPowCafeRank& ststats, m_listMap) + { + QString str = strQuery; + str += "'" + sqlString(ststats.community_id) + "',"; + str += "'" + sqlString(ststats.community_title) + "',"; + str += QString::number(ststats.articlecountall) + ","; + str += QString::number(ststats.articlecountreal) + ","; + str += QString::number(ststats.usercountreal) + ","; + str += "'" + ststats.subject + "',"; + str += QString::number(ststats.rank) + ","; + str += QString::number(_nCompany) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + + +void SPowercafe::SBodyPowercafeMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& strUrl, *orderedUrl) + { + if (streal.m_setRealBodyUrl.contains(strUrl)) + { + stBodyPowercafeMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.community_num = _datagroup.getstBody(strUrl)->community_num; + stbody.category_num = iterPos.key(); + stbody.realin |= REAL_BODY; + m_listMap.append(stbody); + //_datagroup.addRealBodyCategory(strUrl, iterPos.key()); + if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stbody.realin |= REAL_REPLY; + /* + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + */ + } + } + } + } +} + + +void SPowercafe::SReplyPowercafeMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + int num = 0; + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.end(); iterPos1++) + { + stReplyPowercafeMap streply; + streply.num = num++; + streply.body_num = _datagroup.getstBodyNum(iterPos1.key()); + streply.category_num = iterPos.key(); + streply.realin |= REAL_REPLY; + streply.community_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->community_num; + streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num; + m_listMap.append(streply); + + /* + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + */ + } + } +} +/* +bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_powercafe_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_powercafe_map_" + QString::number(_nCompany) + "_update (num,category_num,community_num,body_num,realin)" + " values (:num,:category_num,:community_num,:body_num,:realin)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stBodyPowercafeMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":community_num", stmap.community_num); + query.bindValue(":body_num", stmap.body_num); + query.bindValue(":realin", stmap.realin); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + +bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_powercafe_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_powercafe_map_" + QString::number(_nCompany) + + "_update (num,category_num,community_num,body_num,realin)" + " values ("; + query.clear(); + + foreach (const stBodyPowercafeMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.community_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.realin) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + +/* +bool SPowercafe::SReplyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + + QString strQuery = "truncate reply_powercafe_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_powercafe_map_" + QString::number(_nCompany) + "_update (num,category_num,community_num,body_num,reply_num,realin)" + " values (:num,:category_num,:community_num,:body_num,:reply_num,:realin)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stReplyPowercafeMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":community_num", stmap.community_num); + query.bindValue(":body_num", stmap.body_num); + query.bindValue(":realin", stmap.realin); + query.bindValue(":reply_num", stmap.reply_num); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + + +bool SPowercafe::SReplyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + + QString strQuery = "truncate reply_powercafe_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_powercafe_map_" + QString::number(_nCompany) + + "_update (num,category_num,community_num,body_num,reply_num,realin)" + " values ("; + query.clear(); + + foreach (const stReplyPowercafeMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.community_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.reply_num) + ","; + str += QString::number(stmap.realin) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +bool SPowercafe::SBodyPowercafeMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "body_powercafe_map_" + QString::number(_nCompany) + " TO " + "body_powercafe_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "body_powercafe_map_" + QString::number(_nCompany) + "_update" + " TO " + "body_powercafe_map_" + QString::number(_nCompany) + ","; + strQuery += "body_powercafe_map_" + QString::number(_nCompany) + "_temp" + " TO " + "body_powercafe_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + +bool SPowercafe::SReplyPowercafeMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "reply_powercafe_map_" + QString::number(_nCompany) + " TO " + "reply_powercafe_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "reply_powercafe_map_" + QString::number(_nCompany) + "_update" + " TO " + "reply_powercafe_map_" + QString::number(_nCompany) + ","; + strQuery += "reply_powercafe_map_" + QString::number(_nCompany) + "_temp" + " TO " + "reply_powercafe_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} diff --git a/sfilterprocess/spowercafe.h b/sfilterprocess/spowercafe.h new file mode 100644 index 0000000..9d1735e --- /dev/null +++ b/sfilterprocess/spowercafe.h @@ -0,0 +1,164 @@ +#ifndef SPOWERCAFE +#define SPOWERCAFE +#include "sservice.h" +#include "sfilteralgorithm.h" +#include "sinitializer.h" +class QString; +class QStringList; +class SPowercafe:public SService +{ +public: + struct stReal + { + QSet m_setRealAllUrl; + QSet m_setRealBodyUrl; + QMap m_mapRealReplyUniqueUrl; //multiple + }; + + struct stPowercafe + { + int company_num; + int community_num; + QString community_id; + int bodycountall; + int replycountall; + int allcountall; + int usercountall; + int bodycountreal; + int replycountreal; + int allcountreal; + int usercountreal; + int rank; + stPowercafe():company_num(0), community_num(0), bodycountall(0), replycountall(0), allcountall(0), usercountall(0), bodycountreal(0), + replycountreal(0), allcountreal(0), usercountreal(0), rank(0){} + }; + + struct stStatsPowCafeRank + { + QString community_id; + QString community_title; + int articlecountall; + int articlecountreal; + int usercountreal; + QString subject; + int rank; + int company_num; + stStatsPowCafeRank(): articlecountall(0), articlecountreal(0), usercountreal(0), rank(0), company_num(0){} + }; + + struct stBodyPowercafeMap + { + int num; + int body_num; + int community_num; + int category_num; + int realin; + stBodyPowercafeMap():num(0), body_num(0), community_num(0), category_num(0), realin(0){} + }; + + struct stReplyPowercafeMap + { + int num; + int reply_num; + int body_num; + int community_num; + int category_num; + int realin; + stReplyPowercafeMap():num(0), reply_num(0), body_num(0), community_num(0), category_num(0), realin(0) {} + }; + + class STPowercafe + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + }; + + class SStatsPowCafeRank + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + }; + + class SBodyPowercafeMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + class SReplyPowercafeMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + struct isGreaterCountReal + { + bool operator()(const stPowercafe& first, const stPowercafe& second) + { + return first.allcountreal > second.allcountreal; + } + bool operator()(const stStatsPowCafeRank& first, const stStatsPowCafeRank& second) + { + return first.articlecountreal > second.articlecountreal; + } + }; + + struct isGreaterCountAll + { + bool operator()(const stStatsPowCafeRank& first, const stStatsPowCafeRank& second) + { + return first.articlecountall > second.articlecountall; + } + }; + + + +private: + SFilterAlgorithm m_SFilterAlgorithm; + QMap m_mapCategory; + SBodyPowercafeMap m_SBodyPowercafeMap; + SReplyPowercafeMap m_SReplyPowercafeMap; + STPowercafe m_STPowercafe; + SStatsPowCafeRank m_SStatsPowCafeRank; + +public: + virtual void setInitiaizer(SInitializer &_initializer); + virtual bool uploadData(int _companyNum); + virtual bool changeTable(int _companyNum); + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter); + virtual bool makeOverallCategory(int _categoryNum); + virtual bool makeTable(SDatagroup &_datagroup); +}; + +#endif // SPOWERCAFE + diff --git a/sfilterprocess/sservice.h b/sfilterprocess/sservice.h new file mode 100644 index 0000000..a88c969 --- /dev/null +++ b/sfilterprocess/sservice.h @@ -0,0 +1,20 @@ +#ifndef SSERVICE +#define SSERVICE +#include "sinitializer.h" +#include "sdatagroup.h" + +class SService +{ +protected: + SInitializer* m_pSInitializer; +public: + virtual void setInitiaizer(SInitializer &_initializer) = 0; + virtual bool uploadData(int _companyNum) = 0; + virtual bool changeTable(int _companyNum) = 0; + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter) = 0; + virtual bool makeOverallCategory(int _categoryNum) = 0; + virtual bool makeTable(SDatagroup &_datagroup) = 0; +}; + +#endif // SSERVICE + diff --git a/sfilterprocess/sspammer.cpp b/sfilterprocess/sspammer.cpp new file mode 100644 index 0000000..c055eff --- /dev/null +++ b/sfilterprocess/sspammer.cpp @@ -0,0 +1,757 @@ +#include +#include +#include +#include +#include +#include +#include +#include "sspammer.h" + + +void SSpammer::setInitiaizer(SInitializer &_initializer) +{ + m_pSInitializer = &_initializer; + m_SBodySpammerMap.setInitiaizer(_initializer); + m_SReplySpammerMap.setInitiaizer(_initializer); + m_SStatsSpaSpammerRank.setInitiaizer(_initializer); + m_STSpammer.setInitiaizer(_initializer); +} + +bool SSpammer::uploadData(int _companyNum) +{ + m_pSInitializer->insertLog("Start upload Spammer"); + + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Spammer (Start)"); + if (!m_SBodySpammerMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload Spammer body map"); + return false; + } + if (!m_SReplySpammerMap.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload Spammer reply map"); + return false; + } + if (!m_SStatsSpaSpammerRank.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload stats_spa_spammer_rank"); + return false; + } + if (!m_STSpammer.uploadData(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload spammer"); + return false; + } + m_pSInitializer->insertLog("End upload Spammer"); + m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Spammer (Finish)"); + + return true; +} + +bool SSpammer::changeTable(int _companyNum) +{ + m_pSInitializer->insertLog("Start table change spammer"); + QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB); + if (!m_SBodySpammerMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload spammer body map"); + return false; + } + if (!m_SReplySpammerMap.changeTable(dbWeb, _companyNum)) + { + m_pSInitializer->insertLog("Fail to upload spammer reply map"); + return false; + } + return true; + +} + +bool SSpammer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter) +{ + QVector vecData[SInitializer::E_ARTICLE_MAX]; + bool bFiltered[SInitializer::E_ARTICLE_MAX]; + + for (int i = 0; i < SInitializer::E_ARTICLE_MAX; i++) + bFiltered[i] = false; + _datagroup.copyData(vecData); + foreach (const SInitializer::stFilter& filter, _listFilter) + { + switch(filter.m_nType) + { + case E_FILTER_TYPE_DATE://Date + m_SFilterAlgorithm.FilterDate(vecData, bFiltered, filter.m_vData.toString(), + m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_article_date]); + + break; + case E_FILTER_TYPE_SEARCH: + m_SFilterAlgorithm.FilterSearch(vecData, bFiltered, filter.m_vData.toString()); + + break; + case E_FILTER_TYPE_LENGTH: + m_SFilterAlgorithm.FilterLength(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_REPLACE: + m_SFilterAlgorithm.FilterReplace(vecData, bFiltered, filter.m_vData.toString()); + break; + case E_FILTER_TYPE_COUNTER: + m_SFilterAlgorithm.FilterCounter(vecData, bFiltered, filter.m_vData.toString()); + break; + } + } + stReal streal; + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size(); + qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size(); + const int *anColumn = m_pSInitializer->getColumnIntArray(); + if (bFiltered[SInitializer::E_ARTICLE_BODY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_BODY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_setRealBodyUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + if (bFiltered[SInitializer::E_ARTICLE_REPLY]) + { + foreach (const QStringList& strlist, vecData[SInitializer::E_ARTICLE_REPLY]) + { + streal.m_setRealAllUrl.insert(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + streal.m_mapRealReplyUniqueUrl.insertMulti(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed(), + strlist[anColumn[SInitializer::E_DATA_article_order]].trimmed().toInt()); + _datagroup.addRealUrl(strlist[anColumn[SInitializer::E_DATA_article_url]].trimmed()); + } + } + + m_mapCategory.insert(_nCategory, streal); + return true; +} + +bool SSpammer::makeOverallCategory(int _categoryNum) +{ + return true; +} + +bool SSpammer::makeTable(SDatagroup &_datagroup) +{ + m_pSInitializer->insertLog("Make spammer tables"); + m_pSInitializer->insertLog("Make spammer spammer"); + m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer); + m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables"); + m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory); + m_pSInitializer->insertLog("Make spammer body tables"); + m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer); + m_pSInitializer->insertLog("Make spammer reply tables"); + m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer); + return true; +} + +void SSpammer::STSpammer::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory, QSet &setSpammer) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapScore; + const stReal& streal = iterPos.value(); + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","; + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(_url)->id_num; + } + mapScore[strKey].body++; + mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","; + strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num; + } + mapScore[strKey].reply++; + mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + } + + + QList listSpammer; + + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + stSpammer stspammer; + stspammer.bodycount = iterPos1->body; + stspammer.replycount = iterPos1->reply; + stspammer.nicknamecount = iterPos1->nickname.size(); + stspammer.id_num = iterPos1->id_num; + stspammer.id_id = iterPos1.key().split(",").at(0); + if (stspammer.bodycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] || + stspammer.replycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] || + iterPos1->nickname.size() > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT]) + { + + stspammer.spammervalue = stspammer.bodycount * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO] + + stspammer.replycount * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO] + + iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO]; + listSpammer.push_back(stspammer); + } + } + qSort(listSpammer.begin(), listSpammer.end(), isGreaterSpammerValue()); + + int i = 1; + + for (QList::iterator iterPos1 = listSpammer.begin(); iterPos1 != listSpammer.end(); iterPos1++) + { + iterPos1->rank = i++; + setSpammer.insert(iterPos1->id_num); + m_listMap.push_back(*iterPos1); + + } + } +} + +/* +bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from spammer where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into spammer (rank,id_num,id_id,nicknamecount,bodycount,replycount,spammervalue,company_num) VALUES (" + ":rank,:id_num,:id_id,:nicknamecount,:bodycount,:replycount,:spammervalue,:company_num)"; + + + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stSpammer& stspammer, m_listMap) + { + query.bindValue(":rank", stspammer.rank); + query.bindValue(":id_num", stspammer.id_num); + query.bindValue(":id_id", stspammer.id_id); + query.bindValue(":nicknamecount", stspammer.nicknamecount); + query.bindValue(":bodycount", stspammer.bodycount); + query.bindValue(":replycount", stspammer.replycount); + query.bindValue(":spammervalue", QString::number(stspammer.spammervalue, 'g', 3)); + query.bindValue(":company_num", _nCompany); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + + + strQuery = "update dbinfo set spammercount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + + + return true; +} +*/ + + +bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from spammer where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into spammer (rank,id_num,id_id,nicknamecount,bodycount,replycount,spammervalue,company_num) VALUES ("; + + query.clear(); + + foreach (const stSpammer& stspammer, m_listMap) + { + QString str = strQuery; + str += QString::number(stspammer.rank) + ","; + str += QString::number(stspammer.id_num) + ","; + str += "'" + sqlString(stspammer.id_id) + "',"; + str += QString::number(stspammer.nicknamecount) + ","; + str += QString::number(stspammer.bodycount) + ","; + str += QString::number(stspammer.replycount) + ","; + str += QString::number(stspammer.spammervalue, 'g', 3) + ","; + str += QString::number(_nCompany) + ")"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + + strQuery = "update dbinfo set spammercount = " + QString::number(m_listMap.size()) + " where company_num = " + + QString::number(_nCompany); + query.exec(strQuery); + + return true; +} + + +void SSpammer::SStatsSpaSpammerRank::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory) +{ + const QMap* mapBody = _datagroup.getmapBody(); + const QMap >* mapReply = _datagroup.getmapReply(); + const int* anColumn = m_pSInitializer->getColumnIntArray(); + + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + QMap mapScore; + const stReal& streal = iterPos.value(); + foreach (const QString& _url, streal.m_setRealBodyUrl) + { + QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(_url)->id_num; + } + mapScore[strKey].body++; + mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + } + + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ","; + strKey += ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ","); + strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed(); + if (!mapScore.contains(strKey)) + { + mapScore.insert(strKey, stSpammerScore()); + mapScore[strKey].id_num = _datagroup.getstBody(iterPos1.key())->id_num; + } + mapScore[strKey].reply++; + mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed()); + } + + + QList listSpammer; + + for (QMap::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++) + { + stStatsSpaSpammerRank ststats; + ststats.bodycount = iterPos1->body; + ststats.replycount = iterPos1->reply; + ststats.id_num = iterPos1->id_num; + ststats.nicknamecount = iterPos1->nickname.size(); + ststats.id_id = iterPos1.key().split(",").at(0); + ststats.platformname_name = iterPos1.key().split(",").at(1); + ststats.subject = "spammervalue"; + if (ststats.bodycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] || + ststats.replycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] || + iterPos1->nickname.size() > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT]) + { + + ststats.spammervalue = ststats.bodycount * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO] + + ststats.replycount * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO] + + iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO]; + listSpammer.push_back(ststats); + } + } + + qSort(listSpammer.begin(), listSpammer.end(), isGreaterSpammerValue()); + QMap mapRank; + + for (QList::iterator iterPos1 = listSpammer.begin(); iterPos1 != listSpammer.end(); iterPos1++) + { + if (!mapRank.contains(iterPos1->platformname_name)) + mapRank.insert(iterPos1->platformname_name, 1); + + if (mapRank[iterPos1->platformname_name] <= 10) + { + iterPos1->rank = mapRank[iterPos1->platformname_name]++; + m_listMap.push_back(*iterPos1); + } + } + } +} + +/* +bool SSpammer::SStatsSpaSpammerRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from stats_spa_spammer_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into stats_spa_spammer_rank (rank,id_num,id_id,nicknamecount,bodycount,replycount,spammervalue,company_num,platformname_name,subject) VALUES (" + ":rank,:id_num,:id_id,:nicknamecount,:bodycount,:replycount,:spammervalue,:company_num,:platformname_name,:subject)"; + + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stStatsSpaSpammerRank& ststats, m_listMap) + { + query.bindValue(":rank", ststats.rank); + query.bindValue(":id_num", ststats.id_num); + query.bindValue(":id_id", ststats.id_id); + query.bindValue(":nicknamecount", ststats.nicknamecount); + query.bindValue(":bodycount", ststats.bodycount); + query.bindValue(":replycount", ststats.replycount); + query.bindValue(":spammervalue", QString::number(ststats.spammervalue, 'g', 3)); + query.bindValue(":company_num", _nCompany); + query.bindValue(":platformname_name", ststats.platformname_name); + query.bindValue(":subject", ststats.subject); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} +*/ + +bool SSpammer::SStatsSpaSpammerRank::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + //_db.transaction(); + QString strQuery = "delete from stats_spa_spammer_rank where company_num = " + QString::number(_nCompany); + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + //_db.commit(); + //_db.transaction(); + strQuery = "insert into stats_spa_spammer_rank (rank,id_num,id_id,nicknamecount,bodycount,replycount," + "spammervalue,company_num,platformname_name,subject) VALUES ("; + query.clear(); + + foreach (const stStatsSpaSpammerRank& ststats, m_listMap) + { + QString str = strQuery; + str += QString::number(ststats.rank) + ","; + str += QString::number(ststats.id_num) + ","; + str += "'" + sqlString(ststats.id_id) + "',"; + str += QString::number(ststats.nicknamecount) + ","; + str += QString::number(ststats.bodycount) + ","; + str += QString::number(ststats.replycount) + ","; + str += QString::number(ststats.spammervalue, 'g', 3) + ","; + str += QString::number(_nCompany) + ","; + str += "'" + ststats.platformname_name + "',"; + str += "'" + ststats.subject + "')"; + + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + //_db.commit(); + return true; +} + + +void SSpammer::SBodySpammerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory, QSet& setSpammer) +{ + int num = 1; + const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + foreach (const QString& strUrl, *orderedUrl) + { + if (streal.m_setRealBodyUrl.contains(strUrl)) + { + if (!setSpammer.contains(_datagroup.getstBody(strUrl)->id_num)) + continue; + + stBodySpammerMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.category_num = iterPos.key(); + stbody.realin |= REAL_BODY; + if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stbody.realin |= REAL_REPLY; + /* + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + */ + } + m_listMap.append(stbody); + //_datagroup.addRealBodyCategory(strUrl, iterPos.key()); + } + /* + else if (streal.m_mapRealReplyUniqueUrl.contains(strUrl)) + { + stBodySpammerMap stbody; + stbody.num = num++; + stbody.body_num = _datagroup.getstBodyNum(strUrl); + stbody.category_num = iterPos.key(); + stbody.id_num = _datagroup.getstBody(strUrl)->id_num; + stbody.realin |= REAL_REPLY; + foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl)) + { + _datagroup.addRealReplyCategory(strUrl, order, iterPos.key()); + } + m_listMap.append(stbody); + } + */ + } + } +} + +/* +bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_spammer_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)" + " values (:num,:category_num,:id_num,:body_num,:realin)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stBodySpammerMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":id_num", stmap.id_num); + query.bindValue(":body_num", stmap.body_num); + query.bindValue(":realin", stmap.realin); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + +bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate body_spammer_map_" + QString::number(_nCompany) + "_update"; + + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)" + " values ("; + query.clear(); + + foreach (const stBodySpammerMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.id_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.realin) + ")"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + + +void SSpammer::SReplySpammerMap::makeTable(SDatagroup &_datagroup, const QMap &_mapCategory, QSet& setSpammer) +{ + int num = 1; + //const QStringList* orderedUrl = _datagroup.getOrderedUrl(); + for (QMap::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++) + { + const stReal& streal = iterPos.value(); + //foreach (const QString& strUrl, *orderedUrl) + { + for (QMap::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++) + { + if (!setSpammer.contains(_datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num)) + continue; + + stReplySpammerMap streply; + streply.num = num++; + streply.category_num = iterPos.key(); + streply.body_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->body_num; + streply.id_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num; + streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num; + m_listMap.append(streply); + } + + /* + if (streal.m_setRealAllUrl.contains(strUrl)) + { + if (_datagroup.getstBody(strUrl)->reply_count == 0) + continue; + + const QMap* mapReply = _datagroup.getstReply(strUrl); + + for(QMap::const_iterator iterPos1 = mapReply->constBegin(); iterPos1 != mapReply->constEnd(); iterPos1++) + { + stReplySpammerMap streply; + streply.num = num++; + streply.category_num = iterPos.key(); + streply.body_num = iterPos1.value().body_num; + streply.id_num = iterPos1.value().id_num; + streply.reply_num = iterPos1.value().num; + m_listMap.append(streply); + } + } + */ + } + } +} +/* +bool SSpammer::SReplySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate reply_spammer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,reply_num)" + " values (:num,:category_num,:id_num,:body_num,:reply_num)"; + query.clear(); + query.prepare(strQuery.toUtf8()); + + foreach (const stReplySpammerMap& stmap, m_listMap) + { + query.bindValue(":num", stmap.num); + query.bindValue(":category_num", stmap.category_num); + query.bindValue(":id_num", stmap.id_num); + query.bindValue(":body_num", stmap.body_num); + //query.bindValue(":realin", stmap.realin); + query.bindValue(":reply_num", stmap.reply_num); + if (query.exec() == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} +*/ + +bool SSpammer::SReplySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery = "truncate reply_spammer_map_" + QString::number(_nCompany) + "_update"; + QSqlQuery query(_db); + if (query.exec(strQuery.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + strQuery = "insert into reply_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,reply_num)" + " values ("; + query.clear(); + + foreach (const stReplySpammerMap& stmap, m_listMap) + { + QString str = strQuery; + str += QString::number(stmap.num) + ","; + str += QString::number(stmap.category_num) + ","; + str += QString::number(stmap.id_num) + ","; + str += QString::number(stmap.body_num) + ","; + str += QString::number(stmap.reply_num) + ")"; + if (query.exec(str.toUtf8()) == false) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + } + return true; +} + +bool SSpammer::SBodySpammerMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "body_spammer_map_" + QString::number(_nCompany) + " TO " + "body_spammer_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "body_spammer_map_" + QString::number(_nCompany) + "_update" + " TO " + "body_spammer_map_" + QString::number(_nCompany) + ","; + strQuery += "body_spammer_map_" + QString::number(_nCompany) + "_temp" + " TO " + "body_spammer_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} + + +bool SSpammer::SReplySpammerMap::changeTable(QSqlDatabase &_db, const int &_nCompany) +{ + QString strQuery; + QSqlQuery query(_db); + strQuery = "RENAME TABLE "; + strQuery += "reply_spammer_map_" + QString::number(_nCompany) + " TO " + "reply_spammer_map_" + QString::number(_nCompany) + "_temp" + ","; + strQuery += "reply_spammer_map_" + QString::number(_nCompany) + "_update" + " TO " + "reply_spammer_map_" + QString::number(_nCompany) + ","; + strQuery += "reply_spammer_map_" + QString::number(_nCompany) + "_temp" + " TO " + "reply_spammer_map_" + QString::number(_nCompany) + "_update"; + if (!query.exec(strQuery.toUtf8())) + { + m_pSInitializer->insertLog(query.lastQuery()); + m_pSInitializer->insertLog(query.lastError().text()); + return false; + } + return true; +} diff --git a/sfilterprocess/sspammer.h b/sfilterprocess/sspammer.h new file mode 100644 index 0000000..541155e --- /dev/null +++ b/sfilterprocess/sspammer.h @@ -0,0 +1,163 @@ +#ifndef SSPAMMER +#define SSPAMMER +#include "sservice.h" +#include "sfilteralgorithm.h" +#include "sinitializer.h" +class QString; +class QStringList; +class SSpammer:public SService +{ +public: + struct stReal + { + QSet m_setRealAllUrl; + QSet m_setRealBodyUrl; + QMap m_mapRealReplyUniqueUrl; //multiple + }; + + struct stSpammer + { + int id_num; + int company_num; + QString id_id; + int nicknamecount; + int bodycount; + int replycount; + double spammervalue; + int rank; + stSpammer():id_num(0), company_num(0), nicknamecount(0), bodycount(0), replycount(0), rank(0), spammervalue(0.0){} + + }; + + struct stStatsSpaSpammerRank + { + int id_num; + QString id_id; + QString platformname_name; + int nicknamecount; + int bodycount; + int replycount; + double spammervalue; + int rank; + QString subject; + int company_num; + stStatsSpaSpammerRank():id_num(0),nicknamecount(0),bodycount(0),replycount(0),rank(0), spammervalue(0.0){} + + }; + + struct stBodySpammerMap + { + int num; + int body_num; + int id_num; + int category_num; + int realin; + stBodySpammerMap():num(0), body_num(0), id_num(0), category_num(0), realin(0){} + + }; + + struct stReplySpammerMap + { + int num; + int reply_num; + int body_num; + int id_num; + int category_num; + stReplySpammerMap():num(0), body_num(0), reply_num(0), id_num(0), category_num(0){} + }; + + struct stSpammerScore + { + int body; + int reply; + int id_num; + QSet nickname; + stSpammerScore():body(0),reply(0){} + }; + + class STSpammer + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory, QSet& setSpammer); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + + }; + + class SStatsSpaSpammerRank + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + }; + + class SBodySpammerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory ,QSet& setSpammer); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + class SReplySpammerMap + { + private: + SInitializer *m_pSInitializer; + QList m_listMap; + public: + void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; } + void clear(){ m_listMap.clear(); } + void makeTable(SDatagroup &_datagroup, const QMap& _mapCategory, QSet& setSpammer); + bool uploadData(QSqlDatabase &_db, const int &_nCompany); + bool changeTable(QSqlDatabase &_db, const int &_nCompany); + }; + + struct isGreaterSpammerValue + { + bool operator()(const stSpammer& first, const stSpammer& second) const + { + return first.spammervalue > second.spammervalue; + } + + bool operator()(const stStatsSpaSpammerRank& first, const stStatsSpaSpammerRank& second) const + { + return first.spammervalue > second.spammervalue; + } + }; + +private: + SFilterAlgorithm m_SFilterAlgorithm; + QMap m_mapCategory; + STSpammer m_STSpammer; + SStatsSpaSpammerRank m_SStatsSpaSpammerRank; + SBodySpammerMap m_SBodySpammerMap; + SReplySpammerMap m_SReplySpammerMap; + QSet m_setSpammer; + +public: + virtual void setInitiaizer(SInitializer &_initializer); + virtual bool uploadData(int _companyNum); + virtual bool changeTable(int _companyNum); + virtual bool makeCategory(SDatagroup &_datagroup, int _nCategory, const QList &_listFilter); + virtual bool makeOverallCategory(int _categoryNum); + virtual bool makeTable(SDatagroup &_datagroup); +}; + + + +#endif // SSPAMMER +