graph 업데이트

git-svn-id: svn://192.168.0.12/source@279 8346c931-da38-4b9b-9d4c-e48b93cbd075
2016-07-06 10:19:53 +00:00
parent 3074db4fa0
commit 3a7eb4d829
13 changed files with 1094 additions and 34 deletions
--- a/sfilterprocess/sspammer.cpp
+++ b/sfilterprocess/sspammer.cpp
@@ -5,6 +5,12 @@
 #include <QVariant>
 #include <QString>
 #include <QStringList>
+#include <QDate>
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QJsonValue>
+#include <QSet>
 #include "sspammer.h"


@@ -13,8 +19,9 @@ void SSpammer::setInitiaizer(SInitializer &_initializer)
    m_pSInitializer = &_initializer;
    m_SBodySpammerMap.setInitiaizer(_initializer);
    m_SReplySpammerMap.setInitiaizer(_initializer);
-    m_SStatsSpaSpammerRank.setInitiaizer(_initializer);
+    //m_SStatsSpaSpammerRank.setInitiaizer(_initializer);
    m_STSpammer.setInitiaizer(_initializer);
+    m_SStatsJson.setInitiaizer(_initializer);
 }

 bool SSpammer::uploadData(int _companyNum)
@@ -33,16 +40,23 @@ bool SSpammer::uploadData(int _companyNum)
        m_pSInitializer->insertLog("Fail to upload Spammer reply map");
        return false;
    }
+    /*
    if (!m_SStatsSpaSpammerRank.uploadData(dbWeb, _companyNum))
    {
        m_pSInitializer->insertLog("Fail to upload stats_spa_spammer_rank");
        return false;
    }
+    */
    if (!m_STSpammer.uploadData(dbWeb, _companyNum))
    {
        m_pSInitializer->insertLog("Fail to upload spammer");
        return false;
    }
+    if (!m_SStatsJson.uploadData(dbWeb, _companyNum))
+    {
+        m_pSInitializer->insertLog("Fail to upload statsjson");
+        return false;
+    }
    m_pSInitializer->insertLog("End upload Spammer");
    m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Spammer (Finish)");

@@ -168,13 +182,18 @@ bool SSpammer::makeTable(SDatagroup &_datagroup)
        m_pSInitializer->insertLog("Make spammer tables");
        m_pSInitializer->insertLog("Make spammer spammer");
        m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer);
+        /*
        m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables");
        m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory);
+        */
        m_pSInitializer->insertLog("Make spammer body tables");
        m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
        m_pSInitializer->insertLog("Make spammer reply tables");
        m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
+        m_pSInitializer->insertLog("Make sstatsjson table");
+        m_SStatsJson.makeTable(_datagroup, m_mapCategory);
    }
+
    return true;
 }

@@ -824,3 +843,390 @@ bool SSpammer::SReplySpammerMap::changeTable(QSqlDatabase &_db, const int &_nCom
    }
    return true;
 }
+
+void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
+{
+    const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
+    const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
+    const int* anColumn = m_pSInitializer->getColumnIntArray();
+
+    QDate dateStart(2100,1,1),dateEnd(2000,1,1);
+    QDate dateStartBody(2100,1,1),dateEndBody(2000,1,1);
+    QDate dateStartReply(2100,1,1),dateEndReply(2000,1,1);
+
+    QList<stBuzzy> listBuzzy;
+    // filtered
+    {
+        for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
+        {
+            stBuzzy stbuzzy;
+            stbuzzy.m_nCategory = iterPos.key();
+            const stReal &streal = iterPos.value();
+            foreach (const QString& strUrl, streal.m_setRealBodyUrl)
+            {
+                QDate date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
+                //if (date.isNull())
+                //    date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed(), "yyyy-MM-dd HH:mm:ss").date();
+                if (date.year() == 1990)
+                    continue;
+                dateStart = (date < dateStart ) ? date : dateStart;
+                dateEnd = (date > dateEnd ) ? date : dateEnd;
+                dateStartBody = (date < dateStartBody ) ? date : dateStartBody;
+                dateEndBody = (date > dateEndBody ) ? date : dateEndBody;
+
+
+                if (stbuzzy.m_mapBodyBuzz.contains(date))
+                    stbuzzy.m_mapBodyBuzz[date]++;
+                else
+                    stbuzzy.m_mapBodyBuzz.insert(date, 1);
+            }
+
+            for (QMap<QString, int>::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++)
+            {
+                const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()];
+                QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
+                if (date.year() == 1990)
+                    continue;
+                dateStart = (date < dateStart ) ? date : dateStart;
+                dateEnd = (date > dateEnd ) ? date : dateEnd;
+                dateStartReply = (date < dateStartReply ) ? date : dateStartReply;
+                dateEndReply = (date > dateEndReply ) ? date : dateEndReply;
+
+                if (stbuzzy.m_mapReplyBuzz.contains(date))
+                    stbuzzy.m_mapReplyBuzz[date]++;
+                else
+                    stbuzzy.m_mapReplyBuzz.insert(date, 1);
+            }
+
+            listBuzzy.append(stbuzzy);
+        }
+    }
+    //all buzzy
+    {
+        stBuzzy stbuzzy;
+        stbuzzy.m_nCategory = 0;
+        for (QMap<QString, QStringList>::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++)
+        {
+            QDate date = QDateTime::fromString(iterPos.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
+            if (date.year() == 1990)
+                continue;
+
+            if (stbuzzy.m_mapBodyBuzz.contains(date))
+                stbuzzy.m_mapBodyBuzz[date]++;
+            else
+                stbuzzy.m_mapBodyBuzz.insert(date, 1);
+        }
+
+        for (QMap<QString, QMap<int, QStringList> >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++)
+        {
+            for (QMap<int, QStringList>::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++)
+            {
+                QDate date = QDateTime::fromString(iterPos1.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
+                if (date.year() == 1990)
+                    continue;
+
+                if (stbuzzy.m_mapReplyBuzz.contains(date))
+                    stbuzzy.m_mapReplyBuzz[date]++;
+                else
+                    stbuzzy.m_mapReplyBuzz.insert(date, 1);
+            }
+        }
+        listBuzzy.append(stbuzzy);
+    }
+
+    int nTotalCount = 0, nTotalBodyCount = 0, nTotalReplyCount = 0;
+
+    if (_mapCategory.contains(m_pSInitializer->getAllCategoryNum(3)))
+    {
+        foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(3)].m_setRealAllUrl)
+        {
+            nTotalBodyCount++;
+            nTotalReplyCount += (*mapReply)[strUrl].size();
+        }
+        nTotalCount = nTotalBodyCount + nTotalReplyCount;
+    }
+
+
+    {
+        QJsonArray jsArrayAll;
+        QJsonArray jsArrayReply;
+        QJsonArray jsArrayBody;
+
+        foreach (const stBuzzy &stbuzzy, listBuzzy)
+        {
+            QJsonArray jsValueAll;
+            QJsonArray jsValueBody;
+            QJsonArray jsValueReply;
+
+            for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1))
+            {
+                QJsonObject jsoBody;
+                QJsonObject jsoAll;
+                QJsonObject jsoReply;
+
+                jsoAll["date"] = jsoReply["date"] = jsoBody["date"] = i.toString("yyyyMMdd");
+                jsoBody["count"] = stbuzzy.m_mapBodyBuzz[i];
+                jsoReply["count"] = stbuzzy.m_mapReplyBuzz[i];
+                jsoAll["count"] = stbuzzy.m_mapBodyBuzz[i] + stbuzzy.m_mapReplyBuzz[i];
+
+                jsValueAll.append(jsoAll);
+                jsValueBody.append(jsoBody);
+                jsValueReply.append(jsoReply);
+            }
+
+            QJsonObject jsNameReply;
+            QJsonObject jsNameAll;
+            QJsonObject jsNameBody;
+
+            if (stbuzzy.m_nCategory != 0)
+                jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = m_pSInitializer->getCategoryName(stbuzzy.m_nCategory);
+            else
+                jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALL";
+
+            jsNameAll["value"] = jsValueAll;
+            jsNameBody["value"] = jsValueBody;
+            jsNameReply["value"] = jsValueReply;
+
+            jsArrayAll.append(jsNameAll);
+            jsArrayBody.append(jsNameBody);
+            jsArrayReply.append(jsNameReply);
+        }
+        stStatsJson stbodybuzz;
+        stStatsJson streplybuzz;
+        stStatsJson stallbuzz;
+
+        stallbuzz.title = streplybuzz.title = stbodybuzz.title = "buzz";
+
+        stbodybuzz.subtitle = "body";
+        streplybuzz.subtitle = "reply";
+        stallbuzz.subtitle = "all";
+
+        QJsonObject jobjAll;
+        QJsonObject jobjBody;
+        QJsonObject jobjReply;
+
+        jobjAll["data"] = jsArrayAll;
+        jobjBody["data"] = jsArrayBody;
+        jobjReply["data"] = jsArrayReply;
+
+        jobjAll["mindate"] = dateStart.toString("yyyy.MM.dd");
+        jobjBody["mindate"] = dateStartBody.toString("yyyy.MM.dd");
+        jobjReply["mindate"] = dateStartReply.toString("yyyy.MM.dd");
+
+        jobjAll["maxdate"] = dateEnd.toString("yyyy.MM.dd");
+        jobjBody["maxdate"] = dateEndBody.toString("yyyy.MM.dd");
+        jobjReply["maxdate"] = dateEndReply.toString("yyyy.MM.dd");
+
+        jobjAll["totalcount"] = nTotalCount;
+        jobjBody["totalcount"] = nTotalBodyCount;
+        jobjReply["totalcount"] = nTotalReplyCount;
+
+        QJsonDocument jdAll(jobjAll);
+        QJsonDocument jdBody(jobjBody);
+        QJsonDocument jdReply(jobjReply);
+
+        stbodybuzz.jdata = jdBody.toJson(QJsonDocument::Compact);
+        streplybuzz.jdata = jdReply.toJson(QJsonDocument::Compact);
+        stallbuzz.jdata = jdAll.toJson(QJsonDocument::Compact);
+
+        stallbuzz.platformname_num = stbodybuzz.platformname_num = streplybuzz.platformname_num = 1;
+        m_listMap.append(stbodybuzz);
+        m_listMap.append(streplybuzz);
+        m_listMap.append(stallbuzz);
+    }
+
+    /*
+
+    {
+        for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1))
+        {
+            //body
+            stStatsJson stbodybuzz;
+            stStatsJson streplybuzz;
+            stStatsJson stallbuzz;
+            foreach (const stBuzzy& stbuzzy, listBuzzy)
+            {
+                if (stbuzzy.m_mapBodyBuzz.contains(i))
+                {
+                    stbodybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapBodyBuzz[i];
+                    stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapBodyBuzz[i];
+                }
+
+                if (stbuzzy.m_mapReplyBuzz.contains(i))
+                {
+                    streplybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapReplyBuzz[i];
+                    stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapReplyBuzz[i];
+                }
+            }
+            stallbuzz.date = streplybuzz.date = stbodybuzz.date = i;
+            stbodybuzz.subtitle = "body";
+            streplybuzz.subtitle = "reply";
+            stallbuzz.subtitle = "all";
+            m_listMap.append(stallbuzz);
+            m_listMap.append(stbodybuzz);
+            m_listMap.append(streplybuzz);
+        }
+    }
+    */
+}
+
+void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
+{
+    const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
+    const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
+    const int* anColumn = m_pSInitializer->getColumnIntArray();
+
+    for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
+    {
+        QMap<QString, stSpammerScore> mapScore;
+        const stReal& streal = iterPos.value();
+        QDate dateStart(2030, 1, 1);
+        QDate dateEnd(2000, 1, 1);
+
+        foreach (const QString& _url, streal.m_setRealBodyUrl)
+        {
+            QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ",";
+            strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
+            strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
+            if (!mapScore.contains(strKey))
+            {
+                mapScore.insert(strKey, stSpammerScore());
+                mapScore[strKey].id_num = _datagroup.getstBody(_url)->id_num;
+            }
+            mapScore[strKey].body++;
+            mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed());
+            QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
+            if (dateThis.isValid() && dateThis.year() > 1990)
+            {
+                dateStart = (dateStart < dateThis) ? dateStart : dateThis;
+                dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
+            }
+        }
+
+        for (QMap<QString, int>::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++)
+        {
+            QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ",";
+            strKey += ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
+            strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
+            if (!mapScore.contains(strKey))
+            {
+                mapScore.insert(strKey, stSpammerScore());
+                mapScore[strKey].id_num = _datagroup.getstBody(iterPos1.key())->id_num;
+            }
+            mapScore[strKey].reply++;
+            mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed());
+            QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
+            if (dateThis.isValid() && dateThis.year() > 1990)
+            {
+                dateStart = (dateStart < dateThis) ? dateStart : dateThis;
+                dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
+            }
+        }
+
+        //QList<stSpammerRank> listSpammer;
+        QMap<int, stSpammerRank> mapSpammer;
+        for (QMap<QString, stSpammerScore>::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++)
+        {
+            if (iterPos1->body >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] ||
+                    iterPos1->reply >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] ||
+                    iterPos1->nickname.size() >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT])
+            {
+                double value = iterPos1->body * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO]
+                        + iterPos1->reply * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO]
+                        + iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO];
+                stSpammerRank rank;
+                rank.value = value;
+                rank.id = iterPos1.key().split(",").at(0);
+                rank.platformform_num = m_pSInitializer->getPlatformName(iterPos1.key().split(",").at(1));
+                rank.id_num = iterPos1.value().id_num;
+                mapSpammer.insertMulti(rank.platformform_num, rank);
+            }
+        }
+
+        QList<int> keys = mapSpammer.keys().toSet().toList();
+
+        QJsonArray jsonArray;
+        stStatsJson stjsonRank;
+        {
+            stjsonRank.category_num = iterPos.key();
+            stjsonRank.servicenum = 3;
+            stjsonRank.title = "rank";
+            stjsonRank.subtitle = "";
+            stjsonRank.platformname_num = 1;
+        }
+        int nTotalCount = 0;
+        for (QList<int>::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++)
+        {
+            QList<stSpammerRank> values = mapSpammer.values(*iterPos1);
+            qSort(values.begin(), values.end(), isGreaterSpammerValue());
+            int nRankLimit = (values.size() <= 10) ? values.size() : 10;
+            QJsonObject jsonObj;
+            jsonObj["platformname_num"] = *iterPos1;
+            QJsonArray jsonSubArray;
+            for (int i = 0; i < nRankLimit; i++)
+            {
+                QJsonObject jsonSubObj;
+                jsonSubObj["rank"] = i + 1;
+                jsonSubObj["value"] = values.at(i).value;
+                jsonSubObj["id"] = values.at(i).id;
+                jsonSubObj["id_num"] = values.at(i).id_num;
+                jsonSubArray.append(jsonSubObj);
+            }
+            jsonObj["data"] = jsonSubArray;
+            jsonArray.append(jsonObj);
+
+            nTotalCount += values.size();
+        }
+        QJsonObject jObj;
+        jObj["data"] = jsonArray;
+        jObj["mindate"] = dateStart.toString("yyyy.MM.dd");
+        jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd");
+        jObj["totalcount"] = nTotalCount;
+        QJsonDocument jsonDoc(jObj);
+        stjsonRank.jdata = jsonDoc.toJson();
+        m_listMap.append(stjsonRank);
+    }
+}
+
+
+void SSpammer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
+{
+    makeTableBuzz(_datagroup, _mapCategory);
+    makeTableRank(_datagroup, _mapCategory);
+}
+
+
+
+bool SSpammer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany)
+{
+    QSqlQuery query(_db);
+    {
+        QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 3";
+        if (!query.exec(strQuery.toUtf8()))
+        {
+            m_pSInitializer->insertLog(query.lastQuery());
+            m_pSInitializer->insertLog(query.lastError().text());
+            return false;
+        }
+    }
+
+    foreach (const stStatsJson& ststats, m_listMap)
+    {
+        QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values (";
+        strQuery += QString::number(_nCompany) + ",";
+        strQuery += QString::number(ststats.servicenum) + ",";
+        strQuery += "'" + ststats.title + "',";
+        strQuery += "'" + ststats.subtitle + "',";
+        strQuery += QString::number(ststats.category_num) + ",";
+        strQuery += QString::number(ststats.platformname_num) + ",";
+        strQuery += "'" + ststats.jdata +"')";
+
+        if (!query.exec(strQuery.toUtf8()))
+        {
+            m_pSInitializer->insertLog(query.lastQuery());
+            m_pSInitializer->insertLog(query.lastError().text());
+            return false;
+        }
+    }
+    return true;
+}