etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

graph 업데이트

Browse Source 
git-svn-id: svn://192.168.0.12/source@279 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2016-07-06 10:19:53 +00:00
parent 3074db4fa0
commit 3a7eb4d829
13 changed files with 1094 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
sfilterprocess/sconsumer.cpp
View File

@@ -350,6 +350,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, st
const int* anColumn = m_pSInitializer->getColumnIntArray();
QDate dateStart(2100,1,1),dateEnd(2000,1,1);
QDate dateStartBody(2100,1,1),dateEndBody(2000,1,1);
QDate dateStartReply(2100,1,1),dateEndReply(2000,1,1);
QList<stBuzzy> listBuzzy;
// filtered
{
@@ -367,6 +370,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, st
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
dateStartBody = (date < dateStartBody ) ? date : dateStartBody;
dateEndBody = (date > dateEndBody ) ? date : dateEndBody;
if (stbuzzy.m_mapBodyBuzz.contains(date))
stbuzzy.m_mapBodyBuzz[date]++;
@@ -382,6 +388,9 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, st
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
dateStartReply = (date < dateStartReply ) ? date : dateStartReply;
dateEndReply = (date > dateEndReply ) ? date : dateEndReply;
if (stbuzzy.m_mapReplyBuzz.contains(date))
stbuzzy.m_mapReplyBuzz[date]++;
else
@@ -424,6 +433,19 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, st
listBuzzy.append(stbuzzy);
}
int nTotalCount = 0, nTotalBodyCount = 0, nTotalReplyCount = 0;
if (_mapCategory.contains(m_pSInitializer->getAllCategoryNum(2)))
{
foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(2)].m_setRealAllUrl)
{
nTotalBodyCount++;
nTotalReplyCount += (*mapReply)[strUrl].size();
}
nTotalCount = nTotalBodyCount + nTotalReplyCount;
}
{
QJsonArray jsArrayAll;
QJsonArray jsArrayReply;
@@ -477,14 +499,33 @@ void SConsumer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, st
streplybuzz.subtitle = "reply";
stallbuzz.subtitle = "all";
QJsonDocument jdAll(jsArrayAll);
QJsonDocument jdReply(jsArrayReply);
QJsonDocument jdBody(jsArrayBody);
QJsonObject jobjAll;
QJsonObject jobjBody;
QJsonObject jobjReply;
stbodybuzz.jdata = jdBody.toJson();
streplybuzz.jdata = jdReply.toJson();
stallbuzz.jdata = jdAll.toJson();
streplybuzz.subpagenum = stallbuzz.subpagenum = stbodybuzz.subpagenum = 1;
jobjAll["data"] = jsArrayAll;
jobjBody["data"] = jsArrayBody;
jobjReply["data"] = jsArrayReply;
jobjAll["mindate"] = dateStart.toString("yyyy.MM.dd");
jobjBody["mindate"] = dateStartBody.toString("yyyy.MM.dd");
jobjReply["mindate"] = dateStartReply.toString("yyyy.MM.dd");
jobjAll["maxdate"] = dateEnd.toString("yyyy.MM.dd");
jobjBody["maxdate"] = dateEndBody.toString("yyyy.MM.dd");
jobjReply["maxdate"] = dateEndReply.toString("yyyy.MM.dd");
jobjAll["totalcount"] = nTotalCount;
jobjBody["totalcount"] = nTotalBodyCount;
jobjReply["totalcount"] = nTotalReplyCount;
QJsonDocument jdAll(jobjAll);
QJsonDocument jdBody(jobjBody);
QJsonDocument jdReply(jobjReply);
stbodybuzz.jdata = jdBody.toJson(QJsonDocument::Compact);
streplybuzz.jdata = jdReply.toJson(QJsonDocument::Compact);
stallbuzz.jdata = jdAll.toJson(QJsonDocument::Compact);
stallbuzz.platformname_num = stbodybuzz.platformname_num = streplybuzz.platformname_num = 1;
m_listMap.append(stbodybuzz);
@@ -542,10 +583,9 @@ bool SConsumer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany)
foreach (const stStatsJson& ststats, m_listMap)
{
QString strQuery = "insert into stats_json (company_num, servicenum, subpagenum, title, subtitle, category_num, platformname_num, jdata) values (";
QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values (";
strQuery += QString::number(_nCompany) + ",";
strQuery += QString::number(ststats.servicenum) + ",";
strQuery += QString::number(ststats.subpagenum) + ",";
strQuery += "'" + ststats.title + "',";
strQuery += "'" + ststats.subtitle + "',";
strQuery += QString::number(ststats.category_num) + ",";

3
sfilterprocess/sconsumer.h
View File

@@ -55,13 +55,12 @@ public:
{
int company_num;
int servicenum;
int subpagenum;
QString title;
QString subtitle;
int platformname_num;
QString jdata;
int category_num;
stStatsJson():company_num(0), servicenum(2), subpagenum(0), platformname_num(1), category_num(0){}
stStatsJson():company_num(0), servicenum(2), platformname_num(1), category_num(0){}
};
struct stBuzzy

4
sfilterprocess/sdatagroup.cpp
View File

@@ -292,6 +292,8 @@ bool SDatagroup::makeTable()
stcommunity.platformform_num = m_pSInitializer->getPlatformForm(strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed());
stcommunity.platformname_num = m_pSInitializer->getPlatformName(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed());
stcommunity.title = strListBody[anColumn[SInitializer::E_DATA_platform_title]].trimmed();
if (stcommunity.title.trimmed().isEmpty())
stcommunity.title = strListBody[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
stcommunity.id = strListBody[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
stcommunity.url = m_SCommunity.getCommunityKey(strListBody).trimmed();
if(strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed() == QString("daum") && strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == QString("cafe"))
@@ -1230,7 +1232,7 @@ SDatagroup::stCommunity* SDatagroup::SCommunity::get(const QString &_strKey)
if (m_mapCommunity.contains(_strKey.trimmed()))
return &(m_mapCommunity[_strKey.trimmed()]);
else
return 0;
return &(m_mapCommunity[" "]);
}
bool SDatagroup::SCommunity::changeTable(QSqlDatabase &_db, const int &_nCompany)

2
sfilterprocess/sfilterprocess.cpp
View File

@@ -75,7 +75,7 @@ bool SFilterProcess::run()
m_SDatagroup.makeTable();
m_SConsumer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_CONSUMER));
m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER));
//m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER));
m_SSpammer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_SPAMMER));
m_SPowercafe.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_POWERCAFE));

4
sfilterprocess/sfilterprocess.pro.user
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2016-06-10T19:06:00. -->
<!-- Written by QtCreator 3.3.0, 2016-07-06T15:06:54. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>
@@ -787,7 +787,7 @@
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">sfilterprocess</value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;469&quot; &quot;test2y&quot;</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;510&quot; &quot;testall&quot;</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.ProFile">sfilterprocess.pro</value>
<value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseDyldImageSuffix">false</value>
<value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseTerminal">true</value>

239
sfilterprocess/sinfluencer.cpp
View File

@@ -6,6 +6,10 @@
#include <QDebug>
#include <QDate>
#include <QDateTime>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValue>
#include "sinfluencer.h"
void SInfluencer::setInitiaizer(SInitializer &_initializer)
@@ -13,9 +17,10 @@ void SInfluencer::setInitiaizer(SInitializer &_initializer)
m_pSInitializer = &_initializer;
m_SBodyInfluencerMap.setInitiaizer(_initializer);
m_SReplyInfluencerMap.setInitiaizer(_initializer);
m_SStatsInfBloggerBuzzbyday.setInitiaizer(_initializer);
m_SStatsInfBloggerRank.setInitiaizer(_initializer);
//m_SStatsInfBloggerBuzzbyday.setInitiaizer(_initializer);
//m_SStatsInfBloggerRank.setInitiaizer(_initializer);
m_STInfluencer.setInitiaizer(_initializer);
m_SStatsJson.setInitiaizer(_initializer);
}
bool SInfluencer::uploadData(int _companyNum)
@@ -34,6 +39,7 @@ bool SInfluencer::uploadData(int _companyNum)
m_pSInitializer->insertLog("Fail to upload influencer reply map");
return false;
}
/*
if (!m_SStatsInfBloggerBuzzbyday.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_buzzbyday");
@@ -44,11 +50,18 @@ bool SInfluencer::uploadData(int _companyNum)
m_pSInitializer->insertLog("Fail to upload stats_inf_blogger_rank");
return false;
}
*/
if (!m_STInfluencer.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload influencer");
return false;
}
if (!m_SStatsJson.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload stats_json");
return false;
}
m_pSInitializer->insertLog("End upload influencer");
m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Influence (Finish)");
@@ -170,19 +183,24 @@ bool SInfluencer::makeOverallCategory(int _categoryNum)
bool SInfluencer::makeTable(SDatagroup &_datagroup)
{
if (m_mapCategory.size() > 1)
//if (m_mapCategory.size() > 1)
if (m_mapCategory.size() > 0)
{
m_pSInitializer->insertLog("Make influencer tables");
m_pSInitializer->insertLog("Make influencer body tables");
m_SBodyInfluencerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer reply tables");
m_SReplyInfluencerMap.makeTable(_datagroup, m_mapCategory);
/*
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_buzzy tables");
m_SStatsInfBloggerBuzzbyday.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_rank table");
m_SStatsInfBloggerRank.makeTable(_datagroup, m_mapCategory);
*/
m_pSInitializer->insertLog("Make influencer influencer");
m_STInfluencer.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_json");
m_SStatsJson.makeTable(_datagroup, m_mapCategory);
}
return true;
}
@@ -216,13 +234,19 @@ void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMap<int
mapRank[stprofileurlkey].profile = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profile]].trimmed();
mapRank[stprofileurlkey].profileurl = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profileurl]].trimmed();
mapRank[stprofileurlkey].category_num = iterPos.key();
mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->title;
mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->url;
mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->id;
//mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->title;
//mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->url;
//mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->id;
mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity((*mapBody)[_url])->title;
mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity((*mapBody)[_url])->url;
mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity((*mapBody)[_url])->id;
mapRank[stprofileurlkey].nickname_nickname = _datagroup.getstBody(_url)->nickname_nickname;
mapRank[stprofileurlkey].nickname_exnickname = _datagroup.getstBody(_url)->nickname_exnickname;
mapRank[stprofileurlkey].platformname_num = _datagroup.getstBody(_url)->platformname_num;
mapRank[stprofileurlkey].platformname_name = _datagroup.getstBody(_url)->platformname_name;
//m_pSInitializer->insertLog(_url);
//m_pSInitializer->insertLog(stprofileurlkey.m_strId);
//m_pSInitializer->insertLog(mapRank[stprofileurlkey].community_url);
}
mapRank[stprofileurlkey].bodycount++;
@@ -1041,4 +1065,207 @@ bool SInfluencer::SReplyInfluencerMap::changeTable(QSqlDatabase &_db, const int
return true;
}
namespace
{
QString strSeparator = "!@#";
}
struct stCount
{
unsigned int body;
unsigned int reply;
QSet<QString> id;
stCount():body(0), reply(0){}
};
struct stDate
{
QDate dStart;
QDate dEnd;
stDate():dStart(2020, 1, 1), dEnd(1990, 1, 1){}
};
void SInfluencer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const int* anColumn = m_pSInitializer->getColumnIntArray();
const QStringList *slOrderedUrl = _datagroup.getOrderedUrl();
QMap<QString, stCount> mapCount;
QMap<QString, stDate> mapBodyDate;
foreach (const QString& strUrl, *slOrderedUrl)
{
if (mapBody->contains(strUrl))
{
QString strKey = (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
if (!mapCount.contains(strKey))
mapCount.insert(strKey, stCount());
mapCount[strKey].body += 1;
mapCount[strKey].id.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_id]].trimmed());
if (mapReply->contains(strUrl))
mapCount[strKey].reply += (*mapReply)[strUrl].size();
if (!mapBodyDate.contains(strKey))
mapBodyDate.insert(strKey, stDate());
//m_pSInitializer->insertLog((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_date]].trimmed());
QDate dBodyDate = QDateTime::fromString((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
//m_pSInitializer->insertLog(dBodyDate.toString("yyyy-MM-dd"));
mapBodyDate[strKey].dStart = (mapBodyDate.value(strKey).dStart < dBodyDate) ? mapBodyDate.value(strKey).dStart : dBodyDate;
mapBodyDate[strKey].dEnd = (mapBodyDate.value(strKey).dEnd > dBodyDate) ? mapBodyDate.value(strKey).dEnd : dBodyDate;
}
}
QSet<QString> setRealAllUrl;
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
stStatsJson ststatsjson;
ststatsjson.category_num = iterPos.key();
ststatsjson.platformname_num = 1;
ststatsjson.servicenum = 1;
ststatsjson.title = "info";
ststatsjson.subtitle = "";
const stReal& streal = iterPos.value();
QSet<QString> setPlatformType;
foreach (const QString& strUrl, streal.m_setRealAllUrl)
{
if (mapBody->contains(strUrl))
setPlatformType.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed());
setRealAllUrl.insert(strUrl);
}
unsigned int nBody = 0;
// unsigned int nReply = 0;
unsigned int nUser = 0;
QJsonObject jsonObj;
foreach (const QString& strKey, setPlatformType)
{
if (strKey == "naver!@#blog")
{
// m_pSInitializer->insertLog(mapBodyDate.value(strKey).dStart.toString("yyyy-MM-dd"));
// m_pSInitializer->insertLog(mapBodyDate.value(strKey).dEnd.toString("yyyy-MM-dd"));
// m_pSInitializer->insertLog(QString::number(mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd)));
nBody += (mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd) * 1000);
//nReply += (mapCount[strKey].reply * 1000);
if (setPlatformType.size() == 1)
jsonObj["pagenum"] = m_pSInitializer->getBlogPageNum();
}
else
{
nBody += mapCount[strKey].body;
// nReply += mapCount[strKey].reply;
}
nUser += mapCount[strKey].id.size();
}
/*
unsigned int nRealBody = streal.m_setRealAllUrl.size();
unsigned int nRealReply = 0;
foreach (const QString& strUrl, streal.m_setRealAllUrl)
{
if (mapReply->contains(strUrl))
nRealReply += mapReply->value(strUrl).size();
}
*/
//unsigned int nInfluencerValue = (nRealBody == 0) ? 0 : nRealReply / nRealBody;
unsigned int nTotalCount = nBody;
jsonObj["totalcount"] = static_cast<int>(nTotalCount);
jsonObj["totalusercount"] = static_cast<int>(nUser);
//jsonObj["value"] = static_cast<long>(nInfluencerValue);
if (!jsonObj.contains("pagenum"))
jsonObj["pagenum"] = -1;
QJsonDocument jsonDoc(jsonObj);
ststatsjson.jdata = jsonDoc.toJson();
m_listMap.append(ststatsjson);
}
{
stStatsJson ststatsjson;
ststatsjson.category_num = -1;
ststatsjson.platformname_num = 1;
ststatsjson.servicenum = 1;
ststatsjson.title = "info";
ststatsjson.subtitle = "";
QSet<QString> setPlatformType;
foreach (const QString& strUrl, setRealAllUrl)
{
if (mapBody->contains(strUrl))
setPlatformType.insert((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + strSeparator + (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed());
}
unsigned int nBody = 0;
unsigned int nUser = 0;
QJsonObject jsonObj;
foreach (const QString& strKey, setPlatformType)
{
if (strKey == "naver!@#blog")
{
nBody += (mapBodyDate.value(strKey).dStart.daysTo(mapBodyDate.value(strKey).dEnd) * 1000);
if (setPlatformType.size() == 1)
jsonObj["pagenum"] = m_pSInitializer->getBlogPageNum();
}
else
{
nBody += mapCount[strKey].body;
}
nUser += mapCount[strKey].id.size();
}
unsigned int nTotalCount = nBody;
jsonObj["totalcount"] = static_cast<int>(nTotalCount);
jsonObj["totalusercount"] = static_cast<int>(nUser);
if (!jsonObj.contains("pagenum"))
jsonObj["pagenum"] = -1;
QJsonDocument jsonDoc(jsonObj);
ststatsjson.jdata = jsonDoc.toJson();
m_listMap.append(ststatsjson);
}
}
bool SInfluencer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
QSqlQuery query(_db);
{
QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 1";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
foreach (const stStatsJson& ststats, m_listMap)
{
QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values (";
strQuery += QString::number(_nCompany) + ",";
strQuery += QString::number(ststats.servicenum) + ",";
strQuery += "'" + ststats.title + "',";
strQuery += "'" + ststats.subtitle + "',";
strQuery += QString::number(ststats.category_num) + ",";
strQuery += QString::number(ststats.platformname_num) + ",";
strQuery += "'" + ststats.jdata +"')";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
return true;
}

43
sfilterprocess/sinfluencer.h
View File

@@ -88,6 +88,20 @@ public:
QString body_date;
stReplyInfluencerMap():num(0),reply_num(0),body_num(0),id_num(0),category_num(0),realin(0),community_num(0){}
};
class SReplyInfluencerMap
{
private:
SInitializer *m_pSInitializer;
QList<stReplyInfluencerMap> m_listMap;
public:
void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; }
void clear(){ m_listMap.clear(); }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase &_db, const int &_nCompany);
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
};
class STInfluencer
{
private:
@@ -138,17 +152,29 @@ public:
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
};
class SReplyInfluencerMap
struct stStatsJson
{
int company_num;
int servicenum;
QString title;
QString subtitle;
int platformname_num;
QString jdata;
int category_num;
stStatsJson():company_num(0), servicenum(2), platformname_num(1), category_num(0){}
};
class SStatsJson
{
private:
SInitializer *m_pSInitializer;
QList<stReplyInfluencerMap> m_listMap;
QList<stStatsJson> m_listMap;
public:
void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; }
void clear(){ m_listMap.clear(); }
void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase &_db, const int &_nCompany);
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
bool uploadData(QSqlDatabase& _db, const int &_nCompany);
void clear(){ m_listMap.clear(); }
};
struct isGreaterInfluencerValue
@@ -211,9 +237,10 @@ private:
QMap<int, stReal> m_mapCategory;
SBodyInfluencerMap m_SBodyInfluencerMap;
SReplyInfluencerMap m_SReplyInfluencerMap;
SStatsInfBloggerBuzzbyday m_SStatsInfBloggerBuzzbyday;
SStatsInfBloggerRank m_SStatsInfBloggerRank;
//SStatsInfBloggerBuzzbyday m_SStatsInfBloggerBuzzbyday;
//SStatsInfBloggerRank m_SStatsInfBloggerRank;
STInfluencer m_STInfluencer;
SStatsJson m_SStatsJson;
public:
virtual void setInitiaizer(SInitializer &_initializer);

29
sfilterprocess/sinitializer.cpp
View File

@@ -166,6 +166,25 @@ bool SInitializer::initSpammer()
return true;
}
bool SInitializer::initBlogPageNum()
{
QSettings settings(QString("setup.ini"), QSettings::IniFormat);
settings.setIniCodec("UTF-8");
settings.beginGroup(QString("pagenumber"));
if (settings.childKeys().size() == 0)
m_nBlogPageNum = 10;
else
m_nBlogPageNum = settings.value("pagenumber").toInt();
settings.endGroup();
return true;
}
bool SInitializer::initDBConnection()
{
QSettings settings(QString("setup.ini"), QSettings::IniFormat);
@@ -212,11 +231,13 @@ bool SInitializer::init(const QString& _strFilterProcessGroupID, const QString&
b_ok &= initDays();
b_ok &= initInfluencer();
b_ok &= initSpammer();
b_ok &= initBlogPageNum();
b_ok &= initFilterProcess();
b_ok &= initCompanyNum();
b_ok &= initPlatform();
//b_ok &= initConsumerCategory();
b_ok &= initCategory();
/*
if (!b_ok)
cout << "FALSE" << endl;
@@ -472,6 +493,7 @@ bool SInitializer::initColumn()
return true;
}
bool SInitializer::initCompanyNum()
{
return true;
@@ -523,6 +545,13 @@ const int* SInitializer::getColumnIntArray() const
return m_stColumn.m_anColumn;
}
int SInitializer::getBlogPageNum()
{
return m_nBlogPageNum;
}
QList<SInitializer::stFilterProcess> SInitializer::getFilterProcess()
{
return m_listFilterProcess;

3
sfilterprocess/sinitializer.h
View File

@@ -28,6 +28,7 @@ private:
bool initPlatform();
//bool initConsumerCategory();
bool initCategory();
bool initBlogPageNum();
public:
@@ -177,6 +178,7 @@ private:
int m_nConsumerAllCategory;
int m_anAllCategory[E_SERVICE_MAX];
QMap<int, QString> m_mapCategory;
int m_nBlogPageNum;
public:
bool init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName);
@@ -205,6 +207,7 @@ public:
void updateWebDBInfoComplete(QSqlDatabase _db, QString _mindate, QString _maxdate, QString _lastupdate, int _bodycount, int _replycount, int _nCompany);
int getAllCategoryNum(int _nService);
QString getCategoryName(int _nCategory) const;
int getBlogPageNum();
//bool disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE);
//bool enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE);
//bool disableSettings(QSqlDatabase& db, int _nMode = MODE);

243
sfilterprocess/spowercafe.cpp
View File

@@ -4,6 +4,12 @@
#include <QSqlQuery>
#include <QSqlDatabase>
#include <QSqlError>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValue>
#include <QDate>
#include <QDateTime>
#include "spowercafe.h"
void SPowercafe::setInitiaizer(SInitializer &_initializer)
@@ -12,7 +18,8 @@ void SPowercafe::setInitiaizer(SInitializer &_initializer)
m_SBodyPowercafeMap.setInitiaizer(_initializer);
m_SReplyPowercafeMap.setInitiaizer(_initializer);
m_STPowercafe.setInitiaizer(_initializer);
m_SStatsPowCafeRank.setInitiaizer(_initializer);
//m_SStatsPowCafeRank.setInitiaizer(_initializer);
m_SStatsJson.setInitiaizer(_initializer);
}
bool SPowercafe::uploadData(int _companyNum)
@@ -31,16 +38,23 @@ bool SPowercafe::uploadData(int _companyNum)
m_pSInitializer->insertLog("Fail to upload Powercafe reply map");
return false;
}
/*
if (!m_SStatsPowCafeRank.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload stats_pow_cafe_rank");
return false;
}
*/
if (!m_STPowercafe.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload Powercafe");
return false;
}
if (!m_SStatsJson.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload sstatsJson");
return false;
}
m_pSInitializer->insertLog("End upload Powercafe");
m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Powercafe (Finish)");
@@ -176,10 +190,14 @@ bool SPowercafe::makeTable(SDatagroup &_datagroup)
m_SBodyPowercafeMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe reply tables");
m_SReplyPowercafeMap.makeTable(_datagroup, m_mapCategory);
/*
m_pSInitializer->insertLog("Make powercafe stats_pow_cafe_rank tables");
m_SStatsPowCafeRank.makeTable(_datagroup, m_mapCategory);
*/
m_pSInitializer->insertLog("Make powercafe powercafe table");
m_STPowercafe.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe sstatsjson");
m_SStatsJson.makeTable(_datagroup, m_mapCategory);
}
return true;
@@ -833,3 +851,226 @@ bool SPowercafe::SReplyPowercafeMap::changeTable(QSqlDatabase &_db, const int &_
}
return true;
}
void SPowercafe::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const int* anColumn = m_pSInitializer->getColumnIntArray();
/*
QMap<QString, stStatsPowCafeRank> mapPowercafeAll;
for (QMap<QString, QStringList>::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++)
{
if (iterPos.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe")
continue;
QString strKey = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
if (!mapPowercafeAll.contains(strKey))
{
mapPowercafeAll.insert(strKey, stStatsPowCafeRank());
mapPowercafeAll[strKey].community_id = iterPos.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
mapPowercafeAll[strKey].community_title = iterPos.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed();
mapPowercafeAll[strKey].subject = "articlecountall";
}
mapPowercafeAll[strKey].articlecountall++;
}
for (QMap<QString, QMap<int, QStringList> >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++)
{
for (QMap<int, QStringList>::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++)
{
if (iterPos1.value()[anColumn[SInitializer::E_DATA_platform_form]].trimmed() != "cafe")
continue;
QString strKey = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
if (!mapPowercafeAll.contains(strKey))
{
mapPowercafeAll.insert(strKey, stStatsPowCafeRank());
mapPowercafeAll[strKey].community_id = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_id]].trimmed();
mapPowercafeAll[strKey].community_title = iterPos1.value()[anColumn[SInitializer::E_DATA_platform_title]].trimmed();
mapPowercafeAll[strKey].subject = "articlecountall";
}
mapPowercafeAll[strKey].articlecountall++;
}
}
{
QList<stStatsPowCafeRank> listPowercafe = mapPowercafeAll.values();
qSort(listPowercafe.begin(), listPowercafe.end(), isGreaterCountAll());
int rank = 1;
foreach (stStatsPowCafeRank ststats, listPowercafe)
{
ststats.rank = rank++;
m_listMap.push_back(ststats);
if (rank > 10)
break;
}
}
*/
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
QMap<QString, stPowercafeRank> mapScore;
const stReal& streal = iterPos.value();
QDate dateStart(2030, 1, 1);
QDate dateEnd(2000, 1, 1);
foreach (const QString& _url, streal.m_setRealBodyUrl)
{
if (((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")||
((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" ))
continue;
QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ",";
strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
if (!mapScore.contains(strKey))
{
mapScore.insert(strKey, stPowercafeRank());
mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[_url])->url;
mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->title ;
if (mapScore[strKey].community.trimmed().isEmpty())
mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[_url])->id;
}
mapScore[strKey].count++;
QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
if (dateThis.isValid() && dateThis.year() > 1990)
{
dateStart = (dateStart < dateThis) ? dateStart : dateThis;
dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
}
}
for (QMap<QString, int>::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++)
{
if (((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "blog")||
((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "news" ))
continue;
QString strUrl = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_url]].trimmed();
QString strKey = (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_id]].trimmed() + ",";
strKey += ((*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
strKey += (*mapBody)[strUrl][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
if (!mapScore.contains(strKey))
{
mapScore.insert(strKey, stPowercafeRank());
mapScore[strKey].url = _datagroup.getstCommunity((*mapBody)[strUrl])->url;
mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->title;
if (mapScore[strKey].community.trimmed().isEmpty())
mapScore[strKey].community = _datagroup.getstCommunity((*mapBody)[strUrl])->id;
}
mapScore[strKey].count++;
QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
if (dateThis.isValid() && dateThis.year() > 1990)
{
dateStart = (dateStart < dateThis) ? dateStart : dateThis;
dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
}
}
//QList<stSpammerRank> listSpammer;
QMap<int, QList<stPowercafeRank> > mapRank;
for (QMap<QString, stPowercafeRank>::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++)
{
QString strPlatformName = iterPos1.key().split(",").at(1);
if (!mapRank.contains(m_pSInitializer->getPlatformName(strPlatformName)))
mapRank.insert(m_pSInitializer->getPlatformName(strPlatformName), QList<stPowercafeRank>());
mapRank[m_pSInitializer->getPlatformName(strPlatformName)].append(iterPos1.value());
}
for (QMap<int, QList<stPowercafeRank> >::iterator iterPos1 = mapRank.begin(); iterPos1 != mapRank.end(); iterPos1++)
qSort(iterPos1.value().begin(), iterPos1.value().end(), isGreaterCountReal());
QList<int> keys = mapRank.keys().toSet().toList();
QJsonArray jsonArray;
stStatsJson stjsonRank;
{
stjsonRank.category_num = iterPos.key();
stjsonRank.servicenum = 4;
stjsonRank.title = "rank";
stjsonRank.subtitle = "";
stjsonRank.platformname_num = 1;
}
int nTotalCount = 0;
for (QList<int>::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++)
{
QList<stPowercafeRank> values = mapRank.value(*iterPos1);
int nRankLimit = (values.size() <= 10) ? values.size() : 10;
QJsonObject jsonObj;
jsonObj["platformname_num"] = *iterPos1;
QJsonArray jsonSubArray;
for (int i = 0; i < nRankLimit; i++)
{
QJsonObject jsonSubObj;
jsonSubObj["rank"] = i + 1;
jsonSubObj["realcount"] = values.at(i).count;
jsonSubObj["community"] = values.at(i).community;
if (values.at(i).url.contains("instagram.com"))
jsonSubObj["url"] = "https://www." + values.at(i).url;
else if (values.at(i).url.contains("cafe.naver.com"))
jsonSubObj["url"] = "http://" + values.at(i).url;
else if (values.at(i).url.contains("facebook.com"))
jsonSubObj["url"] = "https://www." + values.at(i).url;
else if (values.at(i).url.contains("kakaostory.com"))
jsonSubObj["url"] = "https://" + values.at(i).url;
else if (values.at(i).url.contains("cafe.daum.net"))
jsonSubObj["url"] = "http://" + values.at(i).url;
else
jsonSubObj["url"] = "http://" + values.at(i).url;
jsonSubArray.append(jsonSubObj);
}
jsonObj["data"] = jsonSubArray;
jsonArray.append(jsonObj);
nTotalCount += values.size();
}
QJsonObject jObj;
jObj["data"] = jsonArray;
jObj["mindate"] = dateStart.toString("yyyy.MM.dd");
jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd");
jObj["totalcount"] = nTotalCount;
QJsonDocument jsonDoc(jObj);
stjsonRank.jdata = jsonDoc.toJson();
m_listMap.append(stjsonRank);
}
}
bool SPowercafe::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
QSqlQuery query(_db);
{
QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 4";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
foreach (const stStatsJson& ststats, m_listMap)
{
QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values (";
strQuery += QString::number(_nCompany) + ",";
strQuery += QString::number(ststats.servicenum) + ",";
strQuery += "'" + ststats.title + "',";
strQuery += "'" + ststats.subtitle + "',";
strQuery += QString::number(ststats.category_num) + ",";
strQuery += QString::number(ststats.platformname_num) + ",";
strQuery += "'" + ststats.jdata +"')";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
return true;
}

40
sfilterprocess/spowercafe.h
View File

@@ -132,6 +132,39 @@ public:
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
};
struct stStatsJson
{
int company_num;
int servicenum;
QString title;
QString subtitle;
int platformname_num;
QString jdata;
int category_num;
stStatsJson():company_num(0), servicenum(2), platformname_num(1), category_num(0){}
};
class SStatsJson
{
private:
SInitializer *m_pSInitializer;
QList<stStatsJson> m_listMap;
public:
void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase& _db, const int &_nCompany);
void clear(){ m_listMap.clear(); }
};
struct stPowercafeRank
{
int count;
QString url;
QString community;
stPowercafeRank(): count(0){}
};
struct isGreaterCountReal
{
bool operator()(const stPowercafe& first, const stPowercafe& second)
@@ -142,6 +175,10 @@ public:
{
return first.articlecountreal > second.articlecountreal;
}
bool operator()(const stPowercafeRank& first, const stPowercafeRank& second)
{
return first.count > second.count;
}
};
struct isGreaterCountAll
@@ -160,7 +197,8 @@ private:
SBodyPowercafeMap m_SBodyPowercafeMap;
SReplyPowercafeMap m_SReplyPowercafeMap;
STPowercafe m_STPowercafe;
SStatsPowCafeRank m_SStatsPowCafeRank;
//SStatsPowCafeRank m_SStatsPowCafeRank;
SStatsJson m_SStatsJson;
public:
virtual void setInitiaizer(SInitializer &_initializer);

408
sfilterprocess/sspammer.cpp
View File

@@ -5,6 +5,12 @@
#include <QVariant>
#include <QString>
#include <QStringList>
#include <QDate>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonValue>
#include <QSet>
#include "sspammer.h"
@@ -13,8 +19,9 @@ void SSpammer::setInitiaizer(SInitializer &_initializer)
m_pSInitializer = &_initializer;
m_SBodySpammerMap.setInitiaizer(_initializer);
m_SReplySpammerMap.setInitiaizer(_initializer);
m_SStatsSpaSpammerRank.setInitiaizer(_initializer);
//m_SStatsSpaSpammerRank.setInitiaizer(_initializer);
m_STSpammer.setInitiaizer(_initializer);
m_SStatsJson.setInitiaizer(_initializer);
}
bool SSpammer::uploadData(int _companyNum)
@@ -33,16 +40,23 @@ bool SSpammer::uploadData(int _companyNum)
m_pSInitializer->insertLog("Fail to upload Spammer reply map");
return false;
}
/*
if (!m_SStatsSpaSpammerRank.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload stats_spa_spammer_rank");
return false;
}
*/
if (!m_STSpammer.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload spammer");
return false;
}
if (!m_SStatsJson.uploadData(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload statsjson");
return false;
}
m_pSInitializer->insertLog("End upload Spammer");
m_pSInitializer->updateWebDBInfoState(dbWeb, _companyNum, "Spammer (Finish)");
@@ -168,13 +182,18 @@ bool SSpammer::makeTable(SDatagroup &_datagroup)
m_pSInitializer->insertLog("Make spammer tables");
m_pSInitializer->insertLog("Make spammer spammer");
m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer);
/*
m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables");
m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory);
*/
m_pSInitializer->insertLog("Make spammer body tables");
m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make spammer reply tables");
m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make sstatsjson table");
m_SStatsJson.makeTable(_datagroup, m_mapCategory);
}
return true;
}
@@ -824,3 +843,390 @@ bool SSpammer::SReplySpammerMap::changeTable(QSqlDatabase &_db, const int &_nCom
}
return true;
}
void SSpammer::SStatsJson::makeTableBuzz(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const int* anColumn = m_pSInitializer->getColumnIntArray();
QDate dateStart(2100,1,1),dateEnd(2000,1,1);
QDate dateStartBody(2100,1,1),dateEndBody(2000,1,1);
QDate dateStartReply(2100,1,1),dateEndReply(2000,1,1);
QList<stBuzzy> listBuzzy;
// filtered
{
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
stBuzzy stbuzzy;
stbuzzy.m_nCategory = iterPos.key();
const stReal &streal = iterPos.value();
foreach (const QString& strUrl, streal.m_setRealBodyUrl)
{
QDate date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
//if (date.isNull())
// date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed(), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
dateStartBody = (date < dateStartBody ) ? date : dateStartBody;
dateEndBody = (date > dateEndBody ) ? date : dateEndBody;
if (stbuzzy.m_mapBodyBuzz.contains(date))
stbuzzy.m_mapBodyBuzz[date]++;
else
stbuzzy.m_mapBodyBuzz.insert(date, 1);
}
for (QMap<QString, int>::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++)
{
const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()];
QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
dateStartReply = (date < dateStartReply ) ? date : dateStartReply;
dateEndReply = (date > dateEndReply ) ? date : dateEndReply;
if (stbuzzy.m_mapReplyBuzz.contains(date))
stbuzzy.m_mapReplyBuzz[date]++;
else
stbuzzy.m_mapReplyBuzz.insert(date, 1);
}
listBuzzy.append(stbuzzy);
}
}
//all buzzy
{
stBuzzy stbuzzy;
stbuzzy.m_nCategory = 0;
for (QMap<QString, QStringList>::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++)
{
QDate date = QDateTime::fromString(iterPos.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
if (stbuzzy.m_mapBodyBuzz.contains(date))
stbuzzy.m_mapBodyBuzz[date]++;
else
stbuzzy.m_mapBodyBuzz.insert(date, 1);
}
for (QMap<QString, QMap<int, QStringList> >::const_iterator iterPos = mapReply->constBegin(); iterPos != mapReply->constEnd(); iterPos++)
{
for (QMap<int, QStringList>::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++)
{
QDate date = QDateTime::fromString(iterPos1.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
if (stbuzzy.m_mapReplyBuzz.contains(date))
stbuzzy.m_mapReplyBuzz[date]++;
else
stbuzzy.m_mapReplyBuzz.insert(date, 1);
}
}
listBuzzy.append(stbuzzy);
}
int nTotalCount = 0, nTotalBodyCount = 0, nTotalReplyCount = 0;
if (_mapCategory.contains(m_pSInitializer->getAllCategoryNum(3)))
{
foreach (const QString& strUrl, _mapCategory[m_pSInitializer->getAllCategoryNum(3)].m_setRealAllUrl)
{
nTotalBodyCount++;
nTotalReplyCount += (*mapReply)[strUrl].size();
}
nTotalCount = nTotalBodyCount + nTotalReplyCount;
}
{
QJsonArray jsArrayAll;
QJsonArray jsArrayReply;
QJsonArray jsArrayBody;
foreach (const stBuzzy &stbuzzy, listBuzzy)
{
QJsonArray jsValueAll;
QJsonArray jsValueBody;
QJsonArray jsValueReply;
for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1))
{
QJsonObject jsoBody;
QJsonObject jsoAll;
QJsonObject jsoReply;
jsoAll["date"] = jsoReply["date"] = jsoBody["date"] = i.toString("yyyyMMdd");
jsoBody["count"] = stbuzzy.m_mapBodyBuzz[i];
jsoReply["count"] = stbuzzy.m_mapReplyBuzz[i];
jsoAll["count"] = stbuzzy.m_mapBodyBuzz[i] + stbuzzy.m_mapReplyBuzz[i];
jsValueAll.append(jsoAll);
jsValueBody.append(jsoBody);
jsValueReply.append(jsoReply);
}
QJsonObject jsNameReply;
QJsonObject jsNameAll;
QJsonObject jsNameBody;
if (stbuzzy.m_nCategory != 0)
jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = m_pSInitializer->getCategoryName(stbuzzy.m_nCategory);
else
jsNameReply["name"] = jsNameBody["name"] = jsNameAll["name"] = "ALL";
jsNameAll["value"] = jsValueAll;
jsNameBody["value"] = jsValueBody;
jsNameReply["value"] = jsValueReply;
jsArrayAll.append(jsNameAll);
jsArrayBody.append(jsNameBody);
jsArrayReply.append(jsNameReply);
}
stStatsJson stbodybuzz;
stStatsJson streplybuzz;
stStatsJson stallbuzz;
stallbuzz.title = streplybuzz.title = stbodybuzz.title = "buzz";
stbodybuzz.subtitle = "body";
streplybuzz.subtitle = "reply";
stallbuzz.subtitle = "all";
QJsonObject jobjAll;
QJsonObject jobjBody;
QJsonObject jobjReply;
jobjAll["data"] = jsArrayAll;
jobjBody["data"] = jsArrayBody;
jobjReply["data"] = jsArrayReply;
jobjAll["mindate"] = dateStart.toString("yyyy.MM.dd");
jobjBody["mindate"] = dateStartBody.toString("yyyy.MM.dd");
jobjReply["mindate"] = dateStartReply.toString("yyyy.MM.dd");
jobjAll["maxdate"] = dateEnd.toString("yyyy.MM.dd");
jobjBody["maxdate"] = dateEndBody.toString("yyyy.MM.dd");
jobjReply["maxdate"] = dateEndReply.toString("yyyy.MM.dd");
jobjAll["totalcount"] = nTotalCount;
jobjBody["totalcount"] = nTotalBodyCount;
jobjReply["totalcount"] = nTotalReplyCount;
QJsonDocument jdAll(jobjAll);
QJsonDocument jdBody(jobjBody);
QJsonDocument jdReply(jobjReply);
stbodybuzz.jdata = jdBody.toJson(QJsonDocument::Compact);
streplybuzz.jdata = jdReply.toJson(QJsonDocument::Compact);
stallbuzz.jdata = jdAll.toJson(QJsonDocument::Compact);
stallbuzz.platformname_num = stbodybuzz.platformname_num = streplybuzz.platformname_num = 1;
m_listMap.append(stbodybuzz);
m_listMap.append(streplybuzz);
m_listMap.append(stallbuzz);
}
/*
{
for (QDate i = dateStart; i <= dateEnd; i = i.addDays(1))
{
//body
stStatsJson stbodybuzz;
stStatsJson streplybuzz;
stStatsJson stallbuzz;
foreach (const stBuzzy& stbuzzy, listBuzzy)
{
if (stbuzzy.m_mapBodyBuzz.contains(i))
{
stbodybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapBodyBuzz[i];
stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapBodyBuzz[i];
}
if (stbuzzy.m_mapReplyBuzz.contains(i))
{
streplybuzz.category[stbuzzy.m_nCategory] = stbuzzy.m_mapReplyBuzz[i];
stallbuzz.category[stbuzzy.m_nCategory] += stbuzzy.m_mapReplyBuzz[i];
}
}
stallbuzz.date = streplybuzz.date = stbodybuzz.date = i;
stbodybuzz.subtitle = "body";
streplybuzz.subtitle = "reply";
stallbuzz.subtitle = "all";
m_listMap.append(stallbuzz);
m_listMap.append(stbodybuzz);
m_listMap.append(streplybuzz);
}
}
*/
}
void SSpammer::SStatsJson::makeTableRank(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const int* anColumn = m_pSInitializer->getColumnIntArray();
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
QMap<QString, stSpammerScore> mapScore;
const stReal& streal = iterPos.value();
QDate dateStart(2030, 1, 1);
QDate dateEnd(2000, 1, 1);
foreach (const QString& _url, streal.m_setRealBodyUrl)
{
QString strKey = (*mapBody)[_url][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ",";
strKey += ((*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
strKey += (*mapBody)[_url][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
if (!mapScore.contains(strKey))
{
mapScore.insert(strKey, stSpammerScore());
mapScore[strKey].id_num = _datagroup.getstBody(_url)->id_num;
}
mapScore[strKey].body++;
mapScore[strKey].nickname.insert((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_nickname]].trimmed());
QDate dateThis = QDateTime::fromString((*mapBody)[_url][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
if (dateThis.isValid() && dateThis.year() > 1990)
{
dateStart = (dateStart < dateThis) ? dateStart : dateThis;
dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
}
}
for (QMap<QString, int>::const_iterator iterPos1 = streal.m_mapRealReplyUniqueUrl.constBegin(); iterPos1 != streal.m_mapRealReplyUniqueUrl.constEnd(); iterPos1++)
{
QString strKey = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_id]].trimmed() + ",";
strKey += ((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed() + ",");
strKey += (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_form]].trimmed();
if (!mapScore.contains(strKey))
{
mapScore.insert(strKey, stSpammerScore());
mapScore[strKey].id_num = _datagroup.getstBody(iterPos1.key())->id_num;
}
mapScore[strKey].reply++;
mapScore[strKey].nickname.insert((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_nickname]].trimmed());
QDate dateThis = QDateTime::fromString((*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd hh:mm:ss").date();
if (dateThis.isValid() && dateThis.year() > 1990)
{
dateStart = (dateStart < dateThis) ? dateStart : dateThis;
dateEnd = (dateEnd > dateThis) ? dateEnd : dateThis;
}
}
//QList<stSpammerRank> listSpammer;
QMap<int, stSpammerRank> mapSpammer;
for (QMap<QString, stSpammerScore>::iterator iterPos1 = mapScore.begin(); iterPos1 != mapScore.end(); iterPos1++)
{
if (iterPos1->body >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] ||
iterPos1->reply >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] ||
iterPos1->nickname.size() >= m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT])
{
double value = iterPos1->body * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_RATIO]
+ iterPos1->reply * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_REPLY_COUNT_RATIO]
+ iterPos1->nickname.size() * m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_RATIO];
stSpammerRank rank;
rank.value = value;
rank.id = iterPos1.key().split(",").at(0);
rank.platformform_num = m_pSInitializer->getPlatformName(iterPos1.key().split(",").at(1));
rank.id_num = iterPos1.value().id_num;
mapSpammer.insertMulti(rank.platformform_num, rank);
}
}
QList<int> keys = mapSpammer.keys().toSet().toList();
QJsonArray jsonArray;
stStatsJson stjsonRank;
{
stjsonRank.category_num = iterPos.key();
stjsonRank.servicenum = 3;
stjsonRank.title = "rank";
stjsonRank.subtitle = "";
stjsonRank.platformname_num = 1;
}
int nTotalCount = 0;
for (QList<int>::iterator iterPos1 = keys.begin(); iterPos1 != keys.end(); iterPos1++)
{
QList<stSpammerRank> values = mapSpammer.values(*iterPos1);
qSort(values.begin(), values.end(), isGreaterSpammerValue());
int nRankLimit = (values.size() <= 10) ? values.size() : 10;
QJsonObject jsonObj;
jsonObj["platformname_num"] = *iterPos1;
QJsonArray jsonSubArray;
for (int i = 0; i < nRankLimit; i++)
{
QJsonObject jsonSubObj;
jsonSubObj["rank"] = i + 1;
jsonSubObj["value"] = values.at(i).value;
jsonSubObj["id"] = values.at(i).id;
jsonSubObj["id_num"] = values.at(i).id_num;
jsonSubArray.append(jsonSubObj);
}
jsonObj["data"] = jsonSubArray;
jsonArray.append(jsonObj);
nTotalCount += values.size();
}
QJsonObject jObj;
jObj["data"] = jsonArray;
jObj["mindate"] = dateStart.toString("yyyy.MM.dd");
jObj["maxdate"] = dateEnd.toString("yyyy.MM.dd");
jObj["totalcount"] = nTotalCount;
QJsonDocument jsonDoc(jObj);
stjsonRank.jdata = jsonDoc.toJson();
m_listMap.append(stjsonRank);
}
}
void SSpammer::SStatsJson::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
makeTableBuzz(_datagroup, _mapCategory);
makeTableRank(_datagroup, _mapCategory);
}
bool SSpammer::SStatsJson::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
QSqlQuery query(_db);
{
QString strQuery = "delete from stats_json where company_num = " + QString::number(_nCompany) + " and servicenum = 3";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
foreach (const stStatsJson& ststats, m_listMap)
{
QString strQuery = "insert into stats_json (company_num, servicenum, title, subtitle, category_num, platformname_num, jdata) values (";
strQuery += QString::number(_nCompany) + ",";
strQuery += QString::number(ststats.servicenum) + ",";
strQuery += "'" + ststats.title + "',";
strQuery += "'" + ststats.subtitle + "',";
strQuery += QString::number(ststats.category_num) + ",";
strQuery += QString::number(ststats.platformname_num) + ",";
strQuery += "'" + ststats.jdata +"')";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
}
return true;
}

52
sfilterprocess/sspammer.h
View File

@@ -77,7 +77,35 @@ public:
int reply;
int id_num;
QSet<QString> nickname;
stSpammerScore():body(0),reply(0){}
stSpammerScore():body(0),reply(0), id_num(0){}
};
struct stSpammerRank
{
double value;
int platformform_num;
QString id;
int id_num;
stSpammerRank():value(0.0), platformform_num(1), id_num(0){}
};
struct stBuzzy
{
int m_nCategory;
QMap<QDate, int> m_mapBodyBuzz;
QMap<QDate, int> m_mapReplyBuzz;
};
struct stStatsJson
{
int company_num;
int servicenum;
QString title;
QString subtitle;
int platformname_num;
QString jdata;
int category_num;
stStatsJson():company_num(0), servicenum(3), platformname_num(1), category_num(0){}
};
class STSpammer
@@ -143,16 +171,36 @@ public:
{
return first.spammervalue > second.spammervalue;
}
bool operator()(const stSpammerRank& first, const stSpammerRank& second) const
{
return first.value > second.value;
}
};
class SStatsJson
{
private:
SInitializer *m_pSInitializer;
QList<stStatsJson> m_listMap;
public:
void setInitiaizer(SInitializer &_SInitializer) { m_pSInitializer = &_SInitializer; }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
void makeTableBuzz(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
void makeTableRank(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase& _db, const int &_nCompany);
void clear(){ m_listMap.clear(); }
};
private:
SFilterAlgorithm m_SFilterAlgorithm;
QMap<int, stReal> m_mapCategory;
STSpammer m_STSpammer;
SStatsSpaSpammerRank m_SStatsSpaSpammerRank;
//SStatsSpaSpammerRank m_SStatsSpaSpammerRank;
SBodySpammerMap m_SBodySpammerMap;
SReplySpammerMap m_SReplySpammerMap;
QSet<int> m_setSpammer;
SStatsJson m_SStatsJson;
public:
virtual void setInitiaizer(SInitializer &_initializer);