etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

카테고리 중간 수정버전

Browse Source 
git-svn-id: svn://192.168.0.12/source@261 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2016-05-11 08:47:24 +00:00
parent d1adf4e8b4
commit bd8a037056
15 changed files with 712 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
sfilterprocess/sconsumer.cpp
View File

@@ -60,8 +60,8 @@ bool SConsumer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList
}
}
stReal streal;
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size();
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size();
m_pSInitializer->insertLog(QString("CategoryNum:") + QString::number(_nCategory) + ",(" +
QString::number(vecData[SInitializer::E_ARTICLE_BODY].size()) + '/' + QString::number(vecData[SInitializer::E_ARTICLE_REPLY].size()) + ")" + "(Body/Reply)" );
const int *anColumn = m_pSInitializer->getColumnIntArray();
if (bFiltered[SInitializer::E_ARTICLE_BODY])
{
@@ -135,11 +135,14 @@ bool SConsumer::makeOverallCategory(int _nCategory)
bool SConsumer::makeTable(SDatagroup &_datagroup)
{
m_pSInitializer->insertLog("Make Consumer Tables");
m_SBodyConsumerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make Body Tables");
m_SStatsConUserBuzzybyday.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make Reply Tables");
if (m_mapCategory.size() > 1)
{
m_pSInitializer->insertLog("Make Consumer Tables");
m_SBodyConsumerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make Body Tables");
m_SStatsConUserBuzzybyday.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make Reply Tables");
}
return true;
}
@@ -159,6 +162,14 @@ void SConsumer::SBodyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap<i
stbody.body_num = _datagroup.getstBodyNum(strUrl);
stbody.category_num = iterPos.key();
stbody.realin |= REAL_BODY;
stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num;
stbody.platformform_num = _datagroup.getstBody(strUrl)->platformform_num;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
stbody.body_replycount = _datagroup.getstBody(strUrl)->replycount;
stbody.body_view = _datagroup.getstBody(strUrl)->view;
stbody.body_id_num = _datagroup.getstBody(strUrl)->id_num;
if (streal.m_mapRealReplyUniqueUrl.contains(strUrl))
{
stbody.realin |= REAL_REPLY;
@@ -179,6 +190,13 @@ void SConsumer::SBodyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap<i
stbody.category_num = iterPos.key();
stbody.realin |= REAL_REPLY;
stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num;
stbody.platformform_num = _datagroup.getstBody(strUrl)->platformform_num;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
stbody.body_replycount = _datagroup.getstBody(strUrl)->replycount;
stbody.body_view = _datagroup.getstBody(strUrl)->view;
stbody.body_id_num = _datagroup.getstBody(strUrl)->id_num;
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
@@ -252,7 +270,7 @@ bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCom
}
strQuery = QString("insert into body_consumer_map_" + QString::number(_nCompany) +
"_update (num,body_num,category_num,realin) VALUES (");
"_update (num,body_num,category_num,realin,platformname_num,platformform_num,body_date,body_replycount,body_view,body_id_num) VALUES (");
query.clear();
@@ -262,7 +280,13 @@ bool SConsumer::SBodyConsumerMap::uploadData(QSqlDatabase &_db, const int &_nCom
str += QString::number(stbody.num) + ",";
str += QString::number(stbody.body_num) + ",";
str += QString::number(stbody.category_num) + ",";
str += QString::number(stbody.realin) + ")";
str += QString::number(stbody.realin) + ",";
str += QString::number(stbody.platformname_num) + ",";
str += QString::number(stbody.platformform_num) + ",";
str += "'" + stbody.body_date + "',";
str += QString::number(stbody.body_replycount) + ",";
str += QString::number(stbody.body_view) + ",";
str += QString::number(stbody.body_id_num) + ")";
if (query.exec(str.toUtf8()) == false)
{
@@ -285,7 +309,7 @@ void SConsumer::SReplyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap<
{
if (streal.m_setRealAllUrl.contains(strUrl))
{
if (_datagroup.getstBody(strUrl)->reply_count == 0)
if (_datagroup.getstBody(strUrl)->replycount == 0)
continue;
const QMap<int, SDatagroup::stReply>* mapReply = _datagroup.getstReply(strUrl);
@@ -303,6 +327,7 @@ void SConsumer::SReplyConsumerMap::makeTable(SDatagroup &_datagroup, const QMap<
void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
/*
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const int* anColumn = m_pSInitializer->getColumnIntArray();
@@ -323,7 +348,8 @@ void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const
QDate date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
//if (date.isNull())
// date = QDateTime::fromString((*mapBody)[strUrl.trimmed()][anColumn[SInitializer::E_DATA_article_date]].trimmed(), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
@@ -337,6 +363,8 @@ void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const
{
const QStringList &slReply = (*mapReply)[iterPos1.key()][iterPos1.value()];
QDate date = QDateTime::fromString(slReply[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
if (stbuzzy.m_mapReplyBuzz.contains(date))
@@ -356,6 +384,9 @@ void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const
for (QMap<QString, QStringList>::const_iterator iterPos = mapBody->constBegin(); iterPos != mapBody->constEnd(); iterPos++)
{
QDate date = QDateTime::fromString(iterPos.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
if (stbuzzy.m_mapBodyBuzz.contains(date))
stbuzzy.m_mapBodyBuzz[date]++;
else
@@ -367,6 +398,9 @@ void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const
for (QMap<int, QStringList>::const_iterator iterPos1 = iterPos.value().constBegin(); iterPos1 != iterPos.value().constEnd(); iterPos1++)
{
QDate date = QDateTime::fromString(iterPos1.value()[anColumn[SInitializer::E_DATA_article_date]].trimmed().replace("T", " "), "yyyy-MM-dd HH:mm:ss").date();
if (date.year() == 1990)
continue;
if (stbuzzy.m_mapReplyBuzz.contains(date))
stbuzzy.m_mapReplyBuzz[date]++;
else
@@ -406,6 +440,7 @@ void SConsumer::SStatsConUserBuzzybyday::makeTable(SDatagroup &_datagroup, const
m_listMap.append(streplybuzz);
}
}
*/
}
/*

8
sfilterprocess/sconsumer.h
View File

@@ -18,7 +18,13 @@ public:
int body_num;
int category_num;
int realin;
stBodyConsumerMap():num(0), body_num(0), category_num(0), realin(0){}
int platformname_num;
int platformform_num;
QString body_date;
int body_replycount;
int body_view;
int body_id_num;
stBodyConsumerMap():num(0), body_num(0), category_num(0), realin(0), platformname_num(0), platformform_num(0), body_replycount(0), body_view(0), body_id_num(0){}
};
struct stReplyConsumerMap
{

132
sfilterprocess/sdatagroup.cpp
View File

@@ -7,6 +7,7 @@
#include <QDebug>
#include <QRegExp>
#include <QTextCodec>
#include <QtAlgorithms>
#include "sdatagroup.h"
#include "sinfluencer.h"
#include "sinitializer.h"
@@ -84,6 +85,7 @@ bool SDatagroup::loadData(const int& _nDatagroupNumber)
for (int j = 0; j < m_pSInitializer->getColumnStringList().size(); j++)
strList.append(" " + query.value(j).toString() + " ");
/*
if(strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "post" ||
strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "story" ||
strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "channel")
@@ -91,7 +93,7 @@ bool SDatagroup::loadData(const int& _nDatagroupNumber)
if(strList[anColumn[SInitializer::E_DATA_platform_form]].trimmed() == "group")
strList[anColumn[SInitializer::E_DATA_platform_form]] = " community ";
*/
switch(i)
{
case E_ARTICLE_BODY:
@@ -256,7 +258,7 @@ bool SDatagroup::uploadData(const int &_nCompany)
bool SDatagroup::makeTable()
{
QRegExp rxHttp("http[s]?://");
m_pSInitializer->insertLog("Start makeTable");
m_pSInitializer->insertLog("Start Datagroup makeTable");
m_pSInitializer->insertLog("#ofALL/#ofFiltered :" + QString::number(m_mapBody.size()) + "/" + QString::number(m_setFilteredUrl.size()) );
const int* anColumn = m_pSInitializer->getColumnIntArray();
int nBody = 0;
@@ -360,7 +362,7 @@ bool SDatagroup::makeTable()
stbody.num = nBody++;
stbody.data = &(strListBody[anColumn[SInitializer::E_DATA_article_data]]);
stbody.title = &(strListBody[anColumn[SInitializer::E_DATA_article_title]]);
stbody.reply_count = mapReply.size();
stbody.replycount = mapReply.size();
stbody.id_id = strListBody[anColumn[SInitializer::E_DATA_article_id]].trimmed();
stbody.date = strListBody[anColumn[SInitializer::E_DATA_article_date]].trimmed();
stbody.lastupdate = lastupdate;
@@ -372,8 +374,18 @@ bool SDatagroup::makeTable()
stbody.id_num = m_SId.get(m_SId.getIdKey(strListBody))->num;
stbody.community_num = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->num;
stbody.nickname_num = m_SNickname.get(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->num;
stbody.reply_startnum = ((mapReply.size() > 0) ? nReply : -1);
stbody.reply_endnum = ((mapReply.size() > 0) ? nReply + mapReply.size() - 1 : -1);
stbody.replystartnum = ((mapReply.size() > 0) ? nReply : -1);
stbody.replyendnum = ((mapReply.size() > 0) ? nReply + mapReply.size() - 1 : -1);
stbody.community_title = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->title;
stbody.community_url = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->url;
stbody.community_id = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->id;
stbody.community_exid = m_SCommunity.get(m_SCommunity.getCommunityKey(strListBody))->exid;
stbody.platformform_form = strListBody[anColumn[SInitializer::E_DATA_platform_form]].trimmed();
stbody.platformname_name = strListBody[anColumn[SInitializer::E_DATA_platform_name]].trimmed();
stbody.nickname_nickname = m_SNickname.get(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->nickname;
stbody.nickname_exnickname = UniCodeToEucKr(m_SNickname.get(strListBody[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->nickname);
m_SBody.put(strUrl, stbody);
m_stDate.m_strMaxdate = (( m_stDate.m_strMaxdate > stbody.date ) ? m_stDate.m_strMaxdate : stbody.date );
@@ -395,6 +407,15 @@ bool SDatagroup::makeTable()
streply.data = &(strListReply[anColumn[SInitializer::E_DATA_article_data]]);
streply.date = strListReply[anColumn[SInitializer::E_DATA_article_date]].trimmed();
streply.parent = strListReply[anColumn[SInitializer::E_DATA_article_parent]].trimmed();
streply.nickname_nickname = m_SNickname.get(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->nickname;
streply.nickname_exnickname = m_SNickname.get(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed())->nickname;
streply.platformform_num = m_pSInitializer->getPlatformForm(strListReply[anColumn[SInitializer::E_DATA_platform_form]].trimmed());
streply.platformform_form = strListReply[anColumn[SInitializer::E_DATA_platform_form]].trimmed();
streply.platformname_num = m_pSInitializer->getPlatformName(strListReply[anColumn[SInitializer::E_DATA_platform_name]].trimmed());
streply.platformname_name = strListReply[anColumn[SInitializer::E_DATA_platform_name]].trimmed();
streply.body_date = m_SBody.get(strUrl)->date;
if (streply.parent.isEmpty())
streply.parent = " ";
if (m_SId.contains(m_SId.getIdKey(strListReply)) && m_SNickname.contains(strListReply[anColumn[SInitializer::E_DATA_article_nickname]].trimmed()))
@@ -556,7 +577,22 @@ void SDatagroup::SBody::clear()
QString SDatagroup::SBody::vecToString(const QVector<int>& _realbycategorynum)
{
QString str = "";
foreach (int real, _realbycategorynum)
QList<int> ilist = _realbycategorynum.toList();
qSort(ilist.begin(), ilist.end(), qLess<int>());
QList<int>::iterator it = std::unique(ilist.begin(), ilist.end());
int totalsize = std::distance(ilist.begin(), it);
for (int i = 0; i < totalsize; i++)
str += ((QString::number(ilist.at(i))) + " ");
return str.trimmed();
}
QString SDatagroup::SBody::vecToString(const QSet<int>& _realbycategorynum)
{
QString str = "";
QList<int> nlist = _realbycategorynum.toList();
qSort(nlist.begin(), nlist.end(), qLess<int>());
foreach (int real, nlist)
str += (QString::number(real) + " ");
return str.trimmed();
}
@@ -625,7 +661,8 @@ bool SDatagroup::SBody::uploadData(QSqlDatabase &_db, const int& _nCompany, cons
}
strQuery = "insert into body_" + QString::number(_nCompany) +
"_update(num,id_num,nickname_num,community_num,id_id,platformname_num,platformform_num, title, data, url, view, date, reply_startnum, reply_endnum, reply_count, lastupdate, realbycategorynum) VALUES(";
"_update(num,id_num,nickname_num,community_num,id_id,platformname_num,platformform_num, title, data, url, view, date, replystartnum, replyendnum, replycount, lastupdate, realbycategorynum,"
"community_title, community_url, community_id, community_exid, platformname_name, platformform_form, nickname_nickname, nickname_exnickname) VALUES(";
foreach (const QString& url, _order)
{
@@ -639,17 +676,28 @@ bool SDatagroup::SBody::uploadData(QSqlDatabase &_db, const int& _nCompany, cons
str += QString::number(body.community_num) + ",";
str += "'" + sqlString(body.id_id) + "',";
str += QString::number(body.platformname_num) + ",";
str += QString::number(body.platformname_num) + ",";
str += QString::number(body.platformform_num) + ",";
str += "'" + sqlString(*(body.title)).replace(QChar(160),"") + "',";
str += "'" + sqlString(*(body.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"").left(65535) + "',";
str += "'" + sqlString((*(body.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"").left(60000)) + "',";
str += "'" + body.url + "',";
str += QString::number(body.view) + ",";
str += "'" + body.date + "',";
str += QString::number(body.reply_startnum) + ",";
str += QString::number(body.reply_endnum) + ",";
str += QString::number(body.reply_count) + ",";
str += QString::number(body.replystartnum) + ",";
str += QString::number(body.replyendnum) + ",";
str += QString::number(body.replycount) + ",";
str += "'" + body.lastupdate + "',";
str += "'" + vecToString(body.realbycategorynum) + "')";
str += "'" + vecToString(body.realbycategorynum) + "',";
str += "'" + sqlString(body.community_title) + "',";
str += "'" + body.community_url + "',";
str += "'" + body.community_id + "',";
str += "'" + sqlString(body.community_exid) + "',";
str += "'" + body.platformname_name + "',";
str += "'" + body.platformform_form + "',";
str += "'" + sqlString(body.nickname_nickname) + "',";
str += "'" + sqlString(body.nickname_exnickname) + "')";
if (!query.exec(str.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
@@ -805,7 +853,9 @@ bool SDatagroup::SReply::uploadData(QSqlDatabase &_db, const int &_nCompany, QSt
return false;
}
strQuery = "insert into reply_" + QString::number(_nCompany) + "_update (num,id_num,nickname_num,community_num,id_id,parent,data,date,body_num,realbycategorynum) VALUES (";
strQuery = "insert into reply_" + QString::number(_nCompany) +
"_update (num,id_num,nickname_num,community_num,id_id,parent,data,date,body_num,realbycategorynum,"
"nickname_nickname,nickname_exnickname,platformname_num,platformname_name,platformform_num,platformform_form,body_date) VALUES (";
foreach (const QString& url, _order)
{
@@ -820,10 +870,18 @@ bool SDatagroup::SReply::uploadData(QSqlDatabase &_db, const int &_nCompany, QSt
str += QString::number(reply.community_num) + ",";
str += "'" + sqlString(reply.id_id) + "',";
str += "'" + sqlString(reply.parent) + "',";
str += "'" + sqlString(*(reply.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"") + "',";
str += "'" + sqlString((*(reply.data)).replace(QRegExp("\n{3,}"),"\n\n").replace(QChar(160),"").left(60000)) + "',";
str += "'" + reply.date + "',";
str += QString::number(reply.body_num) + ",";
str += "'" + vecToString(reply.realbycategorynum) + "')";
str += "'" + vecToString(reply.realbycategorynum) + "',";
str += "'" + sqlString(reply.nickname_nickname) + "',";
str += "'" + sqlString(reply.nickname_exnickname) + "',";
str += QString::number(reply.platformname_num) + ",";
str += "'" + reply.platformname_name + "',";
str += QString::number(reply.platformform_num) + ",";
str += "'" + reply.platformform_form + "',";
str += "'" + reply.body_date + "')";
if (!query.exec(str.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
@@ -840,10 +898,22 @@ bool SDatagroup::SReply::uploadData(QSqlDatabase &_db, const int &_nCompany, QSt
QString SDatagroup::SReply::vecToString(const QVector<int>& _realbycategorynum)
{
/*
QString str = "";
foreach (int real, _realbycategorynum)
str += (QString::number(real) + " ");
return str.trimmed();
*/
QString str = "";
QList<int> ilist = _realbycategorynum.toList();
qSort(ilist.begin(), ilist.end(), qLess<int>());
QList<int>::iterator it = std::unique(ilist.begin(), ilist.end());
int totalsize = std::distance(ilist.begin(), it);
for (int i = 0; i < totalsize; i++)
str += ((QString::number(ilist.at(i))) + " ");
return str.trimmed();
}
@@ -1092,7 +1162,7 @@ bool SDatagroup::SNickname::uploadData(QSqlDatabase &_db, const int &_nCompany)
}
QString SDatagroup::SNickname::UniCodeToEucKr(const QString& _strData)
QString UniCodeToEucKr(const QString& _strData)
{
{
QString strOut;
@@ -1128,11 +1198,29 @@ QString SDatagroup::SNickname::UniCodeToEucKr(const QString& _strData)
QString SDatagroup::SCommunity::getCommunityKey(const QStringList &_strList)
{
QString str;
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_form]].trimmed());
str.append(".");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed());
str.append(".com/");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed());
if (_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed() == "naver" || _strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed() == "daum")
{
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_form]].trimmed());
str.append(".");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed());
str.append(".com/");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed());
}
else if (_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed() == "instagram")
{
str.append("instagram.com/");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed());
}
else if (_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed() == "kakaostory")
{
str.append("story.kakao.com/");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed());
}
else if (_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_name]].trimmed() == "facebook")
{
str.append("facebook.com/");
str.append(_strList[m_pSInitializer->getColumnIntArray()[SInitializer::E_DATA_platform_id]].trimmed());
}
return str.trimmed();
}

36
sfilterprocess/sdatagroup.h
View File

@@ -11,6 +11,7 @@ class QStringList;
class QSqlDatabase;
QString sqlString(QString _str);
QString UniCodeToEucKr(const QString& _strData);
class SDatagroup
{
@@ -35,12 +36,24 @@ public:
QString url;
int view;
QString date;
int reply_startnum;
int reply_endnum;
int reply_count;
QString lastupdate;
int replystartnum;
int replyendnum;
int replycount;
QVector<int> realbycategorynum;
stBody():reply_endnum(-1), reply_startnum(-1), reply_count(0) {}
QString lastupdate;
QString community_title;
QString community_url;
QString community_id;
QString community_exid;
QString platformname_name;
QString platformform_form;
QString nickname_nickname;
QString nickname_exnickname;
//QSet<int> realbycategorynum;
stBody():replyendnum(-1), replystartnum(-1), replycount(0) {}
};
struct stReply
@@ -55,6 +68,17 @@ public:
QString* data;
QString date;
QVector<int> realbycategorynum;
int platformname_num;
int platformform_num;
QString platformname_name;
QString platformform_form;
QString nickname_nickname;
QString nickname_exnickname;
QString body_date;
//QSet<int> realbycategorynum;
stReply():num(0),id_num(0),nickname_num(0),community_num(0),body_num(0),platformname_num(0),platformform_num(0){}
};
struct stCommunity
{
@@ -124,6 +148,7 @@ public:
bool changeTable(QSqlDatabase &_db, const int& nCompany);
int getBodyNum(const QString& _url);
QString vecToString(const QVector<int>& _realbycategorynum);
QString vecToString(const QSet<int> &_realbycategorynum);
void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; }
stBody* get(const QString& _url);
void clear();
@@ -176,7 +201,6 @@ public:
void put(const QString& _strKey, const stNickname& _stnickname);
void put(const stNickname& _nickname);
bool contains(const QString& _strKey);
QString UniCodeToEucKr(const QString& _strData);
void setInitializer(SInitializer &initializer){ m_pSInitializer = &initializer; }
void clear();
};

50
sfilterprocess/sfilterprocess.cpp
View File

@@ -40,14 +40,45 @@ bool SFilterProcess::run()
}
case E_PROCSS_TYPE_CATEGORY_MAKE:
{
m_SConsumer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
switch(process.m_nServiceNum)
{
case E_SERVICE_INFLUENCER:
{
qDebug() << "Influencer:" << process.m_nCategoryNum;
m_SInfluencer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
case E_SERVICE_CONSUMER:
{
qDebug() << "Consumer:" << process.m_nCategoryNum;
m_SConsumer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
case E_SERVICE_SPAMMER:
{
qDebug() << "Spammer:" << process.m_nCategoryNum;
m_SSpammer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
case E_SERVICE_POWERCAFE:
{
qDebug() << "Powercafe:" << process.m_nCategoryNum;
m_SPowercafe.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
}
break;
}
case E_PROCSS_TYPE_SEND:
{
m_SDatagroup.makeTable();
m_SConsumer.makeOverallCategory(m_SInitializer.getConsumerAllCategory());
m_SConsumer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_CONSUMER));
m_SInfluencer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_INFLUENCER));
m_SSpammer.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_SPAMMER));
m_SPowercafe.makeOverallCategory(m_SInitializer.getAllCategoryNum(E_SERVICE_POWERCAFE));
m_SConsumer.makeTable(m_SDatagroup);
m_SInfluencer.makeTable(m_SDatagroup);
m_SSpammer.makeTable(m_SDatagroup);
@@ -72,21 +103,6 @@ bool SFilterProcess::run()
process.m_nCompanyNum);
break;
}
case E_PROCSS_TYPE_INFLUENCE_COUNT:
{
m_SInfluencer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
case E_PROCSS_TYPE_SPAMMERS_COUNT:
{
m_SSpammer.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
case E_PROCSS_TYPE_POWER_CAFE_COUNT:
{
m_SPowercafe.makeCategory(m_SDatagroup, process.m_nCategoryNum, m_SInitializer.getFilterGroup(process.m_nFilterGroupID));
break;
}
}
}
m_SInitializer.insertLog("New Filter END");

8
sfilterprocess/sfilterprocess.h
View File

@@ -19,6 +19,14 @@ private:
SPowercafe m_SPowercafe;
public:
enum E_SERVICE
{
E_SERVICE_INFLUENCER = 1,
E_SERVICE_CONSUMER,
E_SERVICE_SPAMMER,
E_SERVICE_POWERCAFE
};
bool init(QString _strFilterProcessGroupID, QString _strFilterProcessGroupName);
bool run();

4
sfilterprocess/sfilterprocess.pro.user
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2016-03-22T15:30:38. -->
<!-- Written by QtCreator 3.3.0, 2016-05-03T18:01:56. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>
@@ -787,7 +787,7 @@
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">sfilterprocess</value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value>
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.Qt4RunConfiguration:C:/source/sfilterprocess/sfilterprocess.pro</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;370&quot; &quot;test&quot;</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;427&quot; &quot;test&quot;</value>
<value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.ProFile">sfilterprocess.pro</value>
<value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseDyldImageSuffix">false</value>
<value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseTerminal">true</value>

235
sfilterprocess/sinfluencer.cpp
View File

@@ -61,14 +61,20 @@ bool SInfluencer::changeTable(int _companyNum)
QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB);
if (!m_SBodyInfluencerMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload influencer body map");
m_pSInitializer->insertLog("Fail to change influencer body map");
return false;
}
if (!m_SReplyInfluencerMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload influencer reply map");
m_pSInitializer->insertLog("Fail to change influencer reply map");
return false;
}
if (!m_STInfluencer.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to change influencer");
return false;
}
return true;
}
@@ -105,8 +111,8 @@ bool SInfluencer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QLi
}
}
stReal streal;
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size();
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size();
m_pSInitializer->insertLog(QString("CategoryNum:") + QString::number(_nCategory) + ",(" +
QString::number(vecData[SInitializer::E_ARTICLE_BODY].size()) + '/' + QString::number(vecData[SInitializer::E_ARTICLE_REPLY].size()) + ")" + "(Body/Reply)" );
const int *anColumn = m_pSInitializer->getColumnIntArray();
if (bFiltered[SInitializer::E_ARTICLE_BODY])
{
@@ -135,22 +141,46 @@ bool SInfluencer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QLi
bool SInfluencer::makeOverallCategory(int _categoryNum)
{
if (m_mapCategory.size() < 1)
return true;
stReal streal;
for (QMap<int, stReal>::iterator iterPos = m_mapCategory.begin(); iterPos != m_mapCategory.end(); iterPos++ )
{
stReal& val = iterPos.value();
foreach (const QString& _strurl, val.m_setRealBodyUrl )
streal.m_setRealBodyUrl.insert(_strurl);
foreach (const QString& _strurl, val.m_setRealAllUrl )
streal.m_setRealAllUrl.insert(_strurl);
for (QMap<QString, int>::iterator iterPos1 = val.m_mapRealReplyUniqueUrl.begin(); iterPos1 != val.m_mapRealReplyUniqueUrl.end();
iterPos1++)
{
streal.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value());
}
}
m_mapCategory.insert(_categoryNum, streal);
return true;
}
bool SInfluencer::makeTable(SDatagroup &_datagroup)
{
m_pSInitializer->insertLog("Make influencer tables");
m_pSInitializer->insertLog("Make influencer body tables");
m_SBodyInfluencerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer reply tables");
m_SReplyInfluencerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_buzzy tables");
m_SStatsInfBloggerBuzzbyday.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_rank table");
m_SStatsInfBloggerRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer influencer");
m_STInfluencer.makeTable(_datagroup, m_mapCategory);
if (m_mapCategory.size() > 1)
{
m_pSInitializer->insertLog("Make influencer tables");
m_pSInitializer->insertLog("Make influencer body tables");
m_SBodyInfluencerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer reply tables");
m_SReplyInfluencerMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_buzzy tables");
m_SStatsInfBloggerBuzzbyday.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer stats_inf_blogger_rank table");
m_SStatsInfBloggerRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make influencer influencer");
m_STInfluencer.makeTable(_datagroup, m_mapCategory);
}
return true;
}
@@ -159,12 +189,12 @@ void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMap<int
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
const QMap<QString, QMap<int, QStringList> >* mapReply = _datagroup.getmapReply();
const QStringList* orderedUrl = _datagroup.getOrderedUrl();
QMap<SDatagroup::stProfileurlKey, stInfluencer> mapRank;
const int* anColumn = m_pSInitializer->getColumnIntArray();
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
QMap<SDatagroup::stProfileurlKey, stInfluencer> mapRank;
const stReal& streal = iterPos.value();
foreach (const QString& _url, *orderedUrl)
{
@@ -182,6 +212,14 @@ void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMap<int
mapRank[stprofileurlkey].community_num = _datagroup.getstBody(_url)->community_num;
mapRank[stprofileurlkey].profile = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profile]].trimmed();
mapRank[stprofileurlkey].profileurl = (*(_datagroup.getmapBody(stprofileurlkey)))[anColumn[SInitializer::E_DATA_article_profileurl]].trimmed();
mapRank[stprofileurlkey].category_num = iterPos.key();
mapRank[stprofileurlkey].community_title = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->title;
mapRank[stprofileurlkey].community_url = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->url;
mapRank[stprofileurlkey].community_id = _datagroup.getstCommunity(*(_datagroup.getmapBody(stprofileurlkey)))->id;
mapRank[stprofileurlkey].nickname_nickname = _datagroup.getstBody(_url)->nickname_nickname;
mapRank[stprofileurlkey].nickname_exnickname = _datagroup.getstBody(_url)->nickname_exnickname;
mapRank[stprofileurlkey].platformname_num = _datagroup.getstBody(_url)->platformname_num;
mapRank[stprofileurlkey].platformname_name = _datagroup.getstBody(_url)->platformname_name;
}
mapRank[stprofileurlkey].bodycount++;
@@ -195,22 +233,19 @@ void SInfluencer::STInfluencer::makeTable(SDatagroup &_datagroup, const QMap<int
}
}
}
}
for (QMap<SDatagroup::stProfileurlKey, stInfluencer>::iterator iterPos = mapRank.begin(); iterPos != mapRank.end(); iterPos++)
{
stInfluencer &val = iterPos.value();
val.influencervalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1);
}
m_listMap = mapRank.values();
qSort(m_listMap.begin(), m_listMap.end(), isGreaterInfluencerValue());
int rank = 1;
for (QList<stInfluencer>::iterator iterPos = m_listMap.begin(); iterPos != m_listMap.end(); iterPos++)
{
iterPos->rank = rank++;
for (QMap<SDatagroup::stProfileurlKey, stInfluencer>::iterator iterPos1 = mapRank.begin(); iterPos1 != mapRank.end(); iterPos1++)
{
stInfluencer &val = iterPos1.value();
val.influencervalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1);
}
QList<stInfluencer> listmap = mapRank.values();
qSort(listmap.begin(), listmap.end(), isGreaterInfluencerValue());
int rank = 1;
for (QList<stInfluencer>::iterator iterPos1 = listmap.begin(); iterPos1 != listmap.end(); iterPos1++)
{
iterPos1->rank = rank++;
}
m_listMap.append(listmap);
}
}
@@ -264,7 +299,7 @@ bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompa
bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
QString strQuery = "delete from influencer where company_num = " + QString::number(_nCompany);
QString strQuery = "delete from influencer_" + QString::number(_nCompany) + "_update";
QSqlQuery query(_db);
if (query.exec(strQuery.toUtf8()) == false)
{
@@ -272,8 +307,9 @@ bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompa
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
strQuery = "insert into influencer (rank,company_num,id_num,nickname_num,community_num,bodycount,replycount,influencervalue,profile,profileurl,id_id)"
" VALUES (";
strQuery = "insert into influencer_" + QString::number(_nCompany) +
"_update (rank,category_num,id_num,nickname_num,community_num,bodycount,replycount,influencervalue,profile,profileurl,id_id,"
"community_title,community_url,community_id,nickname_nickname,nickname_exnickname,platformname_num,platformname_name) VALUES (";
query.clear();
@@ -281,7 +317,7 @@ bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompa
{
QString str = strQuery;
str += QString::number(stinfluencer.rank) + ",";
str += QString::number(_nCompany) + ",";
str += QString::number(stinfluencer.category_num) + ",";
str += QString::number(stinfluencer.id_num) + ",";
str += QString::number(stinfluencer.nickname_num) + ",";
str += QString::number(stinfluencer.community_num) + ",";
@@ -290,7 +326,15 @@ bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompa
str += QString::number(stinfluencer.influencervalue, 'g', 3) + ",";
str += "'" + sqlString(stinfluencer.profile).trimmed().replace(QChar(160), "") + "',";
str += "'" + stinfluencer.profileurl.trimmed() + "',";
str += "'" + sqlString(stinfluencer.id_id) + "')";
str += "'" + sqlString(stinfluencer.id_id) + "',";
str += "'" + sqlString(stinfluencer.community_title) + "',";
str += "'" + stinfluencer.community_url + "',";
str += "'" + stinfluencer.community_id + "',";
str += "'" + sqlString(stinfluencer.nickname_nickname) + "',";
str += "'" + sqlString(stinfluencer.nickname_exnickname) + "',";
str += QString::number(stinfluencer.platformname_num) + ",";
str += "'" + stinfluencer.platformname_name + "')";
if (query.exec(str.toUtf8()) == false)
{
m_pSInitializer->insertLog(query.lastQuery());
@@ -306,6 +350,23 @@ bool SInfluencer::STInfluencer::uploadData(QSqlDatabase &_db, const int &_nCompa
return true;
}
bool SInfluencer::STInfluencer::changeTable(QSqlDatabase &_db, const int &_nCompany)
{
QString strQuery;
QSqlQuery query(_db);
strQuery = "RENAME TABLE ";
strQuery += "influencer_" + QString::number(_nCompany) + " TO " + "influencer_" + QString::number(_nCompany) + "_temp" + ",";
strQuery += "influencer_" + QString::number(_nCompany) + "_update" + " TO " + "influencer_" + QString::number(_nCompany) + ",";
strQuery += "influencer_" + QString::number(_nCompany) + "_temp" + " TO " + "influencer_" + QString::number(_nCompany) + "_update";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
return true;
}
void SInfluencer::SStatsInfBloggerRank::makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
@@ -313,11 +374,11 @@ void SInfluencer::SStatsInfBloggerRank::makeTable(SDatagroup &_datagroup, const
const int* anColumn = m_pSInitializer->getColumnIntArray();
const int RANK = 10;
QMap<SDatagroup::stProfileurlKey, stStatsInfBloggerRank> mapRank;
for (QMap<int, stReal>::const_iterator iterPos = _mapCategory.constBegin(); iterPos != _mapCategory.constEnd(); iterPos++)
{
const stReal& streal = iterPos.value();
QMap<SDatagroup::stProfileurlKey, stStatsInfBloggerRank> mapRank;
foreach (const QString& _url, streal.m_setRealBodyUrl)
{
@@ -342,46 +403,46 @@ void SInfluencer::SStatsInfBloggerRank::makeTable(SDatagroup &_datagroup, const
}
}
}
}
for (QMap<SDatagroup::stProfileurlKey, stStatsInfBloggerRank>::iterator iterPos = mapRank.begin(); iterPos != mapRank.end(); iterPos++)
{
stStatsInfBloggerRank &val = iterPos.value();
val.influencevalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1);
}
for (QMap<SDatagroup::stProfileurlKey, stStatsInfBloggerRank>::iterator iterPos = mapRank.begin(); iterPos != mapRank.end(); iterPos++)
{
stStatsInfBloggerRank &val = iterPos.value();
val.influencevalue = val.bodycount * m_pSInitializer->getInfluencerParam(0) + val.replycount * m_pSInitializer->getInfluencerParam(1);
}
QList<stStatsInfBloggerRank> listMap = mapRank.values();
QList<stStatsInfBloggerRank> listMap = mapRank.values();
qSort(listMap.begin(), listMap.end(), isGreaterInfluencerValue());
// influencevalue
for(int i = 0; i < RANK; i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "influencevalue";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
}
qSort(listMap.begin(), listMap.end(), isGreaterInfluencerValue());
// influencevalue
for(int i = 0; i < RANK && i < listMap.size(); i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "influencevalue";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
}
qSort(listMap.begin(), listMap.end(), isGreaterBodyCount());
// influencevalue
for(int i = 0; i < RANK; i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "bodycount";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
}
qSort(listMap.begin(), listMap.end(), isGreaterBodyCount());
// influencevalue
for(int i = 0; i < RANK && i < listMap.size(); i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "bodycount";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
}
qSort(listMap.begin(), listMap.end(), isGreaterReplyCount());
// influencevalue
for(int i = 0; i < RANK; i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "replycount";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
qSort(listMap.begin(), listMap.end(), isGreaterReplyCount());
// influencevalue
for(int i = 0; i < RANK && i < listMap.size(); i++)
{
listMap[i].rank = i + 1;
listMap[i].subject = "replycount";
m_listMap.append(listMap.at(i));
//qDebug() << listMap[i].id_id;
}
listMap.clear();
}
listMap.clear();
}
/*
@@ -624,17 +685,17 @@ void SInfluencer::SBodyInfluencerMap::makeTable(SDatagroup &_datagroup, const QM
stbody.id_num = _datagroup.getstBody(strUrl)->id_num;
stbody.category_num = iterPos.key();
stbody.realin |= REAL_BODY;
stbody.community_num = _datagroup.getstBody(strUrl)->community_num;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
m_listMap.append(stbody);
//_datagroup.addRealBodyCategory(strUrl, iterPos.key());
_datagroup.addRealBodyCategory(strUrl, iterPos.key());
if (streal.m_mapRealReplyUniqueUrl.contains(strUrl))
{
stbody.realin |= REAL_REPLY;
/*
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
*/
}
}
}
@@ -687,7 +748,7 @@ bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)"
strQuery = "insert into body_influencer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,community_num,body_date)"
" values (";
query.clear();
@@ -699,7 +760,9 @@ bool SInfluencer::SBodyInfluencerMap::uploadData(QSqlDatabase &_db, const int &_
str += QString::number(stmap.category_num) + ",";
str += QString::number(stmap.id_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.realin) + ")";
str += QString::number(stmap.realin) + ",";
str += QString::number(stmap.community_num) + ",";
str += "'" + stmap.body_date + "')";
if (query.exec(str.toUtf8()) == false)
{
@@ -734,14 +797,11 @@ void SInfluencer::SReplyInfluencerMap::makeTable(SDatagroup &_datagroup, const Q
streply.realin |= REAL_REPLY;
streply.id_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num;
streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num;
streply.community_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->community_num;
streply.body_date = _datagroup.getstBody(iterPos1.key())->date;
m_listMap.append(streply);
/*
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
*/
_datagroup.addRealReplyCategory(iterPos1.key(), iterPos1.value(), iterPos.key());
}
}
else
@@ -757,6 +817,9 @@ void SInfluencer::SReplyInfluencerMap::makeTable(SDatagroup &_datagroup, const Q
streply.realin |= REAL_REPLY;
streply.id_num = _datagroup.getstReply(_url, iterPos1.key())->id_num;
streply.reply_num = _datagroup.getstReply(_url, iterPos1.key())->num;
streply.community_num = _datagroup.getstReply(_url, iterPos1.key())->community_num;
streply.body_date = _datagroup.getstBody(_url)->date;
_datagroup.addRealReplyCategory(iterPos1.value()[SInitializer::E_DATA_article_url].trimmed(), iterPos1.key(), iterPos.key());
m_listMap.append(streply);
}
}
@@ -811,7 +874,7 @@ bool SInfluencer::SReplyInfluencerMap::uploadData(QSqlDatabase &_db, const int &
return false;
}
strQuery = "insert into reply_influencer_map_" + QString::number(_nCompany) +
"_update (num,category_num,id_num,body_num,reply_num,realin)"
"_update (num,category_num,id_num,body_num,reply_num,realin,community_num,body_date)"
" values (";
query.clear();
@@ -823,7 +886,9 @@ bool SInfluencer::SReplyInfluencerMap::uploadData(QSqlDatabase &_db, const int &
str += QString::number(stmap.id_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.reply_num) + ",";
str += QString::number(stmap.realin) + ")";
str += QString::number(stmap.realin) + ",";
str += QString::number(stmap.community_num) + ",";
str += "'" + stmap.body_date + "')";
if (query.exec(str.toUtf8()) == false)
{
m_pSInitializer->insertLog(query.lastQuery());

34
sfilterprocess/sinfluencer.h
View File

@@ -20,7 +20,7 @@ public:
int id_num;
int nickname_num;
int community_num;
int company_num;
int category_num;
QString id_id;
QString profile;
QString profileurl;
@@ -28,11 +28,15 @@ public:
int replycount;
double influencervalue;
int rank;
stInfluencer()
{
id_num = nickname_num = community_num = company_num = bodycount = replycount = rank = 0;
influencervalue = 0.0;
}
QString community_title;
QString community_url;
QString community_id;
QString nickname_nickname;
QString nickname_exnickname;
QString platformname_name;
int platformname_num;
stInfluencer():id_num(0),nickname_num(0),community_num(0),category_num(0),bodycount(0),replycount(0),rank(0),influencervalue(0.0),platformname_num(0){}
};
struct stStatsInfBloggerRank
@@ -45,11 +49,7 @@ public:
QString subject;
int company_num;
int rank;
stStatsInfBloggerRank()
{
bodycount = replycount = rank = 0;
influencevalue = 0.0;
}
stStatsInfBloggerRank():bodycount(0),replycount(0),rank(0),influencevalue(0.0){}
stStatsInfBloggerRank(const QString &_id_id, const QString &_nickname_nickname):id_id(_id_id),nickname_nickname(_nickname_nickname),
bodycount(0), replycount(0), rank(0), influencevalue(0.0) {}
};
@@ -72,7 +72,9 @@ public:
int id_num;
int category_num;
int realin;
stBodyInfluencerMap():num(0),body_num(0), id_num(0), category_num(0), realin(0) {}
int community_num;
QString body_date;
stBodyInfluencerMap():num(0),body_num(0), id_num(0), category_num(0), realin(0), community_num(0){}
};
struct stReplyInfluencerMap
{
@@ -82,10 +84,9 @@ public:
int id_num;
int category_num;
int realin;
stReplyInfluencerMap()
{
num = reply_num = body_num = id_num = category_num = realin = 0;
}
int community_num;
QString body_date;
stReplyInfluencerMap():num(0),reply_num(0),body_num(0),id_num(0),category_num(0),realin(0),community_num(0){}
};
class STInfluencer
{
@@ -96,6 +97,7 @@ public:
void setInitiaizer(SInitializer &_initializer){ m_pSInitializer = &_initializer; }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase &_db, const int &_nCompany);
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
void clear(){ m_listMap.clear(); }
};

85
sfilterprocess/sinitializer.cpp
View File

@@ -214,7 +214,8 @@ bool SInitializer::init(const QString& _strFilterProcessGroupID, const QString&
b_ok &= initFilterProcess();
b_ok &= initCompanyNum();
b_ok &= initPlatform();
b_ok &= initConsumerCategory();
//b_ok &= initConsumerCategory();
b_ok &= initCategory();
/*
if (!b_ok)
cout << "FALSE" << endl;
@@ -234,7 +235,7 @@ bool SInitializer::initInfluencer()
bool SInitializer::initFilterProcess()
{
QString strQuery = QString("select filterprocess_type,datagroup_id,filtergroup_id,company_num,category_num,filterprocess_order "
QString strQuery = QString("select filterprocess_type,datagroup_id,filtergroup_id,company_num,category_num,filterprocess_order,service_num "
"from filterprocess where filterprocessgroup_id = ") + QString::number(m_stFilterProcessGroup.m_nFilterProcessGroupID);
QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_DATA]);
@@ -255,6 +256,7 @@ bool SInitializer::initFilterProcess()
stfilterprocess.m_nCompanyNum = query.value(3).toInt();
stfilterprocess.m_nCategoryNum = query.value(4).toInt();
stfilterprocess.m_nFilterProcessOrder = query.value(5).toInt();
stfilterprocess.m_nServiceNum = query.value(6).toInt();
m_listFilterProcess.append(stfilterprocess);
}
@@ -288,7 +290,79 @@ bool SInitializer::initFilterProcess()
return true;
}
bool SInitializer::initCategory()
{
for (int i = 0; i < E_SERVICE_MAX; i++)
{
m_anAllCategory[i] = -1;
}
int company_num = -1;
foreach (const stFilterProcess& stfilterprocess, m_listFilterProcess)
{
if (stfilterprocess.m_nFilterProcessType == 2)
{
company_num = stfilterprocess.m_nCompanyNum;
}
}
if (company_num == -1)
return false;
QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]);
QString strQuery = "select num, servicenum from category where company_num = ";
strQuery += QString::number(company_num) + " order by servicenum asc, num asc";
if (query.exec(strQuery.toUtf8()) == false)
{
insertLog(query.lastQuery());
insertLog(query.lastError().text());
return false;
}
int nService[E_SERVICE_MAX] = {0,};
while(query.next())
{
switch(query.value(1).toInt())
{
case E_SERVICE_INFLUENCER:
{
if (0 == nService[E_SERVICE_INFLUENCER]++)
{
m_anAllCategory[E_SERVICE_INFLUENCER] = query.value(0).toInt();
}
break;
}
case E_SERVICE_CONSUMER:
{
if (0 == nService[E_SERVICE_CONSUMER]++)
{
m_anAllCategory[E_SERVICE_CONSUMER] = query.value(0).toInt();
}
break;
}
case E_SERVICE_SPAMMER:
{
if (0 == nService[E_SERVICE_SPAMMER]++)
{
m_anAllCategory[E_SERVICE_SPAMMER] = query.value(0).toInt();
}
break;
}
case E_SERVICE_POWERCAFE:
{
if (0 == nService[E_SERVICE_POWERCAFE]++)
{
m_anAllCategory[E_SERVICE_POWERCAFE] = query.value(0).toInt();
}
break;
}
}
}
return true;
}
/*
bool SInitializer::initConsumerCategory()
{
QSqlQuery query(m_sDBManager[SDBManager::E_DATABASE_WEB]);
@@ -330,6 +404,7 @@ bool SInitializer::initConsumerCategory()
}
return true;
}
*/
int SInitializer::getConsumerAllCategory()
{
@@ -525,3 +600,9 @@ bool SInitializer::openConnection(const QString& _str)
{
return m_sDBManager.open(_str);
}
int SInitializer::getAllCategoryNum(int _nService)
{
return m_anAllCategory[_nService];
}

14
sfilterprocess/sinitializer.h
View File

@@ -26,7 +26,8 @@ private:
bool initInfluencer();
bool initSpammer();
bool initPlatform();
bool initConsumerCategory();
//bool initConsumerCategory();
bool initCategory();
public:
@@ -79,6 +80,14 @@ public:
E_DATA_etc,
E_DATA_MAX,
};
enum E_SERVICE
{
E_SERVICE_INFLUENCER = 1,
E_SERVICE_CONSUMER,
E_SERVICE_SPAMMER,
E_SERVICE_POWERCAFE,
E_SERVICE_MAX
};
struct stDataBaseInfo
{
@@ -128,6 +137,7 @@ public:
int m_nFilterProcessOrder;
int m_nFilterProcessType;
int m_nCompanyNum;
int m_nServiceNum;
};
struct stFilter
@@ -165,6 +175,7 @@ private:
stPlatform m_stPlatform;
QMap<int, int> m_mapConsumerCategory;
int m_nConsumerAllCategory;
int m_anAllCategory[E_SERVICE_MAX];
public:
bool init(const QString& _strFilterProcessGroupID, const QString& _strFilterProcessGroupName);
@@ -191,6 +202,7 @@ public:
double getInfluencerParam(int _BodyorReply);
void updateWebDBInfoState(QSqlDatabase &_db, int _nCompany, QString _strStatus);
void updateWebDBInfoComplete(QSqlDatabase _db, QString _mindate, QString _maxdate, QString _lastupdate, int _bodycount, int _replycount, int _nCompany);
int getAllCategoryNum(int _nService);
//bool disableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE);
//bool enableIndex(QSqlDatabase& db, QString _tableName, QStringList _strIndexNames, int _nMode = MODE);
//bool disableSettings(QSqlDatabase& db, int _nMode = MODE);

128
sfilterprocess/spowercafe.cpp
View File

@@ -53,14 +53,20 @@ bool SPowercafe::changeTable(int _companyNum)
QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB);
if (!m_SBodyPowercafeMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload powercafe body map");
m_pSInitializer->insertLog("Fail to change powercafe body map");
return false;
}
if (!m_SReplyPowercafeMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload powercafe reply map");
m_pSInitializer->insertLog("Fail to change powercafe reply map");
return false;
}
if (!m_STPowercafe.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to change powercafe");
return false;
}
return true;
}
@@ -97,8 +103,8 @@ bool SPowercafe::makeCategory(SDatagroup &_datagroup, int _nCategory, const QLis
}
}
stReal streal;
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size();
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size();
m_pSInitializer->insertLog(QString("CategoryNum:") + QString::number(_nCategory) + ",(" +
QString::number(vecData[SInitializer::E_ARTICLE_BODY].size()) + '/' + QString::number(vecData[SInitializer::E_ARTICLE_REPLY].size()) + ")" + "(Body/Reply)" );
const int *anColumn = m_pSInitializer->getColumnIntArray();
if (bFiltered[SInitializer::E_ARTICLE_BODY])
{
@@ -131,20 +137,43 @@ bool SPowercafe::makeCategory(SDatagroup &_datagroup, int _nCategory, const QLis
bool SPowercafe::makeOverallCategory(int _categoryNum)
{
if (m_mapCategory.size() < 1)
return true;
stReal streal;
for (QMap<int, stReal>::iterator iterPos = m_mapCategory.begin(); iterPos != m_mapCategory.end(); iterPos++ )
{
stReal& val = iterPos.value();
foreach (const QString& _strurl, val.m_setRealBodyUrl )
streal.m_setRealBodyUrl.insert(_strurl);
foreach (const QString& _strurl, val.m_setRealAllUrl )
streal.m_setRealAllUrl.insert(_strurl);
for (QMap<QString, int>::iterator iterPos1 = val.m_mapRealReplyUniqueUrl.begin(); iterPos1 != val.m_mapRealReplyUniqueUrl.end();
iterPos1++)
{
streal.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value());
}
}
m_mapCategory.insert(_categoryNum, streal);
return true;
}
bool SPowercafe::makeTable(SDatagroup &_datagroup)
{
m_pSInitializer->insertLog("Make powercafe tables");
m_pSInitializer->insertLog("Make powercafe body tables");
m_SBodyPowercafeMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe reply tables");
m_SReplyPowercafeMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe stats_pow_cafe_rank tables");
m_SStatsPowCafeRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe powercafe table");
m_STPowercafe.makeTable(_datagroup, m_mapCategory);
if (m_mapCategory.size() > 1)
{
m_pSInitializer->insertLog("Make powercafe tables");
m_pSInitializer->insertLog("Make powercafe body tables");
m_SBodyPowercafeMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe reply tables");
m_SReplyPowercafeMap.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe stats_pow_cafe_rank tables");
m_SStatsPowCafeRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make powercafe powercafe table");
m_STPowercafe.makeTable(_datagroup, m_mapCategory);
}
return true;
}
@@ -210,6 +239,12 @@ void SPowercafe::STPowercafe::makeTable(SDatagroup &_datagroup, const QMap<int,
mapPowercafeReal[strKey].replycountall = mapPowercafeAll[strKey].replycountall;
mapPowercafeReal[strKey].allcountall = mapPowercafeAll[strKey].bodycountall + mapPowercafeAll[strKey].replycountall;
mapPowercafeReal[strKey].usercountall = mapUserAll[strKey].size();
mapPowercafeReal[strKey].category_num = iterPos.key();
mapPowercafeReal[strKey].community_title = _datagroup.getstCommunity((*mapBody)[url])->title;
mapPowercafeReal[strKey].community_url = _datagroup.getstCommunity((*mapBody)[url])->url;
mapPowercafeReal[strKey].platformname_name = (*mapBody)[url][anColumn[SInitializer::E_DATA_platform_name]].trimmed();
mapPowercafeReal[strKey].platformname_num = m_pSInitializer->getPlatformName(mapPowercafeReal[strKey].platformname_name);
mapUserReal.insert(strKey, QSet<QString>());
}
mapPowercafeReal[strKey].bodycountreal++;
@@ -228,6 +263,12 @@ void SPowercafe::STPowercafe::makeTable(SDatagroup &_datagroup, const QMap<int,
mapPowercafeReal[strKey].replycountall = mapPowercafeAll[strKey].replycountall;
mapPowercafeReal[strKey].allcountall = mapPowercafeAll[strKey].bodycountall + mapPowercafeAll[strKey].replycountall;
mapPowercafeReal[strKey].usercountall = mapUserAll[strKey].size();
mapPowercafeReal[strKey].category_num = iterPos.key();
mapPowercafeReal[strKey].community_title = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->title;
mapPowercafeReal[strKey].community_url = _datagroup.getstCommunity((*mapReply)[iterPos1.key()][iterPos1.value()])->url;
mapPowercafeReal[strKey].platformname_name = (*mapReply)[iterPos1.key()][iterPos1.value()][anColumn[SInitializer::E_DATA_platform_name]].trimmed();
mapPowercafeReal[strKey].platformname_num = m_pSInitializer->getPlatformName(mapPowercafeReal[strKey].platformname_name);
mapUserReal.insert(strKey, QSet<QString>());
}
mapPowercafeReal[strKey].replycountreal++;
@@ -298,7 +339,7 @@ bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany
bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
QString strQuery = "delete from powercafe where company_num = " + QString::number(_nCompany);
QString strQuery = "delete from powercafe_" + QString::number(_nCompany) + "_update";
QSqlQuery query(_db);
if (query.exec(strQuery.toUtf8()) == false)
{
@@ -306,8 +347,8 @@ bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
strQuery = "insert into powercafe (company_num,rank,community_num,community_id,bodycountall,replycountall,allcountall,usercountall,"
"bodycountreal,replycountreal,allcountreal,usercountreal)"
strQuery = "insert into powercafe_" + QString::number(_nCompany) + "_update (category_num,rank,community_num,community_id,bodycountall,replycountall,allcountall,usercountall,"
"bodycountreal,replycountreal,allcountreal,usercountreal,community_title,community_url,platformname_num,platformname_name)"
" VALUES (";
query.clear();
@@ -315,7 +356,7 @@ bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany
foreach (const stPowercafe& stpowercafe, m_listMap)
{
QString str = strQuery;
str += QString::number(_nCompany) + ",";
str += QString::number(stpowercafe.category_num) + ",";
str += QString::number(stpowercafe.rank) + ",";
str += QString::number(stpowercafe.community_num) + ",";
str += "'" + sqlString(stpowercafe.community_id) + "',";
@@ -326,7 +367,11 @@ bool SPowercafe::STPowercafe::uploadData(QSqlDatabase &_db, const int &_nCompany
str += QString::number(stpowercafe.bodycountreal) + ",";
str += QString::number(stpowercafe.replycountreal) + ",";
str += QString::number(stpowercafe.allcountreal) + ",";
str += QString::number(stpowercafe.usercountreal) + ")";
str += QString::number(stpowercafe.usercountreal) + ",";
str += "'" + sqlString(stpowercafe.community_title) + "',";
str += "'" + stpowercafe.community_url + "',";
str += QString::number(stpowercafe.platformname_num) + ",";
str += "'" + stpowercafe.platformname_name + "')";
if (query.exec(str.toUtf8()) == false)
{
@@ -442,6 +487,25 @@ void SPowercafe::SStatsPowCafeRank::makeTable(SDatagroup &_datagroup, const QMap
}
}
bool SPowercafe::STPowercafe::changeTable(QSqlDatabase &_db, const int &_nCompany)
{
QString strQuery;
QSqlQuery query(_db);
strQuery = "RENAME TABLE ";
strQuery += "powercafe_" + QString::number(_nCompany) + " TO " + "powercafe_" + QString::number(_nCompany) + "_temp" + ",";
strQuery += "powercafe_" + QString::number(_nCompany) + "_update" + " TO " + "powercafe_" + QString::number(_nCompany) + ",";
strQuery += "powercafe_" + QString::number(_nCompany) + "_temp" + " TO " + "powercafe_" + QString::number(_nCompany) + "_update";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
return true;
}
/*
bool SPowercafe::SStatsPowCafeRank::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
@@ -535,17 +599,19 @@ void SPowercafe::SBodyPowercafeMap::makeTable(SDatagroup &_datagroup, const QMap
stbody.community_num = _datagroup.getstBody(strUrl)->community_num;
stbody.category_num = iterPos.key();
stbody.realin |= REAL_BODY;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num;
m_listMap.append(stbody);
//_datagroup.addRealBodyCategory(strUrl, iterPos.key());
_datagroup.addRealBodyCategory(strUrl, iterPos.key());
if (streal.m_mapRealReplyUniqueUrl.contains(strUrl))
{
stbody.realin |= REAL_REPLY;
/*
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
*/
}
}
}
@@ -568,14 +634,10 @@ void SPowercafe::SReplyPowercafeMap::makeTable(SDatagroup &_datagroup, const QMa
streply.realin |= REAL_REPLY;
streply.community_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->community_num;
streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num;
streply.body_date = _datagroup.getstBody(iterPos1.key())->date;
m_listMap.append(streply);
/*
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
*/
_datagroup.addRealReplyCategory(iterPos1.key(), iterPos1.value(), iterPos.key());
}
}
}
@@ -626,7 +688,7 @@ bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nC
return false;
}
strQuery = "insert into body_powercafe_map_" + QString::number(_nCompany)
+ "_update (num,category_num,community_num,body_num,realin)"
+ "_update (num,category_num,community_num,body_num,realin,body_date,platformname_num)"
" values (";
query.clear();
@@ -637,7 +699,10 @@ bool SPowercafe::SBodyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_nC
str += QString::number(stmap.category_num) + ",";
str += QString::number(stmap.community_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.realin) + ")";
str += QString::number(stmap.realin) + ",";
str += "'" + stmap.body_date + "',";
str += QString::number(stmap.platformname_num) + ")";
if (query.exec(str.toUtf8()) == false)
{
@@ -698,7 +763,7 @@ bool SPowercafe::SReplyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_n
return false;
}
strQuery = "insert into reply_powercafe_map_" + QString::number(_nCompany)
+ "_update (num,category_num,community_num,body_num,reply_num,realin)"
+ "_update (num,category_num,community_num,body_num,reply_num,realin,body_date)"
" values (";
query.clear();
@@ -710,7 +775,8 @@ bool SPowercafe::SReplyPowercafeMap::uploadData(QSqlDatabase &_db, const int &_n
str += QString::number(stmap.community_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.reply_num) + ",";
str += QString::number(stmap.realin) + ")";
str += QString::number(stmap.realin) + ",";
str += "'" + stmap.body_date + "')";
if (query.exec(str.toUtf8()) == false)
{

19
sfilterprocess/spowercafe.h
View File

@@ -17,7 +17,7 @@ public:
struct stPowercafe
{
int company_num;
int category_num;
int community_num;
QString community_id;
int bodycountall;
@@ -29,8 +29,14 @@ public:
int allcountreal;
int usercountreal;
int rank;
stPowercafe():company_num(0), community_num(0), bodycountall(0), replycountall(0), allcountall(0), usercountall(0), bodycountreal(0),
replycountreal(0), allcountreal(0), usercountreal(0), rank(0){}
QString community_title;
QString community_url;
int platformname_num;
QString platformname_name;
stPowercafe():category_num(0), community_num(0), bodycountall(0), replycountall(0), allcountall(0), usercountall(0), bodycountreal(0),
replycountreal(0), allcountreal(0), usercountreal(0), rank(0), platformname_num(0){}
};
struct stStatsPowCafeRank
@@ -53,7 +59,9 @@ public:
int community_num;
int category_num;
int realin;
stBodyPowercafeMap():num(0), body_num(0), community_num(0), category_num(0), realin(0){}
QString body_date;
int platformname_num;
stBodyPowercafeMap():num(0), body_num(0), community_num(0), category_num(0), realin(0), platformname_num(0){}
};
struct stReplyPowercafeMap
@@ -64,6 +72,7 @@ public:
int community_num;
int category_num;
int realin;
QString body_date;
stReplyPowercafeMap():num(0), reply_num(0), body_num(0), community_num(0), category_num(0), realin(0) {}
};
@@ -78,6 +87,8 @@ public:
void clear(){ m_listMap.clear(); }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory);
bool uploadData(QSqlDatabase &_db, const int &_nCompany);
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
};
class SStatsPowCafeRank

116
sfilterprocess/sspammer.cpp
View File

@@ -55,12 +55,17 @@ bool SSpammer::changeTable(int _companyNum)
QSqlDatabase dbWeb = m_pSInitializer->getDatabase(SDBManager::E_DATABASE_WEB);
if (!m_SBodySpammerMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload spammer body map");
m_pSInitializer->insertLog("Fail to change spammer body map");
return false;
}
if (!m_SReplySpammerMap.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to upload spammer reply map");
m_pSInitializer->insertLog("Fail to change spammer reply map");
return false;
}
if (!m_STSpammer.changeTable(dbWeb, _companyNum))
{
m_pSInitializer->insertLog("Fail to change spammer");
return false;
}
return true;
@@ -100,8 +105,8 @@ bool SSpammer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList<
}
}
stReal streal;
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_BODY].size();
qDebug() << "filtered size: " << vecData[SInitializer::E_ARTICLE_REPLY].size();
m_pSInitializer->insertLog(QString("CategoryNum:") + QString::number(_nCategory) + ",(" +
QString::number(vecData[SInitializer::E_ARTICLE_BODY].size()) + '/' + QString::number(vecData[SInitializer::E_ARTICLE_REPLY].size()) + ")" + "(Body/Reply)" );
const int *anColumn = m_pSInitializer->getColumnIntArray();
if (bFiltered[SInitializer::E_ARTICLE_BODY])
{
@@ -130,20 +135,43 @@ bool SSpammer::makeCategory(SDatagroup &_datagroup, int _nCategory, const QList<
bool SSpammer::makeOverallCategory(int _categoryNum)
{
if (m_mapCategory.size() < 1)
return true;
stReal streal;
for (QMap<int, stReal>::iterator iterPos = m_mapCategory.begin(); iterPos != m_mapCategory.end(); iterPos++ )
{
stReal& val = iterPos.value();
foreach (const QString& _strurl, val.m_setRealBodyUrl )
streal.m_setRealBodyUrl.insert(_strurl);
foreach (const QString& _strurl, val.m_setRealAllUrl )
streal.m_setRealAllUrl.insert(_strurl);
for (QMap<QString, int>::iterator iterPos1 = val.m_mapRealReplyUniqueUrl.begin(); iterPos1 != val.m_mapRealReplyUniqueUrl.end();
iterPos1++)
{
streal.m_mapRealReplyUniqueUrl.insertMulti(iterPos1.key(), iterPos1.value());
}
}
m_mapCategory.insert(_categoryNum, streal);
return true;
}
bool SSpammer::makeTable(SDatagroup &_datagroup)
{
m_pSInitializer->insertLog("Make spammer tables");
m_pSInitializer->insertLog("Make spammer spammer");
m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables");
m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make spammer body tables");
m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make spammer reply tables");
m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
if (m_mapCategory.size() > 1)
{
m_pSInitializer->insertLog("Make spammer tables");
m_pSInitializer->insertLog("Make spammer spammer");
m_STSpammer.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make spammer stats_spa_spammer_rank tables");
m_SStatsSpaSpammerRank.makeTable(_datagroup, m_mapCategory);
m_pSInitializer->insertLog("Make spammer body tables");
m_SBodySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
m_pSInitializer->insertLog("Make spammer reply tables");
m_SReplySpammerMap.makeTable(_datagroup, m_mapCategory, m_setSpammer);
}
return true;
}
@@ -197,6 +225,9 @@ void SSpammer::STSpammer::makeTable(SDatagroup &_datagroup, const QMap<int, stRe
stspammer.nicknamecount = iterPos1->nickname.size();
stspammer.id_num = iterPos1->id_num;
stspammer.id_id = iterPos1.key().split(",").at(0);
stspammer.category_num = iterPos.key();
stspammer.platformname_name = iterPos1.key().split(",").at(1);
stspammer.platformname_num = m_pSInitializer->getPlatformName(stspammer.platformname_name);
if (stspammer.bodycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_BODY_COUNT_CUT] ||
stspammer.replycount > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT] ||
iterPos1->nickname.size() > m_pSInitializer->getSpammerParam()[SInitializer::E_SPAMER_NICK_COUNT_CUT])
@@ -276,7 +307,7 @@ bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany)
bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany)
{
//_db.transaction();
QString strQuery = "delete from spammer where company_num = " + QString::number(_nCompany);
QString strQuery = "delete from spammer_" + QString::number(_nCompany) + "_update";
QSqlQuery query(_db);
if (query.exec(strQuery.toUtf8()) == false)
{
@@ -286,7 +317,7 @@ bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany)
}
//_db.commit();
//_db.transaction();
strQuery = "insert into spammer (rank,id_num,id_id,nicknamecount,bodycount,replycount,spammervalue,company_num) VALUES (";
strQuery = "insert into spammer_" + QString::number(_nCompany) + "_update (rank,id_num,id_id,nicknamecount,bodycount,replycount,spammervalue,category_num,platformname_name,platformname_num) VALUES (";
query.clear();
@@ -300,7 +331,9 @@ bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany)
str += QString::number(stspammer.bodycount) + ",";
str += QString::number(stspammer.replycount) + ",";
str += QString::number(stspammer.spammervalue, 'g', 3) + ",";
str += QString::number(_nCompany) + ")";
str += QString::number(stspammer.category_num) + ",";
str += "'" + stspammer.platformname_name + "',";
str += QString::number(stspammer.platformname_num) + ")";
if (query.exec(str.toUtf8()) == false)
{
@@ -319,6 +352,26 @@ bool SSpammer::STSpammer::uploadData(QSqlDatabase &_db, const int &_nCompany)
}
bool SSpammer::STSpammer::changeTable(QSqlDatabase &_db, const int &_nCompany)
{
QString strQuery;
QSqlQuery query(_db);
strQuery = "RENAME TABLE ";
strQuery += "spammer_" + QString::number(_nCompany) + " TO " + "spammer_" + QString::number(_nCompany) + "_temp" + ",";
strQuery += "spammer_" + QString::number(_nCompany) + "_update" + " TO " + "spammer_" + QString::number(_nCompany) + ",";
strQuery += "spammer_" + QString::number(_nCompany) + "_temp" + " TO " + "spammer_" + QString::number(_nCompany) + "_update";
if (!query.exec(strQuery.toUtf8()))
{
m_pSInitializer->insertLog(query.lastQuery());
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
return true;
}
void SSpammer::SStatsSpaSpammerRank::makeTable(SDatagroup &_datagroup, const QMap<int, stReal> &_mapCategory)
{
const QMap<QString, QStringList>* mapBody = _datagroup.getmapBody();
@@ -506,20 +559,22 @@ void SSpammer::SBodySpammerMap::makeTable(SDatagroup &_datagroup, const QMap<int
stbody.id_num = _datagroup.getstBody(strUrl)->id_num;
stbody.category_num = iterPos.key();
stbody.realin |= REAL_BODY;
stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
if (streal.m_mapRealReplyUniqueUrl.contains(strUrl))
{
stbody.realin |= REAL_REPLY;
/*
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
*/
}
m_listMap.append(stbody);
//_datagroup.addRealBodyCategory(strUrl, iterPos.key());
_datagroup.addRealBodyCategory(strUrl, iterPos.key());
}
/*
else if (streal.m_mapRealReplyUniqueUrl.contains(strUrl))
{
stBodySpammerMap stbody;
@@ -528,13 +583,16 @@ void SSpammer::SBodySpammerMap::makeTable(SDatagroup &_datagroup, const QMap<int
stbody.category_num = iterPos.key();
stbody.id_num = _datagroup.getstBody(strUrl)->id_num;
stbody.realin |= REAL_REPLY;
stbody.platformname_num = _datagroup.getstBody(strUrl)->platformname_num;
stbody.body_date = _datagroup.getstBody(strUrl)->date;
foreach (int order, streal.m_mapRealReplyUniqueUrl.values(strUrl))
{
_datagroup.addRealReplyCategory(strUrl, order, iterPos.key());
}
m_listMap.append(stbody);
}
*/
}
}
}
@@ -585,7 +643,7 @@ bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompa
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin)"
strQuery = "insert into body_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,realin,body_date,platformname_num)"
" values (";
query.clear();
@@ -596,7 +654,9 @@ bool SSpammer::SBodySpammerMap::uploadData(QSqlDatabase &_db, const int &_nCompa
str += QString::number(stmap.category_num) + ",";
str += QString::number(stmap.id_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.realin) + ")";
str += QString::number(stmap.realin) + ",";
str += "'" + stmap.body_date + "',";
str += QString::number(stmap.platformname_num) + ")";
if (query.exec(str.toUtf8()) == false)
{
m_pSInitializer->insertLog(query.lastQuery());
@@ -628,7 +688,12 @@ void SSpammer::SReplySpammerMap::makeTable(SDatagroup &_datagroup, const QMap<in
streply.body_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->body_num;
streply.id_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->id_num;
streply.reply_num = _datagroup.getstReply(iterPos1.key(), iterPos1.value())->num;
streply.body_date = _datagroup.getstBody(iterPos1.key())->date;
m_listMap.append(streply);
_datagroup.addRealReplyCategory(iterPos1.key(), iterPos1.value(), iterPos.key());
}
/*
@@ -699,7 +764,7 @@ bool SSpammer::SReplySpammerMap::uploadData(QSqlDatabase &_db, const int &_nComp
m_pSInitializer->insertLog(query.lastError().text());
return false;
}
strQuery = "insert into reply_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,reply_num)"
strQuery = "insert into reply_spammer_map_" + QString::number(_nCompany) + "_update (num,category_num,id_num,body_num,reply_num,body_date)"
" values (";
query.clear();
@@ -710,7 +775,8 @@ bool SSpammer::SReplySpammerMap::uploadData(QSqlDatabase &_db, const int &_nComp
str += QString::number(stmap.category_num) + ",";
str += QString::number(stmap.id_num) + ",";
str += QString::number(stmap.body_num) + ",";
str += QString::number(stmap.reply_num) + ")";
str += QString::number(stmap.reply_num) + ",";
str += "'" + stmap.body_date + "')";
if (query.exec(str.toUtf8()) == false)
{
m_pSInitializer->insertLog(query.lastQuery());

12
sfilterprocess/sspammer.h
View File

@@ -18,14 +18,16 @@ public:
struct stSpammer
{
int id_num;
int company_num;
int category_num;
QString id_id;
int nicknamecount;
int bodycount;
int replycount;
double spammervalue;
int rank;
stSpammer():id_num(0), company_num(0), nicknamecount(0), bodycount(0), replycount(0), rank(0), spammervalue(0.0){}
QString platformname_name;
int platformname_num;
stSpammer():id_num(0), category_num(0), nicknamecount(0), bodycount(0), replycount(0), rank(0), spammervalue(0.0), platformname_num(0){}
};
@@ -52,7 +54,9 @@ public:
int id_num;
int category_num;
int realin;
stBodySpammerMap():num(0), body_num(0), id_num(0), category_num(0), realin(0){}
QString body_date;
int platformname_num;
stBodySpammerMap():num(0), body_num(0), id_num(0), category_num(0), realin(0), platformname_num(0){}
};
@@ -63,6 +67,7 @@ public:
int body_num;
int id_num;
int category_num;
QString body_date;
stReplySpammerMap():num(0), body_num(0), reply_num(0), id_num(0), category_num(0){}
};
@@ -85,6 +90,7 @@ public:
void clear(){ m_listMap.clear(); }
void makeTable(SDatagroup &_datagroup, const QMap<int, stReal>& _mapCategory, QSet<int>& setSpammer);
bool uploadData(QSqlDatabase &_db, const int &_nCompany);
bool changeTable(QSqlDatabase &_db, const int &_nCompany);
};