etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

dbinfo 추가 및 버그 수정

Browse Source 
git-svn-id: svn://192.168.0.12/source@171 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-08-10 02:00:56 +00:00
parent a692dd4066
commit ecd221e60a
2 changed files with 192 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
FilterProcess/sfilterprocess.cpp
View File

@@ -335,7 +335,7 @@ void SFilterProcess::FilterLength(QString _strJson)
bFlag = true;
break;
case E_LENGTH_COMP_EQUAL:
if (nLength == nCurrentLength)
if (nCurrentLength == nLength)
bFlag = true;
break;
}
@@ -365,13 +365,9 @@ void SFilterProcess::FilterCounter(QString _strJson)
{
QString str = strList.at(nCatalog).trimmed();
if(mapData.contains(str))
{
mapData[str]++;
}
else
{
mapData.insert(str, 1);
}
}
for(QMap <QString, int>::iterator iterPos = mapData.begin(); iterPos != mapData.end(); iterPos++)
@@ -783,7 +779,11 @@ void SFilterProcess::DataLoad(QString _strDataNum)
SCommunity stCom;
m_mapCommunity.insert(" ",stCom);
QString astrWhere[E_ARTICLE_MAX] = {" where article_form = 'body' Order by article_date DESC"," where article_form = 'reply'"};
QString astrWhere[E_ARTICLE_MAX] = {" where article_form = 'body'"," where article_form = 'reply'"};
astrWhere[E_ARTICLE_BODY] += " and date(article_date) >= date(subdate(now(), INTERVAL ";
astrWhere[E_ARTICLE_BODY] += QString::number(abs(m_nCutDays));
astrWhere[E_ARTICLE_BODY] += " DAY)) and date(article_date) <= date(now()) ";
astrWhere[E_ARTICLE_BODY] += " Order by article_date DESC";
for (int i = 0; i < E_ARTICLE_MAX ; i++)
{
if(query.exec(QString(strSelect + astrWhere[i]).toUtf8()) == false) {InsertLog(query.lastError().text());return;}
@@ -800,14 +800,14 @@ void SFilterProcess::DataLoad(QString _strDataNum)
{
if (m_mapData.contains(strURL) == false)
{
if (strList[m_anData[E_DATA_article_date]].trimmed().isEmpty()) continue;
//if (strList[m_anData[E_DATA_article_date]].trimmed().isEmpty()) continue;
if (strList[m_anData[E_DATA_article_id]].trimmed().isEmpty()) continue;
if (strList[m_anData[E_DATA_article_nickname]].trimmed().isEmpty()) continue;
if (strList[m_anData[E_DATA_platform_id]].trimmed().isEmpty()) continue;
if (strList[m_anData[E_DATA_platform_name]].trimmed() == QString("naver") && strList[m_anData[E_DATA_platform_form]].trimmed() == QString("blog"))
{
QString strID = strList[m_anData[E_DATA_platform_id]].trimmed();
strList[m_anData[E_DATA_article_id]] = strList[m_anData[E_DATA_article_url]].trimmed().split("/").at(3);
if (m_mapNaverBlogProfile.contains(strID) == false )
{
SNaverBlogProfile stProfile;
@@ -833,7 +833,7 @@ void SFilterProcess::DataLoad(QString _strDataNum)
}
SBody stBody;
stBody.m_date = QDateTime::fromString(strList[m_anData[E_DATA_article_date]].trimmed(),"yyyy-MM-dd HH:mm:ss").date();
if (stBody.m_date < QDate::currentDate().addDays(m_nCutDays)) continue;
//if (stBody.m_date < QDate::currentDate().addDays(m_nCutDays)) continue;
stBody.m_dtLast = QDateTime::fromString(strList[m_anData[E_DATA_article_date]].trimmed(),"yyyy-MM-dd HH:mm:ss");
stBody.m_bSelect = false;
@@ -948,6 +948,88 @@ void SFilterProcess::DataLoad(QString _strDataNum)
m_db.close();
}
struct SInfluenceBuzz
{
int m_ncId;
int m_ncBody;
int m_ncReply;
QSet <QString> m_setName;
SInfluenceBuzz()
{
m_ncId = m_ncBody = m_ncReply = 0;
}
};
void SFilterProcess::DataInfluenceBuzzCount(QString _strCompanyNum)
{
QMap <QDate,SInfluenceBuzz> map;
QDate dateStart(2100,1,1),dateEnd(2000,1,1);
foreach(QStringList strlist,m_data[E_ARTICLE_BODY])
{
if (strlist[m_anData[E_DATA_platform_form]].trimmed() != "blog") continue;
QDate date = QDateTime::fromString(strlist[m_anData[E_DATA_article_date]].trimmed(),"yyyy-MM-dd HH:mm:ss").date();
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
if (!map.contains(date))
{
SInfluenceBuzz stBuzz;
map.insert(date,stBuzz);
}
map[date].m_ncBody++;
QString strID = strlist[m_anData[E_DATA_article_id]].trimmed();
if (!map[date].m_setName.contains(strID))
{
map[date].m_setName.insert(strID);
map[date].m_ncId++;
}
}
foreach(QStringList strlist,m_data[E_ARTICLE_REPLY])
{
if (strlist[m_anData[E_DATA_platform_form]].trimmed() != "blog") continue;
QDate date = QDateTime::fromString(strlist[m_anData[E_DATA_article_date]].trimmed(),"yyyy-MM-dd HH:mm:ss").date();
dateStart = (date < dateStart ) ? date : dateStart;
dateEnd = (date > dateEnd ) ? date : dateEnd;
if (!map.contains(date))
{
SInfluenceBuzz stBuzz;
map.insert(date,stBuzz);
}
map[date].m_ncReply++;
}
m_db.setHostName(m_strWebDataBaseHost);
m_db.setUserName(m_strWebDataBaseUser);
m_db.setPassword(m_strWebDataBasePass);
m_db.setDatabaseName(m_strWebDataBaseName);
if (m_db.open() == false)
{
InsertLog(m_db.lastError().text());
return;
}
QSqlQuery query;
QString strQuery;
strQuery = "delete from stats_inf_blogger_buzzbyday where company_num = " + _strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
for(QDate date = dateStart; date < dateEnd; date=date.addDays(1))
{
strQuery = "insert into stats_inf_blogger_buzzbyday (company_num,date,formateddate,idcount,bodycount,replycount) VALUES ( " + _strCompanyNum ;
strQuery += ",'" + date.toString("yyyy-MM-dd") + "'";
strQuery += ",'" + date.toString("yyyyMMdd")+ "'";
if (map.contains(date))
{
strQuery += "," + QString::number(map[date].m_ncId);
strQuery += "," + QString::number(map[date].m_ncBody);
strQuery += "," + QString::number(map[date].m_ncReply) + ")";
}
else
strQuery += ",0,0,0)";
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());qDebug() << strQuery;return;}
}
m_db.close();
}
bool QSortBody(const SInfluenceCount &stInflunce1, const SInfluenceCount &stInflunce2)
{
return stInflunce1.m_ncBody > stInflunce2.m_ncBody;
@@ -966,6 +1048,12 @@ bool QSortInfulrence(const SInfluenceCount &stInflunce1, const SInfluenceCount &
void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNum)
{
ReloadData();
InsertLog("Influence Data Buzz Start ");
DataInfluenceBuzzCount(_strCompanyNum);
InsertLog("Influence Data Buzz End ");
InsertLog("Influence Data Filter Run Start ");
DataFilter(_strFilter);
InsertLog("Influence Data Filter Run End ");
@@ -975,8 +1063,7 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
QMap <QString,SInfluenceCount> mapCount;
{
InsertLog("Influence Rank Count Make Start ");
QString strCompanyNum = _strCompanyNum;
//foreach(SBody stBody, m_mapData.values())
QString strCompanyNum = _strCompanyNum;
foreach(QStringList strlist,m_data[E_ARTICLE_BODY])
{
QString strUrl = strlist[m_nUrlColumn].trimmed();
@@ -1026,6 +1113,7 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
return;
}
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (Delete)");
InsertLog("Influence Rank Count Send Start");
QSqlQuery query;
QString strQuery;
@@ -1040,6 +1128,8 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
strQuery = "insert into influencer (rank,company_num,id_num,nickname_num,community_num,bodycount,replycount,influencervalue,profile,profileurl,id_id)"
" VALUES (:RANK," + strCompanyNum + ",:ID_NUM,:NICKNAME_NUM,:COMMUNITY_NUM,:BODY_COUNT,:REPLY_COUNT,:INPL,:PROFILE,:PROFILE_URL,:ID_ID)";
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (Insert)");
query.prepare(strQuery.toUtf8());
int ncRank = 1;
foreach(SInfluenceCount stIn,list)
@@ -1067,6 +1157,8 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
if(query.exec() == false) {InsertLog(query.lastError().text());return;}
}
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (rank influence value Insert)");
int nCount = 0;
foreach(SInfluenceCount stIn,list)
{
@@ -1076,6 +1168,8 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
}
nCount = 0;
qSort(list.begin(),list.end(),QSortBody);
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (rank body count Insert)");
foreach(SInfluenceCount stIn,list)
{
if (nCount >= 10) break; nCount++;
@@ -1084,6 +1178,7 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
}
qSort(list.begin(),list.end(),QSortReply);
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (rank reply count Insert)");
nCount = 0;
foreach(SInfluenceCount stIn,list)
{
@@ -1096,6 +1191,7 @@ void SFilterProcess::DataInfluenceCount(QString _strFilter,QString _strCompanyNu
"influencercount = " + QString::number(list.size()) + " "
"where company_num = " + strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
UpdateDBInfoState(m_db,_strCompanyNum,"Influence (Finish)");
m_db.close();
InsertLog("Influence Rank Count Send End");
}
@@ -1124,8 +1220,7 @@ bool QSortSpamer(const SSpamer &st1, const SSpamer &st2)
}
void SFilterProcess::DataSpammersCount(QString _strFilter,QString _strCompanyNum)
{
//double dRead[E_SPAMER_MAX]={0.0,};
{
QMap<QString,SSpamer> mapSpamer;
ReloadData();
InsertLog("Spamer Data Filter Run Start ");
@@ -1212,7 +1307,7 @@ void SFilterProcess::DataSpammersCount(QString _strFilter,QString _strCompanyNum
bFlag = true;
if (stSpamer.m_dReplycount > m_adSpammer[E_SPAMER_REPLY_COUNT_CUT])
bFlag = true;
if ((double)stSpamer.m_setNick.size() > m_adSpammer[E_SPAMER_BODY_COUNT_CUT])
if ((double)stSpamer.m_setNick.size() > m_adSpammer[E_SPAMER_NICK_COUNT_CUT])
bFlag = true;
if (bFlag)
{
@@ -1235,6 +1330,8 @@ void SFilterProcess::DataSpammersCount(QString _strFilter,QString _strCompanyNum
InsertLog(m_db.lastError().text());
return;
}
UpdateDBInfoState(m_db,_strCompanyNum,"Spammer (Delete)");
QSqlQuery query;
QString strQuery;
QString strCompanyNum = _strCompanyNum;
@@ -1248,6 +1345,8 @@ void SFilterProcess::DataSpammersCount(QString _strFilter,QString _strCompanyNum
foreach(QString str,m_mapPlatformname.keys())
mapPlatformnameCount.insert(str,1);
UpdateDBInfoState(m_db,_strCompanyNum,"Spammer (Insert)");
int nCount = 1;
foreach(SSpamer stIn,list)
{
@@ -1296,6 +1395,8 @@ void SFilterProcess::DataSpammersCount(QString _strFilter,QString _strCompanyNum
"where company_num = " + strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
UpdateDBInfoState(m_db,_strCompanyNum,"Spammer (Finish)");
m_db.close();
InsertLog("Spamer Rank Count Send End ");
}
@@ -1313,13 +1414,13 @@ bool QSortPowerCafeReal(const SPowerCafeCount &st1, const SPowerCafeCount &st2)
void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNum)
{
ReloadData();
ReloadData();
InsertLog("Power Cafe Data Filter Run Start ");
DataFilter(_strFilter);
InsertLog("Power Cafe Data Filter Run End ");
InsertLog("Power Cafe Data Filter Run End ");
QMap <QString,SPowerCafeCount> mapRealCount;
QMap <QString,SPowerCafeCount> mapAllCount;
{
{
InsertLog("Power Cafe Rank Count Make Start ");
// real make
if (m_bDataFlag[E_ARTICLE_BODY] == true)
@@ -1424,6 +1525,7 @@ void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNu
QSqlQuery query;
QString strQuery;
QString strCompanyNum = _strCompanyNum;
UpdateDBInfoState(m_db,_strCompanyNum,"Power Cafe (Delete)");
strQuery = "delete from powercafe where company_num = " + strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
strQuery = "delete from stats_pow_cafe_rank where company_num = " + strCompanyNum;
@@ -1435,6 +1537,7 @@ void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNu
int nCount = 0;
query.prepare(strQuery.toUtf8());
UpdateDBInfoState(m_db,_strCompanyNum,"Power Cafe (Insert)");
foreach(SPowerCafeCount stIn,list)
{
query.bindValue(":COMMNUNITY_NUM",stIn.m_nCommunityID);
@@ -1453,6 +1556,7 @@ void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNu
}
}
nCount = 0;
UpdateDBInfoState(m_db,_strCompanyNum,"Power Cafe (Rank Insert)");
//articlecountall/articlecountreal/usercountreal
strQuery = "insert into stats_pow_cafe_rank (rank,articlecountall,usercountreal,subject,company_num,community_id,community_title,articlecountreal)"
" VALUES (:RANK,0,0,'articlecountreal'," + strCompanyNum + ",:ID,:TITLE,:COUNT_ALL)";
@@ -1474,6 +1578,8 @@ void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNu
QList<SPowerCafeCount> listAll = mapAllCount.values();
qSort(listAll.begin(),listAll.end(),QSortPowerCafe);
UpdateDBInfoState(m_db,_strCompanyNum,"Power Cafe (Stats Rank Insert)");
nCount = 0;
strQuery = "insert into stats_pow_cafe_rank (rank,articlecountreal,usercountreal,subject,company_num,community_id,community_title,articlecountall)"
" VALUES (:RANK,0,0,'articlecountall'," + strCompanyNum + ",:ID,:TITLE,:COUNT_ALL)";
@@ -1497,6 +1603,8 @@ void SFilterProcess::DataPowerCafeCount(QString _strFilter,QString _strCompanyNu
"where company_num = " + strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
UpdateDBInfoState(m_db,_strCompanyNum,"Power Cafe (Finish)");
m_db.close();
InsertLog("Power Cafe Rank Send End");
}
@@ -1661,6 +1769,39 @@ bool SFilterProcess::RenameTable(QString _strOld, QString _strNew , QString _str
return true;
}
#include <QTextCodec>
QString UniCodeToEuckr(QString _strData)
{
QString strOut;
QChar *pch = _strData.data();
QTextCodec *codec = QTextCodec::codecForName("EUC-KR");
for (int i = 0; i < _strData.length(); i++)
{
if (pch[i].isDigit() || pch[i].isLower() || pch[i].isUpper())
strOut += pch[i];
else if (pch[i].isSpace())
strOut += "%" + codec->fromUnicode(pch[i]).toHex().toUpper();
else
{
QString str;
str = codec->fromUnicode(pch[i]).toHex().toUpper();
if (str.length()%2) continue;
if (str.length() == 2) strOut += pch[i];
else
{
int i = 0;
while (i < str.length())
{
strOut += "%" + str[i++];
strOut += str[i++];
}
}
}
}
return strOut;
}
void SFilterProcess::DataPut(QString _strCompanyNum)
{
m_db.setHostName(m_strWebDataBaseHost);
@@ -1673,10 +1814,12 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
return;
}
InsertLog("Data Put Start");
InsertLog("Data Put Start");
QSqlQuery query;
QString strQuery;
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Delete)");
strQuery = "delete from body_" + _strCompanyNum + "_update";
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
@@ -1701,6 +1844,8 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
QString strMapQuery = QString("insert into body_category_map_" + _strCompanyNum + "_update (num,body_num,category_num,realin) VALUES (:NUM,:BODY_NUM,:CATEGORY_NUM,:REALIN)").toUtf8();
QString strCommunityQuery = QString("insert into community_" + _strCompanyNum + "_update (num,id,title,url,exid,platformname_num,platformform_num) values (");
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Coummunity Insert)");
foreach (SCommunity stCommunity,m_mapCommunity.values())
{
strQuery = strCommunityQuery;
@@ -1717,6 +1862,7 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
}
}
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Id Insert)");
QString strIDQuery = QString("insert into id_" + _strCompanyNum + "_update (num,id,platformname_num,community_num) values (");
foreach (SId stID,m_mapID.values())
{
@@ -1727,17 +1873,20 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
strQuery += "," + QString::number(stID.m_nCommunity);
strQuery += ")";
if(query.exec(strQuery.toUtf8()) == false) {
qDebug() << strQuery;InsertLog(query.lastError().text());m_db.close();return;
InsertLog(query.lastError().text());m_db.close();qDebug() << strQuery;return;
}
}
QString strNickQuery = QString("insert into nickname_" + _strCompanyNum + "_update (num,nickname,id_num) values (");
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Nickname Insert)");
QString strNickQuery = QString("insert into nickname_" + _strCompanyNum + "_update (num,nickname,id_num,exnickname) values (");
foreach (SNickname stNick,m_mapNickname.values())
{
strQuery = strNickQuery;
strQuery += QString::number(stNick.m_nNum);
strQuery += ",'" + SqlString(stNick.m_strNickname) + "'";
strQuery += "," + QString::number(stNick.m_nID);
strQuery += ",'" + SqlString(UniCodeToEuckr(stNick.m_strNickname)) + "'";
strQuery += ")";
if(query.exec(strQuery.toUtf8()) == false)
{
@@ -1745,6 +1894,7 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
}
}
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Body Reply Insert)");
int nReplyCount = 0,nBodyCount = 0;
QDateTime dtMin = QDateTime::currentDateTime().addYears(50);
QDateTime dtMax = QDateTime::currentDateTime().addYears(-50);
@@ -1764,7 +1914,9 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
query.bindValue(":DATE",stReply.m_strlist[m_anData[E_DATA_article_date]].trimmed().toUtf8());
query.bindValue(":BODY_NUM",m_mapData[strUrl].m_nNum);
query.bindValue(":REALBYCATEGORYNUM",GetVectorToString(stReply.m_vecCategory," ").trimmed());
if (query.exec()==false){InsertLog(query.lastError().text());m_db.close();return;}
if (query.exec()==false){
InsertLog(query.lastError().text());m_db.close();return;
}
nReplyCount++;
}
query.prepare(strBodyQuery);
@@ -1818,10 +1970,14 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
query.bindValue(":LASTUPDATE",strDate);
query.bindValue(":REALBYCATEGORYNUM",GetVectorToString(m_mapData[strUrl].m_vecCategory," ").trimmed());
if (query.exec()==false) {InsertLog(query.lastError().text());m_db.close();return;}
if (query.exec()==false) {
InsertLog(query.lastError().text());m_db.close();return;
}
nBodyCount++;
}
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Map Insert)");
query.prepare(strMapQuery);
int nCount = 0;
foreach(SMap stMap,m_vecMap)
@@ -1876,7 +2032,8 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
if (query.exec(strQuery.toUtf8())==false) { InsertLog(query.lastError().text());m_db.close();qDebug() << strQuery;return;}
}
date = date.addDays(1);
}
}
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Table Rename)");
RenameTable("body_" + _strCompanyNum,"body_" + _strCompanyNum + "_update","body_" + _strCompanyNum +"_temp");
RenameTable("reply_" + _strCompanyNum,"reply_" + _strCompanyNum + "_update","reply_" + _strCompanyNum + "_temp");
@@ -1892,6 +2049,15 @@ void SFilterProcess::DataPut(QString _strCompanyNum)
"maxdate = '" + dtMax.toString("yyyy-MM-dd HH:mm:ss") + "' "
"where company_num = " + _strCompanyNum;
if(query.exec(strQuery.toUtf8()) == false) {InsertLog(query.lastError().text());return;}
UpdateDBInfoState(m_db,_strCompanyNum,"Data (Finish)");
m_db.close();
InsertLog("Data Put End");
}
void SFilterProcess::UpdateDBInfoState(QSqlDatabase &_db,QString _strCompanyNum,QString _strStatus)
{
QString strQuery = "update dbinfo set "
"status = '" + _strStatus + "' "
"where company_num = " + _strCompanyNum;
_db.exec(strQuery.toUtf8());
}