일괄처리 추가
platform_name = 'post', 'story', 'channel'일 경우 sns으로 변경 git-svn-id: svn://192.168.0.12/source@244 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
850
Analyzer/sbatchrun.cpp
Normal file
850
Analyzer/sbatchrun.cpp
Normal file
@@ -0,0 +1,850 @@
|
||||
#include "sbatchrun.h"
|
||||
#include <QString>
|
||||
#include <QVBoxLayout>
|
||||
#include <QHBoxLayout>
|
||||
#include <QPushButton>
|
||||
#include <QListWidget>
|
||||
#include "mainwindow.h"
|
||||
#include <QGroupBox>
|
||||
#include <QDateEdit>
|
||||
#include <QLineEdit>
|
||||
#include <QLabel>
|
||||
#include <QCalendarWidget>
|
||||
#include <QMessageBox>
|
||||
#include <QFile>
|
||||
#include "stable.h"
|
||||
#include "scolumn.h"
|
||||
#include "./../Json/sjson.h"
|
||||
#define D_NOT_SELECT -1
|
||||
#include <QSet>
|
||||
|
||||
void SBatchRun::showBatchRun(int nGroupID)
|
||||
{
|
||||
if(nGroupID == -1)
|
||||
return;
|
||||
|
||||
RefreshFilter(nGroupID);
|
||||
this->show();
|
||||
this->setFocus();
|
||||
}
|
||||
|
||||
|
||||
SBatchRun::SBatchRun()
|
||||
{
|
||||
setMainWidgets();
|
||||
ReloadColumn();
|
||||
}
|
||||
|
||||
QVBoxLayout* SBatchRun::setLoadWidgets()
|
||||
{
|
||||
QVBoxLayout *vLoad = new QVBoxLayout;
|
||||
m_plwData = new QListWidget;
|
||||
m_plwData->setSelectionMode(QAbstractItemView::SingleSelection);
|
||||
vLoad->addWidget(m_plwData);
|
||||
|
||||
QPushButton *ppbRefresh = new QPushButton("Refresh");
|
||||
vLoad->addWidget(ppbRefresh);
|
||||
connect(ppbRefresh, SIGNAL(released()),this, SLOT(Refresh()));
|
||||
|
||||
return vLoad;
|
||||
}
|
||||
|
||||
QVBoxLayout* SBatchRun::setDaysWidgets()
|
||||
{
|
||||
QVBoxLayout *vDays = new QVBoxLayout;
|
||||
|
||||
m_pgbFromTo = setFromToWidgets();
|
||||
m_pgbCutDays = setCutDaysWidgets();
|
||||
|
||||
connect(m_pgbFromTo, SIGNAL(clicked(bool)), this, SLOT(clickedFromTo(bool)));
|
||||
connect(m_pgbCutDays, SIGNAL(clicked(bool)), this, SLOT(clickedCutDays(bool)));
|
||||
|
||||
vDays->addWidget(m_pgbFromTo);
|
||||
vDays->addWidget(m_pgbCutDays);
|
||||
|
||||
return vDays;
|
||||
}
|
||||
|
||||
QVBoxLayout* SBatchRun::setFilterWidgets()
|
||||
{
|
||||
m_pleFilter = new QLineEdit;
|
||||
m_plwFilter = new QListWidget;
|
||||
|
||||
QVBoxLayout *vFilter = new QVBoxLayout;
|
||||
vFilter->addWidget(m_pleFilter);
|
||||
vFilter->addWidget(m_plwFilter);
|
||||
|
||||
return vFilter;
|
||||
}
|
||||
|
||||
|
||||
QGroupBox* SBatchRun::setFromToWidgets()
|
||||
{
|
||||
m_pdeFrom = new QDateEdit(QDate::currentDate());
|
||||
m_pdeTo = new QDateEdit(QDate::currentDate());
|
||||
|
||||
m_pdeFrom->setDateRange(QDate(2003, 5, 20),QDate::currentDate());
|
||||
m_pdeTo->setDateRange(QDate(2003, 5, 20),QDate::currentDate().addYears(1));
|
||||
|
||||
QCalendarWidget *pCalender = new QCalendarWidget();
|
||||
m_pdeFrom->setCalendarWidget(pCalender);
|
||||
m_pdeFrom->setCalendarPopup(true);
|
||||
|
||||
m_pdeTo->setCalendarWidget(pCalender);
|
||||
m_pdeTo->setCalendarPopup(true);
|
||||
|
||||
QHBoxLayout *hFromTo = new QHBoxLayout;
|
||||
hFromTo->addWidget(new QLabel("From:"));
|
||||
hFromTo->addWidget(m_pdeFrom);
|
||||
hFromTo->addWidget(new QLabel("To:"));
|
||||
hFromTo->addWidget(m_pdeTo);
|
||||
|
||||
QGroupBox* pgbFromTo = new QGroupBox("FromTo");
|
||||
pgbFromTo->setLayout(hFromTo);
|
||||
pgbFromTo->setCheckable(true);
|
||||
pgbFromTo->setChecked(false);
|
||||
|
||||
return pgbFromTo;
|
||||
}
|
||||
|
||||
QGroupBox* SBatchRun::setCutDaysWidgets()
|
||||
{
|
||||
m_pleCutDays = new QLineEdit;
|
||||
m_pleCutDays->setText("-31");
|
||||
|
||||
QHBoxLayout *hCutDays = new QHBoxLayout;
|
||||
hCutDays->addWidget(new QLabel("CutDays:"));
|
||||
hCutDays->addWidget(m_pleCutDays);
|
||||
|
||||
QGroupBox* pgbCutDays = new QGroupBox("CutDays");
|
||||
pgbCutDays->setLayout(hCutDays);
|
||||
pgbCutDays->setCheckable(true);
|
||||
pgbCutDays->setChecked(true);
|
||||
|
||||
return pgbCutDays;
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::setMainWidgets()
|
||||
{
|
||||
QHBoxLayout *hMain = new QHBoxLayout;
|
||||
hMain->addLayout(setLoadWidgets());
|
||||
hMain->addLayout(setDaysWidgets());
|
||||
hMain->addLayout(setFilterWidgets());
|
||||
QPushButton *ppbRun = new QPushButton;
|
||||
ppbRun->setText("Run");
|
||||
connect(ppbRun, SIGNAL(released()),this, SLOT(run()));
|
||||
hMain->addWidget(ppbRun);
|
||||
|
||||
setLayout(hMain);
|
||||
|
||||
setWindowTitle("Batch Run");
|
||||
Refresh();
|
||||
}
|
||||
|
||||
void SBatchRun::clickedCutDays(bool clicked)
|
||||
{
|
||||
m_pgbFromTo->setChecked(!clicked);
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::clickedFromTo(bool clicked)
|
||||
{
|
||||
m_pgbCutDays->setChecked(!clicked);
|
||||
}
|
||||
|
||||
void SBatchRun::Refresh()
|
||||
{
|
||||
m_plwData->clear();
|
||||
QSqlQuery query("select id,name,count from datagroup");
|
||||
while (query.next())
|
||||
{
|
||||
QString str = query.value(1).toString();
|
||||
str += " ( ";
|
||||
str += query.value(0).toString();
|
||||
str += " , ";
|
||||
str += query.value(2).toString();
|
||||
str += " )";
|
||||
QListWidgetItem *pItem = new QListWidgetItem(str,m_plwData);
|
||||
pItem->setData(Qt::UserRole, QVariant(query.value(0)));
|
||||
}
|
||||
}
|
||||
|
||||
void SBatchRun::run()
|
||||
{
|
||||
if(!loadData())
|
||||
{
|
||||
Message("Check Data DB");
|
||||
return;
|
||||
}
|
||||
if(!execFilter())
|
||||
{
|
||||
Message("Filter Failed");
|
||||
return;
|
||||
}
|
||||
ViewResult();
|
||||
clearData();
|
||||
}
|
||||
|
||||
SBatchRun::~SBatchRun()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
void SBatchRun::ViewResult()
|
||||
{
|
||||
QVector <QStringList> column = GetColumn().data();
|
||||
/*
|
||||
STable *pTable = (STable *)GetMainWindow()->m_pDataDlg->GetCurrentWidget();
|
||||
if (pTable == NULL)
|
||||
pTable = (STable *)GetMainWindow()->m_pDataDlg->AddTable("<Batch>");
|
||||
*/
|
||||
STable *pTable = (STable *)GetMainWindow()->m_pDataDlg->AddTable("<Batch>");
|
||||
QMap<QString, bool> mapFilter;
|
||||
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
if(m_bDataFlag[i])
|
||||
{
|
||||
for(int j = 0; j < m_data[i].length(); j++)
|
||||
{
|
||||
mapFilter[m_data[i][j][m_anData[E_DATA_article_url]].trimmed()] = true;
|
||||
}
|
||||
}
|
||||
qDebug() << m_bDataFlag[i];
|
||||
}
|
||||
|
||||
qDebug() << "MapFilter size :" << mapFilter.size();
|
||||
|
||||
if(!m_bDataFlag[E_ARTICLE_BODY] && !m_bDataFlag[E_ARTICLE_REPLY])
|
||||
{
|
||||
for(int i = 0; i < m_vecData[E_ARTICLE_BODY].size(); i++)
|
||||
{
|
||||
mapFilter[m_vecData[E_ARTICLE_BODY][i][m_anData[E_DATA_article_url]].trimmed()] = true;
|
||||
}
|
||||
}
|
||||
|
||||
int totalRow = 0;
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
for(int j = 0; j < m_vecData[i].size(); j++)
|
||||
{
|
||||
if(mapFilter.value(m_vecData[i][j][m_anData[E_DATA_article_url]].trimmed()))
|
||||
totalRow++;
|
||||
}
|
||||
}
|
||||
qDebug() << totalRow;
|
||||
|
||||
pTable->setArticleSelect(STable::E_ARTICLE_ALL);
|
||||
pTable->clear();
|
||||
pTable->SetHeaderList(&column,SColumn::E_NAME);
|
||||
pTable->setRowCount(totalRow);
|
||||
|
||||
GetMainWindow()->m_progress.setRange(0, totalRow);
|
||||
int nCount = 0;
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
for(int j = 0; j < m_vecData[i].length(); j++)
|
||||
{
|
||||
if(mapFilter.value(m_vecData[i][j][m_anData[E_DATA_article_url]].trimmed()))
|
||||
{
|
||||
for (int k = 0; k < pTable->columnCount() ; k++)
|
||||
{
|
||||
QString str = m_vecData[i][j][k].replace("\n"," ");
|
||||
pTable->setItem(nCount,k,new QTableWidgetItem(" "+str.replace("\t"," ")+" "));
|
||||
}
|
||||
nCount++;
|
||||
GetMainWindow()->m_progress.setValue(nCount);
|
||||
GetMainWindow()->m_progress.repaint();
|
||||
}
|
||||
}
|
||||
}
|
||||
pTable->repaint();
|
||||
//Message(QString("RealBody : ") + QString::number(m_data[E_ARTICLE_BODY].size()) + QString("\nRealReply : ") + QString::number(m_data[E_ARTICLE_REPLY].size()));
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::ViewRawResult()
|
||||
{
|
||||
// check data duplication
|
||||
{
|
||||
QMap<QString, int> mapUrls;
|
||||
for(int i = 0; i < m_vecData[E_ARTICLE_BODY].size(); i++)
|
||||
{
|
||||
if(mapUrls.contains(m_vecData[E_ARTICLE_BODY][i][m_anData[E_DATA_article_url]].trimmed()))
|
||||
{
|
||||
mapUrls[m_vecData[E_ARTICLE_BODY][i][m_anData[E_DATA_article_url]].trimmed()] += 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
mapUrls[m_vecData[E_ARTICLE_BODY][i][m_anData[E_DATA_article_url]].trimmed()] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for(QMap<QString, int>::iterator iterPos = mapUrls.begin(); iterPos != mapUrls.end(); iterPos++)
|
||||
{
|
||||
if(iterPos.value() > 1)
|
||||
qDebug() << iterPos.key() << " : " << iterPos.value() ;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
QVector <QStringList> column = GetColumn().data();
|
||||
/*
|
||||
STable *pTable = (STable *)GetMainWindow()->m_pDataDlg->GetCurrentWidget();
|
||||
if (pTable == NULL)
|
||||
pTable = (STable *)GetMainWindow()->m_pDataDlg->AddTable("<Batch>");
|
||||
*/
|
||||
STable *pTable = (STable *)GetMainWindow()->m_pDataDlg->AddTable("<RawBatch>");
|
||||
|
||||
int totalRow = 0;
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
totalRow += m_vecData[i].size();
|
||||
}
|
||||
qDebug() << totalRow;
|
||||
|
||||
pTable->setArticleSelect(STable::E_ARTICLE_ALL);
|
||||
pTable->clear();
|
||||
pTable->SetHeaderList(&column,SColumn::E_NAME);
|
||||
pTable->setRowCount(totalRow);
|
||||
|
||||
GetMainWindow()->m_progress.setRange(0, totalRow);
|
||||
int nCount = 0;
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
for(int j = 0; j < m_vecData[i].length(); j++)
|
||||
{
|
||||
for (int k = 0; k < pTable->columnCount() ; k++)
|
||||
{
|
||||
QString str = m_vecData[i][j][k].replace("\n"," ");
|
||||
pTable->setItem(nCount,k,new QTableWidgetItem(" "+str.replace("\t"," ")+" "));
|
||||
}
|
||||
nCount++;
|
||||
GetMainWindow()->m_progress.setValue(nCount);
|
||||
GetMainWindow()->m_progress.repaint();
|
||||
}
|
||||
}
|
||||
pTable->repaint();
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::RefreshFilter(int _nGroup)
|
||||
{
|
||||
if (_nGroup == D_NOT_SELECT) return;
|
||||
{
|
||||
QSqlQuery query("select id,type,data from filter where filtergroup_id = " + QString::number(_nGroup));
|
||||
m_plwFilter->clear();
|
||||
while (query.next())
|
||||
{
|
||||
QString str = GetMainWindow()->m_pFilterObjectDlg->JsonToString(query.value(1).toInt(),query.value(2).toString());
|
||||
QListWidgetItem *pItem = new QListWidgetItem(str, m_plwFilter);
|
||||
pItem->setData(Qt::UserRole, QVariant(QList<QVariant>() << query.value(1) << query.value(2)));
|
||||
}
|
||||
}
|
||||
{
|
||||
m_pleFilter->clear();
|
||||
QSqlQuery query("select id,name from filtergroup where id = " + QString::number(_nGroup));
|
||||
while (query.next())
|
||||
m_pleFilter->setText(query.value(1).toString());
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
bool SBatchRun::ReloadColumn()
|
||||
{
|
||||
QVector <QStringList> vecColumn;
|
||||
QFile file("column.txt");
|
||||
if (file.open(QIODevice::ReadOnly | QIODevice::Text) == false) return false;
|
||||
{
|
||||
while (!file.atEnd())
|
||||
{
|
||||
QString str = QString(file.readLine());
|
||||
if (str.at(0) == QChar('#')) continue;
|
||||
if (str.trimmed().isEmpty()) continue;
|
||||
vecColumn.push_back(str.split(","));
|
||||
}
|
||||
}
|
||||
if (vecColumn.size() <= 0) return false;
|
||||
|
||||
foreach(QStringList strList,vecColumn)
|
||||
m_listDataBaseColumn.push_back(strList.at(E_COLUMN_DATABASE));
|
||||
|
||||
for(int i = 0; i < m_listDataBaseColumn.size(); i++)
|
||||
{
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("platform_title")) m_anData[E_DATA_platform_title] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_form")) m_anData[E_DATA_article_form] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_parent")) m_anData[E_DATA_article_parent] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_id")) m_anData[E_DATA_article_id] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_nickname")) m_anData[E_DATA_article_nickname] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_title")) m_anData[E_DATA_article_title] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_data")) m_anData[E_DATA_article_data] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_url")) m_anData[E_DATA_article_url] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_hit")) m_anData[E_DATA_article_hit] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_date")) m_anData[E_DATA_article_date] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_order")) m_anData[E_DATA_article_order] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("platform_id")) m_anData[E_DATA_platform_id] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("platform_name")) m_anData[E_DATA_platform_name] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("platform_form")) m_anData[E_DATA_platform_form] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("reply_url")) m_anData[E_DATA_reply_url] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("keyword_id")) m_anData[E_DATA_keyword_id] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_profileurl")) m_anData[E_DATA_article_profileurl] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("article_profile")) m_anData[E_DATA_article_profile] = i;
|
||||
if (m_listDataBaseColumn[i].trimmed() == QString("etc")) m_anData[E_DATA_etc] = i;
|
||||
}
|
||||
|
||||
m_nUrlColumn = m_anData[E_DATA_article_url];
|
||||
m_nDateColumn = m_anData[E_DATA_article_date];
|
||||
m_nOrderColumn = m_anData[E_DATA_article_order];
|
||||
return true;
|
||||
}
|
||||
*/
|
||||
|
||||
bool SBatchRun::ReloadColumn()
|
||||
{
|
||||
m_anData[E_DATA_platform_title] = GetColumn().getColumnIndex(QString("platform_title"));
|
||||
m_anData[E_DATA_article_form] = GetColumn().getColumnIndex(QString("article_form"));
|
||||
m_anData[E_DATA_article_parent] = GetColumn().getColumnIndex(QString("article_parent"));
|
||||
m_anData[E_DATA_article_id] = GetColumn().getColumnIndex(QString("article_id"));
|
||||
m_anData[E_DATA_article_nickname] = GetColumn().getColumnIndex(QString("article_nickname"));
|
||||
m_anData[E_DATA_article_title] = GetColumn().getColumnIndex(QString("article_title"));
|
||||
m_anData[E_DATA_article_data] = GetColumn().getColumnIndex(QString("article_data"));
|
||||
m_anData[E_DATA_article_url] = GetColumn().getColumnIndex(QString("article_url"));
|
||||
m_anData[E_DATA_article_hit] = GetColumn().getColumnIndex(QString("article_hit"));
|
||||
m_anData[E_DATA_article_date] = GetColumn().getColumnIndex(QString("article_date"));
|
||||
m_anData[E_DATA_article_order] = GetColumn().getColumnIndex(QString("article_order"));
|
||||
m_anData[E_DATA_platform_id] = GetColumn().getColumnIndex(QString("platform_id"));
|
||||
m_anData[E_DATA_platform_name] = GetColumn().getColumnIndex(QString("platform_name"));
|
||||
m_anData[E_DATA_platform_form] = GetColumn().getColumnIndex(QString("platform_form"));
|
||||
m_anData[E_DATA_reply_url] = GetColumn().getColumnIndex(QString("reply_url"));
|
||||
m_anData[E_DATA_keyword_id] = GetColumn().getColumnIndex(QString("keyword_id"));
|
||||
m_anData[E_DATA_article_profileurl] = GetColumn().getColumnIndex(QString("article_profileurl"));
|
||||
m_anData[E_DATA_article_profile] = GetColumn().getColumnIndex(QString("article_profile"));
|
||||
m_anData[E_DATA_etc] = GetColumn().getColumnIndex(QString("etc"));
|
||||
|
||||
m_nUrlColumn = m_anData[E_DATA_article_url];
|
||||
m_nDateColumn = m_anData[E_DATA_article_date];
|
||||
m_nOrderColumn = m_anData[E_DATA_article_order];
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool SBatchRun::loadData()
|
||||
{
|
||||
QList<QListWidgetItem*> items = m_plwData->selectedItems();
|
||||
if(items.length() < 1 || items.length() > 1)
|
||||
return false;
|
||||
|
||||
clearData();
|
||||
|
||||
QString _strDataNum = QString::number(items[0]->data(Qt::UserRole).toInt());
|
||||
int nCutDays = m_pleCutDays->text().toInt();
|
||||
QDate dateFrom = m_pdeFrom->date();
|
||||
QDate dateTo = m_pdeTo->date();
|
||||
|
||||
|
||||
QString strSelect;
|
||||
strSelect = "select ";
|
||||
qDebug() << "GetColumn().data().size() : "<< GetColumn().data().size();
|
||||
foreach (QStringList strListColumn, GetColumn().data())
|
||||
{
|
||||
qDebug() << strListColumn[E_COLUMN_DATABASE];
|
||||
strSelect += "CONVERT(" + strListColumn[E_COLUMN_DATABASE] + " USING utf8),";
|
||||
}
|
||||
/*
|
||||
foreach (QString strColumn,m_listDataBaseColumn)
|
||||
strSelect += "CONVERT(" + strColumn + " USING utf8),";
|
||||
*/
|
||||
strSelect = strSelect.left(strSelect.size() - 1);
|
||||
strSelect += " from ";
|
||||
strSelect += "data_" + _strDataNum;
|
||||
|
||||
QString astrWhere[E_ARTICLE_MAX] = {" where article_form = 'body'"," where article_form = 'reply'"};
|
||||
astrWhere[E_ARTICLE_BODY] += " and date(article_date) >= ";
|
||||
astrWhere[E_ARTICLE_REPLY] += " and date(article_date) >= ";
|
||||
if(m_pgbCutDays->isChecked())
|
||||
{
|
||||
astrWhere[E_ARTICLE_BODY] += "date(subdate(now(), INTERVAL ";
|
||||
astrWhere[E_ARTICLE_BODY] += QString::number(abs(nCutDays));
|
||||
astrWhere[E_ARTICLE_BODY] += " DAY))";
|
||||
astrWhere[E_ARTICLE_BODY] += " and date(article_date) <= date(now())";
|
||||
|
||||
astrWhere[E_ARTICLE_REPLY] += "date(subdate(now(), INTERVAL ";
|
||||
astrWhere[E_ARTICLE_REPLY] += QString::number(abs(nCutDays));
|
||||
astrWhere[E_ARTICLE_REPLY] += " DAY))";
|
||||
}
|
||||
else
|
||||
{
|
||||
astrWhere[E_ARTICLE_BODY] += "date('" + dateFrom.toString("yyyy-MM-dd") + "') ";
|
||||
astrWhere[E_ARTICLE_BODY] += " and date(article_date) <= date('" + dateTo.toString("yyyy-MM-dd") + "')";
|
||||
|
||||
astrWhere[E_ARTICLE_REPLY] += "date('" + dateFrom.toString("yyyy-MM-dd") + "') ";
|
||||
}
|
||||
astrWhere[E_ARTICLE_BODY] += " Order by article_date DESC";
|
||||
|
||||
QSqlQuery query;
|
||||
QSet<QString> setUrl;
|
||||
|
||||
for (int i = 0; i < E_ARTICLE_MAX ; i++)
|
||||
{
|
||||
if(query.exec(QString(strSelect + astrWhere[i]).toUtf8()) == false)
|
||||
{
|
||||
qDebug() << query.lastQuery();
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
while(query.next())
|
||||
{
|
||||
QStringList strList;
|
||||
strList.clear();
|
||||
for(int j = 0; j < GetColumn().data().size(); j++)
|
||||
//strList << " " + query.value(j).toString() + " ";
|
||||
strList.append(query.value(j).toString());
|
||||
|
||||
if(strList[m_anData[E_DATA_platform_form]].trimmed() == "post" || strList[m_anData[E_DATA_platform_form]].trimmed() == "story" || strList[m_anData[E_DATA_platform_form]].trimmed() == "channel")
|
||||
strList[m_anData[E_DATA_platform_form]] = "sns";
|
||||
|
||||
if(strList[m_anData[E_DATA_platform_form]].trimmed() == "group")
|
||||
strList[m_anData[E_DATA_platform_form]] = "community";
|
||||
|
||||
|
||||
if(i == E_ARTICLE_BODY)
|
||||
{
|
||||
if (strList[m_anData[E_DATA_article_date]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_article_id]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_article_nickname]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_platform_id]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_platform_id]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_article_date]].trimmed() == "0000-00-00 00:00:00")
|
||||
strList[m_anData[E_DATA_article_date]] = "1990-01-01 00:00:00";
|
||||
|
||||
setUrl.insert(strList[m_anData[E_DATA_article_url]].trimmed());
|
||||
m_vecData[i].append(strList);
|
||||
}
|
||||
|
||||
else if(i == E_ARTICLE_REPLY)
|
||||
{
|
||||
if (strList[m_anData[E_DATA_article_id]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_article_nickname]].trimmed().isEmpty()) continue;
|
||||
if (strList[m_anData[E_DATA_article_date]].trimmed() == "0000-00-00 00:00:00")
|
||||
strList[m_anData[E_DATA_article_date]] = "1990-01-01 00:00:00";
|
||||
|
||||
if(setUrl.contains(strList[m_anData[E_DATA_article_url]].trimmed()))
|
||||
{
|
||||
m_vecData[i].append(strList);
|
||||
}
|
||||
}
|
||||
}
|
||||
query.clear();
|
||||
}
|
||||
|
||||
setUrl.clear();
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
m_data[i] = m_vecData[i];
|
||||
}
|
||||
|
||||
//ViewRawResult();
|
||||
return true;
|
||||
}
|
||||
|
||||
void SBatchRun::Message(QString strMessage)
|
||||
{
|
||||
QMessageBox msgBox;
|
||||
msgBox.setText(strMessage);
|
||||
msgBox.exec();
|
||||
}
|
||||
|
||||
bool SBatchRun::execFilter()
|
||||
{
|
||||
DataFilter();
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::clearData()
|
||||
{
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
for(int j = 0; j < m_vecData[i].length(); j++)
|
||||
{
|
||||
m_vecData[i][j].clear();
|
||||
}
|
||||
for(int j = 0; j < m_data[i].length(); j++)
|
||||
{
|
||||
m_data[i][j].clear();
|
||||
}
|
||||
m_vecData[i].clear();
|
||||
m_data[i].clear();
|
||||
}
|
||||
for(int i = 0; i < E_ARTICLE_MAX; i++)
|
||||
{
|
||||
m_bDataFlag[i] = false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::DataFilter()
|
||||
{
|
||||
qDebug() << m_plwFilter->count();
|
||||
for(int i = 0; i < m_plwFilter->count(); i++)
|
||||
{
|
||||
QList<QVariant> filterInfo = m_plwFilter->item(i)->data(Qt::UserRole).toList();
|
||||
// qDebug() << filterInfo[0].toInt();
|
||||
// qDebug() << filterInfo[1].toString();
|
||||
switch(filterInfo.at(0).toInt())
|
||||
{
|
||||
case E_FILTER_TYPE_DATE://Date
|
||||
FilterDate(filterInfo.at(1).toString());
|
||||
break;
|
||||
case E_FILTER_TYPE_SEARCH:
|
||||
FilterSerach(filterInfo.at(1).toString());
|
||||
break;
|
||||
case E_FILTER_TYPE_LENGTH:
|
||||
FilterLength(filterInfo.at(1).toString());
|
||||
break;
|
||||
case E_FILTER_TYPE_REPLACE:
|
||||
FilterReplace(filterInfo.at(1).toString());
|
||||
break;
|
||||
case E_FILTER_TYPE_COUNTER:
|
||||
FilterCounter(filterInfo.at(1).toString());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::FilterDate(QString _strJson)
|
||||
{
|
||||
SJson json;
|
||||
int nSelect = json.Get(_strJson,"Article").toInt() - 1;
|
||||
if (nSelect < 0) return;
|
||||
m_bDataFlag[nSelect] = true;
|
||||
QVector <QStringList> copyData;
|
||||
QDate dateStart = QDate::fromString(json.Get(_strJson,"Start"),"yyyy-MM-dd");
|
||||
QDate dateEnd = QDate::fromString(json.Get(_strJson,"End"),"yyyy-MM-dd");
|
||||
foreach(QStringList strList , m_data[nSelect])
|
||||
{
|
||||
bool bFlag = false;
|
||||
QString strTime = strList.at(m_nDateColumn).trimmed();
|
||||
if (strTime.size() >= 10)
|
||||
{
|
||||
QChar ch = strTime.at(4);
|
||||
QString strFormat = QString("yyyy")+ch+QString("MM")+ch+QString("dd");
|
||||
QDate date = QDate::fromString(strTime.left(10),strFormat);
|
||||
if (dateStart <= date && dateEnd >= date)
|
||||
bFlag = true;
|
||||
}
|
||||
if (bFlag)
|
||||
copyData.push_back(strList);
|
||||
}
|
||||
m_data[nSelect].clear();
|
||||
m_data[nSelect] = copyData;
|
||||
}
|
||||
|
||||
void SBatchRun::FilterLength(QString _strJson)
|
||||
{
|
||||
SJson json;
|
||||
int nSelect = json.Get(_strJson,"Article").toInt() - 1;
|
||||
if (nSelect < 0) return;
|
||||
m_bDataFlag[nSelect] = true;
|
||||
|
||||
int nCatalog = json.GetNumber(_strJson,"Category");
|
||||
int nComp = json.GetNumber(_strJson,"Comp");
|
||||
int nFlag = json.GetNumber(_strJson,"InsDel");
|
||||
int nLength = json.Get(_strJson,"String").toInt();
|
||||
|
||||
QVector <QStringList> copyData;
|
||||
foreach(QStringList strList , m_data[nSelect])
|
||||
{
|
||||
bool bFlag = false;
|
||||
int nCurrentLength = strList.at(nCatalog).trimmed().length();
|
||||
switch(nComp)
|
||||
{
|
||||
case E_LENGTH_COMP_GREATER:
|
||||
if (nCurrentLength > nLength)
|
||||
bFlag = true;
|
||||
break;
|
||||
case E_LENGTH_COMP_LESS:
|
||||
if (nCurrentLength < nLength)
|
||||
bFlag = true;
|
||||
break;
|
||||
case E_LENGTH_COMP_EQUAL:
|
||||
if (nCurrentLength == nLength)
|
||||
bFlag = true;
|
||||
break;
|
||||
}
|
||||
if (nFlag == 1) bFlag = !bFlag;
|
||||
if (bFlag)
|
||||
copyData.push_back(strList);
|
||||
}
|
||||
m_data[nSelect].clear();
|
||||
m_data[nSelect] = copyData;
|
||||
}
|
||||
|
||||
|
||||
void SBatchRun::FilterCounter(QString _strJson)
|
||||
{
|
||||
SJson json;
|
||||
int nSelect = json.Get(_strJson,"Article").toInt() - 1;
|
||||
if (nSelect < 0) return;
|
||||
m_bDataFlag[nSelect] = true;
|
||||
|
||||
int nCatalog = json.GetNumber(_strJson,"Category");
|
||||
int nComp = json.GetNumber(_strJson,"Comp");
|
||||
int nFlag = json.GetNumber(_strJson,"InsDel");
|
||||
int nCounter = json.Get(_strJson,"String").toInt();
|
||||
|
||||
QMap <QString, int> mapData, mapDataresult;
|
||||
foreach(QStringList strList , m_data[nSelect])
|
||||
{
|
||||
QString str = strList.at(nCatalog).trimmed();
|
||||
if(mapData.contains(str))
|
||||
mapData[str]++;
|
||||
else
|
||||
mapData.insert(str, 1);
|
||||
}
|
||||
|
||||
for(QMap <QString, int>::iterator iterPos = mapData.begin(); iterPos != mapData.end(); iterPos++)
|
||||
{
|
||||
bool bFlag = false;
|
||||
switch(nComp)
|
||||
{
|
||||
case E_LENGTH_COMP_GREATER:
|
||||
if (iterPos.value() > nCounter)
|
||||
bFlag = true;
|
||||
break;
|
||||
case E_LENGTH_COMP_LESS:
|
||||
if (iterPos.value() < nCounter)
|
||||
bFlag = true;
|
||||
break;
|
||||
case E_LENGTH_COMP_EQUAL:
|
||||
if (iterPos.value() == nCounter)
|
||||
bFlag = true;
|
||||
break;
|
||||
}
|
||||
if (nFlag == 1) bFlag = !bFlag;
|
||||
if (bFlag)
|
||||
mapDataresult.insert(iterPos.key(), iterPos.value());
|
||||
}
|
||||
|
||||
mapData.clear();
|
||||
|
||||
QVector <QStringList> copyData;
|
||||
foreach(QStringList strList , m_data[nSelect])
|
||||
{
|
||||
QString str = strList.at(nCatalog).trimmed();
|
||||
if(mapDataresult.contains(str))
|
||||
copyData.push_back(strList);
|
||||
}
|
||||
m_data[nSelect].clear();
|
||||
m_data[nSelect] = copyData;
|
||||
}
|
||||
|
||||
void SBatchRun::FilterReplace(QString _strJson)
|
||||
{
|
||||
SJson json;
|
||||
int nSelect = json.Get(_strJson,"Article").toInt() - 1;
|
||||
if (nSelect < 0) return;
|
||||
m_bDataFlag[nSelect] = true;
|
||||
|
||||
int nCatalog = json.GetNumber(_strJson,"Category");
|
||||
QString strFind = json.Get(_strJson,"String_Find");
|
||||
QString strReplace = json.Get(_strJson,"String_Replace");
|
||||
|
||||
bool bFlag = false;
|
||||
QStringList strListKeyword;
|
||||
if (json.GetNumber(_strJson,"Find") == E_REPLACE_SPACE)
|
||||
{
|
||||
bFlag = true;
|
||||
strListKeyword = strFind.split(" ");
|
||||
}
|
||||
|
||||
for (int i = 0 ; i < m_data[nSelect].size(); i++)
|
||||
{
|
||||
if (bFlag)
|
||||
{
|
||||
foreach(QString str,strListKeyword)
|
||||
{
|
||||
m_data[nSelect][i][nCatalog] = m_data[nSelect][i][nCatalog].replace(str,strReplace);
|
||||
}
|
||||
}
|
||||
else
|
||||
m_data[nSelect][i][nCatalog] = m_data[nSelect][i][nCatalog].replace(strFind,strReplace);
|
||||
}
|
||||
}
|
||||
|
||||
void SBatchRun::FilterSerach(QString _strJson)
|
||||
{
|
||||
SJson json;
|
||||
int nSelect = json.Get(_strJson,"Article").toInt() - 1;
|
||||
if (nSelect < 0) return;
|
||||
m_bDataFlag[nSelect] = true;
|
||||
{
|
||||
SKeyword stKeyword;
|
||||
stKeyword.m_nCatalog = json.GetNumber(_strJson,"Category");
|
||||
stKeyword.m_nKeyword = json.GetNumber(_strJson,"Keyword");
|
||||
stKeyword.m_strListKeyword = json.Get(_strJson,"String").split(" ");
|
||||
if (json.GetNumber(_strJson,"Method") == 1)
|
||||
{
|
||||
for (int i = 0 ; i < stKeyword.m_strListKeyword.size(); i++ )
|
||||
stKeyword.m_strListKeyword[i] = " " + stKeyword.m_strListKeyword[i] + " ";
|
||||
}
|
||||
m_vecKeyword.push_back(stKeyword);
|
||||
}
|
||||
if (json.GetBool(_strJson,"Last")) //
|
||||
{
|
||||
QVector <QStringList> copyData;
|
||||
|
||||
foreach(QStringList strList , m_data[nSelect])
|
||||
{
|
||||
bool bFlag = false;
|
||||
foreach(SKeyword stKeyword,m_vecKeyword)
|
||||
{
|
||||
QString strData = strList.at(stKeyword.m_nCatalog);
|
||||
switch(stKeyword.m_nKeyword)
|
||||
{
|
||||
case E_SEARCH_KEYWORD_OR:
|
||||
{
|
||||
foreach(QString strKey , stKeyword.m_strListKeyword)
|
||||
if (strData.contains(strKey)){bFlag = true;break;}
|
||||
break;
|
||||
}
|
||||
case E_SEARCH_KEYWORD_AND:
|
||||
{
|
||||
int nKeyCount = 0;
|
||||
foreach(QString strKey , stKeyword.m_strListKeyword)
|
||||
if (strData.contains(strKey)) nKeyCount++;
|
||||
if (nKeyCount == stKeyword.m_strListKeyword.size())
|
||||
bFlag = true;
|
||||
break;
|
||||
}
|
||||
case E_SEARCH_KEYWORD_CELL_DELETE_OR:
|
||||
{
|
||||
bFlag = true;
|
||||
foreach(QString strKey , stKeyword.m_strListKeyword)
|
||||
if (strData.contains(strKey)){bFlag = false;break;}
|
||||
break;
|
||||
}
|
||||
case E_SEARCH_KEYWORD_CELL_DELETE_AND:
|
||||
{
|
||||
bFlag = true;
|
||||
int nCount = 0;
|
||||
foreach(QString strKey , stKeyword.m_strListKeyword)
|
||||
if (strData.contains(strKey)){nCount++;}
|
||||
if (nCount == stKeyword.m_strListKeyword.size())
|
||||
bFlag = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bFlag) copyData.push_back(strList);
|
||||
}
|
||||
m_data[nSelect].clear();
|
||||
m_data[nSelect] = copyData;
|
||||
m_vecKeyword.clear();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user