etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

실시간 중간 버젼

Browse Source 
git-svn-id: svn://192.168.0.12/source@78 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-04-17 05:22:20 +00:00
parent b20c30a93b
commit f553261f78
7 changed files with 272 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CrawlerList/smanage.cpp
View File

@@ -120,6 +120,7 @@ void SManage::CheckLast()
{ {
m_bFinalLast = true; m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)"); m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
} }
m_ncList=1; m_ncList=1;
} }

5
CrawlerList/snavercafemanage.cpp
View File

@@ -100,11 +100,6 @@ void SNaverCafeManage::processFinished(QProcess *_pPro,QString _strOut)
m_bLast = false; m_bLast = false;
if (_strOut.right(4) == "last" || m_ncList >= 1000) if (_strOut.right(4) == "last" || m_ncList >= 1000)
m_bLast = true; m_bLast = true;
// if (_strOut.right(5) == "block")
// //m_bFinalLast = true;
if (_strOut.right(5) == "block") if (_strOut.right(5) == "block")
{ {
ReLoadList(); ReLoadList();

290
CrawlerList/widget.cpp
View File

@@ -6,54 +6,23 @@
#include <QFileDialog> #include <QFileDialog>
#include <QPushButton> #include <QPushButton>
#include <QSqlQuery> #include <QSqlQuery>
#include <QSqlError>
#include <QThread> #include <QThread>
#include "snavercafemanage.h" #include "snavercafemanage.h"
#include "snaverblogmanage.h" #include "snaverblogmanage.h"
#include "sdaumcafemanage.h" #include "sdaumcafemanage.h"
Widget::Widget(QWidget *parent) : QWidget(parent)
Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
{ {
connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update())); connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update()));
m_pcb = new QComboBox;
m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent);
QPushButton *pbtRefresh = new QPushButton("Refresh",this);
QPushButton *pbtStart = new QPushButton("Start",this);
QPushButton *pbtStop = new QPushButton("Stop",this);
{
QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton()));
QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton()));
QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton()));
}
QVBoxLayout *vlayout = new QVBoxLayout; QVBoxLayout *vlayout = new QVBoxLayout;
{ {
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pcb,Qt::AlignLeft);
hlayout->addWidget(pbtRefresh);
vlayout->addLayout(hlayout);
}
m_pedTime = new QLineEdit(this);
m_pedTime->setText(QString("3~10"));
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pedTime);
hlayout->addWidget(pbtStart);
hlayout->addWidget(pbtStop);
vlayout->addLayout(hlayout);
}
{
QHBoxLayout *hlayout = new QHBoxLayout;
m_pResultList = new QListWidget; m_pResultList = new QListWidget;
hlayout->addWidget(m_pResultList); vlayout->addWidget(setRealGroupWidgets());
vlayout->addLayout(hlayout); vlayout->addWidget(m_pResultList);
} }
setLayout(vlayout); setLayout(vlayout);
m_pNaverCafe = new SNaverCafeManage(this); m_pNaverCafe = new SNaverCafeManage(this);
m_pNaverBlog = new SNaverBlogManage(this); m_pNaverBlog = new SNaverBlogManage(this);
m_pDaumCafe = new SDaumCafeManage(this); m_pDaumCafe = new SDaumCafeManage(this);
@@ -76,12 +45,65 @@ Widget::Widget(QWidget *parent) : QWidget(parent)
m_pManage[i]->SetParent(this); m_pManage[i]->SetParent(this);
RefreshButton(); RefreshButton();
m_timer.start(1000);
} }
Widget::~Widget() Widget::~Widget()
{ {
if (m_nMode == E_MODE_RUN)
{
if (m_pgbManual->isChecked() == false)
{
if (m_strCrawlingID.isEmpty() == false)
{
SetCrawlingState("Finish");
}
}
}
UnLockTables();
m_db.close(); m_db.close();
m_pNaverCafe->SetParent(0); for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
m_pManage[i]->SetParent(0);
}
QGroupBox *Widget::setRealGroupWidgets()
{
m_pcb = new QComboBox;
m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent);
QPushButton *pbtRefresh = new QPushButton("Refresh",this);
QPushButton *pbtStart = new QPushButton("Start",this);
QPushButton *pbtStop = new QPushButton("Stop",this);
{
QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton()));
QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton()));
QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton()));
}
QVBoxLayout *vlayout = new QVBoxLayout;
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pcb,Qt::AlignLeft);
hlayout->addWidget(pbtRefresh);
vlayout->addLayout(hlayout);
}
m_pedTime = new QLineEdit(this);
m_pedTime->setText(QString("3"));
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pedTime);
hlayout->addWidget(pbtStart);
hlayout->addWidget(pbtStop);
vlayout->addLayout(hlayout);
}
m_pgbManual = new QGroupBox(tr("Manual"));
m_pgbManual->setCheckable(true);
m_pgbManual->setChecked(true);
m_pgbManual->setLayout(vlayout);
return m_pgbManual;
} }
void Widget::InsertLog(QString str) void Widget::InsertLog(QString str)
@@ -108,55 +130,43 @@ void Widget::InsertLog(QString str)
m_pResultList->setCurrentRow( m_pResultList->count() - 1 ); m_pResultList->setCurrentRow( m_pResultList->count() - 1 );
m_pResultList->repaint(); m_pResultList->repaint();
} }
/*
void Widget::InsertLog(int _nSelect,QString str) void Widget::SetCrawlingState(QString _str)
{ {
if (_nSelect >= C_CRAWLER_MAX) return;
QTime time = QTime::currentTime();
QString strOut = time.toString("[hh:mm:ss] ") + str;
m_aResultList[_nSelect].addItem(strOut);
QDate date = QDate::currentDate();
QFile file(date.toString(Qt::ISODate)+"_"+QString::number(_nSelect)+".log");
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
return;
QTextStream out(&file);
out << strOut << "\n";
file.close();
if (m_aResultList[_nSelect].count() > 1024)
{
m_aResultList[_nSelect].removeItemWidget(m_aResultList[_nSelect].item(0));
QListWidgetItem* item = m_aResultList[_nSelect].takeItem(0);
delete item;
}
m_aResultList[_nSelect].setCurrentRow( m_aResultList[_nSelect].count() - 1 );
m_aResultList[_nSelect].repaint();
}
*/
void Widget::StartButton()
{
m_timer.stop();
m_timer.start(1000);
QSqlQuery query; QSqlQuery query;
query.exec("UPDATE crawling set state = '" + QString("run") + "' where id = '" + m_pcb->currentData().toString() + "'"); if (_str == QString("Finish"))
query.exec("SELECT _keyword.start,_keyword.end, _keyword.searches,_keyword.authorship,_keyword.id,_datagroup.id,_keyword.platform " {
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " if (m_pgbManual->isChecked())
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id " query.exec("UPDATE crawling set state = 2 where id = '" + m_strCrawlingID + "'");
"where _crawling.id = '" + m_pcb->currentData().toString() + "'"); else
query.exec("UPDATE crawling set state = 0 where id = '" + m_strCrawlingID + "'");
m_pgbManual->setEnabled(true);
m_nMode = E_MODE_WAIT;
}
if (_str == QString("Start"))
{
if (m_pgbManual->isChecked() == false)
query.exec("UPDATE crawling set state = 1 where id = '" + m_strCrawlingID + "'");
m_pgbManual->setEnabled(false);
}
}
void Widget::Start()
{
QSqlQuery query;
query.exec("SELECT keyword.start,keyword.end, keyword.searches,keyword.authorship,keyword.id,datagroup.id,keyword.platform "
"FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id "
"inner join datagroup on crawling.datagroup_id = datagroup.id "
"where crawling.id = '" + m_strCrawlingID + "'");
query.next(); query.next();
/*
QString str = + " , "; QString str = + " , ";
str += query.value(1).toString() + " ~ "; str += query.value(1).toString() + " ~ ";
str += query.value(2).toString() + " -> "; str += query.value(2).toString() + " -> ";
str += query.value(3).toString(); str += query.value(3).toString();
*/
m_nPlatform = 0; m_nPlatform = 0;
if(0 <= query.value(6).toInt() && query.value(6).toInt() < C_CRAWLER_MAX) if(0 <= query.value(6).toInt() && query.value(6).toInt() < C_CRAWLER_MAX)
m_nPlatform = query.value(6).toInt(); m_nPlatform = query.value(6).toInt();
else else
@@ -165,24 +175,56 @@ void Widget::StartButton()
return; return;
} }
m_pManage[m_nPlatform]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"), if (m_pgbManual->isChecked())
QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"), {
query.value(2).toString().trimmed(),// keyword m_pManage[m_nPlatform]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"),
query.value(3).toString().trimmed(),// authorship QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"),
query.value(4).toString().trimmed(),// keyword_id query.value(2).toString().trimmed(),// keyword
query.value(5).toString().trimmed(), query.value(3).toString().trimmed(),// authorship
1, query.value(4).toString().trimmed(),// keyword_id
m_pedTime->text().trimmed()); query.value(5).toString().trimmed(),
1,
m_pedTime->text().trimmed());
}
else
{
m_pManage[m_nPlatform]->Start(QDate::currentDate(),QDate::currentDate(),
query.value(2).toString().trimmed(),// keyword
query.value(3).toString().trimmed(),// authorship
query.value(4).toString().trimmed(),// keyword_id
query.value(5).toString().trimmed(),
1,
m_pedTime->text().trimmed());
}
SetCrawlingState("Start");
m_nMode = E_MODE_RUN;
}
void Widget::StartButton()
{
m_strCrawlingID = m_pcb->currentData().toString();
Start();
} }
void Widget::StopButton() void Widget::StopButton()
{
m_nMode = E_MODE_WAIT;
}
bool Widget::LockTable(QString _strTable)
{ {
m_timer.stop(); QSqlQuery query;
return query.exec(QString("lock tables " + _strTable + "write" ).toUtf8());
}
void Widget::UnLockTables()
{
QSqlQuery query;
query.exec(QString("unlock tables").toUtf8());
} }
void Widget::Update() void Widget::Update()
{ {
if (m_db.isOpen() == false) if (m_db.isOpen() == false)
{ {
if (m_db.open()) if (m_db.open())
@@ -193,27 +235,57 @@ void Widget::Update()
} }
} }
//int nCount = 0; if (m_pgbManual->isChecked())
//for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
//nCount +=
m_pManage[m_nPlatform]->Update();
/*
if (nCount == C_CRAWLER_MAX)
{ {
InsertLog("Finish..."); // Menual Mode
m_timer.stop(); switch(m_nMode)
{
case E_MODE_WAIT:return;
}
}
else
{
// RealTime Mode
switch(m_nMode)
{
case E_MODE_WAIT:
LockTable("crawling");
QSqlQuery query;
if (query.exec("SELECT crawling.id,keyword.start,keyword.end "
"FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id "
"where crawling.state = 0 and keyword.realtime = 1 and keyword.state is null") == false)
{
InsertLog(query.lastError().text());
UnLockTables();
return;
}
QDate dateNow = QDate::currentDate();
m_strCrawlingID.clear();
while (query.next())
{
QDate dateStart = QDate::fromString(query.value(1).toString(),"yyyy-MM-dd");
QDate dateEnd = QDate::fromString(query.value(2).toString(),"yyyy-MM-dd");
if (dateNow < dateStart) continue;
if (dateNow > dateEnd) continue;
m_strCrawlingID = query.value(0).toString();
break;
}
if(m_strCrawlingID.isEmpty() == false)
Start();
UnLockTables();
return;
}
} }
*/
m_pManage[m_nPlatform]->Update();
} }
void Widget::RefreshButton() void Widget::RefreshButton()
{ {
m_pcb->clear(); m_pcb->clear();
QSqlQuery query("SELECT _keyword.searches,_keyword.authorship,_keyword.start,_keyword.end, _datagroup.name , _crawling.id,_keyword.platform " QSqlQuery query("SELECT keyword.searches,keyword.authorship,keyword.start,keyword.end,datagroup.name,keyword.platform,crawling.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id where _crawling.state is null"); "inner join datagroup on crawling.datagroup_id = datagroup.id where crawling.state = 0 and keyword.realtime = 0 ");
while (query.next()) while (query.next())
{ {
QString str; QString str;
@@ -222,12 +294,12 @@ void Widget::RefreshButton()
str += query.value(2).toString() + " ~ "; str += query.value(2).toString() + " ~ ";
str += query.value(3).toString() + " -> "; str += query.value(3).toString() + " -> ";
str += query.value(4).toString(); str += query.value(4).toString();
if(query.value(6).toString().toInt() == 0) switch(query.value(5).toString().toInt())
str += ", NaverCafe"; {
else if(query.value(6).toString().toInt() == 1) case 0:str += ", Naver Cafe";break;
str += ", NaverBlog"; case 1:str += ", Naver Blog";break;
else if(query.value(6).toString().toInt() == 2) case 2:str += ", Daum Cafe"; break;
str += ", DaumCafe"; }
m_pcb->addItem(str,query.value(5)); m_pcb->addItem(str,query.value(6));
} }
} }

16
CrawlerList/widget.h
View File

@@ -8,6 +8,7 @@
#include <QListWidget> #include <QListWidget>
#include <QSqlDatabase> #include <QSqlDatabase>
#include <QComboBox> #include <QComboBox>
#include <QGroupBox>
class SNaverCafeManage; class SNaverCafeManage;
class SNaverBlogManage; class SNaverBlogManage;
@@ -24,7 +25,15 @@ public:
~Widget(); ~Widget();
public: public:
void InsertLog(QString str); void InsertLog(QString str);
// void InsertLog(int _nSelect,QString str); void SetCrawlingState(QString _str);
bool LockTable(QString strTable);
void UnLockTables();
enum E_MODE
{
E_MODE_WAIT = 0,
E_MODE_RUN,
};
private: private:
QLineEdit *m_pedTime; QLineEdit *m_pedTime;
@@ -40,8 +49,13 @@ private:
SNaverBlogManage *m_pNaverBlog; SNaverBlogManage *m_pNaverBlog;
SDaumCafeManage *m_pDaumCafe; SDaumCafeManage *m_pDaumCafe;
int m_nStartTime,m_nRangeTime,m_nPlatform; int m_nStartTime,m_nRangeTime,m_nPlatform;
QGroupBox *m_pgbManual;
int m_nMode;
QString m_strCrawlingID;
private: private:
void Start();
QString makeCafeGetListQuery(QString _str,QDate _date,int _nPage); QString makeCafeGetListQuery(QString _str,QDate _date,int _nPage);
QGroupBox *setRealGroupWidgets();
private slots: private slots:
void RefreshButton(); void RefreshButton();
void StartButton(); void StartButton();

74
CrawlerProcess/scrawler.cpp
View File

@@ -34,37 +34,7 @@ void SCrawler::load(QStringList _strlistArgv)
m_strUrl = _strlistArgv[2]; m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_CAFE_LIST; m_nSelect = E_NAVER_CAFE_LIST;
m_strKeywordID = _strlistArgv[4]; m_strKeywordID = _strlistArgv[4];
/*
QFile file("proxy.txt");
if (file.open(QIODevice::ReadOnly | QIODevice::Text))
{
QVector <QStringList> vecProxy;
while (!file.atEnd())
{
QString str = QString(file.readLine());
if (str.isEmpty()) continue;
vecProxy.push_back(str.split(","));
}
if (vecProxy.size() > 0)
{
QStringList strList = vecProxy.at(rand()%vecProxy.size());
switch(strList.size())
{
case 1:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0))));
break;
case 2:
cout << "p : " << strList.at(0).toStdString() << endl;
QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0),strList.at(1).toInt())));
break;
}
}
}*/
setProxy(); setProxy();
} }
if (_strlistArgv[1] == "cafe_data") if (_strlistArgv[1] == "cafe_data")
@@ -79,15 +49,13 @@ void SCrawler::load(QStringList _strlistArgv)
m_strUrl = _strlistArgv[2]; m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_BLOG_LIST; m_nSelect = E_NAVER_BLOG_LIST;
m_strKeywordID = _strlistArgv[4]; m_strKeywordID = _strlistArgv[4];
setProxy(); setProxy();
} }
if (_strlistArgv[1] == "blog_url") if (_strlistArgv[1] == "blog_url")
{ {
m_strUrl = _strlistArgv[2]; m_strUrl = _strlistArgv[2];
m_nSelect = E_NAVER_BLOG_BODY; m_nSelect = E_NAVER_BLOG_BODY;
//m_strReper = _strlistArgv[4];
m_bUse = true; m_bUse = true;
} }
@@ -310,7 +278,10 @@ void SCrawler::saveFrameList(QWebFrame *frame)
QString strBlogMe = "blog.me"; QString strBlogMe = "blog.me";
if ((strList.at(0).compare("blog.naver.com") != 0 ) && (strList.at(0).right(strBlogMe.length()).compare(strBlogMe) != 0)) { cout << "x http://" << strUrl.toStdString() <<endl; continue; }; if ((strList.at(0).compare("blog.naver.com") != 0 ) && (strList.at(0).right(strBlogMe.length()).compare(strBlogMe) != 0))
{
cout << "x http://" << strUrl.toStdString() <<endl; continue;
}
if((strList.at(0).right(strBlogMe.length()).compare(strBlogMe) == 0)) if((strList.at(0).right(strBlogMe.length()).compare(strBlogMe) == 0))
{ {
@@ -321,12 +292,20 @@ void SCrawler::saveFrameList(QWebFrame *frame)
strUrl += strList.at(1); strUrl += strList.at(1);
} }
/*
QString strQuery = "select article_url from "; QString strQuery = "select article_url from ";
strQuery += m_strTable; strQuery += m_strTable;
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl); strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery); sql.exec(strQuery);
*/
if (sql.size() == 0 || sql.size() == -1) QString strQuery = "delete from ";
strQuery += m_strTable;
strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery);
//if (sql.size() == 0 || sql.size() == -1)
{ {
QString str = Find(sub,"a","class","txt84").toPlainText(); QString str = Find(sub,"a","class","txt84").toPlainText();
str = GetSafeUtf(str); str = GetSafeUtf(str);
@@ -349,8 +328,8 @@ void SCrawler::saveFrameList(QWebFrame *frame)
else else
cout << "o "; cout << "o ";
} }
else //else
cout << "v "; // cout << "v ";
cout << "http://" << strUrl.toStdString() << endl; cout << "http://" << strUrl.toStdString() << endl;
m_bUse = true; m_bUse = true;
} }
@@ -741,11 +720,11 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
{ {
QSqlQuery sql; QSqlQuery sql;
QString strQuery = "select article_url from "; QString strQuery = "delete from ";
strQuery += m_strTable; strQuery += m_strTable;
strQuery += QString(" where article_url = '%1'").arg(strUrl); strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery); sql.exec(strQuery);
if (sql.size() == 0 || sql.size() == -1) //if (sql.size() == 0 || sql.size() == -1)
{ {
QString strQuery = QString("insert into "); QString strQuery = QString("insert into ");
strQuery += m_strTable; strQuery += m_strTable;
@@ -758,8 +737,8 @@ void SCrawler::saveFrameCafeList(QWebFrame *frame)
cout << "o " << strUrl.toStdString() << endl; cout << "o " << strUrl.toStdString() << endl;
} }
} }
else //else
cout << "v " << strUrl.toStdString() << endl; // cout << "v " << strUrl.toStdString() << endl;
} }
m_bUse = true; m_bUse = true;
} }
@@ -962,11 +941,10 @@ void SCrawler::saveFrameDaumCafeList(QWebFrame *frame)
{ {
QSqlQuery sql; QSqlQuery sql;
QString strQuery = "select article_url from "; QString strQuery = "delete from ";
strQuery += m_strTable; strQuery += m_strTable;
strQuery += QString(" where article_url = '%1'").arg(strUrl); strQuery += QString(" where article_url = 'http://%1'").arg(strUrl);
sql.exec(strQuery); sql.exec(strQuery);
if (sql.size() == 0 || sql.size() == -1)
{ {
QString strQuery = QString("insert into "); QString strQuery = QString("insert into ");
strQuery += m_strTable; strQuery += m_strTable;
@@ -977,8 +955,8 @@ void SCrawler::saveFrameDaumCafeList(QWebFrame *frame)
else else
cout << "o " << strUrl.toStdString() << endl; cout << "o " << strUrl.toStdString() << endl;
} }
else //else
cout << "v " << strUrl.toStdString() << endl; // cout << "v " << strUrl.toStdString() << endl;
} }
m_bUse = true; m_bUse = true;
} }

192
GroupManager/widget.cpp
View File

@@ -12,6 +12,7 @@
#include <QFile> #include <QFile>
#include <QStack> #include <QStack>
#include <QFileDialog> #include <QFileDialog>
Widget::Widget(QWidget *parent) Widget::Widget(QWidget *parent)
: QWidget(parent) : QWidget(parent)
{ {
@@ -29,18 +30,14 @@ Widget::Widget(QWidget *parent)
m_pmodelGroup = new QSqlQueryModel; m_pmodelGroup = new QSqlQueryModel;
m_pmodelCrawling = new QSqlQueryModel; m_pmodelCrawling = new QSqlQueryModel;
m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform "
"WHEN 0 THEN 'Naver Cafe' "
"WHEN 1 THEN 'Naver Blog' "
"WHEN 2 THEN 'Daum Cafe' "
"ELSE 'UnKnown'"
"END AS platform FROM keyword where state is null");
m_pmodelGroup->setQuery("SELECT * FROM datagroup"); m_pmodelGroup->setQuery("SELECT * FROM datagroup");
/* UpdateCrawling();
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
*/
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state, _keyword.platform "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
QVBoxLayout *vlayout = new QVBoxLayout(); QVBoxLayout *vlayout = new QVBoxLayout();
{ {
@@ -74,10 +71,10 @@ QGroupBox *Widget::setKeywordWidgets()
} }
{ {
m_pdeStart = new QDateEdit(QDate::currentDate()); m_pdeStart = new QDateEdit(QDate::currentDate());
m_pdeEnd = new QDateEdit(QDate::currentDate()); m_pdeEnd = new QDateEdit(QDate::currentDate());
m_pdeStart->setDateRange(QDate(2003, 5, 20),QDate::currentDate()); m_pdeStart->setDateRange(QDate(2003, 5, 20),QDate::currentDate());
m_pdeEnd->setDateRange(QDate(2003, 5, 20),QDate::currentDate()); m_pdeEnd->setDateRange(QDate(2003, 5, 20),QDate::currentDate().addYears(1));
QCalendarWidget *pCalender = new QCalendarWidget(); QCalendarWidget *pCalender = new QCalendarWidget();
m_pdeStart->setCalendarWidget(pCalender); m_pdeStart->setCalendarWidget(pCalender);
@@ -86,11 +83,11 @@ QGroupBox *Widget::setKeywordWidgets()
m_pdeEnd->setCalendarWidget(pCalender); m_pdeEnd->setCalendarWidget(pCalender);
m_pdeEnd->setCalendarPopup(true); m_pdeEnd->setCalendarPopup(true);
m_pcbRealTime = new QComboBox; m_pcbRealTime = new QComboBox;
m_pcbRealTime->addItems(QStringList() << "false" << "true"); m_pcbRealTime->addItems(QStringList() << "false" << "true");
m_pcbPlatform = new QComboBox; m_pcbPlatform = new QComboBox;
m_pcbPlatform->addItems(QStringList() << "NaverCafe" << "NaverBlog" << "DaumCafe"); m_pcbPlatform->addItems(QStringList() << "Naver Cafe" << "Naver Blog" << "Daum Cafe");
m_pleKeyword = new QLineEdit; m_pleKeyword = new QLineEdit;
m_pleAuthorship = new QLineEdit; m_pleAuthorship = new QLineEdit;
@@ -284,7 +281,6 @@ QGroupBox *Widget::setCrawlingWidgets()
return groupBox; return groupBox;
} }
void Widget::on_keyword_currentRowChanged(QModelIndex _index) void Widget::on_keyword_currentRowChanged(QModelIndex _index)
{ {
if (_index.isValid()) if (_index.isValid())
@@ -300,20 +296,6 @@ void Widget::on_keyword_currentRowChanged(QModelIndex _index)
void Widget::on_keyword_button_insert() void Widget::on_keyword_button_insert()
{ {
/*
QString strQuery = QString("insert into keyword set "
"start = STR_TO_DATE('%1', '%Y-%m-%d'),"
"end = STR_TO_DATE('%2', '%Y-%m-%d'),"
"searches = '%3',"
"realtime = %4,"
"authorship = '%5'")
.arg(m_pdeStart->date().toString("yyyy-MM-dd"))
.arg(m_pdeEnd->date().toString("yyyy-MM-dd"))
.arg(m_pleKeyword->text())
.arg(m_pcbRealTime->currentIndex())
.arg(m_pleAuthorship->text());
*/
QString strQuery = QString("insert into keyword set " QString strQuery = QString("insert into keyword set "
"start = STR_TO_DATE('%1', '%Y-%m-%d')," "start = STR_TO_DATE('%1', '%Y-%m-%d'),"
"end = STR_TO_DATE('%2', '%Y-%m-%d')," "end = STR_TO_DATE('%2', '%Y-%m-%d'),"
@@ -328,10 +310,13 @@ void Widget::on_keyword_button_insert()
.arg(m_pleAuthorship->text()) .arg(m_pleAuthorship->text())
.arg(m_pcbPlatform->currentIndex()); .arg(m_pcbPlatform->currentIndex());
//qDebug() << strQuery;
m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); m_pmodelKeyword->setQuery(QString(strQuery.toUtf8()));
m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform "
"WHEN 0 THEN 'Naver Cafe' "
"WHEN 1 THEN 'Naver Blog' "
"WHEN 2 THEN 'Daum Cafe' "
"ELSE 'UnKnown'"
"END AS platform FROM keyword where state is null");
} }
void Widget::on_keyword_button_delete() void Widget::on_keyword_button_delete()
@@ -342,7 +327,12 @@ void Widget::on_keyword_button_delete()
QString strQuery = QString("UPDATE keyword set state = '1' where id = '" + rec.value("id").toString() + "'"); QString strQuery = QString("UPDATE keyword set state = '1' where id = '" + rec.value("id").toString() + "'");
m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); m_pmodelKeyword->setQuery(QString(strQuery.toUtf8()));
} }
m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform "
"WHEN 0 THEN 'Naver Cafe' "
"WHEN 1 THEN 'Naver Blog' "
"WHEN 2 THEN 'Daum Cafe' "
"ELSE 'UnKnown'"
"END AS platform FROM keyword where state is null");
} }
void Widget::on_keyword_button_modify() void Widget::on_keyword_button_modify()
@@ -350,22 +340,6 @@ void Widget::on_keyword_button_modify()
foreach (QModelIndex index,m_ptableKeyword->selectionModel()->selectedIndexes()) foreach (QModelIndex index,m_ptableKeyword->selectionModel()->selectedIndexes())
{ {
QSqlRecord rec = m_pmodelKeyword->record(index.row()); QSqlRecord rec = m_pmodelKeyword->record(index.row());
/*
QString strQuery = QString("update keyword set "
"start = STR_TO_DATE('%1', '%Y-%m-%d'),"
"end = STR_TO_DATE('%2', '%Y-%m-%d'),"
"searches = '%3',"
"realtime = %4,"
"authorship = '%5' "
"where id = '%6'")
.arg(m_pdeStart->date().toString("yyyy-MM-dd"))
.arg(m_pdeEnd->date().toString("yyyy-MM-dd"))
.arg(m_pleKeyword->text())
.arg(m_pcbRealTime->currentIndex())
.arg(m_pleAuthorship->text())
.arg(rec.value("id").toString());
*/
QString strQuery = QString("update keyword set " QString strQuery = QString("update keyword set "
"start = STR_TO_DATE('%1', '%Y-%m-%d')," "start = STR_TO_DATE('%1', '%Y-%m-%d'),"
"end = STR_TO_DATE('%2', '%Y-%m-%d')," "end = STR_TO_DATE('%2', '%Y-%m-%d'),"
@@ -381,11 +355,14 @@ void Widget::on_keyword_button_modify()
.arg(m_pleAuthorship->text()) .arg(m_pleAuthorship->text())
.arg(m_pcbPlatform->currentIndex()) .arg(m_pcbPlatform->currentIndex())
.arg(rec.value("id").toString()); .arg(rec.value("id").toString());
m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); m_pmodelKeyword->setQuery(QString(strQuery.toUtf8()));
} }
m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform "
"WHEN 0 THEN 'Naver Cafe' "
"WHEN 1 THEN 'Naver Blog' "
"WHEN 2 THEN 'Daum Cafe' "
"ELSE 'UnKnown'"
"END AS platform FROM keyword where state is null");
} }
void Widget::on_group_currentRowChanged(QModelIndex _index) void Widget::on_group_currentRowChanged(QModelIndex _index)
@@ -407,27 +384,6 @@ void Widget::on_group_button_insert()
m_pmodelGroup->setQuery(strQuery.toUtf8()); m_pmodelGroup->setQuery(strQuery.toUtf8());
strQuery = "CREATE TABLE data_"; strQuery = "CREATE TABLE data_";
strQuery += sql.value(0).toString(); strQuery += sql.value(0).toString();
/*
strQuery += " ("
"url CHAR(128),"
"keyword_id INT,"
"platformname CHAR(32),"
"platformform CHAR(16),"
"articleform CHAR(16),"
"body_platformtitle CHAR(128),"
"body_platformid CHAR(64),"
"body_articletitle VARCHAR(128),"
"body_articleid VARCHAR(32),"
"body_date DATETIME,"
"body_nickname CHAR(32),"
"body_data VARCHAR(18432),"
"reply_nickname CHAR(32),"
"reply_data VARCHAR(1024),"
"reply_parent CHAR(32),"
"reply_date DATETIME,"
"reply_urlreply VARCHAR(512),"
"reply_rownum INT) CHARSET=utf8";
*/
strQuery += " (" strQuery += " ("
"platform_name CHAR(64)," "platform_name CHAR(64),"
"platform_form CHAR(64)," "platform_form CHAR(64),"
@@ -513,27 +469,14 @@ void Widget::on_group_button_import()
QFile file(strFilename); QFile file(strFilename);
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return; if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return;
//STable *pNew = new STable; //STable *pNew = new STable;
QTextStream in(&file); QTextStream in(&file);
int ncRow=0;
int nCount=0; int nCount=0;
bool bQuit = true;
int nRead = 0;
QStringList attributes; QStringList attributes;
QString strquery; QString strquery;
while(!in.atEnd()) while(!in.atEnd())
{ {
QString strLine; QString strLine;
/*
{
QByteArray byte = file.readLine();
if (byte.isEmpty()) { bQuit = false; continue; }
strLine = codec->toUnicode(byte);
nRead += byte.size();
}*/
strLine = in.readLine(); strLine = in.readLine();
if (nCount == 0) if (nCount == 0)
{ {
@@ -576,8 +519,7 @@ void Widget::on_group_button_import()
QStringList strings = strLine.split(","); QStringList strings = strLine.split(",");
QSqlQuery query; QSqlQuery query;
query.prepare(strquery.toUtf8()); query.prepare(strquery.toUtf8());
// qDebug() << strquery;
for(int i=0; i<attributes.size();i++) for(int i=0; i<attributes.size();i++)
{ {
if(attributes.at(i).trimmed() == "article_order") if(attributes.at(i).trimmed() == "article_order")
@@ -590,8 +532,7 @@ void Widget::on_group_button_import()
else else
{ {
query.bindValue(":"+attributes.at(i).toUpper(), strings.at(i).trimmed().toInt()); query.bindValue(":"+attributes.at(i).toUpper(), strings.at(i).trimmed().toInt());
} }
// qDebug() << " articleorder is executed";
} }
if(strings.at(i).trimmed().length() == 0) if(strings.at(i).trimmed().length() == 0)
@@ -642,12 +583,8 @@ void Widget::on_group_button_refresh()
return; return;
} }
m_pmodelGroup->setQuery("SELECT * FROM datagroup"); m_pmodelGroup->setQuery("SELECT * FROM datagroup");
} }
void Widget::SaveCsv(QString _strName) void Widget::SaveCsv(QString _strName)
{ {
QFile file(_strName + ".csv" ); QFile file(_strName + ".csv" );
@@ -678,12 +615,9 @@ void Widget::SaveCsv(QString _strName)
strSelect += "CONVERT(keyword_id USING utf8),"; strSelect += "CONVERT(keyword_id USING utf8),";
strSelect += "CONVERT(platform_id USING utf8),"; strSelect += "CONVERT(platform_id USING utf8),";
strSelect += "CONVERT(keyword_id USING utf8),"; strSelect += "CONVERT(keyword_id USING utf8),";
strSelect += "CONVERT(reply_url USING utf8)"; strSelect += "CONVERT(reply_url USING utf8)";
//strSelect += "CONVERT(reply_rownum USING utf8)";
strSelect += " from "; strSelect += " from ";
strSelect += _strName; strSelect += _strName;
//strSelect += " Order by body_date";
if (query.exec(strSelect) == false) if (query.exec(strSelect) == false)
{ {
out << query.lastError().text(); out << query.lastError().text();
@@ -719,20 +653,11 @@ void Widget::on_crawling_button_insert()
QString strQuery = "insert into crawling set "; QString strQuery = "insert into crawling set ";
strQuery += "Keyword_id = '" + strKeywordId + "',"; strQuery += "Keyword_id = '" + strKeywordId + "',";
strQuery += "DataGroup_id = '" + strGroupId + "'"; strQuery += "DataGroup_id = '" + strGroupId + "',";
strQuery += "state = 0";
m_pmodelCrawling->setQuery(strQuery.toUtf8()); m_pmodelCrawling->setQuery(strQuery.toUtf8());
/* UpdateCrawling();
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
*/
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state ,_keyword.platform "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
} }
void Widget::on_crawling_button_delete() void Widget::on_crawling_button_delete()
@@ -744,25 +669,13 @@ void Widget::on_crawling_button_delete()
QSqlRecord rec = m_pmodelCrawling->record(index.row()); QSqlRecord rec = m_pmodelCrawling->record(index.row());
strList.push_back(rec.value("id").toString()); strList.push_back(rec.value("id").toString());
//QString strQuery = QString("delete from crawling where id = '" + rec.value("id").toString() + "'"); //QString strQuery = QString("delete from crawling where id = '" + rec.value("id").toString() + "'");
//m_pmodelCrawling->setQuery(QString(strQuery.toUtf8()));
} }
foreach(QString str,strList) foreach(QString str,strList)
{ {
QString strQuery = QString("delete from crawling where id = '" + str + "'"); QString strQuery = QString("delete from crawling where id = '" + str + "'");
m_pmodelCrawling->setQuery(QString(strQuery.toUtf8())); m_pmodelCrawling->setQuery(QString(strQuery.toUtf8()));
} }
/* UpdateCrawling();
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
*/
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state ,_keyword.platform "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
} }
void Widget::on_group_button_copy_clear() void Widget::on_group_button_copy_clear()
@@ -862,7 +775,6 @@ void Widget::on_group_button_copy_start()
return; return;
for(int j = 0; j < strListFrom.length(); j++) for(int j = 0; j < strListFrom.length(); j++)
{ {
strListFrom.at(i).toInt(&ok); strListFrom.at(i).toInt(&ok);
if(!ok) if(!ok)
return; return;
@@ -875,22 +787,6 @@ void Widget::on_group_button_copy_start()
strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '"); strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '");
strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body')"); strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body')");
strQuery += (" and article_date >= '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and article_date is not null)"); strQuery += (" and article_date >= '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and article_date is not null)");
//insert into data_to (select * from data_67 where article_url in (select distinct article_url from data_67 where article_date between '2015-02-01 00:00:00' and '2015-03-21 23:59:59' and article_form='body') and article_date >= '2015-02-01 00:00:00' and article_date is not null)
/*
strQuery = "insert into ";
strQuery += ("data_" + strListTo.at(i).trimmed());
strQuery += (" (select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profile,article_profileurl,platform_id,keyword_id,reply_url from data_" + strListFrom.at(i).trimmed());
strQuery += (" where article_url in (select distinct article_url from data_" + strListFrom.at(i).trimmed());
strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '");
strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body'))");
*/
//m_pdeStart->date().toString("yyyy-MM-dd");
//insert into data_92
//(select * from data_93 where article_url in
//(select distinct article_url from data_93 where article_date between '2015-01-01 00:00:00' and '2015-01-01 23:59:59') and article_date >= '2015-01-01 00:00:00' and article_date is not null);
} }
else else
{ {
@@ -901,9 +797,6 @@ void Widget::on_group_button_copy_start()
strQuery += " where (article_url, article_order) not in (select article_url, article_order from "; strQuery += " where (article_url, article_order) not in (select article_url, article_order from ";
strQuery += ("data_" + strListTo.at(i).trimmed()); strQuery += ("data_" + strListTo.at(i).trimmed());
strQuery += "))"; strQuery += "))";
//insert into data_to from
//(select * from data_from where
//(article_url, article_order) not in (select article_url, article_order from data_to))
} }
qDebug(strQuery.toLatin1()); qDebug(strQuery.toLatin1());
strQueryUtf = strQuery.toUtf8(); strQueryUtf = strQuery.toUtf8();
@@ -918,8 +811,13 @@ void Widget::on_group_button_copy_start()
} }
m_pleCopyFrom->setText("Success"); m_pleCopyFrom->setText("Success");
m_pleCopyTo->setText("Sucess"); m_pleCopyTo->setText("Sucess");
// on_group_button_copy_clear();
} }
void Widget::UpdateCrawling()
{
m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name , "
"(CASE _keyword.platform WHEN 0 THEN 'Naver Cafe' WHEN 1 THEN 'Naver Blog' WHEN 2 THEN 'Daum Cafe' ELSE 'UnKnown' END ) AS platform , "
"(CASE _crawling.state WHEN 0 THEN 'Waiting' WHEN 1 THEN 'Running' WHEN 2 THEN 'Terminated' ELSE 'None' END ) AS state "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id");
}

6
GroupManager/widget.h
View File

@@ -30,11 +30,12 @@ private:
QGroupBox *setGroupWidgets(); QGroupBox *setGroupWidgets();
QGroupBox *setCrawlingWidgets(); QGroupBox *setCrawlingWidgets();
void SaveCsv(QString); void SaveCsv(QString);
void UpdateCrawling();
private slots: private slots:
void on_keyword_currentRowChanged(QModelIndex); void on_keyword_currentRowChanged(QModelIndex);
void on_keyword_button_insert(); void on_keyword_button_insert();
void on_keyword_button_delete(); void on_keyword_button_delete();
void on_keyword_button_modify(); void on_keyword_button_modify();
void on_group_currentRowChanged(QModelIndex); void on_group_currentRowChanged(QModelIndex);
void on_group_button_insert(); void on_group_button_insert();
@@ -48,6 +49,7 @@ private slots:
void on_group_button_copy_from(); void on_group_button_copy_from();
void on_group_button_copy_to(); void on_group_button_copy_to();
void on_group_button_refresh(); void on_group_button_refresh();
void on_crawling_button_insert(); void on_crawling_button_insert();
void on_crawling_button_delete(); void on_crawling_button_delete();
}; };