etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

git-svn-id: svn://192.168.0.12/source@1 8346c931-da38-4b9b-9d4c-e48b93cbd075

Browse Source
This commit is contained in:
admin
2015-01-14 03:17:44 +00:00
commit ec1c8ec329
55 changed files with 5967 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
NaverBlogCrawler/NaverBlogCrawler.pro Normal file
View File

@@ -0,0 +1,22 @@
#-------------------------------------------------
#
# Project created by QtCreator 2014-10-08T10:16:10
#
#-------------------------------------------------
QT += core gui
QT += sql
greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
TARGET = NaverBlogCrawler
TEMPLATE = app
SOURCES += main.cpp\
widget.cpp \
smanage.cpp \
snaverblogmanage.cpp \
HEADERS += widget.h \
smanage.h \
snaverblogmanage.h \

10
NaverBlogCrawler/main.cpp Normal file
View File

@@ -0,0 +1,10 @@
#include <QApplication>
#include "widget.h"
int main(int argc, char *argv[])
{
QApplication a(argc, argv);
Widget w;
w.show();
return a.exec();
}

97
NaverBlogCrawler/smanage.cpp Normal file
View File

@@ -0,0 +1,97 @@
#include "smanage.h"
#include "widget.h"
SManage::SManage(QObject *parent) :
QObject(parent),m_pMain(0)
{
for(int i = 0; i < C_PROCESS_MAX ;i++)
connect(&m_pro[i],SIGNAL(finished(int,QProcess::ExitStatus)),SLOT(processFinished(int,QProcess::ExitStatus)));
}
void SManage::Start(QDate _StartDate,QDate _EndDate,QString _strKeyword,QString _strKeywordID,QString _strGroupID,int _nStart,int _nTime)
{
m_date = _StartDate;
m_dateEnd = _EndDate;
m_strKeyword = _strKeyword;
m_bFinalLast = false;
m_nTime = _nTime;
m_strKeywordID = _strKeywordID;
m_ncList = _nStart;
m_strGroupID = _strGroupID;
Start();
}
void SManage::SetParent(Widget *_pWidget)
{
m_pMain = _pWidget;
}
QString SManage::EncodetoUtf8(QString _str,bool _bExt)
{
QString strOut;
for (int i = 0; i < _str.length(); i++)
{
QString strTran = _str.at(i);
QByteArray byte = strTran.toUtf8();
if (byte.length() == 3)
{
QByteArray byteHan = byte.toHex().toUpper();
for (int j = 0; j < 3 ; j++)
{
strOut += "%";
for (int k = 0; k < 2; k++)
strOut += byteHan.at(j*2+k);
}
}
else if (_str.at(i).isDigit() || _str.at(i).isLower() || _str.at(i).isUpper())
strOut += _str.at(i);
else if (_bExt&&_str.at(i) == '@' || _str.at(i) == '*' || _str.at(i) == '_' || _str.at(i) == '.')
strOut += _str.at(i);
else if (_bExt&&_str.at(i) == ' ')
strOut += '+';
else
{
strOut += "%";
strOut += byte.toHex().toUpper();
}
}
return strOut;
}
bool SManage::UseProcess()
{
for(int i = 0; i < C_PROCESS_MAX ; i++)
{
if (m_pro[i].state() == QProcess::Running)
return true;
}
return false;
}
void SManage::processFinished(int exitCode,QProcess::ExitStatus exitStatus)
{
QProcess *pPro = (QProcess*)sender();
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n");
foreach(QString log,list)
{
if (m_pMain)
m_pMain->InsertLog(m_nID,log);
else
exit(0);
}
processFinished(pPro,str);
}
void SManage::CheckLast()
{
if (m_bLast)
{
m_date = m_date.addDays(1);
if (m_date > m_dateEnd)
{
m_bFinalLast = true;
}
m_ncList=1;
}
}

51
NaverBlogCrawler/smanage.h Normal file
View File

@@ -0,0 +1,51 @@
#ifndef SMANAGE_H
#define SMANAGE_H
#include <QObject>
#include <QSqlDatabase>
#include <QDate>
#include <QStringList>
#include <QVector>
#include <QProcess>
class Widget;
class SManage : public QObject
{
Q_OBJECT
public:
explicit SManage(QObject *parent = 0);
public:
void Start(QDate _StartDate,QDate _EndDate,QString _strKeyword,QString _strKeywordID,QString _strGroupID,int _nStart,int _nTime);
void SetParent(Widget *pWidget);
virtual bool Update() = 0;
signals:
public slots:
protected:
virtual void Start() = 0;
QString EncodetoUtf8(QString _str,bool _bExt=false);
virtual void processFinished(QProcess *pPro,QString _strOut) = 0;
bool UseProcess();
void CheckLast();
protected:
Widget *m_pMain;
QDate m_date,m_dateEnd;
int m_nMode;
QString m_strKeywordID;
QString m_strGroupID;
QString m_strKeyword;
bool m_bFinalLast;
int m_nTime;
int m_nID;
bool m_bLast;
int m_ncList;
int m_ncUrl;
int m_nWait;
static const int C_PROCESS_MAX = 1;
QProcess m_pro[C_PROCESS_MAX];
private:
QVector <QStringList> m_vecList;
private slots:
void processFinished(int exitCode, QProcess::ExitStatus exitStatus);
};
#endif // SMANAGE_H

288
NaverBlogCrawler/snaverblogmanage.cpp Normal file
View File

@@ -0,0 +1,288 @@
#include "snaverblogmanage.h"
#include "widget.h"
#include <QSqlQuery>
#include <QSqlError>
#include <QFile>
#include <QTextStream>
SNaverBlogManage::SNaverBlogManage(QObject *pObject) : SManage(pObject) , C_TABLE_URL("NAVER_BLOG_BODY_") , C_TABLE_COM("NAVER_BLOG_REPLY_")
{
m_nID = 0;
}
QString SNaverBlogManage::makeGetListQuery(QString _str,QDate _date)
{
QString str;
QString strDate = _date.toString("yyyyMMdd");
//http://cafeblog.search.naver.com/search.naver?where=post&sm=tab_pge&query=%EC%95%84%EC%9D%B4%ED%8F%B0&st=date&date_option=6&date_from=20131103&date_to=20131103&dup_remove=1&post_blogurl=&post_blogurl_without=&srchby=all&nso=so%3Add%2Cp%3Afrom20131103to20131103&ie=utf8&start=11
str = "http://cafeblog.search.naver.com/search.naver?where=post&sm=tab_pge&query=";
str += EncodetoUtf8(_str,true);
str += "&st=date&date_option=6&date_from=" + strDate + "&date_to=" + strDate ;
str += "&dup_remove=1&post_blogurl=&post_blogurl_without=&srchby=all&nso=so%3Add%2Cp%3Afrom";
str += strDate + "to" + strDate +"&ie=utf8&start=" + QString::number(m_ncList);
return str;
}
QString SNaverBlogManage::makeGetCommentQuery(QString _strUrl)
{
//http://blog.naver.com/kohaku3533/220149821481/CommentList.nhn?blogId=kohaku3533&logNo=220149821481&currentPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false;
QStringList strList = _strUrl.split("/");
QString strOut = _strUrl;
strOut += "/CommentList.nhn?blogId=";
strOut += strList.at(3);
strOut += "&logNo=";
strOut += strList.at(4);
strOut += "&currentPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false";
return strOut;
}
void SNaverBlogManage::Start()
{
m_nMode = E_PROCESS_LIST_RUN;
m_ncList = 1;
m_bFinalLast = false;
}
void SNaverBlogManage::processFinished(QProcess *pPro,QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_LIST_FINISH_WAIT:
{
m_bLast = false;
if (_strOut.right(4) == "last" || m_ncList >= 991)
m_bLast = true;
m_strListURL.clear();
QSqlQuery query;
if(query.exec("SELECT URL FROM " + C_TABLE_URL + QString::number(m_nUrlTable) + " where ERROR is null"))
{
m_pMain->InsertLog(m_nID,query.lastError().text());
}
while (query.next())
m_strListURL.append(query.value(0).toString());
m_ncUrl = 0;
if (m_strListURL.size() == 0)
{
m_nMode = E_PROCESS_LIST_RUN;
CheckLast();
}
else
m_nMode = E_PROCESS_URL_RUN;
break;
}
case E_PROCESS_URL_FINISH_WAIT:
if (UseProcess() == false)
{
m_nMode = E_PROCESS_COMMENT_RUN;
}
break;
case E_PROCESS_COMMENT_FINISH_WAIT:
//if (UseProcess() == false)
if (m_ncUrl >= m_strListURL.size())
{
m_nMode = E_PROCESS_LIST_RUN;
CheckLast();
m_bLast = false;
}
else
m_nMode = E_PROCESS_URL_RUN;
break;
}
}
bool SNaverBlogManage::Update()
{
if (m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_LIST_RUN:
if (UseProcess() == false)
{
m_strQuery = makeGetListQuery(m_strKeyword,m_date);
m_pMain->InsertLog(m_nID,"Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
//for(int i = 0; i < C_PROCESS_MAX ; i++)
{
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "blog_list" << QString::number(m_nUrlTable) << m_strQuery << m_strKeywordID);
m_ncList+=10;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
m_nWait = 0;
}
break;
case E_PROCESS_URL_RUN:
if (UseProcess() == false)
{
/*
m_pMain->InsertLog(m_nID,"(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
for(int i = 0; i < C_PROCESS_MAX ; i++)
{
m_pro[0].start("CrawlerProcess",QStringList() << m_strListURL.at(m_ncUrl) << "url");
}
*/
/*
int i = 0;
foreach(QString strUrl,m_strListURL)
//for(int i = 0; i < C_PROCESS_MAX ; i++)
{
m_pro[i++].start("CrawlerProcess",QStringList() << strUrl << "url" << C_TABLE_URL + QString::number(m_nUrlTable));
if (i >= C_PROCESS_MAX) break;
}
*/
//m_pro[0].start("CrawlerProcess",QStringList() << m_strListURL.at(m_ncUrl) << "blog_url" << C_TABLE_URL + QString::number(m_nUrlTable));
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "blog_url" << QString::number(m_nUrlTable) << m_strListURL.at(m_ncUrl) << makeGetCommentQuery(m_strListURL.at(m_ncUrl)) << "" );
m_nMode = E_PROCESS_URL_FINISH_WAIT;
m_nWait = 0;
}
break;
case E_PROCESS_COMMENT_RUN:
if (UseProcess() == false)
{
//int i=0;
//foreach(QString strUrl,m_strListURL)
{
//m_pro[i++].start("CrawlerProcess",QStringList() << makeGetCommentQuery(strUrl)<< "comment" << C_TABLE_COM + QString::number(m_nUrlTable));
//if (i >= C_PROCESS_MAX) break;
//m_ncUrl++;
}
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "blog_comm" << QString::number(m_nUrlTable) << makeGetCommentQuery(m_strListURL.at(m_ncUrl++)) << "" );
//m_pro[0].start("CrawlerProcess",QStringList() << makeGetCommentQuery(m_strListURL.at(m_ncUrl++))<< "blog_comm" << C_TABLE_COM + QString::number(m_nUrlTable));
m_nMode = E_PROCESS_COMMENT_FINISH_WAIT;
m_nWait = 0;
}
break;
case E_PROCESS_LIST_FINISH_WAIT:
case E_PROCESS_URL_FINISH_WAIT:
case E_PROCESS_COMMENT_FINISH_WAIT:
m_nWait++;
if (m_nWait > (1000000/m_nTime))
{
for(int i = 0; i < C_PROCESS_MAX ; i++)
{
m_pro[i].kill();
m_pMain->InsertLog(m_nID,"Kill Process.");
}
if (m_nMode == E_PROCESS_LIST_FINISH_WAIT) return m_bFinalLast;
QString strQuery = "update blog set ";
strQuery += "Error ";
strQuery += "='";
strQuery += "Kill Process";
strQuery += "'";
strQuery += " where URL='";
if (m_nMode == E_PROCESS_COMMENT_FINISH_WAIT)
strQuery += m_strListURL.at(m_ncUrl-1);
else
strQuery += m_strListURL.at(m_ncUrl);
strQuery += "'";
QSqlQuery sql;
sql.exec(strQuery);
}
break;
}
return m_bFinalLast;
}
void SNaverBlogManage::MakeTables()
{
QString strQuery = "show tables";
QSqlQuery query;
query.exec(strQuery);
int nUrlMax = -1;
while (query.next())
{
QString str = query.value(0).toString();
if (str.left(C_TABLE_URL.size()) == C_TABLE_URL.toUpper())
{
if (nUrlMax < str.mid(C_TABLE_URL.size()).toInt())
nUrlMax = str.mid(C_TABLE_URL.size()).toInt();
}
}
m_nUrlTable = nUrlMax + 1;
strQuery = "Create table " + C_TABLE_URL + QString::number(m_nUrlTable)+ "(Url CHAR(128) not null primary key,keyword_id INT,PlatformTitle CHAR(128),PlatformID CHAR(64),ArticleTitle VARCHAR(128),ArticleID CHAR(32),Date DATETIME,Nickname CHAR(32),Data VARCHAR(18432),Error CHAR(32)) CHARSET=utf8";
query.exec(strQuery);
strQuery = "Create table " + C_TABLE_COM + QString::number(m_nUrlTable)+ "(Url CHAR(128) not null,Nickname CHAR(32),Data VARCHAR(1024),Parent CHAR(64),Date DATETIME,UrlReply VARCHAR(512),RowNum INT) CHARSET=utf8";
query.exec(strQuery);
/*
strQuery = "Create table " + C_TABLE_URL + QString::number(m_nUrlTable)+ "(URL varchar(128) not null primary key , ID varchar(64) ,NICK varchar(64),TITLE varchar(256), DATA varchar(20480) , DATE char(32) , BLOG_ID varchar(64) , OTHER varchar(128), ERROR char(128)) CHARSET=utf8";
query.exec(strQuery);
strQuery = "Create table " + C_TABLE_COM + QString::number(m_nUrlTable)+ "(URL varchar(128) not null, NICK varchar(64),DATA varchar(20480),PARENT varchar(64),DATE char(32),URL_COMMENT varchar(512) not null,URL_NICK varchar(128)) CHARSET=utf8";
query.exec(strQuery);
*/
m_pMain->setWindowTitle("NaverBlogCrawler " + QString::number(m_nUrlTable));
}
void SNaverBlogManage::DropTables()
{
QString strQuery = "drop table ";
QSqlQuery query;
query.exec(strQuery + C_TABLE_URL + QString::number(m_nUrlTable));
query.exec(strQuery + C_TABLE_COM + QString::number(m_nUrlTable));
}
void SNaverBlogManage::Join()
{
m_pMain->InsertLog(m_nID,"Insert Article Data...");
QString strQuery = "insert into "
"data_" + m_strGroupID +
"(platformname , platformform , articleform ,"
"url , keyword_id , body_platformtitle , body_platformid , body_articletitle , body_articleid , body_date , body_nickname , body_data)"
"select "
"CONVERT('naver' USING utf8),"
"CONVERT('blog' USING utf8),"
"CONVERT('article' USING utf8),"
"CONVERT(url USING utf8),"
"CONVERT(keyword_id USING utf8),"
"CONVERT(PlatformTitle USING utf8),"
"CONVERT(PlatformID USING utf8),"
"CONVERT(ArticleTitle USING utf8),"
"CONVERT(ArticleID USING utf8),"
"CONVERT(Date USING utf8),"
"CONVERT(Nickname USING utf8),"
"CONVERT(Data USING utf8)"
"from NAVER_BLOG_BODY_" + QString::number(m_nUrlTable);
QSqlQuery query;
if (query.exec(strQuery) == false)
{
m_pMain->InsertLog(m_nID,query.lastError().text());
return;
}
m_pMain->InsertLog(m_nID,"Insert Reply Data...");
strQuery = "insert into "
"data_" + m_strGroupID +
"(platformname , platformform , articleform ,"
"url , keyword_id , body_platformtitle , body_platformid , body_articletitle , body_articleid , body_date , body_nickname , body_data ,"
"reply_nickname ,reply_data, reply_parent , reply_date ,reply_urlreply ,reply_rownum )"
"select "
"CONVERT('naver' USING utf8),"
"CONVERT('blog' USING utf8),"
"CONVERT('reply' USING utf8),"
"CONVERT(_body.url USING utf8),"
"CONVERT(_body.keyword_id USING utf8),"
"CONVERT(_body.PlatformTitle USING utf8),"
"CONVERT(_body.PlatformID USING utf8),"
"CONVERT(_body.ArticleTitle USING utf8),"
"CONVERT(_body.ArticleID USING utf8),"
"CONVERT(_body.Date USING utf8),"
"CONVERT(_body.Nickname USING utf8),"
"CONVERT(_body.Data USING utf8),"
"CONVERT(_reply.Nickname USING utf8),"
"CONVERT(_reply.Data USING utf8),"
"CONVERT(_reply.Parent USING utf8),"
"CONVERT(_reply.Date USING utf8),"
"CONVERT(_reply.UrlReply USING utf8),"
"CONVERT(_reply.RowNum USING utf8) "
"from " + C_TABLE_URL + QString::number(m_nUrlTable) + " _body INNER JOIN " + C_TABLE_COM + QString::number(m_nUrlTable) + " _reply ON _body.Url = _reply.Url";
if (query.exec(strQuery) == false)
{
m_pMain->InsertLog(m_nID,query.lastError().text());
return;
}
m_pMain->InsertLog(m_nID,"Delete data ...");
query.exec("delete from NAVER_BLOG_BODY_" + QString::number(m_nUrlTable) );
query.exec("delete from NAVER_BLOG_REPLY_" + QString::number(m_nUrlTable) );
m_pMain->InsertLog(m_nID,"Finish ... ");
}

39
NaverBlogCrawler/snaverblogmanage.h Normal file
View File

@@ -0,0 +1,39 @@
#ifndef SNAVERBLOGMANAGE_H
#define SNAVERBLOGMANAGE_H
#include "SManage.h"
class SNaverBlogManage : public SManage
{
public:
enum E_PROCESS_STATE
{
E_PROCESS_LIST_RUN = 0,
E_PROCESS_LIST_FINISH_WAIT,
E_PROCESS_URL_RUN,
E_PROCESS_URL_FINISH_WAIT,
E_PROCESS_COMMENT_RUN,
E_PROCESS_COMMENT_FINISH_WAIT,
};
public:
SNaverBlogManage(QObject *pObject);
void MakeTables();
void DropTables();
void SaveCsv(QString _strName);
void Join();
int GetTableNumber() {return m_nUrlTable;}
private:
QString makeGetListQuery(QString _str,QDate _date);
QString makeGetCommentQuery(QString _strUrl);
private:
QString m_strQuery;
QVector <QString> m_strListURL;
const QString C_TABLE_URL;
const QString C_TABLE_COM;
int m_nUrlTable;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
};
#endif // SNAVERBLOGMANAGE_H

6
NaverBlogCrawler/sprocess.cpp Normal file
View File

@@ -0,0 +1,6 @@
#include "sprocess.h"
SProcess::SProcess(QObject *parent) :
QProcess(parent) , m_nState(0)
{
}

25
NaverBlogCrawler/sprocess.h Normal file
View File

@@ -0,0 +1,25 @@
#ifndef SPROCESS_H
#define SPROCESS_H
#include <QProcess>
class SProcess : public QProcess
{
Q_OBJECT
public:
enum STATE
{
STATE_WAIT = 0,
STATE_RUNNING,
};
explicit SProcess(QObject *parent = 0);
void SetState(int _nState) { m_nState = _nState; }
int State() {return m_nState;}
signals:
public slots:
private:
int m_nState;
};
#endif // SPROCESS_H

175
NaverBlogCrawler/widget.cpp Normal file
View File

@@ -0,0 +1,175 @@
#include "widget.h"
#include <QHBoxLayout>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QDebug>
#include <QDateEdit>
#include <QFileDialog>
#include "snaverblogmanage.h"
Widget::Widget(QWidget *parent)
: QWidget(parent)
{
connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update()));
m_pcb = new QComboBox;
m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent);
QPushButton *pbtRefresh = new QPushButton("Refresh",this);
QPushButton *pbtStart = new QPushButton("Start",this);
QPushButton *pbtStop = new QPushButton("Stop",this);
{
QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton()));
QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton()));
QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton()));
}
QVBoxLayout *vlayout = new QVBoxLayout;
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pcb,Qt::AlignLeft);
hlayout->addWidget(pbtRefresh);
vlayout->addLayout(hlayout);
}
m_pedTime = new QLineEdit(this);
m_pedTime->setText(QString("500"));
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pedTime);
hlayout->addWidget(pbtStart);
hlayout->addWidget(pbtStop);
vlayout->addLayout(hlayout);
}
{
QHBoxLayout *hlayout = new QHBoxLayout;
for (int i = 0; i < C_CRAWLER_MAX; i++ )
hlayout->addWidget(&m_aResultList[i]);
vlayout->addLayout(hlayout);
}
setLayout(vlayout);
m_pNaverBlog = new SNaverBlogManage(this);
m_pManage[0] = m_pNaverBlog;
m_db = QSqlDatabase::addDatabase("QMYSQL");
m_db.setHostName("bigbird.iptime.org");
m_db.setUserName("admin");
m_db.setPassword("admin123");
m_db.setDatabaseName("concepters");
if (!m_db.open())
{
InsertLog(0,"MySql Error...");
return;
}
for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
m_pManage[i]->SetParent(this);
m_pNaverBlog->MakeTables();
RefreshButton();
}
Widget::~Widget()
{
m_pNaverBlog->DropTables();
m_db.close();
m_pNaverBlog->SetParent(0);
}
void Widget::InsertLog(int _nSelect,QString str)
{
if (_nSelect >= C_CRAWLER_MAX) return;
QTime time = QTime::currentTime();
QString strOut = time.toString("[hh:mm:ss] ") + str;
m_aResultList[_nSelect].addItem(strOut);
QDate date = QDate::currentDate();
QFile file(date.toString(Qt::ISODate)+"_"+QString::number(_nSelect)+".log");
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
return;
QTextStream out(&file);
out << strOut << "\n";
file.close();
if (m_aResultList[_nSelect].count() > 1024)
{
m_aResultList[_nSelect].removeItemWidget(m_aResultList[_nSelect].item(0));
QListWidgetItem* item = m_aResultList[_nSelect].takeItem(0);
delete item;
}
m_aResultList[_nSelect].setCurrentRow( m_aResultList[_nSelect].count() - 1 );
m_aResultList[_nSelect].repaint();
}
void Widget::StartButton()
{
m_timer.stop();
QSqlQuery query;
query.exec("UPDATE crawling set state = '" + QString::number(m_pNaverBlog->GetTableNumber()) + "' where id = '" + m_pcb->currentData().toString() + "'");
query.exec("SELECT _keyword.start,_keyword.end, _keyword.searches,_keyword.id,_datagroup.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id "
"where _crawling.id = '" + m_pcb->currentData().toString() + "'");
query.next();
QString str = + " , ";
str += query.value(1).toString() + " ~ ";
str += query.value(2).toString() + " -> ";
str += query.value(3).toString();
m_timer.start(m_pedTime->text().trimmed().toInt());
m_pManage[0]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"),
QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"),
query.value(2).toString().trimmed(),// keyword
query.value(3).toString().trimmed(),// keyword_id
query.value(4).toString().trimmed(),
1,
m_pedTime->text().trimmed().toInt());
}
void Widget::StopButton()
{
m_timer.stop();
}
void Widget::Update()
{
if (m_db.isOpen() == false)
{
if (m_db.open())
{
InsertLog(0,"MySql Open Error...");
m_timer.stop();
return;
}
}
int nCount = 0;
for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
nCount += m_pManage[i]->Update();
if (nCount == C_CRAWLER_MAX)
{
m_timer.stop();
m_pNaverBlog->Join();
}
}
void Widget::RefreshButton()
{
m_pcb->clear();
QSqlQuery query("SELECT _keyword.searches,_keyword.start,_keyword.end, _datagroup.name , _crawling.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id where _crawling.state is null");
while (query.next())
{
QString str;
str += query.value(0).toString() + " , ";
str += query.value(1).toString() + " ~ ";
str += query.value(2).toString() + " -> ";
str += query.value(3).toString();
m_pcb->addItem(str,query.value(4));
}
}

49
NaverBlogCrawler/widget.h Normal file
View File

@@ -0,0 +1,49 @@
#ifndef WIDGET_H
#define WIDGET_H
#include <QWidget>
#include <QPushButton>
#include <QLineEdit>
#include <QTimer>
#include <QDateEdit>
#include <QListWidget>
#include <QSqlDatabase>
#include <QComboBox>
class SNaverBlogManage;
class SManage;
#define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; }
class Widget : public QWidget
{
Q_OBJECT
public:
Widget(QWidget *parent = 0);
~Widget();
public:
void InsertLog(int _nSelect,QString str);
private:
QLineEdit *m_pedTime;
QTimer m_timer;
QSqlDatabase m_db;
static const int C_CRAWLER_MAX = 1;
SManage *m_pManage[C_CRAWLER_MAX];
QListWidget m_aResultList[C_CRAWLER_MAX];
QString m_strFileName;
QComboBox *m_pcb;
QVector <QString> m_vecSelect;
SNaverBlogManage *m_pNaverBlog;
private:
QString makeCafeGetListQuery(QString _str,QDate _date,int _nPage);
//void SaveBlog(QString _strName);
//void SaveCafe(QString _strName);
private slots:
void RefreshButton();
void StartButton();
void StopButton();
void Update();
};
#endif // WIDGET_H