etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

최초 변경

Browse Source 
git-svn-id: svn://192.168.0.12/source@14 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-01-29 07:04:38 +00:00
parent 7c6bc7df86
commit 26aa931287
9 changed files with 716 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
CrawlerList/CrawlerList.pro Normal file
View File

@@ -0,0 +1,18 @@
#-------------------------------------------------
#
# Project created by QtCreator 2015-01-29T16:00:16
#
#-------------------------------------------------
QT += core gui sql
greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
TARGET = CrawlerList
TEMPLATE = app
SOURCES += main.cpp\
widget.cpp
HEADERS += widget.h

23
CrawlerList/NaverCafeCrawler.pro Normal file
View File

@@ -0,0 +1,23 @@
#-------------------------------------------------
#
# Project created by QtCreator 2014-12-02T17:00:39
#
#-------------------------------------------------
QT += core gui sql
greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
TARGET = NaverCafeCrawler
TEMPLATE = app
SOURCES += main.cpp\
widget.cpp \
sprocess.cpp \
smanage.cpp \
snavercafemanage.cpp
HEADERS += widget.h \
sprocess.h \
smanage.h \
snavercafemanage.h

12
CrawlerList/main.cpp Normal file
View File

@@ -0,0 +1,12 @@
#include "widget.h"
#include <QApplication>
int main(int argc, char *argv[])
{
QApplication a(argc, argv);
Widget w;
w.show();
return a.exec();
}

115
CrawlerList/smanage.cpp Normal file
View File

@@ -0,0 +1,115 @@
#include "smanage.h"
#include "widget.h"
SManage::SManage(QObject *parent) :
QObject(parent),m_pMain(0)
{
for(int i = 0; i < C_PROCESS_MAX ;i++)
connect(&m_pro[i],SIGNAL(finished(int,QProcess::ExitStatus)),SLOT(processFinished(int,QProcess::ExitStatus)));
}
void SManage::Start(QDate _StartDate,QDate _EndDate,QString _strKeyword,QString _strAuthorship,QString _strKeywordID,QString _strGroupID,int _nStart,int _nTime)
{
m_date = _StartDate;
m_dateEnd = _EndDate;
m_strKeyword = _strKeyword;
m_bFinalLast = false;
m_nTime = _nTime;
m_strKeywordID = _strKeywordID;
m_strAuthorship = _strAuthorship;
m_ncList = _nStart;
m_strGroupID = _strGroupID;
Start();
}
void SManage::SetParent(Widget *_pWidget)
{
m_pMain = _pWidget;
}
QString SManage::EncodetoUtf8(QString _str,bool _bExt)
{
QString strOut;
for (int i = 0; i < _str.length(); i++)
{
QString strTran = _str.at(i);
QByteArray byte = strTran.toUtf8();
if (byte.length() == 3)
{
QByteArray byteHan = byte.toHex().toUpper();
for (int j = 0; j < 3 ; j++)
{
strOut += "%";
for (int k = 0; k < 2; k++)
strOut += byteHan.at(j*2+k);
}
}
else if (_str.at(i).isDigit() || _str.at(i).isLower() || _str.at(i).isUpper())
strOut += _str.at(i);
else if (_bExt && _str.at(i) == '@' || _str.at(i) == '*' || _str.at(i) == '_' || _str.at(i) == '.')
strOut += _str.at(i);
else if (_bExt && _str.at(i) == ' ')
strOut += '+';
else
{
strOut += "%";
strOut += byte.toHex().toUpper();
}
}
return strOut;
}
bool SManage::UseProcess()
{
for(int i = 0; i < C_PROCESS_MAX ; i++)
{
if (m_pro[i].state() == QProcess::Running)
return true;
}
return false;
}
void SManage::processFinished(int exitCode,QProcess::ExitStatus exitStatus)
{
SProcess *pPro = (SProcess*)sender();
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n");
foreach(QString log,list)
{
if (m_pMain)
m_pMain->InsertLog(m_nID,log);
else
exit(0);
}
pPro->SetState(SProcess::STATE_WAIT);
processFinished(pPro,str);
pPro->kill();
}
void SManage::CheckLast()
{
if (m_bLast)
{
m_date = m_date.addDays(1);
if (m_date > m_dateEnd)
{
m_bFinalLast = true;
}
m_ncList=1;
}
}
void SManage::WaitExitProcess()
{
bool bQuit = true;
while(bQuit)
{
for(int i = 0; i < C_PROCESS_MAX ; i++)
{
if (m_pro[i].State() != SProcess::STATE_WAIT)
m_pro[i].kill();
}
bQuit = UseProcess();
}
}

53
CrawlerList/smanage.h Normal file
View File

@@ -0,0 +1,53 @@
#ifndef SMANAGE_H
#define SMANAGE_H
#include <QObject>
#include <QSqlDatabase>
#include <QDate>
#include <QStringList>
#include <QVector>
#include "SProcess.h"
class Widget;
class SManage : public QObject
{
Q_OBJECT
public:
explicit SManage(QObject *parent = 0);
public:
void Start(QDate _StartDate,QDate _EndDate,QString _strKeyword,QString _strAuthorship,QString _strKeywordID,QString _strGroupID,int _nStart,int _nTime);
void SetParent(Widget *pWidget);
void WaitExitProcess();
virtual bool Update() = 0;
signals:
public slots:
protected:
virtual void Start() = 0;
QString EncodetoUtf8(QString _str,bool _bExt=false);
virtual void processFinished(SProcess *pPro,QString _strOut) = 0;
bool UseProcess();
void CheckLast();
protected:
Widget *m_pMain;
QDate m_date,m_dateEnd;
int m_nMode;
QString m_strKeyword;
QString m_strKeywordID;
QString m_strGroupID;
QString m_strAuthorship;
bool m_bFinalLast;
int m_nTime;
int m_nID;
bool m_bLast;
int m_ncList;
int m_ncUrl;
int m_nWait;
static const int C_PROCESS_MAX = 1;
SProcess m_pro[C_PROCESS_MAX];
private:
QVector <QStringList> m_vecList;
private slots:
void processFinished(int exitCode, QProcess::ExitStatus exitStatus);
};
#endif // SMANAGE_H

245
CrawlerList/snavercafemanage.cpp Normal file
View File

@@ -0,0 +1,245 @@
#include "snavercafemanage.h"
#include "widget.h"
#include <QSqlQuery>
#include <QSqlError>
#include <qDebug>
#include <QFile>
SNaverCafeManage::SNaverCafeManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
}
QString SNaverCafeManage::makeGetListQuery(QString _str,QDate _date,int _nPage)
{
QString strOut;
strOut = "http://cafeblog.search.naver.com/search.naver?where=article&ie=utf8&query=";
strOut += EncodetoUtf8(_str,true);
strOut += "&t=0&st=date&date_option=6&date_from=";
strOut += _date.toString("yyyy.MM.dd");
strOut += "&date_to=";
strOut += _date.toString("yyyy.MM.dd");
//strOut += "&srchby=text&dup_remove=1&cafe_url=&without_cafe_url=&board=&sm=tab_pge&nso=so%3Add%2Cp%3Afrom";
strOut += "&srchby=text&dup_remove=1&";
strOut += "cafe_url=";
strOut += m_strAuthorship;
strOut += "&without_cafe_url=&board=&sm=tab_pge&nso=so:dd,p:from";
strOut += _date.toString("yyyyMMdd");
strOut += "to";
strOut += _date.toString("yyyyMMdd");
//strOut += "%2Ca%3Aall&start=" + QString::number(_nPage);
strOut += ",a:all&start=" + QString::number(_nPage);
//qDebug() << strOut;
return strOut;
}
void SNaverCafeManage::Start()
{
m_nMode = E_PROCESS_LIST_RUN;
m_bFinalLast = false;
}
bool SNaverCafeManage::Update()
{
if (m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_LIST_RUN:
if (UseProcess() == false)
{
m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
m_pMain->InsertLog(m_nID,"Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
m_pro[0].SetState(SProcess::STATE_RUNNING);
m_ncList+=10;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
m_nWait = 0;
}
break;
case E_PROCESS_URL_RUN:
if (UseProcess() == false)
{
m_pMain->InsertLog(m_nID,"(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
{
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << "" );
m_pro[0].SetState(SProcess::STATE_RUNNING);
m_ncUrl++;
}
m_nMode = E_PROCESS_URL_FINISH_WAIT;
m_nWait = 0;
}
break;
case E_PROCESS_LIST_FINISH_WAIT:
case E_PROCESS_URL_FINISH_WAIT:
m_nWait++;
if (m_nWait > (100000/m_nTime))
{
//for(int i = 0; i < C_PROCESS_MAX ; i++)
{
if (m_pro[0].State() != SProcess::STATE_WAIT)
{
m_pro[0].kill();
m_pro[0].SetState(SProcess::STATE_WAIT);
m_pMain->InsertLog(m_nID,"Kill Process.");
}
}
if (m_nMode == E_PROCESS_LIST_FINISH_WAIT) return m_bFinalLast;
}
break;
}
return m_bFinalLast;
}
void SNaverCafeManage::processFinished(SProcess *_pPro,QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_LIST_FINISH_WAIT:
{
m_bLast = false;
if (_strOut.right(4) == "last" || m_ncList >= 1000)
m_bLast = true;
m_strListURL.clear();
foreach(QString str,_strOut.split("\n"))
{
if (str.isEmpty()) continue;
if (str.at(0) == QChar('o'))
m_strListURL.push_back(str.right(str.length()-2).trimmed());
}
/*
QSqlQuery query;
if (query.exec("SELECT URL FROM " + C_TABLE_URL + QString::number(m_nUrlTable) + " where ERROR is null"))
{
m_pMain->InsertLog(m_nID,query.lastError().text());
}
while (query.next())
m_strListURL.append(query.value(0).toString());
*/
m_ncUrl = 0;
if (m_strListURL.size() == 0)
{
m_nMode = E_PROCESS_LIST_RUN;
CheckLast();
}
else
m_nMode = E_PROCESS_URL_RUN;
break;
}
case E_PROCESS_URL_FINISH_WAIT:
m_nMode = E_PROCESS_LIST_RUN;
if (m_ncUrl >= m_strListURL.size())
{
m_nMode = E_PROCESS_LIST_RUN;
CheckLast();
m_bLast = false;
}
else
m_nMode = E_PROCESS_URL_RUN;
break;
}
}
/*
void SNaverCafeManage::MakeTables()
{
QString strQuery = "show tables";
QSqlQuery query;
query.exec(strQuery);
int nUrlMax = -1;
while (query.next())
{
QString str = query.value(0).toString();
if (str.left(C_TABLE_URL.size()) == C_TABLE_URL.toUpper())
{
if (nUrlMax < str.mid(C_TABLE_URL.size()).toInt())
nUrlMax = str.mid(C_TABLE_URL.size()).toInt();
}
}
m_nUrlTable = nUrlMax + 1;
strQuery = "Create table " + C_TABLE_URL + QString::number(m_nUrlTable)+ "(Url CHAR(128) not null primary key,keyword_id INT,PlatformTitle CHAR(128),PlatformID CHAR(64),ArticleTitle VARCHAR(128),ArticleID CHAR(32),Date DATETIME,Nickname CHAR(32),Data VARCHAR(18432),Error CHAR(32)) CHARSET=utf8";
query.exec(strQuery);
strQuery = "Create table " + C_TABLE_COM + QString::number(m_nUrlTable)+ "(Url CHAR(128) not null,Nickname CHAR(32),Data VARCHAR(1024),Parent CHAR(64),Date DATETIME,UrlReply VARCHAR(512),RowNum INT) CHARSET=utf8";
query.exec(strQuery);
m_pMain->setWindowTitle("NaverCafeCrawler " + QString::number(m_nUrlTable));
}
void SNaverCafeManage::DropTables()
{
QString strQuery = "drop table ";
QSqlQuery query;
query.exec(strQuery + C_TABLE_URL + QString::number(m_nUrlTable));
query.exec(strQuery + C_TABLE_COM + QString::number(m_nUrlTable));
}
void SNaverCafeManage::Join()
{
m_pMain->InsertLog(m_nID,"Insert Article Data...");
QString strQuery = "insert into "
"data_" + m_strGroupID +
"(platformname , platformform , articleform ,"
"url , keyword_id , body_platformtitle , body_platformid , body_articletitle , body_articleid , body_date , body_nickname , body_data)"
"select "
"CONVERT('naver' USING utf8),"
"CONVERT('cafe' USING utf8),"
"CONVERT('article' USING utf8),"
"CONVERT(url USING utf8),"
"CONVERT(keyword_id USING utf8),"
"CONVERT(PlatformTitle USING utf8),"
"CONVERT(PlatformID USING utf8),"
"CONVERT(ArticleTitle USING utf8),"
"CONVERT(ArticleID USING utf8),"
"CONVERT(Date USING utf8),"
"CONVERT(Nickname USING utf8),"
"CONVERT(Data USING utf8)"
"from NAVER_CAFE_BODY_" + QString::number(m_nUrlTable);
QSqlQuery query;
if (query.exec(strQuery) == false)
{
m_pMain->InsertLog(m_nID,query.lastError().text());
return;
}
m_pMain->InsertLog(m_nID,"Insert Reply Data...");
strQuery = "insert into "
"data_" + m_strGroupID +
"(platformname , platformform , articleform ,"
"url , keyword_id , body_platformtitle , body_platformid , body_articletitle , body_articleid , body_date , body_nickname , body_data ,"
"reply_nickname ,reply_data, reply_parent , reply_date ,reply_urlreply ,reply_rownum )"
"select "
"CONVERT('naver' USING utf8),"
"CONVERT('cafe' USING utf8),"
"CONVERT('reply' USING utf8),"
"CONVERT(_body.url USING utf8),"
"CONVERT(_body.keyword_id USING utf8),"
"CONVERT(_body.PlatformTitle USING utf8),"
"CONVERT(_body.PlatformID USING utf8),"
"CONVERT(_body.ArticleTitle USING utf8),"
"CONVERT(_body.ArticleID USING utf8),"
"CONVERT(_body.Date USING utf8),"
"CONVERT(_body.Nickname USING utf8),"
"CONVERT(_body.Data USING utf8),"
"CONVERT(_reply.Nickname USING utf8),"
"CONVERT(_reply.Data USING utf8),"
"CONVERT(_reply.Parent USING utf8),"
"CONVERT(_reply.Date USING utf8),"
"CONVERT(_reply.UrlReply USING utf8),"
"CONVERT(_reply.RowNum USING utf8) "
"from NAVER_CAFE_BODY_" + QString::number(m_nUrlTable) + " _body INNER JOIN NAVER_CAFE_REPLY_" + QString::number(m_nUrlTable) + " _reply ON _body.Url = _reply.Url";
query.exec(strQuery);
if (query.exec(strQuery) == false)
{
m_pMain->InsertLog(m_nID,query.lastError().text());
return;
}
m_pMain->InsertLog(m_nID,"Delete data ...");
query.exec("delete from NAVER_CAFE_BODY_" + QString::number(m_nUrlTable) );
query.exec("delete from NAVER_CAFE_REPLY_" + QString::number(m_nUrlTable) );
m_pMain->InsertLog(m_nID,"Finish ... ");
}
*/

28
CrawlerList/snavercafemanage.h Normal file
View File

@@ -0,0 +1,28 @@
#ifndef SNAVERCAFEMANAGE_H
#define SNAVERCAFEMANAGE_H
#include "SManage.h"
class SNaverCafeManage : public SManage
{
public:
enum E_PROCESS_STATE
{
E_PROCESS_LIST_RUN = 0,
E_PROCESS_LIST_FINISH_WAIT,
E_PROCESS_URL_RUN,
E_PROCESS_URL_FINISH_WAIT,
};
SNaverCafeManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
int m_nUrlTable;
protected:
bool Update();
void Start();
void processFinished(SProcess *pPro,QString _strOut);
};
#endif // SNAVERCAFEMANAGE_H

176
CrawlerList/widget.cpp Normal file
View File

@@ -0,0 +1,176 @@
#include "widget.h"
#include <QHBoxLayout>
#include <QSqlDatabase>
#include <QDebug>
#include <QDateEdit>
#include <QFileDialog>
#include <QPushButton>
#include <QSqlQuery>
#include "snavercafemanage.h"
Widget::Widget(QWidget *parent) : QWidget(parent)
{
connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update()));
m_pcb = new QComboBox;
m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent);
QPushButton *pbtRefresh = new QPushButton("Refresh",this);
QPushButton *pbtStart = new QPushButton("Start",this);
QPushButton *pbtStop = new QPushButton("Stop",this);
{
QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton()));
QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StartButton()));
QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton()));
}
QVBoxLayout *vlayout = new QVBoxLayout;
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pcb,Qt::AlignLeft);
hlayout->addWidget(pbtRefresh);
vlayout->addLayout(hlayout);
}
m_pedTime = new QLineEdit(this);
m_pedTime->setText(QString("500"));
{
QHBoxLayout *hlayout = new QHBoxLayout;
hlayout->addWidget(m_pedTime);
hlayout->addWidget(pbtStart);
hlayout->addWidget(pbtStop);
vlayout->addLayout(hlayout);
}
{
QHBoxLayout *hlayout = new QHBoxLayout;
for (int i = 0; i < C_CRAWLER_MAX; i++ )
hlayout->addWidget(&m_aResultList[i]);
vlayout->addLayout(hlayout);
}
setLayout(vlayout);
m_pNaverCafe = new SNaverCafeManage(this);
m_pManage[0] = m_pNaverCafe;
m_db = QSqlDatabase::addDatabase("QMYSQL");
m_db.setHostName("bigbird.iptime.org");
m_db.setUserName("admin");
m_db.setPassword("admin123");
m_db.setDatabaseName("concepters");
if (!m_db.open())
{
InsertLog(0,"MySql Error...");
return;
}
for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
m_pManage[i]->SetParent(this);
RefreshButton();
}
Widget::~Widget()
{
m_db.close();
m_pNaverCafe->SetParent(0);
}
void Widget::InsertLog(int _nSelect,QString str)
{
if (_nSelect >= C_CRAWLER_MAX) return;
QTime time = QTime::currentTime();
QString strOut = time.toString("[hh:mm:ss] ") + str;
m_aResultList[_nSelect].addItem(strOut);
QDate date = QDate::currentDate();
QFile file(date.toString(Qt::ISODate)+"_"+QString::number(_nSelect)+".log");
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
return;
QTextStream out(&file);
out << strOut << "\n";
file.close();
if (m_aResultList[_nSelect].count() > 1024)
{
m_aResultList[_nSelect].removeItemWidget(m_aResultList[_nSelect].item(0));
QListWidgetItem* item = m_aResultList[_nSelect].takeItem(0);
delete item;
}
m_aResultList[_nSelect].setCurrentRow( m_aResultList[_nSelect].count() - 1 );
m_aResultList[_nSelect].repaint();
}
void Widget::StartButton()
{
m_timer.stop();
m_timer.start(m_pedTime->text().trimmed().toInt());
QSqlQuery query;
query.exec("UPDATE crawling set state = '" + QString("run") + "' where id = '" + m_pcb->currentData().toString() + "'");
query.exec("SELECT _keyword.start,_keyword.end, _keyword.searches,_keyword.authorship,_keyword.id,_datagroup.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id "
"where _crawling.id = '" + m_pcb->currentData().toString() + "'");
query.next();
QString str = + " , ";
str += query.value(1).toString() + " ~ ";
str += query.value(2).toString() + " -> ";
str += query.value(3).toString();
m_pManage[0]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"),
QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"),
query.value(2).toString().trimmed(),// keyword
query.value(3).toString().trimmed(),// authorship
query.value(4).toString().trimmed(),// keyword_id
query.value(5).toString().trimmed(),
1,
m_pedTime->text().trimmed().toInt());
}
void Widget::StopButton()
{
m_timer.stop();
}
void Widget::Update()
{
if (m_db.isOpen() == false)
{
if (m_db.open())
{
InsertLog(0,"MySql Open Error...");
m_timer.stop();
return;
}
}
int nCount = 0;
for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
nCount += m_pManage[i]->Update();
if (nCount == C_CRAWLER_MAX)
{
InsertLog(0,"Finish...");
m_timer.stop();
}
}
void Widget::RefreshButton()
{
m_pcb->clear();
QSqlQuery query("SELECT _keyword.searches,_keyword.authorship,_keyword.start,_keyword.end, _datagroup.name , _crawling.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id where _crawling.state is null");
while (query.next())
{
QString str;
str += query.value(0).toString() + " , ";
str += query.value(1).toString() + " , ";
str += query.value(2).toString() + " ~ ";
str += query.value(3).toString() + " -> ";
str += query.value(4).toString();
m_pcb->addItem(str,query.value(5));
}
}

46
CrawlerList/widget.h Normal file
View File

@@ -0,0 +1,46 @@
#ifndef WIDGET_H
#define WIDGET_H
#include <QWidget>
#include <QLineEdit>
#include <QTimer>
#include <QDateEdit>
#include <QListWidget>
#include <QSqlDatabase>
#include <QComboBox>
class SNaverCafeManage;
class SManage;
#define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; }
class Widget : public QWidget
{
Q_OBJECT
public:
Widget(QWidget *parent = 0);
~Widget();
public:
void InsertLog(int _nSelect,QString str);
private:
QLineEdit *m_pedTime;
QTimer m_timer;
QSqlDatabase m_db;
static const int C_CRAWLER_MAX = 1;
SManage *m_pManage[C_CRAWLER_MAX];
QListWidget m_aResultList[C_CRAWLER_MAX];
QString m_strFileName;
QComboBox *m_pcb;
QVector <QString> m_vecSelect;
SNaverCafeManage *m_pNaverCafe;
private:
QString makeCafeGetListQuery(QString _str,QDate _date,int _nPage);
private slots:
void RefreshButton();
void StartButton();
void StopButton();
void Update();
};
#endif // WIDGET_H