etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Data Tree 구조 변경

Browse Source 
git-svn-id: svn://192.168.0.12/source@10 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-01-21 09:42:59 +00:00
parent fe53bffcf4
commit 4797e2f354
3 changed files with 21 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
NaverCafeCrawler/snavercafemanage.cpp
View File

@@ -5,10 +5,9 @@
#include <qDebug>
#include <QFile>
SNaverCafeManage::SNaverCafeManage(QObject *pObject) : SManage(pObject) , C_TABLE_URL("NAVER_CAFE_BODY_") , C_TABLE_COM("NAVER_CAFE_REPLY_")
SNaverCafeManage::SNaverCafeManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
m_nUrlTable = -1;
m_nID = 0;
}
QString SNaverCafeManage::makeGetListQuery(QString _str,QDate _date,int _nPage)
@@ -49,10 +48,9 @@ bool SNaverCafeManage::Update()
if (UseProcess() == false)
{
m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
m_pMain->InsertLog(m_nID,"Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
//for(int i = 0; i < C_PROCESS_MAX ; i++)
m_pMain->InsertLog(m_nID,"Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "cafe_list" << QString::number(m_nUrlTable) << m_strListQuery << m_strKeywordID);
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
m_pro[0].SetState(SProcess::STATE_RUNNING);
m_ncList+=10;
}
@@ -64,9 +62,8 @@ bool SNaverCafeManage::Update()
if (UseProcess() == false)
{
m_pMain->InsertLog(m_nID,"(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
//for(int i = 0; i < C_PROCESS_MAX ; i++)
{
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "cafe_data" << QString::number(m_nUrlTable) << m_strListURL.at(m_ncUrl) << m_strListQuery << "" );
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << "" );
m_pro[0].SetState(SProcess::STATE_RUNNING);
m_ncUrl++;
}
@@ -105,6 +102,13 @@ void SNaverCafeManage::processFinished(SProcess *_pPro,QString _strOut)
if (_strOut.right(4) == "last" || m_ncList >= 1000)
m_bLast = true;
m_strListURL.clear();
foreach(QString str,_strOut.split("\n"))
{
if (str.at(0) == QChar('o'))
m_strListURL.push_back(str.right(str.length()-2).trimmed());
}
/*
QSqlQuery query;
if (query.exec("SELECT URL FROM " + C_TABLE_URL + QString::number(m_nUrlTable) + " where ERROR is null"))
{
@@ -112,6 +116,7 @@ void SNaverCafeManage::processFinished(SProcess *_pPro,QString _strOut)
}
while (query.next())
m_strListURL.append(query.value(0).toString());
*/
m_ncUrl = 0;
if (m_strListURL.size() == 0)
{
@@ -136,6 +141,7 @@ void SNaverCafeManage::processFinished(SProcess *_pPro,QString _strOut)
}
}
/*
void SNaverCafeManage::MakeTables()
{
QString strQuery = "show tables";
@@ -235,3 +241,4 @@ void SNaverCafeManage::Join()
query.exec("delete from NAVER_CAFE_REPLY_" + QString::number(m_nUrlTable) );
m_pMain->InsertLog(m_nID,"Finish ... ");
}
*/

10
NaverCafeCrawler/snavercafemanage.h
View File

@@ -12,15 +12,9 @@ public:
E_PROCESS_URL_RUN,
E_PROCESS_URL_FINISH_WAIT,
};
SNaverCafeManage(QObject *pObject);
void MakeTables();
void DropTables();
void Join();
int GetTableNumber() {return m_nUrlTable;}
SNaverCafeManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
const QString C_TABLE_URL;
const QString C_TABLE_COM;
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;

20
NaverCafeCrawler/widget.cpp
View File

@@ -69,13 +69,11 @@ Widget::Widget(QWidget *parent) : QWidget(parent)
for (int i = 0 ; i < C_CRAWLER_MAX ; i++)
m_pManage[i]->SetParent(this);
m_pNaverCafe->MakeTables();
RefreshButton();
}
Widget::~Widget()
{
m_pNaverCafe->DropTables();
{
m_db.close();
m_pNaverCafe->SetParent(0);
}
@@ -113,7 +111,7 @@ void Widget::StartButton()
m_timer.start(m_pedTime->text().trimmed().toInt());
QSqlQuery query;
query.exec("UPDATE crawling set state = '" + QString::number(m_pNaverCafe->GetTableNumber()) + "' where id = '" + m_pcb->currentData().toString() + "'");
query.exec("UPDATE crawling set state = '" + QString("run") + "' where id = '" + m_pcb->currentData().toString() + "'");
query.exec("SELECT _keyword.start,_keyword.end, _keyword.searches,_keyword.authorship,_keyword.id,_datagroup.id "
"FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id "
"inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id "
@@ -131,16 +129,6 @@ void Widget::StartButton()
query.value(5).toString().trimmed(),
1,
m_pedTime->text().trimmed().toInt());
/*
m_pManage[0]->Start(QDate::fromString(QString("2014-11-24"),"yyyy-MM-dd"),
QDate::fromString(QString("2014-11-24"),"yyyy-MM-dd"),
QString("양악"),// keyword
QString("2"),// keyword_id
QString("12"),
41,
m_pedTime->text().trimmed().toInt());
*/
}
void Widget::StopButton()
@@ -164,8 +152,8 @@ void Widget::Update()
nCount += m_pManage[i]->Update();
if (nCount == C_CRAWLER_MAX)
{
m_timer.stop();
m_pNaverCafe->Join();
InsertLog(0,"Finish...");
m_timer.stop();
}
}