etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

리눅스에서 Process 들이 ./로 실행되도록 수정

Browse Source 
실시간 크롤러에서 오늘날짜 +1이 되는 오류 수정
crawlerfinish 이후 db가 끊기면 재시도 하지 않는 버그 수정(테스트 꾸준히 해봐야함)


git-svn-id: svn://192.168.0.12/source@216 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2015-10-28 08:13:59 +00:00
parent 0311c2abfc
commit ec55f45ce4
5 changed files with 52 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CrawlerList/sdaumcafemanage.cpp
View File

@@ -53,7 +53,11 @@ bool SDaumCafeManage::Update()
m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
m_pMain->InsertLog("Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#else
m_pro[0].start("./CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#endif
m_ncList+=1;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
@@ -65,7 +69,11 @@ bool SDaumCafeManage::Update()
{
m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
{
#if defined(Q_OS_WIN32)
m_pro[0].start("AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#else
m_pro[0].start("./AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#endif
//m_pro[0].start("CrawlerProcess",QStringList() << "daum" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
m_ncUrl++;
}
@@ -115,7 +123,7 @@ void SDaumCafeManage::processFinished(QProcess *_pPro,QString _strOut)
if (_strOut.right(5) == "block")
{
reloaded = true;
reloaded = true;
ReLoadList();
}

12
CrawlerList/snaverblogmanage.cpp
View File

@@ -186,7 +186,11 @@ bool SNaverBlogManage::Update()
//m_pMain->InsertLog(m_nID,"Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
m_pMain->InsertLog("Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "blog_list" << m_strQuery << m_strGroupID << m_strKeywordID);
#else
m_pro[0].start("./CrawlerProcess",QStringList()<< "naver" << "blog_list" << m_strQuery << m_strGroupID << m_strKeywordID);
#endif
m_ncList+=10;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
@@ -197,7 +201,11 @@ bool SNaverBlogManage::Update()
if (UseProcess() == false)
{
m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "blog_url" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#else
m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "blog_url" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#endif
m_nMode = E_PROCESS_URL_FINISH_WAIT;
m_nWait = 0;
}
@@ -205,7 +213,11 @@ bool SNaverBlogManage::Update()
case E_PROCESS_COMMENT_RUN:
if (UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "blog_comm" << makeGetCommentQuery(m_strListURL.at(m_ncUrl++)) << m_strGroupID << "" );
#else
m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "blog_comm" << makeGetCommentQuery(m_strListURL.at(m_ncUrl++)) << m_strGroupID << "" );
#endif
m_nMode = E_PROCESS_COMMENT_FINISH_WAIT;
m_nWait = 0;
}

9
CrawlerList/snavercafemanage.cpp
View File

@@ -52,7 +52,11 @@ bool SNaverCafeManage::Update()
m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
m_pMain->InsertLog("Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#else
m_pro[0].start("./CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#endif
m_ncList+=10;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
@@ -64,7 +68,11 @@ bool SNaverCafeManage::Update()
{
m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
#else
m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
#endif
m_ncUrl++;
}
m_nMode = E_PROCESS_URL_FINISH_WAIT;
@@ -172,6 +180,7 @@ void SNaverCafeManage::processFinished(QProcess *_pPro,QString _strOut)
break;
}
}
void SNaverCafeManage::ReLoadList()
{
m_nMode = E_PROCESS_LIST_RUN;

8
CrawlerList/snavernewsmanage.cpp
View File

@@ -39,7 +39,11 @@ bool SNaverNewsManage::Update()
m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
m_pMain->InsertLog("Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd"));
{
#if defined(Q_OS_WIN32)
m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "news_list" << m_strListQuery);
#else
m_pro[0].start("./CrawlerProcess",QStringList()<< "naver" << "news_list" << m_strListQuery);
#endif
m_ncList++;
}
m_nMode = E_PROCESS_LIST_FINISH_WAIT;
@@ -51,7 +55,11 @@ bool SNaverNewsManage::Update()
{
m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
{
#if defined(Q_OS_WIN32)
m_pro[0].start("AjaxCrawlerProcess",QStringList() << "naver_news" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#else
m_pro[0].start("./AjaxCrawlerProcess",QStringList() << "naver_news" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#endif
m_ncUrl++;
}
m_nMode = E_PROCESS_URL_FINISH_WAIT;

16
CrawlerList/widget.cpp
View File

@@ -44,7 +44,7 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
//m_db.setConnectOptions("MYSQL_OPT_RECONNECT=1");
if (!m_db.open())
{
InsertLog("MySql Error...");
InsertLog("DB Open Failed");
return;
}
@@ -201,7 +201,7 @@ void Widget::Start()
if (m_pcheckboxReal->isChecked())
{
m_pManage[m_nPlatform]->Start(QDate::currentDate().addDays(m_pedStartDay->text().trimmed().toInt()),QDate::currentDate().addDays(1),
m_pManage[m_nPlatform]->Start(QDate::currentDate().addDays(m_pedStartDay->text().trimmed().toInt()),QDate::currentDate(),
query.value(2).toString().trimmed(),// keyword
query.value(3).toString().trimmed(),// authorship
query.value(4).toString().trimmed(),// keyword_id
@@ -245,7 +245,19 @@ void Widget::Update()
if (m_pcheckboxReal->isChecked())
{
if (m_strCrawlingID.isEmpty() == false)
{
if(!m_db.isOpen())
{
InsertLog("Retry connect DB");
if(!m_db.open())
{
InsertLog("DB open failed");
return;
}
}
Start();
}
}
break;
case E_MODE_RUN: