From 820fa031e2d52b477a5583e865c5fe1d460dc26e Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 24 Dec 2015 01:39:18 +0000 Subject: [PATCH] =?UTF-8?q?naver=20news=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: svn://192.168.0.12/source@230 8346c931-da38-4b9b-9d4c-e48b93cbd075 --- CrawlerList/CrawlerList.pro | 10 +++++++-- CrawlerList/snavercafelistmanager.cpp | 4 ++-- CrawlerList/snavernewsmanage.cpp | 29 +++++++++++++++++++++++---- CrawlerList/snavernewsmanage.h | 2 ++ CrawlerList/widget.cpp | 14 ++++++++++++- CrawlerList/widget.h | 10 ++++++++- 6 files changed, 59 insertions(+), 10 deletions(-) diff --git a/CrawlerList/CrawlerList.pro b/CrawlerList/CrawlerList.pro index dcff204..f919c06 100644 --- a/CrawlerList/CrawlerList.pro +++ b/CrawlerList/CrawlerList.pro @@ -22,7 +22,10 @@ SOURCES += main.cpp\ snavercafelistmanager.cpp \ sdaumcafelistmanage.cpp \ skakaochannelmanage.cpp \ - skakaotagmanage.cpp + skakaotagmanage.cpp \ + sinstausermanage.cpp \ + sinstatagmanage.cpp \ + skakaousermanage.cpp HEADERS += widget.h \ smanage.h \ @@ -33,5 +36,8 @@ HEADERS += widget.h \ snavercafelistmanager.h \ sdaumcafelistmanage.h \ skakaochannelmanage.h \ - skakaotagmanage.h + skakaotagmanage.h \ + sinstausermanage.h \ + sinstatagmanage.h \ + skakaousermanage.h diff --git a/CrawlerList/snavercafelistmanager.cpp b/CrawlerList/snavercafelistmanager.cpp index 60fc6f0..d04d183 100644 --- a/CrawlerList/snavercafelistmanager.cpp +++ b/CrawlerList/snavercafelistmanager.cpp @@ -24,9 +24,9 @@ bool SNaverCafeListManage::Update() if(UseProcess() == false) { #if defined(Q_OS_WIN32) - m_pro[0].start("python", QStringList() << "navercrawl.py" << m_strKeywordID << m_strGroupID << m_pMain->StartDay()); + m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "navercafe" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); #else - m_pro[0].start("/usr/bin/python3", QStringList() << "navercrawl.py" << m_strKeywordID << m_strGroupID << m_pMain->StartDay()); + m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "navercafe" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); #endif m_nMode = E_PROCESS_FINISH_WAIT; } diff --git a/CrawlerList/snavernewsmanage.cpp b/CrawlerList/snavernewsmanage.cpp index 83b0047..1030471 100644 --- a/CrawlerList/snavernewsmanage.cpp +++ b/CrawlerList/snavernewsmanage.cpp @@ -57,18 +57,32 @@ bool SNaverNewsManage::Update() m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")"); { #if defined(Q_OS_WIN32) - m_pro[0].start("AjaxCrawlerProcess",QStringList() << "naver_news" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); + m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "news_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); #else - m_pro[0].start("./AjaxCrawlerProcess",QStringList() << "naver_news" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); + m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "news_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); #endif - m_ncUrl++; + //m_ncUrl++; } m_nMode = E_PROCESS_URL_FINISH_WAIT; m_nWait = 0; } break; + case E_PROCESS_COMMENT_RUN: + if (UseProcess() == false) + { +#if defined(Q_OS_WIN32) + m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "news_comm" << m_strListURL.at(m_ncUrl) + "&m_view=1" << m_strGroupID << "" ); +#else + m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "news_comm" << m_strListURL.at(m_ncUrl) + "&m_view=1" << m_strGroupID << "" ); +#endif + m_ncUrl++; + m_nMode = E_PROCESS_COMMENT_FINISH_WAIT; + m_nWait = 0; + } + break; case E_PROCESS_LIST_FINISH_WAIT: case E_PROCESS_URL_FINISH_WAIT: + case E_PROCESS_COMMENT_FINISH_WAIT: m_nWait++; if (m_nWait >= 300) { @@ -157,7 +171,14 @@ void SNaverNewsManage::processFinished(QProcess *_pPro,QString _strOut) break; } case E_PROCESS_URL_FINISH_WAIT: - m_nMode = E_PROCESS_LIST_RUN; + if (UseProcess() == false) + { + m_nMode = E_PROCESS_COMMENT_RUN; + } + break; + + case E_PROCESS_COMMENT_FINISH_WAIT: + //m_nMode = E_PROCESS_LIST_RUN; if (m_ncUrl >= m_strListURL.size()) { m_nMode = E_PROCESS_LIST_RUN; diff --git a/CrawlerList/snavernewsmanage.h b/CrawlerList/snavernewsmanage.h index 28ee6a2..4dab70e 100644 --- a/CrawlerList/snavernewsmanage.h +++ b/CrawlerList/snavernewsmanage.h @@ -13,6 +13,8 @@ public: E_PROCESS_LIST_FINISH_WAIT, E_PROCESS_URL_RUN, E_PROCESS_URL_FINISH_WAIT, + E_PROCESS_COMMENT_RUN, + E_PROCESS_COMMENT_FINISH_WAIT, }; private: QString makeGetListQuery(QString _str,QDate _date,int _nPage); diff --git a/CrawlerList/widget.cpp b/CrawlerList/widget.cpp index ed30b9b..8a3da52 100644 --- a/CrawlerList/widget.cpp +++ b/CrawlerList/widget.cpp @@ -16,6 +16,9 @@ #include "sdaumcafelistmanage.h" #include "skakaochannelmanage.h" #include "skakaotagmanage.h" +#include "skakaousermanage.h" +#include "sinstatagmanage.h" +#include "sinstausermanage.h" #include #include @@ -39,6 +42,9 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) m_pDaumCafeList = new SDaumCafeListManage(this); m_pKakaoChannel = new SKakaoChannelManage(this); m_pKakaoTag = new SKakaoTagManage(this); + m_pKakaoUser = new SKakaoUserManage(this); + m_pInstaTag = new SInstaTagManage(this); + m_pInstaUser = new SInstaUserManage(this); m_pManage[0] = m_pNaverCafe; m_pManage[1] = m_pNaverBlog; @@ -48,6 +54,9 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) m_pManage[5] = m_pDaumCafeList; m_pManage[6] = m_pKakaoChannel; m_pManage[7] = m_pKakaoTag; + m_pManage[8] = m_pKakaoUser; + m_pManage[9] = m_pInstaTag; + m_pManage[10] = m_pInstaUser; m_db = QSqlDatabase::addDatabase("QMYSQL"); m_db.setHostName("bigbird.iptime.org"); @@ -255,7 +264,7 @@ void Widget::StopButton() SetCrawlingState("Stop"); m_strCrawlingID.clear(); //qDebug() << m_nPlatform; - if(4 <= m_nPlatform && m_nPlatform <= 7) + if(4 <= m_nPlatform && m_nPlatform <= 10) { m_pManage[m_nPlatform]->clossProcess(); } @@ -313,6 +322,9 @@ void Widget::RefreshButton() case 5:str += ", Daum Cafe List"; break; case 6:str += ", Kakao Story Channel"; break; case 7:str += ", Kakao Story Tag"; break; + case 8:str += ", Kakao Story User"; break; + case 9:str += ", Instagram Tag"; break; + case 10:str += ", Instagram User"; break; } m_pcb->addItem(str,query.value(7)); } diff --git a/CrawlerList/widget.h b/CrawlerList/widget.h index 4475219..fa4b02c 100644 --- a/CrawlerList/widget.h +++ b/CrawlerList/widget.h @@ -20,6 +20,11 @@ class SManage; class SDaumCafeListManage; class SKakaoChannelManage; class SKakaoTagManage; +class SKakaoUserManage; +class SInstaTagManage; +class SInstaUserManage; + + #define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; } @@ -49,7 +54,7 @@ private: QLineEdit *m_pedStartDay; QTimer m_timer,m_timerAlive; QSqlDatabase m_db; - static const int C_PLATFORM_MAX = 8; + static const int C_PLATFORM_MAX = 11; SManage *m_pManage[C_PLATFORM_MAX]; QListWidget *m_pResultList; QString m_strFileName; @@ -63,6 +68,9 @@ private: SDaumCafeListManage *m_pDaumCafeList; SKakaoChannelManage *m_pKakaoChannel; SKakaoTagManage *m_pKakaoTag; + SKakaoUserManage *m_pKakaoUser; + SInstaTagManage *m_pInstaTag; + SInstaUserManage *m_pInstaUser; int m_nStartTime,m_nRangeTime,m_nPlatform; //QGroupBox *m_pgbManual;