diff --git a/CrawlerList/CrawlerList.pro b/CrawlerList/CrawlerList.pro index 2a4656c..c1200be 100644 --- a/CrawlerList/CrawlerList.pro +++ b/CrawlerList/CrawlerList.pro @@ -28,7 +28,11 @@ SOURCES += main.cpp\ skakaousermanage.cpp \ sfacebooktagmanage.cpp \ sfacebookusermanage.cpp \ - snaverblogaccuracymanager.cpp + snaverblogaccuracymanager.cpp \ + stwittertagmanage.cpp \ + stwitterusermanage.cpp \ + syoutubetagmanage.cpp \ + syoutubeusermanage.cpp HEADERS += widget.h \ smanage.h \ @@ -45,5 +49,9 @@ HEADERS += widget.h \ skakaousermanage.h \ sfacebooktagmanage.h \ sfacebookusermanage.h \ - snaverblogaccuracymanage.h + snaverblogaccuracymanage.h \ + stwittertagmanage.h \ + stwitterusermanage.h \ + syoutubetagmanage.h \ + syoutubeusermanage.h diff --git a/CrawlerList/stwittertagmanage.cpp b/CrawlerList/stwittertagmanage.cpp new file mode 100644 index 0000000..0074828 --- /dev/null +++ b/CrawlerList/stwittertagmanage.cpp @@ -0,0 +1,89 @@ +#include "stwittertagmanage.h" +#include +#include "widget.h" +STwitterTagManage::STwitterTagManage(QObject *pObject) : SManage(pObject) +{ + m_nID = 0; + connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput())); + connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError())); +} + + +void STwitterTagManage::Start() +{ + m_nMode = E_PROCESS_RUN; + m_bFinalLast = false; +} + +bool STwitterTagManage::Update() +{ + if(m_bFinalLast) return m_bFinalLast; + switch(m_nMode) + { + case E_PROCESS_RUN: + if(UseProcess() == false) + { +#if defined(Q_OS_WIN32) + m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#else + m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#endif + m_nMode = E_PROCESS_FINISH_WAIT; + } + break; + + case E_PROCESS_FINISH_WAIT: + break; + } + + return m_bFinalLast; +} + +void STwitterTagManage::processFinished(QProcess *pPro, QString _strOut) +{ + switch(m_nMode) + { + case E_PROCESS_FINISH_WAIT: + m_nMode = E_PROCESS_RUN; + m_bFinalLast = true; + m_pMain->InsertLog("Finish Crawling :)"); + m_pMain->SetCrawlingState("Finish"); + m_ncList=1; + m_bLast = false; + break; + } +} + +void STwitterTagManage::readStandardOutput() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardOutput(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} + +void STwitterTagManage::readStandardError() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardError(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} diff --git a/CrawlerList/stwittertagmanage.h b/CrawlerList/stwittertagmanage.h new file mode 100644 index 0000000..2680b35 --- /dev/null +++ b/CrawlerList/stwittertagmanage.h @@ -0,0 +1,30 @@ +#ifndef STWITTERTAGMANAGE_H +#define STWITTERTAGMANAGE_H +#include "smanage.h" + +class STwitterTagManage : public SManage +{ + Q_OBJECT +public: + enum E_PROCESS_STATE + { + E_PROCESS_RUN = 0, + E_PROCESS_FINISH_WAIT, + }; + STwitterTagManage(QObject *pObject); +private: + QString makeGetListQuery(QString _str,QDate _date,int _nPage); +private: + QString m_strListQuery; + QVector m_strListURL; +protected: + bool Update(); + void Start(); + void processFinished(QProcess *pPro,QString _strOut); + void ReLoadList(); +private slots: + void readStandardOutput(); + void readStandardError(); +}; +#endif // STWITTERTAGMANAGE_H + diff --git a/CrawlerList/stwitterusermanage.cpp b/CrawlerList/stwitterusermanage.cpp new file mode 100644 index 0000000..28f751b --- /dev/null +++ b/CrawlerList/stwitterusermanage.cpp @@ -0,0 +1,89 @@ +#include "stwitterusermanage.h" +#include +#include "widget.h" +STwitterUserManage::STwitterUserManage(QObject *pObject) : SManage(pObject) +{ + m_nID = 0; + connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput())); + connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError())); +} + + +void STwitterUserManage::Start() +{ + m_nMode = E_PROCESS_RUN; + m_bFinalLast = false; +} + +bool STwitterUserManage::Update() +{ + if(m_bFinalLast) return m_bFinalLast; + switch(m_nMode) + { + case E_PROCESS_RUN: + if(UseProcess() == false) + { +#if defined(Q_OS_WIN32) + m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#else + m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#endif + m_nMode = E_PROCESS_FINISH_WAIT; + } + break; + + case E_PROCESS_FINISH_WAIT: + break; + } + + return m_bFinalLast; +} + +void STwitterUserManage::processFinished(QProcess *pPro, QString _strOut) +{ + switch(m_nMode) + { + case E_PROCESS_FINISH_WAIT: + m_nMode = E_PROCESS_RUN; + m_bFinalLast = true; + m_pMain->InsertLog("Finish Crawling :)"); + m_pMain->SetCrawlingState("Finish"); + m_ncList=1; + m_bLast = false; + break; + } +} + +void STwitterUserManage::readStandardOutput() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardOutput(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} + +void STwitterUserManage::readStandardError() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardError(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} diff --git a/CrawlerList/stwitterusermanage.h b/CrawlerList/stwitterusermanage.h new file mode 100644 index 0000000..5c370ec --- /dev/null +++ b/CrawlerList/stwitterusermanage.h @@ -0,0 +1,30 @@ +#ifndef STWITTERUSERMANAGE_H +#define STWITTERUSERMANAGE_H +#include "smanage.h" + +class STwitterUserManage : public SManage +{ + Q_OBJECT +public: + enum E_PROCESS_STATE + { + E_PROCESS_RUN = 0, + E_PROCESS_FINISH_WAIT, + }; + STwitterUserManage(QObject *pObject); +private: + QString makeGetListQuery(QString _str,QDate _date,int _nPage); +private: + QString m_strListQuery; + QVector m_strListURL; +protected: + bool Update(); + void Start(); + void processFinished(QProcess *pPro,QString _strOut); + void ReLoadList(); +private slots: + void readStandardOutput(); + void readStandardError(); +}; +#endif // STWITTERUSERMANAGE_H + diff --git a/CrawlerList/syoutubetagmanage.cpp b/CrawlerList/syoutubetagmanage.cpp new file mode 100644 index 0000000..3845df4 --- /dev/null +++ b/CrawlerList/syoutubetagmanage.cpp @@ -0,0 +1,89 @@ +#include "syoutubetagmanage.h" +#include +#include "widget.h" +SYoutubeTagManage::SYoutubeTagManage(QObject *pObject) : SManage(pObject) +{ + m_nID = 0; + connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput())); + connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError())); +} + + +void SYoutubeTagManage::Start() +{ + m_nMode = E_PROCESS_RUN; + m_bFinalLast = false; +} + +bool SYoutubeTagManage::Update() +{ + if(m_bFinalLast) return m_bFinalLast; + switch(m_nMode) + { + case E_PROCESS_RUN: + if(UseProcess() == false) + { +#if defined(Q_OS_WIN32) + m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#else + m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#endif + m_nMode = E_PROCESS_FINISH_WAIT; + } + break; + + case E_PROCESS_FINISH_WAIT: + break; + } + + return m_bFinalLast; +} + +void SYoutubeTagManage::processFinished(QProcess *pPro, QString _strOut) +{ + switch(m_nMode) + { + case E_PROCESS_FINISH_WAIT: + m_nMode = E_PROCESS_RUN; + m_bFinalLast = true; + m_pMain->InsertLog("Finish Crawling :)"); + m_pMain->SetCrawlingState("Finish"); + m_ncList=1; + m_bLast = false; + break; + } +} + +void SYoutubeTagManage::readStandardOutput() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardOutput(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} + +void SYoutubeTagManage::readStandardError() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardError(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} diff --git a/CrawlerList/syoutubetagmanage.h b/CrawlerList/syoutubetagmanage.h new file mode 100644 index 0000000..a8f49b2 --- /dev/null +++ b/CrawlerList/syoutubetagmanage.h @@ -0,0 +1,30 @@ +#ifndef STYOUTUBETAGMANAGE_H +#define STYOUTUBETAGMANAGE_H +#include "smanage.h" + +class SYoutubeTagManage : public SManage +{ + Q_OBJECT +public: + enum E_PROCESS_STATE + { + E_PROCESS_RUN = 0, + E_PROCESS_FINISH_WAIT, + }; + SYoutubeTagManage(QObject *pObject); +private: + QString makeGetListQuery(QString _str,QDate _date,int _nPage); +private: + QString m_strListQuery; + QVector m_strListURL; +protected: + bool Update(); + void Start(); + void processFinished(QProcess *pPro,QString _strOut); + void ReLoadList(); +private slots: + void readStandardOutput(); + void readStandardError(); +}; +#endif // STYOUTUBETAGMANAGE_H + diff --git a/CrawlerList/syoutubeusermanage.cpp b/CrawlerList/syoutubeusermanage.cpp new file mode 100644 index 0000000..a53f244 --- /dev/null +++ b/CrawlerList/syoutubeusermanage.cpp @@ -0,0 +1,89 @@ +#include "syoutubeusermanage.h" +#include +#include "widget.h" +SYoutubeUserManage::SYoutubeUserManage(QObject *pObject) : SManage(pObject) +{ + m_nID = 0; + connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput())); + connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError())); +} + + +void SYoutubeUserManage::Start() +{ + m_nMode = E_PROCESS_RUN; + m_bFinalLast = false; +} + +bool SYoutubeUserManage::Update() +{ + if(m_bFinalLast) return m_bFinalLast; + switch(m_nMode) + { + case E_PROCESS_RUN: + if(UseProcess() == false) + { +#if defined(Q_OS_WIN32) + m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#else + m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage()); +#endif + m_nMode = E_PROCESS_FINISH_WAIT; + } + break; + + case E_PROCESS_FINISH_WAIT: + break; + } + + return m_bFinalLast; +} + +void SYoutubeUserManage::processFinished(QProcess *pPro, QString _strOut) +{ + switch(m_nMode) + { + case E_PROCESS_FINISH_WAIT: + m_nMode = E_PROCESS_RUN; + m_bFinalLast = true; + m_pMain->InsertLog("Finish Crawling :)"); + m_pMain->SetCrawlingState("Finish"); + m_ncList=1; + m_bLast = false; + break; + } +} + +void SYoutubeUserManage::readStandardOutput() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardOutput(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} + +void SYoutubeUserManage::readStandardError() +{ + QProcess *pPro = (QProcess*)sender(); + QThread::msleep(100); + QString str = pPro->readAllStandardError(); + QStringList list = str.split("\n", QString::SkipEmptyParts); + foreach(QString log,list) + { + if (m_pMain) + { + m_pMain->InsertLog(log); + } + else + exit(0); + } +} diff --git a/CrawlerList/syoutubeusermanage.h b/CrawlerList/syoutubeusermanage.h new file mode 100644 index 0000000..d3e461f --- /dev/null +++ b/CrawlerList/syoutubeusermanage.h @@ -0,0 +1,30 @@ +#ifndef YOUTUBE_USER_MANAGE_H +#define YOUTUBE_USER_MANAGE_H +#include "smanage.h" + +class SYoutubeUserManage : public SManage +{ + Q_OBJECT +public: + enum E_PROCESS_STATE + { + E_PROCESS_RUN = 0, + E_PROCESS_FINISH_WAIT, + }; + SYoutubeUserManage(QObject *pObject); +private: + QString makeGetListQuery(QString _str,QDate _date,int _nPage); +private: + QString m_strListQuery; + QVector m_strListURL; +protected: + bool Update(); + void Start(); + void processFinished(QProcess *pPro,QString _strOut); + void ReLoadList(); +private slots: + void readStandardOutput(); + void readStandardError(); +}; +#endif // YOUTUBE_USER_MANAGE_H + diff --git a/CrawlerList/widget.cpp b/CrawlerList/widget.cpp index 634f28a..7706626 100644 --- a/CrawlerList/widget.cpp +++ b/CrawlerList/widget.cpp @@ -22,6 +22,10 @@ #include "sfacebooktagmanage.h" #include "sfacebookusermanage.h" #include "snaverblogaccuracymanage.h" +#include "stwittertagmanage.h" +#include "stwitterusermanage.h" +#include "syoutubetagmanage.h" +#include "syoutubeusermanage.h" #include #include @@ -51,6 +55,10 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) m_pFacebookTag = new SFacebookTagManage(this); m_pFacebookUser = new SFacebookUserManage(this); m_pNaverBlogAccuracy = new SNaverBlogAccuracyManage(this); + m_pTwitterTag = new STwitterTagManage(this); + m_pTwitterUser = new STwitterUserManage(this); + m_pYoutubeTag = new SYoutubeTagManage(this); + m_pYoutubeUser = new SYoutubeUserManage(this); m_pManage[0] = m_pNaverCafe; m_pManage[1] = m_pNaverBlog; @@ -66,6 +74,10 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) m_pManage[11] = m_pFacebookTag; m_pManage[12] = m_pFacebookUser; m_pManage[13] = m_pNaverBlogAccuracy; + m_pManage[14] = m_pTwitterTag; + m_pManage[15] = m_pTwitterUser; + m_pManage[16] = m_pYoutubeTag; + m_pManage[17] = m_pYoutubeUser; m_db = QSqlDatabase::addDatabase("QMYSQL"); m_db.setHostName("bigbird.iptime.org"); diff --git a/CrawlerList/widget.h b/CrawlerList/widget.h index 048d5a5..c45b3f7 100644 --- a/CrawlerList/widget.h +++ b/CrawlerList/widget.h @@ -26,6 +26,10 @@ class SInstaUserManage; class SFacebookTagManage; class SFacebookUserManage; class SNaverBlogAccuracyManage; +class STwitterTagManage; +class STwitterUserManage; +class SYoutubeTagManage; +class SYoutubeUserManage; #define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; } @@ -55,7 +59,7 @@ private: QLineEdit *m_pedStartDay; QTimer m_timer,m_timerAlive; QSqlDatabase m_db; - static const int C_PLATFORM_MAX = 14; + static const int C_PLATFORM_MAX = 18; SManage *m_pManage[C_PLATFORM_MAX]; QListWidget *m_pResultList; QString m_strFileName; @@ -75,6 +79,11 @@ private: SFacebookTagManage *m_pFacebookTag; SFacebookUserManage *m_pFacebookUser; SNaverBlogAccuracyManage *m_pNaverBlogAccuracy; + STwitterTagManage* m_pTwitterTag; + STwitterUserManage* m_pTwitterUser; + SYoutubeTagManage* m_pYoutubeTag; + SYoutubeUserManage* m_pYoutubeUser; + int m_nStartTime,m_nRangeTime,m_nPlatform; //QGroupBox *m_pgbManual; QCheckBox *m_pcheckboxReal;