etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

- 트위터, 유튜브 추가

Browse Source
This commit is contained in:
mjjo
2017-07-27 11:30:08 +09:00
parent 82956f7748
commit b0c1d0dabd
11 changed files with 508 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
CrawlerList/CrawlerList.pro
View File

@@ -28,7 +28,11 @@ SOURCES += main.cpp\
skakaousermanage.cpp \
sfacebooktagmanage.cpp \
sfacebookusermanage.cpp \
snaverblogaccuracymanager.cpp
snaverblogaccuracymanager.cpp \
stwittertagmanage.cpp \
stwitterusermanage.cpp \
syoutubetagmanage.cpp \
syoutubeusermanage.cpp
HEADERS += widget.h \
smanage.h \
@@ -45,5 +49,9 @@ HEADERS += widget.h \
skakaousermanage.h \
sfacebooktagmanage.h \
sfacebookusermanage.h \
snaverblogaccuracymanage.h
snaverblogaccuracymanage.h \
stwittertagmanage.h \
stwitterusermanage.h \
syoutubetagmanage.h \
syoutubeusermanage.h

89
CrawlerList/stwittertagmanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "stwittertagmanage.h"
#include <QThread>
#include "widget.h"
STwitterTagManage::STwitterTagManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void STwitterTagManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool STwitterTagManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void STwitterTagManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void STwitterTagManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void STwitterTagManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

30
CrawlerList/stwittertagmanage.h Normal file
View File

@@ -0,0 +1,30 @@
#ifndef STWITTERTAGMANAGE_H
#define STWITTERTAGMANAGE_H
#include "smanage.h"
class STwitterTagManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
STwitterTagManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // STWITTERTAGMANAGE_H

89
CrawlerList/stwitterusermanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "stwitterusermanage.h"
#include <QThread>
#include "widget.h"
STwitterUserManage::STwitterUserManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void STwitterUserManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool STwitterUserManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "twitter" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void STwitterUserManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void STwitterUserManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void STwitterUserManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

30
CrawlerList/stwitterusermanage.h Normal file
View File

@@ -0,0 +1,30 @@
#ifndef STWITTERUSERMANAGE_H
#define STWITTERUSERMANAGE_H
#include "smanage.h"
class STwitterUserManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
STwitterUserManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // STWITTERUSERMANAGE_H

89
CrawlerList/syoutubetagmanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "syoutubetagmanage.h"
#include <QThread>
#include "widget.h"
SYoutubeTagManage::SYoutubeTagManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void SYoutubeTagManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool SYoutubeTagManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void SYoutubeTagManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void SYoutubeTagManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void SYoutubeTagManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

30
CrawlerList/syoutubetagmanage.h Normal file
View File

@@ -0,0 +1,30 @@
#ifndef STYOUTUBETAGMANAGE_H
#define STYOUTUBETAGMANAGE_H
#include "smanage.h"
class SYoutubeTagManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
SYoutubeTagManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // STYOUTUBETAGMANAGE_H

89
CrawlerList/syoutubeusermanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "syoutubeusermanage.h"
#include <QThread>
#include "widget.h"
SYoutubeUserManage::SYoutubeUserManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void SYoutubeUserManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool SYoutubeUserManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "youtube" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void SYoutubeUserManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void SYoutubeUserManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void SYoutubeUserManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

30
CrawlerList/syoutubeusermanage.h Normal file
View File

@@ -0,0 +1,30 @@
#ifndef YOUTUBE_USER_MANAGE_H
#define YOUTUBE_USER_MANAGE_H
#include "smanage.h"
class SYoutubeUserManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
SYoutubeUserManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // YOUTUBE_USER_MANAGE_H

12
CrawlerList/widget.cpp
View File

@@ -22,6 +22,10 @@
#include "sfacebooktagmanage.h"
#include "sfacebookusermanage.h"
#include "snaverblogaccuracymanage.h"
#include "stwittertagmanage.h"
#include "stwitterusermanage.h"
#include "syoutubetagmanage.h"
#include "syoutubeusermanage.h"
#include <QApplication>
#include <QLabel>
@@ -51,6 +55,10 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
m_pFacebookTag = new SFacebookTagManage(this);
m_pFacebookUser = new SFacebookUserManage(this);
m_pNaverBlogAccuracy = new SNaverBlogAccuracyManage(this);
m_pTwitterTag = new STwitterTagManage(this);
m_pTwitterUser = new STwitterUserManage(this);
m_pYoutubeTag = new SYoutubeTagManage(this);
m_pYoutubeUser = new SYoutubeUserManage(this);
m_pManage[0] = m_pNaverCafe;
m_pManage[1] = m_pNaverBlog;
@@ -66,6 +74,10 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
m_pManage[11] = m_pFacebookTag;
m_pManage[12] = m_pFacebookUser;
m_pManage[13] = m_pNaverBlogAccuracy;
m_pManage[14] = m_pTwitterTag;
m_pManage[15] = m_pTwitterUser;
m_pManage[16] = m_pYoutubeTag;
m_pManage[17] = m_pYoutubeUser;
m_db = QSqlDatabase::addDatabase("QMYSQL");
m_db.setHostName("bigbird.iptime.org");

11
CrawlerList/widget.h
View File

@@ -26,6 +26,10 @@ class SInstaUserManage;
class SFacebookTagManage;
class SFacebookUserManage;
class SNaverBlogAccuracyManage;
class STwitterTagManage;
class STwitterUserManage;
class SYoutubeTagManage;
class SYoutubeUserManage;
#define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; }
@@ -55,7 +59,7 @@ private:
QLineEdit *m_pedStartDay;
QTimer m_timer,m_timerAlive;
QSqlDatabase m_db;
static const int C_PLATFORM_MAX = 14;
static const int C_PLATFORM_MAX = 18;
SManage *m_pManage[C_PLATFORM_MAX];
QListWidget *m_pResultList;
QString m_strFileName;
@@ -75,6 +79,11 @@ private:
SFacebookTagManage *m_pFacebookTag;
SFacebookUserManage *m_pFacebookUser;
SNaverBlogAccuracyManage *m_pNaverBlogAccuracy;
STwitterTagManage* m_pTwitterTag;
STwitterUserManage* m_pTwitterUser;
SYoutubeTagManage* m_pYoutubeTag;
SYoutubeUserManage* m_pYoutubeUser;
int m_nStartTime,m_nRangeTime,m_nPlatform;
//QGroupBox *m_pgbManual;
QCheckBox *m_pcheckboxReal;