etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

facebook 추가

Browse Source 
timeout시 재접속 할수 있도록 수정

git-svn-id: svn://192.168.0.12/source@239 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2016-01-19 06:47:32 +00:00
parent ef669df7ad
commit f25230c652
7 changed files with 297 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
CrawlerList/CrawlerList.pro
View File

@@ -25,7 +25,9 @@ SOURCES += main.cpp\
skakaotagmanage.cpp \
sinstausermanage.cpp \
sinstatagmanage.cpp \
skakaousermanage.cpp
skakaousermanage.cpp \
sfacebooktagmanage.cpp \
sfacebookusermanage.cpp
HEADERS += widget.h \
smanage.h \
@@ -39,5 +41,7 @@ HEADERS += widget.h \
skakaotagmanage.h \
sinstausermanage.h \
sinstatagmanage.h \
skakaousermanage.h
skakaousermanage.h \
sfacebooktagmanage.h \
sfacebookusermanage.h

89
CrawlerList/sfacebooktagmanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "sfacebooktagmanage.h"
#include <QThread>
#include "widget.h"
SFacebookTagManage::SFacebookTagManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void SFacebookTagManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool SFacebookTagManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "facebook" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "facebook" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void SFacebookTagManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void SFacebookTagManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void SFacebookTagManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

32
CrawlerList/sfacebooktagmanage.h Normal file
View File

@@ -0,0 +1,32 @@
#ifndef SFACEBOOKTAGMANAGE
#define SFACEBOOKTAGMANAGE
#include "smanage.h"
class SFacebookTagManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
SFacebookTagManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // SFACEBOOKTAGMANAGE

89
CrawlerList/sfacebookusermanage.cpp Normal file
View File

@@ -0,0 +1,89 @@
#include "sfacebookusermanage.h"
#include <QThread>
#include "widget.h"
SFacebookUserManage::SFacebookUserManage(QObject *pObject) : SManage(pObject)
{
m_nID = 0;
connect(&m_pro[0], SIGNAL(readyReadStandardOutput()), this, SLOT(readStandardOutput()));
connect(&m_pro[0], SIGNAL(readyReadStandardError()), this, SLOT(readStandardError()));
}
void SFacebookUserManage::Start()
{
m_nMode = E_PROCESS_RUN;
m_bFinalLast = false;
}
bool SFacebookUserManage::Update()
{
if(m_bFinalLast) return m_bFinalLast;
switch(m_nMode)
{
case E_PROCESS_RUN:
if(UseProcess() == false)
{
#if defined(Q_OS_WIN32)
m_pro[0].start("python", QStringList() << "webbasedcrawler.py" << "facebook" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#else
m_pro[0].start("/usr/bin/python3", QStringList() << "webbasedcrawler.py" << "facebook" << m_strKeywordID << m_strGroupID << m_pMain->StartDay() << m_pMain->UntilPage());
#endif
m_nMode = E_PROCESS_FINISH_WAIT;
}
break;
case E_PROCESS_FINISH_WAIT:
break;
}
return m_bFinalLast;
}
void SFacebookUserManage::processFinished(QProcess *pPro, QString _strOut)
{
switch(m_nMode)
{
case E_PROCESS_FINISH_WAIT:
m_nMode = E_PROCESS_RUN;
m_bFinalLast = true;
m_pMain->InsertLog("Finish Crawling :)");
m_pMain->SetCrawlingState("Finish");
m_ncList=1;
m_bLast = false;
break;
}
}
void SFacebookUserManage::readStandardOutput()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}
void SFacebookUserManage::readStandardError()
{
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardError();
QStringList list = str.split("\n", QString::SkipEmptyParts);
foreach(QString log,list)
{
if (m_pMain)
{
m_pMain->InsertLog(log);
}
else
exit(0);
}
}

30
CrawlerList/sfacebookusermanage.h Normal file
View File

@@ -0,0 +1,30 @@
#ifndef SFACEBOOKUSERMANAGE
#define SFACEBOOKUSERMANAGE
#include "smanage.h"
class SFacebookUserManage : public SManage
{
Q_OBJECT
public:
enum E_PROCESS_STATE
{
E_PROCESS_RUN = 0,
E_PROCESS_FINISH_WAIT,
};
SFacebookUserManage(QObject *pObject);
private:
QString makeGetListQuery(QString _str,QDate _date,int _nPage);
private:
QString m_strListQuery;
QVector <QString> m_strListURL;
protected:
bool Update();
void Start();
void processFinished(QProcess *pPro,QString _strOut);
void ReLoadList();
private slots:
void readStandardOutput();
void readStandardError();
};
#endif // SFACEBOOKUSERMANAGE

52
CrawlerList/widget.cpp
View File

@@ -19,6 +19,8 @@
#include "skakaousermanage.h"
#include "sinstatagmanage.h"
#include "sinstausermanage.h"
#include "sfacebooktagmanage.h"
#include "sfacebookusermanage.h"
#include <QApplication>
#include <QLabel>
@@ -45,6 +47,8 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
m_pKakaoUser = new SKakaoUserManage(this);
m_pInstaTag = new SInstaTagManage(this);
m_pInstaUser = new SInstaUserManage(this);
m_pFacebookTag = new SFacebookTagManage(this);
m_pFacebookUser = new SFacebookUserManage(this);
m_pManage[0] = m_pNaverCafe;
m_pManage[1] = m_pNaverBlog;
@@ -57,6 +61,8 @@ Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT)
m_pManage[8] = m_pKakaoUser;
m_pManage[9] = m_pInstaTag;
m_pManage[10] = m_pInstaUser;
m_pManage[11] = m_pFacebookTag;
m_pManage[12] = m_pFacebookUser;
m_db = QSqlDatabase::addDatabase("QMYSQL");
m_db.setHostName("bigbird.iptime.org");
@@ -149,7 +155,10 @@ void Widget::InsertLog(QString str)
{
QTime time = QTime::currentTime();
QString strOut = time.toString("[hh:mm:ss] ") + str;
m_pResultList->addItem(strOut);
if(strOut.length() <= 1024)
m_pResultList->addItem(strOut);
else
m_pResultList->addItem(strOut.left(1024));
QDate date = QDate::currentDate();
QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
@@ -204,10 +213,18 @@ bool Widget::SetCrawlingState(QString _str)
void Widget::Start()
{
if((!m_db.isOpen()) && (!m_db.open()))
if(!m_db.isOpen())
{
InsertLog("DB is down, retry after a while");
return;
m_db.close();
if(!m_db.open())
{
InsertLog("DB is down, retry after a while");
return;
}
else
{
InsertLog("Reconnect to DB");
}
}
QSqlQuery query;
@@ -216,7 +233,25 @@ void Widget::Start()
"inner join datagroup on crawling.datagroup_id = datagroup.id "
"where crawling.id = '" + m_strCrawlingID + "'");
if (query.next() == false) return;
if (query.next() == false)
{
/*
InsertLog(query.lastError().text());
InsertLog(query.lastQuery());
*/
int err = query.lastError().number();
if (err == 2006 || err == 2013) // Might want to do #2013 here also?
{
m_db.close();
if (!m_db.open())
InsertLog("DB is down, retry after a while");
else
InsertLog("Reconnect to DB");
}
return;
}
qDebug() << m_strCrawlingID;
m_nPlatform = 0;
if(0 <= query.value(6).toInt() && query.value(6).toInt() < C_PLATFORM_MAX)
@@ -264,7 +299,7 @@ void Widget::StopButton()
SetCrawlingState("Stop");
m_strCrawlingID.clear();
//qDebug() << m_nPlatform;
if(4 <= m_nPlatform && m_nPlatform <= 10)
if(4 <= m_nPlatform && m_nPlatform <= 12)
{
m_pManage[m_nPlatform]->clossProcess();
}
@@ -275,15 +310,18 @@ void Widget::Update()
switch(m_nMode)
{
case E_MODE_WAIT:
qDebug() << "MODE : WAIT";
if (m_pcheckboxReal->isChecked())
{
if (m_strCrawlingID.isEmpty() == false)
{
qDebug() << "Start";
Start();
}
}
break;
case E_MODE_RUN:
qDebug() << "MODE : RUN";
if(m_pManage[m_nPlatform]->Update()==true)
m_nMode = E_MODE_WAIT;
break;
@@ -325,6 +363,8 @@ void Widget::RefreshButton()
case 8:str += ", Kakao Story User"; break;
case 9:str += ", Instagram Tag"; break;
case 10:str += ", Instagram User"; break;
case 11:str += ", Facebook Tag"; break;
case 12:str += ", Facebook User"; break;
}
m_pcb->addItem(str,query.value(7));
}

7
CrawlerList/widget.h
View File

@@ -23,7 +23,8 @@ class SKakaoTagManage;
class SKakaoUserManage;
class SInstaTagManage;
class SInstaUserManage;
class SFacebookTagManage;
class SFacebookUserManage;
#define SAFE_DELETE(p) {if(p) delete (p); (p) = NULL; }
@@ -54,7 +55,7 @@ private:
QLineEdit *m_pedStartDay;
QTimer m_timer,m_timerAlive;
QSqlDatabase m_db;
static const int C_PLATFORM_MAX = 11;
static const int C_PLATFORM_MAX = 13;
SManage *m_pManage[C_PLATFORM_MAX];
QListWidget *m_pResultList;
QString m_strFileName;
@@ -71,6 +72,8 @@ private:
SKakaoUserManage *m_pKakaoUser;
SInstaTagManage *m_pInstaTag;
SInstaUserManage *m_pInstaUser;
SFacebookTagManage *m_pFacebookTag;
SFacebookUserManage *m_pFacebookUser;
int m_nStartTime,m_nRangeTime,m_nPlatform;
//QGroupBox *m_pgbManual;