#include "widget.h" #include #include #include #include #include #include #include #include #include #include "snavercafemanage.h" #include "snaverblogmanage.h" #include "sdaumcafemanage.h" #include "snavernewsmanage.h" #include "snavercafelistmanager.h" #include "sdaumcafelistmanage.h" #include "skakaochannelmanage.h" #include "skakaotagmanage.h" #include "skakaousermanage.h" #include "sinstatagmanage.h" #include "sinstausermanage.h" #include "sfacebooktagmanage.h" #include "sfacebookusermanage.h" #include "snaverblogaccuracymanage.h" #include "stwittertagmanage.h" #include "stwitterusermanage.h" #include "syoutubetagmanage.h" #include "syoutubeusermanage.h" #include #include Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) { connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update())); connect(&m_timerAlive, SIGNAL(timeout()), this, SLOT(Alive())); QVBoxLayout *vlayout = new QVBoxLayout; { m_pResultList = new QListWidget; vlayout->addWidget(setRealGroupWidgets()); vlayout->addWidget(m_pResultList); } setLayout(vlayout); m_pNaverCafe = new SNaverCafeManage(this); m_pNaverBlog = new SNaverBlogManage(this); m_pDaumCafe = new SDaumCafeManage(this); m_pNaverNews = new SNaverNewsManage(this); m_pNaverCafeList = new SNaverCafeListManage(this); m_pDaumCafeList = new SDaumCafeListManage(this); m_pKakaoChannel = new SKakaoChannelManage(this); m_pKakaoTag = new SKakaoTagManage(this); m_pKakaoUser = new SKakaoUserManage(this); m_pInstaTag = new SInstaTagManage(this); m_pInstaUser = new SInstaUserManage(this); m_pFacebookTag = new SFacebookTagManage(this); m_pFacebookUser = new SFacebookUserManage(this); m_pNaverBlogAccuracy = new SNaverBlogAccuracyManage(this); m_pTwitterTag = new STwitterTagManage(this); m_pTwitterUser = new STwitterUserManage(this); m_pYoutubeTag = new SYoutubeTagManage(this); m_pYoutubeUser = new SYoutubeUserManage(this); m_pManage[0] = m_pNaverCafe; m_pManage[1] = m_pNaverBlog; m_pManage[2] = m_pDaumCafe; m_pManage[3] = m_pNaverNews; m_pManage[4] = m_pNaverCafeList; m_pManage[5] = m_pDaumCafeList; m_pManage[6] = m_pKakaoChannel; m_pManage[7] = m_pKakaoTag; m_pManage[8] = m_pKakaoUser; m_pManage[9] = m_pInstaTag; m_pManage[10] = m_pInstaUser; m_pManage[11] = m_pFacebookTag; m_pManage[12] = m_pFacebookUser; m_pManage[13] = m_pNaverBlogAccuracy; m_pManage[14] = m_pTwitterTag; m_pManage[15] = m_pTwitterUser; m_pManage[16] = m_pYoutubeTag; m_pManage[17] = m_pYoutubeUser; m_db = QSqlDatabase::addDatabase("QMYSQL"); m_db.setHostName("bigbird.iptime.org"); m_db.setUserName("admin"); m_db.setPassword("admin123"); m_db.setDatabaseName("concepters"); //m_db.setConnectOptions("MYSQL_OPT_RECONNECT=1"); if (!m_db.open()) { InsertLog("DB Open Failed"); return; } for (int i = 0 ; i < C_PLATFORM_MAX ; i++) m_pManage[i]->SetParent(this); setWindowTitle("CrawlerList " + QString::number(QCoreApplication::applicationPid())); RefreshButton(); m_timer.start(1000); m_timerAlive.start(60*60000); } Widget::~Widget() { if (m_nMode == E_MODE_RUN) { if (m_pcheckboxReal->isChecked() == false) { if (m_strCrawlingID.isEmpty() == false) SetCrawlingState("Finish"); } else { if (m_strCrawlingID.isEmpty() == false) SetCrawlingState("RealTime_Exit"); } } m_db.close(); for (int i = 0 ; i < C_PLATFORM_MAX ; i++) m_pManage[i]->SetParent(0); } QGroupBox *Widget::setRealGroupWidgets() { m_pcb = new QComboBox; m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent); QPushButton *pbtRefresh = new QPushButton("Refresh",this); QPushButton *pbtStart = new QPushButton("Start",this); QPushButton *pbtStop = new QPushButton("Stop",this); { QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton())); QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton())); QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton())); } m_pcheckboxReal = new QCheckBox("Real Time",this); QObject::connect(m_pcheckboxReal,SIGNAL(clicked()),this,SLOT(RefreshButton())); QVBoxLayout *vlayout = new QVBoxLayout; vlayout->addWidget(m_pcheckboxReal); { QHBoxLayout *hlayout = new QHBoxLayout; hlayout->addWidget(m_pcb,Qt::AlignLeft); hlayout->addWidget(pbtRefresh); vlayout->addLayout(hlayout); } m_pedUntilPage = new QLineEdit(this); m_pedUntilPage->setText(QString("0")); m_pedStartDay = new QLineEdit(this); m_pedStartDay->setText(QString("-1")); { QHBoxLayout *hlayout = new QHBoxLayout; hlayout->addWidget(new QLabel("Until Page:", this)); hlayout->addWidget(m_pedUntilPage); hlayout->addWidget(new QLabel("Start Day:", this)); hlayout->addWidget(m_pedStartDay); hlayout->addWidget(pbtStart); hlayout->addWidget(pbtStop); vlayout->addLayout(hlayout); } QGroupBox *pGroup = new QGroupBox(); pGroup->setLayout(vlayout); return pGroup; } void Widget::InsertLog(QString str) { QTime time = QTime::currentTime(); QString strOut = time.toString("[hh:mm:ss] ") + str; if(strOut.length() <= 1024) m_pResultList->addItem(strOut); else m_pResultList->addItem(strOut.left(1024)); QDate date = QDate::currentDate(); QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log"); if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) return; QTextStream out(&file); out << strOut << "\n"; file.close(); if (m_pResultList->count() > 1024) { for(int i=0; i<256; i++) { m_pResultList->removeItemWidget(m_pResultList->item(0)); QListWidgetItem* item = m_pResultList->takeItem(0); delete item; } } m_pResultList->setCurrentRow( m_pResultList->count() - 1 ); m_pResultList->repaint(); } void Widget::Debug(QString _strMsg) { QTime time = QTime::currentTime(); QString strOut = time.toString("[hh:mm:ss] ") + _strMsg; QFile file(QDate::currentDate().toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".debug.txt"); if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) return; QTextStream out(&file); out << strOut << "\n"; file.close(); } bool Widget::SetCrawlingState(QString _str) { QSqlQuery query; if (_str == QString("RealTime_Exit") || _str == QString("Stop")) query.exec("UPDATE crawling set state = 0 where id = '" + m_strCrawlingID + "'"); if (_str == QString("Finish")) { query.exec("UPDATE crawling set state = 2 where id = '" + m_strCrawlingID + "'"); m_nMode = E_MODE_WAIT; if (m_pcheckboxReal->isChecked() == false) m_strCrawlingID.clear(); } if (_str == QString("Start")) query.exec("UPDATE crawling set state = 1 where id = '" + m_strCrawlingID + "'"); return true; } void Widget:: Start() { if(!m_db.isOpen()) { m_db.close(); if(!m_db.open()) { InsertLog("DB is down, retry after a while"); return; } else { InsertLog("Reconnect to DB"); } } QSqlQuery query; query.exec("SELECT keyword.start,keyword.end, keyword.searches,keyword.authorship,keyword.id,datagroup.id,keyword.platform " "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id " "inner join datagroup on crawling.datagroup_id = datagroup.id " "where crawling.id = '" + m_strCrawlingID + "'"); if (query.next() == false) { /* InsertLog(query.lastError().text()); InsertLog(query.lastQuery()); */ int err = query.lastError().number(); if (err == 2006 || err == 2013) // Might want to do #2013 here also? { m_db.close(); if (!m_db.open()) InsertLog("DB is down, retry after a while"); else InsertLog("Reconnect to DB"); } return; } qDebug() << m_strCrawlingID; m_nPlatform = 0; if(0 <= query.value(6).toInt() && query.value(6).toInt() < C_PLATFORM_MAX) m_nPlatform = query.value(6).toInt(); else { InsertLog("Starting Crawler is failed"); return; } if (m_pcheckboxReal->isChecked()) { m_pManage[m_nPlatform]->Start(QDate::currentDate().addDays(m_pedStartDay->text().trimmed().toInt()),QDate::currentDate(), query.value(2).toString().trimmed(),// keyword query.value(3).toString().trimmed(),// authorship query.value(4).toString().trimmed(),// keyword_id query.value(5).toString().trimmed(), 1, m_pedUntilPage->text().trimmed().toInt()); } else { m_pManage[m_nPlatform]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"), QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"), query.value(2).toString().trimmed(),// keyword query.value(3).toString().trimmed(),// authorship query.value(4).toString().trimmed(),// keyword_id query.value(5).toString().trimmed(), 1, m_pedUntilPage->text().trimmed().toInt()); } SetCrawlingState("Start"); m_nMode = E_MODE_RUN; } void Widget::StartButton() { m_strCrawlingID = m_pcb->currentData().toString(); Start(); } void Widget::StopButton() { m_nMode = E_MODE_WAIT; SetCrawlingState("Stop"); m_strCrawlingID.clear(); //qDebug() << m_nPlatform; if(m_nPlatform < 4 || m_nPlatform == 13) return; m_pManage[m_nPlatform]->clossProcess(); } void Widget::Update() { switch(m_nMode) { case E_MODE_WAIT: //qDebug() << "MODE : WAIT"; if (m_pcheckboxReal->isChecked()) { if (m_strCrawlingID.isEmpty() == false) { //qDebug() << "Start"; Start(); } } break; case E_MODE_RUN: //qDebug() << "MODE : RUN"; if(m_pManage[m_nPlatform]->Update()==true) m_nMode = E_MODE_WAIT; break; } } void Widget::Alive() { QSqlQuery query; query.exec("SELECT 1"); query.next(); } void Widget::RefreshButton() { m_pcb->clear(); QSqlQuery query("SELECT keyword.searches,keyword.authorship,keyword.start,keyword.end,datagroup.name,keyword.platform,crawling.state,crawling.id " "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id " "inner join datagroup on crawling.datagroup_id = datagroup.id where crawling.state = 0 and keyword.realtime = " + QString::number(m_pcheckboxReal->isChecked())); while (query.next()) { QString str; str += query.value(0).toString() + " , "; str += query.value(1).toString() + " , "; str += query.value(2).toString() + " ~ "; str += query.value(3).toString() + " -> "; str += query.value(4).toString(); switch(query.value(5).toString().toInt()) { case 0:str += ", Naver Cafe";break; case 1:str += ", Naver Blog";break; case 2:str += ", Daum Cafe"; break; case 3:str += ", Naver News"; break; case 4:str += ", Naver Cafe List"; break; case 5:str += ", Daum Cafe List"; break; case 6:str += ", Kakao Story Channel"; break; case 7:str += ", Kakao Story Tag"; break; case 8:str += ", Kakao Story User"; break; case 9:str += ", Instagram Tag"; break; case 10:str += ", Instagram User"; break; case 11:str += ", Facebook Tag"; break; case 12:str += ", Facebook User"; break; case 13:str += ", Naver Blog Accuracy"; break; case 14:str += ", Twitter Tag"; break; case 15:str += ", Twitter User"; break; case 16:str += ", Youtube Tag"; break; case 17:str += ", Youtube User"; break; } m_pcb->addItem(str,query.value(7)); } } QString Widget::StartDay() { return m_pedStartDay->text().trimmed(); } QString Widget::UntilPage() { return m_pedUntilPage->text().trimmed(); }