#include "sdaumcafemanage.h" #include "widget.h" #include #include #include #include #include SDaumCafeManage::SDaumCafeManage(QObject *pObject) : SManage(pObject) { m_nID = 0; } QString SDaumCafeManage::makeGetListQuery(QString _str,QDate _date,int _nPage) { //http://search.daum.net/search?w=cafe&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&sd=20140501000000&ed=20140502235959&period=u&page=1&DA=PGD //http://search.daum.net/search?w=cafe&DA=PGD&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&sd=20140501000000&ed=20140501235959&period=u //http://search.daum.net/search?w=cafe&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&period=u&sd=20140506000000&ed=20140506235959&page=1&DA=PGD QString strOut; strOut = "http://search.daum.net/search?w=cafe&m=board&q="; strOut += EncodetoUtf8(_str,true); strOut += "&find=off&sort=recency&p="; strOut += QString::number(_nPage); strOut += "&period=u&sd="; strOut += _date.toString("yyyyMMdd"); strOut += "000000"; strOut += "&ed="; strOut += _date.toString("yyyyMMdd"); strOut += "235959"; strOut += "&page=1&DA=PGD"; //InsertLog("DaumCafe : URL"); //InsertLog(strOut); qDebug() << strOut; return strOut; } void SDaumCafeManage::Start() { m_nMode = E_PROCESS_LIST_RUN; m_bFinalLast = false; } bool SDaumCafeManage::Update() { if (m_bFinalLast) return m_bFinalLast; switch(m_nMode) { case E_PROCESS_LIST_RUN: if (UseProcess() == false) { m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList); m_pMain->InsertLog("Start : " + QString::number(m_ncList) + " Date : " + m_date.toString("yyyy-MM-dd")); { #if defined(Q_OS_WIN32) m_pro[0].start("CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID); #else m_pro[0].start("./CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID); #endif m_ncList+=1; } m_nMode = E_PROCESS_LIST_FINISH_WAIT; m_nWait = 0; } break; case E_PROCESS_URL_RUN: if (UseProcess() == false) { m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")"); { #if defined(Q_OS_WIN32) m_pro[0].start("AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); #else m_pro[0].start("./AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID ); #endif //m_pro[0].start("CrawlerProcess",QStringList() << "daum" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID ); m_ncUrl++; } m_nMode = E_PROCESS_URL_FINISH_WAIT; m_nWait = 0; } break; case E_PROCESS_LIST_FINISH_WAIT: case E_PROCESS_URL_FINISH_WAIT: m_nWait++; if (m_nWait >= 180) { //for(int i = 0; i < C_PROCESS_MAX ; i++) { // if (m_pro[0].state() == QProcess::Running) { m_pMain->InsertLog("Kill Process."); m_pro[0].kill(); } } ReLoadList(); //return m_bFinalLast; } break; } return m_bFinalLast; } void SDaumCafeManage::processFinished(QProcess *_pPro,QString _strOut) { switch(m_nMode) { case E_PROCESS_LIST_FINISH_WAIT: { m_bLast = false; bool reloaded = false; if (_strOut.right(4) == "last" || m_ncList >= 1000) m_bLast = true; if (m_bLast == false && m_nUntilPage > 0) { if (m_ncList >= m_nUntilPage) m_bLast = true; } // if (_strOut.right(5) == "block") // m_bFinalLast = true; if (_strOut.right(5) == "block") { reloaded = true; ReLoadList(); } if(_strOut.right(7) == "loading") { reloaded = true; ReLoadList(); } QStringList strOutList = _strOut.split("\n"); if(strOutList.length() > 2) { if(_strOut.split("\n").at(2).trimmed().length() == 0) { if(reloaded == false) { ReLoadList(); } } } else { if(reloaded == false) { ReLoadList(); } } m_strListURL.clear(); foreach(QString str,_strOut.split("\n")) { if (str.isEmpty()) continue; if (str.at(0) == QChar('o')) m_strListURL.push_back(str.right(str.length()-2).trimmed()); } m_ncUrl = 0; if (m_strListURL.size() == 0) { m_nMode = E_PROCESS_LIST_RUN; CheckLast(); //if(m_bLast == false) //ReLoadList(); } else m_nMode = E_PROCESS_URL_RUN; break; } case E_PROCESS_URL_FINISH_WAIT: m_nMode = E_PROCESS_LIST_RUN; if (m_ncUrl >= m_strListURL.size()) { m_nMode = E_PROCESS_LIST_RUN; CheckLast(); m_bLast = false; } else m_nMode = E_PROCESS_URL_RUN; break; } } void SDaumCafeManage::ReLoadList() { m_nMode = E_PROCESS_LIST_RUN; m_ncList -= 1; if(m_ncList < 1) m_ncList = 1; m_pMain->InsertLog("Reload List"); }