clients/CrawlerList/sdaumcafemanage.cpp

#include "sdaumcafemanage.h"
#include "widget.h"
#include <QSqlQuery>
#include <QSqlError>
#include <QDebug>
#include <QFile>
#include <QNetworkProxy>

SDaumCafeManage::SDaumCafeManage(QObject *pObject) : SManage(pObject)
{
    m_nID = 0;
}

QString SDaumCafeManage::makeGetListQuery(QString _str,QDate _date,int _nPage)
{
    //http://search.daum.net/search?w=cafe&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&sd=20140501000000&ed=20140502235959&period=u&page=1&DA=PGD
    //http://search.daum.net/search?w=cafe&DA=PGD&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&sd=20140501000000&ed=20140501235959&period=u
    //http://search.daum.net/search?w=cafe&m=board&q=%EC%84%B1%ED%98%95&find=off&sort=recency&p=3&period=u&sd=20140506000000&ed=20140506235959&page=1&DA=PGD
    QString strOut;
    strOut = "http://search.daum.net/search?w=cafe&m=board&q=";
    strOut += EncodetoUtf8(_str,true);
    strOut += "&find=off&sort=recency&p=";
    strOut += QString::number(_nPage);
    strOut += "&period=u&sd=";
    strOut += _date.toString("yyyyMMdd");
    strOut += "000000";
    strOut += "&ed=";
    strOut += _date.toString("yyyyMMdd");
    strOut += "235959";
    strOut += "&page=1&DA=PGD";

    //InsertLog("DaumCafe : URL");
    //InsertLog(strOut);
    qDebug() << strOut;
    return strOut;
}

void SDaumCafeManage::Start()
{
    m_nMode = E_PROCESS_LIST_RUN;
    m_bFinalLast = false;
}

bool SDaumCafeManage::Update()
{
    if (m_bFinalLast) return m_bFinalLast;

    switch(m_nMode)
    {
    case E_PROCESS_LIST_RUN:
        if (UseProcess() == false)
        {
            m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
            m_pMain->InsertLog("Start : " + QString::number(m_ncList) + "   Date : " +  m_date.toString("yyyy-MM-dd"));
            {
#if defined(Q_OS_WIN32)
                m_pro[0].start("CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#else
                m_pro[0].start("./CrawlerProcess",QStringList()<< "daum" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#endif
                m_ncList+=1;
            }
            m_nMode = E_PROCESS_LIST_FINISH_WAIT;
            m_nWait = 0;
        }
        break;
    case E_PROCESS_URL_RUN:
        if (UseProcess() == false)
        {
            m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
            {
#if defined(Q_OS_WIN32)
                m_pro[0].start("AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#else
                m_pro[0].start("./AjaxCrawlerProcess",QStringList() << "daum_cafe" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strKeywordID );
#endif
                //m_pro[0].start("CrawlerProcess",QStringList() << "daum" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
                m_ncUrl++;
            }
            m_nMode = E_PROCESS_URL_FINISH_WAIT;
            m_nWait = 0;
        }
        break;
    case E_PROCESS_LIST_FINISH_WAIT:
    case E_PROCESS_URL_FINISH_WAIT:
        m_nWait++;
        if (m_nWait >= 180)
        {
            //for(int i = 0; i < C_PROCESS_MAX ; i++)
            {
               // if (m_pro[0].state() == QProcess::Running)
                {
                    m_pMain->InsertLog("Kill Process.");
                    m_pro[0].kill();
                }
            }
            ReLoadList(); //return m_bFinalLast;
        }
        break;
    }
    return m_bFinalLast;
}

void SDaumCafeManage::processFinished(QProcess *_pPro,QString _strOut)
{
    switch(m_nMode)
    {
    case E_PROCESS_LIST_FINISH_WAIT:
        {
            m_bLast = false;
            bool reloaded = false;
            if (_strOut.right(4) == "last" || m_ncList >= 1000)
                m_bLast = true;

            if (m_bLast == false && m_nUntilPage > 0)
            {
                if (m_ncList >= m_nUntilPage)
                    m_bLast = true;
            }

 //           if (_strOut.right(5) == "block")
 //               m_bFinalLast = true;

            if (_strOut.right(5) == "block")
            {
               reloaded = true;
               ReLoadList();
            }

            if(_strOut.right(7) == "loading")
            {
                reloaded = true;
                ReLoadList();
            }


            QStringList strOutList = _strOut.split("\n");

            if(strOutList.length() > 2)
            {
                if(_strOut.split("\n").at(2).trimmed().length() == 0)
                {
                    if(reloaded == false)
                    {
                        ReLoadList();
                    }
                }
            }
            else
            {
                if(reloaded == false)
                {
                    ReLoadList();
                }
            }

            m_strListURL.clear();
            foreach(QString str,_strOut.split("\n"))
            {
                if (str.isEmpty()) continue;
                if (str.at(0) == QChar('o'))
                    m_strListURL.push_back(str.right(str.length()-2).trimmed());
            }
            m_ncUrl = 0;
            if (m_strListURL.size() == 0)
            {
                m_nMode = E_PROCESS_LIST_RUN;
                CheckLast();
                //if(m_bLast == false)
                    //ReLoadList();
            }
            else
                m_nMode = E_PROCESS_URL_RUN;
            break;
        }
    case E_PROCESS_URL_FINISH_WAIT:
        m_nMode = E_PROCESS_LIST_RUN;
        if (m_ncUrl >= m_strListURL.size())
        {
            m_nMode = E_PROCESS_LIST_RUN;
            CheckLast();
            m_bLast = false;
        }
        else
            m_nMode = E_PROCESS_URL_RUN;
        break;
    }
}

void SDaumCafeManage::ReLoadList()
{
    m_nMode = E_PROCESS_LIST_RUN;
    m_ncList -= 1;
    if(m_ncList < 1)
        m_ncList = 1;
    m_pMain->InsertLog("Reload List");

}