clients/CrawlerList/snavercafemanage.cpp

#include "snavercafemanage.h"
#include "widget.h"
#include <QSqlQuery>
#include <QSqlError>
#include <QDebug>
#include <QFile>
#include <QNetworkProxy>

SNaverCafeManage::SNaverCafeManage(QObject *pObject) : SManage(pObject)
{
    m_nID = 0;
}

QString SNaverCafeManage::makeGetListQuery(QString _str,QDate _date,int _nPage)
{
    QString strOut;
    //strOut = "http://cafeblog.search.naver.com/search.naver?where=article&ie=utf8&query=";
    strOut = "https://search.naver.com/search.naver?where=article&ie=utf8&query=";
    strOut += EncodetoUtf8(_str,true);
    strOut += "&t=0&st=date&date_option=6&date_from=";
    strOut += _date.toString("yyyy.MM.dd");
    strOut += "&date_to=";
    strOut += _date.toString("yyyy.MM.dd");
    //strOut += "&srchby=text&dup_remove=1&cafe_url=&without_cafe_url=&board=&sm=tab_pge&nso=so%3Add%2Cp%3Afrom";
    strOut += "&srchby=text&dup_remove=1&";
    strOut += "cafe_url=";
    strOut += m_strAuthorship;
    strOut += "&without_cafe_url=&board=&sm=tab_pge&nso=so:dd,p:from";
    strOut += _date.toString("yyyyMMdd");
    strOut += "to";
    strOut += _date.toString("yyyyMMdd");
    //strOut += "%2Ca%3Aall&start=" + QString::number(_nPage);
    strOut += ",a:all&start=" + QString::number(_nPage);
    //qDebug() << strOut;
    return strOut;
}

void SNaverCafeManage::Start()
{    
    m_nMode = E_PROCESS_LIST_RUN;
    m_bFinalLast = false;
}

bool SNaverCafeManage::Update()
{
    if (m_bFinalLast) return m_bFinalLast;

    switch(m_nMode)
    {
    case E_PROCESS_LIST_RUN:
        if (UseProcess() == false)
        {
            m_strListQuery = makeGetListQuery(m_strKeyword,m_date,m_ncList);
            m_pMain->InsertLog("Start : " + QString::number(m_ncList) + "   Date : " +  m_date.toString("yyyy-MM-dd"));
            {
#if defined(Q_OS_WIN32)
                m_pro[0].start("CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#else
                m_pro[0].start("./CrawlerProcess",QStringList()<< "naver" << "cafe_list" << m_strListQuery << m_strGroupID << m_strKeywordID);
#endif
                m_ncList+=10;
            }
            m_nMode = E_PROCESS_LIST_FINISH_WAIT;
            m_nWait = 0;
        }
        break;
    case E_PROCESS_URL_RUN:
        if (UseProcess() == false)
        {
            m_pMain->InsertLog("(" + QString::number(m_ncUrl+1) + "/" + QString::number(m_strListURL.size()) + ")");
            {
#if defined(Q_OS_WIN32)
                m_pro[0].start("CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
#else
                m_pro[0].start("./CrawlerProcess",QStringList() << "naver" << "cafe_data" << m_strListURL.at(m_ncUrl) << m_strGroupID << m_strListQuery << m_strKeywordID );
#endif
                m_ncUrl++;
            }
            m_nMode = E_PROCESS_URL_FINISH_WAIT;
            m_nWait = 0;
        }
        break;
    case E_PROCESS_LIST_FINISH_WAIT:
    case E_PROCESS_URL_FINISH_WAIT:
        m_nWait++;
        if (m_nWait >= 300)
        {
            //for(int i = 0; i < C_PROCESS_MAX ; i++)
            {
               // if (m_pro[0].state() == QProcess::Running)
                {
                    m_pMain->InsertLog("Kill Process.");
                    m_pro[0].kill();
                }
            }
            ReLoadList();
        }
        break;
    }
    return m_bFinalLast;
}

void SNaverCafeManage::processFinished(QProcess *_pPro,QString _strOut)
{    
    switch(m_nMode)
    {
    case E_PROCESS_LIST_FINISH_WAIT:
        {
            m_bLast = false;
            bool reloaded = false;

            if (_strOut.right(4) == "last" || m_ncList >= 1000)
                m_bLast = true;

            if (m_bLast == false && m_nUntilPage > 0)
            {
                if ((m_ncList/10) >= m_nUntilPage)
                    m_bLast = true;
            }

            if (_strOut.right(5) == "block")
            {
                reloaded = true;
                ReLoadList();
            }
            if(_strOut.right(7) == "loading")
            {
                reloaded = true;
                ReLoadList();
            }

            QStringList strOutList = _strOut.split("\n");

            if(strOutList.length() > 2)
            {
                if(_strOut.split("\n").at(2).trimmed().length() == 0)
                {
                    if(reloaded == false)
                    {
                        ReLoadList();
                    }
                }
            }
            else
            {
                if(reloaded == false)
                {
                    ReLoadList();
                }
            }

            m_strListURL.clear();
            foreach(QString str,_strOut.split("\n"))
            {
                if (str.isEmpty()) continue;
                if (str.at(0) == QChar('o'))
                    m_strListURL.push_back(str.right(str.length()-2).trimmed());
            }
            m_ncUrl = 0;
            if (m_strListURL.size() == 0)
            {
                m_nMode = E_PROCESS_LIST_RUN;
                CheckLast();
                //if(m_bLast == false)
                    //ReLoadList();
            }
            else
                m_nMode = E_PROCESS_URL_RUN;
            break;
        }
    case E_PROCESS_URL_FINISH_WAIT:
        m_nMode = E_PROCESS_LIST_RUN;
        if (m_ncUrl >= m_strListURL.size())
        {
            m_nMode = E_PROCESS_LIST_RUN;
            CheckLast();
            m_bLast = false;
        }
        else
            m_nMode = E_PROCESS_URL_RUN;
        break;
    }    
}

void SNaverCafeManage::ReLoadList()
{
    m_nMode = E_PROCESS_LIST_RUN;
    m_ncList -= 10;
    if(m_ncList < 0)
        m_ncList = 1;
    m_pMain->InsertLog("Reload List");
}