clients/ProxyUI/widget.cpp

#include "widget.h"
#include "ui_widget.h"
#include <QSet>
#include <QRegExp>
#define UPDATE_TIMER 43200000
#define CHECK_TIMER 180000
#define MIN_PROXY 200
#define CRAWLING_TIMER 10800000

Widget::Widget(QWidget *parent) :
    QWidget(parent),
    ui(new Ui::Widget)
{
    ui->setupUi(this);

    {
        m_index = 0;
        p_labelUrl = new QLabel(tr("URL: "));
        p_labelTime = new QLabel(tr("Update Time Period(sec): "));
        p_labelProxyFile = new QLabel(tr("Save Location: "));
        p_labelRecentTime = new QLabel("");
        p_labelNextTime = new QLabel("");

        p_labelRecent = new QLabel(tr("Recent Updated Time: "));
        p_labelNext = new QLabel("Next Update Time: ");

        p_lineUrl = new QLineEdit("http://proxylist.hidemyass.com/search-1305249#listable");
        p_lineTime = new QLineEdit("12");
        p_lineProxyFile = new QLineEdit(qApp->applicationDirPath());
        p_btnChoose = new QPushButton("Choose");

        p_FileDialog = new QFileDialog();
        glayout = new QGridLayout();

        p_lineUrl->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);
        p_lineTime->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);
        p_lineProxyFile->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);

        p_chkLocal = new QCheckBox("Local") ;
        p_chkDb = new QCheckBox("DB");
        p_chkLocal->setChecked(true);
        QLabel *blank = new QLabel("Update Location: ");


        glayout->addWidget(p_labelUrl, 0, 0);
        glayout->addWidget(p_lineUrl, 0, 1, 1, 3);
        glayout->addWidget(p_labelTime, 1, 0);
        glayout->addWidget(p_lineTime, 1, 1, 1, 3);
        glayout->addWidget(p_labelProxyFile, 2, 0);
        glayout->addWidget(p_lineProxyFile, 2, 1, 1, 2);
        glayout->addWidget(p_btnChoose, 2, 3);
        glayout->addWidget(p_chkLocal, 3, 1);
        glayout->addWidget(p_chkDb, 3, 2);


        glayout->addWidget(p_labelRecent , 4, 0);
        glayout->addWidget(p_labelRecentTime, 4, 1);
        glayout->addWidget(p_labelNext, 4, 2);
        glayout->addWidget(p_labelNextTime, 4, 3);
        glayout->addWidget(blank, 3, 0);

        connect(p_chkDb,SIGNAL(clicked()),this, SLOT(chkDb()));
        connect(p_chkLocal,SIGNAL(clicked()),this, SLOT(chkLocal()));


        /*
        for(int i = 0; i < 4; i++)
        {
            hlayoutParam[i] = new QHBoxLayout();
        }

        hlayoutParam[0]->addWidget(p_labelUrl);
        hlayoutParam[0]->addWidget(p_lineUrl);
        hlayoutParam[1]->addWidget(p_labelTime);
        hlayoutParam[1]->addWidget(p_lineTime);
        hlayoutParam[2]->addWidget(p_labelProxyFile);
        hlayoutParam[2]->addWidget(p_lineProxyFile);
        hlayoutParam[2]->addWidget(p_btnChoose);
        hlayoutParam[3]->addWidget(p_labelRecent);
        hlayoutParam[3]->addWidget(p_labelRecentTime);
        hlayoutParam[3]->addWidget(p_labelNext);
        hlayoutParam[3]->addWidget(p_labelNextTime);

        vlayoutParam = new QVBoxLayout();

        for(int i = 0; i < 4; i++)
        {
            vlayoutParam->addLayout(hlayoutParam[i]);
        }
        */
    }

    {
        p_btnStart = new QPushButton(tr("Timer Start"));
        p_btnStop = new QPushButton(tr("Timer Stop"));
        p_labelTimerOnOff = new QLabel(tr("Timer OFF"));
//      vlayoutButton = new QVBoxLayout();
        glayout->addWidget(p_btnStart,0, 4);
        glayout->addWidget(p_btnStop, 1, 4);
        glayout->addWidget(p_labelTimerOnOff, 2, 4);

        m_pResultList = new QListWidget;
        m_pCrawlingResultList = new QListWidget;


        //glayout->addWidget(m_pResultList, 5, 0, 1, 5);

        glayout->addWidget(m_pResultList, 5, 0, 1, 5);
        glayout->addWidget(m_pCrawlingResultList, 7, 0, 1, 5);


    }

    p_timer = new QTimer(this);
    p_checkTimer = new QTimer(this);
    p_crawlingTimer = new QTimer(this);
    connect(p_timer,SIGNAL(timeout()),this,SLOT(updateDB()));
    connect(p_checkTimer,SIGNAL(timeout()),this,SLOT(checkProxy()));
    connect(p_crawlingTimer, SIGNAL(timeout()), this, SLOT(crawlingProxy()));
    connect(p_btnStart, SIGNAL(clicked()), this, SLOT(btnStart()));
    connect(p_btnStop, SIGNAL(clicked()), this, SLOT(btnStop()));
    connect(p_btnChoose, SIGNAL(clicked()), this, SLOT(btnChoose()));
    connect(&v_pro,SIGNAL(finished(int,QProcess::ExitStatus)),this,SLOT(processFinished(int, QProcess::ExitStatus)));
//SIGNAL(finished(int,QProcess::ExitStatus)),SLOT(processFinished(int,QProcess::ExitStatus))
/*
    hlayoutDefault = new QHBoxLayout(this);
    hlayoutDefault->addLayout(vlayoutParam);
    hlayoutDefault->addLayout(glayout);
    hlayoutDefault->addLayout(vlayoutButton);
*/
    setLayout(glayout);
    resize(700, 500);
}

Widget::~Widget()
{
    delete ui;
}

void Widget::btnChoose()
{
    qDebug("btnChoose()");
    QString directory = QFileDialog::getExistingDirectory(this,
                               tr("Find Files"), QDir::currentPath());

    if (!directory.isEmpty()) {
        this->p_lineProxyFile->setText(directory);
    }
}

void Widget::btnStart()
{
    qDebug("btnStart()");
    if(p_timer->isActive() && p_checkTimer->isActive() && p_crawlingTimer->isActive())
        return;

    InsertLog("Proxy Update Executed");
    //updateProxy();
    crawlingProxy();

    if(!p_timer->isActive())
    {
        //p_timer->start(p_lineTime->text().toInt()*3600000);
        //p_timer->start(43200000);
        p_timer->start(UPDATE_TIMER);
        InsertLog("Update Timer Start");
    }
    if(!p_checkTimer->isActive())
    {
        //p_checkTimer->start(1200000);
        p_checkTimer->start(CHECK_TIMER);
        InsertLog("Proxy DB Check Timer Start");
    }
    if(!p_crawlingTimer->isActive())
    {
        p_crawlingTimer->start(CRAWLING_TIMER);
        InsertCrawlingLog("Crawling Timer Start");
    }

    p_labelTimerOnOff->setText("Timer ON");
}

void Widget::btnStop()
{
    qDebug("btnStop()");

    if(p_timer->isActive())
    {
        p_timer->stop();
        InsertLog("Update Timer Stop");
    }
    if(p_checkTimer->isActive())
    {
        p_checkTimer->stop();
        InsertLog("Proxy DB Check Timer Timer Stop");
    }
    if(p_crawlingTimer->isActive())
    {
        p_crawlingTimer->stop();
        InsertCrawlingLog("CrawlerTimer stop");
    }
    p_labelTimerOnOff->setText("Timer OFF");
    p_labelNextTime->setText("");
    m_index = 0;
}

void Widget::update()
{
    InsertCrawlingLog("Update Executed");
    if(p_crawlingTimer->isActive())
    {
        p_crawlingTimer->stop();
        InsertCrawlingLog("Crawling Timer Stop");
    }
    InsertCrawlingLog(m_slIpUrl.at(m_index));
#if defined(Q_OS_WIN32)
    v_pro.start("ProxyProcess",QStringList() << m_slIpUrl.at(m_index) << p_lineProxyFile->text());
#else
    v_pro.start("./ProxyProcess",QStringList() << m_slIpUrl.at(m_index) << p_lineProxyFile->text());
#endif
    //p_labelStatus->setText("Updating");
}

void Widget::processFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
    //qDebug("process finished");
    QProcess *pPro = (QProcess*)sender();
    QString str = pPro->readAllStandardOutput();

    if(str.right(2).compare("ok") == 0)
    {
        m_slIpPortListTemp << str.split("\n");
        m_slIpPortListTemp.removeAt(m_slIpPortListTemp.size() - 1);
        InsertCrawlingLog("Successfully Get Proxy : " + QString::number(str.split("\n").length() - 1));
        if(m_slIpPortListTemp.length() > 1)
        {
            mutex.lock();
            m_slIpPortList << m_slIpPortListTemp;
            m_slIpPortList.removeDuplicates();
            mutex.unlock();
            InsertCrawlingLog("Number of IP : " + QString::number(m_slIpPortList.length()));
        }
        /*
        QStringList slIpPortList = str.split("\n");
        slIpPortList.removeAt(slIpPortList.size() - 1);
        QString site;
        if(slIpPortList.length() > 0)
        {
            QStringList sltemp = slIpPortList.at(0).split(",");
            if(sltemp.length() >= 2)
                site = sltemp.at(2).trimmed();
        }

        if(site.isEmpty())
        {
        }
        else
        {
            QRegExp re(QString("\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3},\\d{1,3},(?!" + site + ")"));
            mutex.lock();
            QStringList temp = m_slIpPortList.filter(re);
            m_slIpPortList.clear();
            m_slIpPortList = temp;
            m_slIpPortList << slIpPortList;
            InsertCrawlingLog("Number of Ip List : " + QString::number(m_slIpPortList.length()));
            mutex.unlock();
        }
        InsertCrawlingLog("Successfully Get Proxy : " + QString::number(slIpPortList.length() - 1));
        */
    }
    else if(str.right(7).compare("dbcfail") == 0)
    {
        InsertCrawlingLog("DB Connection Fail");
    }
    else if(str.right(7).compare("dbufail") == 0)
    {
        InsertCrawlingLog("DB Update Fail");
    }
    else if(str.right(8).compare("savefail") == 0)
    {
        InsertCrawlingLog("File Save Failed");
    }
    else if(str.right(7).compare("timeout") == 0)
    {
        InsertCrawlingLog("Timeout, skip this site");
        skipSite();
    }
    else if(str.right(8).compare("sitedown") == 0)
    {
        InsertCrawlingLog("the web site many have problem");
    }
    else
    {
        InsertCrawlingLog("Unknown error occurs");
    }

    pPro->kill();

    m_index++;
    if(m_index < m_slIpUrl.length())
    {
        update();
    }
    else
    {

        if(p_chkLocal->isChecked())
        {
            if(Debug("proxy.txt",m_slIpPortList.join("\n")))
            {
                InsertLog("proxy.txt is saved Successfully : " + QString::number(m_slIpPortList.length()));
            }
            else
            {
                InsertLog("proxy.txt Failed");
            }
        }
        {
            mutex.lock();
            if(m_slIpPortListTemp.length() > 500)
            {
                m_slIpPortList.clear();
                m_slIpPortList = m_slIpPortListTemp;
            }
            mutex.unlock();
            InsertCrawlingLog("Number of IP: " + QString::number(m_slIpPortList.length()));
        }

        p_labelRecentTime->setText(QDateTime::currentDateTime().toLocalTime().toString("yyyy-MM-dd hh:mm:ss"));
        if(!p_crawlingTimer->isActive())
        {
            p_crawlingTimer->start(CRAWLING_TIMER);
            InsertCrawlingLog("Crawler timer start");
        }
        InsertCrawlingLog("Reading proxylist.txt file is complete");
    }
}

void Widget::chkDb()
{
    p_chkLocal->setChecked(true);
}


void Widget::chkLocal()
{
    p_chkDb->setChecked(true);
}

bool Widget::SendIpList(QStringList _slIpList)
{
    QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
    db.setHostName("bigbird.iptime.org");
    db.setUserName("admin");
    db.setPassword("admin123");
    db.setDatabaseName("concepters");

    if (db.open() == false)
    {
        InsertLog("DB open Failed in SendIpList()");
        return false;
    }

    QSqlQuery sql;
    //QString strQuery = "truncate table Proxy";
    /*
    QString strQuery = "delete from Proxy";
    QString strUtf8(strQuery.toUtf8());

    if (sql.exec(strUtf8) == false)
    {
        p_labelStatus->setText("Delete Query\n Fail");
        return false;
    }
    */
    QString strQuery;
    QString strUtf8;
    InsertLog("Sending ip list : " + QString::number(_slIpList.length()));
    foreach(QString str, _slIpList)
    {
        strQuery = "insert into Proxy set Proxy='";
        strQuery += str.split(',').at(0).trimmed();
        strQuery += "', Port=";
        strQuery += str.split(',').at(1).trimmed();
        if(str.split(',').size() > 2)
        {
            strQuery += ", Source='";
            strQuery += str.split(',').at(2).trimmed();
            strQuery += "'";
        }
        strUtf8 = strQuery.toUtf8();
        if (sql.exec(strUtf8) == false)
        {
            InsertLog(sql.lastQuery() + "is Failed");
            //return false;
        }
    }
    db.close();
    return true;
}

bool Widget::Debug(QString _strFilename,QString _strData)
{
    QFile file(_strFilename);
    QFile::remove(_strFilename);
    if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
    {

//    if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate))
       // cout << "savefail" << endl;

        return false;
    }
    QTextStream out(&file);
    out << _strData;
    file.close();
    return true;
}

QStringList Widget::readList(QString strPath)
{
    QFile file(strPath);
    QStringList slProxyList;
    if(!file.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        return slProxyList;
    }
    QTextStream in(&file);

    while(!in.atEnd())
    {
        QString str = in.readLine().trimmed();
        if(str.at(0) == '#')
        {
            continue;
        }
        if(str.size() > 0)
            slProxyList << str;
    }

    file.close();
    return slProxyList;
}

void Widget::checkProxy()
{
    QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
    db.setHostName("bigbird.iptime.org");
    db.setUserName("admin");
    db.setPassword("admin123");
    db.setDatabaseName("concepters");


    if (db.open() == false)
    {
        InsertLog("DB open Failed in checkProxy()");
        return;
    }

    QSqlQuery query;

    QString strQuery;

    InsertLog("Check Proxy DB");
    strQuery = "select count(*) from Proxy";
    if(!query.exec(strQuery.toUtf8()))
    {
        InsertLog("Db open Failed in checkProxy()");
        return;
    }
    int result = 0;
    while(query.next())
        result = query.value(0).toInt();

    InsertLog("Number of Proxy in DB : " + QString::number(result));

    if(result < MIN_PROXY)
    {
        /*
        if(p_timer->isActive())
        {
            p_timer->stop();
            InsertLog("Update Timer Stop in CheckProxy()");
        }
        p_timer->start(UPDATE_TIMER);
        updateProxy();
        */
        updateDB();
    }
    db.close();
}



void Widget::InsertLog(QString str)
{
    QTime time = QTime::currentTime();
    QString strOut = time.toString("[hh:mm:ss] ") + str;
    m_pResultList->addItem(strOut);
    QDate date = QDate::currentDate();

    //QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
    QFile file(date.toString(Qt::ISODate)+ "_" + "proxy" + ".log");
    if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
        return;

    QTextStream out(&file);
    out << strOut << "\n";
    file.close();

    if (m_pResultList->count() > 1024)
    {
        m_pResultList->removeItemWidget(m_pResultList->item(0));
        QListWidgetItem* item = m_pResultList->takeItem(0);
        delete item;
    }
    m_pResultList->setCurrentRow(m_pResultList->count() - 1 );
    m_pResultList->repaint();
}

void Widget::InsertCrawlingLog(QString str)
{
    QTime time = QTime::currentTime();
    QString strOut = time.toString("[hh:mm:ss] ") + str;
    m_pCrawlingResultList->addItem(strOut);
    QDate date = QDate::currentDate();

    //QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
    QFile file(date.toString(Qt::ISODate)+ "_" + "proxyget" + ".log");
    if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
        return;

    QTextStream out(&file);
    out << strOut << "\n";
    file.close();

    if (m_pCrawlingResultList->count() > 1024)
    {
        m_pCrawlingResultList->removeItemWidget(m_pResultList->item(0));
        QListWidgetItem* item = m_pCrawlingResultList->takeItem(0);
        delete item;
    }
    m_pCrawlingResultList->setCurrentRow( m_pCrawlingResultList->count() - 1 );
    m_pCrawlingResultList->repaint();
}



void Widget::updateDB()
{
    if(p_timer->isActive())
    {
        p_timer->stop();
        InsertLog("update timer stop");
    }
    if(p_checkTimer->isActive())
    {
        p_checkTimer->stop();
        InsertLog("check timer stop");
    }

    mutex.lock();
    QStringList iplist = m_slIpPortList;
    mutex.unlock();

    if(SendIpList(iplist))
    {
        InsertLog("Success update ip list to ProxyDB : " + QString::number(iplist.length()));
    }
    else
    {
        InsertLog("Fail to update ip list to proxyDB");
    }

    if(!p_timer->isActive())
    {
        p_timer->start(UPDATE_TIMER);
        InsertLog("update timer start");
    }
    if(!p_checkTimer->isActive())
    {
        p_checkTimer->start(CHECK_TIMER);
        InsertLog("check timer start");
    }
}

void Widget::crawlingProxy()
{
    p_labelNextTime->setText(QDateTime::currentDateTime().toLocalTime().addSecs(p_lineTime->text().toInt()).toString("yyyy-MM-dd hh:mm:ss"));
    InsertCrawlingLog("crawlingProxy Executed");
    //m_slIpPortList.clear();
    m_slIpPortListTemp.clear();
    m_slIpUrl.clear();
    m_slIpUrl = getProxyList();
    m_index = 0;
    InsertCrawlingLog("Ip URL List Clear()");
    update();
}

QStringList Widget::getProxyList()
{
    QStringList slIpUrl = readList("proxylist.txt");
    if(slIpUrl.size() < 1)
    {
        m_slIpUrl //<< "http://proxylist.hidemyass.com/search-1305249#listable"
                  << "http://www.cybersyndrome.net/pla.html"
                  << "https://nordvpn.com/free-proxy-list/?country=&ports=&speed%5B1%5D=on&proto%5BHTTP%5D=on&anon%5BHigh%5D=on&by=l&order=ASC&perpage=500";
    }
    return slIpUrl;
}

void Widget::skipSite()
{
    QStringList temp = m_slIpUrl.at(m_index).split("/");
    if(temp.size() > 3)
    {
        int nindex = m_slIpUrl.lastIndexOf(QRegExp(QString("http[s]?://" + temp.at(2) + "[\\S]*")));
        if(nindex != -1)
            m_index = nindex;
    }
}