git-svn-id: svn://192.168.0.12/source@154 8346c931-da38-4b9b-9d4c-e48b93cbd075

ProxyProcess/ProxyProcess.pro.user

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE QtCreatorProject>
-<!-- Written by QtCreator 3.3.0, 2015-05-28T18:52:51. -->
+<!-- Written by QtCreator 3.3.0, 2015-06-05T18:05:10. -->
 <qtcreator>
  <data>
   <variable>EnvironmentId</variable>
@@ -61,7 +61,7 @@
    <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Desktop Qt 5.4.0 MinGW 32bit</value>
    <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Desktop Qt 5.4.0 MinGW 32bit</value>
    <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">qt.54.win32_mingw491_kit</value>
-   <value type="int" key="ProjectExplorer.Target.ActiveBuildConfiguration">0</value>
+   <value type="int" key="ProjectExplorer.Target.ActiveBuildConfiguration">1</value>
    <value type="int" key="ProjectExplorer.Target.ActiveDeployConfiguration">0</value>
    <value type="int" key="ProjectExplorer.Target.ActiveRunConfiguration">0</value>
    <valuemap type="QVariantMap" key="ProjectExplorer.Target.BuildConfiguration.0">
@@ -227,7 +227,7 @@
     <value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">ProxyProcess</value>
     <value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName"></value>
     <value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">Qt4ProjectManager.Qt4RunConfiguration:C:/source/ProxyProcess/ProxyProcess.pro</value>
-    <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;http://proxylist.hidemyass.com/search-1305249#listable&quot;</value>
+    <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments">&quot;http://www.proxylists.net/us_102_ext.html&quot;</value>
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.ProFile">ProxyProcess.pro</value>
     <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseDyldImageSuffix">false</value>
     <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseTerminal">true</value>

ProxyProcess/scrawler.cpp

@@ -56,33 +56,68 @@ void SCrawler::saveResult(bool ok)
 {
     if (!ok)
     {
-        cout << "Failed loading " << std::endl;
+        cout << "Failed loading";
         emit finished();
         return;
     }
 
     QString strIpList;
-    QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
+    if(m_strUrl.contains("hidemyass"))
-    strIpList = getIpListFromAss(p_parse);
-
-    //Debug("test.html",m_page->currentFrame()->toHtml());
-    //Debug("filtered.html", p_parse.toInnerXml());
-
-    //SendIpList(strIpList);
-
-    if((m_strLocation.compare("local") == 0) || (m_strLocation.compare("both") == 0))
     {
-        if(Debug(m_strFolder + "proxy.txt",strIpList) == false)
+        QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
-            return;
+        if(!p_parse.isNull())
+        {
+            strIpList = getIpListFromAss(p_parse);
+            if(strIpList.trimmed().size() > 0 )
+                strIpList = addSource(strIpList, "hidemyass.com");
+        }
     }
-    if((m_strLocation.compare("db") == 0) || (m_strLocation.compare("both") == 0))
+    else if(m_strUrl.contains("nordvpn"))
     {
-        if(SendIpList(strIpList) == false)
+
-            return;
+        QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"table","class","list");
+        if(!p_parse.isNull())
+        {
+            strIpList = getIpListFromNordVpn(p_parse);
+            if(strIpList.trimmed().size() > 0 )
+                strIpList = addSource(strIpList, "nordvpn.com");
+        }
+    }
+    else if(m_strUrl.contains("cyber"))
+    {
+        QWebElement p_parse = FindLeft(m_page->currentFrame()->documentElement(),"ol","style","list");
+        if(!p_parse.isNull())
+        {
+            strIpList = getIpListFromCyberSyndrom(p_parse);
+            if(strIpList.trimmed().size() > 0 )
+                strIpList = addSource(strIpList, "cybersyndrome.net");
+        }
+    }
+    else if(m_strUrl.contains("proxylists"))
+    {
+
+        QWebElement p_parse = m_page->mainFrame()->findFirstElement("table");
+        if(!p_parse.isNull())
+        {
+            strIpList = getIpListFromProxylists(p_parse);
+            if(strIpList.trimmed().size() > 0 )
+                strIpList = addSource(strIpList, "proxylists.net");
+        }
     }
 
+    if(strIpList.trimmed().size() > 0)
+        cout << strIpList.trimmed().toStdString();
+
+
+    if(strIpList.size() > 8)
+    {
+        cout << endl << "ok";
+    }
+    else
+    {
+        cout << "sitedown";
+    }
     emit finished();
-    cout << endl << "ok";
 }
 
 int SCrawler::GetNumber(QString _str)
@@ -290,19 +325,20 @@ QString SCrawler::getIpListFromAss(const QWebElement _FindElement)
 }
 
 
-QString SCrawler::getIpListFromFreeProxy(const QWebElement _FindElement)
+QString SCrawler::getIpListFromNordVpn(const QWebElement _FindElement)
 {
-    QWebElementCollection trs = _FindElement.findAll("tr");
+    QWebElement tbody = _FindElement.findFirst("tbody");
+    QWebElementCollection trs = tbody.findAll("tr");
     QString totalResult;
-    for(int i = 1; i < trs.count(); i++)
+    for(int i = 0; i < trs.count(); i++)
     {
-        QWebElementCollection tds = trs.at(i).findAll("td");
+        QWebElementCollection ths = trs.at(i).findAll("th");
 
-        if(tds.count() < 2)
+        if(ths.count() < 4)
             continue;
 
-        QString strip = tds.at(0).findFirst("a").toPlainText().trimmed();
+        QString strip = ths.at(2).toPlainText().trimmed();
-        QString strport = tds.at(1).toPlainText().trimmed();
+        QString strport = ths.at(3).toPlainText().trimmed();
         totalResult += strip;
         totalResult += ",";
         totalResult += strport;
@@ -313,6 +349,44 @@ QString SCrawler::getIpListFromFreeProxy(const QWebElement _FindElement)
     return totalResult;
 }
 
+QString SCrawler::getIpListFromCyberSyndrom(const QWebElement _FindElement)
+{
+    QString totalResult;
+    QWebElementCollection lis = _FindElement.findAll("li");
+    for(int i = 0; i < lis.count(); i++)
+    {
+        QString str = lis.at(i).toPlainText().trimmed().replace(":",",");
+        totalResult += str;
+        if(i < lis.count() - 1)
+            totalResult += "\n";
+    }
+    return totalResult;
+}
+
+QString SCrawler::getIpListFromProxylists(const QWebElement _FindElement)
+{
+
+    QString totalResult;
+    QWebElement table = _FindElement.findFirst("table");
+    QWebElementCollection trs = table.findAll("tr");
+    for(int i = 0; i < trs.count() - 1; i++)
+    {
+        QWebElementCollection tds = trs.at(i).findAll("td");
+        if(tds.count() < 4)
+            continue;
+        if(tds.at(2).toPlainText().compare("anonymous",Qt::CaseInsensitive) != 0)
+            continue;
+
+        QString ip = tds.at(0).toPlainText();
+        QString port = tds.at(1).toPlainText();
+
+        totalResult += ip.trimmed() + "," + port.trimmed() + "\n";
+    }
+
+    return totalResult.trimmed();
+}
+
+
 bool SCrawler::SendIpList(QString _strIpList)
 {
     QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
@@ -391,3 +465,22 @@ void SCrawler::killProcess()
     emit finished();
 }
 
+void SCrawler::SearchChildFrame(QWebFrame *frame)
+{
+    Debug("c:/data/nordvpnloop.html", frame->toHtml());
+    foreach(QWebFrame *childFrame, frame->childFrames())
+        SearchChildFrame(childFrame);
+}
+
+QString SCrawler::addSource(QString _strIpList, QString _strSource)
+{
+    QStringList strlistIpList = _strIpList.split("\n");
+    QStringList straddedList;
+    foreach(QString str, strlistIpList)
+    {
+        straddedList << (str + "," + _strSource);
+    }
+    return straddedList.join("\n").trimmed();
+}
+
+

ProxyProcess/scrawler.h

@@ -44,7 +44,11 @@ public:
     bool SendIpList(QString _str);
     void SearchChildFrame(QWebFrame *frame);
     QString getIpListFromAss(const QWebElement _FindElement);
-    QString getIpListFromFreeProxy(const QWebElement _FindElement);
+    QString getIpListFromNordVpn(const QWebElement _FindElement);
+    QString getIpListFromCyberSyndrom(const QWebElement _FindElement);
+    QString getIpListFromProxylists(const QWebElement _FindElement);
+
+    QString addSource(QString _strIpList, QString _strSource);
     QWebElement Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind);
     QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength);
     QWebElement FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart);
@@ -53,6 +57,7 @@ public:
     QList<QWebElement> FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength);
     QWebElementCollection Finds(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind);
     void UpdateError(QString _strError);
+
 };
 
 #endif // SCRAWLER_H