897 lines
27 KiB
C++
897 lines
27 KiB
C++
#include "scrawler.h"
|
|
#include <iostream>
|
|
#include <QSqlQuery>
|
|
#include <QSqlError>
|
|
#include <QByteArray>
|
|
#include <QDebug>
|
|
#include <QRegExp>
|
|
#include <QThread>
|
|
using namespace std;
|
|
|
|
struct SProxyList
|
|
{
|
|
QString m_strAddress;
|
|
int m_nPort;
|
|
};
|
|
|
|
SCrawler::SCrawler():QObject()
|
|
{
|
|
m_page = new QWebPage;
|
|
connect(m_page, SIGNAL(loadFinished(bool)), this, SLOT(saveResult(bool)));
|
|
// p_timer = new QTimer(this);
|
|
// connect(p_timer,SIGNAL(timeout()), this, SLOT(killProcess()));
|
|
QTimer::singleShot(60000, this, SLOT(killProcess()));
|
|
m_bCrawled = false;
|
|
|
|
}
|
|
|
|
SCrawler::~SCrawler()
|
|
{
|
|
}
|
|
|
|
void SCrawler::load(QStringList _strlistArgv)
|
|
{
|
|
QUrl url(_strlistArgv.at(0));
|
|
QNetworkRequest *request = new QNetworkRequest;
|
|
|
|
m_strUrl = _strlistArgv.at(0);
|
|
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, true);
|
|
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, true);
|
|
|
|
if(_strlistArgv.length() > 1)
|
|
m_strFolder = _strlistArgv.at(1).trimmed() + "\\";
|
|
|
|
if(_strlistArgv.length() > 2)
|
|
m_strLocation = _strlistArgv.at(2).trimmed();
|
|
else
|
|
m_strLocation = "local";
|
|
|
|
m_page->mainFrame()->load(url);
|
|
}
|
|
|
|
void SCrawler::UpdateError(QString _strError)
|
|
{
|
|
m_bError = true;
|
|
}
|
|
|
|
void SCrawler::saveResult(bool ok)
|
|
{
|
|
if (!ok)
|
|
{
|
|
cout << "Failed loading";
|
|
emit finished();
|
|
return;
|
|
}
|
|
|
|
QString strIpList;
|
|
if(m_strUrl.contains("hidemyass"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromAss(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "hidemyass.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("nordvpn"))
|
|
{
|
|
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"table","class","list");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromNordVpn(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "nordvpn.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("cyber"))
|
|
{
|
|
QWebElement p_parse = FindLeft(m_page->currentFrame()->documentElement(),"ol","style","list");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromCyberSyndrom(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "cybersyndrome.net");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("proxylists.net"))
|
|
{
|
|
|
|
QWebElement p_parse = m_page->mainFrame()->findFirstElement("table");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromProxylists(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxylists.net");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("txt.proxyspy.net"))
|
|
{
|
|
QString p_parse = m_page->mainFrame()->toPlainText();
|
|
if(!p_parse.isEmpty())
|
|
{
|
|
strIpList = getIpListFromProxySpy(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxyspy.net");
|
|
}
|
|
/*
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromProxylists(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxylists.net");
|
|
}
|
|
*/
|
|
}
|
|
else if(m_strUrl.contains("proxysearcher.sourceforge.net"))
|
|
{
|
|
QWebElement p_parse = m_page->mainFrame()->findFirstElement("body");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromProxySearcher(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxysearcher.sourceforge.net");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("proxylist.ro"))
|
|
{
|
|
QWebElement p_parse = m_page->mainFrame()->findFirstElement("body");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromProxyListro(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxylist.ro");
|
|
}
|
|
}
|
|
|
|
else if(m_strUrl.contains("samair.ru"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(), "div", "id", "content");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromSamuir(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "samair.ru");
|
|
}
|
|
}
|
|
|
|
else if(m_strUrl.contains("nntime.com"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(), "table", "id", "proxylist");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromNntime(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "nntime.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("free-proxy.cz"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(), "table", "id", "proxy_list");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromFreeproxy(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "free-proxy.cz");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("xroxy.com"))
|
|
{
|
|
QWebElement p_parse = m_page->currentFrame()->documentElement().findFirst("body");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromXroxy(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "xroxy.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("cool-proxy.net"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(), "div", "id", "main");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromCoolProxy(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "cool-proxy.net");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("gatherproxy.com"))
|
|
{
|
|
strIpList = getIpListFromGatherProxy();
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "gatherproxy.com");
|
|
}
|
|
|
|
else if(m_strUrl.contains("wait3"))
|
|
{
|
|
QThread::sleep(3);
|
|
}
|
|
else if(m_strUrl.contains("wait5"))
|
|
{
|
|
QThread::sleep(5);
|
|
}
|
|
|
|
|
|
QThread::sleep(5);
|
|
//qDebug() << strIpList;
|
|
//Debug("c:/data/test3.html", m_page->mainFrame()->toHtml());
|
|
|
|
if(strIpList.trimmed().size() > 0)
|
|
cout << strIpList.trimmed().toStdString();
|
|
|
|
|
|
if(strIpList.size() > 8)
|
|
{
|
|
cout << endl << "ok";
|
|
}
|
|
else
|
|
{
|
|
cout << "sitedown";
|
|
}
|
|
emit finished();
|
|
}
|
|
|
|
int SCrawler::GetNumber(QString _str)
|
|
{
|
|
QString strNumber;
|
|
for (int i = 0; i < _str.size();i++)
|
|
{
|
|
if (_str.at(i).isNumber())
|
|
strNumber += _str.at(i);
|
|
}
|
|
return strNumber.toInt();
|
|
}
|
|
|
|
bool SCrawler::Debug(QString _strFilename,QString _strData)
|
|
{
|
|
QFile file(_strFilename);
|
|
QFile::remove(_strFilename);
|
|
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
|
|
{
|
|
// if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate))
|
|
cout << "savefail" << endl;
|
|
emit finished();
|
|
return false;
|
|
}
|
|
QTextStream out(&file);
|
|
out << _strData;
|
|
file.close();
|
|
return true;
|
|
}
|
|
|
|
QString SCrawler::SqlString(QString _str)
|
|
{
|
|
_str = _str.replace("'","\\'");
|
|
_str = _str.replace("\"","\\\"");
|
|
return _str;
|
|
}
|
|
|
|
|
|
QString SCrawler::GetSafeUtf(QString _strData)
|
|
{
|
|
QString str;
|
|
QChar *pch = _strData.data();
|
|
|
|
for (int i = 0; i < _strData.length(); i++)
|
|
{
|
|
if (pch[i].unicode() >= 12593 && pch[i].unicode() <= 12622)
|
|
str += pch[i];
|
|
if (pch[i].unicode() >= 44032 && pch[i].unicode() <= 55203)
|
|
str += pch[i];
|
|
if (pch[i].isDigit() || pch[i].isNumber() || pch[i].isSpace() || pch[i].isLower() || pch[i].isUpper() || pch[i].isSymbol() )
|
|
str += pch[i];
|
|
}
|
|
return str;
|
|
}
|
|
|
|
QWebElement SCrawler::Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
if (element.attribute(_strAttrib) == _strFind)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength);
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart)
|
|
{
|
|
int _strLength = _strFind.length();
|
|
return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart, _strLength);
|
|
}
|
|
|
|
QWebElement SCrawler::FindRight(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().right(_strFind.length());
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindLeft(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
int _strStart = 0;
|
|
return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart);
|
|
}
|
|
|
|
|
|
QString SCrawler::getIpListFromAss(const QWebElement _FindElement)
|
|
{
|
|
|
|
QString DISPLAYINLINE = "{display:inline}";
|
|
QString DISPLAYNONE = "{display:none}";
|
|
QString totalResult;
|
|
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
|
|
for(int i = 1; i < trs.count(); i++)
|
|
{
|
|
QWebElementCollection tds = trs.at(i).findAll("td");
|
|
|
|
QWebElement firstStyle = trs.at(i).findFirst("style");
|
|
|
|
QString strFirstStyle = firstStyle.toInnerXml().trimmed();
|
|
|
|
QStringList disinline;
|
|
QStringList disnone;
|
|
// {display:inline} {display:none}
|
|
// Find display:inline list
|
|
// Find display:none list
|
|
QStringList strlist = strFirstStyle.split(".", QString::SkipEmptyParts);
|
|
|
|
foreach(QString str, strlist)
|
|
{
|
|
if(str.trimmed().right(DISPLAYINLINE.length()).compare(DISPLAYINLINE) == 0)
|
|
{
|
|
disinline.append(str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length()));
|
|
//cout << DISPLAYINLINE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length()).toStdString() << endl;
|
|
}
|
|
else
|
|
{
|
|
disnone.append(str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length()));
|
|
//cout << DISPLAYNONE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length()).toStdString() << endl;
|
|
}
|
|
}
|
|
|
|
//Change class -> display:inline
|
|
//Change class -> display:none
|
|
QString strIpContents = tds.at(1).toInnerXml();
|
|
foreach(QString str, disinline)
|
|
{
|
|
strIpContents.replace(str.trimmed(),"display: inline");
|
|
}
|
|
foreach(QString str, disnone)
|
|
{
|
|
strIpContents.replace(str.trimmed(),"display:none");
|
|
}
|
|
|
|
//cout << strIpContents.toStdString() << endl;
|
|
|
|
//Parsing contents and extracting ip and port each table row
|
|
QStringList strIpContentsList = strIpContents.split(QRegExp("[<>]"), QString::SkipEmptyParts);
|
|
QString result;
|
|
for(int j = 0; j < strIpContentsList.count(); j++)
|
|
{
|
|
QString str1 = strIpContentsList.at(j).trimmed();
|
|
|
|
if(str1.at(0) == '/')
|
|
continue;
|
|
if(str1.left(4).compare("span") == 0)
|
|
continue;
|
|
if(str1.left(3).compare("div") == 0)
|
|
continue;
|
|
if(str1.left(5).compare("style") == 0)
|
|
continue;
|
|
if(str1.mid(1,3).compare("dis") == 0)
|
|
continue;
|
|
|
|
if(j > 0)
|
|
{
|
|
QString str2 = strIpContentsList.at(j-1).trimmed();
|
|
if(str2.right(5).compare("none\"") == 0)
|
|
continue;
|
|
}
|
|
|
|
result += str1;
|
|
}
|
|
|
|
//cout << strIpContentsList.length() << endl;
|
|
//cout << result.toStdString() << endl;
|
|
//Debug("proxy.txt",result);
|
|
|
|
QString strport = tds.at(2).toInnerXml().trimmed();
|
|
result += ("," + strport);
|
|
totalResult += result;
|
|
if(i < trs.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
return totalResult;
|
|
}
|
|
|
|
|
|
QString SCrawler::getIpListFromNordVpn(const QWebElement _FindElement)
|
|
{
|
|
QWebElement tbody = _FindElement.findFirst("tbody");
|
|
QWebElementCollection trs = tbody.findAll("tr");
|
|
QString totalResult;
|
|
for(int i = 0; i < trs.count(); i++)
|
|
{
|
|
QWebElementCollection ths = trs.at(i).findAll("th");
|
|
|
|
if(ths.count() < 4)
|
|
continue;
|
|
|
|
QString strip = ths.at(2).toPlainText().trimmed();
|
|
QString strport = ths.at(3).toPlainText().trimmed();
|
|
totalResult += strip;
|
|
totalResult += ",";
|
|
totalResult += strport;
|
|
if(i < trs.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
|
|
return totalResult;
|
|
}
|
|
|
|
QString SCrawler::getIpListFromCyberSyndrom(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection lis = _FindElement.findAll("li");
|
|
for(int i = 0; i < lis.count(); i++)
|
|
{
|
|
QString str = lis.at(i).toPlainText().trimmed().replace(":",",");
|
|
totalResult += str;
|
|
if(i < lis.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
return totalResult;
|
|
}
|
|
|
|
QString SCrawler::getIpListFromProxylists(const QWebElement _FindElement)
|
|
{
|
|
|
|
QString totalResult;
|
|
QWebElement table = _FindElement.findFirst("table");
|
|
QWebElementCollection trs = table.findAll("tr");
|
|
for(int i = 0; i < trs.count() - 1; i++)
|
|
{
|
|
QWebElementCollection tds = trs.at(i).findAll("td");
|
|
if(tds.count() < 4)
|
|
continue;
|
|
if(tds.at(2).toPlainText().compare("anonymous",Qt::CaseInsensitive) != 0)
|
|
continue;
|
|
|
|
QString ip = tds.at(0).toPlainText();
|
|
QString port = tds.at(1).toPlainText();
|
|
|
|
totalResult += ip.trimmed() + "," + port.trimmed() + "\n";
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromProxySpy(const QString _txt)
|
|
{
|
|
QStringList strlist = _txt.split("\n", QString::SkipEmptyParts);
|
|
QString totalResult;
|
|
//qDebug() << _txt;
|
|
QRegExp re("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}):(\\d{2,5})\\s*[a-zA-Z]+-(N|A|H)\\S*");
|
|
foreach(QString str, strlist)
|
|
{
|
|
int pos = 0;
|
|
while((pos = re.indexIn(str, pos)) != -1)
|
|
{
|
|
if((re.cap(3) == "A") || (re.cap(3)) == "H")
|
|
totalResult += (re.cap(1) + "," + re.cap(2) + "\n");
|
|
pos += re.matchedLength();
|
|
}
|
|
}
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromProxySearcher(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QRegExp re("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}):(\\d{2,5})");
|
|
QString str = _FindElement.toPlainText();
|
|
int pos = 0;
|
|
while((pos = re.indexIn(str, pos)) != -1)
|
|
{
|
|
totalResult += (re.cap(1) + "," + re.cap(2) + "\n");
|
|
pos += re.matchedLength();
|
|
}
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromProxyListro(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QString strclass = tr.attribute("class").trimmed();
|
|
if((strclass.compare("speed1") == 0) || (strclass.compare("speed2") == 0))
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
if(tds.count() < 4)
|
|
continue;
|
|
if((tds.at(3).toPlainText().trimmed() == "Y") || (tds.at(3).toPlainText().trimmed() == "y"))
|
|
{
|
|
totalResult += tds.at(1).toPlainText().trimmed().replace("\"","").trimmed() + "," + tds.at(2).toPlainText().trimmed().replace("\"","").trimmed() + "\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromSamuir(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
|
|
QWebElement table = Find(_FindElement, "table", "id", "proxylist");
|
|
|
|
QWebElementCollection trs = table.findAll("tr");
|
|
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
if(tds.count() < 3)
|
|
continue;
|
|
|
|
if(tds.at(1).toPlainText().contains("anony"))
|
|
{
|
|
QString temp = tds.at(0).toPlainText().replace("\"","").trimmed();
|
|
totalResult += (temp.replace(":",",") + "\n");
|
|
}
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromNntime(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
QRegExp re("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}):(\\d{2,5})");
|
|
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
if(tds.count() < 4)
|
|
continue;
|
|
if(tds.at(2).toPlainText().contains("anony"))
|
|
{
|
|
int pos = 0;
|
|
while((pos = re.indexIn(tds.at(1).toPlainText().replace("\"",""), pos)) != -1)
|
|
{
|
|
totalResult += (re.cap(1) + "," + re.cap(2) + "\n");
|
|
pos += re.matchedLength();
|
|
}
|
|
}
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromFreeproxy(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
if(tds.count() < 7)
|
|
continue;
|
|
|
|
totalResult += (tds.at(0).toPlainText().trimmed() + "," + tds.at(1).toPlainText().trimmed() + "\n");
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
QString SCrawler::getIpListFromXroxy(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
QRegExp reip("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})");
|
|
QRegExp repo("(\\d{2,5})");
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
QString ip;
|
|
QString port;
|
|
if(tds.count() < 7)
|
|
continue;
|
|
{
|
|
int pos = 0;
|
|
while((pos = reip.indexIn(tds.at(1).toPlainText().replace("\"","").trimmed(), pos)) != -1)
|
|
{
|
|
ip = reip.cap(1);
|
|
pos += reip.matchedLength();
|
|
}
|
|
}
|
|
{
|
|
int pos = 0;
|
|
while((pos = repo.indexIn(tds.at(2).toPlainText().replace("\"","").trimmed(), pos)) != -1)
|
|
{
|
|
port = repo.cap(1);
|
|
pos += repo.matchedLength();
|
|
}
|
|
}
|
|
if(!ip.isEmpty() && !port.isEmpty())
|
|
{
|
|
totalResult += (ip + "," + port + "\n");
|
|
}
|
|
|
|
{
|
|
ip.clear();
|
|
port.clear();
|
|
}
|
|
|
|
}
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
|
|
QString SCrawler::getIpListFromCoolProxy(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
QRegExp reip("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})");
|
|
QRegExp repo("(\\d{2,5})");
|
|
|
|
|
|
foreach(QWebElement tr, trs)
|
|
{
|
|
QWebElementCollection tds = tr.findAll("td");
|
|
QString ip;
|
|
QString port;
|
|
if(tds.count() < 7)
|
|
continue;
|
|
{
|
|
int pos = 0;
|
|
while((pos = reip.indexIn(tds.at(0).toPlainText().replace("\"","").trimmed(), pos)) != -1)
|
|
{
|
|
ip = reip.cap(1);
|
|
pos += reip.matchedLength();
|
|
}
|
|
}
|
|
{
|
|
int pos = 0;
|
|
while((pos = repo.indexIn(tds.at(1).toPlainText().replace("\"","").trimmed(), pos)) != -1)
|
|
{
|
|
port = repo.cap(1);
|
|
pos += repo.matchedLength();
|
|
}
|
|
}
|
|
if(!ip.isEmpty() && !port.isEmpty())
|
|
{
|
|
totalResult += (ip + "," + port + "\n");
|
|
}
|
|
|
|
{
|
|
ip.clear();
|
|
port.clear();
|
|
}
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
|
|
|
|
QString SCrawler::getIpListFromGatherProxy()
|
|
{
|
|
QString totalResult;
|
|
QTcpSocket socket;
|
|
socket.connectToHost("65.50.243.103",80);
|
|
if(!socket.waitForConnected())
|
|
{
|
|
qDebug() << "Error: " << socket.errorString();
|
|
}
|
|
|
|
QString index = m_strUrl.right(2);
|
|
QRegExp re("(\\d+)");
|
|
int pos = 0;
|
|
QString num;
|
|
while((pos = re.indexIn(index, pos)) != -1)
|
|
{
|
|
num = re.cap(1);
|
|
pos += re.matchedLength();
|
|
}
|
|
|
|
QString strheader = "POST /proxylist/anonymity/?t=Elite HTTP/1.1\r\n"
|
|
"Host: www.gatherproxy.com\r\n"
|
|
"Connection: keep-alive\r\n"
|
|
"Content-Length: " + QString::number(28+num.length()) + "\r\n"
|
|
"Cache-Control: max-age=0\r\n"
|
|
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n"
|
|
"Origin: http://www.gatherproxy.com\r\n"
|
|
"User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36\r\n"
|
|
"Content-Type: application/x-www-form-urlencoded\r\n"
|
|
"Referer: http://www.gatherproxy.com/proxylist/anonymity/?t=Elite\r\n"
|
|
"Accept-Encoding: deflate\r\n"
|
|
"Accept-Language: ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4\r\n\r\n"
|
|
"Type=elite&PageIdx=" + num + "&Uptime=0";
|
|
socket.write(strheader.toUtf8());
|
|
|
|
QString strPacket;
|
|
while (socket.waitForReadyRead())
|
|
{
|
|
strPacket += QString::fromUtf8(socket.readAll());
|
|
}
|
|
//Debug("c:/data/asdf.html", strPacket);
|
|
{
|
|
int pos = 0;
|
|
QRegExp re("<td><script>document\\.write\\('(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})'\\)</script></td>\\s*<td><script>document\\.write\\(gp\\.dep\\('([A-Fa-f0-9]{2,4})'\\)\\)</script>");
|
|
while((pos = re.indexIn(strPacket, pos)) != -1)
|
|
{
|
|
QString ip = re.cap(1);
|
|
QString port = QString::number(getPort(re.cap(2)));
|
|
|
|
if(!ip.isNull() && !port.isNull())
|
|
{
|
|
totalResult += (ip + "," + port + "\n");
|
|
}
|
|
|
|
pos += re.matchedLength();
|
|
}
|
|
|
|
}
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
|
|
bool SCrawler::SendIpList(QString _strIpList)
|
|
{
|
|
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
|
|
db.setHostName("bigbird.iptime.org");
|
|
db.setUserName("admin");
|
|
db.setPassword("admin123");
|
|
db.setDatabaseName("concepters");
|
|
|
|
if (db.open() == false)
|
|
{
|
|
cout << "error : db open fail..." << endl;
|
|
cout << "dbcfail";
|
|
emit finished();
|
|
return false;
|
|
}
|
|
|
|
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
|
|
|
|
if(strListIpList.size() < 1)
|
|
{
|
|
cout << "sitedown";
|
|
return false;
|
|
}
|
|
|
|
|
|
QSqlQuery sql;
|
|
//QString strQuery = "truncate table Proxy";
|
|
QString strQuery = "delete from Proxy";
|
|
QString strUtf8(strQuery.toUtf8());
|
|
|
|
if (sql.exec(strUtf8) == false)
|
|
{
|
|
cout << "error : " << sql.lastError().text().toStdString();
|
|
cout << endl << "dbdfail";
|
|
//emit finished();
|
|
//return;
|
|
}
|
|
|
|
|
|
foreach(QString str, strListIpList)
|
|
{
|
|
strQuery = "insert into Proxy set Proxy='";
|
|
strQuery += str.split(',').at(0).trimmed();
|
|
strQuery += "', Port=";
|
|
strQuery += str.split(',').at(1).trimmed();
|
|
strUtf8 = strQuery.toUtf8();
|
|
if (sql.exec(strUtf8) == false)
|
|
{
|
|
cout << "error : " << sql.lastError().text().toStdString();
|
|
cout << endl << "dbufail";
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
QList<QWebElement> SCrawler::FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
QList<QWebElement> returnElements = QList<QWebElement>();
|
|
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength);
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
returnElements.append(element);
|
|
}
|
|
}
|
|
return returnElements;
|
|
}
|
|
|
|
void SCrawler::killProcess()
|
|
{
|
|
cout << endl << "timeout";
|
|
emit finished();
|
|
}
|
|
|
|
void SCrawler::SearchChildFrame(QWebFrame *frame)
|
|
{
|
|
Debug("c:/data/nordvpnloop.html", frame->toHtml());
|
|
foreach(QWebFrame *childFrame, frame->childFrames())
|
|
SearchChildFrame(childFrame);
|
|
}
|
|
|
|
QString SCrawler::addSource(QString _strIpList, QString _strSource)
|
|
{
|
|
QStringList strlistIpList = _strIpList.split("\n");
|
|
QStringList straddedList;
|
|
foreach(QString str, strlistIpList)
|
|
{
|
|
straddedList << (str + "," + _strSource);
|
|
}
|
|
return straddedList.join("\n").trimmed();
|
|
}
|
|
|
|
int SCrawler::getPort(QString _strport)
|
|
{
|
|
int result = 0;
|
|
for(int i = 0; i < _strport.length(); i++)
|
|
{
|
|
char strport = _strport.at(i).toLatin1();
|
|
if( 'a' <= strport && strport <= 'f' )
|
|
{
|
|
result += (((int)strport - (int)'a' + 10) << ((_strport.length()-1-i)*4));
|
|
}
|
|
else if( 'A' <= strport && strport <= 'F' )
|
|
{
|
|
result += (((int)strport - (int)'A' + 10) << ((_strport.length()-1-i)*4));
|
|
}
|
|
else if( '0' <= strport && strport <= '9')
|
|
{
|
|
result += (((int)strport - (int)'0') << ((_strport.length()-1-i)*4));
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|