487 lines
14 KiB
C++
487 lines
14 KiB
C++
#include "scrawler.h"
|
|
#include <iostream>
|
|
#include <QSqlQuery>
|
|
#include <QSqlError>
|
|
#include <QByteArray>
|
|
#include <qDebug>
|
|
|
|
using namespace std;
|
|
|
|
struct SProxyList
|
|
{
|
|
QString m_strAddress;
|
|
int m_nPort;
|
|
};
|
|
|
|
SCrawler::SCrawler():QObject()
|
|
{
|
|
m_page = new QWebPage;
|
|
connect(m_page, SIGNAL(loadFinished(bool)), this, SLOT(saveResult(bool)));
|
|
// p_timer = new QTimer(this);
|
|
// connect(p_timer,SIGNAL(timeout()), this, SLOT(killProcess()));
|
|
QTimer::singleShot(90000, this, SLOT(killProcess()));
|
|
m_bCrawled = false;
|
|
}
|
|
|
|
SCrawler::~SCrawler()
|
|
{
|
|
}
|
|
|
|
void SCrawler::load(QStringList _strlistArgv)
|
|
{
|
|
QUrl url(_strlistArgv.at(0));
|
|
QNetworkRequest *request = new QNetworkRequest;
|
|
|
|
m_strUrl = _strlistArgv.at(0);
|
|
m_page->settings()->setAttribute(QWebSettings::AutoLoadImages, true);
|
|
m_page->settings()->setAttribute(QWebSettings::JavaEnabled, true);
|
|
|
|
if(_strlistArgv.length() > 1)
|
|
m_strFolder = _strlistArgv.at(1).trimmed() + "\\";
|
|
|
|
if(_strlistArgv.length() > 2)
|
|
m_strLocation = _strlistArgv.at(2).trimmed();
|
|
else
|
|
m_strLocation = "local";
|
|
|
|
m_page->mainFrame()->load(url);
|
|
}
|
|
|
|
void SCrawler::UpdateError(QString _strError)
|
|
{
|
|
m_bError = true;
|
|
}
|
|
|
|
void SCrawler::saveResult(bool ok)
|
|
{
|
|
if (!ok)
|
|
{
|
|
cout << "Failed loading";
|
|
emit finished();
|
|
return;
|
|
}
|
|
|
|
QString strIpList;
|
|
if(m_strUrl.contains("hidemyass"))
|
|
{
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"div","class","row inner");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromAss(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "hidemyass.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("nordvpn"))
|
|
{
|
|
|
|
QWebElement p_parse = Find(m_page->currentFrame()->documentElement(),"table","class","list");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromNordVpn(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "nordvpn.com");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("cyber"))
|
|
{
|
|
QWebElement p_parse = FindLeft(m_page->currentFrame()->documentElement(),"ol","style","list");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromCyberSyndrom(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "cybersyndrome.net");
|
|
}
|
|
}
|
|
else if(m_strUrl.contains("proxylists"))
|
|
{
|
|
|
|
QWebElement p_parse = m_page->mainFrame()->findFirstElement("table");
|
|
if(!p_parse.isNull())
|
|
{
|
|
strIpList = getIpListFromProxylists(p_parse);
|
|
if(strIpList.trimmed().size() > 0 )
|
|
strIpList = addSource(strIpList, "proxylists.net");
|
|
}
|
|
}
|
|
|
|
if(strIpList.trimmed().size() > 0)
|
|
cout << strIpList.trimmed().toStdString();
|
|
|
|
|
|
if(strIpList.size() > 8)
|
|
{
|
|
cout << endl << "ok";
|
|
}
|
|
else
|
|
{
|
|
cout << "sitedown";
|
|
}
|
|
emit finished();
|
|
}
|
|
|
|
int SCrawler::GetNumber(QString _str)
|
|
{
|
|
QString strNumber;
|
|
for (int i = 0; i < _str.size();i++)
|
|
{
|
|
if (_str.at(i).isNumber())
|
|
strNumber += _str.at(i);
|
|
}
|
|
return strNumber.toInt();
|
|
}
|
|
|
|
bool SCrawler::Debug(QString _strFilename,QString _strData)
|
|
{
|
|
QFile file(_strFilename);
|
|
QFile::remove(_strFilename);
|
|
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
|
|
{
|
|
// if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate))
|
|
cout << "savefail" << endl;
|
|
emit finished();
|
|
return false;
|
|
}
|
|
QTextStream out(&file);
|
|
out << _strData;
|
|
file.close();
|
|
return true;
|
|
}
|
|
|
|
QString SCrawler::SqlString(QString _str)
|
|
{
|
|
_str = _str.replace("'","\\'");
|
|
_str = _str.replace("\"","\\\"");
|
|
return _str;
|
|
}
|
|
|
|
|
|
QString SCrawler::GetSafeUtf(QString _strData)
|
|
{
|
|
QString str;
|
|
QChar *pch = _strData.data();
|
|
|
|
for (int i = 0; i < _strData.length(); i++)
|
|
{
|
|
if (pch[i].unicode() >= 12593 && pch[i].unicode() <= 12622)
|
|
str += pch[i];
|
|
if (pch[i].unicode() >= 44032 && pch[i].unicode() <= 55203)
|
|
str += pch[i];
|
|
if (pch[i].isDigit() || pch[i].isNumber() || pch[i].isSpace() || pch[i].isLower() || pch[i].isUpper() || pch[i].isSymbol() )
|
|
str += pch[i];
|
|
}
|
|
return str;
|
|
}
|
|
|
|
QWebElement SCrawler::Find(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
if (element.attribute(_strAttrib) == _strFind)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength);
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart)
|
|
{
|
|
int _strLength = _strFind.length();
|
|
return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart, _strLength);
|
|
}
|
|
|
|
QWebElement SCrawler::FindRight(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().right(_strFind.length());
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
return element;
|
|
}
|
|
}
|
|
QWebElement element;
|
|
return element;
|
|
}
|
|
|
|
QWebElement SCrawler::FindLeft(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind)
|
|
{
|
|
int _strStart = 0;
|
|
return FindMid(_FindElement, _strElement, _strAttrib, _strFind, _strStart);
|
|
}
|
|
|
|
|
|
QString SCrawler::getIpListFromAss(const QWebElement _FindElement)
|
|
{
|
|
|
|
QString DISPLAYINLINE = "{display:inline}";
|
|
QString DISPLAYNONE = "{display:none}";
|
|
QString totalResult;
|
|
|
|
QWebElementCollection trs = _FindElement.findAll("tr");
|
|
|
|
|
|
for(int i = 1; i < trs.count(); i++)
|
|
{
|
|
QWebElementCollection tds = trs.at(i).findAll("td");
|
|
|
|
QWebElement firstStyle = trs.at(i).findFirst("style");
|
|
|
|
QString strFirstStyle = firstStyle.toInnerXml().trimmed();
|
|
|
|
QStringList disinline;
|
|
QStringList disnone;
|
|
// {display:inline} {display:none}
|
|
// Find display:inline list
|
|
// Find display:none list
|
|
QStringList strlist = strFirstStyle.split(".", QString::SkipEmptyParts);
|
|
|
|
foreach(QString str, strlist)
|
|
{
|
|
if(str.trimmed().right(DISPLAYINLINE.length()).compare(DISPLAYINLINE) == 0)
|
|
{
|
|
disinline.append(str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length()));
|
|
//cout << DISPLAYINLINE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYINLINE.length()).toStdString() << endl;
|
|
}
|
|
else
|
|
{
|
|
disnone.append(str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length()));
|
|
//cout << DISPLAYNONE.toStdString() << " : " << str.trimmed().left(str.trimmed().length() - DISPLAYNONE.length()).toStdString() << endl;
|
|
}
|
|
}
|
|
|
|
//Change class -> display:inline
|
|
//Change class -> display:none
|
|
QString strIpContents = tds.at(1).toInnerXml();
|
|
foreach(QString str, disinline)
|
|
{
|
|
strIpContents.replace(str.trimmed(),"display: inline");
|
|
}
|
|
foreach(QString str, disnone)
|
|
{
|
|
strIpContents.replace(str.trimmed(),"display:none");
|
|
}
|
|
|
|
//cout << strIpContents.toStdString() << endl;
|
|
|
|
//Parsing contents and extracting ip and port each table row
|
|
QStringList strIpContentsList = strIpContents.split(QRegExp("[<>]"), QString::SkipEmptyParts);
|
|
QString result;
|
|
for(int j = 0; j < strIpContentsList.count(); j++)
|
|
{
|
|
QString str1 = strIpContentsList.at(j).trimmed();
|
|
|
|
if(str1.at(0) == '/')
|
|
continue;
|
|
if(str1.left(4).compare("span") == 0)
|
|
continue;
|
|
if(str1.left(3).compare("div") == 0)
|
|
continue;
|
|
if(str1.left(5).compare("style") == 0)
|
|
continue;
|
|
if(str1.mid(1,3).compare("dis") == 0)
|
|
continue;
|
|
|
|
if(j > 0)
|
|
{
|
|
QString str2 = strIpContentsList.at(j-1).trimmed();
|
|
if(str2.right(5).compare("none\"") == 0)
|
|
continue;
|
|
}
|
|
|
|
result += str1;
|
|
}
|
|
|
|
//cout << strIpContentsList.length() << endl;
|
|
//cout << result.toStdString() << endl;
|
|
//Debug("proxy.txt",result);
|
|
|
|
QString strport = tds.at(2).toInnerXml().trimmed();
|
|
result += ("," + strport);
|
|
totalResult += result;
|
|
if(i < trs.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
return totalResult;
|
|
}
|
|
|
|
|
|
QString SCrawler::getIpListFromNordVpn(const QWebElement _FindElement)
|
|
{
|
|
QWebElement tbody = _FindElement.findFirst("tbody");
|
|
QWebElementCollection trs = tbody.findAll("tr");
|
|
QString totalResult;
|
|
for(int i = 0; i < trs.count(); i++)
|
|
{
|
|
QWebElementCollection ths = trs.at(i).findAll("th");
|
|
|
|
if(ths.count() < 4)
|
|
continue;
|
|
|
|
QString strip = ths.at(2).toPlainText().trimmed();
|
|
QString strport = ths.at(3).toPlainText().trimmed();
|
|
totalResult += strip;
|
|
totalResult += ",";
|
|
totalResult += strport;
|
|
if(i < trs.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
|
|
return totalResult;
|
|
}
|
|
|
|
QString SCrawler::getIpListFromCyberSyndrom(const QWebElement _FindElement)
|
|
{
|
|
QString totalResult;
|
|
QWebElementCollection lis = _FindElement.findAll("li");
|
|
for(int i = 0; i < lis.count(); i++)
|
|
{
|
|
QString str = lis.at(i).toPlainText().trimmed().replace(":",",");
|
|
totalResult += str;
|
|
if(i < lis.count() - 1)
|
|
totalResult += "\n";
|
|
}
|
|
return totalResult;
|
|
}
|
|
|
|
QString SCrawler::getIpListFromProxylists(const QWebElement _FindElement)
|
|
{
|
|
|
|
QString totalResult;
|
|
QWebElement table = _FindElement.findFirst("table");
|
|
QWebElementCollection trs = table.findAll("tr");
|
|
for(int i = 0; i < trs.count() - 1; i++)
|
|
{
|
|
QWebElementCollection tds = trs.at(i).findAll("td");
|
|
if(tds.count() < 4)
|
|
continue;
|
|
if(tds.at(2).toPlainText().compare("anonymous",Qt::CaseInsensitive) != 0)
|
|
continue;
|
|
|
|
QString ip = tds.at(0).toPlainText();
|
|
QString port = tds.at(1).toPlainText();
|
|
|
|
totalResult += ip.trimmed() + "," + port.trimmed() + "\n";
|
|
}
|
|
|
|
return totalResult.trimmed();
|
|
}
|
|
|
|
|
|
bool SCrawler::SendIpList(QString _strIpList)
|
|
{
|
|
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL");
|
|
db.setHostName("bigbird.iptime.org");
|
|
db.setUserName("admin");
|
|
db.setPassword("admin123");
|
|
db.setDatabaseName("concepters");
|
|
|
|
if (db.open() == false)
|
|
{
|
|
cout << "error : db open fail..." << endl;
|
|
cout << "dbcfail";
|
|
emit finished();
|
|
return false;
|
|
}
|
|
|
|
QStringList strListIpList = _strIpList.split('\n', QString::SkipEmptyParts);
|
|
|
|
if(strListIpList.size() < 1)
|
|
{
|
|
cout << "sitedown";
|
|
return false;
|
|
}
|
|
|
|
|
|
QSqlQuery sql;
|
|
//QString strQuery = "truncate table Proxy";
|
|
QString strQuery = "delete from Proxy";
|
|
QString strUtf8(strQuery.toUtf8());
|
|
|
|
if (sql.exec(strUtf8) == false)
|
|
{
|
|
cout << "error : " << sql.lastError().text().toStdString();
|
|
cout << endl << "dbdfail";
|
|
//emit finished();
|
|
//return;
|
|
}
|
|
|
|
|
|
foreach(QString str, strListIpList)
|
|
{
|
|
strQuery = "insert into Proxy set Proxy='";
|
|
strQuery += str.split(',').at(0).trimmed();
|
|
strQuery += "', Port=";
|
|
strQuery += str.split(',').at(1).trimmed();
|
|
strUtf8 = strQuery.toUtf8();
|
|
if (sql.exec(strUtf8) == false)
|
|
{
|
|
cout << "error : " << sql.lastError().text().toStdString();
|
|
cout << endl << "dbufail";
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
QList<QWebElement> SCrawler::FindAllMid(const QWebElement _FindElement,const QString _strElement,const QString _strAttrib,const QString _strFind, const int _strStart, const int _strLength)
|
|
{
|
|
QWebElementCollection elements = _FindElement.findAll(_strElement);
|
|
QList<QWebElement> returnElements = QList<QWebElement>();
|
|
|
|
foreach (QWebElement element, elements)
|
|
{
|
|
QString str = element.attribute(_strAttrib).trimmed().mid(_strStart,_strLength);
|
|
if (QString::compare(str,_strFind,Qt::CaseInsensitive)==0)
|
|
{
|
|
returnElements.append(element);
|
|
}
|
|
}
|
|
return returnElements;
|
|
}
|
|
|
|
void SCrawler::killProcess()
|
|
{
|
|
cout << endl << "timeout";
|
|
emit finished();
|
|
}
|
|
|
|
void SCrawler::SearchChildFrame(QWebFrame *frame)
|
|
{
|
|
Debug("c:/data/nordvpnloop.html", frame->toHtml());
|
|
foreach(QWebFrame *childFrame, frame->childFrames())
|
|
SearchChildFrame(childFrame);
|
|
}
|
|
|
|
QString SCrawler::addSource(QString _strIpList, QString _strSource)
|
|
{
|
|
QStringList strlistIpList = _strIpList.split("\n");
|
|
QStringList straddedList;
|
|
foreach(QString str, strlistIpList)
|
|
{
|
|
straddedList << (str + "," + _strSource);
|
|
}
|
|
return straddedList.join("\n").trimmed();
|
|
}
|
|
|
|
|