effectprocess 추가

git-svn-id: svn://192.168.0.12/source@305 8346c931-da38-4b9b-9d4c-e48b93cbd075
This commit is contained in:
admin
2016-10-27 10:11:49 +00:00
parent 0b075003a4
commit 6d62c97b54
31 changed files with 2417 additions and 27 deletions

View File

@@ -600,8 +600,8 @@ bool SCrawler::saveFrameUrl(QWebFrame *frame)
{
//static int cz = 0;
//Debug(frame->frameName() + QString::number(cz++) + ".html",frame->toHtml());
//QSqlQuery sql;
if (frame->frameName().compare(QString("BuddyConnectIframe")) == 0)
{
QWebElement profile = Find(frame->documentElement(),"div","class","profile_name");

View File

@@ -0,0 +1,32 @@
#-------------------------------------------------
#
# Project created by QtCreator 2016-10-25T17:10:10
#
#-------------------------------------------------
QT += webkitwidgets network widgets sql webkit
QT -= gui
TARGET = EffectProcess
CONFIG += console c++11
CONFIG -= app_bundle
TEMPLATE = app
SOURCES += main.cpp \
swebpage.cpp \
sutilclass.cpp \
sutilfunction.cpp \
sresultsender.cpp \
seffectprocess.cpp \
snaverblog.cpp
HEADERS += \
swebpage.h \
sresultsender.h \
sutilclass.h \
sutilfunction.h \
seffectprocess.h \
seffectstatistics.h \
snaverblog.h

File diff suppressed because it is too large Load Diff

35
EffectProcess/main.cpp Normal file
View File

@@ -0,0 +1,35 @@
#include <QCoreApplication>
#include <QSqlDatabase>
#include <QSettings>
#include <QString>
namespace
{
const QString DBNAME = "database";
}
int main(int argc, char *argv[])
{
QCoreApplication a(argc, argv);
a.setApplicationName(QString("Chrome"));
a.setApplicationVersion(QString("53.0.2785.143"));
QSettings settings(QString("effect.ini"), QSettings::IniFormat);
settings.setIniCodec("UTF-8");
settings.beginGroup("database");
QString strHostname = settings.value("host", "182.162.171.147").toString();
QString strUsername = settings.value("user", "admin").toString();
QString strPassword = settings.value("pass", "admin123").toString();
QString strDBName = settings.value("name", "bigbird").toString();
settings.endGroup();
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL", DBNAME);
db.setHostName(strHostname);
db.setUserName(strUsername);
db.setPassword(strPassword);
db.setDatabaseName(strDBName);
return a.exec();
}

View File

@@ -0,0 +1,33 @@
#include "seffectprocess.h"
#include "sutilclass.h"
#include "sutilfunction.h"
#include <iostream>
#include <QApplication>
namespace
{
const QString SEPERATOR = "!@#";
}
SEffectProcess::SEffectProcess(const QString& _platform_name, const QString& _platform_form,
const QString& _event_code, const QString& _url)
{
m_pParam[E_PARAM_URL] = _url;
m_pParam[E_PARAM_EVENT_CODE] = _event_code;
m_pParam[E_PARAM_PLATFORM_FORM] = _platform_form;
m_pParam[E_PARAM_PLATFORM_NAME] = _platform_name;
}
void SEffectProcess::slotOk()
{
std::cout << 'o' << SEPERATOR.toStdString() << m_pParam[E_PARAM_EVENT_CODE].toStdString() <<
SEPERATOR.toStdString() << m_pParam[E_PARAM_URL].toStdString();
QApplication::quit();
}
void SEffectProcess::slotError(E_ERROR_CODE _error, const QString &_msg)
{
std::cout << 'x' << SEPERATOR.toStdString() << m_pParam[E_PARAM_EVENT_CODE].toStdString()
<< SEPERATOR.toStdString() << m_pParam[E_PARAM_URL].toStdString() << EnumErrorCodetostrErrorCode(_error).toStdString()
<< SEPERATOR.toStdString() << _msg.toStdString();
QApplication::quit();
}

View File

@@ -0,0 +1,36 @@
#ifndef SEFFECTPROCESS
#define SEFFECTPROCESS
#include <QObject>
#include <QString>
#include "sutilclass.h"
enum E_PARAM
{
E_PARAM_URL = 0,
E_PARAM_EVENT_CODE,
E_PARAM_PLATFORM_FORM,
E_PARAM_PLATFORM_NAME,
E_PARAM_MAX
};
class SEffectProcess:public QObject
{
Q_OBJECT
public:
SEffectProcess(const QString& _platform_name, const QString& _platform_form,
const QString& _event_code, const QString& _url);
virtual void run() = 0;
protected slots:
void slotOk();
void slotError(E_ERROR_CODE _error, const QString& _msg);
private:
QString m_pParam[E_PARAM_MAX];
};
#endif // SEFFECTPROCESS

View File

@@ -0,0 +1,5 @@
#ifndef SEFFECTSTATISTICS
#define SEFFECTSTATISTICS
#endif // SEFFECTSTATISTICS

View File

@@ -0,0 +1,449 @@
#include "snaverblog.h"
#include <QUrl>
#include <QWebFrame>
#include <QTimer>
#include <QWebElement>
#include <QWebElementCollection>
#include <QChar>
namespace
{
const int TIME = 4000;//4 second
}
QWebElement Find(const QWebElement& _FindElement, const QString& _strElement = "",
const QString& _strAttrib = "" , const QString& _strFind = "")
{
QWebElementCollection elements = _FindElement.findAll(_strElement);
foreach (const QWebElement& element, elements)
{
if (element.attribute(_strAttrib) == _strFind)
{
return element;
}
}
QWebElement element;
return element;
}
QString GetSafeUtf(const QString& _strData)
{
QString str;
const QChar *pch = _strData.data();
for (int i = 0; i < _strData.length(); i++)
{
if (pch[i].unicode() >= 12593 && pch[i].unicode() <= 12622)
str += pch[i];
if (pch[i].unicode() >= 44032 && pch[i].unicode() <= 55203)
str += pch[i];
if (pch[i].isDigit() || pch[i].isNumber() || pch[i].isSpace() || pch[i].isLower() || pch[i].isUpper() || pch[i].isSymbol() )
str += pch[i];
}
return str;
}
class BoolController
{
public:
explicit BoolController(bool& _b):m_b(_b)
{
m_b = true;
}
~BoolController()
{
m_b = false;
}
private:
bool &m_b;
};
void unlock(QMutex* _pm)
{
_pm->unlock();
}
class Lock
{
public:
explicit Lock(QMutex* _pm):m_pMutex(_pm, unlock)
{
_pm->lock();
}
private:
std::shared_ptr<QMutex> m_pMutex;
};
SNaverBlog::SNaverBlog(std::shared_ptr<QWebPage> _page):m_pWebPage(_page),
m_eMode(E_CRAWL_MODE::BODY), m_bReplyProcessed(false), m_bBodyProcessed(false),
//m_bReplyProcessing(false), m_bBodyProcessing(false),
m_pNetworkRequest(new QNetworkRequest), m_nBodyRetry(3)
{
if (!_page)
_page = std::make_shared<QWebPage>(new QWebPage);
initConnect();
}
void SNaverBlog::initConnect()
{
QObject::connect(&(*m_pWebPage), &QWebPage::loadFinished,
this, &SNaverBlog::slotLoadFinished);
m_pWebPage->settings()->setAttribute(QWebSettings::AutoLoadImages, false);
}
void SNaverBlog::go(const QString &_url, E_CRAWL_MODE _mode)
{
m_eMode = _mode;
QUrl url(_url.trimmed());
if (_mode == E_CRAWL_MODE::BODY)
m_strUrl = _url.trimmed();
if (url.scheme().isEmpty())
url.setScheme("http");
m_pNetworkRequest->setUrl(url);
m_pNetworkRequest->setRawHeader(
"Accept-Language",
"ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4,zh-CN;q=0.2,zh;q=0.2"
);
m_pWebPage->mainFrame()->load(*m_pNetworkRequest);
}
void SNaverBlog::slotLoadFinished(bool ok)
{
if (!ok)
{
signalError(E_ERROR_CODE::UNKNOWN_ERROR, "UnkownError/BlockError/ProxyError");
return;
}
switch(m_eMode)
{
case E_CRAWL_MODE::BODY:
{
Lock lock(&m_mutexBody);
if (m_bBodyProcessed)
return;
m_efData.body = getBody();
if (!check(m_efData.body) && m_nBodyRetry-- > 0)
{
QTimer::singleShot(TIME, this, SLOT(slotLoadFinished(bool)));
return;
}
m_bBodyProcessed = true;
break;
}
case E_CRAWL_MODE::REPLY:
{
Lock lock(&m_mutexReply);
if (m_bReplyProcessed)
return;
m_efData.reply = getReply();
m_bReplyProcessed = true;
break;
}
}
emit signalDataOk(m_efData);
}
/*
void SNaverBlog::slotLoadFinished(bool ok)
{
if (!ok)
{
signalError(E_ERROR_CODE::UNKNOWN_ERROR, "UnkownError/BlockError/ProxyError");
return;
}
switch(m_eMode)
{
case E_CRAWL_MODE::BODY:
{
if (m_bBodyProcessing || m_bBodyProcessed)
return;
BoolController(m_bBodyProcessing);
m_efData.body = getBody();
if (!check(m_efData.body) && m_nBodyRetry-- > 0)
{
QTimer::singleShot(TIME, this, &SNaverBlog::slotLoadFinished);
return;
}
m_bBodyProcessed = true;
break;
}
case E_CRAWL_MODE::REPLY:
{
if (m_bReplyProcessed || m_bReplyProcessing)
return;
BoolController(m_bBodyProcessing);
m_efData.reply = getReply();
m_bReplyProcessed = true;
break;
}
}
emit signalDataOk();
}
*/
DataForm SNaverBlog::getBody()
{
DataForm data;
getBody(m_pWebPage->mainFrame(), data);
return data;
}
bool SNaverBlog::check(const DataForm &_data)
{
return true;
}
bool SNaverBlog::check(const QVector<DataForm> &_data)
{
return true;
}
QVector<DataForm> SNaverBlog::getReply()
{
QVector<DataForm> data;
return data;
}
void SNaverBlog::getBody(QWebFrame *frame, DataForm &_data)
{
if (frame->frameName().compare(QString("BuddyConnectIframe")) == 0)
{
QWebElement profile = Find(frame->documentElement(),"div","class","profile_name");
QString str = profile.toPlainText().split("\n").at(0);
if (str.isEmpty() == false)
_data.data[ARTICLE_URL] = m_strUrl;
}
if (frame->frameName().compare(QString("mainFrame")) == 0)
{
QString sympathy;
QString numofReply;
QString strProfile;
QWebElement proTitle = Find(frame->documentElement(),"meta","property","og:article:author");
_data.data[PLATFORM_TITLE] = proTitle.attribute("content").split("|").at(1).trimmed();
if(_data.data[PLATFORM_TITLE].length() > 0)
_data.data[PLATFORM_TITLE] = GetSafeUtf(_data.data[PLATFORM_TITLE]);
else
{
proTitle = Find(frame->documentElement(),"span","id","blogTitleName");
_data.data[PLATFORM_TITLE] = GetSafeUtf(proTitle.toPlainText().trimmed());
}
QWebElement image;
QWebElement profile = Find(frame->documentElement(),"div","id","blog-profile");
{
QWebElement nick = Find(profile,"strong","id","nickNameArea");
if (nick.toPlainText().isEmpty()==false)
_data.data[ARTICLE_NICKNAME] = nick.toPlainText();
if(_data.data[ARTICLE_NICKNAME].isEmpty())
{
QString strHtml = frame->toHtml();
QString strFind = "var nickName = '";
int start = strHtml.indexOf(strFind);
if (start == -1)
{
//cout << "error : nick name can not find and next again connect." << endl;
;
}
if (strHtml.at(start + strFind.length()) == QChar('\''))
{
//cout << "error : nick name can not find and next again connect." << endl;
;
}
else
{
int end = strHtml.indexOf("'",start + strFind.length());
_data.data[ARTICLE_NICKNAME] = strHtml.mid(start + strFind.length(),end-start-strFind.length());
}
}
_data.data[ARTICLE_NICKNAME] = GetSafeUtf(_data.data[ARTICLE_NICKNAME]);
if (m_strUrl.split("/").at(3).trimmed() == _data.data[ARTICLE_NICKNAME].trimmed())
{
_data.data[ARTICLE_ID] = _data.data[ARTICLE_NICKNAME];
}
else
{
if (_data.data[ARTICLE_ID].isEmpty())
{
if((m_strUrl.split("/").at(2).compare("blog.naver.com") == 0))
_data.data[ARTICLE_ID] = m_strUrl.split("/").at(3);
else
_data.data[ARTICLE_ID] = m_strUrl.split("/").at(2).split(".").at(0);
}
}
if(_data.data[ARTICLE_NICKNAME].length() == 0)
_data.data[ARTICLE_NICKNAME] = _data.data[ARTICLE_ID];
image = Find(profile,"img","alt","프로필 이미지");
strProfile = Find(profile, "p", "class", "caption align").toPlainText().trimmed();
}
{
QWebElement post = Find(frame->documentElement(),"div","id","postListBody");
{
QWebElement weCmt = post.findFirst("a[class^='pcol2 _cmtList']");
if (!weCmt.isNull())
{
numofReply = weCmt.toPlainText().replace(",", "").trimmed();
numofReply = numofReply.replace(QRegExp("[\\D]"), "");
}
}
QWebElement post_top = Find(post,"table","class","post-top");
{
QWebElement title = Find(post_top,"span","class","pcol1 itemSubjectBoldfont");
if(title.isNull())
{
title = Find(frame->documentElement(), "div", "class", "se_textView");
}
if(title.isNull())
{
title = Find(frame->documentElement(), "h3", "class", "se_textarea");
}
if (title.toPlainText().isEmpty()==false)
{
_data.data[ARTICLE_TITLE] = title.toPlainText();
_data.data[ARTICLE_TITLE] = GetSafeUtf(_data.data[ARTICLE_TITLE]);
}
}
{
QWebElement date = Find(post_top,"p","class","date fil5 pcol2 _postAddDate");
if(date.isNull())
{
date = Find(frame->documentElement(), "span","class","se_publishDate pcol2 fil5");
}
_data.data[ARTICLE_DATE] = date.toPlainText().trimmed().replace("/","-");
if ( _data.data[ARTICLE_DATE].isEmpty() == false)
{
_data.data[ARTICLE_DATE] += ":00";
}
}
{
//QWebElement body = Find(post,"div","class","post-view pcol2 _param(1)");
QWebElement body = post.findFirst("div[class^='post-view pcol2 _param(1)']");
if(body.isNull())
body = post.findFirst("div[class*='pcol2 _param(1)']");
if(body.isNull())
body = Find(post, "class", "se_component_wrap sect_dsc __se_component_area");
if (body.toPlainText().isEmpty()==false)
{
_data.data[ARTICLE_DATA] = body.toPlainText();
_data.data[ARTICLE_DATA] = GetSafeUtf(_data.data[ARTICLE_DATA]);
}
}
{
QWebElement WEsympathy = Find(frame->documentElement(),"div","class","btn_like pcol2");
if(WEsympathy.isNull())
{
sympathy = "0";
}
else
{
sympathy = WEsympathy.toPlainText().trimmed();
}
//qDebug() << "Sympathy: " << sympathy;
//qDebug() << strProfile;
}
{
//retry if profile is empty and sympathy is empty
/*
if((strProfile.isEmpty() || sympathy.isEmpty()) && (m_nRetryCount < RETRY_MAX))
{
m_nRetryCount++;
qDebug() << m_nRetryCount;
QTimer::singleShot(RETRY_INTERVAL, this, SLOT(reloadPage()));
return false;
}
*/
}
}
//QString strHead[E_DATA_MAX] = {"article_nickname","article_id","article_title","article_date","article_data","platform_title"};
if(image.attribute("src").trimmed().length() != 0)
{
_data.data[ARTICLE_PROFILEURL] = image.attribute("src").trimmed();
}
strProfile = GetSafeUtf(strProfile);
if(strProfile.length() > 0)
{
_data.data[ARTICLE_PROFILE] = strProfile.trimmed();
}
_data.data[REPLY_URL] = sympathy;
_data.data[ARTICLE_ORDER] = numofReply;
}
foreach(QWebFrame *childFrame, frame->childFrames())
getBody(childFrame, _data);
}
QString SNaverBlog::makeReplyUrl(const QString& _url)
{
QStringList strList = _url.split("/");
QString strOut = "";
try
{
if(strList.at(2).compare("blog.naver.com") == 0)
{
//strOut = _strUrl;
strOut = "http://blog.naver.com";
strOut += "/CommentList.nhn?blogId=";
strOut += strList.at(3);
strOut += "&logNo=";
strOut += strList.at(4);
strOut += "&currentPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false";
}
else //id.blog.me
{
strOut = "http://blog.naver.com/";
//strOut += strList.at(2).split(".").at(0);
//strOut += "/";
//strOut += strList.at(3);
strOut += "/CommentList.nhn?blogId=";
strOut += strList.at(2).split(".").at(0);
strOut += "&logNo=";
strOut += strList.at(3);
strOut += "&currentPage=&isMemolog=false&focusingCommentNo=&showLastPage=true&shortestContentAreaWidth=false";
}
}
catch (...)
{
emit signalError(E_ERROR_CODE::OUT_DATED_CRAWLER, "Check ReplyUrl\n" + strOut);
}
return strOut;
}

View File

@@ -0,0 +1,94 @@
#ifndef SNAVERBLOG
#define SNAVERBLOG
#include <QObject>
#include "sutilclass.h"
#include <QWebPage>
#include <QNetworkRequest>
#include <QMutex>
enum E_COLUMN
{
PLATFORM_NAME = 0,
PLATFORM_FORM,
PLATFORM_TITLE,
ARTICLE_FORM,
ARTICLE_PARENT,
ARTICLE_ID,
ARTICLE_NICKNAME,
ARTICLE_TITLE,
ARTICLE_DATA,
ARTICLE_URL,
ARTICLE_HIT,
ARTICLE_DATE,
ARTICLE_ORDER,
ARTICLE_PROFILE,
ARTICLE_PROFILEURL,
PLATFORM_ID,
KEYWORD_ID,
REPLY_URL,
ETC,
TOTAL_COUNT,
};
struct DataForm
{
QString data[TOTAL_COUNT];
};
struct EffectData
{
DataForm body;
QVector<DataForm> reply;
};
enum class E_CRAWL_MODE
{
BODY = 0,
REPLY
};
class SNaverBlog: public QObject
{
Q_OBJECT
public:
explicit SNaverBlog(std::shared_ptr<QWebPage> _page = nullptr);
void go(const QString& _url, E_CRAWL_MODE _mode = E_CRAWL_MODE::BODY);
private:
DataForm getBody();
QVector<DataForm> getReply();
QString makeReplyUrl(const QString& _url);
void initConnect();
bool check(const DataForm& _data);
bool check(const QVector<DataForm>& _data);
void getBody(QWebFrame* frame, DataForm& _data);
signals:
void signalDataOk(const EffectData& _data);
void signalError(E_ERROR_CODE _error, const QString& _msg);
public slots:
void slotLoadFinished(bool ok = true);
private:
std::shared_ptr<QWebPage> m_pWebPage;
std::unique_ptr<QNetworkRequest> m_pNetworkRequest;
E_CRAWL_MODE m_eMode;
bool m_bBodyProcessed;
bool m_bReplyProcessed;
QMutex m_mutexBody;
QMutex m_mutexReply;
/*
bool m_bBodyProcessing;
bool m_bReplyProcessing;
*/
EffectData m_efData;
QString m_strUrl;
int m_nBodyRetry;
};
#endif // SNAVERBLOG

View File

@@ -0,0 +1,19 @@
#ifndef SNAVERBLOGEFFECT
#define SNAVERBLOGEFFECT
#include "seffectprocess.h"
#include <memory>
#include <QWebPage>
#include "sresultsender.h"
class SNaverBlogEffectProcess: public SEffectProcess
{
Q_OBJECT
private:
std::shared_ptr<QWebPage> m_pWebPage;
std::shared_ptr<SResultSender> m_pResultSender;
};
#endif // SNAVERBLOGEFFECT

View File

@@ -0,0 +1,86 @@
#include "sresultsender.h"
#include "sutilfunction.h"
#include <QSqlDatabase>
#include <QSettings>
#include <QSqlError>
#include <QSqlQuery>
namespace
{
const QString SEPERATOR = "!@#";
}
class DBContainer
{
public:
DBContainer(QSqlDatabase& _db):m_db(_db)
{
}
~DBContainer()
{
m_db.close();
}
private:
QSqlDatabase &m_db;
};
namespace
{
const QString DBNAME = "effectprocess";
}
SResultSender::SResultSender(QObject* parent):QObject(parent)
{
init();
}
void SResultSender::init()
{
databaseSetting(DBNAME);
}
bool SResultSender::send(const State_s1_effect& _result)
{
QSqlDatabase db = QSqlDatabase::database(DBNAME);
if (!db.open())
{
emit error(E_ERROR_CODE::DB_OPEN_ERROR, db.lastError().text());
return false;
}
DBContainer container(db);
QString strQuery = "delete from stats_s1_effect where eventcode = " + QString::number(_result.event_num);
QSqlQuery query(db);
if (!query.exec(strQuery))
{
emit error(E_ERROR_CODE::DB_QUERY_ERROR, query.lastError().text() + SEPERATOR + query.lastQuery());
return false;
}
strQuery = "insert into stats_s1_effect ("
"event_num,view_buzz,replybuzz,replycount,likecount,viewcount,interactioncount,reachcount,engagementcount) "
"values "
"(:event_num,:view_buzz,:replybuzz,:replycount,:likecount,:viewcount,:interactioncount,:reachcount,:engagementcount)";
query.prepare(strQuery);
query.bindValue(":event_num", _result.event_num);
query.bindValue(":view_buzz", _result.view_buzz);
query.bindValue(":replybuzz", _result.replybuzz);
query.bindValue(":replycount", _result.replycount);
query.bindValue(":likecount", _result.likecount);
query.bindValue(":viewcount", _result.viewcount);
query.bindValue(":interactioncount", _result.interactioncount);
query.bindValue(":reachcount", _result.reachcount);
query.bindValue(":engagementcount", _result.engagementcount);
if (!query.exec())
{
emit error(E_ERROR_CODE::DB_QUERY_ERROR, query.lastError().text() + SEPERATOR + query.lastQuery());
return false;
}
return true;
}

View File

@@ -0,0 +1,41 @@
#ifndef SRESULTSENDER
#define SRESULTSENDER
#define FB(arg) #arg
#include <QString>
#include <QObject>
#include "sutilclass.h"
struct State_s1_effect
{
int event_num;
QString view_buzz;
QString replybuzz;
int replycount;
int likecount;
int viewcount;
int interactioncount;
int reachcount;
int engagementcount;
State_s1_effect():event_num(0),replycount(0),likecount(0),viewcount(0),
interactioncount(0),reachcount(0),engagementcount(0)
{
}
};
class SResultSender: public QObject
{
Q_OBJECT
public:
SResultSender(QObject *parent = nullptr);
bool send(const State_s1_effect& _result);
signals:
void error(E_ERROR_CODE _error, const QString& _error_msg);
private:
void init();
};
#endif // SUTILCLASS

View File

@@ -0,0 +1,77 @@
#include "sutilclass.h"
STableWidgetItem::STableWidgetItem(int type): QTableWidgetItem(type)
{
}
STableWidgetItem::STableWidgetItem(const QIcon &icon, const QString &text, int type):
QTableWidgetItem(icon, text, type)
{
}
STableWidgetItem::STableWidgetItem(const QString &text, int type):
QTableWidgetItem(text, type)
{
}
STableWidgetItem::STableWidgetItem(const QTableWidgetItem &other):
QTableWidgetItem(other)
{
}
bool STableWidgetItem::operator <(const QTableWidgetItem &other) const
{
bool ok1, ok2;
int nThis = text().toInt(&ok1);
int nOther = other.text().toInt(&ok2);
if (ok1 && ok2)
{
return nThis < nOther;
}
else
{
return text() < other.text();
}
}
SError::SError(): company_num(-1), platformname_num(-1)
{
}
QString ERROR_MESSAGE[static_cast<int>(E_ERROR_CODE::MAX)] =
{
"DB_OPEN_ERROR",
"DB_FULL_ERROR",
"DB_LONG_QUERY_ERROR",
"DB_QUERY_ERROR",
"DB_UNKNOWN_ERROR",
"INTERNET_ERROR",
"OUT_DATED_CRAWLER",
"DELETED_URL_ERROR",
"BLOCK_ERROR",
"TIMEOUT",
"NO_PROGRAM",
"UNKNOWN_ERROR",
};
QString ERROR_MESSAGE_CODE[static_cast<int>(E_ERROR_CODE::MAX)] =
{
"e000",
"e001",
"e002",
"e003",
"e004",
"e005",
"e006",
"e007",
"e008",
"e009",
"e010",
"e011",
};
const QString SEPERATOR = "!@#";

166
EffectProcess/sutilclass.h Normal file
View File

@@ -0,0 +1,166 @@
#ifndef SUTILCLASS
#define SUTILCLASS
#include <QTableWidgetItem>
#include <QDate>
#include <QString>
#include <memory>
class STableWidgetItem : public QTableWidgetItem
{
public:
STableWidgetItem(int type = Type);
STableWidgetItem(const QString & text, int type = Type);
STableWidgetItem(const QIcon & icon, const QString & text, int type = Type);
STableWidgetItem(const QTableWidgetItem & other);
virtual bool operator<(const QTableWidgetItem & other) const override;
};
enum class E_RUN_MODE:unsigned short
{
MODE_WAIT = 0,
MODE_TERMINATING,
MODE_RUN
};
enum class E_CRAWL_STATE:unsigned short
{
OK = 0,
FAIL,
NO
};
enum class E_ERROR_CODE:unsigned short
{
DB_OPEN_ERROR = 0,
DB_FULL_ERROR,
DB_LONG_QUERY_ERROR,
DB_QUERY_ERROR,
DB_UNKNOWN_ERROR,
INTERNET_ERROR,
OUT_DATED_CRAWLER,
DELETED_URL_ERROR,
BLOCK_ERROR,
TIMEOUT,
NO_PROGRAM,
UNKNOWN_ERROR,
MAX
};
struct SReport
{
E_CRAWL_STATE state;
unsigned int company_num;
unsigned int platformname_num;
QString event_code;
QString url;
};
struct SReportSummary
{
unsigned int nOk;
unsigned int nError;
unsigned int nActiveEventCode;
unsigned int nAllGroupEvent;
unsigned int nTotal;
SReportSummary(unsigned int _ok = 0, unsigned int _error = 0,
unsigned int _active_event_code = 0, unsigned int _all_group_event = 0,
unsigned int _total = 0
):nOk(_ok), nError(_error), nActiveEventCode(_active_event_code),
nAllGroupEvent(_all_group_event), nTotal(_total) {}
};
struct SError
{
E_CRAWL_STATE state;
unsigned int company_num;
unsigned int platformname_num;
QString event_code;
QString url;
E_ERROR_CODE error_code;
QString str_error;
SError();
SError(E_ERROR_CODE _error_code): state(E_CRAWL_STATE::FAIL), company_num(-1), platformname_num(-1), error_code(_error_code)
{
}
template <typename T>
SError(E_ERROR_CODE _error_code, T&& _str_error): state(E_CRAWL_STATE::FAIL),
company_num(-1), platformname_num(-1),
error_code(_error_code), str_error(std::forward<T>(_str_error))
{
}
template <typename A, typename B, typename C>
SError(E_CRAWL_STATE _state, unsigned int _company_num, unsigned int _platformname_num,
A&& _event_code, B&& _url, E_ERROR_CODE _error_code, C&& _str_error):
state(_state), company_num(_company_num), platformname_num(_platformname_num),
event_code(std::forward<A>(_event_code)), url(std::forward<B>(_url)), error_code(_error_code),
str_error(std::forward<C>(_str_error))
{
}
};
struct SEffectRow
{
int company_num;
QString company_name;
QDate company_startdate;
QDate company_enddate;
int event_num;
QDate event_date;
QString url;
QString eventcode;
int platformname_num;
SEffectRow():company_num(-1), event_num(-1), platformname_num(-1){}
template <typename A, typename B, typename C, typename D, typename E, typename F>
SEffectRow(int _company_num, A&& _company_name, B&& _company_startdate, C&& _company_enddate, int _event_num,
D&& _event_date, E&& _url, F&& _eventcode, int _platformname_num):
company_num(_company_num), company_name(std::forward<A>(_company_name)), company_startdate(std::forward<B>(_company_startdate)),
company_enddate(std::forward<C>(_company_enddate)), event_num(_event_num), event_date(std::forward<D>(_event_date)),
url(std::forward<E>(_url)), eventcode(std::forward<F>(_eventcode)), platformname_num(_platformname_num)
{
}
};
enum class E_PLATFORM
{
NAVER_BLOG = 0,
NAVER_CAFE,
NAVER_NEWS,
DAUM_CAFE,
DAUM_BLOG,
DAUM_NEWS,
INSTAGRAM,
KAKAOSTORY,
FACEBOOK
};
enum class E_INFO
{
OK = 0,
ERROR,
ACTIVE_EVENT,
ALL_ACTIVE_GROUP,
TOTAL
};
enum E_LOG_COLUMN
{
STATUS = 0,
EVENT_CODE,
URL,
ERROR_CODE,
ERROR_MSG
};
#endif // SUTILCLASS

View File

@@ -0,0 +1,71 @@
#include "sutilfunction.h"
#include <QString>
#include <QSettings>
#include <QSqlDatabase>
#include <QMessageBox>
#include <QRegExp>
extern QString ERROR_MESSAGE[];
extern QString ERROR_MESSAGE_CODE[];
void databaseSetting(const QString &str)
{
QSettings settings(QString("effect.ini"), QSettings::IniFormat);
settings.setIniCodec("UTF-8");
settings.beginGroup("database");
QString strHostName = settings.value("host", "182.162.171.147").toString();
QString strUser = settings.value("user", "admin").toString();
QString strPass = settings.value("pass", "admin123").toString();
QString strDbName = settings.value("name", "bigbird").toString();
settings.endGroup();
QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL", str);
db.setHostName(strHostName);
db.setDatabaseName(strDbName);
db.setUserName(strUser);
db.setPassword(strPass);
}
void messageBoxError(const QString &title, const QString &body)
{
QMessageBox messagebox(QMessageBox::Critical, title, body);
messagebox.exec();
}
QString errorCodeToString(E_ERROR_CODE _error_code)
{
if (0 <= static_cast<int>(_error_code) &&
static_cast<int>(_error_code) < static_cast<int>(E_ERROR_CODE::MAX))
return ERROR_MESSAGE[static_cast<int>(_error_code)];
else
return ERROR_MESSAGE[static_cast<int>(E_ERROR_CODE::UNKNOWN_ERROR)];
}
E_ERROR_CODE strErrorCodeToEnumErrorCode(const QString& _str)
{
static QRegExp re("^[\\D]");
QString str = _str.trimmed().replace(re, "");
bool ok = false;
int n = str.toInt(&ok);
if (ok && (0 <= n && n < static_cast<int>(E_ERROR_CODE::MAX)))
{
return static_cast<E_ERROR_CODE>(n);
}
else
{
return E_ERROR_CODE::UNKNOWN_ERROR;
}
}
QString EnumErrorCodetostrErrorCode(E_ERROR_CODE _code)
{
if (0 <= static_cast<int>(_code) &&
static_cast<int>(_code) < static_cast<int>(E_ERROR_CODE::MAX))
return ERROR_MESSAGE_CODE[static_cast<int>(_code)];
else
return ERROR_MESSAGE_CODE[static_cast<int>(E_ERROR_CODE::UNKNOWN_ERROR)];
}

View File

@@ -0,0 +1,12 @@
#ifndef SUTILFUNCTION
#define SUTILFUNCTION
#include "sutilclass.h"
class QString;
void databaseSetting(const QString& str);
void messageBoxError(const QString& title, const QString &body);
QString errorCodeToString(E_ERROR_CODE _error_code);
E_ERROR_CODE strErrorCodeToEnumErrorCode(const QString& _str);
QString EnumErrorCodetostrErrorCode(E_ERROR_CODE _code);
#endif // SUTILFUNCTION

View File

@@ -0,0 +1,12 @@
#include "swebpage.h"
#include <iostream>
SWebPage::SWebPage(QObject *parent): QWebPage(parent)
{
}
void SWebPage::javaScriptAlert(QWebFrame *frame, const QString &msg)
{
emit signalAlert(msg);
}

22
EffectProcess/swebpage.h Normal file
View File

@@ -0,0 +1,22 @@
#ifndef SWEBPAPGE
#define SWEBPAPGE
#include <QWebPage>
class SWebPage:public QWebPage
{
Q_OBJECT
public:
SWebPage(QObject * parent = 0);
protected:
void javaScriptAlert(QWebFrame * frame, const QString & msg) override;
//bool javaScriptConfirm(QWebFrame * frame, const QString & msg){}
signals:
void signalAlert(const QString& msg);
};
#endif // SWEBPAPGE

View File

@@ -21,7 +21,6 @@ SOURCES += main.cpp\
srunninglistwidget.cpp \
stimer.cpp \
sadddelwidget.cpp \
srunprocess.cpp \
sparentprocess.cpp \
scompanyprocess.cpp \
serrorprocess.cpp \
@@ -36,7 +35,6 @@ HEADERS += widget.h \
srunninglistwidget.h \
stimer.h \
sadddelwidget.h \
srunprocess.h \
sparentprocess.h \
scompanyprocess.h \
serrorprocess.h \

View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2016-10-21T19:00:52. -->
<!-- Written by QtCreator 3.3.0, 2016-10-27T19:07:05. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>

View File

@@ -71,6 +71,9 @@ void SEffectLogic::initConnect()
this, &SEffectLogic::slotCompleteCompany);
QObject::connect(m_pSRunningProcess, &SRunningProcess::signalCompanyOutDated,
this, &SEffectLogic::slotCompanyOutdated);
m_pSRunningListWidget->enableBtnForceStop(false);
m_pSRunningListWidget->enableBtnTimerStop(false);
}
@@ -228,6 +231,9 @@ void SEffectLogic::slotTerminateNormal(const QVector<SEffectRow>& _error,
const QMap<int, SReportSummary>& _report)
{
qDebug() << "slotTerminateNormal";
m_pSLogWidget->writeLog("Number of Erros : " + QString::number(_error.size()));
m_pSLogWidget->writeLog(" ");
for (auto iterPos = _report.cbegin(); iterPos != _report.cend(); ++iterPos)
{
QString str;
@@ -244,17 +250,22 @@ void SEffectLogic::slotTerminateNormal(const QVector<SEffectRow>& _error,
if ((_error.size() > 0) && (nRetry-- > 0))
{
m_pSLogWidget->writeLog("Error Retry : " + QString::number(nRetry + 1));
m_pSLogWidget->writeLog("Start Error Process");
m_pSRunningProcess->setList(_error);
m_pSRunningProcess->start();
}
else
{
m_pSAddDelWidget->enableBtnAdd(true);
m_pSAddDelWidget->enableBtnDel(true);
if (!m_pSTimer->isActive())
{
m_pSRunningListWidget->enableBtnForceStart(true);
m_pSRunningListWidget->enableBtnForceStop(false);
m_pSRunningListWidget->enableBtnTimerStart(true);
m_pSRunningListWidget->enableBtnTimerStop(false);
}
m_pSAddDelWidget->enableBtnAdd(true);
m_pSAddDelWidget->enableBtnDel(true);
nRetry = RETRY;
m_pSLogWidget->writeLog("Complete");
}

View File

@@ -26,7 +26,7 @@ void SLogWidget::writeLog(const QString& _strLog)
m_plwLog->addItem(strOut.left(1024));
QDate date = QDate::currentDate();
QFile file(date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
QFile file("effect_" + date.toString(Qt::ISODate)+ "_" + QString::number(QCoreApplication::applicationPid())+ ".log");
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append))
return;

View File

@@ -8,8 +8,27 @@ extern QString ERROR_MESSAGE[];
namespace
{
#if defined(Q_OS_WIN32)
const QString APPNAME = "effectprocess.exe";
const QString PYTHON = "python";
#else
const QString APPNAME = "./effectprocess";
const QString PYTHON = "python3";
#endif
const int TIMEOUT = 180; // 3 minutes
const QString SPLIT_KEYWORD = "!@#";
const QString PYTAPPNAME = "effectprocess.py";
const QString NAVER_BLOG_1 = "blog.naver.com/";
const QString NAVER_BLOG_2 = ".blog.me/";
const QString NAVER_CAFE = "cafe.naver.com/";
const QString NAVER_NEWS = "news.naver.com";
const QString DAUM_CAFE = "cafe.daum.net/";
const QString INSTAGRAM = "instagram.com/";
const QString KAKAOSTORY = "story.kakao.com/";
const QString FACEBOOK = "www.facebook.com/";
}
@@ -20,6 +39,8 @@ SParentProcess::SParentProcess():m_pProcess(new QProcess(this)), m_bRunning(fals
this, SLOT(finishedProcess(int, QProcess::ExitStatus)));
QObject::connect(this, &SParentProcess::signalTerminateError,
this, &SParentProcess::slotTerminateError);
QObject::connect(m_pProcess, SIGNAL(error(QProcess::ProcessError)),
this, SLOT(finishedProcess(QProcess::ProcessError)));
}
void SParentProcess::clear()
@@ -71,20 +92,63 @@ void SParentProcess::setModeWait()
QStringList SParentProcess::getProcessArguments(const SEffectRow& _row) const
{
return QStringList();
QStringList slargv;
if (_row.url.contains(NAVER_BLOG_1, Qt::CaseInsensitive) ||
_row.url.contains(NAVER_BLOG_2, Qt::CaseInsensitive) ||
_row.url.contains(NAVER_CAFE, Qt::CaseInsensitive) ||
_row.url.contains(NAVER_NEWS, Qt::CaseInsensitive) ||
_row.url.contains(DAUM_CAFE, Qt::CaseInsensitive))
{
slargv.append(APPNAME);
}
else
{
slargv << PYTHON << PYTAPPNAME;
}
QStringList SParentProcess::getProcessArguments(SEffectRow&& _row) const
if (_row.url.contains(NAVER_BLOG_1, Qt::CaseInsensitive) || _row.url.contains(NAVER_BLOG_2, Qt::CaseInsensitive))
{
return QStringList();
slargv << "naver" << "blog";
}
else if (_row.url.contains(NAVER_CAFE, Qt::CaseInsensitive))
{
slargv << "naver" << "cafe";
}
else if (_row.url.contains(NAVER_NEWS, Qt::CaseInsensitive))
{
slargv << "naver" << "news";
}
else if (_row.url.contains(DAUM_CAFE, Qt::CaseInsensitive))
{
slargv << "daum" << "cafe";
}
else if (_row.url.contains(INSTAGRAM, Qt::CaseInsensitive))
{
slargv << "instagram";
}
else if (_row.url.contains(KAKAOSTORY, Qt::CaseInsensitive))
{
slargv << "kakaostory";
}
else if (_row.url.contains(FACEBOOK, Qt::CaseInsensitive))
{
slargv << "facebook";
}
slargv << _row.eventcode << _row.url;
return slargv;
}
void SParentProcess::runProcess(QStringList _arguments)
{
//qDebug () << _arguments;
if (_arguments.size() > 0)
{
QString arg = _arguments.at(0);
_arguments.removeFirst();
qDebug() << arg;
qDebug() << _arguments;
m_pProcess->start(arg, _arguments);
}
@@ -110,6 +174,7 @@ void SParentProcess::setError(SError&& _error)
void SParentProcess::stop()
{
m_bUserAbort = true;
killProcess();
}
void SParentProcess::update()
@@ -165,8 +230,53 @@ void SParentProcess::update()
}
}
void SParentProcess::finishedProcess(QProcess::ProcessError _error)
{
if (m_bUserAbort)
{
setInnerMode(E_INNER_RUN_MODE::MODE_WAIT_HANGING);
emit signalUserAbort();
return;
}
QProcess *pPro = (QProcess*)sender();
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
SError analyzedResult = analyzeStdOut(str.trimmed());
switch (_error)
{
case QProcess::FailedToStart:
{
analyzedResult.error_code = E_ERROR_CODE::NO_PROGRAM;
break;
}
}
QString strLog = makeLog(analyzedResult);
if (analyzedResult.state == E_CRAWL_STATE::OK)
{
if (!m_mapReportSummary.contains(analyzedResult.company_num))
m_mapReportSummary.insert(analyzedResult.company_num, SReportSummary());
m_mapReportSummary[analyzedResult.company_num].nOk++;
}
else
{
if (!m_mapReportSummary.contains(analyzedResult.company_num))
m_mapReportSummary.insert(analyzedResult.company_num, SReportSummary());
m_mapReportSummary[analyzedResult.company_num].nError++;
m_vecError.append(m_erCurrentRow);
}
emit signalLog(strLog);
emit signalTerminateEachEvent(analyzedResult.company_num, analyzedResult.state);
setInnerMode(E_INNER_RUN_MODE::MODE_RUN_EVENTCODE);
}
void SParentProcess::finishedProcess(int exitCode, QProcess::ExitStatus exitStatus)
{
qDebug() << "finishedProcess";
if (m_bUserAbort)
{
setInnerMode(E_INNER_RUN_MODE::MODE_WAIT_HANGING);
@@ -177,7 +287,7 @@ void SParentProcess::finishedProcess(int exitCode, QProcess::ExitStatus exitStat
QThread::msleep(100);
QString str = pPro->readAllStandardOutput();
SError analyzedResult = analyzeStdOut(str);
SError analyzedResult = analyzeStdOut(str.trimmed());
QString strLog = makeLog(analyzedResult);
if (analyzedResult.state == E_CRAWL_STATE::OK)
@@ -220,7 +330,6 @@ SError SParentProcess::analyzeStdOut(const QString& _str)
return errorResult;
}
// unknown type log
// This result is treated like unknown error.
else if (_str.trimmed().at(0) != 'o' && _str.trimmed().at(0) != 'O' &&
@@ -315,6 +424,13 @@ SError SParentProcess::analyzeStdOut(const QString& _str)
}
}
void SParentProcess::killProcess()
{
//if (m_pProcess->state() != QProcess::NotRunning)
m_pProcess->kill();
qDebug() << "kill processed";
}
QString SParentProcess::makeLog(const SError& _error)
{
@@ -363,6 +479,6 @@ void SParentProcess::setCurrentEffectRow(const SEffectRow& _row)
void SParentProcess::setCurrentEffectRow(SEffectRow&& _row)
{
m_erCurrentRow = _row;
//m_erCurrentRow = _row;
m_erCurrentRow = std::move(_row);
}

View File

@@ -44,15 +44,15 @@ public:
bool isWaiting() const;
void setModeWait();
private:
SError analyzeStdOut(const QString& _str);
QString makeLog(const SError& _error);
void killProcess();
protected:
QStringList getProcessArguments(const SEffectRow& _row) const;
QStringList getProcessArguments(SEffectRow&& _row) const;
//QStringList getProcessArguments(SEffectRow&& _row) const;
void setError(const SError& _error);
void setError(SError&& _error);
@@ -67,7 +67,7 @@ public slots:
void stop();
void update();
void finishedProcess(int exitCode, QProcess::ExitStatus exitStatus);
void finishedProcess(QProcess::ProcessError _error);
signals:
void signalUserAbort();

View File

@@ -37,6 +37,7 @@ public:
void start(QVector<T>&& _list);
SError gerError();
public slots:
void stop();
void start();
@@ -44,7 +45,6 @@ public slots:
QVector<SError> getErrorList();
private slots:
void update();

View File

@@ -1,15 +1,48 @@
#include "stimer.h"
#include <QTimer>
#include <QTime>
#include <QSettings>
#include <QDebug>
namespace
{
const int TIME = 300000; // 5 minute
//const int TIME = 300000; // 5 minute
const int TIME = 30000;
}
STimer::STimer(): m_pTimer(new QTimer), m_nPreviousHour(-1)
{
m_pTimer->setInterval(TIME);
connect(&(*m_pTimer), &QTimer::timeout, this, &STimer::checkTimeout);
init();
}
void STimer::init()
{
QSettings settings(QString("effect.ini"), QSettings::IniFormat);
settings.setIniCodec("UTF-8");
settings.beginGroup("timer");
QString strTimes = settings.value("time", "1").toString();
if (strTimes.trimmed().isEmpty())
strTimes = "1";
qDebug() << strTimes;
settings.endGroup();
QStringList slTimes = strTimes.split(QChar('.'), QString::SkipEmptyParts);
qDebug() << "hahaha";
qDebug() << slTimes.size();
bool ok;
int nTime;
foreach (auto& strTime, slTimes)
{
qDebug() << strTime;
nTime = strTime.toInt(&ok);
if (ok)
set(nTime);
}
}
STimer::~STimer()
@@ -86,7 +119,15 @@ void STimer::checkTimeout()
QTime now = QTime::currentTime();
bool bChangedHour = (m_nPreviousHour == now.hour()) ? false : true;
m_nPreviousHour = now.hour();
qDebug() << "timer";
qDebug() << "bChangedHour : " << bChangedHour;
for (auto iter = m_setTime.begin(); iter != m_setTime.end(); ++iter)
qDebug() << *iter;
if (bChangedHour && m_setTime.contains(now.hour()))
{
qDebug() << "TimeOut condition satisfied";
emit timeout();
}
}

View File

@@ -31,10 +31,14 @@ public:
void unset(int time);
void unset(QVector<int> times);
int timerId() const;
~STimer();
private:
void init();
signals:
void timeout();

View File

@@ -57,3 +57,21 @@ QString ERROR_MESSAGE[static_cast<int>(E_ERROR_CODE::MAX)] =
"NO_PROGRAM",
"UNKNOWN_ERROR",
};
QString ERROR_MESSAGE_CODE[static_cast<int>(E_ERROR_CODE::MAX)] =
{
"e000",
"e001",
"e002",
"e003",
"e004",
"e005",
"e006",
"e007",
"e008",
"e009",
"e010",
"e011",
};

View File

@@ -7,10 +7,12 @@
#include <QRegExp>
extern QString ERROR_MESSAGE[];
extern QString ERROR_MESSAGE_CODE[];
void databaseSetting(const QString &str)
{
QSettings settings("effect.ini");
QSettings settings(QString("effect.ini"), QSettings::IniFormat);
settings.setIniCodec("UTF-8");
settings.beginGroup("database");
QString strHostName = settings.value("host", "182.162.171.147").toString();
@@ -58,3 +60,12 @@ E_ERROR_CODE strErrorCodeToEnumErrorCode(const QString& _str)
return E_ERROR_CODE::UNKNOWN_ERROR;
}
}
QString EnumErrorCodetostrErrorCode(E_ERROR_CODE _code)
{
if (0 <= static_cast<int>(_code) &&
static_cast<int>(_code) < static_cast<int>(E_ERROR_CODE::MAX))
return ERROR_MESSAGE_CODE[static_cast<int>(_code)];
else
return ERROR_MESSAGE_CODE[static_cast<int>(E_ERROR_CODE::UNKNOWN_ERROR)];
}

View File

@@ -7,6 +7,6 @@ void databaseSetting(const QString& str);
void messageBoxError(const QString& title, const QString &body);
QString errorCodeToString(E_ERROR_CODE _error_code);
E_ERROR_CODE strErrorCodeToEnumErrorCode(const QString& _str);
QString EnumErrorCodetostrErrorCode(E_ERROR_CODE _code);
#endif // SUTILFUNCTION

View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.3.0, 2016-06-07T15:27:13. -->
<!-- Written by QtCreator 3.3.0, 2016-10-25T11:19:44. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>