diff --git a/CrawlerList/smanage.cpp b/CrawlerList/smanage.cpp index 1300977..e68ef4b 100644 --- a/CrawlerList/smanage.cpp +++ b/CrawlerList/smanage.cpp @@ -120,6 +120,7 @@ void SManage::CheckLast() { m_bFinalLast = true; m_pMain->InsertLog("Finish Crawling :)"); + m_pMain->SetCrawlingState("Finish"); } m_ncList=1; } diff --git a/CrawlerList/snavercafemanage.cpp b/CrawlerList/snavercafemanage.cpp index d5cfb62..3665b06 100644 --- a/CrawlerList/snavercafemanage.cpp +++ b/CrawlerList/snavercafemanage.cpp @@ -100,11 +100,6 @@ void SNaverCafeManage::processFinished(QProcess *_pPro,QString _strOut) m_bLast = false; if (_strOut.right(4) == "last" || m_ncList >= 1000) m_bLast = true; - -// if (_strOut.right(5) == "block") - // //m_bFinalLast = true; - - if (_strOut.right(5) == "block") { ReLoadList(); diff --git a/CrawlerList/widget.cpp b/CrawlerList/widget.cpp index 0d4303e..32462ed 100644 --- a/CrawlerList/widget.cpp +++ b/CrawlerList/widget.cpp @@ -6,54 +6,23 @@ #include #include #include +#include #include #include "snavercafemanage.h" #include "snaverblogmanage.h" #include "sdaumcafemanage.h" -Widget::Widget(QWidget *parent) : QWidget(parent) + +Widget::Widget(QWidget *parent) : QWidget(parent) , m_nMode(E_MODE_WAIT) { connect(&m_timer, SIGNAL(timeout()), this, SLOT(Update())); - - m_pcb = new QComboBox; - m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent); - - QPushButton *pbtRefresh = new QPushButton("Refresh",this); - QPushButton *pbtStart = new QPushButton("Start",this); - QPushButton *pbtStop = new QPushButton("Stop",this); - { - QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton())); - QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton())); - QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton())); - } - QVBoxLayout *vlayout = new QVBoxLayout; { - QHBoxLayout *hlayout = new QHBoxLayout; - hlayout->addWidget(m_pcb,Qt::AlignLeft); - hlayout->addWidget(pbtRefresh); - vlayout->addLayout(hlayout); - } - - m_pedTime = new QLineEdit(this); - m_pedTime->setText(QString("3~10")); - { - QHBoxLayout *hlayout = new QHBoxLayout; - hlayout->addWidget(m_pedTime); - hlayout->addWidget(pbtStart); - hlayout->addWidget(pbtStop); - vlayout->addLayout(hlayout); - } - - { - QHBoxLayout *hlayout = new QHBoxLayout; m_pResultList = new QListWidget; - hlayout->addWidget(m_pResultList); - vlayout->addLayout(hlayout); - } - + vlayout->addWidget(setRealGroupWidgets()); + vlayout->addWidget(m_pResultList); + } setLayout(vlayout); - m_pNaverCafe = new SNaverCafeManage(this); m_pNaverBlog = new SNaverBlogManage(this); m_pDaumCafe = new SDaumCafeManage(this); @@ -76,12 +45,65 @@ Widget::Widget(QWidget *parent) : QWidget(parent) m_pManage[i]->SetParent(this); RefreshButton(); + m_timer.start(1000); } Widget::~Widget() -{ +{ + if (m_nMode == E_MODE_RUN) + { + if (m_pgbManual->isChecked() == false) + { + if (m_strCrawlingID.isEmpty() == false) + { + SetCrawlingState("Finish"); + } + } + } + UnLockTables(); m_db.close(); - m_pNaverCafe->SetParent(0); + for (int i = 0 ; i < C_CRAWLER_MAX ; i++) + m_pManage[i]->SetParent(0); + +} + +QGroupBox *Widget::setRealGroupWidgets() +{ + m_pcb = new QComboBox; + m_pcb->setInsertPolicy(QComboBox::InsertAtCurrent); + + QPushButton *pbtRefresh = new QPushButton("Refresh",this); + QPushButton *pbtStart = new QPushButton("Start",this); + QPushButton *pbtStop = new QPushButton("Stop",this); + { + QObject::connect(pbtStart,SIGNAL(clicked()),this,SLOT(StartButton())); + QObject::connect(pbtStop,SIGNAL(clicked()),this,SLOT(StopButton())); + QObject::connect(pbtRefresh,SIGNAL(clicked()),this,SLOT(RefreshButton())); + } + + QVBoxLayout *vlayout = new QVBoxLayout; + { + QHBoxLayout *hlayout = new QHBoxLayout; + hlayout->addWidget(m_pcb,Qt::AlignLeft); + hlayout->addWidget(pbtRefresh); + vlayout->addLayout(hlayout); + } + + m_pedTime = new QLineEdit(this); + m_pedTime->setText(QString("3")); + { + QHBoxLayout *hlayout = new QHBoxLayout; + hlayout->addWidget(m_pedTime); + hlayout->addWidget(pbtStart); + hlayout->addWidget(pbtStop); + vlayout->addLayout(hlayout); + } + + m_pgbManual = new QGroupBox(tr("Manual")); + m_pgbManual->setCheckable(true); + m_pgbManual->setChecked(true); + m_pgbManual->setLayout(vlayout); + return m_pgbManual; } void Widget::InsertLog(QString str) @@ -108,55 +130,43 @@ void Widget::InsertLog(QString str) m_pResultList->setCurrentRow( m_pResultList->count() - 1 ); m_pResultList->repaint(); } -/* -void Widget::InsertLog(int _nSelect,QString str) + +void Widget::SetCrawlingState(QString _str) { - if (_nSelect >= C_CRAWLER_MAX) return; - - QTime time = QTime::currentTime(); - QString strOut = time.toString("[hh:mm:ss] ") + str; - m_aResultList[_nSelect].addItem(strOut); - QDate date = QDate::currentDate(); - - QFile file(date.toString(Qt::ISODate)+"_"+QString::number(_nSelect)+".log"); - if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append)) - return; - - QTextStream out(&file); - out << strOut << "\n"; - file.close(); - - if (m_aResultList[_nSelect].count() > 1024) - { - m_aResultList[_nSelect].removeItemWidget(m_aResultList[_nSelect].item(0)); - QListWidgetItem* item = m_aResultList[_nSelect].takeItem(0); - delete item; - } - m_aResultList[_nSelect].setCurrentRow( m_aResultList[_nSelect].count() - 1 ); - m_aResultList[_nSelect].repaint(); -} -*/ - -void Widget::StartButton() -{ - m_timer.stop(); - m_timer.start(1000); - QSqlQuery query; - query.exec("UPDATE crawling set state = '" + QString("run") + "' where id = '" + m_pcb->currentData().toString() + "'"); - query.exec("SELECT _keyword.start,_keyword.end, _keyword.searches,_keyword.authorship,_keyword.id,_datagroup.id,_keyword.platform " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id " - "where _crawling.id = '" + m_pcb->currentData().toString() + "'"); + if (_str == QString("Finish")) + { + if (m_pgbManual->isChecked()) + query.exec("UPDATE crawling set state = 2 where id = '" + m_strCrawlingID + "'"); + else + query.exec("UPDATE crawling set state = 0 where id = '" + m_strCrawlingID + "'"); + m_pgbManual->setEnabled(true); + m_nMode = E_MODE_WAIT; + } + if (_str == QString("Start")) + { + if (m_pgbManual->isChecked() == false) + query.exec("UPDATE crawling set state = 1 where id = '" + m_strCrawlingID + "'"); + m_pgbManual->setEnabled(false); + } +} + +void Widget::Start() +{ + QSqlQuery query; + query.exec("SELECT keyword.start,keyword.end, keyword.searches,keyword.authorship,keyword.id,datagroup.id,keyword.platform " + "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id " + "inner join datagroup on crawling.datagroup_id = datagroup.id " + "where crawling.id = '" + m_strCrawlingID + "'"); query.next(); + /* QString str = + " , "; str += query.value(1).toString() + " ~ "; str += query.value(2).toString() + " -> "; str += query.value(3).toString(); + */ m_nPlatform = 0; - - if(0 <= query.value(6).toInt() && query.value(6).toInt() < C_CRAWLER_MAX) m_nPlatform = query.value(6).toInt(); else @@ -165,24 +175,56 @@ void Widget::StartButton() return; } - m_pManage[m_nPlatform]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"), - QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"), - query.value(2).toString().trimmed(),// keyword - query.value(3).toString().trimmed(),// authorship - query.value(4).toString().trimmed(),// keyword_id - query.value(5).toString().trimmed(), - 1, - m_pedTime->text().trimmed()); + if (m_pgbManual->isChecked()) + { + m_pManage[m_nPlatform]->Start(QDate::fromString(query.value(0).toString(),"yyyy-MM-dd"), + QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"), + query.value(2).toString().trimmed(),// keyword + query.value(3).toString().trimmed(),// authorship + query.value(4).toString().trimmed(),// keyword_id + query.value(5).toString().trimmed(), + 1, + m_pedTime->text().trimmed()); + } + else + { + m_pManage[m_nPlatform]->Start(QDate::currentDate(),QDate::currentDate(), + query.value(2).toString().trimmed(),// keyword + query.value(3).toString().trimmed(),// authorship + query.value(4).toString().trimmed(),// keyword_id + query.value(5).toString().trimmed(), + 1, + m_pedTime->text().trimmed()); + } + SetCrawlingState("Start"); + m_nMode = E_MODE_RUN; +} + +void Widget::StartButton() +{ + m_strCrawlingID = m_pcb->currentData().toString(); + Start(); } void Widget::StopButton() +{ + m_nMode = E_MODE_WAIT; +} + +bool Widget::LockTable(QString _strTable) { - m_timer.stop(); + QSqlQuery query; + return query.exec(QString("lock tables " + _strTable + "write" ).toUtf8()); +} + +void Widget::UnLockTables() +{ + QSqlQuery query; + query.exec(QString("unlock tables").toUtf8()); } void Widget::Update() { - if (m_db.isOpen() == false) { if (m_db.open()) @@ -193,27 +235,57 @@ void Widget::Update() } } - //int nCount = 0; - //for (int i = 0 ; i < C_CRAWLER_MAX ; i++) - //nCount += - m_pManage[m_nPlatform]->Update(); - - /* - if (nCount == C_CRAWLER_MAX) + if (m_pgbManual->isChecked()) { - InsertLog("Finish..."); - m_timer.stop(); + // Menual Mode + switch(m_nMode) + { + case E_MODE_WAIT:return; + } + } + else + { + // RealTime Mode + switch(m_nMode) + { + case E_MODE_WAIT: + LockTable("crawling"); + QSqlQuery query; + if (query.exec("SELECT crawling.id,keyword.start,keyword.end " + "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id " + "where crawling.state = 0 and keyword.realtime = 1 and keyword.state is null") == false) + { + InsertLog(query.lastError().text()); + UnLockTables(); + return; + } + QDate dateNow = QDate::currentDate(); + m_strCrawlingID.clear(); + while (query.next()) + { + QDate dateStart = QDate::fromString(query.value(1).toString(),"yyyy-MM-dd"); + QDate dateEnd = QDate::fromString(query.value(2).toString(),"yyyy-MM-dd"); + if (dateNow < dateStart) continue; + if (dateNow > dateEnd) continue; + m_strCrawlingID = query.value(0).toString(); + break; + } + if(m_strCrawlingID.isEmpty() == false) + Start(); + UnLockTables(); + return; + } } - */ + m_pManage[m_nPlatform]->Update(); } void Widget::RefreshButton() { m_pcb->clear(); - QSqlQuery query("SELECT _keyword.searches,_keyword.authorship,_keyword.start,_keyword.end, _datagroup.name , _crawling.id,_keyword.platform " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id where _crawling.state is null"); + QSqlQuery query("SELECT keyword.searches,keyword.authorship,keyword.start,keyword.end,datagroup.name,keyword.platform,crawling.id " + "FROM crawling INNER JOIN keyword ON crawling.keyword_id = keyword.id " + "inner join datagroup on crawling.datagroup_id = datagroup.id where crawling.state = 0 and keyword.realtime = 0 "); while (query.next()) { QString str; @@ -222,12 +294,12 @@ void Widget::RefreshButton() str += query.value(2).toString() + " ~ "; str += query.value(3).toString() + " -> "; str += query.value(4).toString(); - if(query.value(6).toString().toInt() == 0) - str += ", NaverCafe"; - else if(query.value(6).toString().toInt() == 1) - str += ", NaverBlog"; - else if(query.value(6).toString().toInt() == 2) - str += ", DaumCafe"; - m_pcb->addItem(str,query.value(5)); + switch(query.value(5).toString().toInt()) + { + case 0:str += ", Naver Cafe";break; + case 1:str += ", Naver Blog";break; + case 2:str += ", Daum Cafe"; break; + } + m_pcb->addItem(str,query.value(6)); } } diff --git a/CrawlerList/widget.h b/CrawlerList/widget.h index 97365c4..e64e8da 100644 --- a/CrawlerList/widget.h +++ b/CrawlerList/widget.h @@ -8,6 +8,7 @@ #include #include #include +#include class SNaverCafeManage; class SNaverBlogManage; @@ -24,7 +25,15 @@ public: ~Widget(); public: void InsertLog(QString str); - // void InsertLog(int _nSelect,QString str); + void SetCrawlingState(QString _str); + bool LockTable(QString strTable); + void UnLockTables(); + + enum E_MODE + { + E_MODE_WAIT = 0, + E_MODE_RUN, + }; private: QLineEdit *m_pedTime; @@ -40,8 +49,13 @@ private: SNaverBlogManage *m_pNaverBlog; SDaumCafeManage *m_pDaumCafe; int m_nStartTime,m_nRangeTime,m_nPlatform; + QGroupBox *m_pgbManual; + int m_nMode; + QString m_strCrawlingID; private: + void Start(); QString makeCafeGetListQuery(QString _str,QDate _date,int _nPage); + QGroupBox *setRealGroupWidgets(); private slots: void RefreshButton(); void StartButton(); diff --git a/CrawlerProcess/scrawler.cpp b/CrawlerProcess/scrawler.cpp index c917d26..3acafd5 100644 --- a/CrawlerProcess/scrawler.cpp +++ b/CrawlerProcess/scrawler.cpp @@ -34,37 +34,7 @@ void SCrawler::load(QStringList _strlistArgv) m_strUrl = _strlistArgv[2]; m_nSelect = E_NAVER_CAFE_LIST; m_strKeywordID = _strlistArgv[4]; - - /* - QFile file("proxy.txt"); - if (file.open(QIODevice::ReadOnly | QIODevice::Text)) - { - QVector vecProxy; - while (!file.atEnd()) - { - QString str = QString(file.readLine()); - if (str.isEmpty()) continue; - vecProxy.push_back(str.split(",")); - } - if (vecProxy.size() > 0) - { - QStringList strList = vecProxy.at(rand()%vecProxy.size()); - switch(strList.size()) - { - case 1: - cout << "p : " << strList.at(0).toStdString() << endl; - QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0)))); - break; - case 2: - cout << "p : " << strList.at(0).toStdString() << endl; - QNetworkProxy::setApplicationProxy(*(new QNetworkProxy(QNetworkProxy::HttpProxy,strList.at(0),strList.at(1).toInt()))); - break; - } - } - }*/ - setProxy(); - } if (_strlistArgv[1] == "cafe_data") @@ -79,15 +49,13 @@ void SCrawler::load(QStringList _strlistArgv) m_strUrl = _strlistArgv[2]; m_nSelect = E_NAVER_BLOG_LIST; m_strKeywordID = _strlistArgv[4]; - setProxy(); } if (_strlistArgv[1] == "blog_url") { m_strUrl = _strlistArgv[2]; - m_nSelect = E_NAVER_BLOG_BODY; - //m_strReper = _strlistArgv[4]; + m_nSelect = E_NAVER_BLOG_BODY; m_bUse = true; } @@ -310,7 +278,10 @@ void SCrawler::saveFrameList(QWebFrame *frame) QString strBlogMe = "blog.me"; - if ((strList.at(0).compare("blog.naver.com") != 0 ) && (strList.at(0).right(strBlogMe.length()).compare(strBlogMe) != 0)) { cout << "x http://" << strUrl.toStdString() < #include #include + Widget::Widget(QWidget *parent) : QWidget(parent) { @@ -29,18 +30,14 @@ Widget::Widget(QWidget *parent) m_pmodelGroup = new QSqlQueryModel; m_pmodelCrawling = new QSqlQueryModel; - m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); + m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform " + "WHEN 0 THEN 'Naver Cafe' " + "WHEN 1 THEN 'Naver Blog' " + "WHEN 2 THEN 'Daum Cafe' " + "ELSE 'UnKnown'" + "END AS platform FROM keyword where state is null"); m_pmodelGroup->setQuery("SELECT * FROM datagroup"); - /* - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - */ - - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state, _keyword.platform " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - + UpdateCrawling(); QVBoxLayout *vlayout = new QVBoxLayout(); { @@ -74,10 +71,10 @@ QGroupBox *Widget::setKeywordWidgets() } { m_pdeStart = new QDateEdit(QDate::currentDate()); - m_pdeEnd = new QDateEdit(QDate::currentDate()); + m_pdeEnd = new QDateEdit(QDate::currentDate()); m_pdeStart->setDateRange(QDate(2003, 5, 20),QDate::currentDate()); - m_pdeEnd->setDateRange(QDate(2003, 5, 20),QDate::currentDate()); + m_pdeEnd->setDateRange(QDate(2003, 5, 20),QDate::currentDate().addYears(1)); QCalendarWidget *pCalender = new QCalendarWidget(); m_pdeStart->setCalendarWidget(pCalender); @@ -86,11 +83,11 @@ QGroupBox *Widget::setKeywordWidgets() m_pdeEnd->setCalendarWidget(pCalender); m_pdeEnd->setCalendarPopup(true); - m_pcbRealTime = new QComboBox; + m_pcbRealTime = new QComboBox; m_pcbRealTime->addItems(QStringList() << "false" << "true"); m_pcbPlatform = new QComboBox; - m_pcbPlatform->addItems(QStringList() << "NaverCafe" << "NaverBlog" << "DaumCafe"); + m_pcbPlatform->addItems(QStringList() << "Naver Cafe" << "Naver Blog" << "Daum Cafe"); m_pleKeyword = new QLineEdit; m_pleAuthorship = new QLineEdit; @@ -284,7 +281,6 @@ QGroupBox *Widget::setCrawlingWidgets() return groupBox; } - void Widget::on_keyword_currentRowChanged(QModelIndex _index) { if (_index.isValid()) @@ -300,20 +296,6 @@ void Widget::on_keyword_currentRowChanged(QModelIndex _index) void Widget::on_keyword_button_insert() { - /* - QString strQuery = QString("insert into keyword set " - "start = STR_TO_DATE('%1', '%Y-%m-%d')," - "end = STR_TO_DATE('%2', '%Y-%m-%d')," - "searches = '%3'," - "realtime = %4," - "authorship = '%5'") - .arg(m_pdeStart->date().toString("yyyy-MM-dd")) - .arg(m_pdeEnd->date().toString("yyyy-MM-dd")) - .arg(m_pleKeyword->text()) - .arg(m_pcbRealTime->currentIndex()) - .arg(m_pleAuthorship->text()); - */ - QString strQuery = QString("insert into keyword set " "start = STR_TO_DATE('%1', '%Y-%m-%d')," "end = STR_TO_DATE('%2', '%Y-%m-%d')," @@ -328,10 +310,13 @@ void Widget::on_keyword_button_insert() .arg(m_pleAuthorship->text()) .arg(m_pcbPlatform->currentIndex()); - - //qDebug() << strQuery; m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); - m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); + m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform " + "WHEN 0 THEN 'Naver Cafe' " + "WHEN 1 THEN 'Naver Blog' " + "WHEN 2 THEN 'Daum Cafe' " + "ELSE 'UnKnown'" + "END AS platform FROM keyword where state is null"); } void Widget::on_keyword_button_delete() @@ -342,7 +327,12 @@ void Widget::on_keyword_button_delete() QString strQuery = QString("UPDATE keyword set state = '1' where id = '" + rec.value("id").toString() + "'"); m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); } - m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); + m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform " + "WHEN 0 THEN 'Naver Cafe' " + "WHEN 1 THEN 'Naver Blog' " + "WHEN 2 THEN 'Daum Cafe' " + "ELSE 'UnKnown'" + "END AS platform FROM keyword where state is null"); } void Widget::on_keyword_button_modify() @@ -350,22 +340,6 @@ void Widget::on_keyword_button_modify() foreach (QModelIndex index,m_ptableKeyword->selectionModel()->selectedIndexes()) { QSqlRecord rec = m_pmodelKeyword->record(index.row()); - /* - QString strQuery = QString("update keyword set " - "start = STR_TO_DATE('%1', '%Y-%m-%d')," - "end = STR_TO_DATE('%2', '%Y-%m-%d')," - "searches = '%3'," - "realtime = %4," - "authorship = '%5' " - "where id = '%6'") - .arg(m_pdeStart->date().toString("yyyy-MM-dd")) - .arg(m_pdeEnd->date().toString("yyyy-MM-dd")) - .arg(m_pleKeyword->text()) - .arg(m_pcbRealTime->currentIndex()) - .arg(m_pleAuthorship->text()) - .arg(rec.value("id").toString()); - */ - QString strQuery = QString("update keyword set " "start = STR_TO_DATE('%1', '%Y-%m-%d')," "end = STR_TO_DATE('%2', '%Y-%m-%d')," @@ -381,11 +355,14 @@ void Widget::on_keyword_button_modify() .arg(m_pleAuthorship->text()) .arg(m_pcbPlatform->currentIndex()) .arg(rec.value("id").toString()); - - m_pmodelKeyword->setQuery(QString(strQuery.toUtf8())); } - m_pmodelKeyword->setQuery("SELECT * FROM keyword where state is null"); + m_pmodelKeyword->setQuery("SELECT id,realtime,searches,start,end,authorship,CASE platform " + "WHEN 0 THEN 'Naver Cafe' " + "WHEN 1 THEN 'Naver Blog' " + "WHEN 2 THEN 'Daum Cafe' " + "ELSE 'UnKnown'" + "END AS platform FROM keyword where state is null"); } void Widget::on_group_currentRowChanged(QModelIndex _index) @@ -407,27 +384,6 @@ void Widget::on_group_button_insert() m_pmodelGroup->setQuery(strQuery.toUtf8()); strQuery = "CREATE TABLE data_"; strQuery += sql.value(0).toString(); - /* - strQuery += " (" - "url CHAR(128)," - "keyword_id INT," - "platformname CHAR(32)," - "platformform CHAR(16)," - "articleform CHAR(16)," - "body_platformtitle CHAR(128)," - "body_platformid CHAR(64)," - "body_articletitle VARCHAR(128)," - "body_articleid VARCHAR(32)," - "body_date DATETIME," - "body_nickname CHAR(32)," - "body_data VARCHAR(18432)," - "reply_nickname CHAR(32)," - "reply_data VARCHAR(1024)," - "reply_parent CHAR(32)," - "reply_date DATETIME," - "reply_urlreply VARCHAR(512)," - "reply_rownum INT) CHARSET=utf8"; - */ strQuery += " (" "platform_name CHAR(64)," "platform_form CHAR(64)," @@ -513,27 +469,14 @@ void Widget::on_group_button_import() QFile file(strFilename); if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) return; - - //STable *pNew = new STable; QTextStream in(&file); - int ncRow=0; int nCount=0; - bool bQuit = true; - int nRead = 0; QStringList attributes; QString strquery; while(!in.atEnd()) { QString strLine; - /* - { - QByteArray byte = file.readLine(); - if (byte.isEmpty()) { bQuit = false; continue; } - strLine = codec->toUnicode(byte); - nRead += byte.size(); - }*/ - strLine = in.readLine(); if (nCount == 0) { @@ -576,8 +519,7 @@ void Widget::on_group_button_import() QStringList strings = strLine.split(","); QSqlQuery query; - query.prepare(strquery.toUtf8()); - // qDebug() << strquery; + query.prepare(strquery.toUtf8()); for(int i=0; isetQuery("SELECT * FROM datagroup"); - - } - - void Widget::SaveCsv(QString _strName) { QFile file(_strName + ".csv" ); @@ -678,12 +615,9 @@ void Widget::SaveCsv(QString _strName) strSelect += "CONVERT(keyword_id USING utf8),"; strSelect += "CONVERT(platform_id USING utf8),"; strSelect += "CONVERT(keyword_id USING utf8),"; - strSelect += "CONVERT(reply_url USING utf8)"; - //strSelect += "CONVERT(reply_rownum USING utf8)"; + strSelect += "CONVERT(reply_url USING utf8)"; strSelect += " from "; strSelect += _strName; - //strSelect += " Order by body_date"; - if (query.exec(strSelect) == false) { out << query.lastError().text(); @@ -719,20 +653,11 @@ void Widget::on_crawling_button_insert() QString strQuery = "insert into crawling set "; strQuery += "Keyword_id = '" + strKeywordId + "',"; - strQuery += "DataGroup_id = '" + strGroupId + "'"; + strQuery += "DataGroup_id = '" + strGroupId + "',"; + strQuery += "state = 0"; m_pmodelCrawling->setQuery(strQuery.toUtf8()); - /* - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - */ - - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state ,_keyword.platform " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - - + UpdateCrawling(); } void Widget::on_crawling_button_delete() @@ -744,25 +669,13 @@ void Widget::on_crawling_button_delete() QSqlRecord rec = m_pmodelCrawling->record(index.row()); strList.push_back(rec.value("id").toString()); //QString strQuery = QString("delete from crawling where id = '" + rec.value("id").toString() + "'"); - //m_pmodelCrawling->setQuery(QString(strQuery.toUtf8())); - } foreach(QString str,strList) { QString strQuery = QString("delete from crawling where id = '" + str + "'"); m_pmodelCrawling->setQuery(QString(strQuery.toUtf8())); } - /* - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - */ - - m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name ,_crawling.state ,_keyword.platform " - "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " - "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); - - + UpdateCrawling(); } void Widget::on_group_button_copy_clear() @@ -862,7 +775,6 @@ void Widget::on_group_button_copy_start() return; for(int j = 0; j < strListFrom.length(); j++) { - strListFrom.at(i).toInt(&ok); if(!ok) return; @@ -875,22 +787,6 @@ void Widget::on_group_button_copy_start() strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '"); strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body')"); strQuery += (" and article_date >= '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and article_date is not null)"); - - //insert into data_to (select * from data_67 where article_url in (select distinct article_url from data_67 where article_date between '2015-02-01 00:00:00' and '2015-03-21 23:59:59' and article_form='body') and article_date >= '2015-02-01 00:00:00' and article_date is not null) - /* - strQuery = "insert into "; - strQuery += ("data_" + strListTo.at(i).trimmed()); - strQuery += (" (select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profile,article_profileurl,platform_id,keyword_id,reply_url from data_" + strListFrom.at(i).trimmed()); - strQuery += (" where article_url in (select distinct article_url from data_" + strListFrom.at(i).trimmed()); - strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '"); - strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body'))"); - - - */ - //m_pdeStart->date().toString("yyyy-MM-dd"); - //insert into data_92 - //(select * from data_93 where article_url in - //(select distinct article_url from data_93 where article_date between '2015-01-01 00:00:00' and '2015-01-01 23:59:59') and article_date >= '2015-01-01 00:00:00' and article_date is not null); } else { @@ -901,9 +797,6 @@ void Widget::on_group_button_copy_start() strQuery += " where (article_url, article_order) not in (select article_url, article_order from "; strQuery += ("data_" + strListTo.at(i).trimmed()); strQuery += "))"; - //insert into data_to from - //(select * from data_from where - //(article_url, article_order) not in (select article_url, article_order from data_to)) } qDebug(strQuery.toLatin1()); strQueryUtf = strQuery.toUtf8(); @@ -918,8 +811,13 @@ void Widget::on_group_button_copy_start() } m_pleCopyFrom->setText("Success"); m_pleCopyTo->setText("Sucess"); - - - // on_group_button_copy_clear(); } +void Widget::UpdateCrawling() +{ + m_pmodelCrawling->setQuery("SELECT _crawling.id,_keyword.searches,_keyword.start,_keyword.end, _datagroup.name , " + "(CASE _keyword.platform WHEN 0 THEN 'Naver Cafe' WHEN 1 THEN 'Naver Blog' WHEN 2 THEN 'Daum Cafe' ELSE 'UnKnown' END ) AS platform , " + "(CASE _crawling.state WHEN 0 THEN 'Waiting' WHEN 1 THEN 'Running' WHEN 2 THEN 'Terminated' ELSE 'None' END ) AS state " + "FROM crawling _crawling INNER JOIN keyword _keyword ON _crawling.keyword_id = _keyword.id " + "inner join datagroup _datagroup on _crawling.datagroup_id = _datagroup.id"); +} diff --git a/GroupManager/widget.h b/GroupManager/widget.h index 087471a..df11e50 100644 --- a/GroupManager/widget.h +++ b/GroupManager/widget.h @@ -30,11 +30,12 @@ private: QGroupBox *setGroupWidgets(); QGroupBox *setCrawlingWidgets(); void SaveCsv(QString); + void UpdateCrawling(); private slots: - void on_keyword_currentRowChanged(QModelIndex); + void on_keyword_currentRowChanged(QModelIndex); void on_keyword_button_insert(); void on_keyword_button_delete(); - void on_keyword_button_modify(); + void on_keyword_button_modify(); void on_group_currentRowChanged(QModelIndex); void on_group_button_insert(); @@ -48,6 +49,7 @@ private slots: void on_group_button_copy_from(); void on_group_button_copy_to(); void on_group_button_refresh(); + void on_crawling_button_insert(); void on_crawling_button_delete(); };