EffectUI : 크롤링해오는 텀 조절

FilterProcess : ??? 글쎄???
GroupManager : Group 부분 수정 Copy 중복 포함해서 넘김(속도), Clear, Delete 삭제, List Export 추가, Refresh 선택한 것만 갱신하도록 수정

git-svn-id: svn://192.168.0.12/source@328 8346c931-da38-4b9b-9d4c-e48b93cbd075

CrawlerList/widget.cpp

@@ -214,7 +214,7 @@ bool Widget::SetCrawlingState(QString _str)
     return true;
 }
 
-void Widget::Start()
+void Widget:: Start()
 {
     if(!m_db.isOpen())
     {

EffectUI/EffectUI.pro.user

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE QtCreatorProject>
-<!-- Written by QtCreator 4.1.0, 2016-11-21T16:16:29. -->
+<!-- Written by QtCreator 4.1.0, 2016-12-08T16:08:57. -->
 <qtcreator>
  <data>
   <variable>EnvironmentId</variable>
@@ -191,7 +191,7 @@
    <valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.0">
     <value type="bool" key="Analyzer.QmlProfiler.AggregateTraces">false</value>
     <value type="bool" key="Analyzer.QmlProfiler.FlushEnabled">false</value>
-    <value type="uint" key="Analyzer.QmlProfiler.FlushInterval">1000</value>
+    <value type="uint" key="Analyzer.QmlProfiler.FlushInterval">0</value>
     <value type="QString" key="Analyzer.QmlProfiler.LastTraceFile"></value>
     <value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
     <valuelist type="QVariantList" key="Analyzer.Valgrind.AddedSuppressionFiles"/>
@@ -237,6 +237,7 @@
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.CommandLineArguments"></value>
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.ProFile">EffectUI.pro</value>
     <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseDyldImageSuffix">false</value>
+    <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseTerminal">false</value>
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.UserWorkingDirectory"></value>
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.UserWorkingDirectory.default">C:/source/build-EffectUI-Desktop_Qt_5_4_0_MSVC2013_32bit-Debug</value>
     <value type="uint" key="RunConfiguration.QmlDebugServerPort">3768</value>
@@ -623,7 +624,7 @@
     <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseDyldImageSuffix">false</value>
     <value type="bool" key="Qt4ProjectManager.Qt4RunConfiguration.UseTerminal">false</value>
     <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.UserWorkingDirectory"></value>
-    <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.UserWorkingDirectory.default">C:/source/build-EffectUI-Desktop_Qt_5_4_0_MSVC2013_64bit-Debug</value>
+    <value type="QString" key="Qt4ProjectManager.Qt4RunConfiguration.UserWorkingDirectory.default"></value>
     <value type="uint" key="RunConfiguration.QmlDebugServerPort">3768</value>
     <value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
     <value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>

EffectUI/sparentprocess.cpp

@@ -17,7 +17,7 @@ const QString PYTHON = "python3";
 #endif
 
 
-const int TIMEOUT = 180; // 3 minutes
+const int TIMEOUT = 18; // 3 minutes
 const QString SPLIT_KEYWORD = "!@#";
 const QString PYTAPPNAME = "effectprocess.py";
 const QString NAVER_BLOG_1 = "blog.naver.com/";

EffectUI/srunningprocess.cpp

@@ -3,7 +3,7 @@
 #include <QDebug>
 
 namespace{
-const int ONE_SECOND = 1000; // 1 second
+const int TIMER_SECOND = 10000; // 1 second
 const int HALF_SECOND = 500; // 0.5 second
 }
 
@@ -12,7 +12,7 @@ SRunningProcess::SRunningProcess():
     m_pCompanyProcess(new SCompanyProcess), m_eMode(E_PROCESS_MODE::WAIT),
     m_pCurrentProcess(nullptr)
 {
-    m_pTimer->setInterval(ONE_SECOND);
+    m_pTimer->setInterval(TIMER_SECOND);
     initConnect();
 }
 

GroupManager/widget.cpp

@@ -227,13 +227,15 @@ QGroupBox *Widget::setGroupWidgets()
         QPushButton *pModify = new QPushButton("Modify");
         QPushButton *pImport = new QPushButton("Import");
         QPushButton *pExport = new QPushButton("Export");
+        QPushButton *pListExport = new QPushButton("List Export");
         QPushButton *pRefresh = new QPushButton("Refresh");
         hlayout->addWidget(pInsert);
-        hlayout->addWidget(pDelete);
+        //hlayout->addWidget(pDelete);
-        hlayout->addWidget(pClear);
+        //hlayout->addWidget(pClear);
         hlayout->addWidget(pModify);
         hlayout->addWidget(pImport);
         hlayout->addWidget(pExport);        
+        hlayout->addWidget(pListExport);
         hlayout->addWidget(pRefresh);
         connect(pInsert, SIGNAL(released()),this, SLOT(on_group_button_insert()));
         connect(pDelete, SIGNAL(released()),this, SLOT(on_group_button_delete()));
@@ -241,6 +243,7 @@ QGroupBox *Widget::setGroupWidgets()
         connect(pModify, SIGNAL(released()),this, SLOT(on_group_button_modify()));
         connect(pImport, SIGNAL(released()),this, SLOT(on_group_button_import()));
         connect(pExport, SIGNAL(released()),this, SLOT(on_group_button_export()));
+        connect(pListExport, SIGNAL(released()),this, SLOT(on_group_button_list_export()));
         connect(pRefresh, SIGNAL(released()),this, SLOT(on_group_button_refresh()));
         vlayout->addLayout(hlayout);
     }
@@ -249,8 +252,8 @@ QGroupBox *Widget::setGroupWidgets()
         QHBoxLayout *hlayout = new QHBoxLayout();
         m_pleCopyFrom = new QLineEdit();
         m_pleCopyTo = new QLineEdit();
-        m_pleCopyFrom->setReadOnly(true);
+        //m_pleCopyFrom->setReadOnly(true);
-        m_pleCopyTo->setReadOnly(true);
+        //m_pleCopyTo->setReadOnly(true);
         QPushButton *pCopyAddFrom = new QPushButton("Add");
         QPushButton *pCopyAddTo = new QPushButton("Add");
         QPushButton *pStart = new QPushButton("Start");
@@ -774,32 +777,56 @@ void Widget::on_group_button_import()
 }
 
 
+void Widget::on_group_button_list_export()
+{
+    QString strFilename = QFileDialog::getSaveFileName(0,"Import file",QDir::currentPath(),
+            "csv files (*.csv);;All files (*.*)",new QString("Text files (*.csv)"));
+
+    QFile file(strFilename);
+    if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) return;
+
+    //STable *pNew = new STable;
+    QTextStream in(&file);
+
+    for (int i = 0; i < m_pmodelGroup->rowCount(); ++i)
+    {
+        in << m_pmodelGroup->record(i).value("id").toString() + ","
+              + m_pmodelGroup->record(i).value("name").toString() + ","
+              + m_pmodelGroup->record(i).value("count").toString() + "\n";
+    }
+    file.close();
+}
+
 void Widget::on_group_button_refresh()
 {
+    int row = m_ptableGroup->currentIndex().row();
+    if (row == -1)
+        return;
+
+    QString strId = m_pmodelGroup->record(row).value("id").toString();
+    qDebug() << strId;
+
+    if (strId.trimmed().isEmpty()) return;
+
     QString strQuery;
     QString strQueryUtf;
     QSqlQuery query;
 
-    if(m_pmodelGroup->rowCount() < 1)
+    strQuery = "select count(*) from data_" + strId.trimmed();
+    strQueryUtf = strQuery.toUtf8();
+    if(query.exec(strQueryUtf) == false)
+        return;
+    QString strCount;
+    while(query.next())
+        strCount = query.value(0).toString().trimmed();
+
+    strQuery = "update datagroup set count = " + strCount + " where id = " + strId;
+    strQueryUtf = strQuery.toUtf8();
+    if(query.exec(strQueryUtf) == false)
         return;
 
-    for(int i = 0; i < m_pmodelGroup->rowCount(); i++)
-    {
-        QString strId = m_pmodelGroup->record(i).value("id").toString().trimmed();
-        strQuery = "select count(*) from data_" + strId.trimmed();
-        strQueryUtf = strQuery.toUtf8();
-        if(query.exec(strQueryUtf) == false)
-            return;
-        QString strCount;
-        while(query.next())
-            strCount = query.value(0).toString().trimmed();
-
-        strQuery = "update datagroup set count = " + strCount + " where id = " + strId;
-        strQueryUtf = strQuery.toUtf8();
-        if(query.exec(strQueryUtf) == false)
-            return;
-    }
     m_pmodelGroup->setQuery("SELECT * FROM datagroup");
+
 }
 
 void Widget::SaveCsv(QString _strName)
@@ -1017,6 +1044,16 @@ void Widget::on_group_button_copy_start()
                 return;
             if(m_pchbDate->isChecked())
             {
+                strQuery = "insert into ";
+                strQuery += ("data_" + strListTo.at(i).trimmed());
+                strQuery += " (platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profileurl,platform_id,keyword_id,reply_url,article_profile,etc) ";
+                strQuery += "select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profileurl,platform_id,keyword_id,reply_url,article_profile,etc "
+                            "from data_" + strListFrom.at(j).trimmed() + " where article_url in (select distinct article_url from data_";
+                strQuery += strListFrom.at(j).trimmed();
+                strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '");
+                strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body')");
+
+                /*
                 strQuery = "insert into ";
                 strQuery += ("data_" + strListTo.at(i).trimmed());
                 strQuery += (" (select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profile,article_profileurl,platform_id,keyword_id,reply_url,etc from data_" + strListFrom.at(i).trimmed());
@@ -1024,9 +1061,16 @@ void Widget::on_group_button_copy_start()
                 strQuery += (" where article_date between '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and '");
                 strQuery += (m_pdeCopyEnd->date().toString("yyyy-MM-dd") + " 23:59:59' and article_form='body')");
                 strQuery += (" and article_date >= '" + m_pdeCopyStart->date().toString("yyyy-MM-dd") + " 00:00:00' and article_date is not null)");
+                */
             }
             else
             {
+                strQuery = "insert into ";
+                strQuery += ("data_" + strListTo.at(i).trimmed());
+                strQuery += " (platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profileurl,platform_id,keyword_id,reply_url,article_profile,etc) ";
+                strQuery += "select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profileurl,platform_id,keyword_id,reply_url,article_profile,etc "
+                            "from data_" + strListFrom.at(j).trimmed();
+                /*
                 strQuery = "insert into ";
                 strQuery += ("data_" + strListTo.at(i).trimmed());
                 strQuery += " (select platform_name,platform_form,platform_title,article_form,article_parent,article_id,article_nickname,article_title,article_data,article_url,article_hit,article_date,article_order,article_profile,article_profileurl,platform_id,keyword_id,reply_url,etc from ";
@@ -1034,6 +1078,7 @@ void Widget::on_group_button_copy_start()
                 strQuery += " where (article_url, article_order) not in (select article_url, article_order from ";
                 strQuery += ("data_" + strListTo.at(i).trimmed());
                 strQuery += "))";
+                */
             }
             qDebug(strQuery.toLatin1());
             strQueryUtf = strQuery.toUtf8();

GroupManager/widget.h

@@ -46,12 +46,14 @@ private slots:
     void on_group_button_modify();
     void on_group_button_import();
     void on_group_button_export();
+    void on_group_button_list_export();
     void on_group_button_copy_start();
     void on_group_button_copy_clear();
     void on_group_button_copy_from();
     void on_group_button_copy_to();
     void on_group_button_refresh();
 
+
     void on_crawling_button_insert();
     void on_crawling_button_delete();
     void on_crawling_button_modify();

MorphereAnalyzer/sbatchrun.cpp

@@ -529,6 +529,12 @@ bool SBatchRun::loadData()
                     strList[m_anData[E_DATA_article_date]] = " 1990-01-01 00:00:00 ";
 
                 setUrl.insert(strList[m_anData[E_DATA_article_url]].trimmed());
+                if (strList[m_anData[E_DATA_article_url]].trimmed() == "http://blog.naver.com/arim850704/22087584923")
+                {
+                    qDebug() << strList[m_anData[E_DATA_article_url]];
+                    qDebug() << strList[m_anData[E_DATA_article_data]];
+                }
+
                 m_vecData[i].append(strList);
             }