트위터 크롤러 수정

- proxy.txt에 있는 ip 모두 소진하면 새롭게 가져와서 채우는 기능
- db에 넣을 때 128개 모아서 한번에 넣는 기능
- concurrent.future.ThreadPoolExecutor 사용
- qt에서 로그 라인별로 읽어서 출력
- 로그 256개씩 한번에 삭제

CrawlerList/stwittertagmanage.cpp

@@ -56,19 +56,41 @@ void STwitterTagManage::processFinished(QProcess *pPro, QString _strOut)
 
 void STwitterTagManage::readStandardOutput()
 {
+    if(!m_pMain)
+        return;
+
+    static bool bRunning = false;
+    if(bRunning)
+        return;
+
+    bRunning = true;
     QProcess *pPro = (QProcess*)sender();
-    QThread::msleep(100);
-    QString str = pPro->readAllStandardOutput();
-    QStringList list = str.split("\n", QString::SkipEmptyParts);
-    foreach(QString log,list)
+    QStringList lines;
+
+    while(pPro->canReadLine())
     {
-        if (m_pMain)
-        {
-            m_pMain->InsertLog(log);
-        }
-        else
-            exit(0);
+        QString line = pPro->readLine();
+        lines.append(line.simplified());
     }
+
+    foreach(QString line, lines)
+        m_pMain->InsertLog(line);
+
+    bRunning = false;
+
+
+//    QThread::msleep(100);
+//    QString str = pPro->readAllStandardOutput();
+//    QStringList list = str.split("\n", QString::SkipEmptyParts);
+//    foreach(QString log,list)
+//    {
+//        if (m_pMain)
+//        {
+//            m_pMain->InsertLog(log);
+//        }
+//        else
+//            exit(0);
+//    }
 }
 
 void STwitterTagManage::readStandardError()

CrawlerList/widget.cpp

@@ -186,9 +186,12 @@ void Widget::InsertLog(QString str)
 
     if (m_pResultList->count() > 1024)
     {
-        m_pResultList->removeItemWidget(m_pResultList->item(0));
-        QListWidgetItem* item = m_pResultList->takeItem(0);
-        delete item;
+        for(int i=0; i<256; i++)
+        {
+            m_pResultList->removeItemWidget(m_pResultList->item(0));
+            QListWidgetItem* item = m_pResultList->takeItem(0);
+            delete item;
+        }
     }
     m_pResultList->setCurrentRow( m_pResultList->count() - 1 );
     m_pResultList->repaint();