etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

runner exception 추가

Browse Source
This commit is contained in:
mjjo
2017-08-10 11:32:08 +09:00
parent 19cd5bb070
commit 3d829e55b5
2 changed files with 47 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
WebBasedCrawler/base/proxy2.py
View File

@@ -135,7 +135,7 @@ class Proxy2Handler:
else: else:
proxies = proxy_crawler.crawl_proxies() proxies = proxy_crawler.crawl_proxies()
self.insert_all(proxies) self.insert_all(proxies)
self.unlock() self.lock_leave()
return self.get(platform, proc_id) return self.get(platform, proc_id)
def insert(self, ip, port): def insert(self, ip, port):

5
WebBasedCrawler/twitter/twittercrawl.py
View File

@@ -6,6 +6,7 @@ from twitter.twparser import TweetParser
import base.proxy import base.proxy
import base.proxy2 as proxy2 import base.proxy2 as proxy2
import base.baseclasses import base.baseclasses
import base.logger as logger
import requests import requests
import bs4 import bs4
@@ -122,6 +123,7 @@ class TwitterCrawler:
} }
def runner_proc(self, proc_id, content_queue, result_queue, config): def runner_proc(self, proc_id, content_queue, result_queue, config):
try:
print('[{}] {} to {} runner thread start'.format(proc_id, config.start_str, config.end_str)) print('[{}] {} to {} runner thread start'.format(proc_id, config.start_str, config.end_str))
b_continue = True b_continue = True
@@ -173,6 +175,9 @@ class TwitterCrawler:
'count': tweet_count, 'count': tweet_count,
}) })
# self.runner_processing[proc_id].value = False # self.runner_processing[proc_id].value = False
except Exception as e:
logger.log(e, logger.LogLevel.ERROR)
return proc_id, tweet_count, return proc_id, tweet_count,
@staticmethod @staticmethod