etc/clients
1
0
Fork 0
Code Issues Pull Requests Releases Activity

인스타 크롤러 로그에서 문제나는 부분 삭제

Browse Source
This commit is contained in:
mjjo
2017-07-06 11:48:23 +09:00
parent f12c3ca23b
commit ee9b1647e8
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,3 +1,5 @@
**/.idea/ **/.idea/
**/__pycache__/ **/__pycache__/
*.user *.user
**/build-*/
WebBasedCrawler/proxy.txt

6
WebBasedCrawler/insta/instacrawl.py
View File

@@ -72,7 +72,7 @@ body_wait_sec = 0.5
reply_wait_sec = 0.8 reply_wait_sec = 0.8
num_of_page_down = 20 num_of_page_down = 20
num_of_content_process = 10 num_of_content_process = 10
requests_timeout = 60 requests_timeout = 5
num_of_retry_proxy = 5 num_of_retry_proxy = 5
logging.basicConfig(level=logging.INFO, logging.basicConfig(level=logging.INFO,
@@ -226,7 +226,7 @@ def make_list_instance(url, proxies=None):
return list_crawler return list_crawler
except requests.exceptions.ProxyError as e: except requests.exceptions.ProxyError as e:
printd('proxy: '+str(e.args[0].pool.proxy), e) printd('proxy: {}'.format(e))
printd("Fail to make list instance") printd("Fail to make list instance")
return None return None
@@ -243,7 +243,7 @@ def make_content_instance(url, proxies=None):
return content return content
except requests.exceptions.ProxyError as e: except requests.exceptions.ProxyError as e:
printd('proxy: '+str(e.args[0].pool.proxy), e) printd('proxy: {}'.format(e))
printd("Fail to make content instance") printd("Fail to make content instance")
return None return None