From ee9b1647e8dcc6e1713b833cd51b6ad4c2a1c33e Mon Sep 17 00:00:00 2001 From: mjjo Date: Thu, 6 Jul 2017 11:48:23 +0900 Subject: [PATCH] =?UTF-8?q?=EC=9D=B8=EC=8A=A4=ED=83=80=20=ED=81=AC?= =?UTF-8?q?=EB=A1=A4=EB=9F=AC=20=EB=A1=9C=EA=B7=B8=EC=97=90=EC=84=9C=20?= =?UTF-8?q?=EB=AC=B8=EC=A0=9C=EB=82=98=EB=8A=94=20=EB=B6=80=EB=B6=84=20?= =?UTF-8?q?=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ WebBasedCrawler/insta/instacrawl.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 70ec9b1..4401fea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ **/.idea/ **/__pycache__/ *.user +**/build-*/ +WebBasedCrawler/proxy.txt diff --git a/WebBasedCrawler/insta/instacrawl.py b/WebBasedCrawler/insta/instacrawl.py index 1598e7e..248012f 100644 --- a/WebBasedCrawler/insta/instacrawl.py +++ b/WebBasedCrawler/insta/instacrawl.py @@ -72,7 +72,7 @@ body_wait_sec = 0.5 reply_wait_sec = 0.8 num_of_page_down = 20 num_of_content_process = 10 -requests_timeout = 60 +requests_timeout = 5 num_of_retry_proxy = 5 logging.basicConfig(level=logging.INFO, @@ -226,7 +226,7 @@ def make_list_instance(url, proxies=None): return list_crawler except requests.exceptions.ProxyError as e: - printd('proxy: '+str(e.args[0].pool.proxy), e) + printd('proxy: {}'.format(e)) printd("Fail to make list instance") return None @@ -243,7 +243,7 @@ def make_content_instance(url, proxies=None): return content except requests.exceptions.ProxyError as e: - printd('proxy: '+str(e.args[0].pool.proxy), e) + printd('proxy: {}'.format(e)) printd("Fail to make content instance") return None