From 493c44999a66d51fa90d779b0e3051e45763c7b3 Mon Sep 17 00:00:00 2001 From: mjjo Date: Sun, 6 Aug 2017 05:11:53 +0900 Subject: [PATCH] =?UTF-8?q?=EB=A6=AC=EB=88=85=EC=8A=A4=20=ED=99=98?= =?UTF-8?q?=EA=B2=BD=EC=97=90=EC=84=9C=20=EB=94=94=EB=B2=84=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Crawler/Crawler.py | 0 Crawler/ProxyHandler.py | 2 +- Crawler/Setting.py | 4 ++-- requirements.txt | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 Crawler/Crawler.py mode change 100644 => 100755 Crawler/ProxyHandler.py mode change 100644 => 100755 Crawler/Setting.py diff --git a/Crawler/Crawler.py b/Crawler/Crawler.py old mode 100644 new mode 100755 diff --git a/Crawler/ProxyHandler.py b/Crawler/ProxyHandler.py old mode 100644 new mode 100755 index 521546c..2d39adb --- a/Crawler/ProxyHandler.py +++ b/Crawler/ProxyHandler.py @@ -29,7 +29,7 @@ class ProxyHandler: def check_proxy_all(self, proxies, check_url): Logger.log('checking proxies for {}'.format(check_url)) - worker_cnt = 64 + worker_cnt = 16 pool = concurrent.futures.ThreadPoolExecutor(worker_cnt) [pool.submit(self.check_proxy, proxy, check_url) for proxy in proxies] pool.shutdown() diff --git a/Crawler/Setting.py b/Crawler/Setting.py old mode 100644 new mode 100755 index 77cb4f8..7dca1f3 --- a/Crawler/Setting.py +++ b/Crawler/Setting.py @@ -51,8 +51,8 @@ class Setting: if 'download_path' not in self.settings: self.settings['download_path'] = '.' - if self.settings['download_path'][-1] != '\\': - self.settings['download_path'] += '\\' + if self.settings['download_path'][-1] != '/': + self.settings['download_path'] += '/' if not os.path.exists(self.settings['download_path']): try: diff --git a/requirements.txt b/requirements.txt index dc1536f..fddc1a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests bs4 +yaml