diff --git a/Crawler/Crawler.py b/Crawler/Crawler.py old mode 100644 new mode 100755 diff --git a/Crawler/ProxyHandler.py b/Crawler/ProxyHandler.py old mode 100644 new mode 100755 index 521546c..2d39adb --- a/Crawler/ProxyHandler.py +++ b/Crawler/ProxyHandler.py @@ -29,7 +29,7 @@ class ProxyHandler: def check_proxy_all(self, proxies, check_url): Logger.log('checking proxies for {}'.format(check_url)) - worker_cnt = 64 + worker_cnt = 16 pool = concurrent.futures.ThreadPoolExecutor(worker_cnt) [pool.submit(self.check_proxy, proxy, check_url) for proxy in proxies] pool.shutdown() diff --git a/Crawler/Setting.py b/Crawler/Setting.py old mode 100644 new mode 100755 index 77cb4f8..7dca1f3 --- a/Crawler/Setting.py +++ b/Crawler/Setting.py @@ -51,8 +51,8 @@ class Setting: if 'download_path' not in self.settings: self.settings['download_path'] = '.' - if self.settings['download_path'][-1] != '\\': - self.settings['download_path'] += '\\' + if self.settings['download_path'][-1] != '/': + self.settings['download_path'] += '/' if not os.path.exists(self.settings['download_path']): try: diff --git a/requirements.txt b/requirements.txt index dc1536f..fddc1a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests bs4 +yaml