Tools/TorrentCrawler
1
0
Fork 0
Code Issues 3 Pull Requests Releases Activity

리눅스 환경에서 디버깅

Browse Source
This commit is contained in:
mjjo
2017-08-06 05:11:53 +09:00
parent ef0c1e78ac
commit 493c44999a
4 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
Crawler/Crawler.py Normal file → Executable file
View File

2
Crawler/ProxyHandler.py Normal file → Executable file
View File

@@ -29,7 +29,7 @@ class ProxyHandler:
def check_proxy_all(self, proxies, check_url): def check_proxy_all(self, proxies, check_url):
Logger.log('checking proxies for {}'.format(check_url)) Logger.log('checking proxies for {}'.format(check_url))
worker_cnt = 64 worker_cnt = 16
pool = concurrent.futures.ThreadPoolExecutor(worker_cnt) pool = concurrent.futures.ThreadPoolExecutor(worker_cnt)
[pool.submit(self.check_proxy, proxy, check_url) for proxy in proxies] [pool.submit(self.check_proxy, proxy, check_url) for proxy in proxies]
pool.shutdown() pool.shutdown()

4
Crawler/Setting.py Normal file → Executable file
View File

@@ -51,8 +51,8 @@ class Setting:
if 'download_path' not in self.settings: if 'download_path' not in self.settings:
self.settings['download_path'] = '.' self.settings['download_path'] = '.'
if self.settings['download_path'][-1] != '\\': if self.settings['download_path'][-1] != '/':
self.settings['download_path'] += '\\' self.settings['download_path'] += '/'
if not os.path.exists(self.settings['download_path']): if not os.path.exists(self.settings['download_path']):
try: try:

1
requirements.txt
View File

@@ -1,2 +1,3 @@
requests requests
bs4 bs4
yaml