diff --git a/Crawler/Crawler.py b/Crawler/Crawler.py index 441eff0..78c1a30 100755 --- a/Crawler/Crawler.py +++ b/Crawler/Crawler.py @@ -66,7 +66,7 @@ class Crawler: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', + 'Accept-Charset': 'utf-8,ISO-8859-1;q=0.7,*;q=0.3', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4', 'Connection': 'keep-alive', @@ -94,6 +94,7 @@ class Crawler: Logger.log('checking page {}'.format(url)) resp = self.request_get(url) + import pdb; pdb.set_trace() soup = bs4.BeautifulSoup(resp.content.decode('utf8'), 'lxml') re_title = re.compile('|'.join(['(?P'+video['keyword']+')' for i, video in enumerate(self.setting.settings['video'])]), re.I) @@ -179,7 +180,7 @@ class Crawler: self.setting.downloaded[file_link.title].append(file_link.episode) self.setting.save() - Logger.log("downloaded {}".format(file_link.file_name)) + Logger.log("downloaded {}".format(file_name)) except Exception as e: Logger.log(e) diff --git a/Crawler/ProxyHandler.py b/Crawler/ProxyHandler.py index 2d39adb..5281e6b 100755 --- a/Crawler/ProxyHandler.py +++ b/Crawler/ProxyHandler.py @@ -3,6 +3,7 @@ import pickle import requests import bs4 import concurrent.futures +import time from .Logger import Logger @@ -64,7 +65,45 @@ class ProxyHandler: } ) + resp = requests.get('https://www.socks-proxy.net') + soup = bs4.BeautifulSoup(resp.text, 'lxml') + table = soup.select('table.table') + trs = table[0].select('tr') + for tr in trs[1:]: + tds = tr.select('td') + if len(tds) < 2: + continue + + ip, port = tds[0].text, tds[1].text + proxies.append( + { + 'alive': True, + 'http': '{}:{}'.format(ip, port), + 'https': '{}:{}'.format(ip, port), + } + ) + + resp = requests.get('https://www.sslproxies.org') + soup = bs4.BeautifulSoup(resp.text, 'lxml') + table = soup.select('table.table') + trs = table[0].select('tr') + for tr in trs[1:]: + tds = tr.select('td') + if len(tds) < 2: + continue + + ip, port = tds[0].text, tds[1].text + proxies.append( + { + 'alive': True, + 'http': '{}:{}'.format(ip, port), + 'https': '{}:{}'.format(ip, port), + } + ) + + # print(proxies) self.check_proxy_all(proxies, self.check_url) + # print(proxies) alive_proxies = [proxy for proxy in proxies if proxy['alive']] Logger.log('proxies checking end: available : {}'.format(len(alive_proxies))) @@ -77,8 +116,16 @@ class ProxyHandler: if len(self.proxies) <= 0: if self.has_file(): self.proxies = self.load_proxy() - else: + self.proxies = [proxy for proxy in self.proxies if proxy['alive']] + + if len(self.proxies) <= 0: + while True: self.proxies = self.crawl_proxy() + if len(self.proxies) > 0: + break + else: + Logger.log('there is no available proxy. sleep 10secs..') + time.sleep(10) for proxy in self.proxies: if proxy['alive']: diff --git a/Main.py b/Main.py old mode 100644 new mode 100755 index ceec55f..008f4f4 --- a/Main.py +++ b/Main.py @@ -3,3 +3,19 @@ from Crawler.Crawler import Crawler if __name__ == '__main__': crawler = Crawler() crawler.crawl() + + # import requests + # import bs4 + + # resp = requests.get('http://www.gatherproxy.com') + # soup = bs4.BeautifulSoup(resp.text, 'lxml') + # # print(soup) + # table = soup.select('table#tblproxy') + # trs = table[0].select('tr') + # for tr in trs[2:5]: + # tds = tr.select('td') + # if len(tds) < 2: + # continue + + # ip, port = tds[1].text, tds[2].text + # print('ip: {}, port: {}'.format(ip, port))