Tools/TorrentCrawler
1
0
Fork 0
Code Issues 3 Pull Requests Releases Activity

압축 형식 삭제

Browse Source
This commit is contained in:
mjjo53
2017-12-16 21:19:10 +09:00
parent 34c0cc5a29
commit 125c201638
1 changed files with 2 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Crawler/Crawler.py
View File

@@ -66,9 +66,8 @@ class Crawler:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Charset': 'utf-8,ISO-8859-1;q=0.7,*;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'ko-KR,ko;q=0.8,en-US;q=0.6,en;q=0.4',
'Accept-Charset': 'utf-8,euc-kr;q=0.7,*;q=0.3',
'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
'Connection': 'keep-alive',
}
@@ -94,7 +93,6 @@ class Crawler:
Logger.log('checking page {}'.format(url))
resp = self.request_get(url)
import pdb; pdb.set_trace()
soup = bs4.BeautifulSoup(resp.content.decode('utf8'), 'lxml')
re_title = re.compile('|'.join(['(?P<key'+str(i)+'>'+video['keyword']+')' for i, video in enumerate(self.setting.settings['video'])]), re.I)