디스크 용량 확인 및 기타 수정

Crawler/ProxyHandler.py

@@ -6,7 +6,7 @@ import concurrent.futures
 import time
 
 from .Logger import Logger
-
+from .Util import Util
 
 class ProxyHandler:
 	PROXY_FILE_NAME = 'temp/proxy.bin'
@@ -107,8 +107,9 @@ class ProxyHandler:
 		alive_proxies = [proxy for proxy in proxies if proxy['alive']]
 		Logger.log('proxies checking end: available : {}'.format(len(alive_proxies)))
 
-		with open(self.PROXY_FILE_NAME, 'wb') as f:
-			pickle.dump(alive_proxies, f)
+		if Util.get_free_space() >= 1024:
+			with open(self.PROXY_FILE_NAME, 'wb') as f:
+				pickle.dump(alive_proxies, f)
 
 		return alive_proxies
 
@@ -136,7 +137,7 @@ class ProxyHandler:
 	def set_proxy_dead(self, proxy):
 		proxy['alive'] = False
 		for proxy in self.proxies:
-			if proxy['alive']:
+			if proxy['alive'] and Util.get_free_space() >= 1024:
 				with open(self.PROXY_FILE_NAME, 'wb') as f:
 					pickle.dump(self.proxies, f)
 				return

Crawler/Setting.py

@@ -2,7 +2,7 @@ import yaml
 import os
 
 from .Logger import Logger
-
+from .Util import Util
 
 class Setting:
 	SETTING_FILE = 'conf/settings.yml'
@@ -117,6 +117,9 @@ class Setting:
 			yaml.dump(downloaded_ex, outfile, allow_unicode=True)
 
 	def save(self):
+		if Util.get_free_space() < 1024:
+			return
+
 		with open(self.DOWNLOADED_FILE, 'w', encoding='utf-8') as outfile:
 			yaml.dump(self.downloaded, outfile, allow_unicode=True)
 		pass

Crawler/Util.py

@@ -0,0 +1,9 @@
+import subprocess
+
+class Util:
+	@staticmethod
+	def get_free_space():
+		df = subprocess.Popen(["df", "/"], stdout=subprocess.PIPE)
+		output = df.communicate()[0]
+		device, size, used, available, percent, mountpoint = str(output).split("\\n")[1].split()
+		return int(available)