from youtube.youtube import Youtube import queue class YoutubeDBHelper: pymysql = __import__('pymysql.cursors') DB_DUMP_SIZE = 128 def __init__(self): self.youtubes = [] self.buffer = [] self.queue = queue.Queue() pass def __del__(self): self.flush() pass def get_param(self, keyword_id): query = "select * from keyword where id = " + str(keyword_id) params = [] try: conn = self.pymysql.connect(host='bigbird.iptime.org', user='admin', passwd='admin123', db='concepters', charset='utf8', cursorclass=self.pymysql.cursors.DictCursor) with conn.cursor() as cursor: cursor.execute(query) params = cursor.fetchone() except Exception as e: print(e) exit(1) else: conn.close() return params def flush(self): local_buffer = [] while not self.queue.empty(): local_buffer.append(self.queue.get()) print('### db queue dump {}'.format(len(local_buffer))) if len(local_buffer) > 0: while True: try: conn = self.pymysql.connect(host='bigbird.iptime.org', user='admin', passwd='admin123', db='concepters', charset='utf8', cursorclass=self.pymysql.cursors.DictCursor, connect_timeout=5) except Exception as e: print(e) continue else: break try: with conn.cursor() as cursor: for youtube, _db_num in local_buffer: if not youtube.is_reply: query = youtube.get_delete_query(_db_num) cursor.execute(query) query = youtube.get_insert_query(conn, _db_num) cursor.execute(query) conn.commit() except Exception as e: print(e) finally: conn.close() def insert_youtube(self, youtube: Youtube = None, db_num: int = -1, flush=False): self.queue.put((youtube, db_num)) if self.queue.qsize() >= self.DB_DUMP_SIZE: self.flush()