effect update time 업로드하도록 수정

기타 오류 수정


git-svn-id: svn://192.168.0.12/source@333 8346c931-da38-4b9b-9d4c-e48b93cbd075

WebBasedCrawler/effect/effectinstagram.py

@@ -173,12 +173,12 @@ class EffectInsta(object):
         try:
             result_sender = ResultSender(cg['host'], cg['user'], cg['pass'], cg['name']) if cg else ResultSender()
             result_sender.connect()
+            result['lastupdate_filter'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             result_sender.send('stats_s1_effect', result)
             result_sender.close()
         except Exception as e:
             raise effect.effecterror.DBQueryError(str(e))
 
-
     def statistics(self, body, replies):
         result = {}
         result['viewcount'] = int(body.get('article_hit', 0))

WebBasedCrawler/effect/effectkakaostory.py

@@ -448,6 +448,7 @@ class EffectKakaostory(object):
 
         try:
             result = self.statistics(body, replies)
+            result['lastupdate_filter'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         except Exception as e:
             raise effect.effecterror.UnknownError(str(e))
         #pprint.pprint(body)

WebBasedCrawler/facebook/facebookcrawlbs.py

@@ -918,4 +918,4 @@ class FacebookMainCrawler:
     def control_tab(self):
         ac = ActionChains(self.driver)
         ac.key_down(Keys.CONTROL).key_down(Keys.TAB).perform()
-        wait(2)
+        wait(2)

WebBasedCrawler/insta/instaheaders.py

@@ -17,8 +17,13 @@ def get_headers_for_body_html(cookies):
                           " Chrome/50.0.2661.102 Safari/537.36",
             "upgrade-insecure-requests": "1",
             "cache-control": "max-age=0",
-            'cookie': 'mid=' + cookies['mid'] + '; sessionid=' + cookies['sessionid'] +
-                          '; ig_pr=1; ig_vw=1920; csrftoken=' + cookies['csrftoken'] + "; s_network="
+            # 'cookie': 'mid=' + cookies['mid'] + '; sessionid=' + cookies['sessionid'] +
+            #               '; ig_pr=1; ig_vw=1920; csrftoken=' + cookies['csrftoken'] + "; s_network="
+            'cookie': ((('mid=' + cookies['mid'] + '; ') if cookies.get('mid', None) else '') + \
+                   (('sessionid=' + cookies['sessionid'] + '; ') if cookies.get('sessionid', None) else '') + \
+                   (('csrftoken=' + cookies['csrftoken'] + '; ') if cookies.get('csrftoken', None) else '') + \
+                   (('ds_user_id=' + cookies['ds_user_id'] + '; ') if cookies.get('ds_user_id', None) else '')) + \
+                    'ig_pr=1; ig_vw=1920; s_network='
         }
     else:
         request_headers = {
@@ -42,8 +47,11 @@ def get_headers_for_ajax(cookies, referer, form_data):
         'origin': 'https://www.instagram.com',
         'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                       'Chrome/50.0.2661.102 Safari/537.36',
-        'cookie': 'mid=' + cookies['mid'] + '; sessionid=' + cookies['sessionid'] +
-                  '; ig_pr=1; ig_vw=1920; csrftoken=' + cookies['csrftoken'] + "; s_network=",
+        'cookie': ((('mid=' + cookies['mid'] + '; ') if cookies.get('mid', None) else '') + \
+                   (('sessionid=' + cookies['sessionid'] + '; ') if cookies.get('sessionid', None) else '') + \
+                   (('csrftoken=' + cookies['csrftoken'] + '; ') if cookies.get('csrftoken', None) else '') + \
+                   (('ds_user_id=' + cookies['ds_user_id'] + '; ') if cookies.get('ds_user_id', None) else '')) + \
+                    'ig_pr=1; ig_vw=1920; s_network=',
         'x-csrftoken': cookies['csrftoken'],
         'x-instagram-ajax': 1,
         'x-requested-with': 'XMLHttpRequest',

WebBasedCrawler/kakao/kakaocrawl.py

@@ -596,7 +596,7 @@ class ListTag(ListTraverse):
     # Raising exception is intended when fail to find a link to a content
     def open_current_section(self):
         try:
-            # The element to find is a tag. Its class attribute is link_thumb _link or link_txt _link
+            # The element to find is 'a' tag. Its class attribute is link_thumb _link or link_txt _link
             a = self.current_section.find_element_by_css_selector("a[class$=' _link']")
             a.send_keys(Keys.NULL)
             a.send_keys(Keys.ENTER)