update

crawler_sys/site_crawler_by_redis/crawler_douban.py

@@ -109,8 +109,8 @@ class CrawlerDouban():
         sign = "bf7dddc7c9cfe6f7"
         url_limit = url.split("?")[0].replace("http://frodo.douban.com",'')
         url_limit = urllib.parse.quote(url_limit,safe='')
-        # ts = str(int(datetime.datetime.now().timestamp()))
-        ts = '1600650372'
+        ts = str(int(datetime.datetime.now().timestamp()))
+        # ts = '1600650372'
         url_str = 'GET&%s&%s' % (url_limit,ts)
         # print(url_str)
         sig_sha1 = hmac.new(sign.encode('utf-8'), url_str.encode('utf-8'), digestmod='SHA1')
@@ -135,7 +135,7 @@ class CrawlerDouban():
         ts,sig = self.get_sig('/api/v2/group/248952/topics')
         url_dic = {
             # "start": None,
-            "count": "100",
+            "count": "20",
             "sortby": "new",
             # "apple": "389276ed556d40cada2e208482b51cd7",
             # "icecream": "7b92c1aa7b531d1500c6e4905de2ca76",
@@ -247,7 +247,6 @@ class CrawlerDouban():
                         yield res
 if __name__ == '__main__':
     test = CrawlerDouban()
-    url = 'https://weibo.com/p/1644114654/home?from=page_100306&mod=TAB#place'
     # releaserUrl = 'http://v.qq.com/vplus/cfa34d96d1b6609f1dccdea65b26b83d'
     url_list = [
             "https://www.douban.com/people/new_tag"
@@ -259,7 +258,7 @@ if __name__ == '__main__':
     # for r in res:
     #     print(r)
     for u in url_list:
-        ttt = test.releaser_page_by_time(1595755100232, 1595906959333, u, output_to_es_register=False,
+        ttt = test.releaser_page_by_time(1600531200000, 1600660917502, u, output_to_es_register=False,
                                           es_index='crawler-data-raw',
                                           doc_type='doc', releaser_page_num_max=4000,allow=20)
         for t in ttt: