update

crawler_sys/scheduler/weibo_to_rpc.py

@@ -65,7 +65,7 @@ while True:
                 # print(desc_fix)
                 # f.write(json.dumps(res_json) + "\n")
                 # f.flush()
-                res = rds.hset("weibo_with_img", key=pid, value=json.dumps(res_json))
+                # res = rds.hset("weibo_with_img", key=pid, value=json.dumps(res_json))
                 if res_json["releaser_id_str"].replace("weibo_","") in gm_user_id_list:
                     video_dic["level"] = "5"
                 else:
@@ -93,4 +93,5 @@ while True:
                 continue
 
             rds.hdel("weibo", pid)
+            rds.sadd("weibo_exists_set", pid)
 

crawler_sys/scheduler/xiaohongshu_to_rpc.py

@@ -109,5 +109,6 @@ while True:
                 print(e)
                 continue
             rds.hdel("xiaohongshu",pid)
+            rds.sadd("xiaohongshu_exists_set",pid)
     else:
         time.sleep(5)

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -29,7 +29,7 @@ from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_pro
 from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
 import random, urllib
 # from crawler.crawler_sys.utils.rpc_data_to_answer import post_data
-rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
+rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=20, decode_responses=True)
 user_id_list = [29865245,
 36426151,
 36426142,