update

crawler_sys/scheduler/weibo_to_rpc.py

@@ -122,8 +122,8 @@ pid_list = rds.hkeys("weibo")
 # for line in f:
 for pid in pid_list:
     res = rds.hget("weibo", pid)
-    # if rds.hexists("xiaohongshu_with_img", pid):
-    #     continue
+    if rds.hexists("xiaohongshu_with_img", pid):
+        continue
     res_json = json.loads(res)
     video_dic = {}
     qiniu_img_list = []

crawler_sys/utils/rpc_data_to_answer.py

@@ -42,6 +42,8 @@ dic_type = {
     "cims/reply/batch_create":"replies"
 }
 
+invoker = create_default_invoker(debug=True).with_config(dump_curl=True)
+
 def post_muilty_data(data_list:typing.List,rpc_type:str) -> typing.Dict:
     headers = {
         'X-GAIA-HELIOS-VERSION': '0.7.5',
@@ -54,8 +56,6 @@ def post_muilty_data(data_list:typing.List,rpc_type:str) -> typing.Dict:
     #             data_dict_copy.pop(key)
     #     data_list_new.append(data_dict_copy)
     dic = {dic_type[rpc_type]: data_list}
-    invoker = create_default_invoker(debug=True).with_config(dump_curl=True)
-
     res = invoker[rpc_type](**dic)
     # print(res.)
 
@@ -73,7 +73,6 @@ def post_single_data(data_dict:typing.Dict,rpc_type:str) -> typing.Dict:
             data_dict_copy.pop(key)
     print(data_dict_copy)
     dic = {dic_type[rpc_type]: [data_dict_copy]}
-    invoker = create_default_invoker(debug=True).with_config(dump_curl=True)
 
     res = invoker[rpc_type](**dic)
     # print(res.)