update

crawler_sys/scheduler/xiaohongshu_to_rpc.py

@@ -8,7 +8,7 @@ import datetime
 import random
 
 import redis, json
-from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data
+from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
 from crawler_sys.utils.output_results import retry_get_url
 from crawler.gm_upload.gm_upload import upload, upload_file
 
@@ -106,6 +106,8 @@ rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=
 pid_list = rds.hkeys("xiaohongshu")
 for pid in pid_list:
     res = rds.hget("xiaohongshu", pid)
+    if rds.hexists("xiaohongshu_with_img", pid):
+        continue
     res_json = json.loads(res)
     video_dic = {}
     qiniu_img_list = []
@@ -145,6 +147,7 @@ for pid in pid_list:
         video_dic["content"] = desc_fix
         video_dic["user_id"] = random.choice(user_id_list_copy)
         rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
+        comment_list = []
         try:
             if res_json["NoteView"].get("comments"):
                 # print(res_json["NoteView"].get("data"))
@@ -156,7 +159,8 @@ for pid in pid_list:
                     comment_id_list_copy.remove(comment_id)
                     video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0, 24),
                                     minutes=random.randint(0, 60))).timestamp()
-                    rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
+                    comment_list.append(copy.deepcopy(video_dic))
+                    # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
             elif res_json["NoteView"].get("commentInfo"):
                 for comment in res_json["NoteView"]["commentInfo"]["comments"]:
                     video_dic["content"] = comment['content']
@@ -165,7 +169,9 @@ for pid in pid_list:
                     video_dic["user_id"] = comment_id
                     comment_id_list_copy.remove(comment_id)
                     video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0,24),minutes=random.randint(0,60))).timestamp()
-                    rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
+                    comment_list.append(copy.deepcopy(video_dic))
+                    # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
+            rpc_res = post_muilty_data(comment_list, "cims/reply/batch_create")
         except Exception as e:
             print("comment error")
             print(e)