update

crawler_sys/scheduler/xiaohongshu_to_rpc.py

@@ -114,7 +114,7 @@ for pid in pid_list:
     print(pid)
     for img_url in res_json["NoteView"]["content"]["imageList"]:
         try:
-            img_wb = retry_get_url("http:" + img_url["url"]).content
+            img_wb = retry_get_url("http:" + img_url["url"].replace(img_url['fileId'],img_url['traceId'])).content
             res = upload(img_wb, img_type=99)
             # print(res)
             img_info = retry_get_url(res + "-imageinfo")
@@ -125,58 +125,59 @@ for pid in pid_list:
             continue
     # print(qiniu_img_list)
     try:
+    # if True:
         desc_fix = "<p>" + res_json["NoteView"]["content"]['desc'].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
         res_json["NoteView"]["content"]["desc_fix"] = desc_fix
         # print(desc_fix)
         res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json))
-    #     video_dic["platform"] = "9"
-    #     video_dic["platform_id"] = pid
-    #     video_dic["platform_answer_id"] = pid
-    #     video_dic["title"] = res_json["NoteView"]["content"]["title"]
-    #
-    #     user_id_list_copy = copy.deepcopy(user_id_list)
-    #     qustion_id = random.choice(user_id_list_copy)
-    #     user_id_list_copy.remove(qustion_id)
-    #     video_dic["user_id"] = qustion_id
-    #     create_time = datetime.datetime.strptime(res_json["NoteView"]["content"]["time"],
-    #                                                           '%Y-%m-%d %H:%M')
-    #     video_dic["create_time"] = create_time.timestamp()
-    #     rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
-    #     # print(rpc_res)
-    #     video_dic["platform_question_id"] = pid
-    #     video_dic["content"] = desc_fix
-    #     video_dic["user_id"] = random.choice(user_id_list_copy)
-    #     rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
-    #     comment_list = []
-    #     try:
-    #         if res_json["NoteView"].get("comments"):
-    #             # print(res_json["NoteView"].get("data"))
-    #             for comment in res_json["NoteView"]["comments"]["data"]:
-    #                 video_dic["content"] = comment['content']
-    #                 comment_id_list_copy = copy.deepcopy(majiayonghu_list)
-    #                 comment_id = random.choice(comment_id_list_copy)
-    #                 video_dic["user_id"] = comment_id
-    #                 comment_id_list_copy.remove(comment_id)
-    #                 video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0, 24),
-    #                                 minutes=random.randint(0, 60))).timestamp()
-    #                 comment_list.append(copy.deepcopy(video_dic))
-    #                 # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
-    #         elif res_json["NoteView"].get("commentInfo"):
-    #             for comment in res_json["NoteView"]["commentInfo"]["comments"]:
-    #                 video_dic["content"] = comment['content']
-    #                 comment_id_list_copy = copy.deepcopy(majiayonghu_list)
-    #                 comment_id = random.choice(comment_id_list_copy)
-    #                 video_dic["user_id"] = comment_id
-    #                 comment_id_list_copy.remove(comment_id)
-    #                 video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0,24),minutes=random.randint(0,60))).timestamp()
-    #                 comment_list.append(copy.deepcopy(video_dic))
-    #                 # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
-    #         rpc_res = post_muilty_data(comment_list, "cims/reply/batch_create")
-    #     except Exception as e:
-    #         print("comment error")
-    #         print(e)
-    # except Exception as e:
-    #     print(e)
-    #     continue
+        video_dic["platform"] = "9"
+        video_dic["platform_id"] = pid
+        video_dic["platform_answer_id"] = pid
+        video_dic["title"] = res_json["NoteView"]["content"]["title"]
 
-    # break
+        user_id_list_copy = copy.deepcopy(user_id_list)
\ No newline at end of file
+        qustion_id = random.choice(user_id_list_copy)
+        user_id_list_copy.remove(qustion_id)
+        video_dic["user_id"] = qustion_id
+        create_time = datetime.datetime.strptime(res_json["NoteView"]["content"]["time"],
+                                                              '%Y-%m-%d %H:%M')
+        video_dic["create_time"] = create_time.timestamp()
+        rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
+        # print(rpc_res)
+        video_dic["platform_question_id"] = pid
+        video_dic["content"] = desc_fix
+        video_dic["user_id"] = random.choice(user_id_list_copy)
+        rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
+        comment_list = []
+        try:
+            if res_json["NoteView"].get("comments"):
+                # print(res_json["NoteView"].get("data"))
+                for comment in res_json["NoteView"]["comments"]["data"]:
+                    video_dic["content"] = comment['content']
+                    comment_id_list_copy = copy.deepcopy(majiayonghu_list)
+                    comment_id = random.choice(comment_id_list_copy)
+                    video_dic["user_id"] = comment_id
+                    comment_id_list_copy.remove(comment_id)
+                    video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0, 24),
+                                    minutes=random.randint(0, 60))).timestamp()
+                    comment_list.append(copy.deepcopy(video_dic))
+                    # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
+            elif res_json["NoteView"].get("commentInfo"):
+                for comment in res_json["NoteView"]["commentInfo"]["comments"]:
+                    video_dic["content"] = comment['content']
+                    comment_id_list_copy = copy.deepcopy(majiayonghu_list)
+                    comment_id = random.choice(comment_id_list_copy)
+                    video_dic["user_id"] = comment_id
+                    comment_id_list_copy.remove(comment_id)
+                    video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0,24),minutes=random.randint(0,60))).timestamp()
+                    comment_list.append(copy.deepcopy(video_dic))
+                    # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
+            rpc_res = post_muilty_data(comment_list, "cims/reply/batch_create")
+        except Exception as e:
+            print("comment error")
+            print(e)
+    except Exception as e:
+        print(e)
+        continue
+
+    break
\ No newline at end of file