update

crawler_sys/scheduler/xiaohongshu_to_rpc.py

+# -*- coding:UTF-8 -*-
+# @Time  : 2021/1/8 16:41
+# @File  : xiaohongshu_to_rpc.py
+# @email : litao@igengmei.com
+# @author : litao
+import datetime
+import random
+
+import redis, json
+from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data
+from crawler_sys.utils.output_results import retry_get_url
+from crawler.gm_upload.gm_upload import upload, upload_file
+
+majiayonghu_list = [
+    36436814,
+    36436809,
+    36436805,
+    36436803,
+    36436800,
+    36436797,
+    36436794,
+    36436793,
+    36436787,
+    36436782,
+    36436769,
+    36436763,
+    36436758,
+    36436756,
+    36436749,
+    36436745,
+    36436738,
+    36436731,
+    36436729,
+    36436725,
+    36436720,
+    36436717,
+    36436716,
+    36436709,
+    36436703,
+    36436701,
+    36436690,
+    36436689,
+    36436685,
+    36436674,
+    36426171,
+    36426170,
+    36426169,
+    36426168,
+    36426167,
+    36426166,
+    36426165,
+    36426164,
+    36426163,
+    36426162,
+    36426161,
+    36426160,
+    36426159,
+    36426158,
+    36426157,
+    36426156,
+    36426155,
+    36426154,
+    36426153,
+    36426152,
+    36426150,
+    36426149,
+    36426148,
+    36426147,
+    36426146,
+    36426145,
+    36426143,
+    36426141,
+    36368922,
+    36368921,
+    36368920,
+    36368918,
+    36368917,
+
+]
+
+user_id_list = [
+                29865245,
+                36426151,
+                36426142,
+                36427666,
+                36427661,
+                36427657,
+                36427655,
+                36427634,
+                33524762,
+                33524779,
+                33524697,
+                30963358,
+                31293584,
+                31358392,
+                31358396,
+                31358397,
+                31358419,
+                31358448,
+                31358610,
+                31358658,
+                ]
+
+rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
+pid_list = rds.hkeys("xiaohongshu")
+for pid in pid_list:
+    res = rds.hget("xiaohongshu", pid)
+    res_json = json.loads(res)
+    video_dic = {}
+    qiniu_img_list = []
+    print(pid)
+    for img_url in res_json["NoteView"]["content"]["imageList"]:
+        try:
+            img_wb = retry_get_url("http:" + img_url["url"]).content
+            res = upload(img_wb, img_type=99)
+            # print(res)
+            img_info = retry_get_url(res + "-imageinfo")
+            img_info_json = img_info.json()
+            qiniu_img_list.append('<img src="' + res + '-w">')
+        except Exception as e:
+            print("down load img error %s" % e)
+            continue
+    print(qiniu_img_list)
+    desc_fix = "<p>" + res_json["NoteView"]["content"]['desc'].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
+    res_json["NoteView"]["content"]["desc_fix"] = desc_fix
+    print(desc_fix)
+    res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json))
+    video_dic["platform"] = "9"
+    video_dic["platform_id"] = pid
+    video_dic["title"] = res_json["NoteView"]["content"]["title"]
+    # video_dic["content"] = anwser
+    video_dic["user_id"] = random.choice(user_id_list)
+    video_dic["create_time"] = datetime.datetime.strptime(res_json["NoteView"]["content"]["time"],
+                                                          '%Y-%m-%d %H:%M').timestamp()
+    rpc_res = post_single_data(video_dic, "cims/question/batch_create")
+    print(rpc_res)
+    break
+    # rpc_res = post_single_data(video_dic, "cims/answer/batch_create")
+    # rpc_res = post_single_data(video_dic, "cims/reply/batch_create")