update

crawler_sys/scheduler/redis_to_rpc/__init__.py

+# -*- coding:UTF-8 -*-
+# @Time  : 2021/1/14 19:53
+# @File  : __init__.py.py
+# @email : litao@igengmei.com
+# @author : litao
\ No newline at end of file

crawler_sys/scheduler/redis_to_rpc/rpc_config.py

+# -*- coding:UTF-8 -*-
+# @Time  : 2021/1/14 19:54
+# @File  : rpc_config.py
+# @email : litao@igengmei.com
+# @author : litao
+
+majiayonghu_list = [
+    36436814,
+    36436809,
+    36436805,
+    36436803,
+    36436800,
+    36436797,
+    36436794,
+    36436793,
+    36436787,
+    36436782,
+    36436769,
+    36436763,
+    36436758,
+    36436756,
+    36436749,
+    36436745,
+    36436738,
+    36436731,
+    36436729,
+    36436725,
+    36436720,
+    36436717,
+    36436716,
+    36436709,
+    36436703,
+    36436701,
+    36436690,
+    36436689,
+    36436685,
+    36436674,
+    36426171,
+    36426170,
+    36426169,
+    36426168,
+    36426167,
+    36426166,
+    36426165,
+    36426164,
+    36426163,
+    36426162,
+    36426161,
+    36426160,
+    36426159,
+    36426158,
+    36426157,
+    36426156,
+    36426155,
+    36426154,
+    36426153,
+    36426152,
+    36426150,
+    36426149,
+    36426148,
+    36426147,
+    36426146,
+    36426145,
+    36426143,
+    36426141,
+    36368922,
+    36368921,
+    36368920,
+    36368918,
+    36368917,
+
+]
+user_id_list = [
+                29865245,
+                36426151,
+                36426142,
+                36427666,
+                36427661,
+                36427657,
+                36427655,
+                36427634,
+                33524762,
+                33524779,
+                33524697,
+                30963358,
+                31293584,
+                31358392,
+                31358396,
+                31358397,
+                31358419,
+                31358448,
+                31358610,
+                31358658,
+                ]

crawler_sys/scheduler/weibo_to_rpc.py

@@ -12,6 +12,7 @@ import redis, json
 from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
 from crawler_sys.utils.output_results import retry_get_url
 from crawler.gm_upload.gm_upload import upload, upload_file
+from crawler.crawler_sys.scheduler.redis_to_rpc.rpc_config import *
 
 gm_user_id_list = [
 "3236957071",
@@ -26,160 +27,70 @@ gm_user_id_list = [
 "7048594049",
 ]
 
-majiayonghu_list = [
-    36436814,
-    36436809,
-    36436805,
-    36436803,
-    36436800,
-    36436797,
-    36436794,
-    36436793,
-    36436787,
-    36436782,
-    36436769,
-    36436763,
-    36436758,
-    36436756,
-    36436749,
-    36436745,
-    36436738,
-    36436731,
-    36436729,
-    36436725,
-    36436720,
-    36436717,
-    36436716,
-    36436709,
-    36436703,
-    36436701,
-    36436690,
-    36436689,
-    36436685,
-    36436674,
-    36426171,
-    36426170,
-    36426169,
-    36426168,
-    36426167,
-    36426166,
-    36426165,
-    36426164,
-    36426163,
-    36426162,
-    36426161,
-    36426160,
-    36426159,
-    36426158,
-    36426157,
-    36426156,
-    36426155,
-    36426154,
-    36426153,
-    36426152,
-    36426150,
-    36426149,
-    36426148,
-    36426147,
-    36426146,
-    36426145,
-    36426143,
-    36426141,
-    36368922,
-    36368921,
-    36368920,
-    36368918,
-    36368917,
-
-]
-
-user_id_list = [
-                29865245,
-                36426151,
-                36426142,
-                36427666,
-                36427661,
-                36427657,
-                36427655,
-                36427634,
-                33524762,
-                33524779,
-                33524697,
-                30963358,
-                31293584,
-                31358392,
-                31358396,
-                31358397,
-                31358419,
-                31358448,
-                31358610,
-                31358658,
-                ]
-
-# f= open("josnfile.json","r",encoding='utf-8')
 rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
-pid_list = rds.hkeys("weibo")
-# for line in f:
-for pid in pid_list:
-    res = rds.hget("weibo", pid)
-    if rds.hexists("weibo_with_img", pid):
-        continue
-    res_json = json.loads(res)
-    video_dic = {}
-    qiniu_img_list = []
-    # print(res_json)
-    if "http://t.cn/" in res_json["title"]:
-        continue
-    for img_url in res_json["img_list"]:
-        try:
-            img_wb = retry_get_url(img_url.replace("large", "sq480")).content
-            res = upload(img_wb, img_type=99)
-            # print(res)
-            img_info = retry_get_url(res + "-imageinfo")
-            img_info_json = img_info.json()
-            qiniu_img_list.append('<img src="' + res + '-w">')
-        except Exception as e:
-            print("down load img error %s" % e)
-            continue
-    print(qiniu_img_list)
-    try:
-    # if True:
-        try:
-            title = res_json["title"].split("\n")[0]
-        except:
-            title = res_json["title"]
-        desc_fix = "<p>" + res_json["title"].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
-        res_json["desc_fix"] = desc_fix
-        # print(desc_fix)
-        # f.write(json.dumps(res_json) + "\n")
-        # f.flush()
-        res = rds.hset("weibo_with_img", key=pid, value=json.dumps(res_json))
-        if res_json["releaser_id_str"].replace("weibo_","") in gm_user_id_list:
-            video_dic["level"] = "5"
-        else:
-            video_dic["level"] = "3"
-        video_dic["platform"] = "2"
-        video_dic["platform_id"] = pid
-        video_dic["platform_answer_id"] = pid
-        video_dic["title"] = title
+while True:
+    if rds.hlen("weibo"):
+        pid_list = rds.hkeys("weibo")
+        for pid in pid_list:
+            if rds.sismember("weibo_exists_set", pid):
+                rds.hdel("weibo", pid)
+                continue
+            res = rds.hget("weibo", pid)
+            res_json = json.loads(res)
+            video_dic = {}
+            qiniu_img_list = []
+            # print(res_json)
+            if "http://t.cn/" in res_json["title"]:
+                continue
+            for img_url in res_json["img_list"]:
+                try:
+                    img_wb = retry_get_url(img_url.replace("large", "sq480")).content
+                    res = upload(img_wb, img_type=99)
+                    # print(res)
+                    img_info = retry_get_url(res + "-imageinfo")
+                    img_info_json = img_info.json()
+                    qiniu_img_list.append('<img src="' + res + '-w">')
+                except Exception as e:
+                    print("down load img error %s" % e)
+                    continue
+            print(qiniu_img_list)
+            try:
+            # if True:
+                try:
+                    title = res_json["title"].split("\n")[0]
+                except:
+                    title = res_json["title"]
+                desc_fix = "<p>" + res_json["title"].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
+                res_json["desc_fix"] = desc_fix
+                # print(desc_fix)
+                # f.write(json.dumps(res_json) + "\n")
+                # f.flush()
+                res = rds.hset("weibo_with_img", key=pid, value=json.dumps(res_json))
+                if res_json["releaser_id_str"].replace("weibo_","") in gm_user_id_list:
+                    video_dic["level"] = "5"
+                else:
+                    video_dic["level"] = "3"
+                video_dic["platform"] = "2"
+                video_dic["platform_id"] = pid
+                video_dic["platform_answer_id"] = pid
+                video_dic["title"] = title
+
+                user_id_list_copy = copy.deepcopy(user_id_list)
+                qustion_id = random.choice(user_id_list_copy)
+                user_id_list_copy.remove(qustion_id)
+                video_dic["user_id"] = qustion_id
+                create_time = int(res_json["release_time"]/1e3)
+                video_dic["create_time"] = create_time
+                rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
+                # print(rpc_res)
+                video_dic["platform_question_id"] = pid
+                video_dic["content"] = desc_fix
+                video_dic["user_id"] = random.choice(user_id_list_copy)
+                rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
 
-        user_id_list_copy = copy.deepcopy(user_id_list)
-        qustion_id = random.choice(user_id_list_copy)
-        user_id_list_copy.remove(qustion_id)
-        video_dic["user_id"] = qustion_id
-        create_time = int(res_json["release_time"]/1e3)
-        video_dic["create_time"] = create_time
-        rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
-        # print(rpc_res)
-        video_dic["platform_question_id"] = pid
-        video_dic["content"] = desc_fix
-        video_dic["user_id"] = random.choice(user_id_list_copy)
-        rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
+            except Exception as e:
+                print(e)
+                continue
 
-    except Exception as e:
-        print(e)
-        continue
+            rds.hdel("weibo", pid)
 
-    # break
-# f.flush()
-# f.close()
\ No newline at end of file