update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -24,10 +24,10 @@ import random
 
 # from mistune import Renderer, InlineGrammar, InlineLexer, Markdown, escape
 rds = redis.StrictRedis(host='154.8.190.251', port=6379, db=19)
-# conn = pymysql.connect(host='bj-cdb-6slgqwlc.sql.tencentcdb.com', port=62120, user='work', passwd='Gengmei1',
-#                        db='mimas_test', charset='utf8')
-conn = pymysql.connect(host='172.16.30.138', port=3306, user='mimas', passwd='GJL3UJe1Ck9ggL6aKnZCq4cRvM',
-                       db='mimas_prod', charset='utf8mb4')
+conn = pymysql.connect(host='bj-cdb-6slgqwlc.sql.tencentcdb.com', port=62120, user='work', passwd='Gengmei1',
+                       db='mimas_test', charset='utf8')
+# conn = pymysql.connect(host='172.16.30.138', port=3306, user='mimas', passwd='GJL3UJe1Ck9ggL6aKnZCq4cRvM',
+#                        db='mimas_prod', charset='utf8mb4')
 cur = conn.cursor()
 
 
@@ -255,13 +255,14 @@ def push_data_to_user(res_data: Dict) -> Dict:
     if res_data["img_list"]:
         for img_url in res_data["img_list"]:
             try:
-                img_wb = retry_get_url(img_url).content
+                img_wb = retry_get_url(img_url,proxies=2).content
                 res = upload(img_wb)
                 print(res)
                 img_info = retry_get_url(res + "-imageinfo")
                 img_info_json = img_info.json()
                 qiniu_img_list.append((res + "-w", img_info_json))
-            except:
+            except Exception as e:
+                print("down load img error %s" % e)
                 return {}
 
         # 替换图片
@@ -391,7 +392,7 @@ def task_main():
     for res_data in scan_from_redis(rules_list):
         tractate_id = write_data_into_mysql(res_data)
         try:
-            print("line394",res_data)
+            print("line394",tractate_id,res_data)
             if res_data["level"] >= 1 and tractate_id:
                 title_str = res_data["platform"] + "帖子内容审核"
                 body_str = """