update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -312,6 +312,8 @@ def write_data_into_mysql(res_data):
     now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     # 清洗数据为可以入库的格式
     data = push_data_to_user(res_data)
+    if not data["content"]:
+        return None
     tractate_id = 0
     try:
         sql_query = """insert into api_tractate 
@@ -386,19 +388,23 @@ def task_main():
     # 循环处理抓取数据,返回需要添加至后端的数据
     for res_data in scan_from_redis(rules_list):
         tractate_id = write_data_into_mysql(res_data)
-        if res_data["level"] >= 1:
-            title_str = res_data["platform"] + "帖子内容审核"
-            body_str = """
+        try:
+            print(res_data)
+            if res_data["level"] >= 1 and tractate_id:
+                title_str = res_data["platform"] + "帖子内容审核"
+                body_str = """
 问好:
     有一篇新的内容需要审核,帖子号为{tractate_id}
     内容如下:
 
     {content}
-            """.format(tractate_id=tractate_id, content=res_data["content"])
-            send_file_email("", "", email_group=["‎<duanyingrong@igengmei.com>"], cc_group=["litao@igengmei.com"],
-                            email_msg_body_str=body_str, title_str=title_str)
-            print("send to mysql")
-            print(res_data)
+                """.format(tractate_id=tractate_id, content=res_data["content"])
+                send_file_email("", "", email_group=["‎<duanyingrong@igengmei.com>"], cc_group=["litao@igengmei.com"],
+                                email_msg_body_str=body_str, title_str=title_str)
+                print("send to mysql")
+        except Exception as e:
+            print("send email error %s"%e)
+
     # test = {'release_time': 1595952037000, 'fetch_time': 1596012816514, 'url': 'https://www.douban.com/group/topic/186707979/', 'releaser': '🍫', 'repost_count': 40, 'comment_count': 411, 'favorite_count': 144, 'title': '王俊凯终于还是举铁了', 'releaserUrl': 'https://www.douban.com/people/57762442', 'releaser_id_str': 'douban_57762442', 'video_img': 'https://img3.doubanio.com/view/group_topic/sqxs/public/p317684082.webp', 'mid': '186707979', 'platform': 'douban', 'doc_id': 'douban_186707979', 'content': '<div id=\'content\'><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp" width="500"/></div></div><p></p></div>', 'collection_count': 107, 'img_list': ['https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp'], 'level': 5}
     # write_data_into_mysql(test)
     cur.close()