update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -312,6 +312,8 @@ def write_data_into_mysql(res_data):
     now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     # 清洗数据为可以入库的格式
     data = push_data_to_user(res_data)
+    if not data["content"]:
+        return None
     tractate_id = 0
     try:
         sql_query = """insert into api_tractate 
@@ -386,7 +388,9 @@ def task_main():
     # 循环处理抓取数据,返回需要添加至后端的数据
     for res_data in scan_from_redis(rules_list):
         tractate_id = write_data_into_mysql(res_data)
-        if res_data["level"] >= 1:
+        try:
+            print(res_data)
+            if res_data["level"] >= 1 and tractate_id:
                 title_str = res_data["platform"] + "帖子内容审核"
                 body_str = """
 问好:
@@ -398,7 +402,9 @@ def task_main():
                 send_file_email("", "", email_group=["‎<duanyingrong@igengmei.com>"], cc_group=["litao@igengmei.com"],
                                 email_msg_body_str=body_str, title_str=title_str)
                 print("send to mysql")
-            print(res_data)
+        except Exception as e:
+            print("send email error %s"%e)
+
     # test = {'release_time': 1595952037000, 'fetch_time': 1596012816514, 'url': 'https://www.douban.com/group/topic/186707979/', 'releaser': '🍫', 'repost_count': 40, 'comment_count': 411, 'favorite_count': 144, 'title': '王俊凯终于还是举铁了', 'releaserUrl': 'https://www.douban.com/people/57762442', 'releaser_id_str': 'douban_57762442', 'video_img': 'https://img3.doubanio.com/view/group_topic/sqxs/public/p317684082.webp', 'mid': '186707979', 'platform': 'douban', 'doc_id': 'douban_186707979', 'content': '<div id=\'content\'><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp" width="500"/></div></div><p></p></div>', 'collection_count': 107, 'img_list': ['https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp'], 'level': 5}
     # write_data_into_mysql(test)
     cur.close()