update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -153,11 +153,11 @@ def scan_from_redis(push_rule_class_list) -> Dict:
                         set_name = "exists_doc_id_set_%s" % datetime.datetime.fromtimestamp(
                             one_data["release_time"] / 1e3).strftime("%Y-%m-%d")
                         if rds.sismember(set_name, one_data["doc_id"]):
-                            break
+                            continue
                         else:
                             rds.sadd(set_name, one_data["doc_id"])
                             yield one_data
-                            break
+                            continue
 
                 # print(res_list)
 
@@ -464,8 +464,8 @@ def task_main():
     # 循环处理抓取数据,返回需要添加至后端的数据
     for res_data in scan_from_redis(rules_list):
         # 符合规则入库数据
-        tractate_id = write_data_into_mysql(res_data)
         try:
+            tractate_id = write_data_into_mysql(res_data)
             print("line394",tractate_id,res_data)
             if res_data["level"] >= 3 and tractate_id:
                 title_str = res_data["platform"] + "帖子内容审核"