update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -147,10 +147,11 @@ def scan_from_redis(push_rule_class_list) -> Dict:
                         set_name = "exists_doc_id_set_%s" % datetime.datetime.fromtimestamp(
                             one_data["release_time"] / 1e3).strftime("%Y-%m-%d")
                         if rds.sismember(set_name, one_data["doc_id"]):
-                            continue
+                            break
                         else:
                             rds.sadd(set_name, one_data["doc_id"])
                             yield one_data
+                            break
 
                 print(res_list)
 
@@ -380,7 +381,7 @@ def task_main():
     # 循环处理抓取数据,返回需要添加至后端的数据
     for res_data in scan_from_redis(rules_list):
         tractate_id = write_data_into_mysql(res_data)
-        if res_data["level"] == 5:
+        if res_data["level"] >= 5:
             title_str = res_data["platform"] + "帖子内容审核"
             body_str = """
 问好: