update

crawler_sys/scheduler/push_crawler_data_to_mysql.py

@@ -29,8 +29,8 @@ def send_email(query_id_dict: Dict):
     新的query:{search_keyword}抓取内容需要审核,帖子号为\n
 """.format(search_keyword=search_keyword, )
                 for tractate_id in query_id_dict[search_keyword]:
-                    body_str += tractate_id + ", "
-                    print("line25", tractate_id)
+                    body_str += str(tractate_id) + ", "
+                    print("line25", str(tractate_id))
 
                 send_file_email("", "",
                                 email_group=["‎<hongxu@igengmei.com>", "‎<yangjiayue@igengmei.com>",
@@ -54,6 +54,7 @@ def scan_es_to_mysql():
             }
         }
     }
+    count = 0
     scan_res = scan(client=es_framework, query=search_query, index="crawler-data-raw")
     for res in scan_res:
         if_exists = rds.sismember("article_id_list", res["_id"])
@@ -70,6 +71,10 @@ def scan_es_to_mysql():
                 rds.sadd("article_id_list", res["_id"])
                 search_word = data["search_word"]
                 query_id_dict[search_word][tractate_id] = 1
+                count += 1
+        if count % 1000 == 0:
+            send_email(query_id_dict)
+            query_id_dict = {}
     send_email(query_id_dict)