update

crawler_sys/scheduler/cal_ni_and_put_to_backend.py

@@ -477,7 +477,9 @@ def task_main():
 
     {content}
                 """.format(tractate_id=tractate_id, content=res_data["content"],level=res_data["level"])
-                send_file_email("", "", email_group=["<liulingling@igengmei.com>","<liujinhuan@igengmei.com>","‎<hongxu@igengmei.com>","‎<yangjiayue@igengmei.com>","‎<zhangweiwei@igengmei.com>","‎<liuyiting@igengmei.com>"], cc_group=["‎<duanyingrong@igengmei.com>","‎<litao@igengmei.com>"],
+                send_file_email("", "", email_group=["<liulingling@igengmei.com>", "<liujinhuan@igengmei.com>",
+                                                     "‎<hongxu@igengmei.com>", "‎<yangjiayue@igengmei.com>",
+                                                     "‎<zhangweiwei@igengmei.com>","‎<liuyiting@igengmei.com>"], cc_group=["‎<duanyingrong@igengmei.com>","‎<litao@igengmei.com>"],
                                 email_msg_body_str=body_str, title_str=title_str)
                 print("send to mysql")
         except Exception as e:

crawler_sys/scheduler/push_crawler_data_to_mysql.py

@@ -44,6 +44,7 @@ def send_email(query_id_dict: Dict):
 
 def scan_es_to_mysql():
     query_id_dict = {}
+    doc_id_list = []
     search_query = {
         "query": {
             "bool": {
@@ -60,8 +61,14 @@ def scan_es_to_mysql():
         if_exists = rds.sismember("article_id_list", res["_id"])
         tractate_id = None
         if not if_exists:
-            data = res["_source"]
-            data["doc_id"] = res["_id"]
+            doc_id_list.append(res["_id"])
+
+    for doc_id in doc_id_list:
+        try:
+            data = es_framework.get_source(index="crawler-data-raw",id=doc_id)
+        except:
+            continue
+        if data:
             try:
                 tractate_id = write_data_into_mysql(data, user_id_list)
                 print("write data %s %s into sql" % (tractate_id, res["_id"]))
@@ -74,7 +81,7 @@ def scan_es_to_mysql():
                     query_id_dict[search_word] = {}
                 query_id_dict[search_word][tractate_id] = 1
                 count += 1
-        if count % 1000 == 0:
+        if count % 200 == 0:
             send_email(query_id_dict)
             query_id_dict = {}
     send_email(query_id_dict)