update

crawler_sys/scheduler/push_crawler_data_to_mysql.py

@@ -57,6 +57,7 @@ def scan_es_to_mysql():
     scan_res = scan(client=es_framework, query=search_query, index="crawler-data-raw")
     for res in scan_res:
         if_exists = rds.sismember("article_id_list", res["_id"])
+        if_exists = None
         tractate_id = None
         if not if_exists:
             data = res["_source"]
@@ -64,10 +65,10 @@ def scan_es_to_mysql():
             try:
                 tractate_id = write_data_into_mysql(data, user_id_list)
                 print("write data %s %s into sql" % (tractate_id, res["_id"]))
-                rds.sadd("article_id_list", res["_id"])
             except Exception as e:
                 print("send to mysql error %s" % e)
             if tractate_id:
+                rds.sadd("article_id_list", res["_id"])
                 search_word = data["search_word"]
                 query_id_dict[search_word][tractate_id] = 1
     send_email(query_id_dict)
@@ -96,10 +97,11 @@ def send_one_data_to_mysql(_id):
             try:
                 tractate_id = write_data_into_mysql(data, user_id_list)
                 print("write data %s %s into sql"% (tractate_id,res["_id"]))
-                rds.sadd("article_id_list", res["_id"])
+
             except Exception as e:
                 print("send to mysql error %s" % e)
             if tractate_id:
+                rds.sadd("article_id_list", res["_id"])
                 search_word = data["search_word"]
                 query_id_dict[search_word][tractate_id] = 1
     send_email(query_id_dict)