Skip to content

C
crawler
Commit be6cffba authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent e607e45e
Hide whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
crawler_sys/scheduler/push_crawler_data_to_mysql.py
View file @ be6cffba
...@@ -57,7 +57,6 @@ def scan_es_to_mysql(): ...@@ -57,7 +57,6 @@ def scan_es_to_mysql():
scan_res = scan(client=es_framework, query=search_query, index="crawler-data-raw") scan_res = scan(client=es_framework, query=search_query, index="crawler-data-raw")
for res in scan_res: for res in scan_res:
if_exists = rds.sismember("article_id_list", res["_id"]) if_exists = rds.sismember("article_id_list", res["_id"])
if_exists = None
tractate_id = None tractate_id = None
if not if_exists: if not if_exists:
data = res["_source"] data = res["_source"]
...@@ -90,6 +89,7 @@ def send_one_data_to_mysql(_id): ...@@ -90,6 +89,7 @@ def send_one_data_to_mysql(_id):
if search_res["hits"]["hits"]: if search_res["hits"]["hits"]:
res = search_res["hits"]["hits"][0] res = search_res["hits"]["hits"][0]
if_exists = rds.sismember("article_id_list", res["_id"]) if_exists = rds.sismember("article_id_list", res["_id"])
if_exists = None
tractate_id = None tractate_id = None
if not if_exists: if not if_exists:
data = res["_source"] data = res["_source"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment