Skip to content

C
crawler
Commit 381836ae authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent c1a0750f
Hide whitespace changes
Inline Side-by-side
Showing with 11 additions and 11 deletions
crawler_sys/scheduler/push_crawler_data_to_mysql.py
View file @ 381836ae
...@@ -76,17 +76,17 @@ def scan_es_to_mysql(): ...@@ -76,17 +76,17 @@ def scan_es_to_mysql():
print("write data %s %s into sql" % (tractate_id, res["_id"])) print("write data %s %s into sql" % (tractate_id, res["_id"]))
except Exception as e: except Exception as e:
print("send to mysql error %s" % e) print("send to mysql error %s" % e)
if tractate_id: # if tractate_id:
rds.sadd("article_id_list", res["_id"]) # rds.sadd("article_id_list", res["_id"])
search_word = data["search_word"] # search_word = data["search_word"]
if not query_id_dict.get(search_word): # if not query_id_dict.get(search_word):
query_id_dict[search_word] = {} # query_id_dict[search_word] = {}
query_id_dict[search_word][tractate_id] = 1 # query_id_dict[search_word][tractate_id] = 1
count += 1 # count += 1
if count % 200 == 0: # if count % 200 == 0:
send_email(query_id_dict) # send_email(query_id_dict)
query_id_dict = {} # query_id_dict = {}
send_email(query_id_dict) # send_email(query_id_dict)
def send_one_data_to_mysql(_id): def send_one_data_to_mysql(_id):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment