Commit 5c0d4918 authored by litaolemo's avatar litaolemo

update

parent 7c1cfdb5
...@@ -157,6 +157,7 @@ tableDF4 = spark.read.format("jdbc"). \ ...@@ -157,6 +157,7 @@ tableDF4 = spark.read.format("jdbc"). \
task_list = [] task_list = []
task_days = 2 task_days = 2
keyword_dict = {}
for t in range(1, task_days): for t in range(1, task_days):
day_num = 0 - t day_num = 0 - t
now = (datetime.datetime.now() + datetime.timedelta(days=day_num)) now = (datetime.datetime.now() + datetime.timedelta(days=day_num))
...@@ -174,13 +175,15 @@ for t in range(1, task_days): ...@@ -174,13 +175,15 @@ for t in range(1, task_days):
select name from wiki_brand where is_online=True select name from wiki_brand where is_online=True
""" """
keyword_df = spark.sql(keyword_sql) keyword_df = spark.sql(keyword_sql)
keyword_df.createOrReplaceTempView("keywords_table") # keyword_df.createOrReplaceTempView("keywords_table")
res = keyword_df.collect()
for r in res:
keyword_dict[r.name] = 1
sql_search_ctr = r""" sql_search_ctr = r"""
SELECT query, SELECT query,
all_search_uv as all_search_uv, --全部搜索uv all_search_uv as all_search_uv, --全部搜索uv
all_search_pv as all_search_pv, --全部搜索pv all_search_pv as all_search_pv, --全部搜索pv
(if `query` IN(select name from keywords_table), 0, 1) as is_delete
FROM ( FROM (
--搜索pvuv --搜索pvuv
SELECT query SELECT query
...@@ -304,7 +307,7 @@ for t in range(1, task_days): ...@@ -304,7 +307,7 @@ for t in range(1, task_days):
uv = name.all_search_uv uv = name.all_search_uv
is_tag = tag_dict.get(keywords, 0) is_tag = tag_dict.get(keywords, 0)
servise_num = from_es_get_service_num(keywords) servise_num = from_es_get_service_num(keywords)
is_delete = name.is_delete is_delete = 0 if keyword_dict.get(keywords) else 1
# pid = hashlib.md5((today_str + keywords).encode("utf8")).hexdigest() # pid = hashlib.md5((today_str + keywords).encode("utf8")).hexdigest()
tag_names_list_week.append((key_count, keywords, today_str, is_delete, servise_num, pv, is_tag)) tag_names_list_week.append((key_count, keywords, today_str, is_delete, servise_num, pv, is_tag))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment