update

task/daily_search_word_count_last_two_year.py

@@ -157,6 +157,7 @@ tableDF4 = spark.read.format("jdbc"). \
 task_list = []
 task_days = 2
 
+keyword_dict = {}
 for t in range(1, task_days):
     day_num = 0 - t
     now = (datetime.datetime.now() + datetime.timedelta(days=day_num))
@@ -174,13 +175,15 @@ for t in range(1, task_days):
     select name from wiki_brand where is_online=True
     """
     keyword_df = spark.sql(keyword_sql)
-    keyword_df.createOrReplaceTempView("keywords_table")
+    # keyword_df.createOrReplaceTempView("keywords_table")
+    res = keyword_df.collect()
+    for r in res:
+        keyword_dict[r.name] = 1
 
     sql_search_ctr = r"""
     SELECT query,
            all_search_uv as all_search_uv, --全部搜索uv
            all_search_pv as all_search_pv, --全部搜索pv
-           (if `query` IN(select name from keywords_table), 0, 1) as  is_delete
     FROM (
              --搜索pvuv
              SELECT query
@@ -304,7 +307,7 @@ for t in range(1, task_days):
         uv = name.all_search_uv
         is_tag = tag_dict.get(keywords, 0)
         servise_num = from_es_get_service_num(keywords)
-        is_delete = name.is_delete
+        is_delete = 0 if keyword_dict.get(keywords) else 1
         # pid = hashlib.md5((today_str + keywords).encode("utf8")).hexdigest()
         tag_names_list_week.append((key_count, keywords, today_str, is_delete, servise_num, pv, is_tag))