update

task/daily_search_word_count_last_two_year.py

@@ -135,6 +135,18 @@ for t in range(1, task_days):
     today_str = now.strftime("%Y%m%d")
     last_two_year_str = (now + datetime.timedelta(days=-370)).strftime("%Y%m%d")
     # one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
+    keyword_sql = """
+    select name from zhengxing.wiki_item where is_online=True
+    union 
+    select name from zhengxing.wiki_product where is_online=True
+    union 
+    select name from zhengxing.wiki_collect where is_online=True
+    union 
+    select name from zhengxing.wiki_brand where is_online=True
+    """
+    keyword_df = spark.sql(keyword_sql)
+    keyword_df.createOrReplaceTempView("keywords_table")
+
     sql_search_ctr = r"""
     SELECT query,
            partition_date,
@@ -158,7 +170,8 @@ for t in range(1, task_days):
                       WHERE partition_date >= '{start_date}'
                         AND partition_date < '{end_date}'
                         AND ((action = 'do_search' AND params['input_type'] <> 'everyone_watch') or
-                             action = 'search_result_click_search')
+                             action = 'search_result_click_search') 
+                        AND params['query'] in (select * in keywords_table)
 
                       UNION all
                       SELECT cl_id,
@@ -176,6 +189,7 @@ for t in range(1, task_days):
                         and params['tab'] = '精选'
                         and page_name = 'home'
                         AND params['query'] not in ('AI测颜值', 'AI测肤质') --这两个词不跳转搜索结果页
+                         AND params['query'] in (select * in keywords_table)
 
                       union all
                       SELECT cl_id,
@@ -190,6 +204,7 @@ for t in range(1, task_days):
                         AND partition_date < '{end_date}'
                         AND action = 'on_click_card'
                         AND params['page_name'] = 'search_home'
+                        AND params['query'] in (select * in keywords_table)
 
                       union all
                       SELECT cl_id,
@@ -208,6 +223,7 @@ for t in range(1, task_days):
                         AND params['card_type'] = 'search_word'
                         AND params['card_name'] not in ('AI测颜值', 'AI测肤质') --这两个词不跳转搜索结果页
                         --AND page_name='home' android的page_name为空
+                         AND params['card_name'] in (select * in keywords_table)
 
                       union all
                       SELECT cl_id,
@@ -224,6 +240,7 @@ for t in range(1, task_days):
                         AND params['page_name'] = 'welfare_home'
                         AND params['card_type'] = 'search_word'
                         AND params['in_page_pos'] = '大家都在搜'
+                         AND params['card_name'] in (select * in keywords_table)
 
                       union all
                       SELECT cl_id,
@@ -238,6 +255,7 @@ for t in range(1, task_days):
                         AND partition_date < '{end_date}'
                         AND action = 'on_click_card'
                         AND params['card_type'] = 'highlight_word'
+                        AND params['card_name'] in (select * in keywords_table)
                   ) click
              GROUP BY query,partition_date
          ) t3 order by all_search_uv desc