update

task/spark_test.py

@@ -93,14 +93,112 @@ spark.sql("CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDF
 # huidu_device_id_df = spark.sql(huidu_device_id_sql)
 # huidu_device_id_df.createOrReplaceTempView("dev_view")
 sql_search_ctr = r"""
-    SELECT *
-            FROM online.bl_hdfs_maidian_updates
-            WHERE partition_date >= '{start_date}'
-              AND partition_date < '{end_date}'
-              AND action = 'on_click_card'
-              AND params['page_name']='welfare_home'
-              AND params['card_type'] ='search_word'
-              AND params['in_page_pos']='大家都在搜'
+SELECT query
+     , coalesce(t3.all_search_uv, 0) as all_search_uv --全部搜索uv
+     , coalesce(t3.all_search_pv, 0) as all_search_pv --全部搜索pv
+     , t3.partition_date
+FROM (
+         --搜索pvuv
+         SELECT query
+              , click.partition_date        as partition_date
+              , count(click.cl_id)          as all_search_pv
+              , count(distinct click.cl_id) as all_search_uv
+         FROM (
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         params['page_name']  as page_name,
+                         params['input_type'] as input_type,
+                         app_version,
+                         params['query']      as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND ((action = 'do_search' AND params['input_type'] <> 'everyone_watch') or
+                         action = 'search_result_click_search')
+
+                  UNION all
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         params['page_name']  as page_name,
+                         params['input_type'] as input_type,
+                         app_version,
+                         params['query']      as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND action = 'do_search'
+                    and params['input_type'] = 'everyone_watch'
+                    and params['tab'] = '精选'
+                    and page_name = 'home'
+                    AND params['query'] not in ('AI测颜值', 'AI测肤质') --这两个词不跳转搜索结果页
+
+                  union all
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         'search_home'   as page_name,
+                         ''              as input_type,
+                         app_version,
+                         params['query'] as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND action = 'on_click_card'
+                    AND params['page_name'] = 'search_home'
+
+                  union all
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         'home'              as page_name,
+                         '首页-猜你喜欢'           as input_type,
+                         app_version,
+                         params['card_name'] as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND action = 'on_click_card'
+                    AND params['in_page_pos'] = '猜你喜欢'
+                    --AND params['tab_name']='精选'
+                    AND params['card_type'] = 'search_word'
+                    AND params['card_name'] not in ('AI测颜值', 'AI测肤质') --这两个词不跳转搜索结果页
+                    --AND page_name='home' android的page_name为空
+
+                  union all
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         params['page_name'] as page_name,
+                         '美购首页-大家都在搜'        as input_type,
+                         app_version,
+                         params['card_name'] as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND action = 'on_click_card'
+                    AND params['page_name'] = 'welfare_home'
+                    AND params['card_type'] = 'search_word'
+                    AND params['in_page_pos'] = '大家都在搜'
+
+                  union all
+                  SELECT cl_id,
+                         partition_date,
+                         action,
+                         params['page_name'] as page_name,
+                         '高亮词'               as input_type,
+                         app_version,
+                         params['card_name'] as query
+                  FROM online.bl_hdfs_maidian_updates
+                  WHERE partition_date >= '{start_date}'
+                    AND partition_date < '{end_date}'
+                    AND action = 'on_click_card'
+                    AND params['card_type'] = 'highlight_word'
+              ) click
+         GROUP BY click.partition_date, query
+     ) t3
+
 """.format(start_date='20201116',end_date='20201117')
 
 print(sql_search_ctr)