Merge branch 'yindanlei' into 'master'

Yindanlei

See merge request !16

pm/doubtful_channel/en-cn.properties

+doubtful_channel=可疑渠道
\ No newline at end of file

pm/doubtful_channel/etl/create_doubtful_channel.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 可疑渠道（有日活和隐私弹窗浏览，但是无其他任何浏览或点击行为）
+--*业务名称: pm
+--*输入数据:
+--*作者: yindanlei@igengmei.com
+--*更新时间:2020-07-17
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_channel_d
+(
+ day_id string comment '{"chs_name":"日期","description":"","etl":"","value":"","remark":""}',
+ device_id string comment '{"chs_name":"设备ID","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ first_channel_source_type string comment '{"chs_name":"首次激活渠道","description":"","etl":"","value":"","remark":""}',
+ first_city string comment '{"chs_name":"首次定位城市","description":"","etl":"","value":"","remark":""}'
+)comment '可疑渠道'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/doubtful_channel/etl/doubtful_channel.sql

+SET mapreduce.job.queuename=data;
+SET mapreduce.map.memory.mb=8192;
+SET mapreduce.map.java.opts=-Xmx8000m;
+SET mapreduce.reduce.memory.mb=8192;
+SET mapreduce.reduce.java.opts=-Xmx8000m;
+set hive.auto.convert.join=true;
+SET mapred.reduce.tasks=20;
+SET role admin;
+
+
+
+INSERT OVERWRITE TABLE pm.tl_pm_channel_d PARTITION (PARTITION_DAY = ${partition_day})
+
+SELECT a.partition_date as day_id
+    ,a.device_id as device_id
+    ,device_os_type,active_type,first_channel_source_type,first_city
+FROM
+(
+    SELECT
+    partition_date,device_os_type
+    ,device_id,first_channel_source_type,first_city
+    ,case WHEN active_type = '4'  THEN '老活'
+          WHEN active_type  in ('1','2')  then '新增' END as active_type
+    FROM ml_device_day_active_status
+    where partition_date >= '20200101'
+    AND partition_date < regexp_replace(current_date ,'-','')
+    AND active_type in ('1','2','4')
+    and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+    ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
+    AND first_channel_source_type not like 'promotion\_jf\_%'
+)a
+LEFT JOIN
+(
+    SELECT partition_date,cl_id
+    from bl_hdfs_maidian_updates
+    where partition_date >= '20200101'
+    AND partition_date < regexp_replace(current_date ,'-','')
+    AND action = 'popup_view'
+    AND params['popup_name']='service_privacy'
+    GROUP BY partition_date,cl_id
+)b
+    ON a.device_id = b.cl_id
+    AND a.partition_date = b.partition_date
+
+LEFT JOIN
+(
+    SELECT partition_date,cl_id
+    from bl_hdfs_maidian_updates
+    where partition_date >= '20200101'
+    AND partition_date < regexp_replace(current_date ,'-','')
+    AND action = 'page_view'
+    GROUP BY partition_date,cl_id
+)c
+    ON a.device_id = c.cl_id
+    AND a.partition_date = c.partition_date
+LEFT JOIN
+(
+    SELECT partition_day,device_id
+    FROM ml.ml_c_et_ck_click_dimen_d
+    where partition_date >= '20200101'
+    AND partition_date < regexp_replace(current_date ,'-','')
+    GROUP BY partition_day,device_id
+)d
+    on a.device_id = d.device_id
+    AND a.partition_date = d.partition_day
+    WHERE b.cl_id is NOT NULL
+    AND c.cl_id is NULL
+    AND d.device_id is NULL
+
+GROUP BY a.partition_date,a.device_id,device_os_type,active_type,first_channel_source_type,first_city
+
+

pm/doubtful_channel/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/doubtful_channel/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/doubtful_channel/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_ck_click_dimen_d
\ No newline at end of file

pm/doubtful_channel/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3
+command=/home/bi/bi-report/lib/shell/hive doubtful_channel
\ No newline at end of file

pm/search_daily_data/en-cn.properties

+search_daily_data=搜索日报
\ No newline at end of file

pm/search_daily_data/etl/create_search_daily_data.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 搜索日报
+--*业务名称: pm
+--*输入数据:
+--*作者: yindanlei@igengmei.com
+--*更新时间:2020-07-13
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_search_d
+(
+ day_id string comment '{"chs_name":"日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ channel string comment '{"chs_name":"渠道类型","description":"","etl":"","value":"","remark":""}',
+ dau int comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
+ all_search_uv int comment '{"chs_name":"总完成搜索uv(含首页猜你喜欢、美购首页大家都在搜)","description":"","etl":"","value":"","remark":""}',
+ all_search_uv/dau double comment '{"chs_name":"总完成搜索uv/DAU","description":"","etl":"","value":"","remark":""}',
+ all_search_pv/all_search_uv double comment '{"chs_name":"总完成搜索pv/总完成搜索uv","description":"","etl":"","value":"","remark":""}',
+ search_result_more_all_click_pv/search_result_more_pv double comment '{"chs_name":"搜索综合页正向点击pv/搜索综合页面pv","description":"","etl":"","value":"","remark":""}',
+ search_result_more_card_click_pv/search_result_more_pv double comment '{"chs_name":"搜索结果综合页所有卡片点击pv(含AI)/搜索结果综合页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_welfare_card_click_pv/search_result_welfare_pv double comment '{"chs_name":"搜索结果美购页所有卡片点击pv/搜索结果美购页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_hospital_card_click_pv/search_result_hospital_pv double comment '{"chs_name":"搜索结果医院页所有卡片点击pv/搜索结果医院页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_doctor_card_click_pv/search_result_doctor_pv double comment '{"chs_name":"搜索结果医生页所有卡片点击pv/搜索结果医生页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_diary_card_click_pv/search_result_diary_pv double comment '{"chs_name":"搜索结果日记页所有卡片点击pv/搜索结果日记页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_post_card_click_pv/search_result_post_pv double comment '{"chs_name":"搜索结果帖子页所有卡片点击pv/搜索结果帖子页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_wiki_card_click_pv/search_result_wiki_pv double comment '{"chs_name":"搜索结果百科页所有卡片点击pv/搜索结果百科页浏览pv","description":"","etl":"","value":"","remark":""}',
+ all_search_click_pv int comment '{"chs_name":"点击搜索框pv(724及以上)","description":"","etl":"","value":"","remark":""}',
+ navbar_search_pv/all_search_click_pv double comment '{"chs_name":"通过搜索框完成搜索总pv/点击搜索框pv(724及以上)","description":"","etl":"","value":"","remark":""}',
+ sug_search_pv/sug_out_pv double comment '{"chs_name":"sug搜索pv/sug弹窗弹出pv(726及以上)","description":"","etl":"","value":"","remark":""}',
+ home_youlike_search_pv/home_you_like_grey_type_pv double comment '{"chs_name":"首页猜你喜欢点击pv/灰度设备首页浏览pv(723及以上5%灰度)","description":"","etl":"","value":"","remark":""}',
+ faxain_search_pv/search_home_pv double comment '{"chs_name":"搜索首页发现搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
+ lishi_search_pv/search_home_pv double comment '{"chs_name":"搜索首页历史搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
+ allsee_search_pv/search_home_pv double comment '{"chs_name":"搜索首页大家都在看搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
+ home_morenci_search_pv/home_search_click_pv double comment '{"chs_name":"首页默认词搜索pv/首页搜索框点击pv(727及以上)","description":"","etl":"","value":"","remark":""}',
+ welfare_home_morenci_search_pv/welfare_home_search_click_pv double comment '{"chs_name":"美购首页默认词搜索pv/美购首页搜索框点击pv(727及以上)","description":"","etl":"","value":"","remark":""}',
+ content_detail_search_pv/content_detail_search_grey_pv double comment '{"chs_name":"内容详情页点击搜索按钮完成默认词搜索pv/灰度设备内容详情页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_welfare_pv/all_search_result_pv double comment '{"chs_name":"搜索结果美购页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_hospital_pv/all_search_result_pv double comment '{"chs_name":"搜索结果医院页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_doctor_pv/all_search_result_pv double comment '{"chs_name":"搜索结果医生页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_diary_pv/all_search_result_pv double comment '{"chs_name":"搜索结果日记页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_post_pv/all_search_result_pv double comment '{"chs_name":"搜索结果帖子页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
+ search_result_wiki_pv/all_search_result_pv double comment '{"chs_name":"搜索结果百科页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}'
+)comment '搜索日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/search_daily_data/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/search_daily_data/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive al al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
\ No newline at end of file

pm/search_daily_data/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_ydl
\ No newline at end of file

pm/search_daily_data/job/step1_4.job

+#step1_4.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/search_daily_data/job/step1_5.job

+#step1_5.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/search_daily_data/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
+command=/home/bi/bi-report/lib/shell/hive search_daily_data
\ No newline at end of file

pm/search_daily_data/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/search_daily_data/dengguangyu@igengmei.com/liudi@igengmei.com,wangxin@igengmei.com,duanyingrong@igengmei.com,lixiaofang@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,wangzhuo@igengmei.com,wangyan@igengmei.com
\ No newline at end of file

pm/search_daily_data/report/search_daily_data.sql

+--搜索日报
+
+SELECT
+		day_id AS `日期`
+		,device_os_type AS `设备类型`
+		,active_type AS `活跃类型`
+		,channel AS `渠道类型`
+		,dau AS `DAU`
+		,all_search_uv AS `总完成搜索uv(含首页猜你喜欢、美购首页大家都在搜)`
+		,all_search_uv/dau AS `总完成搜索uv/DAU`
+		,all_search_pv/all_search_uv AS `总完成搜索pv/总完成搜索uv`
+		,search_result_more_all_click_pv/search_result_more_pv AS `搜索综合页正向点击pv/搜索综合页面pv`
+		,search_result_more_card_click_pv/search_result_more_pv AS `搜索结果综合页所有卡片点击pv(含AI)/搜索结果综合页浏览pv`
+		,search_result_welfare_card_click_pv/search_result_welfare_pv AS `搜索结果美购页所有卡片点击pv/搜索结果美购页浏览pv`
+		,search_result_hospital_card_click_pv/search_result_hospital_pv AS `搜索结果医院页所有卡片点击pv/搜索结果医院页浏览pv`
+		,search_result_doctor_card_click_pv/search_result_doctor_pv AS `搜索结果医生页所有卡片点击pv/搜索结果医生页浏览pv`
+		,search_result_diary_card_click_pv/search_result_diary_pv AS `搜索结果日记页所有卡片点击pv/搜索结果日记页浏览pv`
+		,search_result_post_card_click_pv/search_result_post_pv AS `搜索结果帖子页所有卡片点击pv/搜索结果帖子页浏览pv`
+		,search_result_wiki_card_click_pv/search_result_wiki_pv AS `搜索结果百科页所有卡片点击pv/搜索结果百科页浏览pv`
+		,all_search_click_pv AS `点击搜索框pv(724及以上)`
+		,navbar_search_pv/all_search_click_pv AS `通过搜索框完成搜索总pv/点击搜索框pv(724及以上)`
+		,sug_search_pv/sug_out_pv AS `sug搜索pv/sug弹窗弹出pv(726及以上)`
+		,home_youlike_search_pv/home_you_like_grey_type_pv AS `首页猜你喜欢点击pv/灰度设备首页浏览pv(723及以上5%灰度)`
+		,faxain_search_pv/search_home_pv AS `搜索首页发现搜索pv/搜索首页浏览pv`
+		,lishi_search_pv/search_home_pv AS `搜索首页历史搜索pv/搜索首页浏览pv`
+		,allsee_search_pv/search_home_pv AS `搜索首页大家都在看搜索pv/搜索首页浏览pv`
+		,home_morenci_search_pv/home_search_click_pv AS `首页默认词搜索pv/首页搜索框点击pv(727及以上)`
+		,welfare_home_morenci_search_pv/welfare_home_search_click_pv AS `美购首页默认词搜索pv/美购首页搜索框点击pv(727及以上)`
+		,content_detail_search_pv/content_detail_search_grey_pv AS `内容详情页点击搜索按钮完成默认词搜索pv/灰度设备内容详情页浏览pv`
+		,search_result_welfare_pv/all_search_result_pv AS `搜索结果美购页浏览pv/搜索结果(全部tab)页浏览pv`
+		,search_result_hospital_pv/all_search_result_pv AS `搜索结果医院页浏览pv/搜索结果(全部tab)页浏览pv`
+		,search_result_doctor_pv/all_search_result_pv AS `搜索结果医生页浏览pv/搜索结果(全部tab)页浏览pv`
+		,search_result_diary_pv/all_search_result_pv AS `搜索结果日记页浏览pv/搜索结果(全部tab)页浏览pv`
+		,search_result_post_pv/all_search_result_pv AS `搜索结果帖子页浏览pv/搜索结果(全部tab)页浏览pv`
+		,search_result_wiki_pv/all_search_result_pv AS `搜索结果百科页浏览pv/搜索结果(全部tab)页浏览pv`
+FROM pm.tl_pm_search_d
+where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')