Skip to content

B
bi-report
Commit 06ab1abf authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 7bf5293a
Hide whitespace changes
Inline Side-by-side
Showing with 31 additions and 21 deletions
pm/daily_content/etl/create_daily_content.sql
View file @ 06ab1abf
......@@ -27,10 +27,11 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_content_v2
home_card_click_uv BIGINT comment '{"chs_name":"首页卡片点击uv","description":"","etl":"","value":"","remark":""}',
home_card_click_pv BIGINT comment '{"chs_name":"首页卡片点击pv","description":"","etl":"","value":"","remark":""}',
home_good_click BIGINT comment '{"chs_name":"来源于首页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
page_stay_per_device BIGINT comment '{"chs_name":"单设备内容浏览时长(m)","description":"","etl":"","value":"","remark":""}',
page_stay_per_device string comment '{"chs_name":"单设备内容浏览时长(m)","description":"","etl":"","value":"","remark":""}',
ai_report_uv BIGINT comment '{"chs_name":"ai结果页uv","description":"","etl":"","value":"","remark":""}',
ai_report_card_click_uv BIGINT comment '{"chs_name":"ai结果页内容卡片点击uv","description":"","etl":"","value":"","remark":""}',
ai_report_card_click_pv BIGINT comment '{"chs_name":"ai结果页内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
ai_report_good_click BIGINT comment '{"chs_name":"来源于结果页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
ai_report_good_click BIGINT comment '{"chs_name":"来源于结果页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}'
)comment '内容日报-简化版'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_content/etl/daily_content.sql
View file @ 06ab1abf
......@@ -23,6 +23,7 @@ SELECT
,NVL(t2.home_click_pv,0) AS home_card_click_pv
,NVL(t2.good_pv,0) AS home_good_click
,NVL(t2.avg_page_stay,0) AS page_stay_per_device
,NVL(t3.ai_report_uv,0) AS ai_report_uv
,NVL(t3.ai_click_uv,0) AS ai_report_card_click_uv
,NVL(t3.ai_click_pv,0) AS ai_report_card_click_pv
,NVL(t3.good_pv,0) AS ai_report_good_click
......@@ -52,9 +53,9 @@ FROM
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -74,7 +75,7 @@ FROM
SELECT device_id
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS partition_date
FROM online.ml_device_day_active_status
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t2
ON t1.device_id=t2.device_id
left join
......@@ -114,9 +115,9 @@ left join
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -137,12 +138,12 @@ left join
,cl_id
,count(distinct time_str) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and referrer='home'
AND action = 'page_view'
-- and params['is_first']=1
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect','welfare_detail')--9月11日新增了可以跳转商详页评论的日记卡片
and page_stay>=20
group by partition_date,cl_id
)t2
......@@ -152,7 +153,7 @@ left join
(
SELECT partition_date,cl_id,count(distinct time_str) as pv
from online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
and params['card_content_type'] in ('diary','user_post','answer','qa')--首页内容卡片点击，未限制tab
......@@ -165,7 +166,7 @@ left join
,cl_id
,sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
......@@ -194,6 +195,7 @@ left join
,device_os_type
,active_type
,channel
,count(distinct t4.cl_id) as ai_report_uv
,count(distinct t3.cl_id) as ai_click_uv
,sum(t3.pv) as ai_click_pv
,count(distinct t2.cl_id) as good_uv
......@@ -213,9 +215,9 @@ left join
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -236,7 +238,7 @@ left join
,cl_id
,count(distinct time_str) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and referrer in ('report_result','face_detect_result','float_tag_detail')
AND action = 'page_view'
-- and params['is_first']=1
......@@ -251,7 +253,7 @@ left join
(
SELECT cl_id,partition_date,count(distinct time_str) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_date >= '20190701' and partition_date <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
and page_name in ('report_result','face_detect_result','float_tag_detail') --ai测颜值、ai测肤质、模拟整形结果页
AND params['card_content_type'] in ('diary','user_post','answer','qa')
......@@ -260,6 +262,18 @@ left join
)t3
ON t1.partition_date=t3.partition_date AND t1.device_id=t3.cl_id
left join
(
SELECT cl_id,partition_date,count(distinct time_str) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20190701' and partition_date <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'page_view'
and page_name in ('report_result','face_detect_result','float_tag_detail') --ai测颜值、ai测肤质、模拟整形结果页
-- AND params['card_content_type'] in ('diary','user_post','answer','qa')
-- AND params['tab_name'] in ('猜你喜欢','为你定制','reference_case')--不限制tab name，只要是从ai结果页到内容页的就算
group by cl_id,partition_date
)t4
ON t1.partition_date=t4.partition_date AND t1.device_id=t4.cl_id
left join
( -- 去掉黑名单设备
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
......
pm/daily_reply_content/report/daily_reply_content.sql
View file @ 06ab1abf
......@@ -72,12 +72,7 @@ left join
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
and is_abnormal_user = 'true'
)c
on a.user_id=c.user_id
where c.user_id is null
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment