Skip to content

B
bi-report
Commit 42c1991d authored by edz's avatar edz
Browse files
Options

Merge branch 'master' of http://git.wanmeizhensuo.com/data/bi-report into hanyingyue

parents 8ea8237b a89761a8
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 608 additions and 121 deletions
pm/daily_content/en-cn.properties
View file @ 42c1991d
......@@ -3,4 +3,5 @@ home_content_detail=首页内容数据-分日明细
home_content_by_month=首页内容数据-月均
ai_content_detail=ai内容数据-分日明细
ai_content_by_month=ai内容数据-月均
home_content_retention=分类用户次留
\ No newline at end of file
home_content_retention=分类用户次留
search_content_detail=搜索到内容的转化
\ No newline at end of file
pm/daily_content/etl/create_daily_content_retention.sql
View file @ 42c1991d
......@@ -21,14 +21,14 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_content_retention
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau BIGINT comment '{"chs_name":"日活","description":"","etl":"","value":"","remark":""}',
retention_rate string comment '{"chs_name":"次留率","description":"","etl":"","value":"","remark":""}',
retention_rate double comment '{"chs_name":"次留率","description":"","etl":"","value":"","remark":""}',
home_good_click_uv BIGINT comment '{"chs_name":"首页good click设备数","description":"","etl":"","value":"","remark":""}',
home_good_click_quality string comment '{"chs_name":"首页gc用户次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
home_good_click_quality double comment '{"chs_name":"首页gc用户次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
home_ungood_click_uv BIGINT comment '{"chs_name":"点击首页卡片但非gc设备数","description":"","etl":"","value":"","remark":""}',
home_ungood_click_quality string comment '{"chs_name":"点击首页卡片但非gc设备次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
home_ungood_click_quality double comment '{"chs_name":"点击首页卡片但非gc设备次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
no_click_uv BIGINT comment '{"chs_name":"未点击首页feed卡片设备数","description":"","etl":"","value":"","remark":""}',
no_click_uv_quality string comment '{"chs_name":"未点击首页feed卡片设备次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
home_good_click_retention_quality string comment '{"chs_name":"当天点击首页feed卡片，且次日依旧点击的次留率/全站次留率","description":"","etl":"","value":"","remark":""}'
no_click_uv_quality double comment '{"chs_name":"未点击首页feed卡片设备次留率/全站次留率","description":"","etl":"","value":"","remark":""}',
home_good_click_retention_quality double comment '{"chs_name":"当天点击首页feed卡片，且次日依旧点击的次留率/全站次留率","description":"","etl":"","value":"","remark":""}'
)comment '内容日报-分用户次留'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_content/etl/create_daily_search_content.sql 0 → 100644
View file @ 42c1991d
--***************************************************************
--*脚本名称:
--*功能: 内容日报-简化版-for 思璟
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_content
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau BIGINT comment '{"chs_name":"日活","description":"","etl":"","value":"","remark":""}',
search_uv BIGINT comment '{"chs_name":"搜索uv","description":"","etl":"","value":"","remark":""}',
search_uv_in_dau string comment '{"chs_name":"搜索uv/dau","description":"","etl":"","value":"","remark":""}',
do_search_uv BIGINT comment '{"chs_name":"完成搜索uv","description":"","etl":"","value":"","remark":""}',
do_search_pv BIGINT comment '{"chs_name":"完成搜索pv","description":"","etl":"","value":"","remark":""}',
do_search_rate string comment '{"chs_name":"完成搜索的用户比例","description":"","etl":"","value":"","remark":""}',
search_times string comment '{"chs_name":"搜索用户人均使用次数","description":"","etl":"","value":"","remark":""}',
search_diary_to_content_uv BIGINT comment '{"chs_name":"来源于搜索日记tab的日记及帖子页uv","description":"","etl":"","value":"","remark":""}',
search_diary_to_content_pv BIGINT comment '{"chs_name":"来源于搜索日记tab的日记及帖子页pv","description":"","etl":"","value":"","remark":""}',
search_diary_to_content_uv_rate string comment '{"chs_name":"来源于搜索日记tab的日记及帖子页uv/搜索uv","description":"","etl":"","value":"","remark":""}',
search_qa_to_content_uv BIGINT comment '{"chs_name":"来源于搜索问答tab的问答页uv","description":"","etl":"","value":"","remark":""}',
search_qa_to_content_pv BIGINT comment '{"chs_name":"来源于搜索问答tab的问答页pv","description":"","etl":"","value":"","remark":""}',
search_qa_to_content_uv_rate string comment '{"chs_name":"来源于搜索问答tab的问答页uv/搜索uv","description":"","etl":"","value":"","remark":""}',
diary_tab_ctr string comment '{"chs_name":"日记tab ctr","description":"","etl":"","value":"","remark":""}',
diary_tab_pv_per_uv string comment '{"chs_name":"日记tab卡片点击pv/uv","description":"","etl":"","value":"","remark":""}',
qa_tab_ctr string comment '{"chs_name":"问答tab ctr","description":"","etl":"","value":"","remark":""}',
qa_tab_pv_per_uv string comment '{"chs_name":"问答tab卡片点击pv/uv","description":"","etl":"","value":"","remark":""}',
content_pv_in_search_uv string comment '{"chs_name":"来源于搜索日记和问答tab的内容页PV/搜索uv","description":"","etl":"","value":"","remark":""}',
diary_second_pv_in_search_uv string comment '{"chs_name":"来源于搜索日记tab的内容二跳PV/搜索uv","description":"","etl":"","value":"","remark":""}',
qa_second_pv_in_search_uv string comment '{"chs_name":"来源于搜索问答tab的内容二跳PV/搜索uv","description":"","etl":"","value":"","remark":""}',
content_pagestay_in_search_uv string comment '{"chs_name":"来源于搜索日记及问答tab的内容总时长/搜索uv(s)","description":"","etl":"","value":"","remark":""}',
avg_contents string comment '{"chs_name":"完成搜索用户人均阅读内容篇数","description":"","etl":"","value":"","remark":""}',
search_retention string comment '{"chs_name":"功能次留","description":"","etl":"","value":"","remark":""}'
)comment '内容日报-搜索到内容的转化'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_content/etl/daily_content.sql
View file @ 42c1991d
This diff is collapsed.
pm/daily_content/job/step3.job
View file @ 42c1991d
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_content/weiyimin@igengmei.com/hanyingyue@igengmei.com,jiaqingqing@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_content/liudi@igengmei.com,shenzheng@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com,hanyingyue@igengmei.com,jiaqingqing@igengmei.com
\ No newline at end of file
pm/daily_content/report/home_content_retention.sql
View file @ 42c1991d
select substr(day_id,1,6) `日期`
,device_os_type `系统`
,active_type `活跃`
,channel `渠道`
,round(avg(home_good_click_uv),0) as `首页good click设备数`
,round(avg(if(home_good_click_quality=0,NULL,home_good_click_quality)),2) as `首页gc用户次留率/全站次留率`
,concat(round(avg(if(home_good_click_quality=0,NULL,home_good_click_quality))*100,2),'%') as `首页gc用户次留率/全站次留率`
,round(avg(home_ungood_click_uv),0) as `点击首页卡片但非gc设备数`
,round(avg(if(home_ungood_click_quality=0,NULL,home_ungood_click_quality)),2) as `点击首页卡片但非gc设备次留率/全站次留率`
,concat(round(avg(if(home_ungood_click_quality=0,NULL,home_ungood_click_quality))*100,2),'%') as `点击首页卡片但非gc设备次留率/全站次留率`
,round(avg(no_click_uv),0) as `未点击首页feed卡片设备数`
,round(avg(if(no_click_uv_quality=0,NULL,no_click_uv_quality)),2) as `未点击首页feed卡片设备次留率/全站次留率`
,round(avg(if(home_good_click_retention_quality=0,NULL,home_good_click_retention_quality)),2) as `当天点击首页feed卡片，且次日依旧点击的次留率/全站次留率`
,concat(round(avg(if(no_click_uv_quality=0,NULL,no_click_uv_quality))*100,2),'%') as `未点击首页feed卡片设备次留率/全站次留率`
,concat(round(avg(if(home_good_click_retention_quality=0,NULL,home_good_click_retention_quality))*100,2),'%') as `当天点击首页feed卡片，且次日依旧点击的次留率/全站次留率`
FROM pm.tl_pm_content_retention
where partition_day>='20201018' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
where ((partition_day='20201110' and day_id<='20201109')
or (partition_day>'20201110' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id=REGEXP_REPLACE(date_sub(concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)),1), '-', ''))
or partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))
group by substr(day_id,1,6),device_os_type,active_type,channel
order by `日期`,`系统`,`活跃`,`渠道`
\ No newline at end of file
pm/daily_content/report/search_content_detail.sql 0 → 100644
View file @ 42c1991d
select day_id as `日期`
,device_os_type as `平台`
,active_type as `活跃`
,channel as `渠道`
,dau as `dau`
,search_uv as `搜索uv`
,search_uv_in_dau as `搜索uv/dau`
,do_search_uv as `完成搜索uv`
,do_search_pv as `完成搜索pv`
,do_search_rate as `完成搜索的用户比例`
,search_times as `搜索用户人均使用次数`
,search_diary_to_content_uv`来源于搜索日记tab的日记及帖子页uv`
,search_diary_to_content_pv `来源于搜索日记tab的日记及帖子页pv`
,search_diary_to_content_uv_rate`来源于搜索日记tab的日记及帖子页uv/搜索uv`
,search_qa_to_content_uv`来源于搜索问答tab的问答页uv`
,search_qa_to_content_pv`来源于搜索问答tab的问答页pv`
,search_qa_to_content_uv_rate`来源于搜索问答tab的问答页uv/搜索uv`
,diary_tab_ctr`日记tab ctr`
,diary_tab_pv_per_uv`日记tab卡片点击pv/uv`
,qa_tab_ctr`问答tab ctr`
,qa_tab_pv_per_uv`问答tab卡片点击pv/uv`
,content_pv_in_search_uv as `来源于搜索日记和问答tab的内容页PV/搜索uv`
,diary_second_pv_in_search_uv`来源于搜索日记tab的内容二跳PV/搜索uv`
,qa_second_pv_in_search_uv`来源于搜索问答tab的内容二跳PV/搜索uv`
,content_pagestay_in_search_uv as `来源于搜索日记及问答tab的内容总时长/搜索uv(s)`
,avg_contents as `完成搜索用户人均阅读内容篇数`
,search_retention as `功能次留`
FROM pm.tl_pm_search_content
where partition_day>='20201109' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期`,`平台`,`活跃`,`渠道`
\ No newline at end of file
pm/daily_grey_recommend/en-cn.properties 0 → 100644
View file @ 42c1991d
daily_grey_recommend=策略灰度实验数据
home_grey_recommend=策略灰度实验数据
pm/daily_grey_recommend/etl/create_daily_grey_recommend.sql 0 → 100644
View file @ 42c1991d
--***************************************************************
--*脚本名称:
--*功能: 策略实验数据日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_grey_recommend_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
retention_rate string comment '{"chs_name":"次留率","description":"","etl":"","value":"","remark":""}',
ctr string comment '{"chs_name":"首页信息流综合ctr","description":"","etl":"","value":"","remark":""}',
card_exp_pv bigint comment '{"chs_name":"卡片曝光pv","description":"","etl":"","value":"","remark":""}',
card_click_pv bigint comment '{"chs_name":"卡片点击pv","description":"","etl":"","value":"","remark":""}',
exp_pv_per_uv double comment '{"chs_name":"人均卡片曝光","description":"","etl":"","value":"","remark":""}',
card_pv_per_uv double comment '{"chs_name":"人均卡片点击","description":"","etl":"","value":"","remark":""}',
card_exp_uv bigint comment '{"chs_name":"卡片曝光uv","description":"","etl":"","value":"","remark":""}',
card_click_uv bigint comment '{"chs_name":"卡片点击uv","description":"","etl":"","value":"","remark":""}',
avg_content_pagestay double comment '{"chs_name":"单内容浏览时长（m）","description":"","etl":"","value":"","remark":""}',
avg_app_duration double comment '{"chs_name":"单设备使用时长（m）","description":"","etl":"","value":"","remark":""}',
pagestay_in_app_duration string comment '{"chs_name":"内容浏览时长在app使用时长上的占比","description":"","etl":"","value":"","remark":""}',
wel_second_in_content_pv string comment '{"chs_name":"来自内容页的商业二跳/内容pv","description":"","etl":"","value":"","remark":""}',
content_second_in_content_pv string comment '{"chs_name":"来自内容页的内容二跳/内容pv","description":"","etl":"","value":"","remark":""}',
home_cardclick_uv_in_home_uv string comment '{"chs_name":"首页卡片点击uv/首页uv","description":"","etl":"","value":"","remark":""}',
home_goodclick_uv_in_homeclick_uv string comment '{"chs_name":"good click卡片数uv/首页点击卡片数uv","description":"","etl":"","value":"","remark":""}'
)comment '策略实验数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_grey_recommend/etl/daily_grey_recommend.sql 0 → 100644
View file @ 42c1991d
This diff is collapsed.
pm/daily_grey_recommend/job/step1_1.job 0 → 100644
View file @ 42c1991d
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_grey_recommend/job/step1_2.job 0 → 100644
View file @ 42c1991d
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
pm/daily_grey_recommend/job/step1_3.job 0 → 100644
View file @ 42c1991d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_grey_recommend/job/step1_4.job 0 → 100644
View file @ 42c1991d
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_grey_recommend/job/step1_5.job 0 → 100644
View file @ 42c1991d
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_grey_recommend/job/step1_6.job 0 → 100644
View file @ 42c1991d
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive DIM DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file
pm/daily_grey_recommend/job/step1_7.job 0 → 100644
View file @ 42c1991d
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
pm/daily_grey_recommend/job/step1_8.job 0 → 100644
View file @ 42c1991d
#step1_8.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
pm/daily_grey_recommend/job/step1_9.job 0 → 100644
View file @ 42c1991d
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/daily_grey_recommend/job/step2.job 0 → 100644
View file @ 42c1991d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=sh /home/bi/bi-report/lib/shell/hive daily_grey_recommend
\ No newline at end of file
pm/daily_grey_recommend/job/step3.job 0 → 100644
View file @ 42c1991d
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_grey_recommend/shenzheng@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,wanglidan@igengmei.com/weiyimin@igengmei.com,hanyingyue@igengmei.com,jiaqingqing@igengmei.com
\ No newline at end of file
pm/daily_grey_recommend/report/home_grey_recommend.sql 0 → 100644
View file @ 42c1991d
SELECT day_id as `日期`
,device_os_type as `系统`
,active_type as `活跃`
,grey_type as `灰度`
,channel as `渠道`
,retention_rate as `次留率`
,ctr as `首页信息流综合ctr`
,card_exp_pv as `卡片曝光pv`
,card_click_pv as `卡片点击pv`
,exp_pv_per_uv`人均卡片曝光`
,card_pv_per_uv`人均卡片点击`
,card_exp_uv as `卡片曝光uv`
,card_click_uv as `卡片点击uv`
,avg_content_pagestay as `单内容浏览时长（m）`
,avg_app_duration as `单设备使用时长（m）`
,pagestay_in_app_duration as `内容浏览时长在app使用时长上的占比`
,wel_second_in_content_pv`来自内容页的商业二跳/内容pv`
,content_second_in_content_pv`来自内容页的内容二跳/内容pv`
,home_cardclick_uv_in_home_uv as `首页卡片点击uv/首页uv`
,home_goodclick_uv_in_homeclick_uv as `good click卡片数uv/首页点击卡片数uv`
from pm.tl_pm_grey_recommend_d
where ((partition_day='20201112' and day_id<='2020-11-11')
or (partition_day>'20201112' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id=date_sub(concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)),1))
or partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))
pm/daily_push/etl/daily_push.sql
View file @ 42c1991d
......@@ -401,6 +401,7 @@ full join
,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (device_type is null or device_type='ios')--在7.35新增了android的埋点，故7.35之后新增了device_type字段
group by partition_day,device_id,msg_id
union all
......
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 42c1991d
......@@ -67,9 +67,9 @@ FROM
cl_id,
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end as card_content_type,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
CASE when transaction_type in ('fmctr','samecity_fmctr') then array('fmctr','合计')
when transaction_type in ('high_quality_fmctr') then array('high_quality_fmctr','合计')
WHEN (transaction_type like '%ctr' and transaction_type not in ('high_quality_ctr','high_quality_fmctr','fmctr')) THEN array('ctr预估','合计')
WHEN (transaction_type like '%ctr' and transaction_type not in ('high_quality_ctr','high_quality_fmctr','fmctr','samecity_fmctr') ) THEN array('ctr预估','合计')
when transaction_type in ('high_quality_ctr') then array('high_quality_ctr','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
......@@ -97,9 +97,9 @@ FROM
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end,
cl_id,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
CASE when transaction_type in ('fmctr','samecity_fmctr') then array('fmctr','合计')
when transaction_type in ('high_quality_fmctr') then array('high_quality_fmctr','合计')
WHEN (transaction_type like '%ctr' and transaction_type not in ('high_quality_ctr','high_quality_fmctr','fmctr')) THEN array('ctr预估','合计')
WHEN (transaction_type like '%ctr' and transaction_type not in ('high_quality_ctr','high_quality_fmctr','fmctr','samecity_fmctr')) THEN array('ctr预估','合计')
when transaction_type in ('high_quality_ctr') then array('high_quality_ctr','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
......@@ -132,9 +132,9 @@ FROM
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end as card_content_type,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
CASE when params['transaction_type'] in ('fmctr','samecity_fmctr') then array('fmctr','合计')
when params['transaction_type'] in ('high_quality_fmctr') then array('high_quality_fmctr','合计')
WHEN (params['transaction_type'] like '%ctr' and params['transaction_type'] not in ('high_quality_ctr','high_quality_fmctr','fmctr')) THEN array('ctr预估','合计')
WHEN (params['transaction_type'] like '%ctr' and params['transaction_type'] not in ('high_quality_ctr','high_quality_fmctr','fmctr','samecity_fmctr')) THEN array('ctr预估','合计')
when params['transaction_type'] in ('high_quality_ctr') then array('high_quality_ctr','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
......@@ -161,9 +161,9 @@ FROM
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
CASE when params['transaction_type'] in ('fmctr','samecity_fmctr') then array('fmctr','合计')
when params['transaction_type'] in ('high_quality_fmctr') then array('high_quality_fmctr','合计')
WHEN (params['transaction_type'] like '%ctr' and params['transaction_type'] not in ('high_quality_ctr','high_quality_fmctr','fmctr')) THEN array('ctr预估','合计')
WHEN (params['transaction_type'] like '%ctr' and params['transaction_type'] not in ('high_quality_ctr','high_quality_fmctr','fmctr','samecity_fmctr')) THEN array('ctr预估','合计')
when params['transaction_type'] in ('high_quality_ctr') then array('high_quality_ctr','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
......@@ -248,7 +248,7 @@ FROM
when page_name in ('custom_special') then 'special' else null end,time_str,page_stay
)a
left join
(
(--在8月份新增了内容专题卡片，需要转换下id
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
......@@ -674,7 +674,7 @@ FROM
partition_date,device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
, '灰度' AS grey_type
,CASE WHEN substr(md5(device_id), -1, 1) IN ('0', '1', '2', '3', '8', 'a', 'b', 'f') THEN '非灰' ELSE '灰度' END AS grey_type
FROM online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
......
pm/daily_reply_content/report/daily_reply_content.sql
View file @ 42c1991d
......@@ -4,14 +4,15 @@ select a.*
FROM
(
--有评论过日记帖的设备,排除疑似广告
SELECT diary_id as content_id,'日记贴' as type,user_id,create_time,content
SELECT t1.id as content_id,'日记贴' as type,user_id,create_time,content
FROM
(
SELECT id,problem_id,user_id,reply_date as create_time,content
FROM online.tl_hdfs_topicreply_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(reply_date,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_online='true'
and regexp_replace(substr(reply_date,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,problem_id,user_id,reply_date,content
)t1
JOIN
......@@ -22,57 +23,60 @@ FROM
group by id,diary_id
)t2
on t2.id=t1.problem_id
group by diary_id,user_id,create_time,content
group by t1.id,user_id,create_time,content
UNION ALL
--有评论过回答的设备,排除疑似广告
SELECT answer_id as content_id,'回答' as type,user_id,create_time,content
SELECT id as content_id,'回答' as type,user_id,create_time,content
FROM online.tl_hdfs_answer_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_fake is NULL or is_fake = 'false')
AND answer_id is not NULL
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by answer_id,user_id,create_time,content
and is_online='true'
and is_spam = 'false' --排除疑似广告
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,user_id,create_time,content
UNION ALL
--有评论过用户帖的设备
SELECT tractate_id as content_id,'帖子' as type,user_id,create_time,content
SELECT id as content_id,'帖子' as type,user_id,create_time,content
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,user_id,create_time,content
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_online='true'
group by id,user_id,create_time,content
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND substr(a.create_time,1,10)= b.partition_date
--JOIN --限制用户是在app进行的回复
--(
-- SELECT a.partition_date,user_id
-- FROM
-- (
-- SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,user_id,device_id,action
-- FROM online.bl_hdfs_operation_updates
-- WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- )a
-- JOIN
-- (
-- SELECT code
-- FROM dim.dim_community_action_type
-- WHERE communityuserbehavior_type_name = '回帖'
-- )type
-- ON a.action = code
-- GROUP BY a.partition_date,user_id
--)b
-- ON a.user_id = b.user_id
-- AND substr(a.create_time,1,10)= b.partition_date
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_abnormal_user = 'true'
and (is_classify_user = 'true' or is_puppet ='true')
)c
on a.user_id=c.user_id
where c.user_id is null
pm/daily_userpost/en-cn.properties
View file @ 42c1991d
daily_userpost=新手精选帖日报
data_by_day=每日数据汇总
key_data=当日数据关键指标
detail_data=当日数据明细
\ No newline at end of file
daily_userpost=新手精选及更美优选帖日报
data_by_day_new=新手精选每日数据汇总
key_data_new=新手精选当日数据关键指标
detail_data_new=新手精选当日数据明细
data_by_day_old=更美优选每日数据汇总
key_data_old=更美优选当日数据关键指标
detail_data_old=更美优选当日数据明细
\ No newline at end of file
pm/daily_userpost/etl/create_userpost_old_v3.sql 0 → 100644
View file @ 42c1991d
--***************************************************************
--*脚本名称:
--*功能: 针对老用户的更美优选标签
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_userpost_old_d_v3
(
post_id string comment '{"chs_name":"帖子id","description":"","etl":"","value":"","remark":""}',
title string comment '{"chs_name":"帖子标题","description":"","etl":"","value":"","remark":""}',
audit_date string comment '{"chs_name":"最新审核时间","description":"","etl":"","value":"","remark":""}',
tag_list string comment '{"chs_name":"关联标签","description":"","etl":"","value":"","remark":""}',
exp_pv_1 bigint comment '{"chs_name":"前1日曝光","description":"","etl":"","value":"","remark":""}',
click_pv_1 bigint comment '{"chs_name":"前1日点击","description":"","etl":"","value":"","remark":""}',
page_pv_1 bigint comment '{"chs_name":"前1日浏览pv","description":"","etl":"","value":"","remark":""}',
page_pv_20s_1 bigint comment '{"chs_name":"前1日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
reply_num_1 bigint comment '{"chs_name":"前1日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_1 bigint comment '{"chs_name":"前1日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_1 bigint comment '{"chs_name":"前1日收藏","description":"","etl":"","value":"","remark":""}',
share_num_1 bigint comment '{"chs_name":"前1日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_1 double comment '{"chs_name":"前1日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
exp_pv_3 bigint comment '{"chs_name":"前3日曝光","description":"","etl":"","value":"","remark":""}',
click_pv_3 bigint comment '{"chs_name":"前3日点击","description":"","etl":"","value":"","remark":""}',
page_pv_3 bigint comment '{"chs_name":"前3日浏览pv","description":"","etl":"","value":"","remark":""}',
page_pv_20s_3 bigint comment '{"chs_name":"前3日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
reply_num_3 bigint comment '{"chs_name":"前3日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_3 bigint comment '{"chs_name":"前3日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_3 bigint comment '{"chs_name":"前3日收藏","description":"","etl":"","value":"","remark":""}',
share_num_3 bigint comment '{"chs_name":"前3日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_3 double comment '{"chs_name":"前3日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
exp_pv_10 bigint comment '{"chs_name":"前10日曝光","description":"","etl":"","value":"","remark":""}',
click_pv_10 bigint comment '{"chs_name":"前10日点击","description":"","etl":"","value":"","remark":""}',
page_pv_10 bigint comment '{"chs_name":"前10日浏览pv","description":"","etl":"","value":"","remark":""}',
page_pv_20s_10 bigint comment '{"chs_name":"前10日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
reply_num_10 bigint comment '{"chs_name":"前10日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_10 bigint comment '{"chs_name":"前10日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_10 bigint comment '{"chs_name":"前10日收藏","description":"","etl":"","value":"","remark":""}',
share_num_10 bigint comment '{"chs_name":"前10日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_10 double comment '{"chs_name":"前10日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
exp_pv bigint comment '{"chs_name":"历史曝光","description":"","etl":"","value":"","remark":""}',
click_pv bigint comment '{"chs_name":"历史点击","description":"","etl":"","value":"","remark":""}',
page_pv bigint comment '{"chs_name":"历史浏览pv","description":"","etl":"","value":"","remark":""}',
page_pv_20s bigint comment '{"chs_name":"历史浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
reply_num bigint comment '{"chs_name":"历史真实评论","description":"","etl":"","value":"","remark":""}',
vote_num bigint comment '{"chs_name":"历史真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num bigint comment '{"chs_name":"历史收藏","description":"","etl":"","value":"","remark":""}',
share_num bigint comment '{"chs_name":"历史转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"历史平均阅读时长(s)","description":"","etl":"","value":"","remark":""}'
)comment '更美优选标签日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_userpost/etl/create_userpost_v2.sql
View file @ 42c1991d
--***************************************************************
--*脚本名称:
--*功能: 线索日报
--*功能: 新手精选标签
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
......
pm/daily_userpost/etl/daily_userpost.sql
View file @ 42c1991d
This diff is collapsed.
pm/daily_userpost/report/data_by_day.sql → pm/daily_userpost/report/data_by_day_new.sql
View file @ 42c1991d
SELECT partition_day as `日期`
,count(post_id) as `帖子数量`
,count(post_id) as `新手精选帖子数量`
,nvl(sum(exp_pv_1),0) as `前1日曝光`
,nvl(concat(round(sum(click_pv_1)/sum(exp_pv_1)*100,2),'%'),0) as `前1日ctr`
,nvl(concat(round(sum(page_pv_20s_1)/sum(exp_pv_1)*100,2),'%'),0) as `日优秀浏览点击ctr`
......
pm/daily_userpost/report/data_by_day_old.sql 0 → 100644
View file @ 42c1991d
SELECT partition_day as `日期`
,count(post_id) as `更美优选帖子数量`
,nvl(sum(exp_pv_1),0) as `前1日曝光`
,nvl(concat(round(sum(click_pv_1)/sum(exp_pv_1)*100,2),'%'),0) as `前1日ctr`
,nvl(concat(round(sum(page_pv_20s_1)/sum(exp_pv_1)*100,2),'%'),0) as `日优秀浏览点击ctr`
,nvl(round(avg(avg_page_stay_1),2),0) as `前1日平均阅读时长(s)`
,nvl(sum(exp_pv),0) as `历史曝光`
,nvl(concat(round(sum(click_pv)/sum(exp_pv)*100,2),'%'),0) as `历史ctr`
,nvl(concat(round(sum(page_pv_20s)/sum(exp_pv)*100,2),'%'),0) as `历史优秀浏览点击ctr`
,nvl(round(avg(avg_page_stay),2),0) as `历史平均阅读时长(s)`
FROM pm.tl_pm_userpost_old_d_v3
where partition_day>='20201102'
group by partition_day
order by `日期`
\ No newline at end of file
pm/daily_userpost/report/detail_data.sql → pm/daily_userpost/report/detail_data_new.sql
View file @ 42c1991d
SELECT post_id as `帖子id`
SELECT post_id as `新手精选帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
......
pm/daily_userpost/report/detail_data_old.sql 0 → 100644
View file @ 42c1991d
SELECT post_id as `更美优选帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
,nvl(concat(round((nvl(reply_num_1,0)+nvl(vote_num_1,0)+nvl(favor_num_1,0)+nvl(share_num_1,0))/page_pv_1*100,2),'%'),0) as `前1日互动率`
,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
,nvl(concat(round(page_pv_20s_1/page_pv_1*100,2),'%'),0) as `前1日浏览20s以上pv/前1日总浏览pv`
,nvl(click_pv_1,0) as `前1日点击`
,nvl(exp_pv_1,0) as `前1日曝光`
,nvl(page_pv_1,0) as `前1日浏览pv`
,nvl(page_pv_20s_1,0) as `前1日浏览20s以上pv`
,nvl(reply_num_1,0) as `前1日真实评论`
,nvl(vote_num_1,0) as `前1日真实点赞`
,nvl(favor_num_1,0) as `前1日收藏`
,nvl(share_num_1,0) as `前1日转发`
,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_3,0)+nvl(vote_num_3,0)+nvl(favor_num_3,0)+nvl(share_num_3,0))/page_pv_3*100,2),'%'),0) as `前3日互动率`
,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
,nvl(concat(round(page_pv_20s_3/page_pv_3*100,2),'%'),0) as `前3日浏览20s以上pv/前3日总浏览pv`
,nvl(click_pv_3,0) as `前3日点击`
,nvl(exp_pv_3,0) as `前3日曝光`
,nvl(page_pv_3,0) as `前3日浏览pv`
,nvl(page_pv_20s_3,0) as `前3日浏览20s以上pv`
,nvl(reply_num_3,0) as `前3日真实评论`
,nvl(vote_num_3,0) as `前3日真实点赞`
,nvl(favor_num_3,0) as `前3日收藏`
,nvl(share_num_3,0) as `前3日转发`
,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_10,0)+nvl(vote_num_10,0)+nvl(favor_num_10,0)+nvl(share_num_10,0))/page_pv_10*100,2),'%'),0) as `前10日互动率`
,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
,nvl(concat(round(page_pv_20s_10/page_pv_10*100,2),'%'),0) as `前10日浏览20s以上pv/前10日总浏览pv`
,nvl(click_pv_10,0) as `前10日点击`
,nvl(exp_pv_10,0) as `前10日曝光`
,nvl(page_pv_10,0) as `前10日浏览pv`
,nvl(page_pv_20s_10,0) as `前10日浏览20s以上pv`
,nvl(reply_num_10,0) as `前10日真实评论`
,nvl(vote_num_10,0) as `前10日真实点赞`
,nvl(favor_num_10,0) as `前10日收藏`
,nvl(share_num_10,0) as `前10日转发`
,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num,0)+nvl(vote_num,0)+nvl(favor_num,0)+nvl(share_num,0))/page_pv*100,2),'%'),0) as `历史互动率`
,nvl(concat(round(click_pv/exp_pv*100,2),'%'),0) as `历史ctr`
,nvl(concat(round(page_pv_20s/page_pv*100,2),'%'),0) as `历史浏览20s以上pv/历史总浏览pv`
,nvl(click_pv,0) as `历史点击`
,nvl(exp_pv,0) as `历史曝光`
,nvl(page_pv,0) as `历史浏览pv`
,nvl(page_pv_20s,0) as `历史浏览20s以上pv`
,nvl(reply_num,0) as `历史真实评论`
,nvl(vote_num,0) as `历史真实点赞`
,nvl(favor_num,0) as `历史收藏`
,nvl(share_num,0) as `历史转发`
,nvl(avg_page_stay,0) as `历史平均阅读时长(s)`
FROM pm.tl_pm_userpost_old_d_v3
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `前10日曝光` desc
pm/daily_userpost/report/key_data.sql → pm/daily_userpost/report/key_data_new.sql
View file @ 42c1991d
SELECT post_id as `帖子id`
SELECT post_id as `新手精选帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
......
pm/daily_userpost/report/key_data_old.sql 0 → 100644
View file @ 42c1991d
SELECT post_id as `更美优选帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
,nvl(concat(round(page_pv_20s_1/page_pv_1*100,2),'%'),0) as `前1日浏览20s以上pv/前1日总浏览pv`
,nvl(exp_pv_1,0) as `前1日曝光`
,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
,nvl(concat(round(page_pv_20s_3/page_pv_3*100,2),'%'),0) as `前3日浏览20s以上pv/前3日总浏览pv`
,nvl(exp_pv_3,0) as `前3日曝光`
,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
,nvl(concat(round(page_pv_20s_10/page_pv_10*100,2),'%'),0) as `前10日浏览20s以上pv/前10日总浏览pv`
,nvl(exp_pv_10,0) as `前10日曝光`
,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
,nvl(concat(round(click_pv/exp_pv*100,2),'%'),0) as `历史ctr`
,nvl(concat(round(page_pv_20s/page_pv*100,2),'%'),0) as `历史浏览20s以上pv/历史总浏览pv`
,nvl(exp_pv,0) as `历史曝光`
,nvl(page_pv,0) as `历史浏览pv`
,nvl(avg_page_stay,0) as `历史平均阅读时长(s)`
FROM pm.tl_pm_userpost_old_d_v3
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `前10日曝光` desc
pm/icon_function_day/etl/icon_function_day.sql
View file @ 42c1991d
......@@ -252,7 +252,10 @@ LEFT JOIN
and referrer='custom_special'
and ((partition_date<'20200924' and params['referrer_id']='605')
or (partition_date>='20200924' and partition_date<'20201020' and params['referrer_id']='707')
or (partition_date>='20201020' and params['referrer_id']='901'))
or (partition_date>='20201020' and partition_date <'20201027' and params['referrer_id']='901')
or (partition_date>='20201027' and partition_date <'20201103' and params['referrer_id']='939')
or (partition_date>='20201103' and partition_date <'20201113' and params['referrer_id']='982')
or (partition_date>='20201113' and partition_date <'20201121' and params['referrer_id']='1055'))
group by partition_date,
cl_id
)t3
......
pm/wel_clue_by_channel/en-cn.properties
View file @ 42c1991d
clue_by_channel=线索pv
welfare_by_channel=核心页pv
wel_clue_by_channel=分渠道3日及7日内核心页和线索pv
\ No newline at end of file
wel_clue_by_channel=分渠道3日及7日内核心页和线索pv
welfare_by_channel_uv=商详页uv
\ No newline at end of file
pm/wel_clue_by_channel/job/step2.job
View file @ 42c1991d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10
command=curl -X GET http://localhost:8553/api/report/email/wel_clue_by_channel/huchunhe@igengmei.com,wangjun@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/wel_clue_by_channel/huchunhe@igengmei.com,wangjun@igengmei.com,zhaoyang@igengmei.com,xukai@igengmei.com,duanchenxia@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/wel_clue_by_channel/report/clue_by_channel.sql
View file @ 42c1991d
......@@ -86,7 +86,7 @@ SELECT
FROM ONLINE.ML_TRADE_ORDER_DETAIL_DAY T
WHERE
T.PARTITION_DATE = regexp_replace(date_sub(current_date(),1),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),8),'MM'),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','')<=regexp_replace(date_sub(current_date,1),'-','')
AND T.is_pure_user='true'
GROUP BY T.USER_ID,T.merchant_id,regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','')
......
pm/wel_clue_by_channel/report/welfare_by_channel.sql
View file @ 42c1991d
......@@ -49,12 +49,12 @@ SELECT
SELECT action_date,cl_id,count(1) as wel_pv
FROM
(
SELECT concat_ws('-',substr(partition_date,0,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS action_date,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),8),'MM'),'-','')
SELECT concat_ws('-',substr(partition_date,0,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS action_date,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),8),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND page_name in ('welfare_detail','organization_detail','expert_detail')
AND action = 'page_view'
AND page_name in ('welfare_detail','organization_detail','expert_detail')
AND action = 'page_view'
)a
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
......
pm/wel_clue_by_channel/report/welfare_by_channel_uv.sql 0 → 100644
View file @ 42c1991d
SELECT
first_active_date as `日期`
,channel as `渠道`
,device_os_type as `平台`
,COUNT(DISTINCT device.device_id) AS `当天新活量`
,nvl(count(distinct IF(pv.action_date=device.first_active_date, pv.cl_id, NULL)),0) AS `当日商详页uv`
,nvl(count(distinct IF(pv.action_date>=device.first_active_date AND pv.action_date<=date_add(device.first_active_date,2), pv.cl_id, NULL)),0) AS `3日内商详页uv`
,nvl(count(distinct IF(pv.action_date>=device.first_active_date AND pv.action_date<=date_add(device.first_active_date,6), pv.cl_id, NULL)),0) AS `7日内商详页uv`
FROM
(
SELECT
first_active_date,device_id,device_os_type
,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type) AS channel
FROM
(
SELECT
device_id,device_os_type
,first_channel_source_type
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS first_active_date
FROM online.ml_device_day_active_status
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),8),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND active_type in ('1','2')
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)device
LEFT JOIN
( SELECT channel,idfa
FROM
(
SELECT channel,idfa,row_number () over (PARTITION BY idfa ORDER BY active_time asc) rn--,substr(active_time,1,7) as new_month
FROM online.tl_hdfs_ios_idfa_tmp--设备表，区分App Store 明细渠道名字
)a
WHERE rn = 1
)ios_device
ON ios_device.idfa = device.device_id
GROUP BY first_active_date,device_id,device_os_type,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type)
)device
LEFT JOIN
(
SELECT action_date,cl_id,count(1) as wel_pv
FROM
(
SELECT concat_ws('-',substr(partition_date,0,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS action_date,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),8),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND page_name in ('welfare_detail')
AND action = 'page_view'
)a
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
SELECT device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY action_date,cl_id
)pv
ON device.device_id = pv.cl_id
GROUP BY
first_active_date,channel,device_os_type
\ No newline at end of file
pm/yeji_day/en-cn.properties
View file @ 42c1991d
yeji_day=业绩基础数据
\ No newline at end of file
yeji_day=分机构转化数据
\ No newline at end of file
pm/yeji_day/etl/create_yeji_day.sql
View file @ 42c1991d
--***************************************************************
--*脚本名称:
--*功能: 业绩基础数据
--*功能: 分机构转化数据
--*业务名称: pm
--*输入数据:
--*作者: yindanlei@igengmei.com
......@@ -17,25 +17,29 @@ USE pm;
CREATE TABLE IF NOT EXISTS pm.tl_pm_yeji_d
(
data string comment '{"chs_name":"日期","description":"","etl":"","value":"","remark":""}',
merchant_name string comment '{"chs_name":"商户名称","description":"","etl":"","value":"","remark":""}',
business_group_name string comment '{"chs_name":"商务组","description":"","etl":"","value":"","remark":""}',
business_partener_name string comment '{"chs_name":"商务名称","description":"","etl":"","value":"","remark":""}',
hospital_name string comment '{"chs_name":"机构名称","description":"","etl":"","value":"","remark":""}',
hospital_id string comment '{"chs_name":"机构id","description":"","etl":"","value":"","remark":""}',
city_name string comment '{"chs_name":"城市","description":"","etl":"","value":"","remark":""}',
hexin_exp_pv BIGINT comment '{"chs_name":"核心卡片曝光pv","description":"","etl":"","value":"","remark":""}',
service_exp_pv BIGINT comment '{"chs_name":"商品卡片曝光pv","description":"","etl":"","value":"","remark":""}',
doc_hos_exp_pv BIGINT comment '{"chs_name":"医生医院卡片曝光pv","description":"","etl":"","value":"","remark":""}',
hexin_pv BIGINT comment '{"chs_name":"核心页pv","description":"","etl":"","value":"","remark":""}',
service_pv BIGINT comment '{"chs_name":"美购页pv","description":"","etl":"","value":"","remark":""}',
service_page_pv BIGINT comment '{"chs_name":"商详页pv","description":"","etl":"","value":"","remark":""}',
doc_hos_page_pv BIGINT comment '{"chs_name":"医生医院页pv","description":"","etl":"","value":"","remark":""}',
click_pv BIGINT comment '{"chs_name":"线索点击人次","description":"","etl":"","value":"","remark":""}',
clue_num BIGINT comment '{"chs_name":"有效线索人次","description":"","etl":"","value":"","remark":""}',
discount BIGINT comment '{"chs_name":"总验证抽成","description":"","etl":"","value":"","remark":""}',
ord_num BIGINT comment '{"chs_name":"验证人次","description":"","etl":"","value":"","remark":""}',
service_price BIGINT comment '{"chs_name":"纯用户支付gmv","description":"","etl":"","value":"","remark":""}',
discount BIGINT comment '{"chs_name":"佣金","description":"","etl":"","value":"","remark":""}',
recharge_amount BIGINT comment '{"chs_name":"广告收款","description":"","etl":"","value":"","remark":""}',
chongzhi_amount BIGINT comment '{"chs_name":"广告消耗","description":"","etl":"","value":"","remark":""}',
all_amount BIGINT comment '{"chs_name":"广告总消耗(含返点)","description":"","etl":"","value":"","remark":""}',
cpc_amount BIGINT comment '{"chs_name":"cpc总消耗","description":"","etl":"","value":"","remark":""}',
cpc_chongzhi_amount BIGINT comment '{"chs_name":"cpc充值消耗","description":"","etl":"","value":"","remark":""}',
budget BIGINT comment '{"chs_name":"cpc商品有效预算","description":"","etl":"","value":"","remark":""}',
cpc_click_num BIGINT comment '{"chs_name":"cpc点击数","description":"","etl":"","value":"","remark":""}',
liulan_amount BIGINT comment '{"chs_name":"浏览收入","description":"","etl":"","value":"","remark":""}',
sixin_amount BIGINT comment '{"chs_name":"私信收入","description":"","etl":"","value":"","remark":""}',
service_price BIGINT comment '{"chs_name":"纯用户支付金额","description":"","etl":"","value":"","remark":""}'
)comment '业绩基础数据'
sixin_amount BIGINT comment '{"chs_name":"私信收入","description":"","etl":"","value":"","remark":""}'
)comment '分机构转化数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
......
pm/yeji_day/etl/yeji_day.sql
View file @ 42c1991d
This diff is collapsed.
pm/yeji_day/job/step1_10.job
View file @ 42c1991d
#step1_10.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_budanluru_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/yeji_day/job/step1_11.job
View file @ 42c1991d
#step1_11.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_maidan_order_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/yeji_day/job/step1_12.job
View file @ 42c1991d
#step1_12.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_mc_merchant_indic_d
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_meigou_service_view
\ No newline at end of file
pm/yeji_day/job/step1_13.job
View file @ 42c1991d
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_mc_merchantadclassify_indic_d
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_budanluru_view
\ No newline at end of file
pm/yeji_day/job/step1_14.job
View file @ 42c1991d
#step1_14.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_cpc_clicklog_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_maidan_order_view
\ No newline at end of file
pm/yeji_day/job/step1_15.job 0 → 100644
View file @ 42c1991d
#step1_14.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_mc_merchant_indic_d
\ No newline at end of file
pm/yeji_day/job/step1_16.job 0 → 100644
View file @ 42c1991d
#step1_14.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_mc_merchantadclassify_indic_d
\ No newline at end of file
pm/yeji_day/job/step1_3.job
View file @ 42c1991d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ML_C_ET_MSG_CONVERSATION_DIMEN_INC_D
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
pm/yeji_day/job/step1_4.job
View file @ 42c1991d
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
pm/yeji_day/job/step1_5.job
View file @ 42c1991d
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ML_C_ET_MSG_CONVERSATION_DIMEN_INC_D
\ No newline at end of file
pm/yeji_day/job/step1_6.job
View file @ 42c1991d
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
pm/yeji_day/job/step1_7.job
View file @ 42c1991d
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/yeji_day/job/step1_8.job
View file @ 42c1991d
#step1_8.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/yeji_day/job/step1_9.job
View file @ 42c1991d
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/yeji_day/job/step2.job
View file @ 42c1991d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16
command=sh /home/bi/bi-report/lib/shell/hive yeji_day
\ No newline at end of file
pm/yeji_day/job/step3.job
View file @ 42c1991d
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/yeji_day/jiaweijie@igengmei.com/zhanghaoyu@igengmei.com,cp-sunyinghe@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/yeji_day/jiaweijie@igengmei.com/cp-sunyinghe@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
pm/yeji_day/report/yeji_day.sql
View file @ 42c1991d
......@@ -2,23 +2,27 @@
SELECT
data AS `日期`
,merchant_name AS `商户名称`
,business_group_name AS `商务组`
,business_partener_name AS `商务名称`
,hospital_name AS `机构名称`
,hospital_id AS `机构id`
,city_name AS `城市`
,hexin_exp_pv AS `核心卡片曝光pv`
,service_exp_pv AS `商品卡片曝光pv`
,doc_hos_exp_pv AS `医生医院卡片曝光pv`
,hexin_pv AS `核心页pv`
,service_pv AS `美购页pv`
,service_page_pv AS `商详页pv`
,doc_hos_page_pv AS `医生医院页pv`
,click_pv AS `线索点击人次`
,clue_num AS `有效线索人次`
,discount AS `总验证抽成`
,ord_num AS `验证人次`
,service_price AS `纯用户支付gmv`
,discount AS `佣金`
,recharge_amount AS `广告收款`
,chongzhi_amount AS `广告消耗`
,all_amount AS `广告总消耗(含返点)`
,cpc_amount AS `cpc总消耗`
,cpc_chongzhi_amount AS `cpc充值消耗`
,budget AS `cpc商品有效预算`
,cpc_click_num AS `cpc点击数`
,liulan_amount AS `浏览收入`
,sixin_amount AS `私信收入`
,service_price AS `纯用户支付金额`
FROM pm.tl_pm_yeji_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment