Skip to content

B
bi-report
Commit 17aa35bf authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'weiyimin' into 'master' 

Weiyimin

See merge request !54
parents 700c5f5d 90695d7b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 140 additions and 15 deletions
pm/daily_content_data/etl/daily_content_data.sql
View file @ 17aa35bf
......@@ -14,6 +14,8 @@ CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_content_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
T1.partition_date AS day_id
,T1.device_os_type AS device_os_type
......@@ -615,7 +617,8 @@ LEFT JOIN
AND page_name = 'home'
AND action = 'on_click_card'
AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
and params['transaction_type'] not like '%ctr' or params['transaction_type'] like '%cvr')
and params['transaction_type'] not like '%ctr'
and params['transaction_type'] not like '%cvr'
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -628,7 +631,8 @@ LEFT JOIN
AND page_name = 'home'
AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
and params['transaction_type'] not like '%ctr' or params['transaction_type'] like '%cvr')
and params['transaction_type'] not like '%ctr'
and params['transaction_type'] not like '%cvr'
UNION ALL
......@@ -640,7 +644,7 @@ LEFT JOIN
AND page_name = 'home'
AND action = 'on_click_card'
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr'))
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -653,7 +657,7 @@ LEFT JOIN
AND page_name = 'home'
AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr'))
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
)t2
ON t1.partition_date=t2.partition_date
AND t1.device_id=t2.cl_id
......@@ -1165,3 +1169,5 @@ ON T1.partition_date=T9.partition_date
AND T1.channel=T9.channel
ORDER BY day_id desc,device_os_type,active_type,is_ai_channel;
pm/daily_push/etl/create_push_quality.sql 0 → 100644
View file @ 17aa35bf
--***************************************************************
--*脚本名称:
--*功能: push质量数据
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_quality_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
op_only_wel double comment '{"chs_name":"只通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_push_first_wel double comment '{"chs_name":"先通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_open_first_wel double comment '{"chs_name":"先主动打开再运营push-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_only_clue double comment '{"chs_name":"只通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
op_push_first_clue double comment '{"chs_name":"先通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
op_open_first_clue double comment '{"chs_name":"先主动打开再运营push-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_only_wel double comment '{"chs_name":"只通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_push_first_wel double comment '{"chs_name":"先通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_open_first_wel double comment '{"chs_name":"先主动打开再自动化push-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_only_clue double comment '{"chs_name":"只通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_push_first_clue double comment '{"chs_name":"先通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_open_first_clue double comment '{"chs_name":"先主动打开再自动化push-漏到线索","description":"","etl":"","value":"","remark":""}',
)comment 'push质量数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/daily_push.sql
View file @ 17aa35bf
This diff is collapsed.
pm/daily_push/job/step1_10.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/daily_push/job/step1_11.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
pm/daily_push/job/step1_12.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/daily_push/job/step1_13.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_14.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_15.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_16.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/daily_push/job/step1_17.job 0 → 100644
View file @ 17aa35bf
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_push/job/step2.job
View file @ 17aa35bf
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17
command=sh /home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
pm/daily_push/job/step3.job
View file @ 17aa35bf
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
pm/daily_recommend_strategy/etl/create_home_retention.sql 0 → 100644
View file @ 17aa35bf
--***************************************************************
--*脚本名称:
--*功能: 首页推荐策略日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
card_click bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
card_exposure bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
navbar_search bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
highlight_word bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
self_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_content_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_special_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
transfer_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
video_consultation bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}'
)comment '内容日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 17aa35bf
......@@ -71,7 +71,7 @@ FROM
when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
when transaction_type in ('aistragegy') then array('新用户AI帖优先')
when transaction_type in ('excestragegy') then array('新用户精华帖优先')
when transaction_type in ('fixedstragegy') then array(' 新氧新用户策略一')
when transaction_type in ('fixedstragegy') then array('新氧新用户策略一')
when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end
AS recommend_type,
card_id,
......@@ -98,7 +98,7 @@ FROM
when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
when transaction_type in ('aistragegy') then array('新用户AI帖优先')
when transaction_type in ('excestragegy') then array('新用户精华帖优先')
when transaction_type in ('fixedstragegy') then array(' 新氧新用户策略一')
when transaction_type in ('fixedstragegy') then array('新氧新用户策略一')
when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end,
card_id,
app_session_id
......@@ -130,7 +130,7 @@ FROM
when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
when params['transaction_type'] in ('fixedstragegy') then array(' 新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy') then array('新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二')
end AS recommend_type,
params['card_id'] as card_id,
......@@ -156,7 +156,7 @@ FROM
when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
when params['transaction_type'] in ('fixedstragegy') then array(' 新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy') then array('新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二') end,
params['card_id'],
app_session_id
......@@ -756,8 +756,8 @@ LEFT JOIN
FROM online.ml_user_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t1
JOIN
( --医生账号
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
......
pm/daily_recommend_strategy/job/step3.job
View file @ 17aa35bf
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment