Skip to content

B
bi-report
Commit 6e9fff6e authored by edz's avatar edz
Browse files
Options

update codes

parents cc62b428 0a1e615b
Hide whitespace changes
Inline Side-by-side
Showing with 2312 additions and 13 deletions
.idea/workspace.xml
View file @ 6e9fff6e
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<<<<<<< HEAD
<list default="true" id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" /> <list default="true" id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" />
=======
<list default="true" id="034b59f7-f608-4144-b0af-f6951647bd51" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/lib/shell/hive.sh" beforeDir="false" afterPath="$PROJECT_DIR$/lib/shell/hive.sh" afterDir="false" />
</list>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" /> <option name="LAST_RESOLUTION" value="IGNORE" />
</component> </component>
<<<<<<< HEAD
<component name="Git.Settings"> <component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" /> <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component> </component>
...@@ -20,12 +27,61 @@ ...@@ -20,12 +27,61 @@
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" /> <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" /> <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" /> <property name="last_opened_file_path" value="$PROJECT_DIR$" />
=======
<component name="CodeStyleSettingsInfer">
<option name="done" value="true" />
</component>
<component name="ComposerSettings">
<execution>
<executable />
</execution>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectCodeStyleSettingsMigration">
<option name="version" value="1" />
</component>
<component name="ProjectId" id="1dfQ0VEii7hX7vb7p4D7taxyaw0" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showExcludedFiles" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
</component>
<component name="RunManager">
<configuration default="true" type="ClojureREPL" factoryName="Local" activateToolWindowBeforeRun="false">
<setting name="replType" value="NREPL" />
<setting name="execution" value="LEININGEN" />
<setting name="jvmArgs" value="" />
<setting name="parameters" value="" />
<setting name="workingDir" value="" />
<setting name="profiles" value="" />
<setting name="aliases" value="" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
<configuration default="true" type="ClojureREPL" factoryName="Remote" activateToolWindowBeforeRun="false">
<setting name="host" value="" />
<setting name="port" value="0" />
<setting name="replType" value="SOCKET" />
<setting name="configType" value="SPECIFY" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
</component> </component>
<component name="SvnConfiguration"> <component name="SvnConfiguration">
<configuration /> <configuration />
</component> </component>
<component name="TaskManager"> <component name="TaskManager">
<task active="true" id="Default" summary="Default task"> <task active="true" id="Default" summary="Default task">
<<<<<<< HEAD
<changelist id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" /> <changelist id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" />
<created>1592816285414</created> <created>1592816285414</created>
<option name="number" value="Default" /> <option name="number" value="Default" />
...@@ -34,4 +90,18 @@ ...@@ -34,4 +90,18 @@
</task> </task>
<servers /> <servers />
</component> </component>
=======
<changelist id="034b59f7-f608-4144-b0af-f6951647bd51" name="Default Changelist" comment="" />
<created>1592813444854</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1592813444854</updated>
<workItem from="1592813446154" duration="2153000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
</project> </project>
\ No newline at end of file
bin/pull_ydl.sh 0 → 100755
View file @ 6e9fff6e
#!/bin/bash
git checkout yindanlei
git pull origin yindanlei
\ No newline at end of file
conf/en-cn.properties
View file @ 6e9fff6e
...@@ -8,4 +8,8 @@ meigou-detail-page-dispense-pv=美购详情页分发pv ...@@ -8,4 +8,8 @@ meigou-detail-page-dispense-pv=美购详情页分发pv
meigou-detail-page-dispense-uv=美购详情页分发uv meigou-detail-page-dispense-uv=美购详情页分发uv
meigou-detail-page-source-pv=美购详情页来源pv meigou-detail-page-source-pv=美购详情页来源pv
meigou-detail-page-source-uv=美购详情页来源uv meigou-detail-page-source-uv=美购详情页来源uv
daily_content_data=内容日报-新 daily_content_data=内容日报-新
\ No newline at end of file search_click_path=分搜索入口的搜索框点击数据
search_result_click_rate=分搜索词转化率数据
search_input_type=分搜索方式近7天数据
daily_recommend_strategy=首页推荐策略日报
jobs/daily_content_data/step4.job
View file @ 6e9fff6e
#step4.job #step4.job
type=command type=command
dependencies=step3 dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaojianwei@igengmei.com,mazhimo@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com/daily_content_data command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaojianwei@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com/daily_content_data
\ No newline at end of file \ No newline at end of file
jobs/daily_recommend_strategy/step1_1.job 0 → 100644
View file @ 6e9fff6e
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/daily_recommend_strategy/step1_2.job 0 → 100644
View file @ 6e9fff6e
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step1_3.job 0 → 100644
View file @ 6e9fff6e
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/daily_recommend_strategy/step1_4.job 0 → 100644
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/daily_recommend_strategy/step1_5.job 0 → 100644
View file @ 6e9fff6e
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
jobs/daily_recommend_strategy/step1_6.job 0 → 100644
View file @ 6e9fff6e
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
jobs/daily_recommend_strategy/step1_7.job 0 → 100644
View file @ 6e9fff6e
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
jobs/daily_recommend_strategy/step1_8.job 0 → 100644
View file @ 6e9fff6e
#step1_8.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step1_9.job 0 → 100644
View file @ 6e9fff6e
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step2.job 0 → 100644
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/shells/daily_recommend_strategy/daily_recommend_strategy.sh
\ No newline at end of file
jobs/daily_recommend_strategy/step3.job 0 → 100644
View file @ 6e9fff6e
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_recommend_strategy_d
\ No newline at end of file
jobs/daily_recommend_strategy/step4.job 0 → 100644
View file @ 6e9fff6e
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/weiyimin@igengmei.com/zhaojianwei@igengmei.com/daily_recommend_strategy
\ No newline at end of file
jobs/search_related/step1_1.job 0 → 100644
View file @ 6e9fff6e
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/search_related/step2.job 0 → 100644
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2
command=curl -X GET http://localhost:8553/api/report/execSql?name=search_related_insert
\ No newline at end of file
jobs/search_related/step3.job 0 → 100644
View file @ 6e9fff6e
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_click_path_d
\ No newline at end of file
jobs/search_related/step4.job 0 → 100644
View file @ 6e9fff6e
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/weiyimin@igengmei.com/search_related
\ No newline at end of file
lib/shell/hive.sh
View file @ 6e9fff6e
#!/bin/bash #!/bin/bash
# 定义变量方便修改 # 定义变量方便修改
hive=/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline hive="/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online -n data"
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天 # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n $1 ] ;then if [ -n "$1" ] ;then
partition_day=$1 partition_day=$1
else else
partition_day=`date --date="$V_PARYMD -1 day" +%Y%m%d` partition_day=`date --date="-1 day" +%Y%m%d`
fi fi
sql=" sql="
...@@ -236,7 +236,7 @@ FROM ...@@ -236,7 +236,7 @@ FROM
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','') AND partition_date<regexp_replace((current_date),'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏' AND (get_json_object(params['extra_param'], '$.type')='���互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link') or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
AND action='on_click_card' AND action='on_click_card'
and params['card_content_type']='service' and params['card_content_type']='service'
......
shells/daily_recommend_strategy/daily_recommend_strategy.sh 0 → 100755
View file @ 6e9fff6e
#!/bin/bash
# 定义变量方便修改
#hive="/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010 -n data"
hive="/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter005:10000 -n data"
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
partition_day=$1
else
partition_day=`date --date="-1 day" +%Y%m%d`
fi
sql="
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY =$partition_day)
SELECT
t1.partition_date as day_id,
t1.device_os_type as device_os_type,
t1.active_type as active_type,
t2.card_content_type as card_content_type,
t2.recommend_type as recommend_type,
NVL(concat(round(sum(t3.session_pv)/sum(t2.session_pv)*100,2),'%'),0) as home_ctr,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t2.session_pv)*100,2),'%'),0) as second_rate,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t3.session_pv)*100,2),'%'),0) as first_rate,
NVL(sum(t3.session_pv),0) as card_click,
NVL(sum(t2.session_pv),0) as card_exposure,
NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)+NVL(sum(self_wel_pv),0)+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0) as total_second_click,
NVL(round(sum(page_stay)/count(distinct t4.cl_id)/60,2),0) as avg_page_stay,
NVL(sum(navbar_pv),0) as navbar_search,
NVL(sum(highlight_pv),0) as highlight_word,
NVL(sum(self_wel_pv),0) as self_welfare_card,
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
JOIN
(--精准曝光，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
END AS recommend_type,
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND card_content_type in ('qa','diary','user_post','answer')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
cl_id,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片' END,
card_id,
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t2
on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
LEFT JOIN
(--卡片，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
END AS recommend_type,
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片' END,
params['card_id'],
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t3
on t2.partition_date=t3.partition_date
and t2.cl_id=t3.cl_id
and t2.card_id=t3.card_id
and t2.card_content_type=t3.card_content_type
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
and t4.business_id=t3.card_id
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
and t5.business_id=t3.card_id
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
and t6.business_id=t3.card_id
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
and t7.business_id=t3.card_id
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
and t8.business_id=t3.card_id
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=t2.cl_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
"
$hive -e "$sql"
\ No newline at end of file
sqls/daily_content_data_insert/daily_content_data_insert.sql
View file @ 6e9fff6e
...@@ -535,7 +535,7 @@ LEFT JOIN ...@@ -535,7 +535,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','') AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home' AND page_name = 'home'
AND action = 'on_click_card' AND action = 'on_click_card'
AND params['transaction_type'] not in ('-1','ctr','smr','newdata') AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article') AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL UNION ALL
...@@ -547,7 +547,7 @@ LEFT JOIN ...@@ -547,7 +547,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','') AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home' AND page_name = 'home'
AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card') AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] not in ('-1','ctr','smr','newdata') AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata')
UNION ALL UNION ALL
...@@ -558,7 +558,7 @@ LEFT JOIN ...@@ -558,7 +558,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','') AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home' AND page_name = 'home'
AND action = 'on_click_card' AND action = 'on_click_card'
AND params['transaction_type'] in ('-1','ctr','smr','newdata') AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article') AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL UNION ALL
...@@ -570,7 +570,7 @@ LEFT JOIN ...@@ -570,7 +570,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','') AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home' AND page_name = 'home'
AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card') AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] in ('-1','ctr','smr','newdata') AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata')
)t2 )t2
ON t1.partition_date=t2.partition_date ON t1.partition_date=t2.partition_date
AND t1.device_id=t2.cl_id AND t1.device_id=t2.cl_id
......
sqls/daily_recommend_strategy/daily_recommend_strategy.sql 0 → 100644
View file @ 6e9fff6e
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
home_ctr as `首页卡片点击PV/首页卡片精准曝光PV`,
second_rate as `来自首页推荐内容卡片的的总商业二跳pv/首页卡片精准曝光PV`,
first_rate as `来自首页推荐内容卡片的的总商业二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
total_second_click as `来自I的总商业二跳pv（包含M列到T列）`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`
sqls/daily_recommend_strategy_insert/daily_recommend_strategy_insert.sql 0 → 100644
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY =#partition_day)
SELECT
t1.partition_date as day_id,
t1.device_os_type as device_os_type,
t1.active_type as active_type,
t2.card_content_type as card_content_type,
t2.recommend_type as recommend_type,
NVL(concat(round(sum(t3.session_pv)/sum(t2.session_pv)*100,2),'%'),0) as home_ctr,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t2.session_pv)*100,2),'%'),0) as second_rate,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t3.session_pv)*100,2),'%'),0) as first_rate,
NVL(sum(t3.session_pv),0) as card_click,
NVL(sum(t2.session_pv),0) as card_exposure,
NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)+NVL(sum(self_wel_pv),0)+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0) as total_second_click,
NVL(round(sum(page_stay)/count(distinct t4.cl_id)/60,2),0) as avg_page_stay,
NVL(sum(navbar_pv),0) as navbar_search,
NVL(sum(highlight_pv),0) as highlight_word,
NVL(sum(self_wel_pv),0) as self_welfare_card,
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
JOIN
(--精准曝光，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
END AS recommend_type,
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND card_content_type in ('qa','diary','user_post','answer')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
cl_id,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片' END,
card_id,
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t2
on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
LEFT JOIN
(--卡片，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
END AS recommend_type,
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片' END,
params['card_id'],
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t3
on t2.partition_date=t3.partition_date
and t2.cl_id=t3.cl_id
and t2.card_id=t3.card_id
and t2.card_content_type=t3.card_content_type
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
and t4.business_id=t3.card_id
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
and t5.business_id=t3.card_id
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
and t6.business_id=t3.card_id
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
and t7.business_id=t3.card_id
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
and t8.business_id=t3.card_id
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=t2.cl_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type
sqls/search_related/search_click_path.sql 0 → 100644
View file @ 6e9fff6e
SELECT
day_id AS `日期`
,device_os_type AS `设备类型`
,active_type AS `活跃类型`
,search_entrance AS `搜索入口`
,click_search_pv AS `点击搜索框pv`
,click_search_uv AS `点击搜索框uv`
FROM pm.tl_pm_search_click_path_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`点击搜索框pv`desc
\ No newline at end of file
sqls/search_related/search_input_type.sql 0 → 100644
View file @ 6e9fff6e
SELECT
day_id AS `日期`
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_pv*100/t2.page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_10_pv*100/t2.page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_diary_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_answer_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_topic_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_10_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_5_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_10_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_5_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
,more_click_10_pv AS `搜索综合结果页前10内容卡片点击PV`
,more_click_5_diary_pv AS `搜索综合结果页前5日记卡片点击PV`
,more_click_5_answer_pv AS `搜索综合结果页前5问答卡片点击PV`
,more_click_5_topic_pv AS `搜索综合结果页前5帖子卡片点击PV`
,search_wel_pv AS `美购tab下的搜索的点击pv`
,page_wel_pv AS `搜索美购结果页浏览pv`
,wel_click_pv AS `搜索美购结果页卡片点击PV`
,wel_click_10_pv AS `搜索美购结果页前10卡片点击PV`
,wel_click_5_pv AS `搜索美购结果页前5卡片点击PV`
,search_diary_pv AS `日记tab下的搜索的点击pv`
,page_diary_pv AS `搜索日记结果页浏览pv`
,diary_click_pv AS `搜索日记结果页卡片点击PV`
,diary_click_10_pv AS `搜索日记结果页前10卡片点击PV`
,diary_click_5_pv AS `搜索日记结果页前5卡片点击PV`
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
sqls/search_related/search_result_click_rate.sql 0 → 100644
View file @ 6e9fff6e
SELECT
day_id AS `日期`
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_pv*100/t2.page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_10_pv*100/t2.page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_diary_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_answer_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_topic_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_10_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_5_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_10_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_5_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
,more_click_10_pv AS `搜索综合结果页前10内容卡片点击PV`
,more_click_5_diary_pv AS `搜索综合结果页前5日记卡片点击PV`
,more_click_5_answer_pv AS `搜索综合结果页前5问答卡片点击PV`
,more_click_5_topic_pv AS `搜索综合结果页前5帖子卡片点击PV`
,search_wel_pv AS `美购tab下的搜索的点击pv`
,page_wel_pv AS `搜索美购结果页浏览pv`
,wel_click_pv AS `搜索美购结果页卡片点击PV`
,wel_click_10_pv AS `搜索美购结果页前10卡片点击PV`
,wel_click_5_pv AS `搜索美购结果页前5卡片点击PV`
,search_diary_pv AS `日记tab下的搜索的点击pv`
,page_diary_pv AS `搜索日记结果页浏览pv`
,diary_click_pv AS `搜索日记结果页卡片点击PV`
,diary_click_10_pv AS `搜索日记结果页前10卡片点击PV`
,diary_click_5_pv AS `搜索日记结果页前5卡片点击PV`
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
sqls/search_related_insert/search_click_path_insert.sql 0 → 100644
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = #partition_day)
SELECT day_id as day_id
,device_os_type as device_os_type
,active_type as active_type
,v.search_entrance as search_entrance
,sum(search_pv) as click_search_pv
,sum(search_uv) as click_search_uv
FROM
(
SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance
,search_pv,search_uv
FROM
(
SELECT t1.partition_date as day_id
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end as search_entrance
,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv
FROM
(
SELECT partition_date,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,device_id
FROM online.ml_device_day_active_status
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
join
(
SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action in ('do_search','on_click_navbar_search')
union all
SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date,cl_id,null as page_name,'热搜词' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='on_click_card'
and params['card_type']='highlight_word'
)t2
on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
group by t1.partition_date
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
)n
LATERAL VIEW explode(n.search_entrance) v AS search_entrance
group by day_id,device_os_type,active_type,v.search_entrance
sqls/search_related_insert/search_input_type_insert.sql 0 → 100644
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_search_result_click_rate_d PARTITION (PARTITION_DAY = #partition_day)
--搜索关键词日报,取近7天数据，每天取关键词搜索量前500
select t1.partition_date as day_id
,t1.query as query
,NVL(t1.search_pv,0) as search_pv
,NVL(t1.search_uv,0) as search_uv
,NVL(t2.search_more_pv,0) as search_more_pv
,NVL(t2.page_more_pv,0) as page_more_pv
,NVL(t2.more_click_pv,0) as more_click_pv
,NVL(t2.more_click_10_pv,0) as more_click_10_pv
,NVL(t2.more_click_5_diary_pv,0) as more_click_5_diary_pv
,NVL(t2.more_click_5_answer_pv,0) as more_click_5_answer_pv
,NVL(t2.more_click_5_topic_pv,0) as more_click_5_topic_pv
,NVL(t2.search_wel_pv,0) as search_wel_pv
,NVL(t2.page_wel_pv,0) as page_wel_pv
,NVL(t2.wel_click_pv,0) as wel_click_pv
,NVL(t2.wel_click_10_pv,0) as wel_click_10_pv
,NVL(t2.wel_click_5_pv,0) as wel_click_5_pv
,NVL(t2.search_diary_pv,0) as search_diary_pv
,NVL(t2.page_diary_pv,0) as page_diary_pv
,NVL(t2.diary_click_pv,0) as diary_click_pv
,NVL(t2.diary_click_10_pv,0) as diary_click_10_pv
,NVL(t2.diary_click_5_pv,0) as diary_click_5_pv
,NVL(t2.search_more_uv,0) as search_more_uv
,NVL(t2.page_more_uv,0) as page_more_uv
,NVL(t2.more_click_uv,0) as more_click_uv
,NVL(t2.more_click_10_uv,0) as more_click_10_uv
,NVL(t2.more_click_5_diary_uv,0) as more_click_5_diary_uv
,NVL(t2.more_click_5_answer_uv,0) as more_click_5_answer_uv
,NVL(t2.more_click_5_topic_uv,0) as more_click_5_topic_uv
,NVL(t2.search_wel_uv,0) as search_wel_uv
,NVL(t2.page_wel_uv,0) as page_wel_uv
,NVL(t2.wel_click_uv,0) as wel_click_uv
,NVL(t2.wel_click_10_uv,0) as wel_click_10_uv
,NVL(t2.wel_click_5_uv,0) as wel_click_5_uv
,NVL(t2.search_diary_uv,0) as search_diary_uv
,NVL(t2.page_diary_uv,0) as page_diary_uv
,NVL(t2.diary_click_uv,0) as diary_click_uv
,NVL(t2.diary_click_10_uv,0) as diary_click_10_uv
,NVL(t2.diary_click_5_uv,0) as diary_click_5_uv
from
(--总搜索量
select partition_date
,query
,search_pv
,search_uv
from
(
select t1.partition_date
,query
,count(t1.cl_id) as search_pv
,count(distinct t1.cl_id) as search_uv
,row_number() over(partition by partition_date order by count(t1.cl_id) desc) as rank
from
(
select partition_date
,params['query'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('do_search','search_result_click_search')
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date
,params['card_name'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and int(split(app_version,'\\.')[1]) >= 27
and action='on_click_card'
and params['card_type']='highlight_word'
)t1
join
(
SELECT device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t2
on t1.cl_id=t2.device_id
group by t1.partition_date,query
)t
where rank<=300
)t1
left join
--综合、美购、日记页面浏览pv
(
select t1.partition_date,t1.query
,sum(case when tab_name='综合' then t1.pv end) as search_more_pv
,count(distinct case when tab_name='综合' then t1.cl_id end) as search_more_uv
,sum(case when tab_name='美购' then t1.pv end) as search_wel_pv
,count(distinct case when tab_name='美购' then t1.cl_id end) as search_wel_uv
,sum(case when tab_name='日记' then t1.pv end) as search_diary_pv
,count(distinct case when tab_name='日记' then t1.cl_id end) as search_diary_uv
,sum(case when t2.page_name='综合' then t2.pv end) as page_more_pv
,count(distinct case when t2.page_name='综合' then t2.cl_id end) as page_more_uv
,sum(case when t2.page_name='美购' then t2.pv end) as page_wel_pv
,count(distinct case when t2.page_name='美购' then t2.cl_id end) as page_wel_uv
,sum(case when t2.page_name='日记' then t2.pv end) as page_diary_pv
,count(distinct case when t2.page_name='日记' then t2.cl_id end) as page_diary_uv
,sum(case when t3.page_name='综合' then t3.pv end) as more_click_pv
,count(distinct case when t3.page_name='综合' then t3.cl_id end) as more_click_uv
,sum(case when t3.page_name='美购' then t3.pv end) as wel_click_pv
,count(distinct case when t3.page_name='美购' then t3.cl_id end) as wel_click_uv
,sum(case when t3.page_name='日记' then t3.pv end) as diary_click_pv
,count(distinct case when t3.page_name='日记' then t3.cl_id end) as diary_click_uv
,sum(case when t4.page_name='综合' then t4.pv end) as more_click_10_pv
,count(distinct case when t4.page_name='综合' then t4.cl_id end) as more_click_10_uv
,sum(case when t4.page_name='美购' then t4.pv end) as wel_click_10_pv
,count(distinct case when t4.page_name='美购' then t4.cl_id end) as wel_click_10_uv
,sum(case when t4.page_name='日记' then t4.pv end) as diary_click_10_pv
,count(distinct case when t4.page_name='日记' then t4.cl_id end) as diary_click_10_uv
,sum(case when t5.page_name='美购' then t5.pv end) as wel_click_5_pv
,count(distinct case when t5.page_name='美购' then t5.cl_id end) as wel_click_5_uv
,sum(case when t5.page_name='日记' then t5.pv end) as diary_click_5_pv
,count(distinct case when t5.page_name='日记' then t5.cl_id end) as diary_click_5_uv
,sum(case when t6.page_name='综合' then t6.pv end) as more_click_5_diary_pv
,count(distinct case when t6.page_name='综合' then t6.cl_id end) as more_click_5_diary_uv
,sum(case when t7.page_name='综合' then t7.pv end) as more_click_5_answer_pv
,count(distinct case when t7.page_name='综合' then t7.cl_id end) as more_click_5_answer_uv
,sum(case when t8.page_name='综合' then t8.pv end) as more_click_5_topic_pv
,count(distinct case when t8.page_name='综合' then t8.cl_id end) as more_click_5_topic_uv
from
(--各tab搜索
select partition_date
,params['query'] as query
,cl_id
,params['tab'] as tab_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and params['tab'] in ('综合','美购','日记')
and action in ('search_result_click_tab','search_result_click_search')
group by partition_date,params['query'],cl_id,params['tab']
)t1
left join
(--结果页浏览
select partition_date
,get_json_object(params['extra_param'],'$.query') as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
and action ='page_view'
group by partition_date
,get_json_object(params['extra_param'],'$.query')
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
and t1.cl_id=t2.cl_id
and t1.tab_name=t2.page_name
left join
(--内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t3
on t2.partition_date=t3.partition_date
and t2.query=t3.query
and t2.cl_id=t3.cl_id
and t2.page_name=t3.page_name
left join
(--前10内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and params['position'] in (0,1,2,3,4,5,6,7,8,9)
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t4
on t2.partition_date=t4.partition_date
and t2.query=t4.query
and t2.cl_id=t4.cl_id
and t2.page_name=t4.page_name
left join
(--日记、美购前5内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and ( action = 'on_click_diary_card' and page_name='search_result_diary'
or action ='search_result_welfare_click_item' and page_name='search_result_welfare')
and params['position'] in (0,1,2,3,4)
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t5
on t2.partition_date=t5.partition_date
and t2.query=t5.query
and t2.cl_id=t5.cl_id
and t2.page_name=t5.page_name
left join
(--综搜页日记卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='diary'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item'and params['business_type']in('diary','0')
or action ='on_click_diary_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t6
on t2.partition_date=t6.partition_date
and t2.query=t6.query
and t2.cl_id=t6.cl_id
and t2.page_name=t6.page_name
left join
(--综搜页回答卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='answer'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action ='search_result_click_infomation_item'
and params['business_type']in ('answer','5')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t7
on t2.partition_date=t7.partition_date
and t2.query=t7.query
and t2.cl_id=t7.cl_id
and t2.page_name=t7.page_name
left join
(--综搜页帖子卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='user_post'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item' and params['business_type'] in ('post','user_post','doctor_post','11')
or action='on_click_topic_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t8
on t2.partition_date=t8.partition_date
and t2.query=t8.query
and t2.cl_id=t8.cl_id
and t2.page_name=t8.page_name
join
(
select device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
on t1.cl_id=dev.device_id
left join
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t2.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
group by t1.partition_date,t1.query
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
order by day_id desc,search_pv desc,query
sqls/search_related_insert/search_result_click_rate_insert.sql 0 → 100644
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_search_result_click_rate_d PARTITION (PARTITION_DAY = #partition_day)
--搜索关键词日报,取近7天数据，每天取关键词搜索量前500
select t1.partition_date as day_id
,t1.query as query
,NVL(t1.search_pv,0) as search_pv
,NVL(t1.search_uv,0) as search_uv
,NVL(t2.search_more_pv,0) as search_more_pv
,NVL(t2.page_more_pv,0) as page_more_pv
,NVL(t2.more_click_pv,0) as more_click_pv
,NVL(t2.more_click_10_pv,0) as more_click_10_pv
,NVL(t2.more_click_5_diary_pv,0) as more_click_5_diary_pv
,NVL(t2.more_click_5_answer_pv,0) as more_click_5_answer_pv
,NVL(t2.more_click_5_topic_pv,0) as more_click_5_topic_pv
,NVL(t2.search_wel_pv,0) as search_wel_pv
,NVL(t2.page_wel_pv,0) as page_wel_pv
,NVL(t2.wel_click_pv,0) as wel_click_pv
,NVL(t2.wel_click_10_pv,0) as wel_click_10_pv
,NVL(t2.wel_click_5_pv,0) as wel_click_5_pv
,NVL(t2.search_diary_pv,0) as search_diary_pv
,NVL(t2.page_diary_pv,0) as page_diary_pv
,NVL(t2.diary_click_pv,0) as diary_click_pv
,NVL(t2.diary_click_10_pv,0) as diary_click_10_pv
,NVL(t2.diary_click_5_pv,0) as diary_click_5_pv
,NVL(t2.search_more_uv,0) as search_more_uv
,NVL(t2.page_more_uv,0) as page_more_uv
,NVL(t2.more_click_uv,0) as more_click_uv
,NVL(t2.more_click_10_uv,0) as more_click_10_uv
,NVL(t2.more_click_5_diary_uv,0) as more_click_5_diary_uv
,NVL(t2.more_click_5_answer_uv,0) as more_click_5_answer_uv
,NVL(t2.more_click_5_topic_uv,0) as more_click_5_topic_uv
,NVL(t2.search_wel_uv,0) as search_wel_uv
,NVL(t2.page_wel_uv,0) as page_wel_uv
,NVL(t2.wel_click_uv,0) as wel_click_uv
,NVL(t2.wel_click_10_uv,0) as wel_click_10_uv
,NVL(t2.wel_click_5_uv,0) as wel_click_5_uv
,NVL(t2.search_diary_uv,0) as search_diary_uv
,NVL(t2.page_diary_uv,0) as page_diary_uv
,NVL(t2.diary_click_uv,0) as diary_click_uv
,NVL(t2.diary_click_10_uv,0) as diary_click_10_uv
,NVL(t2.diary_click_5_uv,0) as diary_click_5_uv
from
(--总搜索量
select partition_date
,query
,search_pv
,search_uv
from
(
select t1.partition_date
,query
,count(t1.cl_id) as search_pv
,count(distinct t1.cl_id) as search_uv
,row_number() over(partition by partition_date order by count(t1.cl_id) desc) as rank
from
(
select partition_date
,params['query'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('do_search','search_result_click_search')
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date
,params['card_name'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and int(split(app_version,'\\.')[1]) >= 27
and action='on_click_card'
and params['card_type']='highlight_word'
)t1
join
(
SELECT device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t2
on t1.cl_id=t2.device_id
group by t1.partition_date,query
)t
where rank<=300
)t1
left join
--综合、美购、日记页面浏览pv
(
select t1.partition_date,t1.query
,sum(case when tab_name='综合' then t1.pv end) as search_more_pv
,count(distinct case when tab_name='综合' then t1.cl_id end) as search_more_uv
,sum(case when tab_name='美购' then t1.pv end) as search_wel_pv
,count(distinct case when tab_name='美购' then t1.cl_id end) as search_wel_uv
,sum(case when tab_name='日记' then t1.pv end) as search_diary_pv
,count(distinct case when tab_name='日记' then t1.cl_id end) as search_diary_uv
,sum(case when t2.page_name='综合' then t2.pv end) as page_more_pv
,count(distinct case when t2.page_name='综合' then t2.cl_id end) as page_more_uv
,sum(case when t2.page_name='美购' then t2.pv end) as page_wel_pv
,count(distinct case when t2.page_name='美购' then t2.cl_id end) as page_wel_uv
,sum(case when t2.page_name='日记' then t2.pv end) as page_diary_pv
,count(distinct case when t2.page_name='日记' then t2.cl_id end) as page_diary_uv
,sum(case when t3.page_name='综合' then t3.pv end) as more_click_pv
,count(distinct case when t3.page_name='综合' then t3.cl_id end) as more_click_uv
,sum(case when t3.page_name='美购' then t3.pv end) as wel_click_pv
,count(distinct case when t3.page_name='美购' then t3.cl_id end) as wel_click_uv
,sum(case when t3.page_name='日记' then t3.pv end) as diary_click_pv
,count(distinct case when t3.page_name='日记' then t3.cl_id end) as diary_click_uv
,sum(case when t4.page_name='综合' then t4.pv end) as more_click_10_pv
,count(distinct case when t4.page_name='综合' then t4.cl_id end) as more_click_10_uv
,sum(case when t4.page_name='美购' then t4.pv end) as wel_click_10_pv
,count(distinct case when t4.page_name='美购' then t4.cl_id end) as wel_click_10_uv
,sum(case when t4.page_name='日记' then t4.pv end) as diary_click_10_pv
,count(distinct case when t4.page_name='日记' then t4.cl_id end) as diary_click_10_uv
,sum(case when t5.page_name='美购' then t5.pv end) as wel_click_5_pv
,count(distinct case when t5.page_name='美购' then t5.cl_id end) as wel_click_5_uv
,sum(case when t5.page_name='日记' then t5.pv end) as diary_click_5_pv
,count(distinct case when t5.page_name='日记' then t5.cl_id end) as diary_click_5_uv
,sum(case when t6.page_name='综合' then t6.pv end) as more_click_5_diary_pv
,count(distinct case when t6.page_name='综合' then t6.cl_id end) as more_click_5_diary_uv
,sum(case when t7.page_name='综合' then t7.pv end) as more_click_5_answer_pv
,count(distinct case when t7.page_name='综合' then t7.cl_id end) as more_click_5_answer_uv
,sum(case when t8.page_name='综合' then t8.pv end) as more_click_5_topic_pv
,count(distinct case when t8.page_name='综合' then t8.cl_id end) as more_click_5_topic_uv
from
(--各tab搜索
select partition_date
,params['query'] as query
,cl_id
,params['tab'] as tab_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and params['tab'] in ('综合','美购','日记')
and action in ('search_result_click_tab','search_result_click_search')
group by partition_date,params['query'],cl_id,params['tab']
)t1
left join
(--结果页浏览
select partition_date
,get_json_object(params['extra_param'],'$.query') as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
and action ='page_view'
group by partition_date
,get_json_object(params['extra_param'],'$.query')
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
and t1.cl_id=t2.cl_id
and t1.tab_name=t2.page_name
left join
(--内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t3
on t2.partition_date=t3.partition_date
and t2.query=t3.query
and t2.cl_id=t3.cl_id
and t2.page_name=t3.page_name
left join
(--前10内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and params['position'] in (0,1,2,3,4,5,6,7,8,9)
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t4
on t2.partition_date=t4.partition_date
and t2.query=t4.query
and t2.cl_id=t4.cl_id
and t2.page_name=t4.page_name
left join
(--日记、美购前5内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and ( action = 'on_click_diary_card' and page_name='search_result_diary'
or action ='search_result_welfare_click_item' and page_name='search_result_welfare')
and params['position'] in (0,1,2,3,4)
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t5
on t2.partition_date=t5.partition_date
and t2.query=t5.query
and t2.cl_id=t5.cl_id
and t2.page_name=t5.page_name
left join
(--综搜页日记卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='diary'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item'and params['business_type']in('diary','0')
or action ='on_click_diary_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t6
on t2.partition_date=t6.partition_date
and t2.query=t6.query
and t2.cl_id=t6.cl_id
and t2.page_name=t6.page_name
left join
(--综搜页回答卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='answer'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action ='search_result_click_infomation_item'
and params['business_type']in ('answer','5')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t7
on t2.partition_date=t7.partition_date
and t2.query=t7.query
and t2.cl_id=t7.cl_id
and t2.page_name=t7.page_name
left join
(--综搜页帖子卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='user_post'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item' and params['business_type'] in ('post','user_post','doctor_post','11')
or action='on_click_topic_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t8
on t2.partition_date=t8.partition_date
and t2.query=t8.query
and t2.cl_id=t8.cl_id
and t2.page_name=t8.page_name
join
(
select device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
on t1.cl_id=dev.device_id
left join
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t2.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
group by t1.partition_date,t1.query
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
order by day_id desc,search_pv desc,query
tables/create_meigou_source-pv.sql 0 → 100644
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_meigou_source_pv_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
is_ai_channel string comment '{"chs_name":"是否AI渠道","description":"","etl":"","value":"","remark":""}',
welfare_pv int comment '{"chs_name":"美购详情页pv","description":"","etl":"","value":"","remark":""}',
to_welfare_pv int comment '{"chs_name":"从上级页面转化到美购详情页的pv总量","description":"","etl":"","value":"","remark":""}',
total_conversion_rate double comment '{"chs_name":"从上级页面转化到美购详情页的pv比美购详情页总pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_pv int comment '{"chs_name":"搜索结果美购页pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_to_welfare_pv int comment '{"chs_name":"从搜索结果美购页转化到美购详情页pv","description":"","etl":"","value":"","remark":""}',
result_wel_add_pv int comment '{"chs_name":"搜索结果美购页-转化点击加车pv","description":"","etl":"","value":"","remark":""}',
result_wel_buy_pv int comment '{"chs_name":"内容用户APP7留","description":"","etl":"","value":"","remark":""}',
result_wel_msg_pv int comment '{"chs_name":"内容用户APP30留","description":"","etl":"","value":"","remark":""}',
avg_app_duration double comment '{"chs_name":"内容用户单设备App时长(m)","description":"","etl":"","value":"","remark":""}',
avg_content_stay double comment '{"chs_name":"内容用户单设备内容时长(m)","description":"","etl":"","value":"","remark":""}',
avg_open_times double comment '{"chs_name":"内容用户单设备打开次数","description":"","etl":"","value":"","remark":""}',
search_related_stay double comment '{"chs_name":"内容用户搜索相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
welfare_stay double comment '{"chs_name":"内容用户美购详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_question_stay double comment '{"chs_name":"内容用户问题详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
ai_related_stay double comment '{"chs_name":"内容用户AI相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_diary_stay double comment '{"chs_name":"内容用户日记详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
home_stay double comment '{"chs_name":"内容用户首页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
conv_related_stay double comment '{"chs_name":"内容用户咨询相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
recommend_rate double comment '{"chs_name":"首页feeds推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_recommend_pv double comment '{"chs_name":"首页feeds推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
feeds_rate double comment '{"chs_name":"首页feeds非推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_feeds_pv double comment '{"chs_name":"首页feeds非推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
search_rate double comment '{"chs_name":"搜索进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_search_pv double comment '{"chs_name":"搜索进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
zone_rate double comment '{"chs_name":"内容聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_zone_pv double comment '{"chs_name":"内容聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
content_rate double comment '{"chs_name":"内容详情页推荐板块进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_from_content_pv double comment '{"chs_name":"内容详情页推荐板块进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
blank_rate double comment '{"chs_name":"无来源页面(大多数为push)进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_blank_pv double comment '{"chs_name":"无来源页面(大多数为push)进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
comment_rate double comment '{"chs_name":"评论列表页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_comment_pv double comment '{"chs_name":"评论列表页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
org_rate double comment '{"chs_name":"医生医院主页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_org_pv double comment '{"chs_name":"医生医院主页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
category_rate double comment '{"chs_name":"品类聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_category_pv double comment '{"chs_name":"品类聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
my_diary_rate double comment '{"chs_name":"我的日记页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_my_diary_pv double comment '{"chs_name":"我的日记页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
ai_rate double comment '{"chs_name":"AI报告页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_ai_pv double comment '{"chs_name":"AI报告页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
create_topic_num int comment '{"chs_name":"真实发帖数","description":"","etl":"","value":"","remark":""}',
create_reply_num int comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
diary_uv int comment '{"chs_name":"日记UV","description":"","etl":"","value":"","remark":""}',
diary_pv int comment '{"chs_name":"日记PV","description":"","etl":"","value":"","remark":""}',
diary_rate double comment '{"chs_name":"日记UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_diary_pv double comment '{"chs_name":"日记PV/日记UV","description":"","etl":"","value":"","remark":""}',
diary_stay double comment '{"chs_name":"日记单设备时长(m)","description":"","etl":"","value":"","remark":""}',
post_uv int comment '{"chs_name":"帖子UV","description":"","etl":"","value":"","remark":""}',
post_pv int comment '{"chs_name":"帖子PV","description":"","etl":"","value":"","remark":""}',
post_rate double comment '{"chs_name":"帖子UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_post_pv double comment '{"chs_name":"帖子PV/帖子UV","description":"","etl":"","value":"","remark":""}',
post_stay double comment '{"chs_name":"帖子单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_uv int comment '{"chs_name":"问题UV","description":"","etl":"","value":"","remark":""}',
question_pv int comment '{"chs_name":"问题PV","description":"","etl":"","value":"","remark":""}',
question_rate double comment '{"chs_name":"问题UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_question_pv double comment '{"chs_name":"问题PV/问题UV","description":"","etl":"","value":"","remark":""}',
question_stay double comment '{"chs_name":"问题单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_answer_uv int comment '{"chs_name":"问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_pv int comment '{"chs_name":"问答PV","description":"","etl":"","value":"","remark":""}',
question_answer_rate double comment '{"chs_name":"问答UV/问答UV","description":"","etl":"","value":"","remark":""}',
per_question_answer_pv double comment '{"chs_name":"问答PV/问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_stay double comment '{"chs_name":"问答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
answer_uv int comment '{"chs_name":"回答UV","description":"","etl":"","value":"","remark":""}',
answer_pv int comment '{"chs_name":"回答PV","description":"","etl":"","value":"","remark":""}',
answer_rate double comment '{"chs_name":"回答UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_answer_pv double comment '{"chs_name":"回答PV/回答UV","description":"","etl":"","value":"","remark":""}',
answer_stay double comment '{"chs_name":"回答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
video_uv int comment '{"chs_name":"视频UV","description":"","etl":"","value":"","remark":""}',
video_pv int comment '{"chs_name":"视频PV","description":"","etl":"","value":"","remark":""}',
video_rate double comment '{"chs_name":"视频UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_video_pv double comment '{"chs_name":"视频PV/视频UV","description":"","etl":"","value":"","remark":""}',
video_stay double comment '{"chs_name":"视频单设备时长(m)","description":"","etl":"","value":"","remark":""}',
wiki_uv int comment '{"chs_name":"百科UV","description":"","etl":"","value":"","remark":""}',
wiki_pv int comment '{"chs_name":"百科PV","description":"","etl":"","value":"","remark":""}',
wiki_rate double comment '{"chs_name":"百科UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_wiki_pv double comment '{"chs_name":"百科PV/百科UV","description":"","etl":"","value":"","remark":""}',
wiki_stay double comment '{"chs_name":"百科单设备时长(m)","description":"","etl":"","value":"","remark":""}',
article_uv int comment '{"chs_name":"专栏UV","description":"","etl":"","value":"","remark":""}',
article_pv int comment '{"chs_name":"专栏PV","description":"","etl":"","value":"","remark":""}',
article_rate double comment '{"chs_name":"专栏UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_article_pv double comment '{"chs_name":"专栏PV/专栏UV","description":"","etl":"","value":"","remark":""}',
article_stay double comment '{"chs_name":"专栏单设备时长(m)","description":"","etl":"","value":"","remark":""}'
)comment '内容日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/create_recommend_daily_data.sql 0 → 100644
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
home_ctr string comment '{"chs_name":"首页卡片点击PV/首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
second_rate string comment '{"chs_name":"来自首页推荐内容卡片的的总商业二跳pv/首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
first_rate string comment '{"chs_name":"来自首页推荐内容卡片的的总商业二跳pv/首页卡片点击PV","description":"","etl":"","value":"","remark":""}',
card_click BIGINT comment '{"chs_name":"首页卡片点击PV","description":"","etl":"","value":"","remark":""}',
card_exposure BIGINT comment '{"chs_name":"首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
total_second_click BIGINT comment '{"chs_name":"来自首页卡片的总商业二跳pv（包含页面上搜索及卡片点击）","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"来自首页卡片的单PV平均浏览时长","description":"","etl":"","value":"","remark":""}',
navbar_search BIGINT comment '{"chs_name":"来自首页卡片的搜索框+搜索按钮点击PV","description":"","etl":"","value":"","remark":""}',
highlight_word BIGINT comment '{"chs_name":"来自首页卡片的文内搜索点击PV","description":"","etl":"","value":"","remark":""}',
self_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的商品卡片点击PV","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的推荐商品+查看全部商品点击pv","description":"","etl":"","value":"","remark":""}',
recommend_content_card BIGINT comment '{"chs_name":"来自首页卡片的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
recommend_special_card BIGINT comment '{"chs_name":"来自首页卡片的推荐专题点击pv","description":"","etl":"","value":"","remark":""}',
transfer_card BIGINT comment '{"chs_name":"来自首页卡片的转诊点击pv","description":"","etl":"","value":"","remark":""}',
video_consultation BIGINT comment '{"chs_name":"来自首页卡片的视频面诊点击pv","description":"","etl":"","value":"","remark":""}'
)comment '首页推荐策略日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/create_search_click_path.sql 0 → 100644
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_click_path_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
search_entrance string comment '{"chs_name":"搜索入口","description":"","etl":"","value":"","remark":""}',
click_search_pv int comment '{"chs_name":"点击搜索或搜索框pv","description":"","etl":"","value":"","remark":""}',
click_search_uv int comment '{"chs_name":"点击搜索或搜索框uv","description":"","etl":"","value":"","remark":""}'
)comment '分搜索入口的搜索点击数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/create_search_input_type.sql 0 → 100644
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_input_type_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
query string comment '{"chs_name":"搜索词","description":"","etl":"","value":"","remark":""}',
input_type string comment '{"chs_name":"搜索方式","description":"","etl":"","value":"","remark":""}',
search_pv bigint comment '{"chs_name":"搜索量","description":"","etl":"","value":"","remark":""}'
)comment '分方式搜索日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/create_search_result_click_rate.sql 0 → 100644
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_result_click_rate_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
query string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
search_pv bigint comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
search_uv bigint comment '{"chs_name":"搜索入口","description":"","etl":"","value":"","remark":""}',
search_more_pv bigint comment '{"chs_name":"综合tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_more_pv bigint comment '{"chs_name":"搜索综合结果页浏览pv","description":"","etl":"","value":"","remark":""}',
more_click_pv bigint comment '{"chs_name":"搜索综合结果页内容卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_10_pv bigint comment '{"chs_name":"搜索综合结果页前10内容卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_diary_pv bigint comment '{"chs_name":"搜索综合结果页前5日记卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_answer_pv bigint comment '{"chs_name":"搜索综合结果页前5回答卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_topic_pv bigint comment '{"chs_name":"搜索综合结果页前5帖子卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_wel_pv bigint comment '{"chs_name":"美购tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_wel_pv bigint comment '{"chs_name":"搜索美购结果页浏览pv","description":"","etl":"","value":"","remark":""}',
wel_click_pv bigint comment '{"chs_name":"搜索美购结果页卡片点击PV","description":"","etl":"","value":"","remark":""}',
wel_click_10_pv bigint comment '{"chs_name":"搜索美购结果页前10卡片点击PV","description":"","etl":"","value":"","remark":""}',
wel_click_5_pv bigint comment '{"chs_name":"搜索美购结果页前5卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_diary_pv bigint comment '{"chs_name":"日记tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_diary_pv bigint comment '{"chs_name":"搜索日记结果页浏览pv","description":"","etl":"","value":"","remark":""}',
diary_click_pv bigint comment '{"chs_name":"搜索日记结果页卡片点击PV","description":"","etl":"","value":"","remark":""}',
diary_click_10_pv bigint comment '{"chs_name":"搜索日记结果页前10卡片点击PV","description":"","etl":"","value":"","remark":""}',
diary_click_5_pv bigint comment '{"chs_name":"搜索日记结果页前5卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_more_uv bigint comment '{"chs_name":"综合tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_more_uv bigint comment '{"chs_name":"搜索综合结果页浏览uv","description":"","etl":"","value":"","remark":""}',
more_click_uv bigint comment '{"chs_name":"搜索综合结果页内容卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_10_uv bigint comment '{"chs_name":"搜索综合结果页前10内容卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_diary_uv bigint comment '{"chs_name":"搜索综合结果页前5日记卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_answer_uv bigint comment '{"chs_name":"搜索综合结果页前5回答卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_topic_uv bigint comment '{"chs_name":"搜索综合结果页前5帖子卡片点击UV","description":"","etl":"","value":"","remark":""}',
search_wel_uv bigint comment '{"chs_name":"美购tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_wel_uv bigint comment '{"chs_name":"搜索美购结果页浏览uv","description":"","etl":"","value":"","remark":""}',
wel_click_uv bigint comment '{"chs_name":"搜索美购结果页卡片点击UV","description":"","etl":"","value":"","remark":""}',
wel_click_10_uv bigint comment '{"chs_name":"搜索美购结果页前10卡片点击UV","description":"","etl":"","value":"","remark":""}',
wel_click_5_uv bigint comment '{"chs_name":"搜索美购结果页前5卡片点击UV","description":"","etl":"","value":"","remark":""}',
search_diary_uv bigint comment '{"chs_name":"日记tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_diary_uv bigint comment '{"chs_name":"搜索日记结果页浏览uv","description":"","etl":"","value":"","remark":""}',
diary_click_uv bigint comment '{"chs_name":"搜索日记结果页卡片点击UV","description":"","etl":"","value":"","remark":""}',
diary_click_10_uv bigint comment '{"chs_name":"搜索日记结果页前10卡片点击UV","description":"","etl":"","value":"","remark":""}',
diary_click_5_uv bigint comment '{"chs_name":"搜索日记结果页前5卡片点击UV","description":"","etl":"","value":"","remark":""}'
)comment '分搜索词日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment