Skip to content

F
ffm-baseline
Commit 752ca9da authored by 张彦钊's avatar 张彦钊
Browse files
Options

Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline 

change test file
parents 2082c673 cac9748d
Hide whitespace changes
Inline Side-by-side
Showing with 117 additions and 3 deletions
eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala
View file @ 752ca9da
......@@ -1061,8 +1061,8 @@ object icon_data {
sc.sql("use jerry_prod")
val stat_date = GmeiConfig.getMinusNDate(1)
// val stat_date = param.date
// val stat_date = GmeiConfig.getMinusNDate(1)
val stat_date = param.date
//println(param.date)
val partition_date = stat_date.replace("-","")
......@@ -1087,6 +1087,13 @@ object icon_data {
|from online.tl_hdfs_maidian_view
|where partition_date='${partition_date}'
|and action='home_click_section'
|and channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
""".stripMargin
)
icon_click_all.createOrReplaceTempView("icon_click_all")
......@@ -1099,12 +1106,56 @@ object icon_data {
|from online.ml_community_exposure_detail_new
|where partition_date='${partition_date}'
|and business_type='icon'
|and channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
""".stripMargin
)
icon_expoure_all.createOrReplaceTempView("icon_expoure_all")
icon_expoure_all.show()
val device_id_oldUser = sc.sql(
s"""
|select distinct(device_id) as device_id
|from online.ml_device_day_active_status
|where active_type = '4'
|and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
|and partition_date ='${partition_date}'
""".stripMargin
)
device_id_oldUser.show()
device_id_oldUser.createOrReplaceTempView("device_id_old")
val device_id_newUser = sc.sql(
s"""
|select distinct(device_id) as device_id
|from online.ml_device_day_active_status
|where active_type != '4'
|and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
|and partition_date ='${partition_date}'
""".stripMargin
)
device_id_newUser.createOrReplaceTempView("device_id_new")
//指定用户的点击
val icon_click_device = sc.sql(
s"""
......@@ -1114,9 +1165,16 @@ object icon_data {
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.action='home_click_section'
|and a.channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
""".stripMargin
)
icon_click_all.createOrReplaceTempView("icon_click_all")
icon_click_device.createOrReplaceTempView("icon_click_device")
icon_click_device.show()
//指定用户的曝光
val icon_expoure_device = sc.sql(
......@@ -1127,14 +1185,70 @@ object icon_data {
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.business_type='icon'
|and a.channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
| ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
| ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
| ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
| ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
| ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
| ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
""".stripMargin
)
icon_expoure_device.createOrReplaceTempView("icon_expoure_device")
icon_expoure_device.show()
val icon_click_device_new = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(*) as icon_click_num_device_new
|from online.tl_hdfs_maidian_view a
|inner join device_id_new b
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.action='home_click_section'
""".stripMargin
)
val icon_click_device_old = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(*) as icon_click_num_device_old
|from online.tl_hdfs_maidian_view a
|inner join device_id_old b
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.action='home_click_section'
""".stripMargin
)
val icon_expoure_device_new = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(*) as icon_expoure_num_device_new
|from online.ml_community_exposure_detail_new a
|inner join device_id_new b
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.business_type='icon'
""".stripMargin
)
val icon_expoure_device_old = sc.sql(
s"""
|select '${stat_date}' as stat_date,count(*) as icon_expoure_num_device_old
|from online.ml_community_exposure_detail_new a
|inner join device_id_old b
|on a.cl_id=b.device_id
|where a.partition_date='${partition_date}'
|and a.business_type='icon'
""".stripMargin
)
val result = icon_click_all.join(icon_expoure_all,"stat_date")
.join(icon_click_device,"stat_date")
.join(icon_expoure_device,"stat_date")
.join(icon_click_device_new,"stat_date")
.join(icon_click_device_old,"stat_date")
.join(icon_expoure_device_new,"stat_date")
.join(icon_expoure_device_old,"stat_date")
println("开始写入")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment