Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

change test file

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

@@ -1061,8 +1061,8 @@ object icon_data {
       sc.sql("use jerry_prod")
 
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-      //      val stat_date = param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
@@ -1087,6 +1087,13 @@ object icon_data {
            |from online.tl_hdfs_maidian_view
            |where partition_date='${partition_date}'
            |and action='home_click_section'
+           |and channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
          """.stripMargin
       )
       icon_click_all.createOrReplaceTempView("icon_click_all")
@@ -1099,12 +1106,56 @@ object icon_data {
            |from online.ml_community_exposure_detail_new
            |where partition_date='${partition_date}'
            |and business_type='icon'
+           |and channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
           """.stripMargin
       )
       icon_expoure_all.createOrReplaceTempView("icon_expoure_all")
       icon_expoure_all.show()
 
 
+      val device_id_oldUser = sc.sql(
+        s"""
+           |select distinct(device_id) as device_id
+           |from online.ml_device_day_active_status
+           |where active_type = '4'
+           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
+           |and partition_date ='${partition_date}'
+         """.stripMargin
+      )
+      device_id_oldUser.show()
+      device_id_oldUser.createOrReplaceTempView("device_id_old")
+
+      val device_id_newUser = sc.sql(
+        s"""
+           |select distinct(device_id) as device_id
+           |from online.ml_device_day_active_status
+           |where active_type != '4'
+           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
+           |and partition_date ='${partition_date}'
+         """.stripMargin
+      )
+      device_id_newUser.createOrReplaceTempView("device_id_new")
+
+
+
 //指定用户的点击
       val icon_click_device = sc.sql(
         s"""
@@ -1114,9 +1165,16 @@ object icon_data {
            |on a.cl_id=b.device_id
            |where a.partition_date='${partition_date}'
            |and a.action='home_click_section'
+           |and a.channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
          """.stripMargin
       )
-      icon_click_all.createOrReplaceTempView("icon_click_all")
+      icon_click_device.createOrReplaceTempView("icon_click_device")
       icon_click_device.show()
 //指定用户的曝光
       val icon_expoure_device = sc.sql(
@@ -1127,14 +1185,70 @@ object icon_data {
            |on a.cl_id=b.device_id
            |where a.partition_date='${partition_date}'
            |and a.business_type='icon'
+           |and a.channel not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |    ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |    ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |    ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |    ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |    ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |    ,'promotion_shike','promotion_julang_jl03','','unknown','promotion_zuimei')
           """.stripMargin
       )
       icon_expoure_device.createOrReplaceTempView("icon_expoure_device")
       icon_expoure_device.show()
 
+      val icon_click_device_new = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(*) as icon_click_num_device_new
+           |from online.tl_hdfs_maidian_view a
+           |inner join device_id_new b
+           |on a.cl_id=b.device_id
+           |where a.partition_date='${partition_date}'
+           |and a.action='home_click_section'
+         """.stripMargin
+      )
+
+      val icon_click_device_old = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(*) as icon_click_num_device_old
+           |from online.tl_hdfs_maidian_view a
+           |inner join device_id_old b
+           |on a.cl_id=b.device_id
+           |where a.partition_date='${partition_date}'
+           |and a.action='home_click_section'
+         """.stripMargin
+      )
+
+      val icon_expoure_device_new = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(*) as icon_expoure_num_device_new
+           |from online.ml_community_exposure_detail_new a
+           |inner join device_id_new b
+           |on a.cl_id=b.device_id
+           |where a.partition_date='${partition_date}'
+           |and a.business_type='icon'
+          """.stripMargin
+      )
+
+      val icon_expoure_device_old = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(*) as icon_expoure_num_device_old
+           |from online.ml_community_exposure_detail_new a
+           |inner join device_id_old b
+           |on a.cl_id=b.device_id
+           |where a.partition_date='${partition_date}'
+           |and a.business_type='icon'
+          """.stripMargin
+      )
+
+
       val result = icon_click_all.join(icon_expoure_all,"stat_date")
         .join(icon_click_device,"stat_date")
         .join(icon_expoure_device,"stat_date")
+        .join(icon_click_device_new,"stat_date")
+        .join(icon_click_device_old,"stat_date")
+        .join(icon_expoure_device_new,"stat_date")
+        .join(icon_expoure_device_old,"stat_date")
 
 
       println("开始写入")