Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

修改训练集sql

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

@@ -51,8 +51,8 @@ object data_feed_exposure_precise {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-//      val stat_date = GmeiConfig.getMinusNDate(1)
-      val stat_date = param.date
+      val stat_date = GmeiConfig.getMinusNDate(1)
+//      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -559,7 +559,7 @@ object alpha_ctr {
 
       val click_count_recommend = sc.sql(
         s"""
-           |select '${stat_date}' as stat_date,count(*) as count_recommend
+           |select '${stat_date}' as stat_date,count(*) as click_count_recommend
            |from bl.bl_alpha_et_mg_maidianlog_inc_d
            |where params['tab_name']='recommend'
            |and params['page_name']='home'
@@ -571,7 +571,7 @@ object alpha_ctr {
 
       val click_count_focus = sc.sql(
         s"""
-           |select '${stat_date}' as stat_date,count(*) as count_focus
+           |select '${stat_date}' as stat_date,count(*) as click_count_focus
            |from bl.bl_alpha_et_mg_maidianlog_inc_d
            |where params['tab_name']='focus'
            |and params['page_name']='home'
@@ -581,7 +581,20 @@ object alpha_ctr {
       )
       click_count_focus.show()
 
-      sc.sql(
+
+      def parse_json(str:String): Int ={
+        var t = List[Map[String, Any]]()
+        val result = JSON.parseFull(str)
+        result match {
+          case Some(b: List[Map[String, Any]]) => t = t ++ b
+          case None => println("Parsing failed")
+          case other => println("Unknown data structure: " + other)
+        }
+        t.size
+
+      }
+
+      val expoure_cards=sc.sql(
         s"""
            |select params['exposure_cards'] as exposure_cards
            |from bl.bl_alpha_et_mg_maidianlog_inc_d
@@ -590,25 +603,34 @@ object alpha_ctr {
            |and type = 'page_precise_exposure'
            |and partition_day='${partition_date}'
        """.stripMargin
-      ).rdd.map(row => {
-        val lenth=row.toSeq.length
-        (stat_date,lenth)
-      }).toDF("stat_date","count").show()
-        /*.rdd
-        .map(row =>{
-        val s = row.getAs[List[String]]("exposure_cards")
-        (stat_date,s)
-      })
-        //.reduceByKey((x,y) => x + y)
-        .toDF("stat_date","count").show()
-*/
+      )
+      val a =expoure_cards.rdd.map(row => row(0).toString).map(row=>parse_json(row)).collect().sum
+      val result1=List((stat_date,a))
+      val df1 = sc.createDataFrame(result1).toDF("stat_date","expoure_count_recommend")
+
+      val expoure_cards2=sc.sql(
+        s"""
+           |select params['exposure_cards'] as exposure_cards
+           |from bl.bl_alpha_et_mg_maidianlog_inc_d
+           |where params['tab_name'] = 'focus'
+           |and params['page_name'] = 'home'
+           |and type = 'page_precise_exposure'
+           |and partition_day='${partition_date}'
+       """.stripMargin
+      )
+      val b =expoure_cards2.rdd.map(row => row(0).toString).map(row=>parse_json(row)).collect().sum
+      val result2=List((stat_date,b))
+      val df2 = sc.createDataFrame(result2).toDF("stat_date","expoure_count_focus")
 
 
 
+      val result=click_count_recommend.join(click_count_focus,"stat_date")
+        .join(df1,"stat_date")
+        .join(df2,"stat_date")
 
 
 
-//      GmeiConfig.writeToJDBCTable(click_count_recommend, "smart_rank_count", SaveMode.Append)
+      GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
 
 
     }

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -512,6 +512,7 @@ object diary_clk_card {
          """.stripMargin
       )
       device_id_oldUser.createOrReplaceTempView("device_id_old")
+      device_id_oldUser.show()
 
 
       val clk_count_oldUser_Contrast_a = sc.sql(
@@ -526,6 +527,7 @@ object diary_clk_card {
            |and ot.partition_date ='${partition_date}'
          """.stripMargin
       )
+      clk_count_oldUser_Contrast_a.show()
 
       val clk_count_oldUser_Contrast_b = sc.sql(
         s"""