Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

add multi file

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -553,8 +553,8 @@ object alpha_ctr {
 
 
       import sc.implicits._
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date = param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
@@ -631,32 +631,31 @@ object alpha_ctr {
 
 
 
-      GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
-      GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
+//      GmeiConfig.writeToJDBCTable(result, "alpha_ctr", SaveMode.Append)
+//      GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="alpha_ctr",SaveMode.Append)
 
 
 
-      val device_duration_avge = sc.sql(
+      val device_num_count = sc.sql(
         s"""
-           |SELECT '${stat_date}' as stat_date,sum(a.time_all)/count(a.device_id) as device_duration_avge,count(distinct(a.device_id)) as device_num from (select device_id,sum(params['duration']) as time_all
-           |from bl.bl_alpha_et_mg_maidianlog_inc_d
-           |where type='on_app_session_over'
-           |and partition_day='${partition_date}'
-           |GROUP BY device_id) a
+           |select '${stat_date}' as stat_date,count(DISTINCT(device_id))
+           |from ML_ALPHA_C_CT_DV_DEVICE_DIMEN_D
+           |where partition_day='${partition_date}'
+           |and is_today_active="1"
        """.stripMargin
       )
-      device_duration_avge.show()
+      device_num_count.show()
 
-      val duration_everytime_avge=sc.sql(
+      val duration_device=sc.sql(
         s"""
-           |SELECT '${stat_date}' as stat_date,sum(a.time_duration)/count(a.device_id) as duration_everytime_avge from (select device_id,params['duration'] as time_duration
-           |from bl.bl_alpha_et_mg_maidianlog_inc_d
-           |where type='on_app_session_over'
-           |and partition_day='${partition_date}') a
+           |select '${stat_date}' as stat_date,sum(user_duration)/count(DISTINCT(device_id))
+           |from ML_ALPHA_C_CT_DV_DEVICE_INDIC_INC_D
+           |WHERE partition_day='${partition_date}'
+           |and open_times!="0"
        """.stripMargin
       )
 
-      val result3=device_duration_avge.join(duration_everytime_avge,"stat_date")
+      val result3=device_num_count.join(duration_device,"stat_date")
       GmeiConfig.writeToJDBCTable(result3, "alpha_duration", SaveMode.Append)
       GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result3, table="alpha_duration",SaveMode.Append)
 

eda/feededa/src/main/scala/com/gmei/temp_count.scala

@@ -410,11 +410,11 @@ object Repeated_content_recommendation_moreday {
       val sc = spark_env._2
 
       val ti = new TiContext(sc)
-//      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure_precise")
-//      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
-//      ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
-//      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
-//      ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
+      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure_precise")
+      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_click")
+      ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
+      ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
+      ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
       val stat_date = GmeiConfig.getMinusNDate(1)