Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

add print

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -14,7 +14,8 @@ object testt {
   Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
   Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
 
-  case class Params(env: String = "dev"
+  case class Params(env: String = "dev",
+                    date: String = "2018-08-01"
                    ) extends AbstractParams[Params] with Serializable
 
   val defaultParams = Params()
@@ -24,6 +25,9 @@ object testt {
     opt[String]("env")
       .text(s"the databases environment you used")
       .action((x, c) => c.copy(env = x))
+    opt[String] ("date")
+      .text(s"the date you used")
+      .action((x,c) => c.copy(date = x))
     note(
       """
         |For example, the following command runs this app on a tidb dataset:
@@ -46,63 +50,103 @@ object testt {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "blacklist")
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
 
-
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date=param.date
+      val partition_date = stat_date.replace("-","")
       //机构id
-//      val agency_id = sc.sql(
-//        s"""
-//           |SELECT DISTINCT(cl_id) as device_id
-//           |FROM online.ml_hospital_spam_pv_day
-//           |WHERE partition_date >= '20180402'
-//           |AND partition_date <= '20181203'
-//           |AND pv_ratio >= 0.95
-//           |UNION ALL
-//           |SELECT DISTINCT(cl_id) as device_id
-//           |FROM online.ml_hospital_spam_pv_month
-//           |WHERE partition_date >= '20171101'
-//           |AND partition_date <= '20181203'
-//           |AND pv_ratio >= 0.95
-//         """.stripMargin
-//      )
-////      agency_id.show()
-//      agency_id.createOrReplaceTempView("agency_id")
-      val a = Array("answer_detail","article_detail","diary_detail","home_page","question_detail","search_content","search_diary")
-      for (i <- 0 until a.length){
-        val diary_pv = sc.sql(
+      val agency_id = sc.sql(
         s"""
-             |select partition_date,count(cl_id)
-             |from online.ml_community_exposure_detail
-             |where business_type ='diary'
-             |and partition_date >='20181120'
-             |and event='${a(i)}'
-             |GROUP BY partition_date
-             |order by partition_date
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_day
+           |WHERE partition_date >= '20180402'
+           |AND partition_date <= '20181203'
+           |AND pv_ratio >= 0.95
+           |UNION ALL
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_month
+           |WHERE partition_date >= '20171101'
+           |AND partition_date <= '20181203'
+           |AND pv_ratio >= 0.95
          """.stripMargin
       )
-        println("该来源的数据为："+ a(i))
-        diary_pv.show()
-      }
+//      agency_id.show()
+      agency_id.createOrReplaceTempView("agency_id")
 
+      //日记本转化美购
+      //1.日记本到美购转化数
+      val diary_meigou_count = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date, count(page_name) as diary_meigou_count
+           |from online.bl_hdfs_page_view_updates ou left join agency_id
+           |on ou.cl_id = agency_id.device_id
+           |where ou.partition_date = '${partition_date}'
+           |and ou.page_name='welfare_detail'
+           |and ou.referrer='diary_detail'
+           |and agency_id.device_id is not null
+           |and ou.cl_id not in (select device_id from blacklist)
+         """.stripMargin
+      )
 
-//      val diary_pv = sc.sql(
-//        s"""
-//           |select partition_date,count(cl_id)
-//           |from online.tl_hdfs_maidian_view
-//           |where action ='on_click_diary_card'
-//           |and params["page_name"]="home"
-//           |and params["tab_name"]="精选"
-//           |and partition_date >='20181120'
-//           |GROUP BY partition_date
-//           |order by partition_date
-//           """.stripMargin
-//      )
-//      diary_pv.show()
+      //2.日记本点击数
+      val diary_clk = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(cl_id) as diary_clk
+           |from online.tl_hdfs_maidian_view ov left join agency_id
+           |on ov.cl_id = agency_id.device_id
+           |where ov.action = 'on_click_diary_card'
+           |and ov.cl_id != "NULL"
+           |and ov.partition_date='${partition_date}'
+           |and agency_id.device_id is not null
+           |and ov.cl_id not in (select device_id from blacklist)
+       """.stripMargin
+      )
 
+      //3.日记本曝光数
+      val diary_expoure=sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(cl_id) as diary_expoure
+           |from online.ml_community_exposure_detail od left join agency_id
+           |on od.cl_id = agency_id.device_id
+           |where od.business_type = "diary"
+           |and od.cl_id != "NULL"
+           |and od.partition_date='${partition_date}'
+           |and agency_id.device_id is not null
+           |and od.cl_id not in (select device_id from blacklist)
+       """.stripMargin
+      )
 
+      //4.搜索次数统计
+      val search_count = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(params['query']) as search_num
+           |from online.tl_hdfs_maidian_view ov left join agency_id
+           |on ov.cl_id = agency_id.device_id
+           |where (ov.action = 'do_search' or ov.action = 'search_result_click_search')
+           |and ov.partition_date ='${partition_date}'
+           |and agency_id.device_id is not null
+           |and ov.cl_id  not in (select device_id from blacklist)
+         """.stripMargin
+      )
 
+      //5.登录人数
 
-//      GmeiConfig.writeToJDBCTable(view_count, "avg", SaveMode.Overwrite)
+      val log_num = sc.sql(
+        s"""
+           |select '${stat_date}' as stat_date,count(distinct(device_id)) as log_num
+           |from data_feed_exposure oe left join agency_id
+           |on oe.device_id = agency_id.device_id
+           |and oe.stat_date ='${stat_date}'
+           |and agency_id.device_id is not null
+           |and oe.device_id not in (select device_id from blacklist)
+         """.stripMargin
+      )
 
+      val result = diary_meigou_count.join(diary_clk,"stat_date")
+        .join(diary_expoure,"stat_date")
+        .join(search_count,"stat_date")
+        .join(log_num,"stat_date")
 
+      GmeiConfig.writeToJDBCTable(result, "diary_meigou_crv", SaveMode.Overwrite)
 
 
       }