Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

change date

eda/feededa/src/main/scala/com/gmei/EsmmData.scala

@@ -548,25 +548,39 @@ object GetDevicePortrait {
 
       val device_search_tag = sc.sql(
         s"""
-           |select a.cl_id as device_id,COALESCE(a.params['diary_id'], a.params['business_id'], 0) as cid_id,split(c.level1_ids,',')[0] as level1_id
-           |from online.tl_hdfs_maidian_view as a
-           |left join
-           |	(select cl_id as device_id,max(partition_date) as stat_date
-           |	from online.tl_hdfs_maidian_view
-           |	where action = 'on_click_diary_card'
-           |	and (params['page_name']='search_result_diary' or params['page_name']='search_result_more')
-           |	and partition_date > '20180601' group by cl_id) b
-           |on a.cl_id = b.device_id and a.partition_date=b.stat_date
-           |left join diary_feat c
-           |on COALESCE(a.params['diary_id'], a.params['business_id'], 0) = c.diary_id
-           |where a.partition_date > '20180601'
-           |and COALESCE(a.params['diary_id'], a.params['business_id'], 0) != 0
-           |and c.level1_ids != ""
+           |select c.device_id,c.stat_date,c.level1_id,count(c.level1_id) as level1_count
+           |from (select
+           |		a.cl_id as device_id,a.partition_date as stat_date,
+           |		COALESCE(a.params['diary_id'], a.params['business_id'], 0) as cid_id,
+           |		b.level1_ids as level1_id
+           |	from online.tl_hdfs_maidian_view a
+           |	left join diary_feat b
+           |	on COALESCE(a.params['diary_id'], a.params['business_id'], 0) = b.diary_id
+           |	where
+           |    b.level1_ids is not null and
+           |		a.partition_date = '20181224'
+           |		and (a.action = 'on_click_diary_card' or (a.action="full_stack_click_video_card_full_screen_play" and a.params["card_type"]="diary"))) c
+           |group by c.device_id,c.level1_id,c.stat_date
          """.stripMargin
       )
       device_search_tag.show()
       println(device_search_tag.count())
 
+      device_search_tag.createOrReplaceTempView("tag_count")
+
+      val max_count_tag = sc.sql(
+      s"""
+         |select a.device_id,a.stat_date,a.level1_id,a.level1_count
+         |from tag_count a
+         |inner join
+         |(select device_id,max(level1_count) as max_count from tag_count group by device_id) b
+         |on a.level1_count = b.max_count and a.device_id = b.device_id
+       """.stripMargin
+    )
+      max_count_tag.show()
+      println(max_count_tag.count())
+
+
 //      GmeiConfig.writeToJDBCTable(diary_tag,"diary_feat",SaveMode.Overwrite)
 
 

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -467,23 +467,48 @@ object meigou_xiaofei_renshu {
       final_id.createOrReplaceTempView("final_id")
 
 
+//      val meigou_price = sc.sql(
+//        s"""
+//           |select  md.user_id,sum(md.gengmei_price) as pay_all
+//           |from online.ml_meigou_order_detail md left join final_id
+//           |on md.device_id = final_id.device_id
+//           |where md.status= 2
+//           |and final_id.device_id is  null
+//           |and md.partition_date = '20181223'
+//           |and  md.pay_time  is  not  null
+//           |and  md.validate_time>'2017-01-01 00:00:00.0'
+//           |group by md.user_id
+//           |order by sum(md.gengmei_price)
+//       """.stripMargin
+//      )
+//      meigou_price.show(80)
+
+
       val meigou_price = sc.sql(
         s"""
            |select  md.user_id,sum(md.gengmei_price) as pay_all
-           |from online.ml_meigou_order_detail md left join final_id
-           |on md.device_id = final_id.device_id
+           |from online.ml_meigou_order_detail md
+           |left join
+           |(
+           |        SELECT
+           |        order_id
+           |        FROM mining.ml_order_spam_recognize
+           |        WHERE partition_date='20181223' AND
+           |        self_support=0 AND dayBitsGetW1(predict_result,'20181223')=0
+           |)spam
+           |on md.order_id = spam.order_id
            |where md.status= 2
-           |and final_id.device_id is  null
+           |and  spam.order_id is null
            |and md.partition_date = '20181223'
            |and  md.pay_time  is  not  null
-           |and  md.validate_time>'2018-01-01 00:00:00.0'
+           |and  md.validate_time>'2017-01-01 00:00:00.0'
            |group by md.user_id
            |order by sum(md.gengmei_price)
        """.stripMargin
       )
       meigou_price.show(80)
 
-      GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Append)
+      GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite)
 
     }