计算多天的推荐重复问题

eda/feededa/src/main/scala/com/gmei/temp_count.scala

 package com.gmei
 
 import java.io.Serializable
+import java.text.SimpleDateFormat
 
 import com.gmei.WeafareStat.{defaultParams, parser}
 import org.apache.spark.sql.{SaveMode, TiContext}
 import org.apache.log4j.{Level, Logger}
 import scopt.OptionParser
 import com.gmei.lib.AbstractParams
+import com.google.`type`.Date
 
 object temp_count {
 
@@ -298,40 +300,30 @@ object Repeated_content_recommendation_moreday {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date = param.date
-      val partition_date = stat_date.replace("-","")
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
+//      val partition_date = stat_date.replace("-","")
+
+
+//      val now: Date = new Date()
+//      val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd")
+//      val date = dateFormat.format(now.getTime - 86400000L * 18)
 
 
       val exp_diary = sc.sql(
         s"""
-           |select concat_ws('|',device_id,cid_id)
-           |from data_feed_exposure
+           |select stat_date,device_id,group_concat(distinct cid_id SEPARATOR ',')
+           |from data_feed_exposure_precise
            |where cid_type = 'diary'
            |and device_id not in (select device_id from blacklist)
-           |and stat_date ='${stat_date}'
+           |and stat_date >='${stat_date}'
          """.stripMargin
       )
       exp_diary.show()
-      val get_result =exp_diary.rdd.map((_, 1)).reduceByKey(_ + _)
-              .sortBy(_._2,false)
 
-      val more_than2=get_result.filter(_._2 >=2).map(_._2).reduce((x,y)=>x+y)
-      println(more_than2)
-      val all =get_result.map(_._2).reduce((x,y)=>x+y)
-      println(all)
-      val repeated_rate= more_than2 / all.toDouble
-      println(repeated_rate)
-      val test=List((stat_date,repeated_rate))
-      val df = sc.createDataFrame(test)
 
-      GmeiConfig.writeToJDBCTable(df, table = "Repeated_evaluation_indicator_moreday", SaveMode.Append)
+//      GmeiConfig.writeToJDBCTable(df, table = "Repeated_evaluation_indicator_moreday", SaveMode.Append)
 
-
-//      val temp=get_result.collect()
-//            for (i <- 0 until 30 ) {
-//              println(temp(i))
-//            }
     }