清洗点击数据

eda/feededa/src/main/scala/com/gmei/Recommendation_strategy_all.scala

@@ -13,7 +13,8 @@ object Recommendation_strategy_all {
   Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
   Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
 
-  case class Params(env: String = "dev"
+  case class Params(env: String = "dev",
+                    date: String = "2018-08-01"
                    ) extends AbstractParams[Params] with Serializable
 
   val defaultParams = Params()
@@ -23,6 +24,9 @@ object Recommendation_strategy_all {
     opt[String]("env")
       .text(s"the databases environment you used")
       .action((x, c) => c.copy(env = x))
+    opt[String] ("date")
+      .text(s"the date you used")
+      .action((x,c) => c.copy(date = x))
     note(
       """
         |For example, the following command runs this app on a tidb dataset:
@@ -48,9 +52,8 @@ object Recommendation_strategy_all {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-      import sc.implicits._
-      val stat_date = GmeiConfig.getMinusNDate(1)
-      println(stat_date)
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -426,14 +426,6 @@ object meigou_xiaofei_renshu {
 
 
 
-      sc.sparkContext.addJar("hdfs:///user/hive/share/lib/udf/daybits-1.0.0-SNAPSHOT.jar")
-      sc.sparkContext.addJar("hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
-
-
-      sc.sql("CREATE TEMPORARY FUNCTION dayBitsGetW1 AS 'com.gmei.data.daybits.udf.DayBitsGetsW1'")
-      sc.sql("SELECT order_id FROM mining.ml_order_spam_recognize WHERE partition_date='20181212' AND self_support=0 AND dayBitsGetW1(predict_result,'20181212')=0").show()
-
-
       import sc.implicits._
       val stat_date = GmeiConfig.getMinusNDate(1)
       //println(param.date)