增加分区

eda/feededa/src/main/scala/com/gmei/EsmmData.scala

@@ -65,7 +65,7 @@ object EsmmData {
            |where cid_type = 'diary'
            |and stat_date >'${stat_date}'
          """.stripMargin
-      )
+      ).repartition(200)
 //      imp_data.show()
 //      println("imp_data.count()")
 //      println(imp_data.count())
@@ -79,7 +79,7 @@ object EsmmData {
            |where cid_type = 'diary'
            |and stat_date >'${stat_date}'
          """.stripMargin
-      )
+      ).repartition(200)
 //      clk_data.show()
 //      println("clk_data.count()")
 //      println(clk_data.count())
@@ -106,7 +106,7 @@ object EsmmData {
            |and params['page_name'] = 'welfare_detail'
            |and params['referrer'] = 'diary_detail'
          """.stripMargin
-      )
+      ).repartition(200)
 
       val cvr_data_filter = cvr_data.withColumn("y",lit(1)).withColumn("z",lit(1))
 //      cvr_data_filter.createOrReplaceTempView("cvr_data_filter")
@@ -268,7 +268,7 @@ object EsmmPredData {
            |select device_id,city_id,search_queue as merge_queue from search_queue) as tmp1
            |where tmp1.device_id in (select distinct device_id from esmm_train_data)
          """.stripMargin
-      )
+      ).repartition(200)
       raw_data.show()
 
 
@@ -286,7 +286,7 @@ object EsmmPredData {
         s"""
            |select device_id,city_id,merge_queue from raw_data1 limit 10000
          """.stripMargin
-      )
+      ).repartition(200)
       raw_data2.createOrReplaceTempView("raw_data2")
       println(raw_data2.count())
       raw_data2.show()
@@ -295,7 +295,7 @@ object EsmmPredData {
         s"""
            |select device_id,city_id as ucity_id,explode(split(merge_queue, ',')) as cid_id from raw_data2
          """.stripMargin
-      )
+      ).repartition(200)
       raw_data3.createOrReplaceTempView("raw_data")
       println(raw_data3.count())