统计like项目

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -581,6 +581,19 @@ object alpha_ctr {
       )
       click_count_focus.show()
 
+
+      def parse_json(str:String): Int ={
+        var t = List[Map[String, Any]]()
+        val result = JSON.parseFull(str)
+        result match {
+          case Some(b: List[Map[String, Any]]) => t = t ++ b
+          case None => println("Parsing failed")
+          case other => println("Unknown data structure: " + other)
+        }
+        t.size
+
+      }
+
       sc.sql(
         s"""
            |select params['exposure_cards'] as exposure_cards
@@ -590,11 +603,7 @@ object alpha_ctr {
            |and type = 'page_precise_exposure'
            |and partition_day='${partition_date}'
        """.stripMargin
-      )
-        .rdd.map(row => {
-        val lenth=JSON.parseFull(row.toString())
-        (stat_date,lenth)
-      }).toDF("stat_date","count").show()
+      ).rdd.map(row => row(0).toString).map(row=>parse_json(row)).toDF().show()
         /*.rdd
         .map(row =>{
         val s = row.getAs[List[String]]("exposure_cards")