filter out

src/main/scala/com/gmei/data/dq/pvCheker.scala

@@ -15,19 +15,32 @@ object pvCheker {
     val x = sc.sql(
       s"""
         |select * from (
-        |select
+        | select
         |     params['page_name'] as page_name,
-        |    (case when params['referrer'] = '' then 1 else 0 end),
+        |     (case when params['referrer'] = '' then 1 else 0 end) as has_referrer,
         |     cl_type,
-        |    count(1) as c
-        |from online.tl_hdfs_maidian_view
-        |where partition_date = '$partition_date' and action = 'page_view'
-        |group by params['page_name'], (case when params['referrer'] = '' then 1 else 0 end),cl_type
-        |having c > $vault
+        |     count(1) as c,
+        | from online.tl_hdfs_maidian_view
+        | where partition_date = '$partition_date' and action = 'page_view'
+        | group by params['page_name'], (case when params['referrer'] = '' then 1 else 0 end), cl_type
         |) c
         |order by c.page_name
       """.stripMargin)
 
+    x.createTempView("pv_refer")
+    val y = sc.sql(
+      """
+        |select page_name, has_referrer, cl_type, c
+        |from (
+        | select
+        |  page_name, has_referrer, cl_type, c
+        |  count(page_name) over (partition by page_name, has_referrer) as d
+        | from pv_refer
+        |) t
+        |where t.d > 1
+      """.stripMargin)
+    y.show()
+
     // demo data
     /*
       about_me_message_list	0	ios	335
@@ -39,6 +52,7 @@ object pvCheker {
       all_case_service_comment	0	ios	13163
       all_case_service_comment	1	ios	75
       all_case_service_comment	0	android	8115
+      all_case_service_comment	1	android	8115
 
       all_cases	0	ios	221
       all_cases	0	android	179
@@ -48,6 +62,5 @@ object pvCheker {
       all_wiki	0	android	249
       all_wiki	0	ios	563
      */
-    val m = collection.mutable.Map[String, Double]()
   }
 }