过滤异常数据

eda/feededa/src/main/scala/com/gmei/Search_keywords_count.scala

@@ -9,6 +9,7 @@ import scopt.OptionParser
 import com.gmei.lib.AbstractParams
 import org.dmg.pmml.True
 
+import scala.util.Try
 import scala.util.parsing.json.JSON
 
 object Search_keywords_count {
@@ -81,7 +82,7 @@ object Search_keywords_count {
       )
       search_count.show()
 
-      val get_result =search_keywords.rdd.map((_, 1)).reduceByKey(_ + _)
+      val get_result =search_keywords.rdd.map(x=>Try(JSON.parseFull(x.toString()))).filter(_.isSuccess).map((_, 1)).reduceByKey(_ + _)
         .sortBy(_._2,false)
       val temp=get_result.collect()
       for (i <- 0 until temp.length ) {