Commit 8247d5ec authored by 王志伟's avatar 王志伟

搜索关键词统计bug修复

parent 247ec5de
...@@ -52,6 +52,7 @@ object Search_keywords_count { ...@@ -52,6 +52,7 @@ object Search_keywords_count {
} }
} }
def main(args: Array[String]): Unit = { def main(args: Array[String]): Unit = {
parser.parse(args, defaultParams).map { param => parser.parse(args, defaultParams).map { param =>
GmeiConfig.setup(param.env) GmeiConfig.setup(param.env)
...@@ -61,16 +62,18 @@ object Search_keywords_count { ...@@ -61,16 +62,18 @@ object Search_keywords_count {
//println(param.date) //println(param.date)
val partition_date = stat_date.replace("-","") val partition_date = stat_date.replace("-","")
sc.udf.register("test_udf",test.test_udf _)
//搜索关键词提取 //搜索关键词提取
val search_keywords = sc.sql( val search_keywords = sc.sql(
s""" s"""
|select params as search_keywords |select test_udf(params) as search_keywords
|from online.tl_hdfs_maidian_view |from online.tl_hdfs_maidian_view
|where (action = 'do_search' or action = 'search_result_click_search') |where (action = 'do_search' or action = 'search_result_click_search')
|and partition_date ='20190109' |and partition_date ='20190109'
""".stripMargin """.stripMargin
).rdd.map(x=>{ ).show(20)/*.rdd.map(x=>{
val b=JSON.parseFull(x.toString()) val b=JSON.parseFull(x.toString())
b match { b match {
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any // Matches if jsonStr is valid JSON and represents a Map of Strings to Any
...@@ -78,7 +81,7 @@ object Search_keywords_count { ...@@ -78,7 +81,7 @@ object Search_keywords_count {
case None => println("Parsing failed") case None => println("Parsing failed")
case other => println("Unknown data structure: " + other) case other => println("Unknown data structure: " + other)
} }
}).foreach(println) }).foreach(println)*/
// search_keywords.show() // search_keywords.show()
// val tempp=search_keywords.collect() // val tempp=search_keywords.collect()
...@@ -109,3 +112,14 @@ object Search_keywords_count { ...@@ -109,3 +112,14 @@ object Search_keywords_count {
} }
} }
object test{
def test_udf(param:String) ={
val b=JSON.parseFull(param)
b match {
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any
case Some(map: Map[String, Any]) => param
case None => ""
case _ => ""
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment