Commit 8247d5ec authored by 王志伟's avatar 王志伟

搜索关键词统计bug修复

parent 247ec5de
......@@ -52,6 +52,7 @@ object Search_keywords_count {
}
}
def main(args: Array[String]): Unit = {
parser.parse(args, defaultParams).map { param =>
GmeiConfig.setup(param.env)
......@@ -61,16 +62,18 @@ object Search_keywords_count {
//println(param.date)
val partition_date = stat_date.replace("-","")
sc.udf.register("test_udf",test.test_udf _)
//搜索关键词提取
val search_keywords = sc.sql(
s"""
|select params as search_keywords
|select test_udf(params) as search_keywords
|from online.tl_hdfs_maidian_view
|where (action = 'do_search' or action = 'search_result_click_search')
|and partition_date ='20190109'
""".stripMargin
).rdd.map(x=>{
).show(20)/*.rdd.map(x=>{
val b=JSON.parseFull(x.toString())
b match {
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any
......@@ -78,7 +81,7 @@ object Search_keywords_count {
case None => println("Parsing failed")
case other => println("Unknown data structure: " + other)
}
}).foreach(println)
}).foreach(println)*/
// search_keywords.show()
// val tempp=search_keywords.collect()
......@@ -108,4 +111,15 @@ object Search_keywords_count {
}
}
object test{
def test_udf(param:String) ={
val b=JSON.parseFull(param)
b match {
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any
case Some(map: Map[String, Any]) => param
case None => ""
case _ => ""
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment