Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
F
ffm-baseline
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ML
ffm-baseline
Commits
8247d5ec
Commit
8247d5ec
authored
Jan 11, 2019
by
王志伟
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
搜索关键词统计bug修复
parent
247ec5de
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
18 additions
and
3 deletions
+18
-3
Search_keywords_count.scala
...ededa/src/main/scala/com/gmei/Search_keywords_count.scala
+18
-3
No files found.
eda/feededa/src/main/scala/com/gmei/Search_keywords_count.scala
View file @
8247d5ec
...
...
@@ -52,6 +52,7 @@ object Search_keywords_count {
}
}
def
main
(
args
:
Array
[
String
])
:
Unit
=
{
parser
.
parse
(
args
,
defaultParams
).
map
{
param
=>
GmeiConfig
.
setup
(
param
.
env
)
...
...
@@ -61,16 +62,18 @@ object Search_keywords_count {
//println(param.date)
val
partition_date
=
stat_date
.
replace
(
"-"
,
""
)
sc
.
udf
.
register
(
"test_udf"
,
test
.
test_udf
_
)
//搜索关键词提取
val
search_keywords
=
sc
.
sql
(
s
"""
|select
params
as search_keywords
|select
test_udf(params)
as search_keywords
|from online.tl_hdfs_maidian_view
|where (action = 'do_search' or action = 'search_result_click_search')
|and partition_date ='20190109'
"""
.
stripMargin
).
rdd
.
map
(
x
=>{
).
show
(
20
)
/*.
rdd.map(x=>{
val b=JSON.parseFull(x.toString())
b match {
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any
...
...
@@ -78,7 +81,7 @@ object Search_keywords_count {
case None => println("Parsing failed")
case other => println("Unknown data structure: " + other)
}
}).
foreach
(
println
)
}).foreach(println)
*/
// search_keywords.show()
// val tempp=search_keywords.collect()
...
...
@@ -108,4 +111,15 @@ object Search_keywords_count {
}
}
object
test
{
def
test_udf
(
param
:
String
)
={
val
b
=
JSON
.
parseFull
(
param
)
b
match
{
// Matches if jsonStr is valid JSON and represents a Map of Strings to Any
case
Some
(
map
:
Map
[
String
,
Any
])
=>
param
case
None
=>
""
case
_
=>
""
}
}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment