Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
F
ffm-baseline
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ML
ffm-baseline
Commits
fceab030
Commit
fceab030
authored
Aug 16, 2019
by
高雅喆
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update
parent
7fc880d8
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
14 additions
and
12 deletions
+14
-12
dist_update_user_portrait.py
eda/smart_rank/dist_update_user_portrait.py
+14
-12
No files found.
eda/smart_rank/dist_update_user_portrait.py
View file @
fceab030
...
...
@@ -142,17 +142,18 @@ if __name__ == '__main__':
device_ids_lst_rdd
=
spark
.
sparkContext
.
parallelize
(
device_ids_lst
)
gm_kv_cli
=
redis
.
Redis
(
host
=
"172.16.40.135"
,
port
=
5379
,
db
=
6
,
socket_timeout
=
2000
)
result
=
device_ids_lst_rdd
.
repartition
(
100
)
.
map
(
lambda
x
:
get_user_tag_score
(
x
,
all_log_df
))
.
toDF
()
result_rename
=
result
.
selectExpr
(
"_1 as cl_id"
,
"_2 as tag_list"
)
stat_date
=
datetime
.
datetime
.
today
()
.
strftime
(
'
%
Y-
%
m-
%
d'
)
result_last
=
result_rename
.
withColumn
(
"stat_date"
,
lit
(
stat_date
))
result_last
.
show
()
df
=
result_last
.
select
(
"stat_date"
,
"cl_id"
,
concat_ws
(
','
,
'tag_list'
)
.
alias
(
"tag_list"
))
df
.
show
()
df
.
write
.
jdbc
(
mode
=
"overwrite"
,
url
=
"jdbc:mysql://172.16.40.158:4000/jerry_test?user=root&password=3SYz54LS9#^9sBvC&useSSL=true"
,
table
=
"user_portrait_tags"
,
properties
=
{
"driver"
:
'com.mysql.jdbc.Driver'
})
result
=
device_ids_lst_rdd
.
repartition
(
100
)
.
map
(
lambda
x
:
get_user_tag_score
(
x
,
all_log_df
))
a
=
result
.
collect
()
print
(
a
)
# stat_date = datetime.datetime.today().strftime('%Y-%m-%d')
# result_last = result_rename.withColumn("stat_date", lit(stat_date))
# result_last.show()
# df = result_last.select("stat_date", "cl_id", concat_ws(',', 'tag_list').alias("tag_list"))
# df.show()
# df.write.jdbc(
# mode="overwrite",
# url="jdbc:mysql://172.16.40.158:4000/jerry_test?user=root&password=3SYz54LS9#^9sBvC&useSSL=true",
# table="user_portrait_tags",
# properties={"driver": 'com.mysql.jdbc.Driver'})
except
Exception
as
e
:
print
(
e
)
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment