Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
crawler
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Chengyang Zhong
crawler
Commits
a9f89e25
Commit
a9f89e25
authored
4 years ago
by
litaolemo
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update
parent
d2ef16bb
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
2 deletions
+4
-2
from_sparksql_to_mysql.py
tasks/from_sparksql_to_mysql.py
+4
-2
No files found.
tasks/from_sparksql_to_mysql.py
View file @
a9f89e25
...
...
@@ -78,7 +78,7 @@ device_df.show(1, False)
sql_res
=
device_df
.
collect
()
print
(
"-----------------------------------------------------------------------------"
)
for
res
in
sql_res
:
print
(
res
)
#
print(res)
day_id
=
res
.
day_id
device_os_type
=
res
.
device_os_type
active_type
=
res
.
active_type
...
...
@@ -100,12 +100,13 @@ for res in sql_res:
wel_click_pv,content_click_pv,slide_wel_click_pv,partition_day,pid
) VALUES('{day_id}','{device_os_type}',
'{active_type}','{grey_type}','{page_name}',{content_pv},{content_uv},{wel_exp_pv},{content_exp_pv},{wel_click_pv},{content_click_pv},
{wel_click_pv},{content_click_pv},{slide_wel_click_pv},'{partition_day}','{pid}
)'
"""
.
format
(
{wel_click_pv},{content_click_pv},{slide_wel_click_pv},'{partition_day}','{pid}
')
"""
.
format
(
day_id
=
day_id
,
device_os_type
=
device_os_type
,
active_type
=
active_type
,
grey_type
=
grey_type
,
page_name
=
page_name
,
content_pv
=
content_pv
,
content_uv
=
content_uv
,
wel_exp_pv
=
wel_exp_pv
,
content_exp_pv
=
content_exp_pv
,
wel_click_pv
=
wel_click_pv
,
content_click_pv
=
content_click_pv
,
slide_wel_click_pv
=
slide_wel_click_pv
,
self_wel_click_pv
=
self_wel_click_pv
,
partition_day
=
partition_day
,
pid
=
pid
)
print
(
sql
)
cursor
.
execute
(
sql
)
# cursor.executemany()
db
.
close
()
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment