Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
crawler
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
backend
crawler
Commits
233c49d3
Commit
233c49d3
authored
4 years ago
by
litaolemo
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update
parent
e3f24ff6
master
litao
mr/develop/xiaohongshu
soyang
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
6 additions
and
5 deletions
+6
-5
cal_ni_and_put_to_backend.py
crawler_sys/scheduler/cal_ni_and_put_to_backend.py
+6
-5
No files found.
crawler_sys/scheduler/cal_ni_and_put_to_backend.py
View file @
233c49d3
...
...
@@ -29,9 +29,7 @@ import random
rds
=
redis
.
StrictRedis
(
host
=
'192.144.194.190'
,
port
=
6379
,
db
=
19
)
# conn = pymysql.connect(host='bj-cdb-6slgqwlc.sql.tencentcdb.com', port=62120, user='work', passwd='Gengmei1',
# db='mimas_test', charset='utf8')
conn
=
pymysql
.
connect
(
host
=
'172.16.30.138'
,
port
=
3306
,
user
=
'mimas'
,
passwd
=
'GJL3UJe1Ck9ggL6aKnZCq4cRvM'
,
db
=
'mimas_prod'
,
charset
=
'utf8mb4'
)
cur
=
conn
.
cursor
()
class
push_rule
(
object
):
...
...
@@ -313,6 +311,9 @@ img_type = {
def
write_data_into_mysql
(
res_data
):
conn
=
pymysql
.
connect
(
host
=
'172.16.30.138'
,
port
=
3306
,
user
=
'mimas'
,
passwd
=
'GJL3UJe1Ck9ggL6aKnZCq4cRvM'
,
db
=
'mimas_prod'
,
charset
=
'utf8mb4'
)
cur
=
conn
.
cursor
()
now_str
=
datetime
.
datetime
.
now
()
.
strftime
(
"
%
Y-
%
m-
%
d
%
H:
%
M:
%
S"
)
# 清洗数据为可以入库的格式
data
=
push_data_to_user
(
res_data
)
...
...
@@ -381,6 +382,8 @@ def write_data_into_mysql(res_data):
except
Exception
as
e
:
print
(
"commit error
%
s"
%
e
)
conn
.
rollback
()
cur
.
close
()
conn
.
close
()
if
tractate_id
:
return
tractate_id
else
:
...
...
@@ -480,8 +483,6 @@ def task_main():
# test = {'release_time': 1595952037000, 'fetch_time': 1596012816514, 'url': 'https://www.douban.com/group/topic/186707979/', 'releaser': '🍫', 'repost_count': 40, 'comment_count': 411, 'favorite_count': 144, 'title': '王俊凯终于还是举铁了', 'releaserUrl': 'https://www.douban.com/people/57762442', 'releaser_id_str': 'douban_57762442', 'video_img': 'https://img3.doubanio.com/view/group_topic/sqxs/public/p317684082.webp', 'mid': '186707979', 'platform': 'douban', 'doc_id': 'douban_186707979', 'content': '<div id=\'content\'><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp" width="500"/></div></div><p></p></div>', 'collection_count': 107, 'img_list': ['https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp'], 'level': 5}
# write_data_into_mysql(test)
cur
.
close
()
conn
.
close
()
if
__name__
==
"__main__"
:
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment