Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
crawler
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
backend
crawler
Commits
602fada3
Commit
602fada3
authored
4 years ago
by
litaolemo
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
update
parent
faab8b9d
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
8 additions
and
5 deletions
+8
-5
cal_ni_and_put_to_backend.py
crawler_sys/scheduler/cal_ni_and_put_to_backend.py
+2
-0
run.sh
run.sh
+3
-3
start_crawler.sh
start_crawler.sh
+3
-2
No files found.
crawler_sys/scheduler/cal_ni_and_put_to_backend.py
View file @
602fada3
...
...
@@ -391,6 +391,8 @@ def task_main():
"""
.
format
(
tractate_id
=
tractate_id
,
content
=
res_data
[
"content"
])
send_file_email
(
""
,
""
,
email_group
=
[
"<duanyingrong@igengmei.com>"
],
cc_group
=
[
"litao@igengmei.com"
],
email_msg_body_str
=
body_str
,
title_str
=
title_str
)
print
(
"send to mysql"
)
print
(
res_data
)
# test = {'release_time': 1595952037000, 'fetch_time': 1596012816514, 'url': 'https://www.douban.com/group/topic/186707979/', 'releaser': '🍫', 'repost_count': 40, 'comment_count': 411, 'favorite_count': 144, 'title': '王俊凯终于还是举铁了', 'releaserUrl': 'https://www.douban.com/people/57762442', 'releaser_id_str': 'douban_57762442', 'video_img': 'https://img3.doubanio.com/view/group_topic/sqxs/public/p317684082.webp', 'mid': '186707979', 'platform': 'douban', 'doc_id': 'douban_186707979', 'content': '<div id=\'content\'><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp" width="500"/></div></div><div class="image-container image-float-center"><div class="image-wrapper"><img src="https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp" width="500"/></div></div><p></p></div>', 'collection_count': 107, 'img_list': ['https://img3.doubanio.com/view/group_topic/l/public/p317684082.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684064.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684093.webp', 'https://img9.doubanio.com/view/group_topic/l/public/p317684095.webp', 'https://img3.doubanio.com/view/group_topic/l/public/p317684052.webp'], 'level': 5}
# write_data_into_mysql(test)
cur
.
close
()
...
...
This diff is collapsed.
Click to expand it.
run.sh
View file @
602fada3
...
...
@@ -2,5 +2,5 @@
#sudo su - gmuser
source
/root/anaconda3/bin/activate
conda activate crawler_env
python /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py
-p
weibo
-d
1
-proxies
5
>
/data/log/write_task.log &
python /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py
-p
douban
-d
1
-proxies
5
>
/data/log/write_task.log &
\ No newline at end of file
python /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py
-p
weibo
-d
1
-proxies
5
>
/data/log/crawler/write_task.log &
python /srv/apps/crawler/crawler_sys/framework/write_releasers_to_redis.py
-p
douban
-d
1
-proxies
5
>
/data/log/crawler/write_task.log &
\ No newline at end of file
This diff is collapsed.
Click to expand it.
start_crawler.sh
View file @
602fada3
...
...
@@ -2,4 +2,5 @@
source
/root/anaconda3/bin/activate
conda activate crawler_env
current_date
=
`
date
-d
"-1 day"
"+%Y%m%d"
`
python /srv/apps/crawler/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py
>
/data/log/write_task_
${
current_date
}
_.log &
\ No newline at end of file
python /srv/apps/crawler/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py
>
/data/log/crawler/write_task_
${
current_date
}
_.log &
python /srv/apps/crawler/crawler_sys/scheduler/cal_ni_and_put_to_backend.py
>
/data/log/crawler/.log &
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment