Skip to content

C
crawler
Commit 79bbb8d0 authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent cfb67895
Expand all Show whitespace changes
Inline Side-by-side
Showing with 140 additions and 225 deletions
crawler_sys/scheduler/redis_to_rpc/__init__.py 0 → 100644
View file @ 79bbb8d0
# -*- coding:UTF-8 -*-
# @Time : 2021/1/14 19:53
# @File : __init__.py.py
# @email : litao@igengmei.com
# @author : litao
\ No newline at end of file
crawler_sys/scheduler/redis_to_rpc/rpc_config.py 0 → 100644
View file @ 79bbb8d0
# -*- coding:UTF-8 -*-
# @Time : 2021/1/14 19:54
# @File : rpc_config.py
# @email : litao@igengmei.com
# @author : litao
majiayonghu_list = [
36436814,
36436809,
36436805,
36436803,
36436800,
36436797,
36436794,
36436793,
36436787,
36436782,
36436769,
36436763,
36436758,
36436756,
36436749,
36436745,
36436738,
36436731,
36436729,
36436725,
36436720,
36436717,
36436716,
36436709,
36436703,
36436701,
36436690,
36436689,
36436685,
36436674,
36426171,
36426170,
36426169,
36426168,
36426167,
36426166,
36426165,
36426164,
36426163,
36426162,
36426161,
36426160,
36426159,
36426158,
36426157,
36426156,
36426155,
36426154,
36426153,
36426152,
36426150,
36426149,
36426148,
36426147,
36426146,
36426145,
36426143,
36426141,
36368922,
36368921,
36368920,
36368918,
36368917,
]
user_id_list = [
29865245,
36426151,
36426142,
36427666,
36427661,
36427657,
36427655,
36427634,
33524762,
33524779,
33524697,
30963358,
31293584,
31358392,
31358396,
31358397,
31358419,
31358448,
31358610,
31358658,
]
crawler_sys/scheduler/weibo_to_rpc.py
View file @ 79bbb8d0
......@@ -12,6 +12,7 @@ import redis, json
from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
from crawler_sys.utils.output_results import retry_get_url
from crawler.gm_upload.gm_upload import upload, upload_file
from crawler.crawler_sys.scheduler.redis_to_rpc.rpc_config import *
gm_user_id_list = [
"3236957071",
......@@ -26,104 +27,15 @@ gm_user_id_list = [
"7048594049",
]
majiayonghu_list = [
36436814,
36436809,
36436805,
36436803,
36436800,
36436797,
36436794,
36436793,
36436787,
36436782,
36436769,
36436763,
36436758,
36436756,
36436749,
36436745,
36436738,
36436731,
36436729,
36436725,
36436720,
36436717,
36436716,
36436709,
36436703,
36436701,
36436690,
36436689,
36436685,
36436674,
36426171,
36426170,
36426169,
36426168,
36426167,
36426166,
36426165,
36426164,
36426163,
36426162,
36426161,
36426160,
36426159,
36426158,
36426157,
36426156,
36426155,
36426154,
36426153,
36426152,
36426150,
36426149,
36426148,
36426147,
36426146,
36426145,
36426143,
36426141,
36368922,
36368921,
36368920,
36368918,
36368917,
]
user_id_list = [
29865245,
36426151,
36426142,
36427666,
36427661,
36427657,
36427655,
36427634,
33524762,
33524779,
33524697,
30963358,
31293584,
31358392,
31358396,
31358397,
31358419,
31358448,
31358610,
31358658,
]
# f= open("josnfile.json","r",encoding='utf-8')
rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
pid_list = rds.hkeys("weibo")
# for line in f:
for pid in pid_list:
res = rds.hget("weibo", pid)
if rds.hexists("weibo_with_img", pid):
while True:
if rds.hlen("weibo"):
pid_list = rds.hkeys("weibo")
for pid in pid_list:
if rds.sismember("weibo_exists_set", pid):
rds.hdel("weibo", pid)
continue
res = rds.hget("weibo", pid)
res_json = json.loads(res)
video_dic = {}
qiniu_img_list = []
......@@ -180,6 +92,5 @@ for pid in pid_list:
print(e)
continue
# break
# f.flush()
# f.close()
\ No newline at end of file
rds.hdel("weibo", pid)
crawler_sys/scheduler/xiaohongshu_to_rpc.py
View file @ 79bbb8d0
......@@ -6,11 +6,13 @@
import copy
import datetime
import random
import time
import redis, json
from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
# from crawler_sys.utils.output_results import retry_get_url
# from crawler.gm_upload.gm_upload import upload, upload_file
# from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
from crawler_sys.utils.output_results import retry_get_url
from crawler.gm_upload.gm_upload import upload, upload_file
from crawler.crawler_sys.scheduler.redis_to_rpc.rpc_config import *
gm_user_id_list = [
'5cca9b3700000000120314c9',
......@@ -18,133 +20,38 @@ gm_user_id_list = [
'5c20dd200000000007027c07',
'5fe1c1ba0000000001006e65']
majiayonghu_list = [
36436814,
36436809,
36436805,
36436803,
36436800,
36436797,
36436794,
36436793,
36436787,
36436782,
36436769,
36436763,
36436758,
36436756,
36436749,
36436745,
36436738,
36436731,
36436729,
36436725,
36436720,
36436717,
36436716,
36436709,
36436703,
36436701,
36436690,
36436689,
36436685,
36436674,
36426171,
36426170,
36426169,
36426168,
36426167,
36426166,
36426165,
36426164,
36426163,
36426162,
36426161,
36426160,
36426159,
36426158,
36426157,
36426156,
36426155,
36426154,
36426153,
36426152,
36426150,
36426149,
36426148,
36426147,
36426146,
36426145,
36426143,
36426141,
36368922,
36368921,
36368920,
36368918,
36368917,
]
rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
while True:
if rds.hlen("xiaohongshu"):
pid_list = rds.hkeys("xiaohongshu")
for pid in pid_list:
user_id_list = [
29865245,
36426151,
36426142,
36427666,
36427661,
36427657,
36427655,
36427634,
33524762,
33524779,
33524697,
30963358,
31293584,
31358392,
31358396,
31358397,
31358419,
31358448,
31358610,
31358658,
]
f= open("josnfile.json","r",encoding='utf-8')
# rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
# pid_list = rds.hkeys("xiaohongshu")
for line in f:
# for pid in f:
# res = rds.hget("xiaohongshu", pid)
# if rds.hexists("xiaohongshu_with_img", pid):
# continue
if rds.sismember("xiaohongshu_exists_set", pid):
rds.hdel("xiaohongshu", pid)
continue
line = rds.hget("xiaohongshu", pid)
res_json = json.loads(line)
video_dic = {}
qiniu_img_list = []
# print(res_json)
try:
pid = res_json["NoteView"]["commentInfo"]["targetNoteId"]
except:
pid = res_json["NoteView"]["content"]["id"]
# for img_url in res_json["NoteView"]["content"]["imageList"]:
# try:
# img_wb = retry_get_url("http:" + img_url["url"].replace(img_url['fileId'],img_url['traceId'])).content
# res = upload(img_wb, img_type=99)
# # print(res)
# img_info = retry_get_url(res + "-imageinfo")
# img_info_json = img_info.json()
# qiniu_img_list.append('<img src="' + res + '-w">')
# except Exception as e:
# print("down load img error %s" % e)
# continue
# print(qiniu_img_list)
for img_url in res_json["NoteView"]["content"]["imageList"]:
try:
# if True:
# desc_fix = "<p>" + res_json["NoteView"]["content"]['desc'].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
# res_json["NoteView"]["content"]["desc_fix"] = desc_fix
desc_fix = res_json["NoteView"]["content"]["desc_fix"]
# print(desc_fix)
# f.write(json.dumps(res_json) + "\n")
# f.flush()
# res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json))
img_wb = retry_get_url("http:" + img_url["url"].replace(img_url['fileId'],img_url['traceId'])).content
res = upload(img_wb, img_type=99)
# print(res)
img_info = retry_get_url(res + "-imageinfo")
img_info_json = img_info.json()
qiniu_img_list.append('<img src="' + res + '-w">')
except Exception as e:
print("down load img error %s" % e)
continue
try:
desc_fix = "<p>" + res_json["NoteView"]["content"]['desc'].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
res_json["NoteView"]["content"]["desc_fix"] = desc_fix
if res_json["NoteView"]["author"]['id'] in gm_user_id_list:
video_dic["level"] = "5"
else:
......@@ -201,7 +108,6 @@ for line in f:
except Exception as e:
print(e)
continue
# break
# f.flush()
# f.close()
\ No newline at end of file
rds.hdel("xiaohongshu",pid)
else:
time.sleep(5)
crawler_sys/site_crawler_by_redis/crawler_xiaohongshu.py 0 → 100644
View file @ 79bbb8d0
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment