Skip to content

C
crawler
Commit f8fac55c authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent e2fe4476
Show whitespace changes
Inline Side-by-side
Showing with 53 additions and 52 deletions
crawler_sys/scheduler/xiaohongshu_to_rpc.py
View file @ f8fac55c
......@@ -114,7 +114,7 @@ for pid in pid_list:
print(pid)
for img_url in res_json["NoteView"]["content"]["imageList"]:
try:
img_wb = retry_get_url("http:" + img_url["url"]).content
img_wb = retry_get_url("http:" + img_url["url"].replace(img_url['fileId'],img_url['traceId'])).content
res = upload(img_wb, img_type=99)
# print(res)
img_info = retry_get_url(res + "-imageinfo")
......@@ -125,58 +125,59 @@ for pid in pid_list:
continue
# print(qiniu_img_list)
try:
# if True:
desc_fix = "<p>" + res_json["NoteView"]["content"]['desc'].replace('\n', '<br>') + "".join(qiniu_img_list) + "</p>"
res_json["NoteView"]["content"]["desc_fix"] = desc_fix
# print(desc_fix)
res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json))
# video_dic["platform"] = "9"
# video_dic["platform_id"] = pid
# video_dic["platform_answer_id"] = pid
# video_dic["title"] = res_json["NoteView"]["content"]["title"]
#
# user_id_list_copy = copy.deepcopy(user_id_list)
# qustion_id = random.choice(user_id_list_copy)
# user_id_list_copy.remove(qustion_id)
# video_dic["user_id"] = qustion_id
# create_time = datetime.datetime.strptime(res_json["NoteView"]["content"]["time"],
# '%Y-%m-%d %H:%M')
# video_dic["create_time"] = create_time.timestamp()
# rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
# # print(rpc_res)
# video_dic["platform_question_id"] = pid
# video_dic["content"] = desc_fix
# video_dic["user_id"] = random.choice(user_id_list_copy)
# rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
# comment_list = []
# try:
# if res_json["NoteView"].get("comments"):
# # print(res_json["NoteView"].get("data"))
# for comment in res_json["NoteView"]["comments"]["data"]:
# video_dic["content"] = comment['content']
# comment_id_list_copy = copy.deepcopy(majiayonghu_list)
# comment_id = random.choice(comment_id_list_copy)
# video_dic["user_id"] = comment_id
# comment_id_list_copy.remove(comment_id)
# video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0, 24),
# minutes=random.randint(0, 60))).timestamp()
# comment_list.append(copy.deepcopy(video_dic))
# # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
# elif res_json["NoteView"].get("commentInfo"):
# for comment in res_json["NoteView"]["commentInfo"]["comments"]:
# video_dic["content"] = comment['content']
# comment_id_list_copy = copy.deepcopy(majiayonghu_list)
# comment_id = random.choice(comment_id_list_copy)
# video_dic["user_id"] = comment_id
# comment_id_list_copy.remove(comment_id)
# video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0,24),minutes=random.randint(0,60))).timestamp()
# comment_list.append(copy.deepcopy(video_dic))
# # rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
# rpc_res = post_muilty_data(comment_list, "cims/reply/batch_create")
# except Exception as e:
# print("comment error")
# print(e)
# except Exception as e:
# print(e)
# continue
video_dic["platform"] = "9"
video_dic["platform_id"] = pid
video_dic["platform_answer_id"] = pid
video_dic["title"] = res_json["NoteView"]["content"]["title"]
# break
\ No newline at end of file
user_id_list_copy = copy.deepcopy(user_id_list)
qustion_id = random.choice(user_id_list_copy)
user_id_list_copy.remove(qustion_id)
video_dic["user_id"] = qustion_id
create_time = datetime.datetime.strptime(res_json["NoteView"]["content"]["time"],
'%Y-%m-%d %H:%M')
video_dic["create_time"] = create_time.timestamp()
rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/question/batch_create")
# print(rpc_res)
video_dic["platform_question_id"] = pid
video_dic["content"] = desc_fix
video_dic["user_id"] = random.choice(user_id_list_copy)
rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/answer/batch_create")
comment_list = []
try:
if res_json["NoteView"].get("comments"):
# print(res_json["NoteView"].get("data"))
for comment in res_json["NoteView"]["comments"]["data"]:
video_dic["content"] = comment['content']
comment_id_list_copy = copy.deepcopy(majiayonghu_list)
comment_id = random.choice(comment_id_list_copy)
video_dic["user_id"] = comment_id
comment_id_list_copy.remove(comment_id)
video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0, 24),
minutes=random.randint(0, 60))).timestamp()
comment_list.append(copy.deepcopy(video_dic))
# rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
elif res_json["NoteView"].get("commentInfo"):
for comment in res_json["NoteView"]["commentInfo"]["comments"]:
video_dic["content"] = comment['content']
comment_id_list_copy = copy.deepcopy(majiayonghu_list)
comment_id = random.choice(comment_id_list_copy)
video_dic["user_id"] = comment_id
comment_id_list_copy.remove(comment_id)
video_dic["create_time"] = (create_time + datetime.timedelta(hours=random.randint(0,24),minutes=random.randint(0,60))).timestamp()
comment_list.append(copy.deepcopy(video_dic))
# rpc_res = post_single_data(copy.deepcopy(video_dic), "cims/reply/batch_create")
rpc_res = post_muilty_data(comment_list, "cims/reply/batch_create")
except Exception as e:
print("comment error")
print(e)
except Exception as e:
print(e)
continue
break
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment