Skip to content

C
crawler
Commit 5634a341 authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent 9208da8a
Hide whitespace changes
Inline Side-by-side
Showing with 142 additions and 126 deletions
crawler_sys/scheduler/xiaohongshu_to_rpc.py
View file @ 5634a341
...@@ -137,9 +137,9 @@ for pid in pid_list: ...@@ -137,9 +137,9 @@ for pid in pid_list:
# print(desc_fix) # print(desc_fix)
res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json)) res = rds.hset("xiaohongshu_with_img", key=pid, value=json.dumps(res_json))
if res_json["NoteView"]["author"]['id'] in gm_user_id_list: if res_json["NoteView"]["author"]['id'] in gm_user_id_list:
video_dic["level"] = "4" video_dic["level"] = "5"
else: else:
video_dic["level"] = "2" video_dic["level"] = "3"
video_dic["platform"] = "9" video_dic["platform"] = "9"
video_dic["platform_id"] = pid video_dic["platform_id"] = pid
video_dic["platform_answer_id"] = pid video_dic["platform_answer_id"] = pid
......
crawler_sys/site_crawler/crawler_xiaohongshu.py
View file @ 5634a341
...@@ -197,8 +197,7 @@ class Crawler_xiaohongshu(): ...@@ -197,8 +197,7 @@ class Crawler_xiaohongshu():
releaser_id = self.get_releaser_id(releaserUrl) releaser_id = self.get_releaser_id(releaserUrl)
releaserUrl = 'http://www.xiaohongshu.com/user/profile/%s' % releaser_id releaserUrl = 'http://www.xiaohongshu.com/user/profile/%s' % releaser_id
pcursor = 0 pcursor = 0
cookie_dic = {'timestamp2': '2021010899964852bd70ca4c0c991c6c', 'xhsuid': 'cqq3glNpFsMgH50j', 'xhs_spses.5dde': '*', 'xhs_spid.5dde': 'fa1043ce96194610.1610072893.1.1610072895.1610072893.3536bab9-1e85-4a3a-8a46-37e694100de1', 'extra_exp_ids': 'gif_clt1,ques_clt1', 'xhsTrackerId': '591fba69-1884-4ab2-ca05-9ae70ab77d2e'} cookie_dic = {'timestamp2': '2021010899964852bd70ca4c0c991c6c'}
# print(proxies)
# proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'} # proxies = {'http': 'http://hanye:i9mmu0a3@58.55.159.141:16085/', 'https': 'http://hanye:i9mmu0a3@58.55.159.141:16085/'}
while count <= releaser_page_num_max and count <= 1: while count <= releaser_page_num_max and count <= 1:
try: try:
...@@ -252,106 +251,106 @@ if __name__ == '__main__': ...@@ -252,106 +251,106 @@ if __name__ == '__main__':
test = Crawler_xiaohongshu() test = Crawler_xiaohongshu()
releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae' releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
url_list =[ url_list =[
# "https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae", "https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae",
# "https://www.xiaohongshu.com/user/profile/5ea6909900000000010057a3", "https://www.xiaohongshu.com/user/profile/5ea6909900000000010057a3",
# "https://www.xiaohongshu.com/user/profile/5a03b1f4b1da1412dd070a86", "https://www.xiaohongshu.com/user/profile/5a03b1f4b1da1412dd070a86",
# "https://www.xiaohongshu.com/user/profile/5b6e76419276ee0001bd5740", "https://www.xiaohongshu.com/user/profile/5b6e76419276ee0001bd5740",
# "https://www.xiaohongshu.com/user/profile/5c4140500000000006006cb7", "https://www.xiaohongshu.com/user/profile/5c4140500000000006006cb7",
# "https://www.xiaohongshu.com/user/profile/5bd2beff7da0890001b5408a", "https://www.xiaohongshu.com/user/profile/5bd2beff7da0890001b5408a",
# "https://www.xiaohongshu.com/user/profile/5b5edc5211be1044bcce7824", "https://www.xiaohongshu.com/user/profile/5b5edc5211be1044bcce7824",
# "https://www.xiaohongshu.com/user/profile/5b35cce84eacab52fbe15c0b", "https://www.xiaohongshu.com/user/profile/5b35cce84eacab52fbe15c0b",
# "https://www.xiaohongshu.com/user/profile/5efec35c000000000101d75a", "https://www.xiaohongshu.com/user/profile/5efec35c000000000101d75a",
# "https://www.xiaohongshu.com/user/profile/5f91428a000000000101d909", "https://www.xiaohongshu.com/user/profile/5f91428a000000000101d909",
# "https://www.xiaohongshu.com/user/profile/5ed49f1200000000010017f0", "https://www.xiaohongshu.com/user/profile/5ed49f1200000000010017f0",
# "https://www.xiaohongshu.com/user/profile/5ae3f47b11be105fae4b854c", "https://www.xiaohongshu.com/user/profile/5ae3f47b11be105fae4b854c",
# "https://www.xiaohongshu.com/user/profile/5a9e10fb11be1006adc5b9d5", "https://www.xiaohongshu.com/user/profile/5a9e10fb11be1006adc5b9d5",
# "https://www.xiaohongshu.com/user/profile/5d0c3b900000000012013409", "https://www.xiaohongshu.com/user/profile/5d0c3b900000000012013409",
# "https://www.xiaohongshu.com/user/profile/5f1013a70000000001005b16", "https://www.xiaohongshu.com/user/profile/5f1013a70000000001005b16",
# "https://www.xiaohongshu.com/user/profile/5f5c6d860000000001001787", "https://www.xiaohongshu.com/user/profile/5f5c6d860000000001001787",
# "https://www.xiaohongshu.com/user/profile/5eeb18e600000000010062b6", "https://www.xiaohongshu.com/user/profile/5eeb18e600000000010062b6",
# "https://www.xiaohongshu.com/user/profile/5bab62e9ee80fc0001505980", "https://www.xiaohongshu.com/user/profile/5bab62e9ee80fc0001505980",
# "https://www.xiaohongshu.com/user/profile/5f262a610000000001004ea9", "https://www.xiaohongshu.com/user/profile/5f262a610000000001004ea9",
# "https://www.xiaohongshu.com/user/profile/5eb6779300000000010045f5", "https://www.xiaohongshu.com/user/profile/5eb6779300000000010045f5",
# "https://www.xiaohongshu.com/user/profile/5c855374000000001202ef0c", "https://www.xiaohongshu.com/user/profile/5c855374000000001202ef0c",
# "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4", "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4",
# "https://www.xiaohongshu.com/user/profile/5f100b2d000000000100138d", "https://www.xiaohongshu.com/user/profile/5f100b2d000000000100138d",
# "https://www.xiaohongshu.com/user/profile/5c14ae400000000006016f5d", "https://www.xiaohongshu.com/user/profile/5c14ae400000000006016f5d",
# "https://www.xiaohongshu.com/user/profile/5bbd28de4c26220001881cbd", "https://www.xiaohongshu.com/user/profile/5bbd28de4c26220001881cbd",
# "https://www.xiaohongshu.com/user/profile/5f86b6fc000000000100a5d8", "https://www.xiaohongshu.com/user/profile/5f86b6fc000000000100a5d8",
# "https://www.xiaohongshu.com/user/profile/5db16ca20000000001004c02", "https://www.xiaohongshu.com/user/profile/5db16ca20000000001004c02",
# "https://www.xiaohongshu.com/user/profile/5ad553bb4eacab34ee9f7d4a", "https://www.xiaohongshu.com/user/profile/5ad553bb4eacab34ee9f7d4a",
# "https://www.xiaohongshu.com/user/profile/5f12cffd000000000101da61", "https://www.xiaohongshu.com/user/profile/5f12cffd000000000101da61",
# "https://www.xiaohongshu.com/user/profile/596d7e4f5e87e722ff1bfd32", "https://www.xiaohongshu.com/user/profile/596d7e4f5e87e722ff1bfd32",
# "https://www.xiaohongshu.com/user/profile/5ef17ad00000000001005e1c", "https://www.xiaohongshu.com/user/profile/5ef17ad00000000001005e1c",
# "https://www.xiaohongshu.com/user/profile/5f75a5700000000001007679", "https://www.xiaohongshu.com/user/profile/5f75a5700000000001007679",
# "https://www.xiaohongshu.com/user/profile/5c639f59000000001000c731", "https://www.xiaohongshu.com/user/profile/5c639f59000000001000c731",
# "https://www.xiaohongshu.com/user/profile/5f865cbd0000000001002f01", "https://www.xiaohongshu.com/user/profile/5f865cbd0000000001002f01",
# "https://www.xiaohongshu.com/user/profile/5eccc58f000000000100753e", "https://www.xiaohongshu.com/user/profile/5eccc58f000000000100753e",
# "https://www.xiaohongshu.com/user/profile/5fbe05b4000000000101c88d", "https://www.xiaohongshu.com/user/profile/5fbe05b4000000000101c88d",
# "https://www.xiaohongshu.com/user/profile/5b7d1da7e8ac2b471ee6fef3", "https://www.xiaohongshu.com/user/profile/5b7d1da7e8ac2b471ee6fef3",
# "https://www.xiaohongshu.com/user/profile/5a11b22211be101018ba7125", "https://www.xiaohongshu.com/user/profile/5a11b22211be101018ba7125",
# "https://www.xiaohongshu.com/user/profile/5a76c3c611be107f08bd35b3", "https://www.xiaohongshu.com/user/profile/5a76c3c611be107f08bd35b3",
# "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4", "https://www.xiaohongshu.com/user/profile/5ecb6d7300000000010016a4",
# "https://www.xiaohongshu.com/user/profile/5f2539e80000000001009d9e", "https://www.xiaohongshu.com/user/profile/5f2539e80000000001009d9e",
# "https://www.xiaohongshu.com/user/profile/561b1fd8e4b1cf0295755d05", "https://www.xiaohongshu.com/user/profile/561b1fd8e4b1cf0295755d05",
# "https://www.xiaohongshu.com/user/profile/5beeba1ff7e8b93bc0405234", "https://www.xiaohongshu.com/user/profile/5beeba1ff7e8b93bc0405234",
# "https://www.xiaohongshu.com/user/profile/5c87785f000000001000ed51", "https://www.xiaohongshu.com/user/profile/5c87785f000000001000ed51",
# "https://www.xiaohongshu.com/user/profile/5efdba65000000000101c79c", "https://www.xiaohongshu.com/user/profile/5efdba65000000000101c79c",
# "https://www.xiaohongshu.com/user/profile/5507e7dfa46e9616260827f6", "https://www.xiaohongshu.com/user/profile/5507e7dfa46e9616260827f6",
# "https://www.xiaohongshu.com/user/profile/567573470bf90c27957dd73c", "https://www.xiaohongshu.com/user/profile/567573470bf90c27957dd73c",
# "https://www.xiaohongshu.com/user/profile/5fd1821b000000000100381a", "https://www.xiaohongshu.com/user/profile/5fd1821b000000000100381a",
# "https://www.xiaohongshu.com/user/profile/5f5f6b1c00000000010064dc", "https://www.xiaohongshu.com/user/profile/5f5f6b1c00000000010064dc",
# "https://www.xiaohongshu.com/user/profile/5aea4d31e8ac2b4a44e1d2d4", "https://www.xiaohongshu.com/user/profile/5aea4d31e8ac2b4a44e1d2d4",
# "https://www.xiaohongshu.com/user/profile/5f39eabb00000000010076ca", "https://www.xiaohongshu.com/user/profile/5f39eabb00000000010076ca",
# "https://www.xiaohongshu.com/user/profile/5cda11d7000000001703780c", "https://www.xiaohongshu.com/user/profile/5cda11d7000000001703780c",
# "https://www.xiaohongshu.com/user/profile/5cbc3e9f000000001701d7bf", "https://www.xiaohongshu.com/user/profile/5cbc3e9f000000001701d7bf",
# "https://www.xiaohongshu.com/user/profile/5e7886930000000001003f7f", "https://www.xiaohongshu.com/user/profile/5e7886930000000001003f7f",
# "https://www.xiaohongshu.com/user/profile/566fbc3550c4b435f51f637b", "https://www.xiaohongshu.com/user/profile/566fbc3550c4b435f51f637b",
# "https://www.xiaohongshu.com/user/profile/5e86cb34000000000100a223", "https://www.xiaohongshu.com/user/profile/5e86cb34000000000100a223",
# "https://www.xiaohongshu.com/user/profile/558e15b2f5a263490c65cdaa", "https://www.xiaohongshu.com/user/profile/558e15b2f5a263490c65cdaa",
# "https://www.xiaohongshu.com/user/profile/5d9eef320000000001001615", "https://www.xiaohongshu.com/user/profile/5d9eef320000000001001615",
# "https://www.xiaohongshu.com/user/profile/5a6ba3214eacab4eee8e627a", "https://www.xiaohongshu.com/user/profile/5a6ba3214eacab4eee8e627a",
# "https://www.xiaohongshu.com/user/profile/5f58cacb000000000100bdf5", "https://www.xiaohongshu.com/user/profile/5f58cacb000000000100bdf5",
# "https://www.xiaohongshu.com/user/profile/5f954030000000000100780c", "https://www.xiaohongshu.com/user/profile/5f954030000000000100780c",
# "https://www.xiaohongshu.com/user/profile/5f5745bf000000000100351d", "https://www.xiaohongshu.com/user/profile/5f5745bf000000000100351d",
# "https://www.xiaohongshu.com/user/profile/5c74a2b9000000001002e667", "https://www.xiaohongshu.com/user/profile/5c74a2b9000000001002e667",
# "https://www.xiaohongshu.com/user/profile/595ee5b882ec397553103dd3", "https://www.xiaohongshu.com/user/profile/595ee5b882ec397553103dd3",
# "https://www.xiaohongshu.com/user/profile/5a5e20324eacab30f03654fb", "https://www.xiaohongshu.com/user/profile/5a5e20324eacab30f03654fb",
# "https://www.xiaohongshu.com/user/profile/55743bedc2bdeb1a16844741", "https://www.xiaohongshu.com/user/profile/55743bedc2bdeb1a16844741",
# "https://www.xiaohongshu.com/user/profile/5f0d523800000000010056de", "https://www.xiaohongshu.com/user/profile/5f0d523800000000010056de",
# "https://www.xiaohongshu.com/user/profile/59d5b03e44363b61a050532f", "https://www.xiaohongshu.com/user/profile/59d5b03e44363b61a050532f",
# "https://www.xiaohongshu.com/user/profile/5ebdd5f40000000001002a67", "https://www.xiaohongshu.com/user/profile/5ebdd5f40000000001002a67",
# "https://www.xiaohongshu.com/user/profile/5f1c1b7b0000000001006cbf", "https://www.xiaohongshu.com/user/profile/5f1c1b7b0000000001006cbf",
# "https://www.xiaohongshu.com/user/profile/5ae404944eacab794dfb95b1", "https://www.xiaohongshu.com/user/profile/5ae404944eacab794dfb95b1",
# "https://www.xiaohongshu.com/user/profile/5d26276a0000000012017538", "https://www.xiaohongshu.com/user/profile/5d26276a0000000012017538",
# "https://www.xiaohongshu.com/user/profile/5ed5aa8f0000000001001f1e", "https://www.xiaohongshu.com/user/profile/5ed5aa8f0000000001001f1e",
# "https://www.xiaohongshu.com/user/profile/5f92cf4f000000000100a846", "https://www.xiaohongshu.com/user/profile/5f92cf4f000000000100a846",
# "https://www.xiaohongshu.com/user/profile/5a75d42011be10344b917ffe", "https://www.xiaohongshu.com/user/profile/5a75d42011be10344b917ffe",
# "https://www.xiaohongshu.com/user/profile/5ccea0ff000000001002b753", "https://www.xiaohongshu.com/user/profile/5ccea0ff000000001002b753",
# "https://www.xiaohongshu.com/user/profile/5c4418750000000005006717", "https://www.xiaohongshu.com/user/profile/5c4418750000000005006717",
# "https://www.xiaohongshu.com/user/profile/5ec582d60000000001005315", "https://www.xiaohongshu.com/user/profile/5ec582d60000000001005315",
# "https://www.xiaohongshu.com/user/profile/594a93835e87e72f3e2ded11", "https://www.xiaohongshu.com/user/profile/594a93835e87e72f3e2ded11",
# "https://www.xiaohongshu.com/user/profile/5b8ab07606311b000184195a", "https://www.xiaohongshu.com/user/profile/5b8ab07606311b000184195a",
# "https://www.xiaohongshu.com/user/profile/54e7413ea46e96122dab7674", "https://www.xiaohongshu.com/user/profile/54e7413ea46e96122dab7674",
# "https://www.xiaohongshu.com/user/profile/5f3657900000000001002181", "https://www.xiaohongshu.com/user/profile/5f3657900000000001002181",
# "https://www.xiaohongshu.com/user/profile/5a65d6554eacab6864e2749e", "https://www.xiaohongshu.com/user/profile/5a65d6554eacab6864e2749e",
# "https://www.xiaohongshu.com/user/profile/5a745dc911be101d9ceab748", "https://www.xiaohongshu.com/user/profile/5a745dc911be101d9ceab748",
# "https://www.xiaohongshu.com/user/profile/59b2033550c4b45e5d43c3d9", "https://www.xiaohongshu.com/user/profile/59b2033550c4b45e5d43c3d9",
# "https://www.xiaohongshu.com/user/profile/59a97aaa5e87e760e012dcd0", "https://www.xiaohongshu.com/user/profile/59a97aaa5e87e760e012dcd0",
# "https://www.xiaohongshu.com/user/profile/5a5de03611be100219719b0f", "https://www.xiaohongshu.com/user/profile/5a5de03611be100219719b0f",
# "https://www.xiaohongshu.com/user/profile/5f40a5170000000001008577", "https://www.xiaohongshu.com/user/profile/5f40a5170000000001008577",
# "https://www.xiaohongshu.com/user/profile/597e82aa5e87e73c4915db81", "https://www.xiaohongshu.com/user/profile/597e82aa5e87e73c4915db81",
# "https://www.xiaohongshu.com/user/profile/580e0bc36a6a69043935369d", "https://www.xiaohongshu.com/user/profile/580e0bc36a6a69043935369d",
# "https://www.xiaohongshu.com/user/profile/5d1a17670000000012021d8e", "https://www.xiaohongshu.com/user/profile/5d1a17670000000012021d8e",
# "https://www.xiaohongshu.com/user/profile/59a830be82ec39155146f421", "https://www.xiaohongshu.com/user/profile/59a830be82ec39155146f421",
# "https://www.xiaohongshu.com/user/profile/55efc1b73397db0e969c8fbd", "https://www.xiaohongshu.com/user/profile/55efc1b73397db0e969c8fbd",
# "https://www.xiaohongshu.com/user/profile/5c8c55220000000010005810", "https://www.xiaohongshu.com/user/profile/5c8c55220000000010005810",
# "https://www.xiaohongshu.com/user/profile/5f337df2000000000101e2b2", "https://www.xiaohongshu.com/user/profile/5f337df2000000000101e2b2",
# "https://www.xiaohongshu.com/user/profile/5f2111500000000001009b7b", "https://www.xiaohongshu.com/user/profile/5f2111500000000001009b7b",
# "https://www.xiaohongshu.com/user/profile/59c840ff44363b497f335cd4", "https://www.xiaohongshu.com/user/profile/59c840ff44363b497f335cd4",
# "https://www.xiaohongshu.com/user/profile/5f8e8508000000000101d70e", "https://www.xiaohongshu.com/user/profile/5f8e8508000000000101d70e",
# "https://www.xiaohongshu.com/user/profile/5a163e3511be10234e1abffd", "https://www.xiaohongshu.com/user/profile/5a163e3511be10234e1abffd",
# "https://www.xiaohongshu.com/user/profile/5e71f6870000000001005e52", "https://www.xiaohongshu.com/user/profile/5e71f6870000000001005e52",
'https://www.xiaohongshu.com/user/profile/5cca9b3700000000120314c9', 'https://www.xiaohongshu.com/user/profile/5cca9b3700000000120314c9',
'https://www.xiaohongshu.com/user/profile/5aa0f7bae8ac2b65bfcdaf0e', 'https://www.xiaohongshu.com/user/profile/5aa0f7bae8ac2b65bfcdaf0e',
'https://www.xiaohongshu.com/user/profile/5c20dd200000000007027c07', 'https://www.xiaohongshu.com/user/profile/5c20dd200000000007027c07',
......
crawler_sys/utils/rpc_data_to_answer.py
View file @ 5634a341
...@@ -33,7 +33,7 @@ platfrom_id_dict = { ...@@ -33,7 +33,7 @@ platfrom_id_dict = {
data_type_dict = { data_type_dict = {
"cims/question/batch_create": ["platform","platform_id","title","content","user_id","create_time","is_online"], "cims/question/batch_create": ["platform","platform_id","title","content","user_id","create_time","is_online"],
"cims/answer/batch_create": ["platform","platform_id","platform_question_id","content","user_id","create_time","is_online"], "cims/answer/batch_create": ["platform","platform_id","platform_question_id","content","user_id","create_time","is_online",'level'],
"cims/reply/batch_create": ["platform","platform_id","platform_answer_id","content","user_id","create_time","is_online"] "cims/reply/batch_create": ["platform","platform_id","platform_answer_id","content","user_id","create_time","is_online"]
} }
dic_type = { dic_type = {
......
test/xiaohongshu_t.py
View file @ 5634a341
...@@ -5,6 +5,9 @@ ...@@ -5,6 +5,9 @@
# @author : litao # @author : litao
import hashlib import hashlib
import json
import redis
from crawler_sys.utils.output_results import retry_get_url from crawler_sys.utils.output_results import retry_get_url
...@@ -39,24 +42,37 @@ def cc(): ...@@ -39,24 +42,37 @@ def cc():
i2 += 1 i2 += 1
if __name__ == "__main__": if __name__ == "__main__":
url = "https://www.xiaohongshu.com/user/profile/5ea6909900000000010057a3" # url = "https://www.xiaohongshu.com/user/profile/5ea6909900000000010057a3"
headers = { # headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", # "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-encoding": "gzip, deflate", # "accept-encoding": "gzip, deflate",
"accept-language": "zh-CN,zh;q=0.9", # "accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache", # "cache-control": "no-cache",
"cookie": "xhsTracker=url=user-profile&xhsshare=CopyLink; xhsTrackerId=38ec6dcb-d882-45e8-c539-834386696a14; xhsuid=Dv4OTnGbDg9LivGu; timestamp2=202101062497d4bed842476b2618e0ea; timestamp2.sig=-Jax1vd_iNZtToaWYMOMoFUmCJwojKQPnfP8iMeOpAc; xhs_spses.5dde=*; extra_exp_ids=gif_clt1,ques_clt1; xhs_spid.5dde=59a50d47116c4333.1609921946.3.1610074315.1609997760.3be8232b-0407-44f8-8036-d40a4c47b120", # "cookie": "xhsTracker=url=user-profile&xhsshare=CopyLink; xhsTrackerId=38ec6dcb-d882-45e8-c539-834386696a14; xhsuid=Dv4OTnGbDg9LivGu; timestamp2=202101062497d4bed842476b2618e0ea; timestamp2.sig=-Jax1vd_iNZtToaWYMOMoFUmCJwojKQPnfP8iMeOpAc; xhs_spses.5dde=*; extra_exp_ids=gif_clt1,ques_clt1; xhs_spid.5dde=59a50d47116c4333.1609921946.3.1610074315.1609997760.3be8232b-0407-44f8-8036-d40a4c47b120",
"pragma": "no-cache", # "pragma": "no-cache",
"referer": "https://www.xiaohongshu.com/discovery/item/5f90384e000000000100b8f0", # "referer": "https://www.xiaohongshu.com/discovery/item/5f90384e000000000100b8f0",
"sec-ch-ua": '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"', # "sec-ch-ua": '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
"sec-ch-ua-mobile": "?0", # "sec-ch-ua-mobile": "?0",
"sec-fetch-dest": "document", # "sec-fetch-dest": "document",
"sec-fetch-mode": "navigate", # "sec-fetch-mode": "navigate",
"sec-fetch-site": "same-origin", # "sec-fetch-site": "same-origin",
"sec-fetch-user": "?1", # "sec-fetch-user": "?1",
"upgrade-insecure-requests": "1", # "upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", # "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
"Hm_lvt_900d393eff703909946efe28447affd3": "1596187047", # "Hm_lvt_900d393eff703909946efe28447affd3": "1596187047",
} # }
res = retry_get_url(url, headers=headers, proxies=0) # res = retry_get_url(url, headers=headers, proxies=0)
print(res.text) # print(res.text)
rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
pid_list = rds.hkeys("xiaohongshu")
for pid in pid_list:
res = rds.hget("xiaohongshu", pid)
res_json = json.loads(res)
if res_json["NoteView"].get("comments"):
# print(res_json["NoteView"].get("data"))
for comment in res_json["NoteView"]["comments"]["data"]:
print(comment)
elif res_json["NoteView"].get("commentInfo"):
for comment in res_json["NoteView"]["commentInfo"]["comments"]:
print(comment)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment