update

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -151,6 +151,7 @@ class Crawler_xiaohongshu():
 
             # time.sleep(random.randint(1, 2))
             page_text = res.text
+            print(page_text)
             data_list = re.findall("window.__INITIAL_SSR_STATE__=(.*?)</script>", page_text)[0]
             data_json = json.loads(data_list)
             # # print(data_list)
@@ -161,7 +162,7 @@ class Crawler_xiaohongshu():
                     video_dic = {}
                     page_id = info_dic["id"]
                     title = info_dic["title"]
-                    desc = info_dic["desc"]
+                    desc = info_dic.get("desc","")
                     time_ts = datetime.datetime.strptime(info_dic["time"],'%Y-%m-%d %H:%M').timestamp()
                     if info_dic["type"] != "normal":
                         continue