update

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -155,6 +155,13 @@ class Crawler_xiaohongshu():
         time.sleep(1)
         page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
         print(page_element)
+
+        self.driver.find_element_by_xpath(
+            "/html/body/div/div/div/div/div[2]/div[2]/div[1]/span/div[2]/h6/span").click()
+        self.driver.implicitly_wait(2)
+        time.sleep(1)
+        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
+        print(page_element)
         cookie = self.driver.get_cookies()
         #
         # return True
@@ -164,18 +171,22 @@ class Crawler_xiaohongshu():
             cookie_dic[k["name"]] = k["value"]
         print(cookie_dic)
         headers = {
-            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-            "accept-encoding": "gzip, deflate",
-            "accept-language": "zh-CN,zh;q=0.9",
-            "cache-control": "max-age=0",
-            # "cookie": "xhsTracker=url=user-profile&xhsshare=CopyLink; xhsTrackerId=38ec6dcb-d882-45e8-c539-834386696a14; xhsuid=Dv4OTnGbDg9LivGu; timestamp2=202101062497d4bed842476b2618e0ea; timestamp2.sig=-Jax1vd_iNZtToaWYMOMoFUmCJwojKQPnfP8iMeOpAc; extra_exp_ids=gif_clt1,ques_clt1; xhs_spses.5dde=*; xhs_spid.5dde=59a50d47116c4333.1609921946.2.1609997747.1609922354.16a344a9-3e6e-43d8-b894-33a7205e181f",
-            "referer": releaserUrl,
-            "sec-fetch-dest": "document",
-            "sec-fetch-mode": "navigate",
-            "sec-fetch-site": "same-origin",
-            "sec-fetch-user": "?1",
-            "upgrade-insecure-requests": "1",
-            "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
+"host":"www.xiaohongshu.com",
+"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+"accept-encoding": "gzip, deflate, br",
+"accept-language": "zh-CN,zh;q=0.9",
+"cache-control": "no-cache",
+# "cookie": "xhsTrackerId=a81077f9-661a-4731-c790-ac6fbbeaa44b; extra_exp_ids=gif_clt1,ques_exp2; xhsuid=2EFsw5qOMk70l1we; timestamp2=2021010899964852bd70ca4c0c991c6c; timestamp2.sig=Lj3xTHgJ-JO20IUULPRnAhACddlzUtd7AsUzrlJQbWc; xhs_spses.5dde=*; xhsTracker=url=index&searchengine=baidu; xhs_spid.5dde=4dc700089fbdde46.1610082780.1.1610083480.1610082780.d70776d0-eac9-4684-912e-130f0cdb86a1",
+"pragma": "no-cache",
+"sec-ch-ua": '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
+"sec-ch-ua-mobile": "?0",
+"sec-fetch-dest": "document",
+"sec-fetch-mode": "navigate",
+"sec-fetch-site": "none",
+"sec-fetch-user": "?1",
+"upgrade-insecure-requests": "1",
+"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
+
         }
 
         count = 1