update

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -67,17 +67,17 @@ class Crawler_xiaohongshu():
         self.chrome_options.add_argument('--headless')
         self.chrome_options.add_argument('--disable-gpu')
         self.chrome_options.add_argument("--no-sandbox")
-        self.chrome_options.add_argument('--proxy-server=http://{ip}:{port}'.format(ip='113.121.242.91', port=19319))
-        self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
-        self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
-        self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
-        self.chrome_options.add_argument('accept-language="zh-CN,zh;q=0.9"')
-        self.chrome_options.add_argument('cache-control="max-age=0"')
-        self.chrome_options.add_argument('referer="https://www.xiaohongshu.com/explore"')
-        self.chrome_options.add_argument('sec-fetch-dest="document"')
-        self.chrome_options.add_argument('sec-fetch-mode="navigate"')
-        self.chrome_options.add_argument('sec-fetch-user="?1"')
-        self.chrome_options.add_argument('upgrade-insecure-requests="1"')
+        # self.chrome_options.add_argument('--proxy-server=http://{ip}:{port}'.format(ip='113.121.242.91', port=19319))
+        # self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
+        # self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
+        # self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
+        # self.chrome_options.add_argument('accept-language="zh-CN,zh;q=0.9"')
+        # self.chrome_options.add_argument('cache-control="max-age=0"')
+        # self.chrome_options.add_argument('referer="https://www.xiaohongshu.com/explore"')
+        # self.chrome_options.add_argument('sec-fetch-dest="document"')
+        # self.chrome_options.add_argument('sec-fetch-mode="navigate"')
+        # self.chrome_options.add_argument('sec-fetch-user="?1"')
+        # self.chrome_options.add_argument('upgrade-insecure-requests="1"')
 
         # self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
         self.timestamp = str(datetime.datetime.now().timestamp() * 1e3)
@@ -142,6 +142,8 @@ class Crawler_xiaohongshu():
 
         time.sleep(1)
         self.driver.implicitly_wait(2)
+        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
+        print(page_element)
         self.driver.get(releaserUrl)
         self.driver.implicitly_wait(2)
         time.sleep(1)