update

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -65,8 +65,10 @@ class Crawler_xiaohongshu():
         self.video_data['platform'] = self.platform
         self.chrome_options = webdriver.ChromeOptions()
         # self.chrome_options.add_argument('--headless')
-        # self.chrome_options.add_argument('--disable-gpu')
-        # self.chrome_options.add_argument("--no-sandbox")
+        self.chrome_options.add_argument('--disable-gpu')
+        self.chrome_options.add_argument("--no-sandbox")
+        self.chrome_options.add_argument( '--proxy-server=http://{ip}:{port}'.format(ip='171.80.186.177',
+                                                                          port=15455))
         self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
         self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
         self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
@@ -82,7 +84,7 @@ class Crawler_xiaohongshu():
         self.timestamp = str(datetime.datetime.now().timestamp() * 1e3)
         prefs = {"profile.managed_default_content_settings.images": 2}
         self.chrome_options.add_experimental_option("prefs", prefs)
-        # self.driver = webdriver.Chrome(options=self.chrome_options)
+        self.driver = webdriver.Chrome(options=self.chrome_options)
 
     def __exit__(self):
         self.driver.close()
@@ -137,24 +139,24 @@ class Crawler_xiaohongshu():
                       doc_type=None,
                       output_to_es_register=False,
                       push_to_redis=False, proxies_num=None, **kwargs):
-        # self.driver.get("https://www.xiaohongshu.com/")
-        # self.driver.implicitly_wait(2)
-        # time.sleep(1)
-        # self.driver.get(releaserUrl)
-        # self.driver.implicitly_wait(2)
-        # time.sleep(1)
-        #
-        #
-        # self.driver.refresh()
-        # page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
-        # print(page_element)
-        # cookie = self.driver.get_cookies()
+        self.driver.get("https://www.xiaohongshu.com/")
+        self.driver.implicitly_wait(2)
+        time.sleep(1)
+        self.driver.get(releaserUrl)
+        self.driver.implicitly_wait(2)
+        time.sleep(1)
 
+
+        # self.driver.refresh()
+        page_element = self.driver.find_element_by_xpath("//*").get_attribute("outerHTML")
+        print(page_element)
+        cookie = self.driver.get_cookies()
+        #
         # print(self.driver.get_log("performance"))
-        # cookie_dic={}
-        # for k in cookie:
-        #     cookie_dic[k["name"]] = k["value"]
-        # print(cookie_dic)
+        cookie_dic={}
+        for k in cookie:
+            cookie_dic[k["name"]] = k["value"]
+        print(cookie_dic)
         headers = {
             "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
             "accept-encoding": "gzip, deflate",