update

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -67,7 +67,11 @@ class Crawler_xiaohongshu():
         self.chrome_options.add_argument('--headless')
         self.chrome_options.add_argument('--disable-gpu')
         self.chrome_options.add_argument("--no-sandbox")
-        # self.chrome_options.add_argument('--proxy-server=http://{ip}:{port}'.format(ip='113.121.242.91', port=19319))
+        proxies_dic = get_proxy(1)
+        proxies_dic_list = proxies_dic["http"].split(":")
+        proxy_server = '--proxy-server=http://{ip}:{port}'.format(ip=proxies_dic_list[1].replace("/",""), port=int(proxies_dic_list[2]))
+        print(proxy_server)
+        self.chrome_options.add_argument(proxy_server)
         # self.chrome_options.add_argument('User-Agent="Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"')
         # self.chrome_options.add_argument('accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"')
         # self.chrome_options.add_argument('accept-encoding="gzip, deflate"')
@@ -237,4 +241,4 @@ class Crawler_xiaohongshu():
 if __name__ == '__main__':
     test = Crawler_xiaohongshu()
     releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
-    res = test.releaser_page(releaserurl,proxies_num=0)
+    res = test.releaser_page(releaserurl,proxies_num=1)