更新小红书爬虫

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -5,6 +5,7 @@
 # @author : litao
 import copy
 
+import execjs
 import redis
 import requests
 import json
@@ -248,7 +249,16 @@ class Crawler_xiaohongshu():
                     # break
 
 if __name__ == '__main__':
-    test = Crawler_xiaohongshu()
+    # try:
+    #     with open(r'D:\work_file\gengmei\crawler\crawler_sys\site_crawler_by_redis\xiaohongshu_js.js', 'r', encoding='utf-8') as f:
+    #         js = f.read()
+    # except:
+    #     with open('/srv/apps/crawler/crawler_sys/site_crawler_by_redis/xiaohongshu.js', 'r', encoding='utf-8') as f:
+    #         js = f.read()
+    # # print(js)
+    # exec_js = execjs.compile(js)
+    # exec_js.call("get_sign", "https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae")
+    # test = Crawler_xiaohongshu()
     releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
     url_list =[
         "https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae",
@@ -358,4 +368,8 @@ if __name__ == '__main__':
     ]
     for url in url_list:
         print(url)
-        res = test.releaser_page(url,proxies_num=0)
+        try:
+            res = test.releaser_page(url,proxies_num=0)
+        except Exception as e:
+            print(e)
+            continue

crawler_sys/site_crawler_by_redis/xiaohongshu_js.js

+const crypto = require('crypto');
+
+/**
+ * 生成 x-sign header
+ * `x-sign: 'X' + md5(url + 'WSUDD')`
+ * @param {string} url url
+ * @param {object} params 参数
+ */
+function generateXSign(url, params = {}) {
+  const searchString = new URLSearchParams(params).toString();
+  const realUrl = `${url}${searchString ? '?' : ''}${searchString}WSUDD`;
+  const md5 = crypto.createHash('md5').update(realUrl).digest('hex');
+  return `X${md5}`;
+}
+
+module.exports = {
+  generateXSign,
+};
\ No newline at end of file