update

crawler_sys/scheduler/xiaohongshu_to_rpc.py

@@ -9,7 +9,7 @@ import random
 import time
 
 import redis, json
-# from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
+from crawler.crawler_sys.utils.rpc_data_to_answer import post_single_data,post_muilty_data
 from crawler_sys.utils.output_results import retry_get_url
 from crawler.gm_upload.gm_upload import upload, upload_file
 from crawler.crawler_sys.scheduler.redis_to_rpc.rpc_config import *

crawler_sys/site_crawler_by_redis/crawler_xiaohongshu.py

@@ -129,7 +129,7 @@ class Crawler_xiaohongshu():
     def releaser_page_by_xiaochengxu(self, releaserUrl,
                       output_to_file=False,
                       filepath=None,
-                      releaser_page_num_max=30,
+                      releaser_page_num_max=200,
                       output_to_es_raw=False,
                       es_index=None,
                       doc_type=None,
@@ -165,7 +165,8 @@ class Crawler_xiaohongshu():
 
             time.sleep(random.randint(1, 2))
             data_list = res.json()
-
+            if data_list["code"] != 0 or not data_list["success"]:
+                break
             if data_list:
                 print("get data at releaser: %s page: %s" % (releaser_id, count))
                 count += 1