Merge branch 'shangwu/hotfix-xiaohongshu' into 'master'

修复小红书数据入库阻塞问题

See merge request alpha/saturn!109

api/management/commands/reply_commands.py

@@ -49,7 +49,6 @@ class Command(BaseCommand):
             start_user_id=self.user_id_start,
             end_user_id=self.user_id_start + 5000
         ).unwrap()
-        print(data, ">>>>>")
         self.shadow_user_ids = data
 
     def get_random_user_id(self):
@@ -68,19 +67,19 @@ class Command(BaseCommand):
     def get_user_id(self, id_, platform):
         # 获取用户ID 缓存记录保留用户关系
         cache_key = 'grap:{}:{}'.format(platform, id_)
-        exist_key = 'grap:{}:{}'
+        # exist_key = 'grap:{}:{}'
         value = ins_cache.get(cache_key)
-        user_id = None
+        # user_id = None
         if not value:
-            while True:
+            # while True:
                 user_id = self.get_random_user_id()
-                exist = exist_key.format(platform, user_id)
-                if not ins_cache.get(exist):
-                    ins_cache.set(exist, id_)
-                    self.del_cache_keys.append(exist)
-                    break
-            ins_cache.set(cache_key, user_id)
-            self.del_cache_keys.append(exist)
+            #     exist = exist_key.format(platform, user_id)
+            #     if not ins_cache.get(exist):
+            #         ins_cache.set(exist, id_)
+            #         self.del_cache_keys.append(exist)
+            #         break
+            # ins_cache.set(cache_key, user_id)
+            # self.del_cache_keys.append(exist)
         else:
             user_id = int(value)
         return user_id
@@ -117,7 +116,7 @@ class Command(BaseCommand):
         # 获取图片宽高
         try:
             url = image_url + IMAGE_SUFFIX + '?imageInfo'
-            response = requests.request("GET", url)
+            response = requests.request("GET", url, timeout=3)
             info = response.json()
             return info.get('width'), info.get('height')
         except Exception as e:
@@ -163,7 +162,7 @@ class Command(BaseCommand):
 
     def create_topic(self, topics, platform):
         count = 0
-        for topic in topics[:100]:
+        for topic in topics:
             count += 1
             topic_comments = topic.pop('comments', None)
             images = topic.pop('image')
@@ -279,7 +278,7 @@ class Command(BaseCommand):
         print('----- start deal pictorial at {} -----'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S %f')))
         pictorial_data = self.get_json_data_from_dir(is_pictorial=1)
         count = 0
-        for pictorial in pictorial_data[:50]:
+        for pictorial in pictorial_data:
             count += 1
             print('------- current pictorial count :', count)
             self.create_pictorial(pictorial=pictorial, platform=platform)

api/utils/upload.py

@@ -9,7 +9,7 @@ from PIL import Image
 def upload_image(url, img_type=IMG_TYPE.TOPIC):
     '''非站内图片处理'''
     try:
-        response = requests.get(url)
+        response = requests.get(url, timeout=3)
         return upload(response.content, img_type=img_type)
     except:
         return None