Merge branch 'feature/weibo' into 'test'

微博数据入库

See merge request alpha/saturn!101

api/management/commands/weibo_commands.py

@@ -15,7 +15,7 @@ from engine.logger import info_logger, error_logger, logging_exception
 
 
 IMAGE_SUFFIX = '-w'
-FILE_PATH = '/Users/zhongshangwu/workspace/gengmei/like/saturn/api/management/commands/'
+FILE_PATH = '/Users/zhongshangwu/workspace/gengmei/like/saturn/weibo/'
 
 # TODO
 # 1. 图片裁剪上传
@@ -72,10 +72,7 @@ class Command(BaseCommand):
     def get_json_data_from_dir(self, is_topic=None, is_pictorial=None):
         # 获取目录文件数据
         ret = []
-        if is_topic:
-            file_path = FILE_PATH + 'topic/'
-        if is_pictorial:
-            file_path = FILE_PATH + 'pictorial/'
+        file_path = FILE_PATH
         filenames = []
         for _, _, names in os.walk(file_path):
             filenames = names
@@ -141,7 +138,7 @@ class Command(BaseCommand):
             return comment, replies
 
         images = []
-        if not comment["images"]:
+        if not comment["images"] and len(reply) > 1:
 
             normal_images = set(reply[0]["images"])
             for info in reply[1:]:
@@ -213,7 +210,7 @@ class Command(BaseCommand):
 
             if not comment:
                 continue
-            if comment["images"]:  # -> to topic
+            if comment["images"] and len(self.stats[weibo_id]["topics"]) < 50:  # -> to topic
                 images = self.image_info(comment.pop('images'))
                 topic = {
                     # 'id': pictorial_id + str(idx),
@@ -244,7 +241,6 @@ class Command(BaseCommand):
 
             else:  # -> to pictorial comment
                 top_comments_obj = rpc_invoker['venus/community/crawl/replys'](data=[comment], platform=platform, pictorial_id=pictorial_id).unwrap()
-
                 if not top_comments_obj.get("reply_ids"):
                     self.top_pictorial_error_comments.append(comment)
                 else: