Merge branch 'haow/dev' into 'dev'

Haow/dev

See merge request !79

api/urls.py

@@ -4,6 +4,9 @@ from django.conf.urls import url
 from .views import user
 from .views import topic
 from .views import tag
+from .views import reply
+from .views import product
+
 
 urlpatterns = [
     # grasp
@@ -22,4 +25,11 @@ urlpatterns = [
     url(r'^v1/validate_3party_account$', user.Validate3PartyAccount.as_view(), name='validate_3party_account$'),
     url(r'^v1/user/batch_create_shadow_user$', user.BatchCreateShadowUser.as_view(), name='batch_create_shadow_user$'),
     url(r'^v1/user/batch_update_user_level_fans$', user.BatchUpdateUserLevelFansNum.as_view(), name='batch_update_user_level_fans$'),
+
+    # reply
+    url(r'^v1/reply/create_for_inner$', reply.CreateReplyForInner.as_view(), name='create_reply_for_inner'),
+
+    # product
+    url(r'^v1/product/batch_create$', product.ProductBatchCreate.as_view(), name='product_batch_create'),
+    url(r'^v1/brand/batch_create$', product.BrandBatchCreate.as_view(), name='brand_batch_create'),
 ]

api/views/product.py

+import json
+
+from api.views.base_view import BaseView
+
+from libs.user import get_user_by_ids
+from alpha_types.neptune import ERROR
+from alpha_types.neptune import PRODUCT_FLATFORM
+from engine.logger import info_logger, error_logger
+
+
+class ProductBatchCreate(BaseView):
+    '''批量创建商品 爬虫入口'''
+    def post(self, request):
+        product_info = json.loads(request.POST.get('data', '[]'))
+
+        if not product_info:
+            return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+
+        result = []
+
+        for info in product_info:
+            error, ret = self.process(info)
+            if error:
+                return self.error(error)
+            result.append(ret)
+
+        return self.ok(data=result)
+
+    def process(self, info):
+        if not info:
+            return None, None
+        brand_info = info.pop('brand', {})
+        composition_infos = info.pop('composition', [])
+        category_info = info.pop('category', {})
+        effect_info = info.pop('effect', {})
+
+        product_id = None
+        if info:
+            error, result = self.create_product(info)
+            if error:
+                return error, None
+            product_id = result.get('id')
+            if not product_id:
+                return error, None # 创建商品失败 未获取到商品ID
+        
+        if brand_info:
+            self.add_parallel_rpc_call_info('create_brand', 'neptune/commodity/brand/create',
+                                            cn_name=brand_info.get('cnName'), icon=brand_info.get('imgSrc'),
+                                            description=brand_info.get('description'), en_name=brand_info.get('en_name'),
+                                            origin_brand_id=brand_info.get('id'), platform=PRODUCT_FLATFORM.BEVOL, product_id=product_id)
+        if composition_infos:
+            self.add_parallel_rpc_call_info('batch_create_composition', 'neptune/commodity/composition/batch_create',
+                                            infos=composition_infos, platform=PRODUCT_FLATFORM.BEVOL, product_id=product_id)
+
+        if category_info:
+            self.add_parallel_rpc_call_info('create_category', 'neptune/commodity/category/create',
+                                            cn_name=category_info.get('name'), platform=PRODUCT_FLATFORM.BEVOL, 
+                                            origin_category_id=category_info.get('id'), product_id=product_id)
+        
+        if effect_info:
+            self.add_parallel_rpc_call_info('create_effect', 'neptune/commodity/effect/create',
+                                            cn_name=effect_info.get('name'), origin_effect_id=effect_info.get('id'),
+                                            platform=PRODUCT_FLATFORM.BEVOL, product_id=product_id)
+
+        self.shoot_rpc_calls_in_parallel()
+        
+        if brand_info:
+            error, _ = self.parallel_rpc_call_result['create_brand']
+            if error:
+                return error, None
+    
+        if composition_infos:
+            error, _ = self.parallel_rpc_call_result['batch_create_composition']
+            if error:
+                return error, None
+    
+        if category_info:
+            error, _ = self.parallel_rpc_call_result['create_category']
+            if error:
+                return error, None
+
+        if effect_info:
+            error, _ = self.parallel_rpc_call_result['create_effect']
+            if error:
+                return error, None
+
+        return None, result
+
+    def create_product(self, product_info):
+        error, ret = self.call_rpc('neptune/commodity/product/create', 
+                                   cn_name=product_info.get('cn_name'), en_name=product_info.get('en_name'), 
+                                   image=product_info.get('image', ''), norms=product_info.get('norms'),
+                                   grade=product_info.get('grade'), price=product_info.get('price'),
+                                   country=product_info.get('country'),
+                                   comment_nums=product_info.get('comment_nums'),
+                                   origin_product_id=product_info.get('id'), platform=PRODUCT_FLATFORM.BEVOL)
+        return error, ret
+
+
+class BrandBatchCreate(BaseView):
+    '''批量创建品牌'''
+    def post(self, request):
+        brand_infos = json.loads(request.POST.get('data', '[]'))
+        if not brand_infos:
+            return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+        infos = [{
+            'id': item.get('origin_brand_id'),
+            'cn_name': item.get('cn_name'),
+            'en_name': item.get('en_name'),
+            'icon': item.get('icon'),
+            'description': item.get('description')
+            } for item in brand_infos]
+        error, _ = self.call_rpc('neptune/commodity/brand/batch_create', 
+                                   infos=infos, platform=PRODUCT_FLATFORM.BEVOL)
+        if error:
+            return self.error(error)
+
+        return self.ok()

api/views/reply.py

+import json
+
+from api.views.base_view import BaseView
+
+
+class CreateReplyForInner(BaseView):
+    """
+    内部使用，创建评论回复
+    """
+    def post(self, request):
+        user_id = request.POST.get('user_id')
+        topic_id = request.POST.get('topic_id')
+        content = request.POST.get('content', '')
+        replied_id = request.POST.get('replied_id')
+
+        
+        error, data = self.call_rpc('venus/community/reply/for_inner_create', user_id=user_id, topic_id=topic_id, content=content, replied_id=replied_id)
+
+        if error:
+            return self.error(error)
+
+        return self.ok(data=data)

api/views/tag.py

@@ -15,11 +15,11 @@ class CreateTagForBatch(BaseView):
         tags = []
         if need_check:
             # 敏感词检测，获取可用的帖子
-            check_info = Sensitive.check([item["name"] for item in tag_list if item.get("name")])
+            check_info = Sensitive.check([item for item in tag_list if item])
             for tag in tag_list:
-                succ = check_info.get(tag.get("name")) if tag.get("name") else True
+                succ = check_info.get(tag) if tag else True
                 if not succ:
-                    tags.append(tag)
+                    tags.append({"name": tag})
         else:
             tags = tag_list
         error, data = self.call_rpc('venus/community/tag/batch_create_not_classify', data=tags)

api/views/topic.py

@@ -7,9 +7,10 @@ from api.cache.cache import ins_cache
 from libs.user import get_user_by_ids
 from alpha_types.venus import ERROR as CODES
 from alpha_types.venus import GRAP_PLATFORM
+from engine.logger import info_logger, error_logger
 
 
-ins_account_cache = "ins_account_cache"
+topic_id_cache = "ins_account_cache"
 
 
 class CreateTopicForBatch(BaseView):
@@ -17,6 +18,101 @@ class CreateTopicForBatch(BaseView):
     内部使用，批量建帖
     """
 
+    def batch_create_tags(self, tags, is_location=0, is_own=0):
+
+        info_logger.info({
+            'api': 'venus/community/tag/batch_create_tag_by_name',
+            'tags': tags,
+            'is_own': 1,
+        })
+        try:
+            _tag_error, _tag_data = self.call_rpc(
+                "venus/community/tag/batch_create_tag_by_name",
+                tags=tags, is_own=is_own, is_location=is_location,
+            )
+            if _tag_error:
+                error_logger.error({'api': 'venus/community/tag/batch_create_tag_by_name',
+                                    'error': _tag_error})
+                _tag_data = {}
+        except Exception as e:
+            error_logger.error({'api': 'venus/community/tag/batch_create_tag_by_name',
+                                'information': e})
+            _tag_data = {}
+        info_logger.info({
+            'api': 'venus/community/tag/batch_create_tag_by_name',
+            '_tag_data': _tag_data,
+        })
+        return _tag_error, _tag_data
+
+    def replace_tag_info(self, topic_list, tag_data):
+        # 处理标签，将文本中的标签处理成现有标签
+        for item in topic_list:
+            tags = item.get("tags") or []
+            tags = [tag.replace("#", '').strip() for tag in tags]
+
+            #添加地域标签
+            if item.get("location") and item.get("location").get("name"):
+                tags.append(item.get("location").get("name"))
+            content = item.get("content")
+            if not tag_data:
+                continue
+            for tag_name, tag_id  in tag_data.items():
+                if tag_name in tags:
+                    alpha_tag = '<topic>{' + '"id":{},"name":"{}"'.format(tag_id, tag_name) + '}</topic>'
+                    content = content.replace('#' + tag_name, alpha_tag)
+            item["content"] = content.replace('#', '')
+
+            item["tag_ids"] = [
+                tag_data[tag_name]
+                for tag_name in tags if tag_data.get(tag_name)
+            ]
+        return topic_list
+
+    def check_exist_ids(self, topics, platform):
+        # 帖子去重 redis层面
+        not_exists_ids = []
+        topic_list = []
+        if platform == GRAP_PLATFORM.INS:
+            cache_key = topic_id_cache
+        elif platform == GRAP_PLATFORM.PIN:
+            cache_key = topic_id_cache + ":2"
+        elif platform == GRAP_PLATFORM.FASHION:
+            cache_key = topic_id_cache + ":3"
+        else:
+            return
+
+        if platform in (GRAP_PLATFORM.INS, GRAP_PLATFORM.PIN, GRAP_PLATFORM.FASHION):
+            for item in topics:
+                _id = item.get("id")
+                if not _id:
+                    continue
+
+                exists = ins_cache.sismember(cache_key, _id)
+                if exists:
+                    continue
+
+                item["platform"] = platform
+                item["platform_id"] = _id
+                topic_list.append(item)
+                not_exists_ids.append(_id)
+
+        return not_exists_ids, topic_list, cache_key
+
+    def check_sensitive_content(self, topic_list):
+        # 敏感词检测，获取可用的帖子
+        topics = []
+    
+        check_info = Sensitive.check([topic["content"] for topic in topic_list if topic.get("content")])
+        for topic in topic_list:
+            if topic.get('content'):
+                succ = check_info.get(topic.get("content"))
+                if not succ:
+                    topics.append(topic)
+            else:
+                if topic.get('images') or topic.get('video'):
+                    topics.append(topic)
+        return topics
+
     def post(self, request):
 
         user_id = request.POST.get("user_id", 0)
@@ -25,6 +121,17 @@ class CreateTopicForBatch(BaseView):
         is_online = request.POST.get("is_online", 0)
         platform = int(request.POST.get("platform"))
         topic_list = json.loads(request.POST.get("topic_list", '[]'))
+        pictorial_tag_ids = json.loads(request.POST.get("tag_ids", '[]'))
+
+        info_logger.info({
+            'user_id': user_id,
+            'card_level': card_level,
+            'tag_id': tag_id,
+            'is_online': is_online,
+            'platform': platform,
+            'topic_list': topic_list,
+            'pictorial_tag_ids': pictorial_tag_ids,
+        })
 
         if not user_id:
             return self.parameter_invalid_response()
@@ -36,18 +143,12 @@ class CreateTopicForBatch(BaseView):
         if not topic_list:
             return self.ok()
 
-        topics = []
         # 敏感词检测，获取可用的帖子
-        # check_info = Sensitive.check([topic["content"] for topic in topic_list if topic.get("content")])
-        # for topic in topic_list:
-        #     if topic.get('content'):
-        #         succ = check_info.get(topic.get("content"))
-        #         if not succ:
-        #             topics.append(topic)
-        #     else:
-        #         if topic.get('images') or topic.get('video'):
-        #             topics.append(topic)
-        for topic in topic_list:
+        # checked_topics = self.check_sensitive_content(topic_list)
+        checked_topics = topic_list
+
+        topics = []
+        for topic in checked_topics:
             if topic.get('content'):
                 topics.append(topic)
             else:
@@ -57,6 +158,7 @@ class CreateTopicForBatch(BaseView):
         if not topics:
             return self.ok()
 
+        location_tags = []
         tag_names = []
         for item in topics:
             tags = item.get("tags") or []
@@ -65,53 +167,37 @@ class CreateTopicForBatch(BaseView):
             item["card_level"] = card_level
             item["tag_id"] = tag_id if tag_id else None
             item["is_online"] = is_online
+            item["pictorial_tag_ids"] = pictorial_tag_ids
+            if item.get("location") and item.get("location").get("name"):
+                _tag_error, _location_tag_data = self.batch_create_tags(tags=[item.get("location").get("name")], is_location=1, is_own=1)
+                if _location_tag_data:
+                    location_tags.append(_location_tag_data)
 
-        not_exists_ids = []
-        topic_list = []
-        if platform == GRAP_PLATFORM.INS:
-            for item in topics:
-                _id = item.get("id")
-                if not _id:
-                    continue
-
-                exists = ins_cache.sismember(ins_account_cache, _id)
-                if exists:
-                    continue
-
-                item["platform"] = platform
-                item["platform_id"] = _id
-                topic_list.append(item)
-                not_exists_ids.append(_id)
+        not_exists_ids, topic_list, cache_key = self.check_exist_ids(topics=topics, platform=platform)
 
         # check_info = Sensitive.check(tag_names)
         # tags  = [tag_name for tag_name, succ in check_info.items() if not succ]
-        tags  = tag_names
+        tags = tag_names
 
         # 先创建标签
-        _tag_error, _tag_data = self.call_rpc(
-            "venus/community/tag/batch_create_tag_by_name",
-            tag_names=tags, is_own=1
-        )
-        if _tag_error:
-            return self.error(_tag_error)
+        _tag_error, _tag_data = self.batch_create_tags(tags=tags, is_own=1)
+        if not _tag_data:
+            _tag_data = {}
+
+        # 地域标签
+        if location_tags:
+            info_logger.info({'location_tags': location_tags})
+            for item in location_tags:
+                _tag_data.update(item)
 
         # 更新发帖
         # 处理标签，将文本中的标签处理成现有标签
-        for item in topic_list:
-            tags = item.get("tags") or []
-            tags = [tag.replace("#", '').strip() for tag in tags]
-            content = item["content"]
-            for tag_name, tag_id  in _tag_data.items():
-                if tag_name in tags:
-                    alpha_tag = '<topic>{' + '"id":{},"name":"{}"'.format(tag_id, tag_name) + '}</topic>'
-                    content = content.replace('#' + tag_name, alpha_tag)
-            item["content"] = content.replace('#', '')
-
-            item["tag_ids"] = [
-                _tag_data[tag_name]
-                for tag_name in tags if _tag_data.get(tag_name)
-            ]
+        topic_list = self.replace_tag_info(topic_list=topic_list, tag_data=_tag_data)
 
+        info_logger.info({
+            'api': 'venus/community/topic/batch_create_for_inner',
+            'topic_list': topic_list,
+        })
         create_err, result = self.call_rpc(
             "venus/community/topic/batch_create_for_inner",
             topic_list=topic_list
@@ -121,7 +207,7 @@ class CreateTopicForBatch(BaseView):
 
         # 将已经跑了的数据添加到缓存
         if not_exists_ids:
-            ins_cache.sadd(ins_account_cache, *not_exists_ids)
+            ins_cache.sadd(cache_key, *not_exists_ids)
 
         return self.ok(data=result)
 
@@ -174,12 +260,11 @@ class CreateTopicForBatchByOne(BaseView):
         # check_info = Sensitive.check(tag_names)
         # tags  = [tag_name for tag_name, succ in check_info.items() if not succ]
         # check_info = Sensitive.check(tag_names)
-        tags  = tag_names
 
         # 先创建标签
         _tag_error, _tag_data = self.call_rpc(
             "venus/community/tag/batch_create_tag_by_name",
-            tag_names=tags
+            tags=tag_names
         )
         if _tag_error:
             return self.error(_tag_error)
@@ -189,7 +274,7 @@ class CreateTopicForBatchByOne(BaseView):
         for item in need_create_topics:
             tags = item.get("tags") or []
             tags = [tag.replace("#", '').strip() for tag in tags]
-            content = item["content"]
+            content = item.get("content")
             for tag_name, tag_id  in _tag_data.items():
                 if tag_name in tags:
                     alpha_tag = '<topic>{' + '"id":{},"name":"{}"'.format(tag_id, tag_name) + '}</topic>'