Merge branch 'haow/dev' into 'test'

add crawl api about xiaohongshu

See merge request alpha/saturn!83

api/urls.py

@@ -6,6 +6,7 @@ from .views import topic
 from .views import tag
 from .views import reply
 from .views import product
+from .views import pictorial
 
 
 urlpatterns = [
@@ -32,4 +33,7 @@ urlpatterns = [
     # product
     url(r'^v1/product/batch_create$', product.ProductBatchCreate.as_view(), name='product_batch_create'),
     url(r'^v1/brand/batch_create$', product.BrandBatchCreate.as_view(), name='brand_batch_create'),
+
+    # pictorial
+    url(r'^v1/pictorial/create$', pictorial.CreatePictorial.as_view(), name='pictorial_create'),
 ]

api/utils/upload.py

+import requests
+from gm_upload import upload, upload_file
+from gm_upload import IMG_TYPE
+
+
+def upload_image(url, img_type=IMG_TYPE.TOPIC):
+    '''非站内图片处理'''
+    try:
+        response = requests.get(url)
+        return upload(response.content, img_type=img_type)
+    except:
+        return None

api/views/pictorial.py

+import json
+import requests
+from random import randint
+from api.views.base_view import BaseView
+from api.utils.sensitive import Sensitive
+from api.utils.upload import upload_image
+from api.cache.cache import ins_cache
+
+from libs.user import get_user_by_ids
+from alpha_types.venus import ERROR
+from alpha_types.venus import GRAP_PLATFORM
+from engine.logger import info_logger, error_logger, logging_exception
+
+
+pictorial_id_cache = "pictorial_cache"
+IMAGE_SUFFIX = '-w'
+
+
+class CreatePictorial(BaseView):
+    """
+        画报爬取接口
+    """
+    user_id_list = [241407684, 241407653, 241407671, 241592897]
+    del_cache_keys = []
+
+    def del_cache(self):
+        for obj in self.del_cache_keys:
+            ins_cache.delete(obj)
+
+    def get_random_user_id(self):
+        index = randint(0, 3)
+        return self.user_id_list[index]
+
+    def get_user_id(self, id_, platform):
+        cache_key = 'grap:{}:{}'.format(platform, id_)
+        exist_key = 'grap:{}:{}'
+        value = ins_cache.get(cache_key)
+        user_id = None
+        if not value:
+            while True:
+                user_id = self.get_random_user_id()
+                exist = exist_key.format(platform, user_id)
+                if not ins_cache.get(exist):
+                    ins_cache.set(exist, id_)
+                    self.del_cache_keys.append(exist)
+                    break
+            ins_cache.set(cache_key, user_id)
+            self.del_cache_keys.append(exist)
+        else:
+            user_id = int(value)
+        return user_id
+
+    
+    def get_image_size(self, image_url):
+        # 获取图片宽高
+        try:
+            url = image_url + IMAGE_SUFFIX + '?imageInfo'
+            response = requests.request("GET", url)
+            info = response.json()
+            return info.get('width'), info.get('height')
+        except Exception as e:
+            logging_exception()
+            return None
+
+    def image_info(self, urls):
+        ret = []
+        for url in urls:
+            image_url = upload_image(url)
+            while not image_url:
+                image_url = upload_image(url)
+            width, height = self.get_image_size(image_url)
+            while not width and not height:
+                width, height = self.get_image_size(image_url)
+            ret.append(
+                {
+                    'url': image_url.replace('http://alpha.gmeiapp.com/', ''),
+                    'height': height,
+                    'width': width,
+                }
+            )
+        return ret
+
+    def revise_comments(self, comments, from_id):
+        ret = []
+        for obj in comments:
+            obj['from_id'] = from_id
+            reply = obj.pop('reply', None)
+            if not reply:
+                ret.append(obj)
+                continue
+            for info in reply:
+                ret.append(obj)
+                info['from_id'] = obj.get('from_id')
+                info['reply_id'] = obj.get('id')
+                info['type'] = obj.get('type')
+            
+                ret.append(info)
+        return ret
+
+    def post(self, request):
+    
+        pictorial = json.loads(request.POST.get('pictorial', '{}'))
+        topics = json.loads(request.POST.get('topics', '[]'))
+        platform = int(request.POST.get('platform', GRAP_PLATFORM.XIAOHONGSHU))
+
+        if not topics:
+            return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+        if pictorial:
+            if not pictorial.get('id'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not pictorial.get('name'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not pictorial.get('description'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not pictorial.get('create_time'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not pictorial.get('user'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not pictorial.get('user').get('id'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+        
+        for topic in topics:
+            if not topic.get('id'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not topic.get('content'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not topic.get('images'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not topic.get('create_time'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+            if not topic.get('user').get('id'):
+                return self.error(self.get_ErrorInfo(ERROR.PARAMS_INCOMPLETE))
+
+        pictorial_id = None
+        if pictorial:
+            pictorial_comments = pictorial.pop('comments', None)
+            pictorial['user_id'] = self.get_user_id(id_=pictorial.get('id'), platform=platform)
+            error, pictorial_obj = self.call_rpc('venus/community/crawl/pictorial', data=pictorial, platform=platform)
+            if error:
+                self.del_cache()
+                return self.error(error=error)
+            pictorial_id = pictorial_obj.get('id')
+
+            if pictorial_comments:
+                error, _ = self.call_rpc('venus/community/crawl/replys', data=pictorial_comments, platform=platform, pictorial_id=pictorial_id)
+                if error:
+                    self.del_cache()
+                    return self.error(error=error)
+
+        for topic in topics:
+            topic_comments = topic.pop('comments', None)
+            images = topic.pop('images')
+            topic['images'] = self.image_info(images)
+            topic['user_id'] = self.get_user_id(id_=topic.get('id'), platform=platform)
+            error, topic_obj = self.call_rpc('venus/community/crawl/topic', data=topic, platform=platform, pictorial_id=pictorial_id)
+            if error:
+                self.del_cache()
+                return self.error(error=error)
+            if not topic_comments:
+                continue
+            import ipdb
+            ipdb.set_trace()
+            from_id = topic.get('id')
+            for obj in topic_comments:
+                if platform == GRAP_PLATFORM.XIAOHONGSHU:
+                    topic_comments = self.revise_comments(topic_comments, from_id)
+                obj['user_id'] = self.get_user_id(id_=obj.get('id'), platform=platform)
+            error, _ = self.call_rpc('venus/community/crawl/replys', data=topic_comments, platform=platform, topic_id=topic_obj.get('id'))
+            if error:
+                self.del_cache()
+                return self.error(error=error)
+
+        self.del_cache()
+        return self.ok()