Merge branch 'test' into 'pictorial_topic_recomment'

# Conflicts:
#   libs/es.py
#   search/utils/group.py
#   search/views/group.py
#   trans2es/mapping/pictorial.json
#   trans2es/models/pictorial.py
#   trans2es/models/topic.py
#   trans2es/utils/pictorial_transfer.py

.idea/codeStyles/Project.xml

@@ -24,29 +24,5 @@
         <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
       </formatting-settings>
     </DBN-SQL>
-    <DBN-PSQL>
-      <case-options enabled="false">
-        <option name="KEYWORD_CASE" value="lower" />
-        <option name="FUNCTION_CASE" value="lower" />
-        <option name="PARAMETER_CASE" value="lower" />
-        <option name="DATATYPE_CASE" value="lower" />
-        <option name="OBJECT_CASE" value="preserve" />
-      </case-options>
-      <formatting-settings enabled="false" />
-    </DBN-PSQL>
-    <DBN-SQL>
-      <case-options enabled="false">
-        <option name="KEYWORD_CASE" value="lower" />
-        <option name="FUNCTION_CASE" value="lower" />
-        <option name="PARAMETER_CASE" value="lower" />
-        <option name="DATATYPE_CASE" value="lower" />
-        <option name="OBJECT_CASE" value="preserve" />
-      </case-options>
-      <formatting-settings enabled="false">
-        <option name="STATEMENT_SPACING" value="one_line" />
-        <option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
-        <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
-      </formatting-settings>
-    </DBN-SQL>
   </code_scheme>
 </component>
\ No newline at end of file

app_conf.xml

@@ -18,5 +18,6 @@
         <element value="injection.data_sync.tasks"/>
         <element value="search.views.contrast_similar"/>
         <element value="search.views.search_hotword"/>
+        <element value="search.views.product"/>
     </config>
 </gm_rpcd_config>

injection/data_sync/tasks.py

@@ -132,3 +132,26 @@ def get_tag_count():
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
+
+def del_pictorial_read_ten_redis():
+    try:
+        now = datetime.datetime.now().date()
+        yes_time = now - datetime.timedelta(days=10)
+        logging.info("get yes_time:%s" % yes_time)
+        device_id = "123456"
+        redis_key = "physical:home_recommend_pictorial" + ":device_id:" + str(device_id)
+        logging.info("get redis:%s" % redis_key)
+        redis_question_val_list = redis_client.hgetall(redis_key)
+
+        logging.info("get redis_question_val_list:%s" % redis_question_val_list)
+        for item in redis_question_val_list:
+            items = item.encode(encoding='utf-8')
+            if items < str(yes_time):
+
+                redis_client.hdel(redis_key, item)
+
+
+
+
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())

libs/es.py

@@ -415,7 +415,7 @@ class ESPerform(object):
                 },
                 "sort": [
                     {"_score": {"order": "desc"}},
-                    {"latest_reply_time":{"order": "desc"}},
+                    {"latest_reply_time":{"order":"desc"}},
                     {"create_time_val": {"order": "desc"}},
                     {"language_type": {"order": "asc"}},
                 ],

libs/tools.py

@@ -22,15 +22,13 @@ def tzlc(dt, truncate_to_sec=True):
         return timezone(settings.TIME_ZONE).normalize(dt)
 
 
-
-def get_have_read_topic_id_list(device_id,user_id,query_type):
+def get_have_read_topic_id_list(device_id, user_id, query_type):
     try:
-        if user_id and int(user_id)>0:
+        if user_id and int(user_id) > 0:
             redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
         else:
             redis_key = "physical:home_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
 
-
         have_read_topic_id_list = list()
         redis_field_list = [b'have_read_topic_list']
         redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)

linucb/views/collect_data.py

@@ -159,7 +159,7 @@ class CollectData(object):
 
                                 click_topic_tag_list = list()
                                 if "on_click_feed_topic_card" == raw_val_dict["type"]:
-                                    topic_id = raw_val_dict["params"]["business_id"] or raw_val_dict["params"]["topic_id"]
+                                    topic_id = raw_val_dict["params"]["topic_id"]
                                     device_id = raw_val_dict["device"]["device_id"]
                                     user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
 

physical/DBRouter.py

 # !/usr/bin/env python
 # -*- coding: utf-8 -*-
+DATABASE_APPS_MAPPING = {'face': 'face', 'commodity': 'commodity'}
 
 
 class DBRouter:
@@ -12,25 +13,36 @@ class DBRouter:
         """
         Attempts to read user models go to users_db.
         """
-        if model._meta.app_label == 'face':
-            return 'face'
+        # if model._meta.app_label == 'face':
+        #     return 'face'
+
+        if model._meta.app_label in DATABASE_APPS_MAPPING:
+            return DATABASE_APPS_MAPPING[model._meta.app_label]
         return None
 
     def db_for_write(self, model, **hints):
         """
         Attempts to write user models go to users_db.
         """
-        if model._meta.app_label == 'face':
-            return 'face'
+        # if model._meta.app_label == 'face':
+        #     return 'face'
+
+        if model._meta.app_label in DATABASE_APPS_MAPPING:
+            return DATABASE_APPS_MAPPING[model._meta.app_label]
         return None
 
     def allow_relation(self, obj1, obj2, **hints):
         """
         Allow relations if a model in the user app is involved.
-        """
-        if obj1._meta.app_label == 'face' or \
-                obj2._meta.app_label == 'face':
+        # """
+        db_obj1 = DATABASE_APPS_MAPPING.get(obj1._meta.app_label)
+        db_obj2 = DATABASE_APPS_MAPPING.get(obj2._meta.app_label)
+        if db_obj1 and db_obj2:
+            if db_obj1 == db_obj2:
                 return True
+            else:
+                return False
+        else:
             return None
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
@@ -38,6 +50,9 @@ class DBRouter:
         Make sure the auth app only appears in the 'users_db'
         database.
         """
-        if app_label == 'face':
-            return db == 'face'
+        if db in DATABASE_APPS_MAPPING.values():
+            return DATABASE_APPS_MAPPING.get(app_label) == db
+        elif app_label in DATABASE_APPS_MAPPING:
+            return False
+
         return None

search/utils/group.py

@@ -93,7 +93,8 @@ class GroupUtils(object):
                                     {"term": {"effective": True}}
                                 ],
                                 "must_not": [
-                                    {"term": {"is_default": 1}}
+                                    {"term": {"is_default": 1}},
+                                    # {"terms": {"id": have_read_pictorial_id_list}}
                                 ]
                             }
                         },

search/utils/product.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from libs.cache import redis_client
+from libs.es import ESPerform
+from search.utils.group import GroupUtils
+from search.utils.common import GroupSortTypes
+from libs.es import ESPerform
+from trans2es.models.pictorial import PictorialTopics
+
+
+class ProductUtils(object):
+
+    @classmethod
+    def get_product_sku(cls, query='', offset=0, size=10, filters={}):
+        try:
+            multi_fields = {
+                'cn_name': 2,
+                'en_name': 2,
+                'alias': 2,
+                'brand_cn_name': 2,
+                'brand_en_name': 2,
+                'brand_alias': 2,
+                'category_cn_name': 2,
+                "effect_cn_name": 2
+            }
+            query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
+            multi_match = {
+                'query': query,
+                'type': 'cross_fields',
+                'operator': 'and',
+                'fields': query_fields,
+            }
+
+            sku_must_flter = cls.sku_filter(filters)
+            logging.info("get  sku_must_flter:%s " % sku_must_flter)
+            logging.info("get  sku_must_flter:%s " % type(sku_must_flter))
+
+            q = {
+                "query": {
+                    "bool": {
+                        "must": sku_must_flter
+                    }
+                }
+            }
+
+            if query != '':
+                q = {
+                    "query": {
+                        "bool": {
+                            "must": sku_must_flter,
+                            "should": {
+                                "multi_match": multi_match
+                            },
+                            "minimum_should_match": 1
+                        }
+                    }
+                }
+
+            q["sort"] = [{"comment_nums": {"order": "desc"}}, {"cn_name_sort": {"order": "asc"}}]
+
+            logging.info("get product query:%s" % q)
+
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="product", query_body=q,
+                                                       offset=offset, size=size)
+
+            return result_dict
+        except:
+            logging.error("catch exception, query_sku:%s" % traceback.format_exc())
+            return []
+
+    @classmethod
+    def sku_filter(cls, filters):
+
+        """处理过滤器部分。"""
+        logging.info("get filters:%s" % filters)
+
+        f = [
+            {'term': {"have_image": True}},
+            {'term': {"is_online": True}},
+            {"term": {"is_deleted": False}},
+        ]
+
+        if not filters:
+            return f
+
+        for k, v in filters.items():
+
+            if v in (None, '', []):
+                continue
+
+            if k == "brand":
+                f.append({"term": {"brand_cn_name_pre": v}})
+
+            if k == "effect":
+                f.append({"term": {"effect_cn_name_pre": v}})
+
+            if k == "category":
+                f.append({"term": {"category_cn_name_pre": v}})
+
+        return f

search/utils/topic.py

@@ -1124,6 +1124,7 @@ class TopicUtils(object):
                     if sorts:
                         q["sort"] = sorts
 
+            logging.info("get --------qqqq:%s" % q)
             result_dict = ESPerform.get_search_results(
                 ESPerform.get_cli(), sub_index_name=index_name,
                 query_body=q, offset=offset, size=size

search/views/group.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from gm_rpcd.all import bind
 import logging
 import traceback
 import json
+import time
+import datetime
+from libs.cache import redis_client
+from gm_rpcd.all import bind
 from libs.cache import redis_client
 from libs.es import ESPerform
 from search.utils.group import GroupUtils
 from search.utils.common import GroupSortTypes
 from libs.es import ESPerform
 from trans2es.models.pictorial import PictorialTopics
+from trans2es.models.pictorial import CommunityPictorialActivity
+from alpha_types.venus import PICTORIAL_ACTIVITY_SORT
 
 
 @bind("physical/search/query_pictorial")
@@ -512,6 +517,108 @@ def search_physical(query="", offset=0, size=10):
         return {"search_pictorial_ids": []}
 
 
+@bind("physical/search/pictorial_activity")
+def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
+    """
+    榜单活动的排名和最新tab下的排序
+    排名根据该榜单下的帖子的总票数倒排
+    最新根据榜单的创建时间排序
+    过滤掉下线的无封面的帖子
+
+    :param activity_id:
+    :param offset:
+    :param size:
+    :param sort_type:
+    :return:
+    """
+    try:
+        now = datetime.datetime.now()
+        activity_status = CommunityPictorialActivity.objects.filter(id=activity_id).values_list("end_time",
+                                                                                                flat=True).first()
+
+        now = int(time.mktime(now.timetuple()))
+        activity_status = int(time.mktime(activity_status.timetuple()))
+
+        logging.info("get now:%s" % now)
+        logging.info("get activity_status:%s" % activity_status)
+
+        pictorial_ids_list = []
+        if sort_type == 1:
+            redis_key = "pictorial:activity:vote:id:" + str(activity_id)
+        else:
+            redis_key = "pictorial:activity:create:id:" + str(activity_id)
+
+        if now < activity_status:
+            q = {
+                "query": {
+                    "bool": {
+                        "must": [
+                            {
+                                "term": {
+                                    "activity_join": activity_id
+                                }
+                            },
+                            {
+                                "term": {
+                                    "is_cover": True
+                                }
+                            },
+                            {
+                                "term": {
+                                    "is_online": True
+                                }
+                            }
+                        ]
+                    }
+                }
+            }
+            q["sort"] = process_sort(sort_type)
+            es_cli_obj = ESPerform.get_cli()
+            result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
+            result_dicts = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
+
+            if len(result_dict["hits"]) > 0:
+                for item in result_dict["hits"]:
+                    pictorial_id = item["_source"]["id"]
+                    vote_num = item["_source"]["topic_vote_number"]
+                    pictorial_ids_list.append({"pictorial_id": pictorial_id, "vote_num": vote_num})
+            redis_client.set(redis_key, json.dumps(pictorial_ids_list))
+
+        else:
+            redis_field_val_list = redis_client.get(redis_key)
+            pictorial_ids_list_all = json.loads(str(redis_field_val_list, encoding="utf-8"))
+            pictorial_ids_list = pictorial_ids_list_all[offset:offset + size]
+
+        return {"pictorial_activity_sort": pictorial_ids_list}
+
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"pictorial_activity_sort": []}
+
+
+def process_sort(sorts_by):
+    """处理排序部分。"""
+
+    sort_rule = []
+    if isinstance(sorts_by, int):
+        if sorts_by == PICTORIAL_ACTIVITY_SORT.VOTE_NUM:
+            sort_rule.append({
+                "topic_vote_number": {
+                    "order": "desc"
+                }
+            })
+        if sorts_by == PICTORIAL_ACTIVITY_SORT.CREATE_TIME:
+            sort_rule.append({
+                "create_time": {
+                    "order": "desc"
+                }
+            })
+
+    logging.info("get picotirial:%s" % sort_rule)
+
+    return sort_rule
+
+
 @bind("physical/search/pictorialid_topic")
 def pictorial_topic_sort_peoplehot(pictorial_id=-1, user_id=-1, offset=0, size=10):
     try:
@@ -578,3 +685,63 @@ def pictorial_topic_sort_peoplehot(pictorial_id=-1, user_id=-1, offset=0, size=1
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"pict_pictorial_ids_list": []}
+
+
+@bind("physical/search/pictorial_activity_sort")
+def get_pictorial_activeity_sort(activity_id=-1, pictorial_id=-1):
+    try:
+        pictorial_activity_sort_only = []
+        """
+        判断在不在活动中
+        """
+        q = {
+            "query": {
+                "bool": {
+                    "must": [
+                        {
+                            "term": {
+                                "activity_join": activity_id
+                            }
+                        },
+                        {
+                            "term": {
+                                "id": pictorial_id
+                            }
+                        },
+                        {
+                            "term": {
+                                "is_cover": True
+                            }
+                        },
+                        {
+                            "term": {
+                                "is_online": True
+                            }
+                        }
+                    ]
+                }
+            }
+        }
+
+        result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "pictorial", q, 0, 1)
+        if len(result_dict["hits"]) == 0:
+            pictorial_activity_sort_only.append({"pictorial_id": pictorial_id, "pictorial_sort": 0})
+            return {"pictorial_activity_sort": pictorial_activity_sort_only}
+
+        pictorial_ids_list = pictorial_activity_sort(activity_id=activity_id, size=999, offset=0, sort_type=1)
+        data = pictorial_ids_list.get("pictorial_activity_sort", [])
+        if data:
+            pictorial_ids = []
+            for item in data:
+                pictorial_ids.append(item["pictorial_id"])
+            if pictorial_id in pictorial_ids:
+                pictorial_sort = pictorial_ids.index(pictorial_id)
+                pictorial_activity_sort_only.append(
+                    {"pictorial_id": pictorial_id, "pictorial_sort": pictorial_sort + 1})
+            else:
+                pictorial_activity_sort_only.append({"pictorial_id": pictorial_id, "pictorial_sort": 1000})
+        return {"pictorial_activity_sort": pictorial_activity_sort_only}
+
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"pictorial_activity_sort": []}

search/views/product.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from libs.cache import redis_client
+from libs.es import ESPerform
+from search.utils.product import ProductUtils
+from search.utils.common import GroupSortTypes
+from libs.es import ESPerform
+
+
+@bind("physical/search/query_product_sku")
+def product_hot_sort(query='', offset=0, size=10, filters={}):
+    '''
+    商品sku排序
+    :param query:
+    :param offset:
+    :param size:
+    :param sort_type:
+    :param filters:
+    :return:
+    '''
+    try:
+        res = ProductUtils.get_product_sku(query=query, offset=offset, size=size, filters=filters)
+
+        product_list = []
+        res_hit = res["hits"]
+
+        for item in res_hit:
+            product_id = item["_source"]["id"]
+            product_list.append(product_id)
+
+        return {"product_hot_ids": product_list}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"product_hot_ids": []}

search/views/search_hotword.py

@@ -41,7 +41,7 @@ def search_hotword(device_id=-1):
         for item in results_registr_tag:
             for word in item.split():
                 tag_val_list.add(word)
-        tag_id_list = random.sample(range(0, len(tag_val_list)), 6)
+        tag_id_list = random.sample(range(0, len(tag_val_list)),1)
         for tag_id in tag_id_list:
             tag_val = list(tag_val_list)[tag_id]
             all_tag_name_list.add(tag_val)

trans2es/mapping/product.json

+{
+  "dynamic":"strict",
+   "_routing": {"required": false},
+  "properties": {
+    "id":{"type":"long"}, //id
+    "is_online":{"type":"boolean"},//上线
+    "create_time":{"type":"date", "format":"date_time_no_millis"},
+    "update_time":{"type":"date", "format":"date_time_no_millis"},
+    "create_time_val":{"type":"long"},//创建时间
+    "update_time_val":{"type":"long"},//更新时间
+    "is_deleted":{"type":"boolean"}, //是否被删除
+    "price":{"type":"double"}, //价格
+    "cn_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},  //商品名称
+    "en_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "alias":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "brand_cn_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "brand_en_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "brand_alias":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "category_cn_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "effect_cn_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+
+    "cn_name_pre":{"type": "text","analyzer":"keyword"},  //商品名称
+    "cn_name_sort":{"type": "text", "fielddata":"true"},  //商品名称
+    "en_name_pre":{"type": "text",  "analyzer":"keyword"},  //商品原名
+    "alias_pre":{"type": "text",  "analyzer":"keyword"},
+    "description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
+    "have_image":{"type":"boolean"},
+    "comment_nums":{"type":"long"},
+    "brand_cn_name_pre":{"type": "text", "analyzer":"keyword"}, //所属品牌名称
+    "brand_en_name_pre":{"type": "text", "analyzer":"keyword"}, //所属品牌原名
+    "brand_alias_pre":{"type": "text",  "analyzer":"keyword"}, //所属品牌别名
+    "category_cn_name_pre":{"type": "text", "analyzer":"keyword"}, //所属类目的名称
+    "effect_cn_name_pre":{"type": "text", "analyzer":"keyword"}//所属功效的名称
+  }
+}
+
+
+

trans2es/mapping/topic-high-star.json

@@ -19,6 +19,7 @@
     "tag_list":{"type":"long"},//标签属性
     "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
     "useful_tag_list":{"type":"long"},//有用标签属性
+    "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
     "edit_tag_list":{"type":"long"},//编辑标签
     "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
     "share_num":{"type":"long"},

trans2es/mapping/topic-star-routing.json

@@ -8,19 +8,19 @@
     "vote_num":{"type":"long"},
     "total_vote_num":{"type":"long"},
     "reply_num":{"type":"long"},
-    "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
-    "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
-    "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
+    "description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
+    "content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
-    "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
     "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "useful_tag_list":{"type":"long"},//有用标签属性
     "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
     "edit_tag_list":{"type":"long"},//编辑标签
-    "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
     "share_num":{"type":"long"},
     "pick_id_list":{"type":"long"},
     "offline_score":{"type":"double"},//离线算分
@@ -45,14 +45,14 @@
     "platform": {"type": "long"},
     "platform_id": {"type": "long"},
 
-    "drop_score":{"type": "double"},  // 人工降分
+    "drop_score":{"type": "long"},  // 人工降分
     "sort_score":{"type": "double"},  // 排序分
 
     "pictorial_id":{"type": "long"}, //所在组ID
     "pictorial_name":{  // 所在组名称
         "type": "text",
-        "analyzer": "gm_default_index",
-        "search_analyzer": "gm_default_index"
+        "analyzer": "keyword",
+        "search_analyzer": "keyword"
     },
     "is_excellent":{"type": "long"},
     "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐

trans2es/mapping/topic.json

@@ -8,19 +8,20 @@
     "vote_num":{"type":"long"},
     "total_vote_num":{"type":"long"},
     "reply_num":{"type":"long"},
-    "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
-    "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
-    "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
+    "description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
+    "content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
-    "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
     "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
     "useful_tag_list":{"type":"long"},//有用标签属性
+    "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
     "edit_tag_list":{"type":"long"},//编辑标签
-    "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
     "share_num":{"type":"long"},
     "pick_id_list":{"type":"long"},
     "offline_score":{"type":"double"},//离线算分
@@ -45,14 +46,14 @@
     "platform": {"type": "long"},
     "platform_id": {"type": "long"},
 
-    "drop_score":{"type": "double"},  // 人工降分
+    "drop_score":{"type": "long"},  // 人工降分
     "sort_score":{"type": "double"},  // 排序分
 
     "pictorial_id":{"type": "long"}, //所在组ID
     "pictorial_name":{  // 所在组名称
         "type": "text",
-        "analyzer": "gm_default_index",
-        "search_analyzer": "gm_default_index"
+        "analyzer": "keyword",
+        "search_analyzer": "keyword"
     },
     "is_excellent":{"type": "long"},
     "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐

trans2es/models/product.py

+import datetime
+from django.db import models
+import logging
+import traceback
+
+from .tag import Tag
+from .topic import Topic
+
+
+class CommodityProduct(models.Model):
+    """画报关注"""
+
+    class Meta:
+        verbose_name = u"商品"
+        app_label = "commodity"
+        db_table = "commodity_product"
+
+    id = models.IntegerField(verbose_name=u'商品ID', primary_key=True)
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    cn_name = models.CharField(verbose_name=u'商品名称', max_length=64)
+    en_name = models.CharField(verbose_name=u'商品原名', max_length=64, default="")
+    alias = models.CharField(verbose_name=u'别名', max_length=64)
+    image = models.CharField(verbose_name=u'图片', max_length=120)
+    description = models.CharField(verbose_name=u'商品描述', max_length=200)
+    comment_nums = models.IntegerField(verbose_name=u'评论数', max_length=11)
+    price = models.IntegerField(verbose_name="价格", max_length=11)
+
+    def get_brand_name(self):
+        try:
+            brand_id = CommodityProductBrand.objects.filter(product_id=self.id, is_deleted=False).values_list(
+                "brand_id", flat=True)
+            result_name = CommodityBrand.objects.filter(id=brand_id, is_online=True, is_deleted=False).values("cn_name",
+                                                                                                              "en_name",
+                                                                                                              "alias").first()
+            return result_name
+
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return []
+
+    def get_category_en_name(self):
+        try:
+            category_id = CommodityProductCategory.objects.filter(product_id=self.id, is_deleted=False).values_list(
+                "category_id", flat=True)
+            result_name = CommodityCategory.objects.filter(id=category_id, is_online=True,
+                                                           is_deleted=False).values_list("cn_name", flat=True)
+
+            return list(result_name)
+
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return ""
+
+    def get_effect_cn_name(self):
+        try:
+            effect_id = CommodityProductEffect.objects.filter(product_id=self.id, is_deleted=False).values_list(
+                "effect_id", flat=True)
+            result_name = CommodityEffect.objects.filter(id=effect_id, is_deleted=False).values_list("cn_name",
+                                                                                                     flat=True)
+
+            return list(result_name)
+
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return ""
+
+
+class CommodityBrand(models.Model):
+    """品牌"""
+
+    class Meta:
+        verbose_name = u'品牌'
+        app_label = 'commodity'
+        db_table = 'commodity_brand'
+
+    id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
+    en_name = models.CharField(verbose_name=u'品牌原名', max_length=64)
+    alias = models.CharField(verbose_name=u'别名', max_length=64)
+    description = models.CharField(verbose_name=u'品牌描述', max_length=200)
+
+
+class CommodityCategory(models.Model):
+    """类目"""
+
+    class Meta:
+        verbose_name = u'类目'
+        app_label = 'commodity'
+        db_table = 'commodity_category'
+
+    id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
+
+
+class CommodityEffect(models.Model):
+    """功效"""
+
+    class Meta:
+        verbose_name = u'功效'
+        app_label = 'commodity'
+        db_table = 'commodity_effect'
+
+    id = models.IntegerField(verbose_name=u'品牌ID', primary_key=True)
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    cn_name = models.CharField(verbose_name=u'品牌名称', max_length=64)
+
+
+class CommodityProductBrand(models.Model):
+    """商品品牌关系"""
+
+    class Meta:
+        verbose_name = u'商品品牌关系'
+        app_label = 'commodity'
+        db_table = 'commodity_productbrand'
+
+    id = models.IntegerField(verbose_name=u'ID', primary_key=True)
+    product_id = models.BigIntegerField(verbose_name=u'商品ID')
+    brand_id = models.BigIntegerField(verbose_name=u'品牌ID')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+
+
+class CommodityProductCategory(models.Model):
+    """画报关注"""
+
+    class Meta:
+        verbose_name = u"画报"
+        app_label = "commodity"
+        db_table = "commodity_productcategory"
+
+    id = models.IntegerField(verbose_name=u'关注ID', primary_key=True)
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    product_id = models.BigIntegerField(verbose_name=u'商品ID')
+    category_id = models.BigIntegerField(verbose_name=u'分类ID')
+
+
+class CommodityProductEffect(models.Model):
+    """画报关注标签"""
+
+    class Meta:
+        verbose_name = u"画报标签"
+        app_label = "commodity"
+        db_table = "commodity_producteffect"
+
+    id = models.IntegerField(verbose_name=u'关注ID', primary_key=True)
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    product_id = models.BigIntegerField(verbose_name=u'商品ID')
+    effect_id = models.BigIntegerField(verbose_name=u'功效ID')

trans2es/models/topic.py

@@ -6,12 +6,10 @@ from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 import traceback
 import logging
+import datetime
 from libs.es import ESPerform
 from django.db import models
-import datetime
-
 from alpha_types.venus import GRAP_PLATFORM
-
 from .pick_topic import PickTopic
 from .tag import TopicTag, Tag
 from .user_extra import UserExtra

trans2es/type_info.py

@@ -14,7 +14,7 @@ import sys
 from libs.cache import redis_client
 import copy
 
-from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar,pictorial
+from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial, product
 from trans2es.utils.user_transfer import UserTransfer
 from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
 from trans2es.utils.group_transfer import GroupTransfer
@@ -24,7 +24,7 @@ from trans2es.utils.pictorial_transfer import PictorialTransfer
 from trans2es.utils.celebrity_transfer import CelebrityTransfer
 from trans2es.utils.tag_transfer import TagTransfer
 from trans2es.utils.contrast_similar_transfer import Contrast_Similar_Transfer
-
+from trans2es.utils.product_transfer import ProductTransfer
 __es = None
 
 
@@ -160,12 +160,11 @@ class TypeInfo(object):
                                     old_data["is_history"] = True
 
                                     data_list.append(old_data)
-                                    if int_ori_topic_star>=4:
+                                    if int_ori_topic_star >= 4:
                                         topic_data_high_star_list.append(old_data)
                                     redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
 
-
-                            if data["content_level"] and int(data["content_level"])>=4:
+                            if data["content_level"] and int(data["content_level"]) >= 4:
                                 topic_data_high_star_list.append(data)
                         elif self.type == "tag" or self.type == "tag_v1":
                             (res, begin_res) = data
@@ -174,7 +173,7 @@ class TypeInfo(object):
                         else:
                             data_list.append(data)
 
-        return (data_list,topic_data_high_star_list)
+        return (data_list, topic_data_high_star_list)
 
     def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
 
@@ -217,17 +216,17 @@ class TypeInfo(object):
         else:
             qs = self.model.objects.all()
         end = time.time()
-        time0=end-begin
+        time0 = end - begin
 
         begin = time.time()
         instance_list = qs.filter(pk__in=pk_list)
         end = time.time()
-        time1=end-begin
+        time1 = end - begin
 
         begin = time.time()
         data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
         end = time.time()
-        time2=end-begin
+        time2 = end - begin
 
         begin = time.time()
         # logging.info("get sub_index_name:%s"%sub_index_name)
@@ -239,7 +238,7 @@ class TypeInfo(object):
             es=es,
         )
 
-        if sub_index_name=="topic":
+        if sub_index_name == "topic":
             self.elasticsearch_bulk_insert_data(
                 sub_index_name="topic-star-routing",
                 data_list=data_list,
@@ -247,7 +246,7 @@ class TypeInfo(object):
             )
 
         # 同时写4星及以上的帖子
-        if len(topic_data_high_star_list)>0:
+        if len(topic_data_high_star_list) > 0:
             self.elasticsearch_bulk_insert_data(
                 sub_index_name="topic-high-star",
                 data_list=topic_data_high_star_list,
@@ -255,9 +254,8 @@ class TypeInfo(object):
             )
 
         end = time.time()
-        time3=end-begin
-        logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
-
+        time3 = end - begin
+        logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0, time1, time2, time3))
 
     def insert_table_chunk(self, sub_index_name, table_chunk, es=None):
         try:
@@ -279,7 +277,7 @@ class TypeInfo(object):
                 auto_create_index=True
             )
 
-            logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name,len(data_list)))
+            logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name, len(data_list)))
 
             stage_3_time = time.time()
             end_clock = time.clock()
@@ -317,8 +315,8 @@ def get_type_info_map():
             name='topic-star',
             type='topic-star',
             model=topic.Topic,
-            query_deferred=lambda: topic.Topic.objects.all().query,#假的
-            get_data_func=TopicTransfer.get_topic_data,#假的
+            query_deferred=lambda: topic.Topic.objects.all().query,  # 假的
+            get_data_func=TopicTransfer.get_topic_data,  # 假的
             bulk_insert_chunk_size=100,
             round_insert_chunk_size=5,
             round_insert_period=2,
@@ -467,7 +465,7 @@ def get_type_info_map():
             bulk_insert_chunk_size=100,
             round_insert_chunk_size=5,
             round_insert_period=2,
-        )
+        ),
         # TypeInfo(
         #     name="account_user_tag",  # 用户标签
         #     type="account_user_tag",
@@ -478,6 +476,16 @@ def get_type_info_map():
         #     round_insert_chunk_size=5,
         #     round_insert_period=2,
         # )
+        TypeInfo(
+            name="product",  # 商品
+            type="product",
+            model=product.CommodityProduct,
+            query_deferred=lambda: product.CommodityProduct.objects.all().query,
+            get_data_func=ProductTransfer.get_product_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        )
     ]
 
     type_info_map = {
@@ -487,4 +495,3 @@ def get_type_info_map():
 
     _get_type_info_map_result = type_info_map
     return type_info_map
-

trans2es/utils/product_transfer.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import sys
+import logging
+import traceback
+import time
+from libs.tools import tzlc
+from pypinyin import lazy_pinyin
+
+
+class ProductTransfer(object):
+
+    @classmethod
+    def get_product_data(cls, instance):
+        try:
+            res = dict()
+
+            res["id"] = instance.id
+            res["is_online"] = instance.is_online
+            res["is_deleted"] = instance.is_deleted
+            res["create_time"] = tzlc(instance.create_time)
+            res["update_time"] = tzlc(instance.update_time)
+            res["create_time_val"] = int(time.mktime(instance.create_time.timetuple()))
+            res["update_time_val"] = int(time.mktime(instance.update_time.timetuple()))
+            res["price"] = instance.price
+            res["cn_name_sort"] = ''
+            for i in lazy_pinyin(instance.cn_name):
+                res["cn_name_sort"] += str(i[0])
+            res["cn_name_pre"] = instance.cn_name
+            res["en_name_pre"] = instance.en_name
+            res["alias"] = instance.alias
+            res["cn_name"] = instance.cn_name
+            res["en_name"] = instance.en_name
+            res["alias_pre"] = instance.alias
+            res['description'] = instance.description
+            res["have_image"] = True if instance.image else False
+            res["comment_nums"] = instance.comment_nums
+            result_name = instance.get_brand_name()
+            if result_name:
+                res["brand_cn_name"] = result_name.get("cn_name", "")
+                res["brand_en_name"] = result_name.get("en_name", "")
+                res["brand_alias"] = result_name.get("alias", "")
+                res["brand_cn_name_pre"] = result_name.get("cn_name", "")
+                res["brand_en_name_pre"] = result_name.get("en_name", "")
+                res["brand_alias_pre"] = result_name.get("alias", "")
+            else:
+                res["brand_cn_name"] = ""
+                res["brand_en_name"] = ""
+                res["brand_alias"] = ""
+                res["brand_cn_name_pre"] = ""
+                res["brand_en_name_pre"] = ""
+                res["brand_alias_pre"] = ""
+            category_cn_name = instance.get_category_en_name()
+            if category_cn_name:
+                res["category_cn_name"] = category_cn_name
+                res["category_cn_name_pre"] = category_cn_name
+            else:
+                res["category_cn_name"] = []
+                res["category_cn_name_pre"] = []
+            effect_cn_name = instance.get_effect_cn_name()
+            if effect_cn_name:
+                res['effect_cn_name'] = effect_cn_name
+                res['effect_cn_name_pre'] = effect_cn_name
+            else:
+                res["effect_cn_name"] = []
+                res["effect_cn_name_pre"] = []
+            logging.info("get product:%s" % res)
+            return res
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return dict()