Merge branch 'test' into 'wsy2'

# Conflicts:
#   README.md

README.md

@@ -55,8 +55,10 @@ celery -A physical beat
 
 ### physical策略端脚本
 在Alp-Test-Cos-test001服务器上，通过crontab -l查看
+<<<<<<< README.md
 
 更新脚本
 ```bash
 make crontab
 ```
+

search/utils/topic.py

@@ -1339,7 +1339,19 @@ class TopicUtils(object):
                             }
                         }
                     })
+            elif k == "is_kol":
+                f.append({
+                    "term": {
+                        "user_is_kol": True
+                    }
+                })
 
+            elif k == "is_edit":
+                f.append({
+                    "term": {
+                        "user_is_edit": True
+                    }
+                })
             # elif k == "pictorial_id":
             #     f.append({
             #         "nested": {

search/views/tag.py

@@ -16,6 +16,7 @@ from trans2es.models.tag import CommunityTagSetRelation
 from django.conf import settings
 from libs.error import logging_exception
 from django.db import connection
+from trans2es.models.account_reg_extra import AccountRegExtra
 
 
 def get_highlight(fields=[]):
@@ -143,6 +144,27 @@ def choice_pictorial_push_tag(device_id, user_id):
         return {"pictorial_tag_list": []}
 
 
+@bind("physical/search/lintag_by_user_id")
+def get_lintags_by_user_id(user_id):
+    try:
+        devices = AccountRegExtra.objects.filter(user_id=user_id, is_online=True, is_deleted=False).values_list("device_id", flat=True)
+        if devices:
+            linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
+            device_id = devices[0]
+            redis_key = linucb_recommend_redis_prefix + str(device_id)
+            tag_data = redis_client.get(redis_key)
+            lintags = []
+            if tag_data is None:
+                lintags = []
+            else:
+                lintags = json.loads(str(tag_data, encoding="utf-8"))
+            return {"lin_tag_list": lintags[:3]}
+        return {"lin_tag_list": []}
+    except Exception as e:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"lin_tag_list": []}
+
+
 @bind("physical/search/choice_push_tag")
 def choice_push_tag(device_id, user_id):
     """

search/views/topic.py

@@ -55,7 +55,7 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
 
 def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposure_lin_topic_count=0, size=0, query=None,
                                  query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
-                                 usefulrecall=-1, useful_tag_list=[], has_score=False, gray_list=[]):
+                                 usefulrecall=-1, useful_tag_list=[], has_score=False, gray_list=[1]):
     try:
         topic_star_routing = "6"
         index_type = "topic-high-star"
@@ -276,7 +276,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposu
                 return [], [], []
 
 
-def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[]):
+def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[1]):
     try:
         if STRATEGY_TYPE.CTR_GRAY in gray_list:
             pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:" + str(device_id)
@@ -316,20 +316,22 @@ def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list
         return []
 
 
-def get_topic_and_pictorial_detail_for_debug(device_id="", linucb_topic_id_list=[], rank_topic_id_list=[],
-                                             linucb_pictorial_id_list=[], rank_topic_score=[]):
+def get_topic_and_pictorial_detail_for_debug(device_id="", underexposure_lin_topic_ids=[], linucb_topic_id_list=[],
+                                                 rank_topic_id_list=[], linucb_pictorial_id_list=[],
+                                                 rank_topic_score=[], gray_list=[1]):
     try:
-        linucb_tag_dict = get_device_linucb_tag(device_id, size=20, has_score=True)
+        linucb_tag_dict = get_device_linucb_tag(device_id, size=20, has_score=True, gray_list=gray_list)
         linucb_topic_results = dict()
         linucb_pictorial_results = dict()
         rank_topic_results = dict()
-        if len(linucb_topic_id_list) > 0:
-            linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list),
-                                                                           linucb_topic_list=linucb_topic_id_list)
-            linucb_topic_results = dict()
+        linucb_topic_id_list_all = linucb_topic_id_list + underexposure_lin_topic_ids
+        underexposure_lin_topic_results = dict()
+        exposure_lin_topic_results = dict()
+        if len(linucb_topic_id_list_all) > 0:
+            linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list_all),
+                                                                           linucb_topic_list=linucb_topic_id_list_all)
             if len(linucb_topic_dict) > 0:
-                # num = 1
-                for topic_id in linucb_topic_id_list:
+                for topic_id in linucb_topic_id_list_all:
                     topic_id = topic_id
                     edit_tag_List = linucb_topic_dict[topic_id]['edit_tag_list']
                     content_level = linucb_topic_dict[topic_id]['content_level']
@@ -345,27 +347,25 @@ def get_topic_and_pictorial_detail_for_debug(device_id="", linucb_topic_id_list=
                                 {"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
                     linucb_topic_results.update({str(topic_id): {"id": topic_id, "content_level": content_level,
                                                                  "edit_tags": edit_tag_name_score_list}})
-                logging.info("linucb_topic:{}".format(linucb_topic_results))
-                # num += 1
+                # logging.info("linucb_topic:{}".format(linucb_topic_results))
+                underexposure_lin_topic_results = {k: v for k, v in linucb_topic_results.items() if
+                                                   int(k) in underexposure_lin_topic_ids}
+                exposure_lin_topic_results = {k: v for k, v in linucb_topic_results.items() if
+                                              int(k) in linucb_topic_id_list}
         if len(rank_topic_id_list) > 0:
             rank_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(rank_topic_id_list),
                                                                          linucb_topic_list=rank_topic_id_list)
-            rank_topic_results = dict()
             if len(rank_topic_dict) > 0:
-                # num = 1
                 for i in range(len(rank_topic_id_list)):
                     topic_id = rank_topic_id_list[i]
                     score = rank_topic_score[i]
                     rank_topic_results.update({str(topic_id): {"id": topic_id, "rank": score}})
-                # num += 1
-                logging.info("rank_topic:{}".format(rank_topic_results))
+                # logging.info("rank_topic:{}".format(rank_topic_results))
         if len(linucb_pictorial_id_list) > 0:
             if len(linucb_pictorial_id_list) > 0:
                 linucb_pictorial_dict = TopicUtils.get_linucb_pictorial_info_for_debug(len(linucb_pictorial_id_list),
                                                                                        linucb_pictorial_list=linucb_pictorial_id_list)
-                linucb_pictorial_results = dict()
                 if len(linucb_pictorial_dict) > 0:
-                    # num = 1
                     for pictorial_id in linucb_pictorial_id_list:
                         pictorial_id = pictorial_id
                         edit_tag_List = linucb_pictorial_dict[pictorial_id]['edit_tag_list']
@@ -381,15 +381,15 @@ def get_topic_and_pictorial_detail_for_debug(device_id="", linucb_topic_id_list=
                                     {"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
                         linucb_pictorial_results.update({str(pictorial_id): {"id": pictorial_id,
                                                                              "edit_tags": edit_tag_name_score_list}})
-                    logging.info("linucb_pictorial:{}".format(linucb_pictorial_results))
-                    # num += 1
-        result = {"linucb_topic": linucb_topic_results, "rank_topic": rank_topic_results,
+                    # logging.info("linucb_pictorial:{}".format(linucb_pictorial_results))
+        result = {"underexposure_lin_topic": underexposure_lin_topic_results,
+                  "exposure_lin_topic": exposure_lin_topic_results, "rank_topic": rank_topic_results,
                   "linucb_pictorial": linucb_pictorial_results}
         logging.info("get_topic_and_pictorial_detail_for_debug:result:{}:device_id:{}".format(result, device_id))
         return result
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return {"linucb_topic": {}, "rank_topic": {}, "linucb_pictorial": {}}
+        return {"underexposure_lin_topic": {}, "exposure_lin_topic": {}, "rank_topic": {}, "linucb_pictorial": {}}
 
 
 @bind("physical/search/query_tag_id_by_topic")
@@ -403,7 +403,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
 
 @bind("physical/search/home_recommend")
 def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_count =4, lin_pictorial_count=4, size=10, query_type=TopicPageType.HOME_RECOMMEND,
-                   promote_topic_list=[], usefulrecall=-1, useful_tag_list=[], is_debug=False,gray_list=[]):
+                   promote_topic_list=[], usefulrecall=-1, useful_tag_list=[], is_debug=False,gray_list=[1]):
     """
     :remark:首页推荐，目前只推荐日记
     :param session_id:
@@ -477,8 +477,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
                             "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids}
                 else:
-                    results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
-                                                                       recommend_pictorial_ids, rank_topic_score)
+                    results = get_topic_and_pictorial_detail_for_debug(device_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids,
+                                                                       recommend_pictorial_ids, rank_topic_score, gray_list)
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
                             "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids,
                             "debug_model_data": results}
@@ -515,8 +515,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
                             "linucb_pictorial_ids": recommend_pictorial_ids}
                 else:
-                    results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
-                                                                       recommend_pictorial_ids, rank_topic_score)
+                    results = get_topic_and_pictorial_detail_for_debug(device_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids,
+                                                                       recommend_pictorial_ids, rank_topic_score, gray_list)
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
                             "linucb_pictorial_ids": recommend_pictorial_ids, "debug_model_data": results}
     except:
@@ -806,11 +806,11 @@ def clear_history(device_id=None, user_id=-1):
         msg = list()
         redis_key_list = list()
         if device_id:
-            redis_key_list.extend(('physical:linucb:device_id:%s', 'physical:home_recommend:device_id:%s:query_type:1',
-                                   'physical:linucb:tag_recommend:device_id:%s',
-                                   'physical:linucb:topic_recommend:device_id:%s',
-                                   'physical:linucb:pictorial_recommend:device_id:%s',
-                                   'physical:home_pictorial_recommend:device_id:%s:query_type:1'))
+            redis_key_list.extend(('physical:linucb:device_id:%s', 'ctr_physical:linucb:device_id:%s',
+                                   'physical:linucb:tag_recommend:device_id:%s', 'ctr_physical:linucb:tag_recommend:device_id:%s',
+                                   'physical:linucb:topic_recommend:device_id:%s', 'ctr_physical:linucb:topic_recommend:device_id:%s',
+                                   'physical:linucb:pictorial_recommend:device_id:%s', 'ctr_physical:linucb:pictorial_recommend:device_id:%s',
+                                   'physical:home_recommend:device_id:%s:query_type:1', 'physical:home_pictorial_recommend:device_id:%s:query_type:1'))
             for redis_key in redis_key_list:
                 key = redis_key % device_id
                 if redis_client.type(key) != "b'none'":
@@ -830,37 +830,13 @@ def clear_history(device_id=None, user_id=-1):
         return ['fail']
 
 
-@bind("physical/search/home_recommend_debug")
-def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids=[], rank_topic_ids=[],
-                         recommend_pictorial_ids=[]):
-    """
-    :remark:首页推荐，目前只推荐日记
-    :param session_id:
-    :param user_id:
-    :param offset:
-    :param size:
-    :return:
-    """
-    try:
-        if device_id:
-            results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
-                                                               recommend_pictorial_ids)
-            return results
-        else:
-            return {}
-    except:
-        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return {}
-
-
-def get_device_linucb_tag(device_id=None, size=20, has_score=False):
-    # {
-    #     "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
-    #                {"id": 6, "name": 名字}, ]
-    # }
+def get_device_linucb_tag(device_id=None, size=20, has_score=False, gray_list=[1]):
     try:
         user_feature = [1, 1]
         if device_id:
+            if STRATEGY_TYPE.CTR_GRAY in gray_list:
+                linucb_matrix_redis_prefix = "ctr_physical:linucb:device_id:"
+            else:
                 linucb_matrix_redis_prefix = "physical:linucb:device_id:"
             redis_key = linucb_matrix_redis_prefix + str(device_id)
             redis_linucb_tag_data_dict = redis_client.hgetall(redis_key)
@@ -875,7 +851,7 @@ def get_device_linucb_tag(device_id=None, size=20, has_score=False):
                 linucb_tag_dict_list = list()
                 linucb_tag_dict = dict()
                 part_recommend_tag_list = recommend_tag_list[0:size]
-                tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
+                tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
                     id__in=part_recommend_tag_list, is_online=True, is_deleted=False).values_list("id", "name")
                 if has_score:
                     for id, name in tag_query_results:
@@ -885,7 +861,6 @@ def get_device_linucb_tag(device_id=None, size=20, has_score=False):
                     for id, name in tag_query_results:
                         linucb_tag_dict_list.append({"id": id, "name": name})
                     return linucb_tag_dict_list
-
         else:
             return {"linucb": []}
     except:
@@ -894,7 +869,7 @@ def get_device_linucb_tag(device_id=None, size=20, has_score=False):
 
 def get_edit_tag_name(tag_lst):
     try:
-        tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
+        tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
             id__in=tag_lst, is_online=True, is_deleted=False).values_list("id", "name")
         tag_dict = dict()
         for id, name in tag_query_results:
@@ -905,14 +880,13 @@ def get_edit_tag_name(tag_lst):
 
 
 @bind("physical/search/device_linucb_tag_debug")
-def get_device_linucb_tag2(device_id=None, size=20, has_score=False):
-    # {
-    #     "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
-    #                {"id": 6, "name": 名字}, ]
-    # }
+def get_device_linucb_tag2(device_id=None, size=20, has_score=False, gray_list=[1]):
     try:
         user_feature = [1, 1]
         if device_id:
+            if STRATEGY_TYPE.CTR_GRAY in gray_list:
+                linucb_matrix_redis_prefix = "ctr_physical:linucb:device_id:"
+            else:
                 linucb_matrix_redis_prefix = "physical:linucb:device_id:"
             redis_key = linucb_matrix_redis_prefix + str(device_id)
             redis_linucb_tag_data_dict = redis_client.hgetall(redis_key)
@@ -927,7 +901,7 @@ def get_device_linucb_tag2(device_id=None, size=20, has_score=False):
                 linucb_tag_dict_list = list()
                 linucb_tag_dict = dict()
                 part_recommend_tag_list = recommend_tag_list[0:size]
-                tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
+                tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
                     id__in=part_recommend_tag_list, is_online=True, is_deleted=False).values_list("id", "name")
                 if has_score:
                     result_lst = []

trans2es/models/__init__.py

@@ -14,3 +14,5 @@ from .user import User
 from .group import Group
 from .pictorial import PictorialFollow
 from .pictorial import Pictorial
+
+from .account_reg_extra import AccountRegExtra

trans2es/models/account_reg_extra.py

+import datetime
+
+from django.db import models
+
+
+class AccountRegExtra(models.Model):
+    class Meta:
+        verbose_name = u"设备用户关系表"
+        db_table = "account_reg_extra"
+
+    id = models.IntegerField(verbose_name="主键ID", primary_key=True)
+    is_online = models.BooleanField(verbose_name=u"是否上线")
+    create_time = models.DateTimeField(verbose_name=u"创建时间", default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u"更新时间", default=datetime.datetime.fromtimestamp(0))
+    is_deleted = models.BooleanField(verbose_name=u"")
+    geo = models.CharField(verbose_name=u"", max_length=300)
+    model = models.CharField(verbose_name=u"", max_length=64)
+    device_id = models.CharField(verbose_name=u"设备ID", max_length=64)
+    share_code = models.CharField(verbose_name=u"", max_length=64)
+    user_id = models.IntegerField(verbose_name="用户ID")