Merge branch 'master' into gyz_es

libs/es.py

@@ -412,14 +412,14 @@ class ESPerform(object):
                     }
                 },
                 "_source": {
-                    "include": ["id", "user_id", "latest_reply_time"]
+                    "include": ["id", "user_id", "latest_reply_time", "topic_ctr_30", "like_rate_30"]
                 },
                 "sort": [
                     {
                         "_script": {
                             "order": "desc",
                             "script": {
-                                "inline": "doc['topic_ctr_30'].value+doc['like_rate_30'].value"
+                                "inline": "5*doc['topic_ctr_30'].value+doc['like_rate_30'].value"
                             },
                             "type": "number"
                         }
@@ -456,7 +456,7 @@ class ESPerform(object):
 
             logging.info("topic_id_list:%s" % str(topic_id_dict))
 
-            logging.info("linucb_tag_id_list_2_same_tagset_ids:" + str(tag_id))
+            logging.info("get_tag_topic_list_dict:gyz" + str(q) + str(result_dict))
             logging.info("linucb_tag_id_list_2_same_tagset_ids_2_topics_detail:" + str(topic_id_dict_latest_reply_time))
 
             return topic_id_list, topic_id_dict

linucb/views/collect_data.py

@@ -263,6 +263,7 @@ class CollectData(object):
                             if "type" in raw_val_dict and \
                                     (raw_val_dict["type"] in ("on_click_feed_topic_card","on_click_button")):
                                 click_topic_tag_list = list()
+                                device_id = ""
                                 if "on_click_feed_topic_card" == raw_val_dict["type"]:
                                     topic_id =  raw_val_dict["params"]["topic_id"]
                                     device_id = raw_val_dict["device"]["device_id"]

search/views/group.py

@@ -303,7 +303,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
 
 
 @bind("physical/search/pictorial_topic_sort")
-def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL_TOPIC_SORT.HOT):
+def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL_TOPIC_SORT.HOT, user_id=-1):
     """
     :remark 画报排序 人气 部分
     人气按照票数从大到小排序，相同票数按照图片票数更新时间由旧到新排序

search/views/topic.py

@@ -55,7 +55,7 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
 
 def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposure_lin_topic_count, size, query=None,
                                  query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
-                                 usefulrecall=-1, useful_tag_list=[], has_score= False,gray_list=[]):
+                                 usefulrecall=-1, useful_tag_list=[], has_score=False, gray_list=[]):
     try:
         topic_star_routing = "6"
         index_type = "topic-high-star"
@@ -278,7 +278,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposu
 def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[]):
     try:
         if STRATEGY_TYPE.CTR_GRAY in gray_list:
-            pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:"+str(device_id)
+            pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:" + str(device_id)
         else:
             pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id)
         have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id, user_id,
@@ -315,14 +315,16 @@ def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list
         return []
 
 
-def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list = [],rank_topic_id_list = [],linucb_pictorial_id_list = [],rank_topic_score = []):
+def get_topic_and_pictorial_detail_for_debug(device_id="", linucb_topic_id_list=[], rank_topic_id_list=[],
+                                             linucb_pictorial_id_list=[], rank_topic_score=[]):
     try:
-        linucb_tag_dict = get_device_linucb_tag(device_id,size=20,has_score=True)
+        linucb_tag_dict = get_device_linucb_tag(device_id, size=20, has_score=True)
         linucb_topic_results = dict()
         linucb_pictorial_results = dict()
         rank_topic_results = dict()
-        if len(linucb_topic_id_list) >0:
-            linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list),linucb_topic_list=linucb_topic_id_list)
+        if len(linucb_topic_id_list) > 0:
+            linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list),
+                                                                           linucb_topic_list=linucb_topic_id_list)
             linucb_topic_results = dict()
             if len(linucb_topic_dict) > 0:
                 # num = 1
@@ -338,11 +340,13 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
                                 {"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
                                  "name": linucb_tag_dict[edit_tag]["name"]})
                         else:
-                            edit_tag_name_score_list.append({"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
-                    linucb_topic_results.update({str(topic_id):{"id":topic_id,"content_level":content_level,"edit_tags":edit_tag_name_score_list}})
+                            edit_tag_name_score_list.append(
+                                {"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
+                    linucb_topic_results.update({str(topic_id): {"id": topic_id, "content_level": content_level,
+                                                                 "edit_tags": edit_tag_name_score_list}})
                 logging.info("linucb_topic:{}".format(linucb_topic_results))
                 # num += 1
-        if len(rank_topic_id_list) >0:
+        if len(rank_topic_id_list) > 0:
             rank_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(rank_topic_id_list),
                                                                          linucb_topic_list=rank_topic_id_list)
             rank_topic_results = dict()
@@ -351,10 +355,10 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
                 for i in range(len(rank_topic_id_list)):
                     topic_id = rank_topic_id_list[i]
                     score = rank_topic_score[i]
-                    rank_topic_results.update({str(topic_id): {"id": topic_id, "rank":score}})
+                    rank_topic_results.update({str(topic_id): {"id": topic_id, "rank": score}})
                 # num += 1
                 logging.info("rank_topic:{}".format(rank_topic_results))
-        if len(linucb_pictorial_id_list) >0:
+        if len(linucb_pictorial_id_list) > 0:
             if len(linucb_pictorial_id_list) > 0:
                 linucb_pictorial_dict = TopicUtils.get_linucb_pictorial_info_for_debug(len(linucb_pictorial_id_list),
                                                                                        linucb_pictorial_list=linucb_pictorial_id_list)
@@ -372,17 +376,19 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
                                     {"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
                                      "name": linucb_tag_dict[edit_tag]["name"]})
                             else:
-                                edit_tag_name_score_list.append({"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
+                                edit_tag_name_score_list.append(
+                                    {"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
                         linucb_pictorial_results.update({str(pictorial_id): {"id": pictorial_id,
                                                                              "edit_tags": edit_tag_name_score_list}})
                     logging.info("linucb_pictorial:{}".format(linucb_pictorial_results))
                     # num += 1
-        result = { "linucb_topic":linucb_topic_results,"rank_topic":rank_topic_results,"linucb_pictorial":linucb_pictorial_results}
+        result = {"linucb_topic": linucb_topic_results, "rank_topic": rank_topic_results,
+                  "linucb_pictorial": linucb_pictorial_results}
         logging.info("get_topic_and_pictorial_detail_for_debug:result:{}:device_id:{}".format(result, device_id))
         return result
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return { "linucb_topic":{},"rank_topic":{},"linucb_pictorial":{}}
+        return {"linucb_topic": {}, "rank_topic": {}, "linucb_pictorial": {}}
 
 
 @bind("physical/search/query_tag_id_by_topic")
@@ -437,7 +443,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
 
         else:
             if usefulrecall != -1 and len(useful_tag_list) > 0:
-                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count, gray_list=gray_list)
+                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
+                                                                           gray_list=gray_list)
                 size = size - len(recommend_pictorial_ids)
                 if is_debug:
                     has_score = True
@@ -449,7 +456,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                         query_type=query_type,
                         promote_topic_list=promote_topic_list,
                         usefulrecall=usefulrecall,
-                        useful_tag_list=useful_tag_list, has_score=has_score,gray_list=gray_list)
+                        useful_tag_list=useful_tag_list, has_score=has_score, gray_list=gray_list)
 
                 else:
                     has_score = False
@@ -463,7 +470,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                                                                                                          promote_topic_list=promote_topic_list,
                                                                                                          usefulrecall=usefulrecall,
                                                                                                          useful_tag_list=useful_tag_list,
-                                                                                                         has_score=has_score,gray_list=gray_list)
+                                                                                                         has_score=has_score,
+                                                                                                         gray_list=gray_list)
                 if not is_debug:
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
                             "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids}
@@ -475,7 +483,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                             "debug_model_data": results}
 
             else:
-                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count, gray_list=gray_list)
+                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
+                                                                           gray_list=gray_list)
                 size = size - len(recommend_pictorial_ids)
                 if is_debug:
                     has_score = True
@@ -487,7 +496,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                                                                                                          size=size,
                                                                                                          query_type=query_type,
                                                                                                          promote_topic_list=promote_topic_list,
-                                                                                                         has_score=has_score,gray_list=gray_list)
+                                                                                                         has_score=has_score,
+                                                                                                         gray_list=gray_list)
 
                 else:
                     has_score = False
@@ -497,7 +507,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
                                                                                        size=size,
                                                                                        query_type=query_type,
                                                                                        promote_topic_list=promote_topic_list,
-                                                                                       has_score=has_score,gray_list=gray_list)
+                                                                                       has_score=has_score,
+                                                                                       gray_list=gray_list)
 
                 if not is_debug:
                     return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
@@ -515,6 +526,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_c
             return {"underexposure_lin_topic_ids": [], "linucb_topic_ids": [], "rank_topic_ids": [], "linucb_pictorial_ids": [],"debug_model_data": {}}
 
 
+
 @bind("physical/search/discover_page")
 def discover_page(device_id="", user_id=-1, size=10):
     """
@@ -746,7 +758,7 @@ def query_topic_by_tag_aggregation(user_id, tag_id, offset, size):
 
 
 @bind("physical/search/topic")
-def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
+def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10, user_id=-1):
     """帖子搜索。"""
 
     try:
@@ -788,12 +800,12 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
 
 
 @bind("physical/search/clear_history_for_debug")
-def clear_history(device_id=None,user_id=-1):
+def clear_history(device_id=None, user_id=-1):
     try:
         msg = list()
         redis_key_list = list()
         if device_id:
-            redis_key_list.extend(('physical:linucb:device_id:%s','physical:home_recommend:device_id:%s:query_type:1',
+            redis_key_list.extend(('physical:linucb:device_id:%s', 'physical:home_recommend:device_id:%s:query_type:1',
                                    'physical:linucb:tag_recommend:device_id:%s',
                                    'physical:linucb:topic_recommend:device_id:%s',
                                    'physical:linucb:pictorial_recommend:device_id:%s',
@@ -804,7 +816,8 @@ def clear_history(device_id=None,user_id=-1):
                     redis_client.delete(key)
             msg.append('device_id_clear')
         if user_id != -1:
-            redis_user_key_list = ["physical:home_recommend:user_id:%s:query_type:1",'physical:home_pictorial_recommend:user_id:%s:query_type:1']
+            redis_user_key_list = ["physical:home_recommend:user_id:%s:query_type:1",
+                                   'physical:home_pictorial_recommend:user_id:%s:query_type:1']
             for redis_key in redis_user_key_list:
                 key = redis_key % user_id
                 if redis_client.type(key) != "b'none'":
@@ -817,7 +830,8 @@ def clear_history(device_id=None,user_id=-1):
 
 
 @bind("physical/search/home_recommend_debug")
-def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank_topic_ids = [],recommend_pictorial_ids =[]):
+def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids=[], rank_topic_ids=[],
+                         recommend_pictorial_ids=[]):
     """
     :remark:首页推荐，目前只推荐日记
     :param session_id:
@@ -828,7 +842,8 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank
     """
     try:
         if device_id:
-            results = get_topic_and_pictorial_detail_for_debug(device_id,recommend_topic_ids,rank_topic_ids,recommend_pictorial_ids)
+            results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
+                                                               recommend_pictorial_ids)
             return results
         else:
             return {}
@@ -837,13 +852,13 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank
         return {}
 
 
-def get_device_linucb_tag(device_id = None,size = 20,has_score = False):
+def get_device_linucb_tag(device_id=None, size=20, has_score=False):
     # {
     #     "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
     #                {"id": 6, "name": 名字}, ]
     # }
     try:
-        user_feature = [1,1]
+        user_feature = [1, 1]
         if device_id:
             linucb_matrix_redis_prefix = "physical:linucb:device_id:"
             redis_key = linucb_matrix_redis_prefix + str(device_id)
@@ -851,7 +866,10 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False):
             if len(redis_linucb_tag_data_dict) == 0:
                 return {"linucb": []}
             else:
-                (recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys()))
+                (recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
+                                                                                      redis_linucb_tag_data_dict,
+                                                                                      user_feature, list(
+                        redis_linucb_tag_data_dict.keys()))
                 recommend_tag_list = list(recommend_tag_dict.keys())
                 linucb_tag_dict_list = list()
                 linucb_tag_dict = dict()
@@ -868,7 +886,7 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False):
                     return linucb_tag_dict_list
 
         else:
-            return {"linucb":[]}
+            return {"linucb": []}
     except:
         return {"linucb": []}
 
@@ -886,13 +904,13 @@ def get_edit_tag_name(tag_lst):
 
 
 @bind("physical/search/device_linucb_tag_debug")
-def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
+def get_device_linucb_tag2(device_id=None, size=20, has_score=False):
     # {
     #     "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
     #                {"id": 6, "name": 名字}, ]
     # }
     try:
-        user_feature = [1,1]
+        user_feature = [1, 1]
         if device_id:
             linucb_matrix_redis_prefix = "physical:linucb:device_id:"
             redis_key = linucb_matrix_redis_prefix + str(device_id)
@@ -900,7 +918,10 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
             if len(redis_linucb_tag_data_dict) == 0:
                 return {"linucb": []}
             else:
-                (recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys()))
+                (recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
+                                                                                      redis_linucb_tag_data_dict,
+                                                                                      user_feature, list(
+                        redis_linucb_tag_data_dict.keys()))
                 recommend_tag_list = list(recommend_tag_dict.keys())
                 linucb_tag_dict_list = list()
                 linucb_tag_dict = dict()
@@ -911,7 +932,7 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
                     result_lst = []
                     for id, name in tag_query_results:
                         result_lst.append({"name": name, "score": recommend_tag_dict[str(id)]})
-                    linucb_tag_dict = {"linucb":result_lst}
+                    linucb_tag_dict = {"linucb": result_lst}
                     logging.info("physical/search/device_linucb_tag_debug:%s" % str(linucb_tag_dict))
                     return linucb_tag_dict
                 else:
@@ -920,6 +941,6 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
                     return linucb_tag_dict_list
 
         else:
-            return {"linucb":[]}
+            return {"linucb": []}
     except:
         return {"linucb": []}

trans2es/management/commands/redis_new_topic_sub.py

@@ -4,7 +4,7 @@ import datetime
 from libs.cache import redis_client
 from libs.error import logging_exception
 from django.conf import settings
-from trans2es.models.portrait_stat import LikeDeviceTagStat
+from trans2es.models.portrait_stat import LikeTopicStat
 
 try:
     ps = redis_client.pubsub()
@@ -28,9 +28,9 @@ try:
                 insert_list = []
                 for topic in insert_topic_ids:
                     insert_list.append(
-                        LikeDeviceTagStat(create_time=datetime.datetime.today(), update_time=datetime.datetime.today(),
+                        LikeTopicStat(create_time=datetime.datetime.today(), update_time=datetime.datetime.today(),
                                           topic_id=topic, is_new_topic=0, topic_ctr_30=0.0, like_rate_30=0.0))
-                LikeDeviceTagStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list)
+                LikeTopicStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list)
                 logging.info("impressions count gt 100 topic ids" + str(insert_topic_ids))
             json_all_new_topic_impression_count = json.dumps(all_new_topic_impression_count)
             logging.info("all_new_topic_impression_count" + str(all_new_topic_impression_count))

trans2es/management/commands/trans2es_data2es_parallel.py

@@ -28,10 +28,13 @@ from trans2es.models.topic import Topic,ActionSumAboutTopic
 from search.utils.common import *
 from linucb.views.collect_data import CollectData
 from injection.data_sync.tasks import sync_user_similar_score
+import datetime
 
 from trans2es.models.tag import Tag
 from libs.cache import redis_client
 from trans2es.models.tag import TopicTag
+from libs.error import logging_exception
+from trans2es.models.portrait_stat import LikeTopicStat
 
 
 
@@ -215,6 +218,42 @@ class Command(BaseCommand):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
 
+    def sub_redis_new_topic_ids(self):
+        try:
+            ps = redis_client.pubsub()
+            ps.subscribe("new_topic_impression")
+            all_new_topic_impression_count_key = "all_new_topic_impression_count_key"
+            for item in ps.listen():
+                if item['type'] == 'message':
+                    new_topic_ids = json.loads(item["data"])
+                    all_new_topic_impression_count = json.loads(redis_client.get(all_new_topic_impression_count_key))
+                    insert_topic_ids = []
+                    for topic in new_topic_ids:
+                        topic = str(topic)
+                        if topic in all_new_topic_impression_count:
+                            all_new_topic_impression_count[topic] = all_new_topic_impression_count[topic] + 1
+                            if all_new_topic_impression_count[topic] > 100:
+                                insert_topic_ids.append(int(topic))
+                                all_new_topic_impression_count.pop(topic)
+                        else:
+                            all_new_topic_impression_count[topic] = 1
+                    if insert_topic_ids:
+                        insert_list = []
+                        for topic in insert_topic_ids:
+                            insert_list.append(
+                                LikeTopicStat(create_time=datetime.datetime.today(),
+                                                  update_time=datetime.datetime.today(),
+                                                  topic_id=topic, is_new_topic=0, topic_ctr_30=0.0, like_rate_30=0.0))
+                        LikeTopicStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list)
+                        logging.info("impressions count gt 100 topic ids" + str(insert_topic_ids))
+                    json_all_new_topic_impression_count = json.dumps(all_new_topic_impression_count)
+                    logging.info("all_new_topic_impression_count" + str(all_new_topic_impression_count))
+                    redis_client.set(all_new_topic_impression_count_key, json_all_new_topic_impression_count)
+        except:
+            logging_exception()
+            logging.error("redis new topic sub error!")
+
+
     def handle(self, *args, **options):
         try:
             type_name_list = get_type_info_map().keys()
@@ -241,5 +280,8 @@ class Command(BaseCommand):
             if len(options["sync_type"]) and options["sync_type"]=="tagname":
                 self.sync_tag_collecction_name_set()
 
+            if len(options["sync_type"]) and options["sync_type"] == "new_topic_sub":
+                self.sub_redis_new_topic_ids()
+
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())