Merge branch 'linucb_pictorial' into 'master'

linucb_pictorial

See merge request alpha/physical!429

libs/es.py

@@ -447,6 +447,53 @@ class ESPerform(object):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return list()
 
+    @classmethod
+    def get_tag_pictorial_id_list(cls, tag_id, have_read_lin_pictorial_id_list, size=100):
+        try:
+            q = {
+                "query": {
+                    "function_score": {
+                        "query": {
+                            "bool": {
+                                "must": [
+                                    {"term": {"is_online": True}},
+                                    {"term": {"is_deleted": False}},
+                                    {"terms": {"edit_tag_id": tag_id}},
+                                    {"term": {"is_recommend": True}}
+                                ]
+                            }
+                        },
+                        "boost_mode": "sum",
+                        "score_mode": "sum",
+                    }
+                },
+                "_source": {
+                    "include": ["id"]
+                },
+                "sort": [
+                    {"real_user_activate_time": {"order": "desc"}},
+                    {"create_time": {"order": "desc"}},
+                ],
+            }
+
+            if len(have_read_lin_pictorial_id_list) > 0:
+                q["query"]["function_score"]["query"]["bool"]["must_not"] = {
+                    "terms": {
+                        "id": have_read_lin_pictorial_id_list
+                    }
+                }
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="pictorial",
+                                                       query_body=q,
+                                                       offset=0, size=size)
+
+            pictorial_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
+            # logging.info("topic_id_list:%s" % str(topic_id_list))
+            # topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
+            return pictorial_id_list
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return list()
+
     @classmethod
     def get_highlight(cls, fields=[]):
         field_highlight = {

libs/tools.py

@@ -39,3 +39,22 @@ def get_have_read_topic_id_list(device_id, user_id, query_type):
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return list()
+
+def get_have_read_lin_pictorial_id_list(device_id,user_id,query_type):
+    try:
+        if user_id and int(user_id) > 0:
+            redis_key = "physical:home_pictorial_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+        else:
+            redis_key = "physical:home_pictorial_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
+
+        have_read_lin_pictorial_id_list = list()
+        redis_field_list = ['have_read_pictorial_list']
+        redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
+        if redis_field_val_list[0]:
+            have_read_lin_pictorial_id_list = list(json.loads(redis_field_val_list[0]))
+
+        return have_read_lin_pictorial_id_list
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return list()
+

linucb/views/collect_data.py

@@ -14,6 +14,7 @@ from django.conf import settings
 from libs.es import ESPerform
 from search.utils.common import *
 import libs.tools as Tools
+from trans2es.models.pictorial import CommunityPictorialHomeFeed
 
 class KafkaManager(object):
     consumser_obj = None
@@ -36,6 +37,7 @@ class CollectData(object):
         self.linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
         #推荐帖子
         self.linucb_recommend_topic_id_prefix = "physical:linucb:topic_recommend:device_id:"
+        self.linucb_recommend_pictorial_id_prefix = "physical:linucb:pictorial_recommend:device_id:"
         self.tag_topic_id_redis_prefix = "physical:tag_id:topic_id_list:"
         self.click_recommend_redis_key_prefix = "physical:click_recommend:device_id:"
         # 默认
@@ -71,13 +73,19 @@ class CollectData(object):
                 redis_client.expire(tag_recommend_redis_key, 7*24*60*60)
 
                 have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,user_id,TopicPageType.HOME_RECOMMEND)
+                have_read_lin_pictorial_id_list = Tools.get_have_read_lin_pictorial_id_list(device_id, user_id,
+                                                                                            TopicPageType.HOME_RECOMMEND)
                 promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(is_online=1).values_list("topic_id",flat=True)
+                promote_lin_pictorial_id_list = CommunityPictorialHomeFeed.objects.using(settings.SLAVE_DB_NAME).filter(
+                    is_deleted=0, is_online=1).values_list("pictorial_id", flat=True)
                 have_read_topic_id_list.extend(promote_recommend_topic_id_list)
+                have_read_lin_pictorial_id_list.extend(promote_lin_pictorial_id_list)
 
                 recommend_topic_id_list = list()
                 recommend_topic_id_list_dict = dict()
                 recommend_topic_id_list_click = list()
                 recommend_topic_id_list_click_dict = dict()
+                recommend_lin_pictorial_id_list = list()
 
                 if click_topic_tag_list and len(click_topic_tag_list)>0:
                     recommend_topic_id_list_click,recommend_topic_id_list_click_dict = ESPerform.get_tag_topic_list_dict(click_topic_tag_list,
@@ -95,7 +103,7 @@ class CollectData(object):
                         # redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
 
                 tag_id_list = recommend_tag_list[0:20]
-
+                pictorial_recommend_redis_key = self.linucb_recommend_pictorial_id_prefix + str(device_id)
                 topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
                 # redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
                 # redis_topic_list = list()
@@ -110,10 +118,12 @@ class CollectData(object):
                 #     have_read_topic_id_list.extend(redis_topic_list[:2])
                 if len(new_user_click_tag_list)>0:
                      tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(new_user_click_tag_list, have_read_topic_id_list)
-
+                     recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(new_user_click_tag_list,
+                                                                                      have_read_lin_pictorial_id_list)
                 else:
                      tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(tag_id_list,have_read_topic_id_list)
-
+                     recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(tag_id_list,
+                                                                                      have_read_lin_pictorial_id_list)
 
                 if len(recommend_topic_id_list)>0 or len(tag_topic_id_list)>0 or len(new_user_click_tag_list) > 0:
                     tag_topic_id_list = recommend_topic_id_list + tag_topic_id_list
@@ -124,6 +134,12 @@ class CollectData(object):
                         "cursor":0
                     }
                     redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
+                if len(recommend_lin_pictorial_id_list) > 0:
+                     pictorial_data_dict = {
+                    "data": json.dumps(recommend_lin_pictorial_id_list),
+                    "cursor": 0
+                      }
+                     redis_client.hmset(pictorial_recommend_redis_key, pictorial_data_dict)
 
             return True
         except:

search/views/topic.py

@@ -13,6 +13,7 @@ from libs.cache import redis_client
 from search.utils.common import *
 from libs.es import ESPerform
 from django.conf import settings
+from libs.tools import get_have_read_lin_pictorial_id_list
 import datetime
 
 
@@ -215,6 +216,40 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         else:
             return [], []
 
+def get_home_recommend_pictorial_ids(user_id=-1,device_id="",size=4):
+    try:
+        pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id)
+        have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id,user_id,TopicPageType.HOME_RECOMMEND)
+        pictorial_recommend_dict = redis_client.hgetall(pictorial_recommend_redis_key)
+        pictorial_recommend_list = list()
+        if b"data" in pictorial_recommend_dict:
+            pictorial_recommend_list = json.loads(pictorial_recommend_dict[b"data"])
+            if pictorial_recommend_list == None:
+                pictorial_recommend_list = list()
+            recommend_pictorial_id_list = list(set(pictorial_recommend_list) - set(have_read_lin_pictorial_id_list))
+            recommend_pictorial_id_list.sort(key=pictorial_recommend_list.index)
+            recommend_pictorial_ids = recommend_pictorial_id_list[0:size]
+            have_read_lin_pictorial_id_list.extend(recommend_pictorial_ids)
+            if user_id and int(user_id) > 0:
+                redis_key = "physical:home_pictorial_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(
+                    TopicPageType.HOME_RECOMMEND)
+            else:
+                redis_key = "physical:home_pictorial_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(
+                    TopicPageType.HOME_RECOMMEND)
+            redis_field_list = 'have_read_pictorial_list'
+            redis_client.hset(redis_key, redis_field_list,have_read_lin_pictorial_id_list)
+            today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
+            tomorrow = today + datetime.timedelta(days=1)
+            nowTime = datetime.datetime.now()
+            expire_time = (tomorrow - nowTime).seconds + 3 * 60 * 60
+            redis_client.expire(redis_key, expire_time)
+            return recommend_pictorial_ids
+        else:
+            return []
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return []
+
 
 @bind("physical/search/query_tag_id_by_topic")
 def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@@ -243,6 +278,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
             device_id = ""
 
         recommend_topic_ids = list()
+        recommend_pictorial_ids = list()
         rank_topic_ids = list()
         useful_topic_ids = list()
         es_node_load_high_flag = False
@@ -266,6 +302,8 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
 
         else:
             if usefulrecall != -1 and len(useful_tag_list) > 0:
+                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, 4)
+                size = size - len(recommend_pictorial_ids)
                 recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
                                                                                                      tag_id=0, offset=0,
                                                                                                      size=size,
@@ -274,20 +312,22 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
                                                                                                      usefulrecall=usefulrecall,
                                                                                                      useful_tag_list=useful_tag_list)
                 return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
-                        "useful_topic_ids": useful_topic_ids}
+                        "useful_topic_ids": useful_topic_ids,"linucb_pictorial_ids":recommend_pictorial_ids}
             else:
+                recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, 4)
+                size = size - len(recommend_pictorial_ids)
                 recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
                                                                                    offset=0, size=size,
                                                                                    query_type=query_type,
                                                                                    promote_topic_list=promote_topic_list)
 
-                return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
+                return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,"linucb_pictorial_ids":recommend_pictorial_ids}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         if usefulrecall != -1:
-            return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
+            return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [],"linucb_pictorial_ids":[]}
         else:
-            return {"linucb_topic_ids": [], "rank_topic_ids": []}
+            return {"linucb_topic_ids": [], "rank_topic_ids": [],"linucb_pictorial_ids":[]}
 
 
 @bind("physical/search/discover_page")

trans2es/models/pictorial.py

@@ -340,3 +340,15 @@ class CommunityPictorialActivityRelation(models.Model):
     is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
     pictorial_activity_id = models.BigIntegerField(verbose_name=u'活动ID', max_length=20)
     pictorial_id = models.BigIntegerField(verbose_name=u'榜单ID', max_length=20)
+
+
+class CommunityPictorialHomeFeed(models.Model):
+    """运营榜单"""
+
+    class Meta:
+        verbose_name = u"运营画报"
+        app_label = "community"
+        db_table = "community_pictorial_home_feed"
+    is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
+    is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
+    pictorial_id = models.BigIntegerField(verbose_name=u'榜单ID', max_length=20)