Merge branch 'test' of git.wanmeizhensuo.com:alpha/physical into test

.idea/codeStyles/Project.xml

@@ -24,29 +24,5 @@
         <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
       </formatting-settings>
     </DBN-SQL>
-    <DBN-PSQL>
-      <case-options enabled="false">
-        <option name="KEYWORD_CASE" value="lower" />
-        <option name="FUNCTION_CASE" value="lower" />
-        <option name="PARAMETER_CASE" value="lower" />
-        <option name="DATATYPE_CASE" value="lower" />
-        <option name="OBJECT_CASE" value="preserve" />
-      </case-options>
-      <formatting-settings enabled="false" />
-    </DBN-PSQL>
-    <DBN-SQL>
-      <case-options enabled="false">
-        <option name="KEYWORD_CASE" value="lower" />
-        <option name="FUNCTION_CASE" value="lower" />
-        <option name="PARAMETER_CASE" value="lower" />
-        <option name="DATATYPE_CASE" value="lower" />
-        <option name="OBJECT_CASE" value="preserve" />
-      </case-options>
-      <formatting-settings enabled="false">
-        <option name="STATEMENT_SPACING" value="one_line" />
-        <option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
-        <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
-      </formatting-settings>
-    </DBN-SQL>
   </code_scheme>
 </component>
\ No newline at end of file

linucb/views/collect_data.py

@@ -8,6 +8,7 @@ import logging
 from linucb.views.linucb import LinUCB
 import json
 from trans2es.models.tag import TopicTag,Tag
+from trans2es.models.topic import TopicHomeRecommend
 import traceback
 from django.conf import settings
 from libs.es import ESPerform
@@ -71,8 +72,10 @@ class CollectData(object):
                 # Todo:设置过期时间，调研set是否支持
                 redis_client.expire(tag_recommend_redis_key, 7*24*60*60)
 
-                have_read_topic_id_list = list()
                 have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,-1,TopicPageType.HOME_RECOMMEND)
+                promote_recommend_topic_id_list = list()
+                promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(is_online=1).values_list("topic_id")
+                have_read_topic_id_list.extend(promote_recommend_topic_id_list)
 
                 recommend_topic_id_list = list()
                 recommend_topic_id_list_click = list()

search/utils/topic.py

@@ -221,7 +221,7 @@ class TopicUtils(object):
                     {
                         "filter": {"bool": {
                             "should": {"terms": {"tag_list": attention_tag_list}}}},
-                        "weight": 1000000
+                        "weight": 100
                     }
                 )
             # if len(recommend_tag_list)>0:
@@ -653,20 +653,37 @@ class TopicUtils(object):
                         }
                     }
                 })
+            elif k.endswith("__exclude"):
+                filed = k[:-5]
+                op = "lt"
+                f.append({
+                    "range": {
+                        filed: {
+                            op: v,
+                        }
+                    }
+                })
+                op = "gt"
+                f.append({
+                    "range": {
+                        filed: {
+                            op: v,
+                        }
+                    }
+                })
 
-            elif k.startswith("__gte") or k.startswith("__lte") or \
-               k.startswith("__gt") or k.startswith("__lt"):
-                if k.startswith("__gte"):
+            elif k.endswith("__gte") or k.endswith("__lte") or k.endswith("__gt") or k.endswith("__lt"):
+                if k.endswith("__gte"):
                     op = "gte"
                     filed = k[:-5]
-                elif k == "__lte":
+                elif k.endswith("__lte"):
                     op = "lte"
                     filed = k[:-5]
-                elif k == "__gt":
-                    op = "lte"
+                elif k.endswith("__gt"):
+                    op = "gt"
                     filed = k[:-4]
-                elif k == "__lt":
-                    op = "lte"
+                elif k.endswith("__lt"):
+                    op = "lt"
                     filed = k[:-4]
 
                 f.append({

search/views/tag.py

@@ -10,6 +10,8 @@ from search.utils.topic import TopicUtils
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
+from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow
+import time
 
 
 def get_highlight(fields=[]):
@@ -95,9 +97,52 @@ def query_by_tag_type(tag_type_id,offset,size):
 
 
 @bind("physical/search/choice_push_tag")
-def choice_push_tag(device_id,user_id):
+def choice_push_tag(device_id, user_id):
+    """
+    :remark 挑选push标签
+    :param device_id:
+    :param user_id:
+    :return:
+    """
     try:
-        pass
+        redis_key_prefix = "physical:push_tag:user_id:"
+        redis_push_tag_key = redis_key_prefix + str(user_id)
+
+        redis_push_tag_data = redis_client.get(redis_push_tag_key)
+        redis_push_tag_dict = json.loads(redis_push_tag_data) if redis_push_tag_data else {}
+
+        now_sec = int(time.time())
+        valid_time = 8*7*24*60*60
+        for tag_id in redis_push_tag_dict:
+            if now_sec - redis_push_tag_dict[tag_id] >= valid_time:
+                redis_push_tag_dict.popitem(tag_id)
+
+        redis_push_tag_list = list(redis_push_tag_dict.keys())
+        account_user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
+        community_tag_follow_list = CommunityTagFollow.objects.filter(user_id=user_id).values_list("tag_id",flat=True)
+
+        linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
+        tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
+        linucb_recommend_tag_data = redis_client.get(tag_recommend_redis_key)
+        linucb_recommend_tag_list = json.loads(linucb_recommend_tag_data) if linucb_recommend_tag_data else []
+
+        account_user_tag_list.extend(community_tag_follow_list)
+        account_user_tag_list.extend(linucb_recommend_tag_list)
+        unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list))
+
+        ret_tag_set = set()
+        if len(unread_tag_list)>0:
+            for tag_id in unread_tag_list:
+                valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count()
+                if valid_tag_topic_num>100:
+                    ret_tag_set.add(tag_id)
+                    redis_push_tag_dict[tag_id] = now_sec
+                    if len(ret_tag_set)>=2:
+                        break
+
+        redis_client.set(redis_push_tag_key, json.dumps(redis_push_tag_dict))
+
+        return {"tag_list": list(ret_tag_set)}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"tag_list": []}

search/views/topic.py

@@ -7,10 +7,13 @@ import logging
 import traceback
 import json
 from search.utils.topic import TopicUtils
+from trans2es.models.topic import TopicHomeRecommend
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
 from libs.es import ESPerform
+from django.conf import settings
+
 
 
 def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
@@ -46,13 +49,15 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
                                  query_type=TopicPageType.HOME_RECOMMEND):
     try:
         if query is None:
-            # redis_key = "physical:home_recommend" + ":user_id:" + str(
-            #     user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
-            redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
+            if user_id>0:
+                redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+            else:
+                redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
         else:
-            # redis_key = "physical:home_query" + ":user_id:" + str(
-            #     user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
-            redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
+            if user_id>0:
+                redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
+            else:
+                redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
 
 
         redis_field_list = [b'have_read_topic_list']
@@ -72,7 +77,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         if b"data" in recommend_topic_dict:
             recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
             cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
-            newcursor = cursor + 5
+            newcursor = cursor + 6
             if len(recommend_topic_id_list) > newcursor:
                 recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
                 redis_client.hset(topic_recommend_redis_key,"cursor",newcursor)
@@ -112,12 +117,37 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         if len(recommend_topic_list)>0:
             size = size-len(recommend_topic_list)
             have_read_topic_id_list.extend(recommend_topic_list)
-        topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
+
+        have_read_topic_id_list_add_promote = list()
+        promote_recommend_topic_id_list = list()
+        have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
+        promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
+            is_online=1).values_list("topic_id",flat=True)
+        if len(promote_recommend_topic_id_list)>0:
+          for topic_id in promote_recommend_topic_id_list:
+             have_read_topic_id_list_add_promote.append(topic_id)
+        topic_id_list = list()
+
+        rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
                                                            single_size=size,query=query, query_type=query_type,
-                                                           filter_topic_id_list=have_read_topic_id_list,
+                                                           filter_topic_id_list=have_read_topic_id_list_add_promote,
                                                            recommend_tag_list=recommend_topic_list,
                                                            user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5",attention_tag_list=attention_tag_list)
 
+        if (len(recommend_topic_list) == 6):
+            if (size < 11):
+                topic_id_list.extend(rank_topic_id_list[0:3])
+                topic_id_list.extend(recommend_topic_list[0:3])
+                topic_id_list.extend(rank_topic_id_list[3:size])
+                topic_id_list.extend(recommend_topic_list[3:6])
+            else:
+                topic_id_list.extend(rank_topic_id_list[0:size - 7])
+                topic_id_list.extend(recommend_topic_list[0:3])
+                topic_id_list.extend(rank_topic_id_list[size - 7:size])
+                topic_id_list.extend(recommend_topic_list[3:6])
+        else:
+            topic_id_list.extend(rank_topic_id_list)
+
         have_read_group_id_set = set()
         have_read_user_id_set = set()
         unread_topic_id_dict = dict()
@@ -159,7 +189,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         #         else:
         #             break
 
-        topic_id_list.extend(recommend_topic_list)
+        # topic_id_list.extend(recommend_topic_list)
         # recommend_topic_list.extend(topic_id_list)
         have_read_topic_id_list.extend(topic_id_list)
         if len(have_read_topic_id_list) > 30000:
@@ -169,7 +199,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
             "have_read_topic_list": json.dumps(have_read_topic_id_list),
         }
         redis_client.hmset(redis_key, redis_dict)
-        # 每个session key保存15分钟
+        # 每个session key保存30天
         redis_client.expire(redis_key, 60 * 60 * 24 * 30)
 
         return topic_id_list

trans2es/models/tag.py

@@ -22,7 +22,6 @@ class TopicTag(models.Model):
     is_collection = models.IntegerField(verbose_name=u"是否编辑标签")
 
 
-
 class AccountUserTag(models.Model):
     class Meta:
         verbose_name=u"用户标签"

trans2es/models/topic.py

@@ -281,3 +281,12 @@ class ExcellentTopic(models.Model):
     create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
+class TopicHomeRecommend(models.Model):
+    """运营位帖子"""
+
+    class Meta:
+        verbose_name = "运营位帖子"
+        db_table = "topic_home_recommend"
+    id = models.IntegerField(verbose_name=u"id",primary_key=True)
+    topic_id = models.IntegerField(verbose_name=u"帖子ID")
+    is_online = models.BooleanField(verbose_name=u'是否上线')
\ No newline at end of file