Merge branch 'test' into 'master'

revise

See merge request alpha/physical!325

.gitignore

@@ -85,4 +85,5 @@ gaia/rpcd.json
 dbmw_deploy/config.dir/
 .idea/codeStyles/Project.xml
 .idea/misc.xml
-.idea/physical.iml
\ No newline at end of file
+.idea/physical.iml
+.idea/codeStyles/Project.xml
\ No newline at end of file

.idea/misc.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

.idea/physical.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 3.6 (venv)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

app_conf.xml

@@ -17,6 +17,5 @@
         <element value="search.views.contrast_similar"/>
         <element value="injection.data_sync.tasks"/>
         <element value="search.views.contrast_similar"/>
-        <!--<element value="majia..tasks.tasks"/>-->
     </config>
 </gm_rpcd_config>

injection/data_sync/tasks.py

@@ -2,6 +2,10 @@
 import logging
 import traceback
 import json
+import pymysql
+import threading
+import random
+import datetime
 from celery import shared_task
 from django.conf import settings
 from django.core import serializers
@@ -9,9 +13,10 @@ from trans2es.type_info import get_type_info_map
 # from rpc.all import get_rpc_remote_invoker
 from libs.es import ESPerform
 from libs.cache import redis_client
-from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
+from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
 from linucb.utils.register_user_tag import RegisterUserTag
 
+
 @shared_task
 def write_to_es(es_type, pk_list, use_batch_query_set=False):
     try:
@@ -67,23 +72,26 @@ def sync_face_similar_data_to_redis():
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
+
 @shared_task
 def sync_user_similar_score():
     try:
-        results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True)
+        results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
+            "user_id", flat=True)
         redis_key_prefix = "physical:user_similar_score:user_id:"
 
         logging.info("begin sync user_similar_score!")
         for user_id in results_items:
             redis_key = redis_key_prefix + str(user_id)
-            similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score")
+            similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
+                "-score")
 
             item_list = list()
             for item in similar_results_items:
                 contrast_user_id = item.contrast_user_id
                 score = item.score
                 item_list.append(
-                    [contrast_user_id,score]
+                    [contrast_user_id, score]
                 )
 
             redis_client.set(redis_key, json.dumps(item_list))

libs/es.py

@@ -366,3 +366,78 @@ class ESPerform(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return list()
+
+    @classmethod
+    def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
+        try:
+            functions_list = list()
+            for id in tag_id:
+                functions_list.append(
+                    {
+                        "filter": {"term": {"tag_list": id}},
+                        "weight": 1
+                    }
+                )
+            functions_list += [
+                {
+                    "filter": {"term": {"content_level": 6}},
+                    "weight": 6000
+                },
+                {
+                    "filter": {"term": {"content_level": 5}},
+                    "weight": 5000
+                },
+                {
+                    "filter": {"term": {"content_level": 4}},
+                    "weight": 4000
+                }
+            ]
+            q = {
+                "query": {
+                    "function_score": {
+                        "query": {
+                            "bool": {
+                                "must": [
+                                    {"range": {"content_level": {"gte": 4, "lte": 6}}},
+                                    {"term": {"is_online": True}},
+                                    {"term": {"is_deleted": False}},
+                                    {"terms": {"tag_list": tag_id}}
+                                ]
+                            }
+                        },
+                        "boost_mode": "sum",
+                        "score_mode": "sum",
+                        "functions": functions_list
+                    }
+                },
+                "_source": {
+                    "include": ["id","user_id"]
+                },
+                "sort": [
+                    {"_score": {"order": "desc"}},
+                    {"create_time_val": {"order": "desc"}},
+                    {"language_type": {"order": "asc"}},
+                ]
+            }
+            if len(have_read_topic_id_list) > 0:
+                q["query"]["function_score"]["query"]["bool"]["must_not"] = {
+                    "terms": {
+                        "id": have_read_topic_id_list
+                    }
+                }
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
+                                                       offset=0, size=size, routing="4,5,6")
+
+            topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
+            # logging.info("topic_id_list:%s" % str(topic_id_list))
+            # topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
+            topic_id_dict = dict()
+            for item in result_dict["hits"]:
+                topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
+
+            logging.info("topic_id_list:%s" % str(topic_id_dict))
+
+            return topic_id_list,topic_id_dict
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return list()

linucb/views/collect_data.py

@@ -76,18 +76,23 @@ class CollectData(object):
                 have_read_topic_id_list.extend(promote_recommend_topic_id_list)
 
                 recommend_topic_id_list = list()
+                recommend_topic_id_list_dict = dict()
                 recommend_topic_id_list_click = list()
+                recommend_topic_id_list_click_dict = dict()
+
 
                 if click_topic_tag_list:
                     if len(click_topic_tag_list)>0:
-                     recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list,
+                     recommend_topic_id_list_click,recommend_topic_id_list_click_dict = ESPerform.get_tag_topic_list_dict(click_topic_tag_list,
                                                                                  have_read_topic_id_list,size=2)
                     if len(recommend_topic_id_list_click) > 0:
                         recommend_topic_id_list.extend(recommend_topic_id_list_click)
-                        have_read_topic_id_list.extend(recommend_topic_id_list)
+                        recommend_topic_id_list_dict.update(recommend_topic_id_list_click_dict)
+                        have_read_topic_id_list.extend(recommend_topic_id_list_click)
                         click_recommend_redis_key = self.click_recommend_redis_key_prefix + str(device_id)
                         click_redis_data_dict = {
                             "data": json.dumps(recommend_topic_id_list),
+                            "datadict":json.dumps(recommend_topic_id_list_dict),
                             "cursor": 0
                           }
                         redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
@@ -103,25 +108,31 @@ class CollectData(object):
                         b"data"] else []
                     cursor = int(str(redis_topic_data_dict[b"cursor"], encoding="utf-8"))
 
+
                 if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
                     have_read_topic_id_list.extend(redis_topic_list[:2])
+                tag_topic_dict = dict()
                 if len(new_user_click_tag_list)>0:
-                     tag_topic_id_list = ESPerform.get_tag_topic_list(new_user_click_tag_list, have_read_topic_id_list)
+                     tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(new_user_click_tag_list, have_read_topic_id_list)
+
                 else:
-                     tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
+                     tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(tag_id_list,have_read_topic_id_list)
 
 
                 if len(recommend_topic_id_list)>0 or len(new_user_click_tag_list) > 0:
                     tag_topic_id_list = recommend_topic_id_list + tag_topic_id_list
+                    tag_topic_dict.update(recommend_topic_id_list_dict)
                     redis_data_dict = {
                         "data": json.dumps(tag_topic_id_list),
+                        "datadict":json.dumps(tag_topic_dict),
                         "cursor":0
                     }
                     redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
                 else:
                     if cursor<=0 and len(redis_topic_list)>0:
-                        tag_topic_id_list = redis_topic_list[:2] + tag_topic_id_list
-                        tag_topic_id_list = list(set(tag_topic_id_list))
+                        tag_topic_dict = list()
+                        tag_topic_dict = redis_topic_list[:2]
+                        tag_topic_dict = list(set(tag_topic_dict))
 
             return True
         except:

majia/__init__.py

-default_app_config = 'majia.apps.MaJiaApp'

majia/admin.py

-from django.contrib import admin
-
-# Register your models here.

majia/apps.py

-# coding=utf-8
-from __future__ import unicode_literals, print_function, absolute_import
-
-from django.apps import AppConfig
-
-
-class MaJiaApp(AppConfig):
-    name = 'majia'
-    label = 'majia'

majia/get_session.py

-import requests
-import time
-import datetime
-import random
-import traceback
-from django.conf import settings
-import logging
-
-DATABASES = settings.DATABASES
-
-# host = DATABASES['default']['HOST']
-#
-# user = DATABASES['default']['USER']
-#
-# port = DATABASES['default']['PORT']
-#
-# db = DATABASES['default']['NAME']
-#
-# passwd = DATABASES['default']['PASSWORD']
-
-auto_click_url = "http://earth.igengmei.com/api/v1/like"
-auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
-auto_follow_url = "http://earth.igengmei.com/api/v1/follow"
-auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
-
-
-def get_majia():
-    try:
-        data = open("/srv/apps/physical/majia/majia_email.txt")
-        list = []
-        for i in data:
-            list.append(i.strip('\n').strip(','))
-        maj = random.randint(1, len(list))
-        user_id = list[maj - 1]
-        return user_id
-
-    except:
-        logging.error("catch exception,get_majia:%s" % traceback.format_exc())
-        return None
-
-
-def get_cookies(user_id):
-    try:
-        with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
-            try:
-                data = f.readlines()
-            except:
-                data = None
-
-        list = []
-        for i in data:
-            list.append(i.strip('\n').split(","))
-
-        for i in list:
-
-            if user_id == i[0]:
-                return (i[1])
-    except:
-
-        return None
-
-
-def login():
-    try:
-        user_id = get_majia()
-
-        cookies = get_cookies(user_id)
-
-        if cookies is not None:
-            return {'sessionid': cookies}
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-        return None
-
-
-def click(cookies_get, id):
-    try:
-        post_dict = {
-            'type': 0,
-            'id': id
-        }
-        response = requests.post(url=auto_click_url,
-                                 cookies=cookies_get,
-                                 data=post_dict)
-
-        logging.info("response.text:%s" % response.text)
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-
-def reply(cookies_get, id, content):
-    try:
-        post_dict = {
-            'topic_id': id,
-            'content': content
-        }
-        response = requests.post(url=auto_reply_url,
-                                 cookies=cookies_get,
-                                 data=post_dict)
-
-        logging.info("response.text:%s" % response.text)
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-
-def time_conv(minutest):
-    try:
-        now = datetime.datetime.now()
-        yes_time = now - datetime.timedelta(minutes=minutest)
-        print(yes_time)
-        return yes_time
-
-    except:
-        return None
-
-
-def get_comments():
-    try:
-        data = open("/srv/apps/physical/majia/guanshui.txt", "r")
-        list_guanshui = []
-        for i in data:
-            list_guanshui.append(i)
-        num1 = random.randint(0, 30)
-        num2 = random.randint(30, 70)
-        num3 = random.randint(70, len(list_guanshui))
-
-        comment1 = list_guanshui[num1 - 1]
-        comment2 = list_guanshui[num2 - 1]
-        comment3 = list_guanshui[num3 - 1]
-
-        comment_list = [comment1, comment2, comment3]
-        return comment_list
-
-    except:
-
-        return None

majia/guanshui.txt

-照片色调很好看
-用的什么调色工具呀
-立马去get同款
-我一个一看就会，动手就废的girl
-为什么我拍不出来你的效果？
-Po主用的什么拍照软件
-这个好，学习了
-真的是元气满满
-很帅气！
-这样好显气质！
-好可爱啊，完全没有抵抗力了。
-莫名好感中
-满分颜值
-超酷！
-马住回看
-请问发色是啥
-请问是什么app拍的啊
-滤镜要怎么选？
-卖萌打滚求翻牌
-有没有常用自拍姿势教程
-拍照技术满分
-日常夸奖博主
-日常勾搭
-背景是在哪里？
-照片构图很好看
-主要还是看颜值
-你长得很像明星
-支持一下
-转需留存
-拍照表情很自然
-风格很好
-这里是哪里呀？
-种草了！
-搭配小技巧get！
-怎么都学不会的路过
-太好了，新手学习了
-一直支持，只求被翻牌！
-好喜欢这种风格
-好看，在哪儿买的
-衣好有气质哦
-这个我也会
-拍照pose求出教程
-小姐姐，作图app是啥
-终于更新了
-搭配的好好看
-求同款
-日常催更
-get√
-什么时候更新啊
-果然只要瘦怎么都好看。
-不错的哈哈
-同款达不到楼主的效果
-喜欢你的搭配
-我拍出来为什么感觉比本人丑很多？
-能多发点全身正面照吗？想看整体效果
-这个真的很真实了
-大圆脸合适吗？
-成功引起了我的注意！
-感觉很少女~
-不错，继续加油
-好看的人做什么都好看
-都是人才！！！！
-道理我都懂，就差一个脸！
-不赞我良心都过意不去啊
-爱你哟~
-用手机可以拍出来吗？
-使用相机拍的还是手机
-我觉得很赞
-颜色搭配很舒服
-小姐姐有戴美瞳吗
-看见这个帖心情变好了
-不管怎么样先收藏了
-更新更新更新
-你和我朋友长得有点像
-你的风格我都很喜欢
-好看
-我要有你这条件还需要啥拍照技术
-太美了！
-被圈粉了……
-美而不作，我喜欢
-楼主真的real真实了。
-持续等更新
-楼主加油，照片拍的不错，非常值得推荐。
-大家花呗还了吗？
-来啦来啦来啦来啦来啦
-画面太治愈了
-楼主辛苦啦
-不错，跟着学习一下
-心动，我要去试试
-满满的干货，收藏好下次用
-新技能get√
-感觉不错mark一下
-适合自己的才是最重要的
-羡慕一下作者的颜
-Omaga太好看了吧~
-这张拍的不错
-这个特效挺可爱的
-翻我翻我~
-这样很显脸小
-关注啦

majia/models.py

-from django.db import models
-
-# Create your models here.

majia/tasks.py

-# -*- coding: UTF-8 -*-
-
-import pymysql
-import random
-import threading
-import traceback
-import logging
-from celery import shared_task
-from django.conf import settings
-from majia.get_session import get_comments, click, login, reply
-# from majia.get_session import host, user, passwd, db
-
-exec_count = 0
-
-def auto_click(pk_list):
-
-    logging.info("get----------------------:%s"%pk_list)
-    # exec_count = 0
-    # try:
-    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
-    #     cursor = pc.cursor()
-    #     cursor.execute(
-    #         "SELECT id,user_id FROM topic WHERE id  in  " + str(
-    #             tuple(pk_list)) + "  and user_id in (select user_id from user_extra  where is_shadow = 0)")
-    #     data = cursor.fetchall()
-    #     topic_id_list = list(data)
-    #     logging.info("Database version : %s " % topic_id_list)
-    #     pc.close()
-    #     exec_count = 0
-    #     if topic_id_list:
-    #
-    #         def fun_timer():
-    #             cookies = login()
-    #             if cookies:
-    #                 click(cookies, topic_id_list[0])
-    #
-    #             global timer
-    #             global exec_count
-    #
-    #             exec_count += 1
-    #             if exec_count == 1:
-    #                 # sleep_time = random.randint(300, 540)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_timer)
-    #                 timer.start()
-    #
-    #             if exec_count == 2:
-    #                 # sleep_time = random.randint(1000, 1900)
-    #                 sleep_time = 50
-    #                 timer = threading.Timer(sleep_time, fun_timer)
-    #                 timer.start()
-    #
-    #         sleep_time = random.randint(300, 540)
-    #         timer = threading.Timer(10, fun_timer)  # 首次启动
-    #         timer.start()
-    #
-    #     else:
-    #         pass
-    # except:
-    #     logging.error("catch exception,main:%s" % traceback.format_exc())
-
-
-def auto_reply(pk_list):
-
-    logging.info("get----------------------:%s"%pk_list)
-
-    # exec_count = 0
-    # try:
-    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
-    #     cursor = pc.cursor()
-    #     cursor.execute(
-    #         "SELECT id,user_id FROM topic WHERE id in  " + str(
-    #             tuple(pk_list)) + " and user_id in (select user_id from user_extra  where is_shadow = 0)")
-    #     data = cursor.fetchall()
-    #     topic_id_list = list(data)
-    #     logging.info("Database version : %s " % topic_id_list)
-    #     pc.close()
-    #     exec_count = 0
-    #     if topic_id_list:
-    #
-    #         def fun_comment():
-    #             cookies = login()
-    #             if cookies:
-    #                 comment_content = get_comments()
-    #                 comment = comment_content[0]
-    #                 reply(cookies, topic_id_list[0], comment)
-    #
-    #             global timer
-    #             global exec_count
-    #
-    #             exec_count += 1
-    #             if exec_count == 1:
-    #                 sleep_time = random.randint(300, 540)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_comment)
-    #                 timer.start()
-    #
-    #             if exec_count == 2:
-    #                 sleep_time = random.randint(1000, 1900)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_comment)
-    #                 timer.start()
-    #
-    #         sleep_time = random.randint(300, 540)
-    #         timer = threading.Timer(10, fun_comment)  # 首次启动
-    #         timer.start()
-    #
-    #
-    # except:
-    #     logging.error("catch exception,main:%s" % traceback.format_exc())
-
-
-@shared_task
-def auto_majia_effect(pk_list):
-    auto_click(pk_list)
-    auto_reply(pk_list)

majia/tests.py

-from django.test import TestCase
-
-# Create your tests here.

majia/views.py

-from django.shortcuts import render
-
-# Create your views here.

physical/celery_task_router.py

@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
     queue_task_map = {
         "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
-        ],
-        "majia-alpha": [
-            'majia.tasks.auto_majia_effect',
-        ],
+        ]
     }
 
     # Map[TaskName, QueueName]

physical/settings_local.py.template

@@ -48,7 +48,6 @@ INSTALLED_APPS = (
     'trans2es',
     'search',
     'injection.data_sync',
-    'majia',
 )
 
 CELERYBEAT_SCHEDULE = {

search/utils/group.py

@@ -47,23 +47,76 @@ class GroupUtils(object):
             return {"total_count": 0, "hits": []}
 
     @classmethod
-    def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None):
+    def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None,attention_tag_list=[]):
         try:
             if not es_cli_obj:
                 es_cli_obj = ESPerform.get_cli()
 
-            q = dict()
-            q["query"] = {
-                "bool":{
-                    "must":[
-                        {"term": {"is_online": True}},
-                        {"term":{"is_deleted": False}},
-                        {"term": {"effective": True}}
-                    ]
+            functions_list = list()
+            for tag_id in attention_tag_list:
+                functions_list.append({
+                    "filter":{
+                        "constant_score":{
+                            "filter":{
+                                "term":{
+                                    "tag_id": tag_id
+                                }
+                            }
+                        }
+                    },
+                    "weight": 20
+                })
+
+                if len(functions_list)>=20:
+                    break
+
+            functions_list.append(
+                {
+                    "gauss": {
+                        "create_time": {
+                            "scale": "1d",
+                            "decay": 0.99
+                        }
+                    },
+                    "weight": 60
+                }
+            )
+            q = {
+                "query":{
+                    "function_score":{
+                        "query":{
+                            "bool":{
+                                "filter":[
+                                    {"term": {"is_online": True}},
+                                    {"term": {"is_deleted": False}},
+                                    {"term": {"effective": True}}
+                                ]
+                            }
+                        },
+                        "score_mode": "sum",
+                        "boost_mode": "sum",
+                        "functions": functions_list
+                    }
                 }
             }
             q["sort"] = [
-                {"high_quality_topic_num":{"order":"desc"}}
+                {
+                    "_script": {
+                        "type": "number",
+                        "script": {
+                            "lang": "expression",
+                            "source": "_score+doc['offline_score']"
+                            # "lang":"painless",
+                            # "source":"_score+params._source.offline_score"
+                        },
+                        "order": "desc"
+                    }
+                },
+                {
+                    "_score":{
+                        "order": "desc"
+                    }
+                }
             ]
             q["_source"] = {
                 "includes":["id"]

search/utils/topic.py

@@ -122,7 +122,7 @@ class TopicUtils(object):
     @classmethod
     def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.FIND_PAGE,
                                 filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
-                                user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[],current_topic_id =-1,topic_tag_list =[],topic_user_id =-1,disable_collpase=False):
+                                user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[],linucb_user_id_list = [],disable_collpase=False):
         """
         :remark:获取首页推荐帖子列表
         :param user_id:
@@ -216,25 +216,7 @@ class TopicUtils(object):
                         "weight": 100
                     }
                 )
-            if current_topic_id != -1:
 
-                if len(topic_tag_list) > 0:
-                    functions_list.append(
-                        {
-                            "filter": {"bool": {
-                                "should": {"terms": {"tag_list": topic_tag_list}}}},
-                            "weight": 2000
-
-                        }
-                    )
-                if topic_user_id != -1:
-                    functions_list.append(
-                        {
-                            "filter": {"bool": {
-                                "should": {"term": {"user_id": topic_user_id}}}},
-                            "weight": 1500
-                        }
-                    )
             query_function_score = {
                 "query": {
                     "bool": {
@@ -280,12 +262,16 @@ class TopicUtils(object):
                 query_function_score["query"]["bool"]["must_not"] = [
                     {"terms":{"id":filter_topic_id_list}}
                 ]
-            if current_topic_id != -1:
-                query_function_score["query"]["bool"]["must_not"] = [{
-                    "term": {
-                        "id": current_topic_id
-                    }
-                }]
+
+            if len(linucb_user_id_list)>0:
+                if "must_not" in query_function_score["query"]["bool"]:
+                    query_function_score["query"]["bool"]["must_not"] += [
+                        {"terms": {"user_id": linucb_user_id_list}}
+                    ]
+                else:
+                    query_function_score["query"]["bool"]["must_not"] = [
+                        {"terms": {"user_id": linucb_user_id_list}}
+                    ]
             if query is not None:  # 搜索帖子
                 multi_fields = {
                     'description': 200,

search/views/group.py

@@ -52,9 +52,15 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
         es_cli_obj = ESPerform.get_cli()
 
         if sort_type == GroupSortTypes.HOT_RECOMMEND:
-            pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj)
-            return {"pictorial_recommend_ids": pictorial_ids_list}
+            # 用户关注标签
+            attention_tag_list = list()
+            if user_id>0:
+                redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
+                attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
+
+            pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,attention_tag_list)
 
+            return {"pictorial_recommend_ids": pictorial_ids_list}
         elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
             attention_pictorial_list = GroupUtils.get_user_attention_pictorial_list(user_id, offset=0, size=1,
                                                                                     es_cli_obj=es_cli_obj)

search/views/topic.py

@@ -46,7 +46,7 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
 
 
 def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
-                                 query_type=TopicPageType.HOME_RECOMMEND,disable_collpase=False):
+                                 query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False):
     try:
         if query is None:
             if user_id>0:
@@ -77,6 +77,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
         # user_similar_score_redis_list = json.loads(
         #     redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
+        recommend_topic_user_list = list()
 
         attention_tag_list = list()
         recommend_topic_list = list()
@@ -87,14 +88,21 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
             recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
             if b"data" in recommend_topic_dict:
                 recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
+
+
                 # 推荐帖子是强插的，要保证推荐帖子不在已读里
-                recommend_topic_id_list = list(set(recommend_topic_id_list) - set(have_read_topic_id_list))
+                # recommend_topic_id_list = list(set(recommend_topic_id_list) - set(have_read_topic_id_list))
                 cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
                 newcursor = cursor + 6
                 if len(recommend_topic_id_list) > newcursor:
                     recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
                     redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
 
+            if b"datadict" in recommend_topic_dict:
+                recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
+                if len(recommend_topic_list) == 6 and recommend_topic_id_dict is not None:
+                  for i in recommend_topic_list:
+                    recommend_topic_user_list.append(recommend_topic_id_dict[str(i)])
             # 用户关注标签
             redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
             attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
@@ -109,27 +117,28 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
             #
             # for topic_id in promote_recommend_topic_id_list:
             #     have_read_topic_id_list_add_promote.append(topic_id)
-
+        have_read_topic_id_list.extend(promote_topic_list)
         topic_id_list = list()
         rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
                                                            single_size=size,query=query, query_type=query_type,
-                                                           filter_topic_id_list=have_read_topic_id_list,index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list,disable_collpase=disable_collpase)
-
-        if len(recommend_topic_list) == 6 and query is None:
-            if (size < 11):
-                topic_id_list.extend(rank_topic_id_list[0:3])
-                topic_id_list.extend(recommend_topic_list[0:3])
-                topic_id_list.extend(rank_topic_id_list[3:size])
-                topic_id_list.extend(recommend_topic_list[3:6])
-            else:
-                topic_id_list.extend(rank_topic_id_list[0:size - 7])
-                topic_id_list.extend(recommend_topic_list[0:3])
-                topic_id_list.extend(rank_topic_id_list[size - 7:size])
-                topic_id_list.extend(recommend_topic_list[3:6])
-        else:
-            topic_id_list.extend(rank_topic_id_list)
-
-        have_read_topic_id_list.extend(topic_id_list)
+                                                           filter_topic_id_list=have_read_topic_id_list,
+                                                           index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
+
+        # if len(recommend_topic_list) == 6 and query is None:
+        #     if (size < 11):
+        #         topic_id_list.extend(rank_topic_id_list[0:3])
+        #         topic_id_list.extend(recommend_topic_list[0:3])
+        #         topic_id_list.extend(rank_topic_id_list[3:size])
+        #         topic_id_list.extend(recommend_topic_list[3:6])
+        #     else:
+        #         topic_id_list.extend(rank_topic_id_list[0:size - 7])
+        #         topic_id_list.extend(recommend_topic_list[0:3])
+        #         topic_id_list.extend(rank_topic_id_list[size - 7:size])
+        #         topic_id_list.extend(recommend_topic_list[3:6])
+        # else:
+        #     topic_id_list.extend(rank_topic_id_list)
+
+        have_read_topic_id_list.extend(rank_topic_id_list)
         if len(have_read_topic_id_list) > 30000:
             cut_len = len(have_read_topic_id_list)-30000
             have_read_topic_id_list = have_read_topic_id_list[cut_len:]
@@ -140,10 +149,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         # 每个session key保存30天
         redis_client.expire(redis_key, 60 * 60 * 24 * 30)
 
-        return topic_id_list
+        return recommend_topic_list,rank_topic_id_list
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return []
+        return [],[]
 
 
 @bind("physical/search/query_tag_id_by_topic")
@@ -156,7 +165,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
 
 
 @bind("physical/search/home_recommend")
-def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND):
+def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[]):
     """
     :remark:首页推荐，目前只推荐日记
     :param session_id:
@@ -172,7 +181,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
             device_id = ""
 
         recommend_topic_ids = list()
-
+        rank_topic_ids = list()
         es_node_load_high_flag = False
         # try:
         #     es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
@@ -193,10 +202,10 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
                     recommend_topic_ids = have_read_topic_id_list[0:size]
 
         else:
-            recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
-                                                           query_type=query_type)
+            recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
+                                                           query_type=query_type,promote_topic_list=promote_topic_list)
 
-        return {"recommend_topic_ids": recommend_topic_ids}
+        return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"recommend_topic_ids": []}
@@ -243,13 +252,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
         if not isinstance(device_id, str):
             device_id = ""
 
-        recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
-
-        #todo 因为按用户打散，会出现召回数量少于请求数的情况，再次去掉打散请求
-        if len(recommend_topic_ids)>0 and len(recommend_topic_ids)<size:
-            recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size,
-                                                               query=query,disable_collpase=True)
-        return {"recommend_topic_ids": recommend_topic_ids}
+        recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
+        if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
+            recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
+                                                                                    offset=offset, size=size,
+                                                                                    query=query,disable_collpase=True)
+        return {"recommend_topic_ids": rank_topic_id_list}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"recommend_topic_ids": []}

trans2es/mapping/pictorial.json

@@ -16,6 +16,7 @@
     "tag_id":{"type":"long"},
     "tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
     "topic_id_list":{"type":"long"},
-    "effective":{"type":"boolean"}
+    "effective":{"type":"boolean"},
+    "offline_score":{"type":"long"}
   }
 }
\ No newline at end of file

trans2es/mapping/topic-high-star.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/mapping/topic-star-routing.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/mapping/topic-star.json

 {
   "dynamic":"strict",
+  "_routing": {"required": true},
   "properties": {
     "id":{"type":"long"},
-    "content_level":{"type":"text"}
+    "is_online":{"type":"boolean"},//上线
+    "is_deleted":{"type":"boolean"},
+    "vote_num":{"type":"long"},
+    "total_vote_num":{"type":"long"},
+    "reply_num":{"type":"long"},
+    "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content_level":{"type":"text"},
+    "user_id":{"type":"long"},
+    "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","index":"not_analyzed"}, //不切词的用户名
+    "group_id":{"type":"long"}, //所在组ID
+    "tag_list":{"type":"long"},//标签属性
+    "edit_tag_list":{"type":"long"},//编辑标签
+    "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "share_num":{"type":"long"},
+    "pick_id_list":{"type":"long"},
+    "offline_score":{"type":"double"},//离线算分
+    "manual_score":{"type":"double"},//人工赋分
+    "has_image":{"type":"boolean"},//是否有图
+    "has_video":{"type":"boolean"},//是否是视频
+    "create_time":{"type":"date", "format":"date_time_no_millis"},
+    "update_time":{"type":"date", "format":"date_time_no_millis"},
+    "create_time_val":{"type":"long"},
+    "update_time_val":{"type":"long"},
+    "language_type":{"type":"long"},
+
+    "is_shadow": {"type": "boolean"},
+    "is_recommend": {"type": "boolean"},
+    "is_complaint": {"type": "boolean"},  // 是否被举报
+
+    "virtual_content_level":{"type": "text"},
+    "like_num_crawl": {"type": "long"}, // 爬取点赞数
+    "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+    "is_crawl": {"type": "boolean"},
+    "platform": {"type": "long"},
+    "platform_id": {"type": "long"},
+
+    "drop_score":{"type": "double"},  // 人工降分
+    "sort_score":{"type": "double"},  // 排序分
+
+    "pictorial_id":{"type": "long"}, //所在组ID
+    "pictorial_name":{  // 所在组名称
+        "type": "text",
+        "analyzer": "gm_default_index",
+        "search_analyzer": "gm_default_index"
+    },
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic-v1.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签
@@ -52,6 +53,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/models/pictorial.py

@@ -36,7 +36,6 @@ class PictorialTopics(models.Model):
     pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
     topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
     is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
-    is_online = models.BooleanField(verbose_name=u'是否上线')
     is_deleted = models.BooleanField(verbose_name=u'是否删除')
 
 
@@ -59,18 +58,15 @@ class Pictorial(models.Model):
     creator_id = models.BigIntegerField(verbose_name=u'画报用户ID')
     icon = models.CharField(verbose_name=u'画报名称', max_length=255)
     topic_num = models.IntegerField(verbose_name=u'次数')
+    add_score = models.IntegerField(verbose_name=u'人工权重')
 
     def get_topic_id(self):
         try:
-            topic_id =[]
-            topic_id_list = PictorialTopics.objects.filter(pictorial_id=self.id).values_list("topic_id", flat=True)
-            for i in topic_id_list:
-                topic_id.append(i)
-            return topic_id
-
+            topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=self.id,is_online=True,is_deleted=False).values_list("topic_id", flat=True))
+            return topic_id_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
 
     def get_effective(self,topic_id_list):
@@ -112,29 +108,21 @@ class Pictorial(models.Model):
 
     def get_tag_by_id(self):
         try:
-            tag_id_list = []
-            tags = PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True)
-            for i in tags:
-                tag_id_list.append(i)
+            tag_id_list = list(PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
 
             return tag_id_list
-
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
     def get_tag_by_name(self, tag_id):
         try:
-            tag_name_list = []
-            tags = Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True)
-            for i in tags:
-                tag_name_list.append(i)
+            tag_name_list = list(Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True))
 
             return tag_name_list
-
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
 
 class PictorialTag(models.Model):

trans2es/models/topic.py

@@ -130,6 +130,20 @@ class Topic(models.Model):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return False
 
+    @classmethod
+    def get_topic_image_num(cls,topic_id):
+        """
+        :remark 获取指定帖子的图片数量
+        :param topic_id:
+        :return:
+        """
+        try:
+            query_list = list(TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False, is_online=True).values_list("url",flat=True))
+            return len(query_list)
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return 0
+
     def topic_has_image(self):
         try:
             has_image = False

trans2es/utils/pictorial_transfer.py

@@ -5,13 +5,39 @@ import sys
 import logging
 import traceback
 from libs.tools import tzlc
-
+from trans2es.models.topic import Topic
 
 class PictorialTransfer(object):
 
     def __init__(self):
         pass
 
+    @classmethod
+    def get_offline_score(cls,instance,topic_id_list):
+        try:
+            total_offline_score = 0
+            topic_image_num = 0
+            for topic_id in topic_id_list:
+                topic_image_num += Topic.get_topic_image_num(topic_id)
+
+            if topic_image_num>=6 and topic_image_num<=10:
+                total_offline_score += 30
+            elif topic_image_num>10 and topic_image_num<=20:
+                total_offline_score += 60
+            elif topic_image_num>20 and topic_image_num<=50:
+                total_offline_score += 80
+            elif topic_image_num>50:
+                total_offline_score += 100
+
+            total_offline_score += instance.add_score
+            if instance.is_recommend:
+                total_offline_score += 100
+
+            return total_offline_score
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return 0
+
     @classmethod
     def get_poctorial_data(cls, instance):
         try:
@@ -33,12 +59,12 @@ class PictorialTransfer(object):
             tzlc_udpate_time = tzlc(update_time)
             res["update_time"] = tzlc_udpate_time
             res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
-            tag_id = instance.get_tag_by_id()
-            res["tag_id"] = tag_id
-            res["tag_name"] = instance.get_tag_by_name(tag_id)
+            tag_id_list = instance.get_tag_by_id()
+            res["tag_id"] = tag_id_list
+            res["tag_name"] = instance.get_tag_by_name(tag_id_list)
             res["topic_id_list"] =instance.get_topic_id()
             res["effective"] = instance.get_effective(res["topic_id_list"])
-
+            res["offline_score"] = cls.get_offline_score(instance,res["topic_id_list"])
             return res
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/utils/topic_transfer.py

@@ -29,6 +29,7 @@ class TopicTransfer(object):
             res["content_level"] = instance.content_level
             res["user_id"] = instance.user_id
             res["user_nick_name"] = User.get_user_nick_name(instance.user_id)
+            res["user_nick_name_pre"] = res["user_nick_name"]
 
             if instance.group:
                 res["group_id"] = instance.group.id