update

.gitignore

@@ -85,4 +85,5 @@ gaia/rpcd.json
 dbmw_deploy/config.dir/
 .idea/codeStyles/Project.xml
 .idea/misc.xml
-.idea/physical.iml
\ No newline at end of file
+.idea/physical.iml
+.idea/codeStyles/Project.xml
\ No newline at end of file

.idea/codeStyles/Project.xml

@@ -24,5 +24,29 @@
         <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
       </formatting-settings>
     </DBN-SQL>
+    <DBN-PSQL>
+      <case-options enabled="false">
+        <option name="KEYWORD_CASE" value="lower" />
+        <option name="FUNCTION_CASE" value="lower" />
+        <option name="PARAMETER_CASE" value="lower" />
+        <option name="DATATYPE_CASE" value="lower" />
+        <option name="OBJECT_CASE" value="preserve" />
+      </case-options>
+      <formatting-settings enabled="false" />
+    </DBN-PSQL>
+    <DBN-SQL>
+      <case-options enabled="false">
+        <option name="KEYWORD_CASE" value="lower" />
+        <option name="FUNCTION_CASE" value="lower" />
+        <option name="PARAMETER_CASE" value="lower" />
+        <option name="DATATYPE_CASE" value="lower" />
+        <option name="OBJECT_CASE" value="preserve" />
+      </case-options>
+      <formatting-settings enabled="false">
+        <option name="STATEMENT_SPACING" value="one_line" />
+        <option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
+        <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
+      </formatting-settings>
+    </DBN-SQL>
   </code_scheme>
 </component>
\ No newline at end of file

.idea/misc.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

.idea/physical.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 3.6 (venv)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

app_conf.xml

@@ -17,6 +17,5 @@
         <element value="search.views.contrast_similar"/>
         <element value="injection.data_sync.tasks"/>
         <element value="search.views.contrast_similar"/>
-        <!--<element value="majia..tasks.tasks"/>-->
     </config>
 </gm_rpcd_config>

injection/data_sync/tasks.py

@@ -2,6 +2,10 @@
 import logging
 import traceback
 import json
+import pymysql
+import threading
+import random
+import datetime
 from celery import shared_task
 from django.conf import settings
 from django.core import serializers
@@ -9,9 +13,10 @@ from trans2es.type_info import get_type_info_map
 # from rpc.all import get_rpc_remote_invoker
 from libs.es import ESPerform
 from libs.cache import redis_client
-from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
+from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
 from linucb.utils.register_user_tag import RegisterUserTag
 
+
 @shared_task
 def write_to_es(es_type, pk_list, use_batch_query_set=False):
     try:
@@ -55,35 +60,47 @@ def sync_face_similar_data_to_redis():
 
             item_list = list()
             for item in similar_result_items:
+                weight_score = int(item.similarity * 100)
                 item_list.append(
                     {
-                        "contrast_user_id": item.contrast_user_id,
-                        "similarity": item.similarity
+                        "filter":{
+                            "constant_score":{
+                                "filter":{
+                                    "term":{"user_id": item.contrast_user_id}
+                                }
+                            }
+                        },
+                        "weight": weight_score*2
                     }
                 )
+                if len(item_list)>=100:
+                    break
             redis_client.set(redis_key, json.dumps(item_list))
 
             logging.info("participant_user_id:%d set data done!" % participant_user_id)
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
+
 @shared_task
 def sync_user_similar_score():
     try:
-        results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True)
+        results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
+            "user_id", flat=True)
         redis_key_prefix = "physical:user_similar_score:user_id:"
 
         logging.info("begin sync user_similar_score!")
         for user_id in results_items:
             redis_key = redis_key_prefix + str(user_id)
-            similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score")
+            similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
+                "-score")
 
             item_list = list()
             for item in similar_results_items:
                 contrast_user_id = item.contrast_user_id
                 score = item.score
                 item_list.append(
-                    [contrast_user_id,score]
+                    [contrast_user_id, score]
                 )
 
             redis_client.set(redis_key, json.dumps(item_list))

libs/error.py

@@ -18,4 +18,4 @@ def logging_exception(send_to_sentry=True):
         # send exception info to sentry, fail silently
         _sentry_client.captureException()
     except:
-        pass
+        passraven.contrib.django.raven_compat.models

libs/es.py

@@ -366,3 +366,78 @@ class ESPerform(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return list()
+
+    @classmethod
+    def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
+        try:
+            functions_list = list()
+            for id in tag_id:
+                functions_list.append(
+                    {
+                        "filter": {"term": {"tag_list": id}},
+                        "weight": 1
+                    }
+                )
+            # functions_list += [
+            #     {
+            #         "filter": {"term": {"content_level": 6}},
+            #         "weight": 6000
+            #     },
+            #     {
+            #         "filter": {"term": {"content_level": 5}},
+            #         "weight": 5000
+            #     },
+            #     {
+            #         "filter": {"term": {"content_level": 4}},
+            #         "weight": 4000
+            #     }
+            # ]
+            q = {
+                "query": {
+                    "function_score": {
+                        "query": {
+                            "bool": {
+                                "must": [
+                                    {"term": {"content_level": 6}},
+                                    {"term": {"is_online": True}},
+                                    {"term": {"is_deleted": False}},
+                                    {"terms": {"tag_list": tag_id}}
+                                ]
+                            }
+                        },
+                        "boost_mode": "sum",
+                        "score_mode": "sum",
+                        "functions": functions_list
+                    }
+                },
+                "_source": {
+                    "include": ["id","user_id"]
+                },
+                "sort": [
+                    {"_score": {"order": "desc"}},
+                    {"create_time_val": {"order": "desc"}},
+                    {"language_type": {"order": "asc"}},
+                ]
+            }
+            if len(have_read_topic_id_list) > 0:
+                q["query"]["function_score"]["query"]["bool"]["must_not"] = {
+                    "terms": {
+                        "id": have_read_topic_id_list
+                    }
+                }
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
+                                                       offset=0, size=size, routing="6")
+
+            topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
+            # logging.info("topic_id_list:%s" % str(topic_id_list))
+            # topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
+            topic_id_dict = dict()
+            for item in result_dict["hits"]:
+                topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
+
+            logging.info("topic_id_list:%s" % str(topic_id_dict))
+
+            return topic_id_list,topic_id_dict
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return list()

majia/__init__.py

-default_app_config = 'majia.apps.MaJiaApp'

majia/admin.py

-from django.contrib import admin
-
-# Register your models here.

majia/apps.py

-# coding=utf-8
-from __future__ import unicode_literals, print_function, absolute_import
-
-from django.apps import AppConfig
-
-
-class MaJiaApp(AppConfig):
-    name = 'majia'
-    label = 'majia'

majia/get_session.py

-import requests
-import time
-import datetime
-import random
-import traceback
-from django.conf import settings
-import logging
-
-DATABASES = settings.DATABASES
-
-# host = DATABASES['default']['HOST']
-#
-# user = DATABASES['default']['USER']
-#
-# port = DATABASES['default']['PORT']
-#
-# db = DATABASES['default']['NAME']
-#
-# passwd = DATABASES['default']['PASSWORD']
-
-auto_click_url = "http://earth.igengmei.com/api/v1/like"
-auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
-auto_follow_url = "http://earth.igengmei.com/api/v1/follow"
-auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
-
-
-def get_majia():
-    try:
-        data = open("/srv/apps/physical/majia/majia_email.txt")
-        list = []
-        for i in data:
-            list.append(i.strip('\n').strip(','))
-        maj = random.randint(1, len(list))
-        user_id = list[maj - 1]
-        return user_id
-
-    except:
-        logging.error("catch exception,get_majia:%s" % traceback.format_exc())
-        return None
-
-
-def get_cookies(user_id):
-    try:
-        with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
-            try:
-                data = f.readlines()
-            except:
-                data = None
-
-        list = []
-        for i in data:
-            list.append(i.strip('\n').split(","))
-
-        for i in list:
-
-            if user_id == i[0]:
-                return (i[1])
-    except:
-
-        return None
-
-
-def login():
-    try:
-        user_id = get_majia()
-
-        cookies = get_cookies(user_id)
-
-        if cookies is not None:
-            return {'sessionid': cookies}
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-        return None
-
-
-def click(cookies_get, id):
-    try:
-        post_dict = {
-            'type': 0,
-            'id': id
-        }
-        response = requests.post(url=auto_click_url,
-                                 cookies=cookies_get,
-                                 data=post_dict)
-
-        logging.info("response.text:%s" % response.text)
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-
-def reply(cookies_get, id, content):
-    try:
-        post_dict = {
-            'topic_id': id,
-            'content': content
-        }
-        response = requests.post(url=auto_reply_url,
-                                 cookies=cookies_get,
-                                 data=post_dict)
-
-        logging.info("response.text:%s" % response.text)
-
-    except:
-
-        logging.error("catch exception,logins:%s" % traceback.format_exc())
-
-
-def time_conv(minutest):
-    try:
-        now = datetime.datetime.now()
-        yes_time = now - datetime.timedelta(minutes=minutest)
-        print(yes_time)
-        return yes_time
-
-    except:
-        return None
-
-
-def get_comments():
-    try:
-        data = open("/srv/apps/physical/majia/guanshui.txt", "r")
-        list_guanshui = []
-        for i in data:
-            list_guanshui.append(i)
-        num1 = random.randint(0, 30)
-        num2 = random.randint(30, 70)
-        num3 = random.randint(70, len(list_guanshui))
-
-        comment1 = list_guanshui[num1 - 1]
-        comment2 = list_guanshui[num2 - 1]
-        comment3 = list_guanshui[num3 - 1]
-
-        comment_list = [comment1, comment2, comment3]
-        return comment_list
-
-    except:
-
-        return None

majia/guanshui.txt

-照片色调很好看
-用的什么调色工具呀
-立马去get同款
-我一个一看就会，动手就废的girl
-为什么我拍不出来你的效果？
-Po主用的什么拍照软件
-这个好，学习了
-真的是元气满满
-很帅气！
-这样好显气质！
-好可爱啊，完全没有抵抗力了。
-莫名好感中
-满分颜值
-超酷！
-马住回看
-请问发色是啥
-请问是什么app拍的啊
-滤镜要怎么选？
-卖萌打滚求翻牌
-有没有常用自拍姿势教程
-拍照技术满分
-日常夸奖博主
-日常勾搭
-背景是在哪里？
-照片构图很好看
-主要还是看颜值
-你长得很像明星
-支持一下
-转需留存
-拍照表情很自然
-风格很好
-这里是哪里呀？
-种草了！
-搭配小技巧get！
-怎么都学不会的路过
-太好了，新手学习了
-一直支持，只求被翻牌！
-好喜欢这种风格
-好看，在哪儿买的
-衣好有气质哦
-这个我也会
-拍照pose求出教程
-小姐姐，作图app是啥
-终于更新了
-搭配的好好看
-求同款
-日常催更
-get√
-什么时候更新啊
-果然只要瘦怎么都好看。
-不错的哈哈
-同款达不到楼主的效果
-喜欢你的搭配
-我拍出来为什么感觉比本人丑很多？
-能多发点全身正面照吗？想看整体效果
-这个真的很真实了
-大圆脸合适吗？
-成功引起了我的注意！
-感觉很少女~
-不错，继续加油
-好看的人做什么都好看
-都是人才！！！！
-道理我都懂，就差一个脸！
-不赞我良心都过意不去啊
-爱你哟~
-用手机可以拍出来吗？
-使用相机拍的还是手机
-我觉得很赞
-颜色搭配很舒服
-小姐姐有戴美瞳吗
-看见这个帖心情变好了
-不管怎么样先收藏了
-更新更新更新
-你和我朋友长得有点像
-你的风格我都很喜欢
-好看
-我要有你这条件还需要啥拍照技术
-太美了！
-被圈粉了……
-美而不作，我喜欢
-楼主真的real真实了。
-持续等更新
-楼主加油，照片拍的不错，非常值得推荐。
-大家花呗还了吗？
-来啦来啦来啦来啦来啦
-画面太治愈了
-楼主辛苦啦
-不错，跟着学习一下
-心动，我要去试试
-满满的干货，收藏好下次用
-新技能get√
-感觉不错mark一下
-适合自己的才是最重要的
-羡慕一下作者的颜
-Omaga太好看了吧~
-这张拍的不错
-这个特效挺可爱的
-翻我翻我~
-这样很显脸小
-关注啦

majia/models.py

-from django.db import models
-
-# Create your models here.

majia/tasks.py

-# -*- coding: UTF-8 -*-
-
-import pymysql
-import random
-import threading
-import traceback
-import logging
-from celery import shared_task
-from django.conf import settings
-from majia.get_session import get_comments, click, login, reply
-# from majia.get_session import host, user, passwd, db
-
-exec_count = 0
-
-def auto_click(pk_list):
-
-    logging.info("get----------------------:%s"%pk_list)
-    # exec_count = 0
-    # try:
-    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
-    #     cursor = pc.cursor()
-    #     cursor.execute(
-    #         "SELECT id,user_id FROM topic WHERE id  in  " + str(
-    #             tuple(pk_list)) + "  and user_id in (select user_id from user_extra  where is_shadow = 0)")
-    #     data = cursor.fetchall()
-    #     topic_id_list = list(data)
-    #     logging.info("Database version : %s " % topic_id_list)
-    #     pc.close()
-    #     exec_count = 0
-    #     if topic_id_list:
-    #
-    #         def fun_timer():
-    #             cookies = login()
-    #             if cookies:
-    #                 click(cookies, topic_id_list[0])
-    #
-    #             global timer
-    #             global exec_count
-    #
-    #             exec_count += 1
-    #             if exec_count == 1:
-    #                 # sleep_time = random.randint(300, 540)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_timer)
-    #                 timer.start()
-    #
-    #             if exec_count == 2:
-    #                 # sleep_time = random.randint(1000, 1900)
-    #                 sleep_time = 50
-    #                 timer = threading.Timer(sleep_time, fun_timer)
-    #                 timer.start()
-    #
-    #         sleep_time = random.randint(300, 540)
-    #         timer = threading.Timer(10, fun_timer)  # 首次启动
-    #         timer.start()
-    #
-    #     else:
-    #         pass
-    # except:
-    #     logging.error("catch exception,main:%s" % traceback.format_exc())
-
-
-def auto_reply(pk_list):
-
-    logging.info("get----------------------:%s"%pk_list)
-
-    # exec_count = 0
-    # try:
-    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
-    #     cursor = pc.cursor()
-    #     cursor.execute(
-    #         "SELECT id,user_id FROM topic WHERE id in  " + str(
-    #             tuple(pk_list)) + " and user_id in (select user_id from user_extra  where is_shadow = 0)")
-    #     data = cursor.fetchall()
-    #     topic_id_list = list(data)
-    #     logging.info("Database version : %s " % topic_id_list)
-    #     pc.close()
-    #     exec_count = 0
-    #     if topic_id_list:
-    #
-    #         def fun_comment():
-    #             cookies = login()
-    #             if cookies:
-    #                 comment_content = get_comments()
-    #                 comment = comment_content[0]
-    #                 reply(cookies, topic_id_list[0], comment)
-    #
-    #             global timer
-    #             global exec_count
-    #
-    #             exec_count += 1
-    #             if exec_count == 1:
-    #                 sleep_time = random.randint(300, 540)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_comment)
-    #                 timer.start()
-    #
-    #             if exec_count == 2:
-    #                 sleep_time = random.randint(1000, 1900)
-    #                 sleep_time = 30
-    #                 timer = threading.Timer(sleep_time, fun_comment)
-    #                 timer.start()
-    #
-    #         sleep_time = random.randint(300, 540)
-    #         timer = threading.Timer(10, fun_comment)  # 首次启动
-    #         timer.start()
-    #
-    #
-    # except:
-    #     logging.error("catch exception,main:%s" % traceback.format_exc())
-
-
-@shared_task
-def auto_majia_effect(pk_list):
-    auto_click(pk_list)
-    auto_reply(pk_list)

majia/tests.py

-from django.test import TestCase
-
-# Create your tests here.

majia/views.py

-from django.shortcuts import render
-
-# Create your views here.

physical/celery_task_router.py

@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
     queue_task_map = {
         "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
-        ],
-        "majia-alpha": [
-            'majia.tasks.auto_majia_effect',
-        ],
+        ]
     }
 
     # Map[TaskName, QueueName]

physical/settings_local.py.template

@@ -48,7 +48,6 @@ INSTALLED_APPS = (
     'trans2es',
     'search',
     'injection.data_sync',
-    'majia',
 )
 
 CELERYBEAT_SCHEDULE = {

search/utils/group.py

@@ -47,23 +47,79 @@ class GroupUtils(object):
             return {"total_count": 0, "hits": []}
 
     @classmethod
-    def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None):
+    def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None,attention_tag_list=[]):
         try:
             if not es_cli_obj:
                 es_cli_obj = ESPerform.get_cli()
 
-            q = dict()
-            q["query"] = {
-                "bool":{
-                    "must":[
-                        {"term": {"is_online": True}},
-                        {"term":{"is_deleted": False}},
-                        {"term": {"effective": True}}
-                    ]
+            functions_list = list()
+            for tag_id in attention_tag_list:
+                functions_list.append({
+                    "filter":{
+                        "constant_score":{
+                            "filter":{
+                                "term":{
+                                    "tag_id": tag_id
+                                }
+                            }
+                        }
+                    },
+                    "weight": 20
+                })
+
+                if len(functions_list)>=20:
+                    break
+
+            functions_list.append(
+                {
+                    "gauss": {
+                        "create_time": {
+                            "scale": "1d",
+                            "decay": 0.99
+                        }
+                    },
+                    "weight": 60
+                }
+            )
+            q = {
+                "query":{
+                    "function_score":{
+                        "query":{
+                            "bool":{
+                                "filter":[
+                                    {"term": {"is_online": True}},
+                                    {"term": {"is_deleted": False}},
+                                    {"term": {"effective": True}}
+                                ],
+                                "must_not":[
+                                    {"term":{"is_default":1}}
+                                ]
+                            }
+                        },
+                        "score_mode": "sum",
+                        "boost_mode": "sum",
+                        "functions": functions_list
+                    }
                 }
             }
             q["sort"] = [
-                {"high_quality_topic_num":{"order":"desc"}}
+                {
+                    "_script": {
+                        "type": "number",
+                        "script": {
+                            "lang": "expression",
+                            "source": "_score+doc['offline_score']"
+                            # "lang":"painless",
+                            # "source":"_score+params._source.offline_score"
+                        },
+                        "order": "desc"
+                    }
+                },
+                {
+                    "_score":{
+                        "order": "desc"
+                    }
+                }
             ]
             q["_source"] = {
                 "includes":["id"]

search/views/business_topic.py

@@ -25,7 +25,6 @@ def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=
             index_name="topic"
         )
 
-        logging.info("get result_lsit:%s"%result_list)
         topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
         return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
     except:

search/views/group.py

@@ -52,9 +52,15 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
         es_cli_obj = ESPerform.get_cli()
 
         if sort_type == GroupSortTypes.HOT_RECOMMEND:
-            pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj)
-            return {"pictorial_recommend_ids": pictorial_ids_list}
+            # 用户关注标签
+            attention_tag_list = list()
+            if user_id>0:
+                redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
+                attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
+
+            pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,attention_tag_list)
 
+            return {"pictorial_recommend_ids": pictorial_ids_list}
         elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
             attention_pictorial_list = GroupUtils.get_user_attention_pictorial_list(user_id, offset=0, size=1,
                                                                                     es_cli_obj=es_cli_obj)

trans2es/mapping/pictorial.json

@@ -16,6 +16,8 @@
     "tag_id":{"type":"long"},
     "tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
     "topic_id_list":{"type":"long"},
-    "effective":{"type":"boolean"}
+    "effective":{"type":"boolean"},
+    "offline_score":{"type":"long"},
+    "is_default":{"type":"long"}
   }
 }
\ No newline at end of file

trans2es/mapping/topic-high-star.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/mapping/topic-star-routing.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/mapping/topic-star.json

 {
   "dynamic":"strict",
+  "_routing": {"required": true},
   "properties": {
     "id":{"type":"long"},
-    "content_level":{"type":"text"}
+    "is_online":{"type":"boolean"},//上线
+    "is_deleted":{"type":"boolean"},
+    "vote_num":{"type":"long"},
+    "total_vote_num":{"type":"long"},
+    "reply_num":{"type":"long"},
+    "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content_level":{"type":"text"},
+    "user_id":{"type":"long"},
+    "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","index":"not_analyzed"}, //不切词的用户名
+    "group_id":{"type":"long"}, //所在组ID
+    "tag_list":{"type":"long"},//标签属性
+    "edit_tag_list":{"type":"long"},//编辑标签
+    "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "share_num":{"type":"long"},
+    "pick_id_list":{"type":"long"},
+    "offline_score":{"type":"double"},//离线算分
+    "manual_score":{"type":"double"},//人工赋分
+    "has_image":{"type":"boolean"},//是否有图
+    "has_video":{"type":"boolean"},//是否是视频
+    "create_time":{"type":"date", "format":"date_time_no_millis"},
+    "update_time":{"type":"date", "format":"date_time_no_millis"},
+    "create_time_val":{"type":"long"},
+    "update_time_val":{"type":"long"},
+    "language_type":{"type":"long"},
+
+    "is_shadow": {"type": "boolean"},
+    "is_recommend": {"type": "boolean"},
+    "is_complaint": {"type": "boolean"},  // 是否被举报
+
+    "virtual_content_level":{"type": "text"},
+    "like_num_crawl": {"type": "long"}, // 爬取点赞数
+    "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+    "is_crawl": {"type": "boolean"},
+    "platform": {"type": "long"},
+    "platform_id": {"type": "long"},
+
+    "drop_score":{"type": "double"},  // 人工降分
+    "sort_score":{"type": "double"},  // 排序分
+
+    "pictorial_id":{"type": "long"}, //所在组ID
+    "pictorial_name":{  // 所在组名称
+        "type": "text",
+        "analyzer": "gm_default_index",
+        "search_analyzer": "gm_default_index"
+    },
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic-v1.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签
@@ -52,6 +53,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic.json

@@ -14,6 +14,7 @@
     "content_level":{"type":"text"},
     "user_id":{"type":"long"},
     "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
+    "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
     "group_id":{"type":"long"}, //所在组ID
     "tag_list":{"type":"long"},//标签属性
     "edit_tag_list":{"type":"long"},//编辑标签

trans2es/models/pictorial.py

@@ -36,7 +36,6 @@ class PictorialTopics(models.Model):
     pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
     topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
     is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
-    is_online = models.BooleanField(verbose_name=u'是否上线')
     is_deleted = models.BooleanField(verbose_name=u'是否删除')
 
 
@@ -59,18 +58,16 @@ class Pictorial(models.Model):
     creator_id = models.BigIntegerField(verbose_name=u'画报用户ID')
     icon = models.CharField(verbose_name=u'画报名称', max_length=255)
     topic_num = models.IntegerField(verbose_name=u'次数')
+    add_score = models.IntegerField(verbose_name=u'人工权重')
+    is_default = models.IntegerField(verbose_name=u"是否是默认画报")
 
     def get_topic_id(self):
         try:
-            topic_id =[]
-            topic_id_list = PictorialTopics.objects.filter(pictorial_id=self.id).values_list("topic_id", flat=True)
-            for i in topic_id_list:
-                topic_id.append(i)
-            return topic_id
-
+            topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=self.id,is_online=True,is_deleted=False).values_list("topic_id", flat=True))
+            return topic_id_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
 
     def get_effective(self,topic_id_list):
@@ -112,29 +109,21 @@ class Pictorial(models.Model):
 
     def get_tag_by_id(self):
         try:
-            tag_id_list = []
-            tags = PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True)
-            for i in tags:
-                tag_id_list.append(i)
+            tag_id_list = list(PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
 
             return tag_id_list
-
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
     def get_tag_by_name(self, tag_id):
         try:
-            tag_name_list = []
-            tags = Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True)
-            for i in tags:
-                tag_name_list.append(i)
+            tag_name_list = list(Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True))
 
             return tag_name_list
-
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return False
+            return []
 
 
 class PictorialTag(models.Model):

trans2es/models/topic.py

@@ -130,6 +130,20 @@ class Topic(models.Model):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return False
 
+    @classmethod
+    def get_topic_image_num(cls,topic_id):
+        """
+        :remark 获取指定帖子的图片数量
+        :param topic_id:
+        :return:
+        """
+        try:
+            query_list = list(TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False, is_online=True).values_list("url",flat=True))
+            return len(query_list)
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return 0
+
     def topic_has_image(self):
         try:
             has_image = False
@@ -213,10 +227,6 @@ class Topic(models.Model):
                 elif user_query_results[0].is_shadow:
                     user_is_shadow = True
 
-            # 是否官方推荐小组
-            # if self.group and self.group.is_recommend:
-            #     offline_score += 4.0
-
             # 帖子等级
             if self.content_level == '5':
                 offline_score += 100.0 *3
@@ -225,20 +235,8 @@ class Topic(models.Model):
             elif self.content_level == '6':
                 offline_score += 200.0 *3
 
-            # is_excellent = self.judge_if_excellent_topic(self.id)
-            # if is_excellent:
-            #     offline_score += 200.0
-
             if self.language_type == 1:
                 offline_score += 60.0
-            # exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
-            # click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
-            # uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
-            #
-            # if exposure_count > 0:
-            #     offline_score += click_count / exposure_count
-            # if uv_num > 0:
-            #     offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
 
             """
                 1:马甲账号是否对总分降权？

trans2es/type_info.py

@@ -161,20 +161,20 @@ class TypeInfo(object):
                             ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
                             if ori_topic_star:
                                 ori_topic_star = str(ori_topic_star, encoding="utf-8")
-                            else:
-                                q = {
-                                    "query":{
-                                        "term":{
-                                            "id": data["id"]
-                                        }
-                                    },
-                                    "_source":{
-                                        "includes":["content_level"]
-                                    }
-                                }
-                                search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
-                                if len(search_results["hits"]) > 0:
-                                    ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
+                            # else:
+                            #     q = {
+                            #         "query":{
+                            #             "term":{
+                            #                 "id": data["id"]
+                            #             }
+                            #         },
+                            #         "_source":{
+                            #             "includes":["content_level"]
+                            #         }
+                            #     }
+                            #     search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
+                            #     if len(search_results["hits"]) > 0:
+                            #         ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
 
                             if not ori_topic_star:
                                 # data_list = [

trans2es/utils/pictorial_transfer.py

@@ -5,13 +5,39 @@ import sys
 import logging
 import traceback
 from libs.tools import tzlc
-
+from trans2es.models.topic import Topic
 
 class PictorialTransfer(object):
 
     def __init__(self):
         pass
 
+    @classmethod
+    def get_offline_score(cls,instance,topic_id_list):
+        try:
+            total_offline_score = 0
+            topic_image_num = 0
+            for topic_id in topic_id_list:
+                topic_image_num += Topic.get_topic_image_num(topic_id)
+
+            if topic_image_num>=6 and topic_image_num<=10:
+                total_offline_score += 30
+            elif topic_image_num>10 and topic_image_num<=20:
+                total_offline_score += 60
+            elif topic_image_num>20 and topic_image_num<=50:
+                total_offline_score += 80
+            elif topic_image_num>50:
+                total_offline_score += 100
+
+            total_offline_score += instance.add_score
+            if instance.is_recommend:
+                total_offline_score += 100
+
+            return total_offline_score
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return 0
+
     @classmethod
     def get_poctorial_data(cls, instance):
         try:
@@ -33,12 +59,13 @@ class PictorialTransfer(object):
             tzlc_udpate_time = tzlc(update_time)
             res["update_time"] = tzlc_udpate_time
             res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
-            tag_id = instance.get_tag_by_id()
-            res["tag_id"] = tag_id
-            res["tag_name"] = instance.get_tag_by_name(tag_id)
+            tag_id_list = instance.get_tag_by_id()
+            res["tag_id"] = tag_id_list
+            res["tag_name"] = instance.get_tag_by_name(tag_id_list)
             res["topic_id_list"] =instance.get_topic_id()
             res["effective"] = instance.get_effective(res["topic_id_list"])
-
+            res["offline_score"] = cls.get_offline_score(instance,res["topic_id_list"])
+            res["is_default"] = instance.is_default
             return res
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/utils/topic_transfer.py

@@ -29,6 +29,10 @@ class TopicTransfer(object):
             res["content_level"] = instance.content_level
             res["user_id"] = instance.user_id
             res["user_nick_name"] = User.get_user_nick_name(instance.user_id)
+            try:
+                res["user_nick_name_pre"] = res["user_nick_name"].lower()
+            except:
+                res["user_nick_name_pre"] = ""
 
             if instance.group:
                 res["group_id"] = instance.group.id