update

.gitignore

@@ -83,3 +83,6 @@ gaia/rpcd.json
 *.swp
 
 dbmw_deploy/config.dir/
+.idea/codeStyles/Project.xml
+.idea/misc.xml
+.idea/physical.iml
\ No newline at end of file

.idea/misc.xml

 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

.idea/physical.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.6 (venv)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

app_conf.xml

@@ -17,5 +17,6 @@
         <element value="search.views.contrast_similar"/>
         <element value="injection.data_sync.tasks"/>
         <element value="search.views.contrast_similar"/>
+        <!--<element value="majia..tasks.tasks"/>-->
     </config>
 </gm_rpcd_config>

injection/data_sync/tasks.py

 # -*- coding: UTF-8 -*-
-
+import logging
+import traceback
+import json
 from celery import shared_task
 from django.conf import settings
 from django.core import serializers
 from trans2es.type_info import get_type_info_map
 # from rpc.all import get_rpc_remote_invoker
 from libs.es import ESPerform
-import logging
-import traceback
 from libs.cache import redis_client
 from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
-import json
 from linucb.utils.register_user_tag import RegisterUserTag
 
 @shared_task

linucb/utils/register_user_tag.py

@@ -29,8 +29,12 @@ class RegisterUserTag(object):
     linucb_device_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:device_id:"
     linucb_user_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:user_id:"
 
+    # 用户关注标签
     linucb_register_user_tag_key = "physical:linucb:register_user_tag_info"
-
+    # # 用户关注标签
+    # linucb_user_attention_tag_key = "physical:linucb:user_attention_tag_info"
+    # # 用户喜好标签（注册勾选 + 关注）
+    # linucb_user_like_tag_key = "physical:linucb:user_like_tag_info"
     @classmethod
     def get_user_attention_tag(cls, pk_list):
         """
@@ -45,14 +49,17 @@ class RegisterUserTag(object):
                 tag_id = item.tag_id
                 user_id = item.user_id
 
-                user_tag_list = CommunityTagFollow.objects.filter(user=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True)
+                user_tag_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True))
                 user_id_dict[user_id] = user_tag_list
 
+                user_register_tag_list = list(AccountUserTag.objects.filter(user=user_id,is_deleted=False).values_list("tag_id", flat=True))
+                user_id_dict[user_id].extend(user_register_tag_list)
+
             for user_id in user_id_dict:
-                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
-                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
-                redis_user_tag_id_list.extend(user_id_dict[user_id])
-                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
+                # redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                # redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                # redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(user_id_dict[user_id]))))
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
@@ -65,60 +72,22 @@ class RegisterUserTag(object):
         :return:
         """
         try:
-            # user_id_set = set()
             user_id_dict = dict()
             query_results = AccountUserTag.objects.filter(pk__in=pk_list)
             for item in query_results:
                 tag_id = item.tag_id
                 user_id = item.user
-                user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id", flat=True)
+                user_tag_list = list(AccountUserTag.objects.filter(user=user_id).values_list("tag_id", flat=True))
                 user_id_dict[user_id] = user_tag_list
 
+                user_follow_tag_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True))
+                user_id_dict[user_id].extend(user_follow_tag_list)
+
             for user_id in user_id_dict:
-                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
-                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
-                redis_user_tag_id_list.extend(user_id_dict[user_id])
-                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
+                # redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                # redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                # redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(user_id_dict[user_id]))))
 
-        #         if user_id not in user_id_set:
-        #             user_id_set.add(user_id)
-        #
-        #             user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
-        #
-        #             have_read_topic_id_list = Tools.get_have_read_topic_id_list(-1, user_id,
-        #                                                                         TopicPageType.HOME_RECOMMEND)
-        #             recommend_topic_id_list = list()
-        #             cycle_num = int(10000/len(user_tag_list))
-        #             for index in range(0,cycle_num):
-        #                 for tag_id in user_tag_list:
-        #                     redis_tag_id_key = cls.tag_topic_id_redis_prefix + str(tag_id)
-        #                     redis_tag_id_data = redis_client.get(redis_tag_id_key)
-        #                     tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
-        #                     if not redis_tag_id_data:
-        #                         tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
-        #                         redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
-        #                         redis_client.expire(redis_tag_id_key,1*24*60*60)
-        #
-        #                     if len(tag_topic_id_list)>index:
-        #                         for topic_id in tag_topic_id_list[index:]:
-        #                             if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list:
-        #                                 recommend_topic_id_list.append(topic_id)
-        #                                 break
-        #
-        #             redis_register_tag_topic_data = {
-        #                 "data": json.dumps(recommend_topic_id_list),
-        #                 "cursor": 0
-        #             }
-        #             redis_client.hmset(cls.linucb_user_id_register_tag_topic_id_prefix,redis_register_tag_topic_data)
-        #             redis_client.expire(cls.linucb_user_id_register_tag_topic_id_prefix,30*24*60*60)
-        #
-        #             topic_recommend_redis_key = cls.linucb_user_id_recommend_topic_id_prefix + str(user_id)
-        #             redis_data_dict = {
-        #                 "data": json.dumps(recommend_topic_id_list),
-        #                 "cursor":0
-        #             }
-        #             redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
-        #             redis_client.expire(topic_recommend_redis_key,30*24*60*60)
-        #
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

linucb/views/collect_data.py

@@ -103,12 +103,9 @@ class CollectData(object):
 
                 if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
                     have_read_topic_id_list.extend(redis_topic_list[:2])
-                if len(tag_id_list) > 0:
-                 if len(new_user_click_tag_list)>0:
+                if len(new_user_click_tag_list)>0:
                      tag_topic_id_list = ESPerform.get_tag_topic_list(new_user_click_tag_list, have_read_topic_id_list)
-                     logging.warning("tag_topic_id_list:%s" % str(new_user_click_tag_list))
-                     logging.warning("tag_id_list:%s" % str(tag_id_list))
-                 else:
+                else:
                      tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
 
 
@@ -251,9 +248,7 @@ class CollectData(object):
                                 else:
                                     tagid_list = list()
                                 logging.warning("unknown type msg:%s" % raw_val_dict.get("type", "missing type"))
-                                logging.info(
-                                    "consume click topic_id:%s,device_id:%s" % (
-                                    str(tagid_list), str(device_id)))
+
                                 device_id = raw_val_dict["device"]["device_id"]
                                 user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
 

majia/__init__.py

+default_app_config = 'majia.apps.MaJiaApp'

majia/admin.py

+from django.contrib import admin
+
+# Register your models here.

majia/apps.py

+# coding=utf-8
+from __future__ import unicode_literals, print_function, absolute_import
+
+from django.apps import AppConfig
+
+
+class MaJiaApp(AppConfig):
+    name = 'majia'
+    label = 'majia'

majia/get_session.py

+import requests
+import time
+import datetime
+import random
+import traceback
+from django.conf import settings
+import logging
+
+DATABASES = settings.DATABASES
+
+# host = DATABASES['default']['HOST']
+#
+# user = DATABASES['default']['USER']
+#
+# port = DATABASES['default']['PORT']
+#
+# db = DATABASES['default']['NAME']
+#
+# passwd = DATABASES['default']['PASSWORD']
+
+auto_click_url = "http://earth.igengmei.com/api/v1/like"
+auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
+auto_follow_url = "http://earth.igengmei.com/api/v1/follow"
+auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
+
+
+def get_majia():
+    try:
+        data = open("/srv/apps/physical/majia/majia_email.txt")
+        list = []
+        for i in data:
+            list.append(i.strip('\n').strip(','))
+        maj = random.randint(1, len(list))
+        user_id = list[maj - 1]
+        return user_id
+
+    except:
+        logging.error("catch exception,get_majia:%s" % traceback.format_exc())
+        return None
+
+
+def get_cookies(user_id):
+    try:
+        with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
+            try:
+                data = f.readlines()
+            except:
+                data = None
+
+        list = []
+        for i in data:
+            list.append(i.strip('\n').split(","))
+
+        for i in list:
+
+            if user_id == i[0]:
+                return (i[1])
+    except:
+
+        return None
+
+
+def login():
+    try:
+        user_id = get_majia()
+
+        cookies = get_cookies(user_id)
+
+        if cookies is not None:
+            return {'sessionid': cookies}
+
+    except:
+
+        logging.error("catch exception,logins:%s" % traceback.format_exc())
+
+        return None
+
+
+def click(cookies_get, id):
+    try:
+        post_dict = {
+            'type': 0,
+            'id': id
+        }
+        response = requests.post(url=auto_click_url,
+                                 cookies=cookies_get,
+                                 data=post_dict)
+
+        logging.info("response.text:%s" % response.text)
+
+    except:
+
+        logging.error("catch exception,logins:%s" % traceback.format_exc())
+
+
+def reply(cookies_get, id, content):
+    try:
+        post_dict = {
+            'topic_id': id,
+            'content': content
+        }
+        response = requests.post(url=auto_reply_url,
+                                 cookies=cookies_get,
+                                 data=post_dict)
+
+        logging.info("response.text:%s" % response.text)
+
+    except:
+
+        logging.error("catch exception,logins:%s" % traceback.format_exc())
+
+
+def time_conv(minutest):
+    try:
+        now = datetime.datetime.now()
+        yes_time = now - datetime.timedelta(minutes=minutest)
+        print(yes_time)
+        return yes_time
+
+    except:
+        return None
+
+
+def get_comments():
+    try:
+        data = open("/srv/apps/physical/majia/guanshui.txt", "r")
+        list_guanshui = []
+        for i in data:
+            list_guanshui.append(i)
+        num1 = random.randint(0, 30)
+        num2 = random.randint(30, 70)
+        num3 = random.randint(70, len(list_guanshui))
+
+        comment1 = list_guanshui[num1 - 1]
+        comment2 = list_guanshui[num2 - 1]
+        comment3 = list_guanshui[num3 - 1]
+
+        comment_list = [comment1, comment2, comment3]
+        return comment_list
+
+    except:
+
+        return None

majia/guanshui.txt

+照片色调很好看
+用的什么调色工具呀
+立马去get同款
+我一个一看就会，动手就废的girl
+为什么我拍不出来你的效果？
+Po主用的什么拍照软件
+这个好，学习了
+真的是元气满满
+很帅气！
+这样好显气质！
+好可爱啊，完全没有抵抗力了。
+莫名好感中
+满分颜值
+超酷！
+马住回看
+请问发色是啥
+请问是什么app拍的啊
+滤镜要怎么选？
+卖萌打滚求翻牌
+有没有常用自拍姿势教程
+拍照技术满分
+日常夸奖博主
+日常勾搭
+背景是在哪里？
+照片构图很好看
+主要还是看颜值
+你长得很像明星
+支持一下
+转需留存
+拍照表情很自然
+风格很好
+这里是哪里呀？
+种草了！
+搭配小技巧get！
+怎么都学不会的路过
+太好了，新手学习了
+一直支持，只求被翻牌！
+好喜欢这种风格
+好看，在哪儿买的
+衣好有气质哦
+这个我也会
+拍照pose求出教程
+小姐姐，作图app是啥
+终于更新了
+搭配的好好看
+求同款
+日常催更
+get√
+什么时候更新啊
+果然只要瘦怎么都好看。
+不错的哈哈
+同款达不到楼主的效果
+喜欢你的搭配
+我拍出来为什么感觉比本人丑很多？
+能多发点全身正面照吗？想看整体效果
+这个真的很真实了
+大圆脸合适吗？
+成功引起了我的注意！
+感觉很少女~
+不错，继续加油
+好看的人做什么都好看
+都是人才！！！！
+道理我都懂，就差一个脸！
+不赞我良心都过意不去啊
+爱你哟~
+用手机可以拍出来吗？
+使用相机拍的还是手机
+我觉得很赞
+颜色搭配很舒服
+小姐姐有戴美瞳吗
+看见这个帖心情变好了
+不管怎么样先收藏了
+更新更新更新
+你和我朋友长得有点像
+你的风格我都很喜欢
+好看
+我要有你这条件还需要啥拍照技术
+太美了！
+被圈粉了……
+美而不作，我喜欢
+楼主真的real真实了。
+持续等更新
+楼主加油，照片拍的不错，非常值得推荐。
+大家花呗还了吗？
+来啦来啦来啦来啦来啦
+画面太治愈了
+楼主辛苦啦
+不错，跟着学习一下
+心动，我要去试试
+满满的干货，收藏好下次用
+新技能get√
+感觉不错mark一下
+适合自己的才是最重要的
+羡慕一下作者的颜
+Omaga太好看了吧~
+这张拍的不错
+这个特效挺可爱的
+翻我翻我~
+这样很显脸小
+关注啦

majia/models.py

+from django.db import models
+
+# Create your models here.

majia/tasks.py

+# -*- coding: UTF-8 -*-
+
+import pymysql
+import random
+import threading
+import traceback
+import logging
+from celery import shared_task
+from django.conf import settings
+from majia.get_session import get_comments, click, login, reply
+# from majia.get_session import host, user, passwd, db
+
+exec_count = 0
+
+def auto_click(pk_list):
+
+    logging.info("get----------------------:%s"%pk_list)
+    # exec_count = 0
+    # try:
+    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    #     cursor = pc.cursor()
+    #     cursor.execute(
+    #         "SELECT id,user_id FROM topic WHERE id  in  " + str(
+    #             tuple(pk_list)) + "  and user_id in (select user_id from user_extra  where is_shadow = 0)")
+    #     data = cursor.fetchall()
+    #     topic_id_list = list(data)
+    #     logging.info("Database version : %s " % topic_id_list)
+    #     pc.close()
+    #     exec_count = 0
+    #     if topic_id_list:
+    #
+    #         def fun_timer():
+    #             cookies = login()
+    #             if cookies:
+    #                 click(cookies, topic_id_list[0])
+    #
+    #             global timer
+    #             global exec_count
+    #
+    #             exec_count += 1
+    #             if exec_count == 1:
+    #                 # sleep_time = random.randint(300, 540)
+    #                 sleep_time = 30
+    #                 timer = threading.Timer(sleep_time, fun_timer)
+    #                 timer.start()
+    #
+    #             if exec_count == 2:
+    #                 # sleep_time = random.randint(1000, 1900)
+    #                 sleep_time = 50
+    #                 timer = threading.Timer(sleep_time, fun_timer)
+    #                 timer.start()
+    #
+    #         sleep_time = random.randint(300, 540)
+    #         timer = threading.Timer(10, fun_timer)  # 首次启动
+    #         timer.start()
+    #
+    #     else:
+    #         pass
+    # except:
+    #     logging.error("catch exception,main:%s" % traceback.format_exc())
+
+
+def auto_reply(pk_list):
+
+    logging.info("get----------------------:%s"%pk_list)
+
+    # exec_count = 0
+    # try:
+    #     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    #     cursor = pc.cursor()
+    #     cursor.execute(
+    #         "SELECT id,user_id FROM topic WHERE id in  " + str(
+    #             tuple(pk_list)) + " and user_id in (select user_id from user_extra  where is_shadow = 0)")
+    #     data = cursor.fetchall()
+    #     topic_id_list = list(data)
+    #     logging.info("Database version : %s " % topic_id_list)
+    #     pc.close()
+    #     exec_count = 0
+    #     if topic_id_list:
+    #
+    #         def fun_comment():
+    #             cookies = login()
+    #             if cookies:
+    #                 comment_content = get_comments()
+    #                 comment = comment_content[0]
+    #                 reply(cookies, topic_id_list[0], comment)
+    #
+    #             global timer
+    #             global exec_count
+    #
+    #             exec_count += 1
+    #             if exec_count == 1:
+    #                 sleep_time = random.randint(300, 540)
+    #                 sleep_time = 30
+    #                 timer = threading.Timer(sleep_time, fun_comment)
+    #                 timer.start()
+    #
+    #             if exec_count == 2:
+    #                 sleep_time = random.randint(1000, 1900)
+    #                 sleep_time = 30
+    #                 timer = threading.Timer(sleep_time, fun_comment)
+    #                 timer.start()
+    #
+    #         sleep_time = random.randint(300, 540)
+    #         timer = threading.Timer(10, fun_comment)  # 首次启动
+    #         timer.start()
+    #
+    #
+    # except:
+    #     logging.error("catch exception,main:%s" % traceback.format_exc())
+
+
+@shared_task
+def auto_majia_effect(pk_list):
+    auto_click(pk_list)
+    auto_reply(pk_list)

majia/tests.py

+from django.test import TestCase
+
+# Create your tests here.

majia/views.py

+from django.shortcuts import render
+
+# Create your views here.

physical/celery_task_router.py

 # coding=utf-8                                                                           
 from __future__ import unicode_literals, print_function, absolute_import
-
-import itertools
-
 from django.conf import settings
+import itertools
 import logging
 
 
@@ -11,7 +9,10 @@ class CeleryTaskRouter(object):
     queue_task_map = {
         "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
-        ]
+        ],
+        "majia-alpha": [
+            'majia.tasks.auto_majia_effect',
+        ],
     }
 
     # Map[TaskName, QueueName]

physical/settings.py

@@ -10,8 +10,5 @@ For the full list of settings and their values, see
 https://docs.djangoproject.com/en/1.10/ref/settings/
 """
 
-import os
 from .log_settings import *
-from datetime import timedelta
-from celery.schedules import crontab
 from .settings_local import *

physical/settings_local.py.template

@@ -48,6 +48,7 @@ INSTALLED_APPS = (
     'trans2es',
     'search',
     'injection.data_sync',
+    'majia',
 )
 
 CELERYBEAT_SCHEDULE = {
@@ -57,6 +58,7 @@ CELERYBEAT_SCHEDULE = {
         'args': ()
     },
 }
+
 """
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',

search/views/tag.py

@@ -10,8 +10,9 @@ from search.utils.topic import TopicUtils
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
-from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow
+from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow,Tag
 import time
+from django.conf import settings
 
 
 def get_highlight(fields=[]):
@@ -113,13 +114,16 @@ def choice_push_tag(device_id, user_id):
 
         now_sec = int(time.time())
         valid_time = 8*7*24*60*60
-        for tag_id in redis_push_tag_dict:
+
+        ori_key_list = list(redis_push_tag_dict.keys())
+        for tag_id in ori_key_list:
             if now_sec - redis_push_tag_dict[tag_id] >= valid_time:
-                redis_push_tag_dict.popitem(tag_id)
+                redis_push_tag_dict.pop(tag_id)
 
         redis_push_tag_list = list(redis_push_tag_dict.keys())
-        account_user_tag_list = list(AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True))
-        community_tag_follow_list = list(CommunityTagFollow.objects.filter(user_id=user_id).values_list("tag_id",flat=True))
+        redis_push_tag_list = [int(item) for item in redis_push_tag_list]
+        account_user_tag_list = list(AccountUserTag.objects.filter(user=user_id,is_deleted=False).values_list("tag_id",flat=True))
+        community_tag_follow_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_online=True,is_deleted=False).values_list("tag_id",flat=True))
 
         linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
         tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
@@ -130,11 +134,12 @@ def choice_push_tag(device_id, user_id):
         account_user_tag_list.extend(linucb_recommend_tag_list)
         unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list))
 
+        unread_tag_list = list(Tag.objects.filter(id__in=unread_tag_list, is_online=True, is_deleted=False).values_list("id",flat=True))
         ret_tag_set = set()
         if len(unread_tag_list)>0:
             for tag_id in unread_tag_list:
                 valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count()
-                if valid_tag_topic_num>0:
+                if valid_tag_topic_num>100:
                     ret_tag_set.add(tag_id)
                     redis_push_tag_dict[tag_id] = now_sec
                     if len(ret_tag_set)>=1:
@@ -146,3 +151,30 @@ def choice_push_tag(device_id, user_id):
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"tag_list": []}
+
+@bind("physical/search/identity_tag_name")
+def identity_tag_name(topic_content):
+    try:
+        ret_tag_set = set()
+        redis_key_name = "physical:tag_name_set"
+
+        body = {
+            'text': topic_content,
+            'analyzer': "gm_default_index"
+        }
+
+        cli_info  = settings.TAG_ES_INFO_LIST
+        res = ESPerform.get_analyze_results(es_cli=ESPerform.get_cli(cli_info=cli_info), sub_index_name="tag", query_body=body)
+
+        logging.info("duan add,res:%s" % str(res).encode("utf-8"))
+        for item in res["tokens"]:
+            token_word = item["token"]
+            is_member = redis_client.sismember(redis_key_name, token_word)
+            if is_member:
+                ret_tag_set.add(token_word)
+
+        return {"tag_name_list": list(ret_tag_set)}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"tag_name_list": []}
+

search/views/topic.py

@@ -73,10 +73,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
                 if offset>0: # 首次搜索时不需要过滤已读
                     have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
 
-        user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
-        redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
-        user_similar_score_redis_list = json.loads(
-            redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
+        # user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
+        # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
+        # user_similar_score_redis_list = json.loads(
+        #     redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
 
         attention_tag_list = list()
         recommend_topic_list = list()
@@ -113,8 +113,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         topic_id_list = list()
         rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
                                                            single_size=size,query=query, query_type=query_type,
-                                                           filter_topic_id_list=have_read_topic_id_list,
-                                                           user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5,6",attention_tag_list=attention_tag_list)
+                                                           filter_topic_id_list=have_read_topic_id_list,index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list)
 
         if len(recommend_topic_list) == 6 and query is None:
             if (size < 11):
@@ -253,7 +252,7 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
 
 @bind("physical/search/topic_detail_page_recommend")
 def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pictorial_id=-1, topic_user_id=-1,
-                                filter_topic_user_id=False, offset=0, size=10):
+                                filter_topic_user_id=False, offset=0, size=10, topic_tag_list=[]):
     """
     :remark:帖子详情页推荐策略，缺少第一个卡片策略
     :param user_id:
@@ -265,31 +264,90 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
     try:
         if not isinstance(user_id, int):
             user_id = -1
+        if user_id > 0:
+                redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id)
+        else:
+                redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id)
+        if int(offset) == 0:
+            have_read_topic_list = list()
+            # redis_dict = {
+            #     "have_read_topic_id": json.dumps(have_read_topic_list)
+            # }
+            redis_client.delete(redis_key)
+            # redis_client.expire(redis_key, 60 * 60 * 24)
 
-        redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id)
-        have_read_topic_redis_data = redis_client.get(redis_key)
-        have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
+        else:
+            have_read_topic_list = list()
+            redis_field_list = [b'have_read_topic_list']
 
-        # 获取es链接对象
+            have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list)
+            have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
         es_cli_obj = ESPerform.get_cli()
+        have_read_topic_list.append(topic_id)
+        topic_user_result = list()
+        topic_tag_result = list()
+        result = list()
+        if len(topic_tag_list) != 0:
+             topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
+                                                                               index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
+             topic_tag_size = len(topic_tag_result)
+             have_read_topic_list.extend(topic_tag_result)
+        else:
+             topic_tag_size = 0
+        if topic_tag_size <size:
+           size = size - topic_tag_size
+           if topic_user_id != -1:
+              topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
+                                                                              size, es_cli_obj,
+                                                                              index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
+                                                                              )
+              topic_user_size = len(topic_user_result)
+              have_read_topic_list.extend(topic_user_result)
+              if topic_user_size < size:
+                 size = size - topic_user_size
+                 result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
+                                                                                  have_read_topic_list,
+                                                                                  size, es_cli_obj,
+                                                                                  index_type="topic", routing="4,5,6")
+                 have_read_topic_list.extend(result)
+
+        # have_read_topic_redis_data = redis_client.get(redis_key)
+        # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
+        #
+        # redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id)
+        # have_read_topic_redis_data = redis_client.get(redis_key)
+        # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
+        #
+        # # 获取es链接对象
+        # es_cli_obj = ESPerform.get_cli()
+        #
+        # # 获取帖子标签列表
+        # topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id, es_cli_obj)
+
+        # result_list = TopicUtils.get_topic_detail_recommend_list(user_id, topic_id, topic_tag_list, topic_pictorial_id,
+        #                                                          topic_user_id, filter_topic_user_id,
+        #                                                          have_read_topic_list, offset, size, es_cli_obj,index_type="topic",routing="4,5,6")
+        # result_list = TopicUtils.get_recommend_topic_ids(user_id, tag_id=-1, single_size=size, offset=offset, size=size,
+        #                                                  topic_user_id=topic_user_id, current_topic_id=topic_id,
+        #                                                  topic_tag_list=topic_tag_list)
+        #
+        # user_recommend_list = TopicUtils.get_recommend_topic_ids(user_id, tag_id=-1, single_size=size, offset=offset, size=size,
+        #                                                  topic_user_id=topic_user_id, current_topic_id=topic_id,
+        #                                                  topic_tag_list=topic_tag_list)
+        redis_dict = {
+            "have_read_topic_list": json.dumps(have_read_topic_list),
+        }
+        redis_client.hmset(redis_key, redis_dict)
+        redis_client.expire(redis_key, 60 * 60 * 24)
 
-        # 获取帖子标签列表
-        topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id, es_cli_obj)
-
-        result_list = TopicUtils.get_topic_detail_recommend_list(user_id, topic_id, topic_tag_list, topic_pictorial_id,
-                                                                 topic_user_id, filter_topic_user_id,
-                                                                 have_read_topic_list, offset, size, es_cli_obj,index_type="topic",routing="4,5,6")
+        result_list = []
+        logging.warning("topic_tag_result:%s" % str(topic_tag_result))
+        logging.warning("topic_user_result:%s" % str(topic_user_result))
+        logging.warning("result:%s" % str(result))
         recommend_topic_ids_list = list()
-        if len(result_list) > 0:
-            recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
-
-        have_read_topic_list.extend(recommend_topic_ids_list)
-
-        have_read_topic_len = len(have_read_topic_list)
-        if have_read_topic_len > 5000:
-            have_read_topic_list = have_read_topic_list[(have_read_topic_len - 5000):]
-
-        redis_client.set(redis_key, json.dumps(have_read_topic_list))
+        recommend_topic_ids_list.extend(topic_tag_result)
+        recommend_topic_ids_list.extend(topic_user_result)
+        recommend_topic_ids_list.extend(result)
         return {"recommend_topic_ids": recommend_topic_ids_list}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/mapping/tag-name.json

+{
+  "dynamic":"strict",
+  "properties": {
+    "name":{"type":"keyword"}
+  }
+}
\ No newline at end of file

trans2es/mapping/topic-high-star.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic-star-routing.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic-star.json

+{
+  "dynamic":"strict",
+  "properties": {
+    "id":{"type":"long"},
+    "content_level":{"type":"text"}
+  }
+}

trans2es/mapping/topic.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/models/topic.py

@@ -43,6 +43,20 @@ class TopicImage(models.Model):
     create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
+class ExcellentTopic(models.Model):
+    """优质帖子"""
+
+    class Meta:
+        verbose_name = "优质帖子"
+        db_table = "excellent_topic"
+
+    id = models.IntegerField(verbose_name=u'ID', primary_key=True)
+    topic_id = models.IntegerField(verbose_name=u"帖子ID",db_index=True)
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    excellent_type = models.IntegerField(verbose_name=u"优质类型",db_index=True)
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
 class Topic(models.Model):
     class Meta:
@@ -175,6 +189,17 @@ class Topic(models.Model):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
 
+    def judge_if_excellent_topic(self, topic_id):
+        try:
+            excelllect_object = ExcellentTopic.objects.filter(topic_id=topic_id).first()
+            if excelllect_object and excelllect_object.is_online and not excelllect_object.is_deleted:
+                return True
+            else:
+                return False
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return False
+
     def get_topic_offline_score(self):
         try:
             offline_score = 0.0
@@ -184,7 +209,7 @@ class Topic(models.Model):
             user_query_results = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id)
             if user_query_results.count() > 0:
                 if user_query_results[0].is_recommend:
-                    offline_score += 2.0
+                    offline_score += 2.0 *10
                 elif user_query_results[0].is_shadow:
                     user_is_shadow = True
 
@@ -194,12 +219,18 @@ class Topic(models.Model):
 
             # 帖子等级
             if self.content_level == '5':
-                offline_score += 6.0
+                offline_score += 100.0 *3
             elif self.content_level == '4':
-                offline_score += 5.0
+                offline_score += 60.0 *3
             elif self.content_level == '6':
-                offline_score += 100.0
+                offline_score += 200.0 *3
+
+            # is_excellent = self.judge_if_excellent_topic(self.id)
+            # if is_excellent:
+            #     offline_score += 200.0
 
+            if self.language_type == 1:
+                offline_score += 60.0
             # exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
             # click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
             # uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
@@ -266,21 +297,6 @@ class TopicExtra(models.Model):
     virtual_vote_num = models.IntegerField(verbose_name="帖子虚拟点赞")
 
 
-class ExcellentTopic(models.Model):
-    """优质帖子"""
-
-    class Meta:
-        verbose_name = "优质帖子"
-        db_table = "excellent_topic"
-
-    id = models.IntegerField(verbose_name=u'ID', primary_key=True)
-    topic_id = models.IntegerField(verbose_name=u"帖子ID",db_index=True)
-    is_online = models.BooleanField(verbose_name=u'是否上线')
-    is_deleted = models.BooleanField(verbose_name=u'是否删除')
-    excellent_type = models.IntegerField(verbose_name=u"优质类型",db_index=True)
-    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
-    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
-
 class TopicHomeRecommend(models.Model):
     """运营位帖子"""
 

trans2es/type_info.py

@@ -84,7 +84,7 @@ class TypeInfo(object):
 
     def bulk_get_data(self, instance_iterable):
         data_list = []
-        # 4，5星帖子单独索引
+        # 4星以上帖子单独索引
         topic_data_high_star_list = list()
 
         if self.batch_get_data_func:
@@ -144,23 +144,73 @@ class TypeInfo(object):
                 else:
                     if data:
                         if self.type == "topic":
+                            # q = {
+                            #     "query":{
+                            #         "term":{
+                            #             "id": data["id"]
+                            #         }
+                            #     },
+                            #     "_source":{
+                            #         "include":["content_level"]
+                            #     }
+                            # }
+                            # search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
+                            # if len(search_results["hits"]) > 0:
+                            #     ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
+                            # else:
                             ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
+                            if ori_topic_star:
+                                ori_topic_star = str(ori_topic_star, encoding="utf-8")
+                            else:
+                                q = {
+                                    "query":{
+                                        "term":{
+                                            "id": data["id"]
+                                        }
+                                    },
+                                    "_source":{
+                                        "includes":["content_level"]
+                                    }
+                                }
+                                search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
+                                if len(search_results["hits"]) > 0:
+                                    ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
+
                             if not ori_topic_star:
+                                # data_list = [
+                                #     {
+                                #         "id": data["id"],
+                                #         "content_level": data["content_level"]
+                                #     }
+                                # ]
+                                # ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
                                 redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
                             else:
-                                int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
+                                # int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
+                                int_ori_topic_star = int(ori_topic_star)
                                 if int_ori_topic_star != data["content_level"]:
                                     old_data = copy.deepcopy(data)
                                     old_data["is_online"] = False
                                     old_data["is_deleted"] = True
                                     old_data["content_level"] = int_ori_topic_star
+                                    old_data["is_history"] = True
 
                                     data_list.append(old_data)
+                                    if int_ori_topic_star>=4:
+                                        topic_data_high_star_list.append(old_data)
                                     redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
-                        data_list.append(data)
+                                    # data_list = [
+                                    #     {
+                                    #         "id": data["id"],
+                                    #         "content_level": data["content_level"]
+                                    #     }
+                                    # ]
+                                    # ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
 
-                    # if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
-                    #     topic_data_high_star_list.append(data)
+
+                            if data["content_level"] and int(data["content_level"])>=4:
+                                topic_data_high_star_list.append(data)
+                        data_list.append(data)
 
         return (data_list,topic_data_high_star_list)
 
@@ -234,13 +284,13 @@ class TypeInfo(object):
                 es=es,
             )
 
-        # # 同时写4星及以上的帖子
-        # if len(topic_data_high_star_list)>0:
-        #     self.elasticsearch_bulk_insert_data(
-        #         sub_index_name="topic-high-star",
-        #         data_list=topic_data_high_star_list,
-        #         es=es,
-        #     )
+        # 同时写4星及以上的帖子
+        if len(topic_data_high_star_list)>0:
+            self.elasticsearch_bulk_insert_data(
+                sub_index_name="topic-high-star",
+                data_list=topic_data_high_star_list,
+                es=es,
+            )
 
         end = time.time()
         time3=end-begin
@@ -301,6 +351,16 @@ def get_type_info_map():
         return _get_type_info_map_result
 
     type_info_list = [
+        TypeInfo(
+            name='topic-star',
+            type='topic-star',
+            model=topic.Topic,
+            query_deferred=lambda: topic.Topic.objects.all().query,#假的
+            get_data_func=TopicTransfer.get_topic_data,#假的
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        ),
         TypeInfo(
             name='topic-star-routing',
             type='topic-star-routing',
@@ -406,6 +466,16 @@ def get_type_info_map():
             round_insert_chunk_size=5,
             round_insert_period=2,
         ),
+        TypeInfo(
+            name="tag-name",  # 标签名字
+            type="tag-name",
+            model=tag.Tag,
+            query_deferred=lambda: tag.Tag.objects.all().query,
+            get_data_func=TagTransfer.get_tag_name_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        ),
         TypeInfo(
             name='contrast_similar',  # facesimilar
             type='contrast_similar',

trans2es/utils/tag_transfer.py

@@ -16,6 +16,17 @@ from django.conf import settings
 
 class TagTransfer(object):
 
+    @classmethod
+    def get_tag_name_data(cls,instance):
+        try:
+            res = dict()
+            res["name"] = instance.name
+
+            return res
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return dict()
+
     @classmethod
     def get_tag_data(cls,instance):
         try:

trans2es/utils/topic_transfer.py

@@ -115,8 +115,10 @@ class TopicTransfer(object):
             if is_excellect:
                 res["is_excellent"] = 1
             else:
-                excelllect_object = ExcellentTopic.objects.filter(topic_id=instance.id).first()
-                if excelllect_object and excelllect_object.is_online and not excelllect_object.is_deleted:
+                # excelllect_object = ExcellentTopic.objects.filter(topic_id=instance.id).first()
+                # if excelllect_object and excelllect_object.is_online and not excelllect_object.is_deleted:
+                is_excellent = instance.judge_if_excellent_topic(instance.id)
+                if is_excellent:
                     res["is_excellent"] = 1
                 else:
                     res["is_excellent"] = 0