Skip to content

P
physical
Commit 7cf6330b authored by Kai's avatar Kai
Browse files
Options

update

parents 1856f464 374fb99d
Hide whitespace changes
Inline Side-by-side
Showing with 640 additions and 712 deletions
.gitignore
View file @ 7cf6330b
......@@ -85,4 +85,5 @@ gaia/rpcd.json
dbmw_deploy/config.dir/
.idea/codeStyles/Project.xml
.idea/misc.xml
.idea/physical.iml
\ No newline at end of file
.idea/physical.iml
.idea/codeStyles/Project.xml
\ No newline at end of file
.idea/codeStyles/Project.xml
View file @ 7cf6330b
......@@ -24,5 +24,29 @@
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme>
</component>
\ No newline at end of file
.idea/misc.xml
View file @ 7cf6330b
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>
......
.idea/physical.iml
View file @ 7cf6330b
......@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.6 (venv)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
......
app_conf.xml
View file @ 7cf6330b
......@@ -17,6 +17,5 @@
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
<!--<element value="majia..tasks.tasks"/>-->
</config>
</gm_rpcd_config>
injection/data_sync/tasks.py
View file @ 7cf6330b
......@@ -2,6 +2,10 @@
import logging
import traceback
import json
import pymysql
import threading
import random
import datetime
from celery import shared_task
from django.conf import settings
from django.core import serializers
......@@ -9,9 +13,10 @@ from trans2es.type_info import get_type_info_map
# from rpc.all import get_rpc_remote_invoker
from libs.es import ESPerform
from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag
@shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False):
try:
......@@ -55,35 +60,47 @@ def sync_face_similar_data_to_redis():
item_list = list()
for item in similar_result_items:
weight_score = int(item.similarity * 100)
item_list.append(
{
"contrast_user_id": item.contrast_user_id,
"similarity": item.similarity
"filter":{
"constant_score":{
"filter":{
"term":{"user_id": item.contrast_user_id}
}
}
},
"weight": weight_score*2
}
)
if len(item_list)>=100:
break
redis_client.set(redis_key, json.dumps(item_list))
logging.info("participant_user_id:%d set data done!" % participant_user_id)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@shared_task
def sync_user_similar_score():
try:
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True)
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
"user_id", flat=True)
redis_key_prefix = "physical:user_similar_score:user_id:"
logging.info("begin sync user_similar_score!")
for user_id in results_items:
redis_key = redis_key_prefix + str(user_id)
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score")
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
"-score")
item_list = list()
for item in similar_results_items:
contrast_user_id = item.contrast_user_id
score = item.score
item_list.append(
[contrast_user_id,score]
[contrast_user_id, score]
)
redis_client.set(redis_key, json.dumps(item_list))
......
libs/error.py
View file @ 7cf6330b
......@@ -18,4 +18,4 @@ def logging_exception(send_to_sentry=True):
# send exception info to sentry, fail silently
_sentry_client.captureException()
except:
pass
passraven.contrib.django.raven_compat.models
libs/es.py
View file @ 7cf6330b
......@@ -366,3 +366,78 @@ class ESPerform(object):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id","user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list,topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
linucb/views/collect_data.py
View file @ 7cf6330b
......@@ -32,7 +32,9 @@ class CollectData(object):
def __init__(self):
self.linucb_matrix_redis_prefix = "physical:linucb:device_id:"
#废弃
self.linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
#推荐帖子
self.linucb_recommend_topic_id_prefix = "physical:linucb:topic_recommend:device_id:"
self.tag_topic_id_redis_prefix = "physical:tag_id:topic_id_list:"
self.click_recommend_redis_key_prefix = "physical:click_recommend:device_id:"
......@@ -66,7 +68,6 @@ class CollectData(object):
if len(recommend_tag_list) > 0:
tag_recommend_redis_key = self.linucb_recommend_redis_prefix + str(device_id)
redis_client.set(tag_recommend_redis_key, json.dumps(recommend_tag_list))
# Todo:设置过期时间，调研set是否支持
redis_client.expire(tag_recommend_redis_key, 7*24*60*60)
have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,user_id,TopicPageType.HOME_RECOMMEND)
......@@ -74,52 +75,55 @@ class CollectData(object):
have_read_topic_id_list.extend(promote_recommend_topic_id_list)
recommend_topic_id_list = list()
recommend_topic_id_list_dict = dict()
recommend_topic_id_list_click = list()
recommend_topic_id_list_click_dict = dict()
if click_topic_tag_list:
if len(click_topic_tag_list)>0:
recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list,
if click_topic_tag_list and len(click_topic_tag_list)>0:
recommend_topic_id_list_click,recommend_topic_id_list_click_dict = ESPerform.get_tag_topic_list_dict(click_topic_tag_list,
have_read_topic_id_list,size=2)
if len(recommend_topic_id_list_click) > 0:
recommend_topic_id_list.extend(recommend_topic_id_list_click)
have_read_topic_id_list.extend(recommend_topic_id_list)
click_recommend_redis_key = self.click_recommend_redis_key_prefix + str(device_id)
click_redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"cursor": 0
}
redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
recommend_topic_id_list_dict.update(recommend_topic_id_list_click_dict)
# have_read_topic_id_list.extend(recommend_topic_id_list_click)
# click_recommend_redis_key = self.click_recommend_redis_key_prefix + str(device_id)
# click_redis_data_dict = {
# "data": json.dumps(recommend_topic_id_list),
# "datadict":json.dumps(recommend_topic_id_list_dict),
# "cursor": 0
# }
# redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
tag_id_list = recommend_tag_list[0:100]
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
redis_topic_list = list()
cursor = -1
if b"data" in redis_topic_data_dict:
redis_topic_list = json.loads(redis_topic_data_dict[b"data"]) if redis_topic_data_dict[
b"data"] else []
cursor = int(str(redis_topic_data_dict[b"cursor"], encoding="utf-8"))
if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
have_read_topic_id_list.extend(redis_topic_list[:2])
# redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
# redis_topic_list = list()
# cursor = -1
# if b"data" in redis_topic_data_dict:
# redis_topic_list = json.loads(redis_topic_data_dict[b"data"]) if redis_topic_data_dict[
# b"data"] else []
# cursor = int(str(redis_topic_data_dict[b"cursor"], encoding="utf-8"))
# if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
# have_read_topic_id_list.extend(redis_topic_list[:2])
if len(new_user_click_tag_list)>0:
tag_topic_id_list = ESPerform.get_tag_topic_list(new_user_click_tag_list, have_read_topic_id_list)
tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(new_user_click_tag_list, have_read_topic_id_list)
else:
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(tag_id_list,have_read_topic_id_list)
if len(recommend_topic_id_list)>0 or len(new_user_click_tag_list) > 0:
if len(recommend_topic_id_list)>0 or len(tag_topic_id_list)>0 or len(new_user_click_tag_list) > 0:
tag_topic_id_list = recommend_topic_id_list + tag_topic_id_list
tag_topic_dict.update(recommend_topic_id_list_dict)
redis_data_dict = {
"data": json.dumps(tag_topic_id_list),
"datadict":json.dumps(tag_topic_dict),
"cursor":0
}
redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
else:
if cursor<=0 and len(redis_topic_list)>0:
tag_topic_id_list = redis_topic_list[:2] + tag_topic_id_list
tag_topic_id_list = list(set(tag_topic_id_list))
return True
except:
......@@ -150,46 +154,49 @@ class CollectData(object):
raw_val_dict = json.loads(ori_msg.value)
if "type" in raw_val_dict and "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["business_id"] or raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id)))
if "type" in raw_val_dict and \
(raw_val_dict["type"] in ("on_click_feed_topic_card","tag_zone_click_focus")):
tag_list = list()
click_topic_tag_list = list()
collection_tag_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online","is_collection")
# if len(collection_tag_sql_query_results)>0:
for tag_id,is_online,is_collection in collection_tag_sql_query_results:
if is_online and is_collection == 1:
click_topic_tag_list.append(tag_id)
if "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["business_id"] or raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id)))
topic_tag_list = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id,is_online=True).values_list("tag_id",flat=True))
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_tag_list,is_online=True,is_deleted=False).values_list("id","collection","is_ai")
for id,collection,is_ai in tag_query_results:
if collection and is_ai:
click_topic_tag_list.append(id)
logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % (
str(device_id), str(topic_id), str(click_topic_tag_list)))
else:
tag_name = raw_val_dict["params"]["query"]
query_type = raw_val_dict["params"]["type"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
if query_type=="do":
tag_list = list(Tag.objects.using(settings.SLAVE_DB_NAME).filter(name=tag_name,is_online=True,is_deleted=False).values_list("id",flat=True))
click_topic_tag_list.extend(tag_list)
logging.info("query tag attention,positive tag_list,device_id:%s,query_name:%s,tag_list:%s" % (
str(device_id), tag_name, str(click_topic_tag_list)))
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results:
if is_online:
tag_list.append(tag_id)
tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id=tag_id).values_list("id", "collection", "is_ai")
for id, collection, is_ai in tag_sql_query_results:
if (is_ai == 1) and id not in click_topic_tag_list:
click_topic_tag_list.append(id)
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
is_click = 1
is_vote = 0
reward = 1 if is_click or is_vote else 0
logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % (
str(device_id), str(topic_id), str(click_topic_tag_list)))
for tag_id in click_topic_tag_list:
self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature)
# 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后
self.update_recommend_tag_list(device_id, user_feature, user_id,click_topic_tag_list=click_topic_tag_list)
if len(click_topic_tag_list)>0:
self.update_recommend_tag_list(device_id, user_feature, user_id,click_topic_tag_list=click_topic_tag_list)
elif "type" in raw_val_dict and "page_precise_exposure" == raw_val_dict["type"]:
if isinstance(raw_val_dict["params"]["exposure_cards"],str):
exposure_cards_list = json.loads(raw_val_dict["params"]["exposure_cards"])
......
majia/__init__.py deleted 100644 → 0
View file @ 1856f464
default_app_config = 'majia.apps.MaJiaApp'
majia/admin.py deleted 100644 → 0
View file @ 1856f464
from django.contrib import admin
# Register your models here.
majia/apps.py deleted 100644 → 0
View file @ 1856f464
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from django.apps import AppConfig
class MaJiaApp(AppConfig):
name = 'majia'
label = 'majia'
majia/get_session.py deleted 100644 → 0
View file @ 1856f464
import requests
import time
import datetime
import random
import traceback
from django.conf import settings
import logging
DATABASES = settings.DATABASES
# host = DATABASES['default']['HOST']
#
# user = DATABASES['default']['USER']
#
# port = DATABASES['default']['PORT']
#
# db = DATABASES['default']['NAME']
#
# passwd = DATABASES['default']['PASSWORD']
auto_click_url = "http://earth.igengmei.com/api/v1/like"
auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
auto_follow_url = "http://earth.igengmei.com/api/v1/follow"
auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
def get_majia():
try:
data = open("/srv/apps/physical/majia/majia_email.txt")
list = []
for i in data:
list.append(i.strip('\n').strip(','))
maj = random.randint(1, len(list))
user_id = list[maj - 1]
return user_id
except:
logging.error("catch exception,get_majia:%s" % traceback.format_exc())
return None
def get_cookies(user_id):
try:
with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
try:
data = f.readlines()
except:
data = None
list = []
for i in data:
list.append(i.strip('\n').split(","))
for i in list:
if user_id == i[0]:
return (i[1])
except:
return None
def login():
try:
user_id = get_majia()
cookies = get_cookies(user_id)
if cookies is not None:
return {'sessionid': cookies}
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
return None
def click(cookies_get, id):
try:
post_dict = {
'type': 0,
'id': id
}
response = requests.post(url=auto_click_url,
cookies=cookies_get,
data=post_dict)
logging.info("response.text:%s" % response.text)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
def reply(cookies_get, id, content):
try:
post_dict = {
'topic_id': id,
'content': content
}
response = requests.post(url=auto_reply_url,
cookies=cookies_get,
data=post_dict)
logging.info("response.text:%s" % response.text)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
def time_conv(minutest):
try:
now = datetime.datetime.now()
yes_time = now - datetime.timedelta(minutes=minutest)
print(yes_time)
return yes_time
except:
return None
def get_comments():
try:
data = open("/srv/apps/physical/majia/guanshui.txt", "r")
list_guanshui = []
for i in data:
list_guanshui.append(i)
num1 = random.randint(0, 30)
num2 = random.randint(30, 70)
num3 = random.randint(70, len(list_guanshui))
comment1 = list_guanshui[num1 - 1]
comment2 = list_guanshui[num2 - 1]
comment3 = list_guanshui[num3 - 1]
comment_list = [comment1, comment2, comment3]
return comment_list
except:
return None
majia/guanshui.txt deleted 100644 → 0
View file @ 1856f464
照片色调很好看
用的什么调色工具呀
立马去get同款
我一个一看就会，动手就废的girl
为什么我拍不出来你的效果？
Po主用的什么拍照软件
这个好，学习了
真的是元气满满
很帅气！
这样好显气质！
好可爱啊，完全没有抵抗力了。
莫名好感中
满分颜值
超酷！
马住回看
请问发色是啥
请问是什么app拍的啊
滤镜要怎么选？
卖萌打滚求翻牌
有没有常用自拍姿势教程
拍照技术满分
日常夸奖博主
日常勾搭
背景是在哪里？
照片构图很好看
主要还是看颜值
你长得很像明星
支持一下
转需留存
拍照表情很自然
风格很好
这里是哪里呀？
种草了！
搭配小技巧get！
怎么都学不会的路过
太好了，新手学习了
一直支持，只求被翻牌！
好喜欢这种风格
好看，在哪儿买的
衣好有气质哦
这个我也会
拍照pose求出教程
小姐姐，作图app是啥
终于更新了
搭配的好好看
求同款
日常催更
get√
什么时候更新啊
果然只要瘦怎么都好看。
不错的哈哈
同款达不到楼主的效果
喜欢你的搭配
我拍出来为什么感觉比本人丑很多？
能多发点全身正面照吗？想看整体效果
这个真的很真实了
大圆脸合适吗？
成功引起了我的注意！
感觉很少女~
不错，继续加油
好看的人做什么都好看
都是人才！！！！
道理我都懂，就差一个脸！
不赞我良心都过意不去啊
爱你哟~
用手机可以拍出来吗？
使用相机拍的还是手机
我觉得很赞
颜色搭配很舒服
小姐姐有戴美瞳吗
看见这个帖心情变好了
不管怎么样先收藏了
更新更新更新
你和我朋友长得有点像
你的风格我都很喜欢
好看
我要有你这条件还需要啥拍照技术
太美了！
被圈粉了……
美而不作，我喜欢
楼主真的real真实了。
持续等更新
楼主加油，照片拍的不错，非常值得推荐。
大家花呗还了吗？
来啦来啦来啦来啦来啦
画面太治愈了
楼主辛苦啦
不错，跟着学习一下
心动，我要去试试
满满的干货，收藏好下次用
新技能get√
感觉不错mark一下
适合自己的才是最重要的
羡慕一下作者的颜
Omaga太好看了吧~
这张拍的不错
这个特效挺可爱的
翻我翻我~
这样很显脸小
关注啦
majia/models.py deleted 100644 → 0
View file @ 1856f464
from django.db import models
# Create your models here.
majia/tasks.py deleted 100644 → 0
View file @ 1856f464
# -*- coding: UTF-8 -*-
import pymysql
import random
import threading
import traceback
import logging
from celery import shared_task
from django.conf import settings
from majia.get_session import get_comments, click, login, reply
# from majia.get_session import host, user, passwd, db
exec_count = 0
def auto_click(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_timer():
# cookies = login()
# if cookies:
# click(cookies, topic_id_list[0])
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# # sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# if exec_count == 2:
# # sleep_time = random.randint(1000, 1900)
# sleep_time = 50
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_timer) # 首次启动
# timer.start()
#
# else:
# pass
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_comment():
# cookies = login()
# if cookies:
# comment_content = get_comments()
# comment = comment_content[0]
# reply(cookies, topic_id_list[0], comment)
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# if exec_count == 2:
# sleep_time = random.randint(1000, 1900)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_comment) # 首次启动
# timer.start()
#
#
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
@shared_task
def auto_majia_effect(pk_list):
auto_click(pk_list)
auto_reply(pk_list)
majia/tests.py deleted 100644 → 0
View file @ 1856f464
from django.test import TestCase
# Create your tests here.
majia/views.py deleted 100644 → 0
View file @ 1856f464
from django.shortcuts import render
# Create your views here.
physical/celery_task_router.py
View file @ 7cf6330b
......@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
queue_task_map = {
"tapir-alpha": [
'injection.data_sync.tasks.write_to_es',
],
"majia-alpha": [
'majia.tasks.auto_majia_effect',
],
]
}
# Map[TaskName, QueueName]
......
physical/settings_local.py.template
View file @ 7cf6330b
......@@ -48,7 +48,6 @@ INSTALLED_APPS = (
'trans2es',
'search',
'injection.data_sync',
'majia',
)
CELERYBEAT_SCHEDULE = {
......
search/utils/group.py
View file @ 7cf6330b
......@@ -47,23 +47,79 @@ class GroupUtils(object):
return {"total_count": 0, "hits": []}
@classmethod
def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None):
def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None,attention_tag_list=[]):
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["query"] = {
"bool":{
"must":[
{"term": {"is_online": True}},
{"term":{"is_deleted": False}},
{"term": {"effective": True}}
]
functions_list = list()
for tag_id in attention_tag_list:
functions_list.append({
"filter":{
"constant_score":{
"filter":{
"term":{
"tag_id": tag_id
}
}
}
},
"weight": 20
})
if len(functions_list)>=20:
break
functions_list.append(
{
"gauss": {
"create_time": {
"scale": "1d",
"decay": 0.99
}
},
"weight": 60
}
)
q = {
"query":{
"function_score":{
"query":{
"bool":{
"filter":[
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"effective": True}}
],
"must_not":[
{"term":{"is_default":1}}
]
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
}
}
q["sort"] = [
{"high_quality_topic_num":{"order":"desc"}}
{
"_script": {
"type": "number",
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
# "lang":"painless",
# "source":"_score+params._source.offline_score"
},
"order": "desc"
}
},
{
"_score":{
"order": "desc"
}
}
]
q["_source"] = {
"includes":["id"]
......
search/utils/topic.py
View file @ 7cf6330b
......@@ -10,6 +10,9 @@ from alpha_types.venus import TOPIC_SEARCH_SORT
from libs.es import ESPerform
from .common import TopicDocumentField
from search.utils.common import *
from trans2es.models.pictorial import PictorialTopics
from libs.cache import redis_client
class TopicUtils(object):
......@@ -120,9 +123,11 @@ class TopicUtils(object):
return {}
@classmethod
def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.FIND_PAGE,
filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[],current_topic_id =-1,topic_tag_list =[],topic_user_id =-1):
def get_recommend_topic_ids(cls, user_id, tag_id, offset, size, single_size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,
filter_topic_id_list=[], test_score=False, must_topic_id_list=[], recommend_tag_list=[],
user_similar_score_list=[], index_type="topic", routing=None, attention_tag_list=[],
linucb_user_id_list=[], disable_collpase=False):
"""
:remark:获取首页推荐帖子列表
:param user_id:
......@@ -158,14 +163,6 @@ class TopicUtils(object):
q["query"] = dict()
functions_list = [
{
"filter": {
"term": {
"language_type": 1
}
},
"weight": 60
},
{
"gauss": {
"create_time": {
......@@ -174,38 +171,22 @@ class TopicUtils(object):
}
},
"weight": 60
},
{
"filter": {
"constant_score":{
"filter":{
"term": {
"content_level": 6
}
}
}
},
"weight": 600
}
]
# if len(user_similar_score_list) > 0:
# for item in user_similar_score_list[:100]:
# score_item = 2 + item[1]
# functions_list.append(
# {
# "filter": {"bool": {
# "should": {"term": {"user_id": item[0]}}}},
# "weight": score_item,
# }
# )
if user_id and user_id>0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list)>0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 30,
"filter": {"constant_score":{"filter":{"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
......@@ -216,31 +197,11 @@ class TopicUtils(object):
"weight": 100
}
)
if current_topic_id != -1:
if len(topic_tag_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"tag_list": topic_tag_list}}}},
"weight": 2000
}
)
if topic_user_id != -1:
functions_list.append(
{
"filter": {"bool": {
"should": {"term": {"user_id": topic_user_id}}}},
"weight": 1500
}
)
query_function_score = {
"query": {
"bool": {
"filter": [
{"range": {"content_level": {"gte": 4, "lte": 6}}},
# {"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
......@@ -260,7 +221,10 @@ class TopicUtils(object):
}
}
],
"minimum_should_match": 1
"minimum_should_match": 1,
"must_not": [
{"term": {"is_history": True}}
]
}
},
"score_mode": "sum",
......@@ -275,14 +239,18 @@ class TopicUtils(object):
}
if len(filter_topic_id_list) > 0:
query_function_score["query"]["bool"]["must_not"] = [
{"terms":{"id":filter_topic_id_list}}
{"terms": {"id": filter_topic_id_list}}
]
if current_topic_id != -1:
query_function_score["query"]["bool"]["must_not"] = [{
"term": {
"id": current_topic_id
}
}]
if len(linucb_user_id_list) > 0:
if "must_not" in query_function_score["query"]["bool"]:
query_function_score["query"]["bool"]["must_not"] += [
{"terms": {"user_id": linucb_user_id_list}}
]
else:
query_function_score["query"]["bool"]["must_not"] = [
{"terms": {"user_id": linucb_user_id_list}}
]
if query is not None: # 搜索帖子
multi_fields = {
'description': 200,
......@@ -300,9 +268,13 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match},
{"term": {"tag_list": tag_id}}
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
query_function_score["query"]["bool"]["minimum_should_match"] = 1
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte":3,"lte":6}}}
)
else:
if "must_not" in query_function_score["query"]["bool"]:
query_function_score["query"]["bool"]["must_not"] += [
......@@ -312,11 +284,19 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["must_not"] = [
{"term": {"is_operation_home_recommend": True}}
]
if query_type == TopicPageType.FIND_PAGE:
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte": 4, "lte": 6}}}
)
else:
query_function_score["query"]["bool"]["filter"].append(
{"term": {"content_level": 6}}
)
q["query"]["function_score"] = query_function_score
q["collapse"] = {
"field": "user_id"
}
# "includes": ["id", "pictorial_id", "offline_score", "user_id", "edit_tag_list"]
if not disable_collpase:
q["collapse"] = {
"field": "user_id"
}
q["_source"] = {
"includes": ["id"]
}
......@@ -334,18 +314,18 @@ class TopicUtils(object):
# }
# },
{
"_score": {
"offline_score": {
"order": "desc"
}
},
{
"offline_score":{
"_score": {
"order": "desc"
}
}
]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing)
offset=offset, size=size, routing=routing)
topic_id_list = list()
......@@ -358,7 +338,8 @@ class TopicUtils(object):
@classmethod
def get_topic_detail_recommend_list(cls, user_id, topic_id, topic_tag_list, topic_pictorial_id, topic_user_id,
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None,index_type="topic",routing=None):
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None,
index_type="topic", routing=None):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -405,7 +386,7 @@ class TopicUtils(object):
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 4, "lte": 5}}},
{"range": {"content_level": {"gte": 4, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
......@@ -435,7 +416,7 @@ class TopicUtils(object):
}
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing)
offset=offset, size=size, routing=routing)
return result_dict["hits"]
except:
......@@ -443,9 +424,10 @@ class TopicUtils(object):
return []
@classmethod
def top_get_topic_detail_recommend_list(cls, user_id, topic_id,have_read_topic_list, size, es_cli_obj=None,
index_type="topic", routing=None,collection_topic_tag_list = [],topic_tag_list = [],
topic_user_id =-1):
def top_get_topic_detail_recommend_list(cls, user_id, topic_id, have_read_topic_list, size, es_cli_obj=None,
index_type="topic", routing=None, collection_topic_tag_list=[],
topic_tag_list=[],
topic_user_id=-1):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -489,26 +471,26 @@ class TopicUtils(object):
# "weight": 5000
# }
# )
if len(topic_tag_list) != 0 or topic_user_id!= -1:
query_function_score = {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
if len(topic_tag_list) != 0 or topic_user_id != -1:
query_function_score = {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
}
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
else:
query_function_score = {
"query": {
......@@ -544,9 +526,9 @@ class TopicUtils(object):
}
q["query"]["function_score"] = query_function_score
if topic_user_id == -1:
q["collapse"] = {
"field": "user_id"
}
q["collapse"] = {
"field": "user_id"
}
q["_source"] = {
"includes": ["id", "pictorial_id", "user_id", "_score"]
}
......@@ -557,7 +539,8 @@ class TopicUtils(object):
# {"create_time": {"order": "desc"}}
# ]
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size, routing=routing)
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
routing=routing)
topic_id_list = list()
for item in result_dict["hits"]:
......@@ -700,7 +683,7 @@ class TopicUtils(object):
{"term": {"is_deleted": False}},
]
logging.info("get filters:%s"%filters)
logging.info("get filters:%s" % filters)
if not filters:
return f
......@@ -710,12 +693,6 @@ class TopicUtils(object):
"term": {k: v},
})
if k == "is_complaint":
f.append({
"term": {k: v},
})
if v in (None, '', []):
continue
......@@ -875,8 +852,50 @@ class TopicUtils(object):
@classmethod
def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
try:
must = cls.process_filters(filters, filter_online=filter_online)
q = {
"query": {
"bool": {
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
}
}
if sorts_by:
sorts = cls.process_sort(sorts_by)
if sorts:
q["sort"] = sorts
result_dict = ESPerform.get_search_results(
ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
)
if len(result_dict["hits"]) > 0:
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
must = cls.process_filters(filters, filter_online=filter_online)
return (topic_id_list, result_dict["total_count"])
elif offset == 0 and "pictorial_id" in filters: # 防止帖子同步延迟，画报详情页为空
pictorial_id = int(filters["pictorial_id"])
topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=pictorial_id, is_online=True,
is_deleted=False).values_list("topic_id",
flat=True)[
offset:size])
return (topic_id_list, len(topic_id_list))
else:
return ([], 0)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([], 0)
@classmethod
def business_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.business_filters(filters, filter_online=filter_online)
logging.info("get must:%s" % must)
q = {
"query": {
"bool": {
......@@ -884,6 +903,7 @@ class TopicUtils(object):
"must_not": cls.process_nfilters(nfilters),
}
}
}
if sorts_by:
......@@ -909,64 +929,77 @@ class TopicUtils(object):
}
@classmethod
def business_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
def business_filters(cls, filters, filter_online=True):
"""处理过滤器部分。"""
logging.info("get filters:%s" % filters)
f = [
{"term": {"is_deleted": False}},
]
if not filters:
return f
if filter_online:
f.append({"term": {"is_online": True}})
must = cls.process_filters(filters, filter_online=filter_online)
query = ''
for k, v in filters.items():
if v in (None, '', []):
continue
if k == "content":
query = filters[k]
q = {}
q["query"] = {
"function_score": {
"functions": [{
"filter": {
"bool": {
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
},
"weight": 1
}],
"query": {
f.append({
"multi_match": {
"fields":["content"],
"fields": ["content"],
"type": "cross_fields",
"operator": "and",
"query": query
"query": v
}
}
}
})
}
if query == '':
q["query"] = {
"bool": {
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
elif k == "virtual_content_level":
f.append({
"match": {k: v}
})
}
elif k in ["create_time_gte", "create_time_lte"]:
if sorts_by:
sorts = cls.process_sort(sorts_by)
if sorts:
q["sort"] = sorts
if k == "create_time_gte":
op = "gte"
elif k == "create_time_lte":
op = "lte"
try:
result_dict = ESPerform.get_search_results(
ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
)
f.append({
"range": {
"create_time_val": {
op: v,
}
}
})
elif k == "drop_score":
if v == "0":
f.append({
"term": {k: v}
})
return {
"hits": result_dict["hits"],
"total_count": result_dict["total_count"]
}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {
"hits": [],
"total_count": 0
}
else:
f.append({
"range": {
"drop_score": {
"gte": v,
}
}
})
else:
if isinstance(v, list):
f.append({
"terms": {k: v},
})
else:
f.append({
"term": {k: v},
})
return f
search/views/business_topic.py
View file @ 7cf6330b
......@@ -25,7 +25,6 @@ def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=
index_name="topic"
)
logging.info("get result_lsit:%s"%result_list)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except:
......
search/views/group.py
View file @ 7cf6330b
......@@ -52,9 +52,15 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
es_cli_obj = ESPerform.get_cli()
if sort_type == GroupSortTypes.HOT_RECOMMEND:
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj)
return {"pictorial_recommend_ids": pictorial_ids_list}
# 用户关注标签
attention_tag_list = list()
if user_id>0:
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,attention_tag_list)
return {"pictorial_recommend_ids": pictorial_ids_list}
elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
attention_pictorial_list = GroupUtils.get_user_attention_pictorial_list(user_id, offset=0, size=1,
es_cli_obj=es_cli_obj)
......
search/views/topic.py
View file @ 7cf6330b
......@@ -46,14 +46,18 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list =[]):
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
if query is None:
if user_id>0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
......@@ -77,7 +81,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
have_read_topic_id_list.extend(promote_topic_list)
recommend_topic_user_list = list()
attention_tag_list = list()
recommend_topic_list = list()
if query is None:
......@@ -86,15 +91,22 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
if b"data" in recommend_topic_dict:
recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
# 推荐帖子是强插的，要保证推荐帖子不在已读里
recommend_topic_id_list = list(set(recommend_topic_id_list) - set(have_read_topic_id_list))
cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
newcursor = cursor + 6
if len(recommend_topic_id_list) > newcursor:
recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
# 推荐帖子是强插的，要保证推荐帖子不在已读里
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if len(recommend_topic_list) == 6 and linucb_recommend_topic_id_dict is not None:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
......@@ -109,11 +121,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
topic_id_list = list()
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list)
have_read_topic_id_list.extend(promote_topic_list)
rank_topic_id_list = list()
if size>0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
......@@ -129,7 +143,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# else:
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:]
......@@ -137,13 +151,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
"have_read_topic_list": json.dumps(have_read_topic_id_list),
}
redis_client.hmset(redis_key, redis_dict)
# 每个session key保存30天
redis_client.expire(redis_key, 60 * 60 * 24 * 30)
# 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60)
return recommend_topic_list,rank_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
return [],[]
@bind("physical/search/query_tag_id_by_topic")
......@@ -156,7 +170,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND):
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[]):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
......@@ -172,7 +186,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
device_id = ""
recommend_topic_ids = list()
rank_topic_ids = list()
es_node_load_high_flag = False
# try:
# es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
......@@ -193,13 +207,13 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type)
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list)
return {"recommend_topic_ids": recommend_topic_ids}
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
return {"linucb_topic_ids": [],"rank_topic_ids":[]}
@bind("physical/search/discover_page")
......@@ -243,8 +257,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
return {"recommend_topic_ids": recommend_topic_ids}
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size,
query=query,disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
......@@ -385,13 +403,12 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
logging.info("get result_list:%s"%result_list)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {
"topic_ids": topic_ids,
"total_count": result_list["total_count"]
"topic_ids": topic_id_list,
"total_count": total_count
}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
......
trans2es/mapping/pictorial.json
View file @ 7cf6330b
......@@ -16,6 +16,8 @@
"tag_id":{"type":"long"},
"tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"topic_id_list":{"type":"long"},
"effective":{"type":"boolean"}
"effective":{"type":"boolean"},
"offline_score":{"type":"long"},
"is_default":{"type":"long"}
}
}
\ No newline at end of file
trans2es/mapping/topic-high-star.json
View file @ 7cf6330b
......@@ -14,6 +14,7 @@
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic-star-routing.json
View file @ 7cf6330b
......@@ -14,6 +14,7 @@
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic-star.json
View file @ 7cf6330b
{
"dynamic":"strict",
"_routing": {"required": true},
"properties": {
"id":{"type":"long"},
"content_level":{"type":"text"}
"is_online":{"type":"boolean"},//上线
"is_deleted":{"type":"boolean"},
"vote_num":{"type":"long"},
"total_vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","index":"not_analyzed"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"has_video":{"type":"boolean"},//是否是视频
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"},
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
trans2es/mapping/topic-v1.json
View file @ 7cf6330b
......@@ -14,6 +14,7 @@
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
......@@ -52,6 +53,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
trans2es/mapping/topic.json
View file @ 7cf6330b
......@@ -14,6 +14,7 @@
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/models/pictorial.py
View file @ 7cf6330b
......@@ -36,7 +36,6 @@ class PictorialTopics(models.Model):
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
......@@ -59,18 +58,16 @@ class Pictorial(models.Model):
creator_id = models.BigIntegerField(verbose_name=u'画报用户ID')
icon = models.CharField(verbose_name=u'画报名称', max_length=255)
topic_num = models.IntegerField(verbose_name=u'次数')
add_score = models.IntegerField(verbose_name=u'人工权重')
is_default = models.IntegerField(verbose_name=u"是否是默认画报")
def get_topic_id(self):
try:
topic_id =[]
topic_id_list = PictorialTopics.objects.filter(pictorial_id=self.id).values_list("topic_id", flat=True)
for i in topic_id_list:
topic_id.append(i)
return topic_id
topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=self.id,is_online=True,is_deleted=False).values_list("topic_id", flat=True))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
return []
def get_effective(self,topic_id_list):
......@@ -112,29 +109,21 @@ class Pictorial(models.Model):
def get_tag_by_id(self):
try:
tag_id_list = []
tags = PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True)
for i in tags:
tag_id_list.append(i)
tag_id_list = list(PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
return tag_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
return []
def get_tag_by_name(self, tag_id):
try:
tag_name_list = []
tags = Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True)
for i in tags:
tag_name_list.append(i)
tag_name_list = list(Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True))
return tag_name_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
return []
class PictorialTag(models.Model):
......
trans2es/models/topic.py
View file @ 7cf6330b
......@@ -130,6 +130,20 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
@classmethod
def get_topic_image_num(cls,topic_id):
"""
:remark 获取指定帖子的图片数量
:param topic_id:
:return:
"""
try:
query_list = list(TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False, is_online=True).values_list("url",flat=True))
return len(query_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
def topic_has_image(self):
try:
has_image = False
......@@ -213,10 +227,6 @@ class Topic(models.Model):
elif user_query_results[0].is_shadow:
user_is_shadow = True
# 是否官方推荐小组
# if self.group and self.group.is_recommend:
# offline_score += 4.0
# 帖子等级
if self.content_level == '5':
offline_score += 100.0 *3
......@@ -225,20 +235,8 @@ class Topic(models.Model):
elif self.content_level == '6':
offline_score += 200.0 *3
# is_excellent = self.judge_if_excellent_topic(self.id)
# if is_excellent:
# offline_score += 200.0
if self.language_type == 1:
offline_score += 60.0
# exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
# click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
# uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
#
# if exposure_count > 0:
# offline_score += click_count / exposure_count
# if uv_num > 0:
# offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
"""
1:马甲账号是否对总分降权？
......
trans2es/type_info.py
View file @ 7cf6330b
......@@ -161,20 +161,20 @@ class TypeInfo(object):
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8")
else:
q = {
"query":{
"term":{
"id": data["id"]
}
},
"_source":{
"includes":["content_level"]
}
}
search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
if len(search_results["hits"]) > 0:
ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
# else:
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "includes":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star:
# data_list = [
......
trans2es/utils/pictorial_transfer.py
View file @ 7cf6330b
......@@ -5,13 +5,39 @@ import sys
import logging
import traceback
from libs.tools import tzlc
from trans2es.models.topic import Topic
class PictorialTransfer(object):
def __init__(self):
pass
@classmethod
def get_offline_score(cls,instance,topic_id_list):
try:
total_offline_score = 0
topic_image_num = 0
for topic_id in topic_id_list:
topic_image_num += Topic.get_topic_image_num(topic_id)
if topic_image_num>=6 and topic_image_num<=10:
total_offline_score += 30
elif topic_image_num>10 and topic_image_num<=20:
total_offline_score += 60
elif topic_image_num>20 and topic_image_num<=50:
total_offline_score += 80
elif topic_image_num>50:
total_offline_score += 100
total_offline_score += instance.add_score
if instance.is_recommend:
total_offline_score += 100
return total_offline_score
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
@classmethod
def get_poctorial_data(cls, instance):
try:
......@@ -33,12 +59,13 @@ class PictorialTransfer(object):
tzlc_udpate_time = tzlc(update_time)
res["update_time"] = tzlc_udpate_time
res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
tag_id = instance.get_tag_by_id()
res["tag_id"] = tag_id
res["tag_name"] = instance.get_tag_by_name(tag_id)
tag_id_list = instance.get_tag_by_id()
res["tag_id"] = tag_id_list
res["tag_name"] = instance.get_tag_by_name(tag_id_list)
res["topic_id_list"] =instance.get_topic_id()
res["effective"] = instance.get_effective(res["topic_id_list"])
res["offline_score"] = cls.get_offline_score(instance,res["topic_id_list"])
res["is_default"] = instance.is_default
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/topic_transfer.py
View file @ 7cf6330b
......@@ -29,6 +29,10 @@ class TopicTransfer(object):
res["content_level"] = instance.content_level
res["user_id"] = instance.user_id
res["user_nick_name"] = User.get_user_nick_name(instance.user_id)
try:
res["user_nick_name_pre"] = res["user_nick_name"].lower()
except:
res["user_nick_name_pre"] = ""
if instance.group:
res["group_id"] = instance.group.id
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment