Skip to content

P
physical
Commit 7cf6330b authored by Kai's avatar Kai
Browse files
Options

update

parents 1856f464 374fb99d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 340 additions and 469 deletions
.gitignore
View file @ 7cf6330b
...@@ -85,4 +85,5 @@ gaia/rpcd.json ...@@ -85,4 +85,5 @@ gaia/rpcd.json
dbmw_deploy/config.dir/ dbmw_deploy/config.dir/
.idea/codeStyles/Project.xml .idea/codeStyles/Project.xml
.idea/misc.xml .idea/misc.xml
.idea/physical.iml .idea/physical.iml
\ No newline at end of file .idea/codeStyles/Project.xml
\ No newline at end of file
.idea/codeStyles/Project.xml
View file @ 7cf6330b
...@@ -24,5 +24,29 @@ ...@@ -24,5 +24,29 @@
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" /> <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings> </formatting-settings>
</DBN-SQL> </DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme> </code_scheme>
</component> </component>
\ No newline at end of file
.idea/misc.xml
View file @ 7cf6330b
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser"> <component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" /> <option name="shown" value="true" />
</component> </component>
......
.idea/physical.iml
View file @ 7cf6330b
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4"> <module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager"> <component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" /> <content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.6 (venv)" jdkType="Python SDK" /> <orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
<component name="TestRunnerService"> <component name="TestRunnerService">
......
app_conf.xml
View file @ 7cf6330b
...@@ -17,6 +17,5 @@ ...@@ -17,6 +17,5 @@
<element value="search.views.contrast_similar"/> <element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/> <element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/> <element value="search.views.contrast_similar"/>
<!--<element value="majia..tasks.tasks"/>-->
</config> </config>
</gm_rpcd_config> </gm_rpcd_config>
injection/data_sync/tasks.py
View file @ 7cf6330b
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
import logging import logging
import traceback import traceback
import json import json
import pymysql
import threading
import random
import datetime
from celery import shared_task from celery import shared_task
from django.conf import settings from django.conf import settings
from django.core import serializers from django.core import serializers
...@@ -9,9 +13,10 @@ from trans2es.type_info import get_type_info_map ...@@ -9,9 +13,10 @@ from trans2es.type_info import get_type_info_map
# from rpc.all import get_rpc_remote_invoker # from rpc.all import get_rpc_remote_invoker
from libs.es import ESPerform from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag from linucb.utils.register_user_tag import RegisterUserTag
@shared_task @shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False): def write_to_es(es_type, pk_list, use_batch_query_set=False):
try: try:
...@@ -55,35 +60,47 @@ def sync_face_similar_data_to_redis(): ...@@ -55,35 +60,47 @@ def sync_face_similar_data_to_redis():
item_list = list() item_list = list()
for item in similar_result_items: for item in similar_result_items:
weight_score = int(item.similarity * 100)
item_list.append( item_list.append(
{ {
"contrast_user_id": item.contrast_user_id, "filter":{
"similarity": item.similarity "constant_score":{
"filter":{
"term":{"user_id": item.contrast_user_id}
}
}
},
"weight": weight_score*2
} }
) )
if len(item_list)>=100:
break
redis_client.set(redis_key, json.dumps(item_list)) redis_client.set(redis_key, json.dumps(item_list))
logging.info("participant_user_id:%d set data done!" % participant_user_id) logging.info("participant_user_id:%d set data done!" % participant_user_id)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@shared_task @shared_task
def sync_user_similar_score(): def sync_user_similar_score():
try: try:
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True) results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
"user_id", flat=True)
redis_key_prefix = "physical:user_similar_score:user_id:" redis_key_prefix = "physical:user_similar_score:user_id:"
logging.info("begin sync user_similar_score!") logging.info("begin sync user_similar_score!")
for user_id in results_items: for user_id in results_items:
redis_key = redis_key_prefix + str(user_id) redis_key = redis_key_prefix + str(user_id)
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score") similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
"-score")
item_list = list() item_list = list()
for item in similar_results_items: for item in similar_results_items:
contrast_user_id = item.contrast_user_id contrast_user_id = item.contrast_user_id
score = item.score score = item.score
item_list.append( item_list.append(
[contrast_user_id,score] [contrast_user_id, score]
) )
redis_client.set(redis_key, json.dumps(item_list)) redis_client.set(redis_key, json.dumps(item_list))
......
libs/error.py
View file @ 7cf6330b
...@@ -18,4 +18,4 @@ def logging_exception(send_to_sentry=True): ...@@ -18,4 +18,4 @@ def logging_exception(send_to_sentry=True):
# send exception info to sentry, fail silently # send exception info to sentry, fail silently
_sentry_client.captureException() _sentry_client.captureException()
except: except:
pass passraven.contrib.django.raven_compat.models
libs/es.py
View file @ 7cf6330b
...@@ -366,3 +366,78 @@ class ESPerform(object): ...@@ -366,3 +366,78 @@ class ESPerform(object):
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list() return list()
@classmethod
def get_tag_topic_list_dict(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# functions_list += [
# {
# "filter": {"term": {"content_level": 6}},
# "weight": 6000
# },
# {
# "filter": {"term": {"content_level": 5}},
# "weight": 5000
# },
# {
# "filter": {"term": {"content_level": 4}},
# "weight": 4000
# }
# ]
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id","user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
# logging.info("topic_id_list:%s" % str(topic_id_list))
# topic_id_dict = [{str(item["_source"]["id"]):item["_source"]["user_id"]} for item in result_dict["hits"]]
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list,topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
linucb/views/collect_data.py
View file @ 7cf6330b
This diff is collapsed.
majia/__init__.py deleted 100644 → 0
View file @ 1856f464
default_app_config = 'majia.apps.MaJiaApp'
majia/admin.py deleted 100644 → 0
View file @ 1856f464
from django.contrib import admin
# Register your models here.
majia/apps.py deleted 100644 → 0
View file @ 1856f464
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from django.apps import AppConfig
class MaJiaApp(AppConfig):
name = 'majia'
label = 'majia'
majia/get_session.py deleted 100644 → 0
View file @ 1856f464
import requests
import time
import datetime
import random
import traceback
from django.conf import settings
import logging
DATABASES = settings.DATABASES
# host = DATABASES['default']['HOST']
#
# user = DATABASES['default']['USER']
#
# port = DATABASES['default']['PORT']
#
# db = DATABASES['default']['NAME']
#
# passwd = DATABASES['default']['PASSWORD']
auto_click_url = "http://earth.igengmei.com/api/v1/like"
auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
auto_follow_url = "http://earth.igengmei.com/api/v1/follow"
auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
def get_majia():
try:
data = open("/srv/apps/physical/majia/majia_email.txt")
list = []
for i in data:
list.append(i.strip('\n').strip(','))
maj = random.randint(1, len(list))
user_id = list[maj - 1]
return user_id
except:
logging.error("catch exception,get_majia:%s" % traceback.format_exc())
return None
def get_cookies(user_id):
try:
with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
try:
data = f.readlines()
except:
data = None
list = []
for i in data:
list.append(i.strip('\n').split(","))
for i in list:
if user_id == i[0]:
return (i[1])
except:
return None
def login():
try:
user_id = get_majia()
cookies = get_cookies(user_id)
if cookies is not None:
return {'sessionid': cookies}
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
return None
def click(cookies_get, id):
try:
post_dict = {
'type': 0,
'id': id
}
response = requests.post(url=auto_click_url,
cookies=cookies_get,
data=post_dict)
logging.info("response.text:%s" % response.text)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
def reply(cookies_get, id, content):
try:
post_dict = {
'topic_id': id,
'content': content
}
response = requests.post(url=auto_reply_url,
cookies=cookies_get,
data=post_dict)
logging.info("response.text:%s" % response.text)
except:
logging.error("catch exception,logins:%s" % traceback.format_exc())
def time_conv(minutest):
try:
now = datetime.datetime.now()
yes_time = now - datetime.timedelta(minutes=minutest)
print(yes_time)
return yes_time
except:
return None
def get_comments():
try:
data = open("/srv/apps/physical/majia/guanshui.txt", "r")
list_guanshui = []
for i in data:
list_guanshui.append(i)
num1 = random.randint(0, 30)
num2 = random.randint(30, 70)
num3 = random.randint(70, len(list_guanshui))
comment1 = list_guanshui[num1 - 1]
comment2 = list_guanshui[num2 - 1]
comment3 = list_guanshui[num3 - 1]
comment_list = [comment1, comment2, comment3]
return comment_list
except:
return None
majia/guanshui.txt deleted 100644 → 0
View file @ 1856f464
照片色调很好看
用的什么调色工具呀
立马去get同款
我一个一看就会，动手就废的girl
为什么我拍不出来你的效果？
Po主用的什么拍照软件
这个好，学习了
真的是元气满满
很帅气！
这样好显气质！
好可爱啊，完全没有抵抗力了。
莫名好感中
满分颜值
超酷！
马住回看
请问发色是啥
请问是什么app拍的啊
滤镜要怎么选？
卖萌打滚求翻牌
有没有常用自拍姿势教程
拍照技术满分
日常夸奖博主
日常勾搭
背景是在哪里？
照片构图很好看
主要还是看颜值
你长得很像明星
支持一下
转需留存
拍照表情很自然
风格很好
这里是哪里呀？
种草了！
搭配小技巧get！
怎么都学不会的路过
太好了，新手学习了
一直支持，只求被翻牌！
好喜欢这种风格
好看，在哪儿买的
衣好有气质哦
这个我也会
拍照pose求出教程
小姐姐，作图app是啥
终于更新了
搭配的好好看
求同款
日常催更
get√
什么时候更新啊
果然只要瘦怎么都好看。
不错的哈哈
同款达不到楼主的效果
喜欢你的搭配
我拍出来为什么感觉比本人丑很多？
能多发点全身正面照吗？想看整体效果
这个真的很真实了
大圆脸合适吗？
成功引起了我的注意！
感觉很少女~
不错，继续加油
好看的人做什么都好看
都是人才！！！！
道理我都懂，就差一个脸！
不赞我良心都过意不去啊
爱你哟~
用手机可以拍出来吗？
使用相机拍的还是手机
我觉得很赞
颜色搭配很舒服
小姐姐有戴美瞳吗
看见这个帖心情变好了
不管怎么样先收藏了
更新更新更新
你和我朋友长得有点像
你的风格我都很喜欢
好看
我要有你这条件还需要啥拍照技术
太美了！
被圈粉了……
美而不作，我喜欢
楼主真的real真实了。
持续等更新
楼主加油，照片拍的不错，非常值得推荐。
大家花呗还了吗？
来啦来啦来啦来啦来啦
画面太治愈了
楼主辛苦啦
不错，跟着学习一下
心动，我要去试试
满满的干货，收藏好下次用
新技能get√
感觉不错mark一下
适合自己的才是最重要的
羡慕一下作者的颜
Omaga太好看了吧~
这张拍的不错
这个特效挺可爱的
翻我翻我~
这样很显脸小
关注啦
majia/models.py deleted 100644 → 0
View file @ 1856f464
from django.db import models
# Create your models here.
majia/tasks.py deleted 100644 → 0
View file @ 1856f464
# -*- coding: UTF-8 -*-
import pymysql
import random
import threading
import traceback
import logging
from celery import shared_task
from django.conf import settings
from majia.get_session import get_comments, click, login, reply
# from majia.get_session import host, user, passwd, db
exec_count = 0
def auto_click(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_timer():
# cookies = login()
# if cookies:
# click(cookies, topic_id_list[0])
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# # sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# if exec_count == 2:
# # sleep_time = random.randint(1000, 1900)
# sleep_time = 50
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_timer) # 首次启动
# timer.start()
#
# else:
# pass
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_comment():
# cookies = login()
# if cookies:
# comment_content = get_comments()
# comment = comment_content[0]
# reply(cookies, topic_id_list[0], comment)
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# if exec_count == 2:
# sleep_time = random.randint(1000, 1900)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_comment) # 首次启动
# timer.start()
#
#
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
@shared_task
def auto_majia_effect(pk_list):
auto_click(pk_list)
auto_reply(pk_list)
majia/tests.py deleted 100644 → 0
View file @ 1856f464
from django.test import TestCase
# Create your tests here.
majia/views.py deleted 100644 → 0
View file @ 1856f464
from django.shortcuts import render
# Create your views here.
physical/celery_task_router.py
View file @ 7cf6330b
...@@ -9,10 +9,7 @@ class CeleryTaskRouter(object): ...@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
queue_task_map = { queue_task_map = {
"tapir-alpha": [ "tapir-alpha": [
'injection.data_sync.tasks.write_to_es', 'injection.data_sync.tasks.write_to_es',
], ]
"majia-alpha": [
'majia.tasks.auto_majia_effect',
],
} }
# Map[TaskName, QueueName] # Map[TaskName, QueueName]
......
physical/settings_local.py.template
View file @ 7cf6330b
...@@ -48,7 +48,6 @@ INSTALLED_APPS = ( ...@@ -48,7 +48,6 @@ INSTALLED_APPS = (
'trans2es', 'trans2es',
'search', 'search',
'injection.data_sync', 'injection.data_sync',
'majia',
) )
CELERYBEAT_SCHEDULE = { CELERYBEAT_SCHEDULE = {
......
search/utils/group.py
View file @ 7cf6330b
...@@ -47,23 +47,79 @@ class GroupUtils(object): ...@@ -47,23 +47,79 @@ class GroupUtils(object):
return {"total_count": 0, "hits": []} return {"total_count": 0, "hits": []}
@classmethod @classmethod
def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None): def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None,attention_tag_list=[]):
try: try:
if not es_cli_obj: if not es_cli_obj:
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
q = dict() functions_list = list()
q["query"] = { for tag_id in attention_tag_list:
"bool":{ functions_list.append({
"must":[ "filter":{
{"term": {"is_online": True}}, "constant_score":{
{"term":{"is_deleted": False}}, "filter":{
{"term": {"effective": True}} "term":{
] "tag_id": tag_id
}
}
}
},
"weight": 20
})
if len(functions_list)>=20:
break
functions_list.append(
{
"gauss": {
"create_time": {
"scale": "1d",
"decay": 0.99
}
},
"weight": 60
}
)
q = {
"query":{
"function_score":{
"query":{
"bool":{
"filter":[
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"effective": True}}
],
"must_not":[
{"term":{"is_default":1}}
]
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
} }
} }
q["sort"] = [ q["sort"] = [
{"high_quality_topic_num":{"order":"desc"}} {
"_script": {
"type": "number",
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
# "lang":"painless",
# "source":"_score+params._source.offline_score"
},
"order": "desc"
}
},
{
"_score":{
"order": "desc"
}
}
] ]
q["_source"] = { q["_source"] = {
"includes":["id"] "includes":["id"]
......
search/utils/topic.py
View file @ 7cf6330b
This diff is collapsed.
search/views/business_topic.py
View file @ 7cf6330b
...@@ -25,7 +25,6 @@ def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size= ...@@ -25,7 +25,6 @@ def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=
index_name="topic" index_name="topic"
) )
logging.info("get result_lsit:%s"%result_list)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]] topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]} return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except: except:
......
search/views/group.py
View file @ 7cf6330b
...@@ -52,9 +52,15 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, ...@@ -52,9 +52,15 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
if sort_type == GroupSortTypes.HOT_RECOMMEND: if sort_type == GroupSortTypes.HOT_RECOMMEND:
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj) # 用户关注标签
return {"pictorial_recommend_ids": pictorial_ids_list} attention_tag_list = list()
if user_id>0:
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,attention_tag_list)
return {"pictorial_recommend_ids": pictorial_ids_list}
elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND: elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
attention_pictorial_list = GroupUtils.get_user_attention_pictorial_list(user_id, offset=0, size=1, attention_pictorial_list = GroupUtils.get_user_attention_pictorial_list(user_id, offset=0, size=1,
es_cli_obj=es_cli_obj) es_cli_obj=es_cli_obj)
......
search/views/topic.py
View file @ 7cf6330b
This diff is collapsed.
trans2es/mapping/pictorial.json
View file @ 7cf6330b
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
"tag_id":{"type":"long"}, "tag_id":{"type":"long"},
"tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"topic_id_list":{"type":"long"}, "topic_id_list":{"type":"long"},
"effective":{"type":"boolean"} "effective":{"type":"boolean"},
"offline_score":{"type":"long"},
"is_default":{"type":"long"}
} }
} }
\ No newline at end of file
trans2es/mapping/topic-high-star.json
View file @ 7cf6330b
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic-star-routing.json
View file @ 7cf6330b
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic-star.json
View file @ 7cf6330b
{ {
"dynamic":"strict", "dynamic":"strict",
"_routing": {"required": true},
"properties": { "properties": {
"id":{"type":"long"}, "id":{"type":"long"},
"content_level":{"type":"text"} "is_online":{"type":"boolean"},//上线
"is_deleted":{"type":"boolean"},
"vote_num":{"type":"long"},
"total_vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","index":"not_analyzed"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"has_video":{"type":"boolean"},//是否是视频
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"},
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
trans2es/mapping/topic-v1.json
View file @ 7cf6330b
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
...@@ -52,6 +53,7 @@ ...@@ -52,6 +53,7 @@
"search_analyzer": "gm_default_index" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
trans2es/mapping/topic.json
View file @ 7cf6330b
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/models/pictorial.py
View file @ 7cf6330b
...@@ -36,7 +36,6 @@ class PictorialTopics(models.Model): ...@@ -36,7 +36,6 @@ class PictorialTopics(models.Model):
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID') pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
topic_id = models.BigIntegerField(verbose_name=u'帖子ID') topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True) is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除') is_deleted = models.BooleanField(verbose_name=u'是否删除')
...@@ -59,18 +58,16 @@ class Pictorial(models.Model): ...@@ -59,18 +58,16 @@ class Pictorial(models.Model):
creator_id = models.BigIntegerField(verbose_name=u'画报用户ID') creator_id = models.BigIntegerField(verbose_name=u'画报用户ID')
icon = models.CharField(verbose_name=u'画报名称', max_length=255) icon = models.CharField(verbose_name=u'画报名称', max_length=255)
topic_num = models.IntegerField(verbose_name=u'次数') topic_num = models.IntegerField(verbose_name=u'次数')
add_score = models.IntegerField(verbose_name=u'人工权重')
is_default = models.IntegerField(verbose_name=u"是否是默认画报")
def get_topic_id(self): def get_topic_id(self):
try: try:
topic_id =[] topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=self.id,is_online=True,is_deleted=False).values_list("topic_id", flat=True))
topic_id_list = PictorialTopics.objects.filter(pictorial_id=self.id).values_list("topic_id", flat=True) return topic_id_list
for i in topic_id_list:
topic_id.append(i)
return topic_id
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return []
def get_effective(self,topic_id_list): def get_effective(self,topic_id_list):
...@@ -112,29 +109,21 @@ class Pictorial(models.Model): ...@@ -112,29 +109,21 @@ class Pictorial(models.Model):
def get_tag_by_id(self): def get_tag_by_id(self):
try: try:
tag_id_list = [] tag_id_list = list(PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
tags = PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True)
for i in tags:
tag_id_list.append(i)
return tag_id_list return tag_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return []
def get_tag_by_name(self, tag_id): def get_tag_by_name(self, tag_id):
try: try:
tag_name_list = [] tag_name_list = list(Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True))
tags = Tag.objects.filter(id__in=tag_id, is_online=True).values_list("name", flat=True)
for i in tags:
tag_name_list.append(i)
return tag_name_list return tag_name_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return []
class PictorialTag(models.Model): class PictorialTag(models.Model):
......
trans2es/models/topic.py
View file @ 7cf6330b
...@@ -130,6 +130,20 @@ class Topic(models.Model): ...@@ -130,6 +130,20 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return False
@classmethod
def get_topic_image_num(cls,topic_id):
"""
:remark 获取指定帖子的图片数量
:param topic_id:
:return:
"""
try:
query_list = list(TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False, is_online=True).values_list("url",flat=True))
return len(query_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
def topic_has_image(self): def topic_has_image(self):
try: try:
has_image = False has_image = False
...@@ -213,10 +227,6 @@ class Topic(models.Model): ...@@ -213,10 +227,6 @@ class Topic(models.Model):
elif user_query_results[0].is_shadow: elif user_query_results[0].is_shadow:
user_is_shadow = True user_is_shadow = True
# 是否官方推荐小组
# if self.group and self.group.is_recommend:
# offline_score += 4.0
# 帖子等级 # 帖子等级
if self.content_level == '5': if self.content_level == '5':
offline_score += 100.0 *3 offline_score += 100.0 *3
...@@ -225,20 +235,8 @@ class Topic(models.Model): ...@@ -225,20 +235,8 @@ class Topic(models.Model):
elif self.content_level == '6': elif self.content_level == '6':
offline_score += 200.0 *3 offline_score += 200.0 *3
# is_excellent = self.judge_if_excellent_topic(self.id)
# if is_excellent:
# offline_score += 200.0
if self.language_type == 1: if self.language_type == 1:
offline_score += 60.0 offline_score += 60.0
# exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
# click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
# uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
#
# if exposure_count > 0:
# offline_score += click_count / exposure_count
# if uv_num > 0:
# offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
""" """
1:马甲账号是否对总分降权？ 1:马甲账号是否对总分降权？
......
trans2es/type_info.py
View file @ 7cf6330b
...@@ -161,20 +161,20 @@ class TypeInfo(object): ...@@ -161,20 +161,20 @@ class TypeInfo(object):
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"]) ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star: if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8") ori_topic_star = str(ori_topic_star, encoding="utf-8")
else: # else:
q = { # q = {
"query":{ # "query":{
"term":{ # "term":{
"id": data["id"] # "id": data["id"]
} # }
}, # },
"_source":{ # "_source":{
"includes":["content_level"] # "includes":["content_level"]
} # }
} # }
search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1) # search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
if len(search_results["hits"]) > 0: # if len(search_results["hits"]) > 0:
ori_topic_star = search_results["hits"][0]["_source"]["content_level"] # ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star: if not ori_topic_star:
# data_list = [ # data_list = [
......
trans2es/utils/pictorial_transfer.py
View file @ 7cf6330b
...@@ -5,13 +5,39 @@ import sys ...@@ -5,13 +5,39 @@ import sys
import logging import logging
import traceback import traceback
from libs.tools import tzlc from libs.tools import tzlc
from trans2es.models.topic import Topic
class PictorialTransfer(object): class PictorialTransfer(object):
def __init__(self): def __init__(self):
pass pass
@classmethod
def get_offline_score(cls,instance,topic_id_list):
try:
total_offline_score = 0
topic_image_num = 0
for topic_id in topic_id_list:
topic_image_num += Topic.get_topic_image_num(topic_id)
if topic_image_num>=6 and topic_image_num<=10:
total_offline_score += 30
elif topic_image_num>10 and topic_image_num<=20:
total_offline_score += 60
elif topic_image_num>20 and topic_image_num<=50:
total_offline_score += 80
elif topic_image_num>50:
total_offline_score += 100
total_offline_score += instance.add_score
if instance.is_recommend:
total_offline_score += 100
return total_offline_score
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
@classmethod @classmethod
def get_poctorial_data(cls, instance): def get_poctorial_data(cls, instance):
try: try:
...@@ -33,12 +59,13 @@ class PictorialTransfer(object): ...@@ -33,12 +59,13 @@ class PictorialTransfer(object):
tzlc_udpate_time = tzlc(update_time) tzlc_udpate_time = tzlc(update_time)
res["update_time"] = tzlc_udpate_time res["update_time"] = tzlc_udpate_time
res["high_quality_topic_num"] = instance.get_high_quality_topic_num() res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
tag_id = instance.get_tag_by_id() tag_id_list = instance.get_tag_by_id()
res["tag_id"] = tag_id res["tag_id"] = tag_id_list
res["tag_name"] = instance.get_tag_by_name(tag_id) res["tag_name"] = instance.get_tag_by_name(tag_id_list)
res["topic_id_list"] =instance.get_topic_id() res["topic_id_list"] =instance.get_topic_id()
res["effective"] = instance.get_effective(res["topic_id_list"]) res["effective"] = instance.get_effective(res["topic_id_list"])
res["offline_score"] = cls.get_offline_score(instance,res["topic_id_list"])
res["is_default"] = instance.is_default
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/topic_transfer.py
View file @ 7cf6330b
...@@ -29,6 +29,10 @@ class TopicTransfer(object): ...@@ -29,6 +29,10 @@ class TopicTransfer(object):
res["content_level"] = instance.content_level res["content_level"] = instance.content_level
res["user_id"] = instance.user_id res["user_id"] = instance.user_id
res["user_nick_name"] = User.get_user_nick_name(instance.user_id) res["user_nick_name"] = User.get_user_nick_name(instance.user_id)
try:
res["user_nick_name_pre"] = res["user_nick_name"].lower()
except:
res["user_nick_name_pre"] = ""
if instance.group: if instance.group:
res["group_id"] = instance.group.id res["group_id"] = instance.group.id
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment