Skip to content

P
physical
Commit af32cc9d authored by Kai's avatar Kai
Browse files
Options

Merge branch 'hk' of git.wanmeizhensuo.com:alpha/physical into hk

parents ad489dea 5f58738a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 214 additions and 191 deletions
app_conf.xml
View file @ af32cc9d
...@@ -17,6 +17,5 @@ ...@@ -17,6 +17,5 @@
<element value="search.views.contrast_similar"/> <element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/> <element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/> <element value="search.views.contrast_similar"/>
<!--<element value="majia..tasks.tasks"/>-->
</config> </config>
</gm_rpcd_config> </gm_rpcd_config>
majia/get_session.py → injection/data_sync/get_session.py
View file @ af32cc9d
...@@ -8,15 +8,15 @@ import logging ...@@ -8,15 +8,15 @@ import logging
DATABASES = settings.DATABASES DATABASES = settings.DATABASES
# host = DATABASES['default']['HOST'] host = DATABASES['default']['HOST']
#
# user = DATABASES['default']['USER'] user = DATABASES['default']['USER']
#
# port = DATABASES['default']['PORT'] port = DATABASES['default']['PORT']
#
# db = DATABASES['default']['NAME'] db = DATABASES['default']['NAME']
#
# passwd = DATABASES['default']['PASSWORD'] passwd = DATABASES['default']['PASSWORD']
auto_click_url = "http://earth.igengmei.com/api/v1/like" auto_click_url = "http://earth.igengmei.com/api/v1/like"
auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create" auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
...@@ -26,7 +26,7 @@ auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge" ...@@ -26,7 +26,7 @@ auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
def get_majia(): def get_majia():
try: try:
data = open("/srv/apps/physical/majia/majia_email.txt") data = open("/srv/apps/physical/injection/data_sync/majia_email.txt")
list = [] list = []
for i in data: for i in data:
list.append(i.strip('\n').strip(',')) list.append(i.strip('\n').strip(','))
...@@ -41,7 +41,7 @@ def get_majia(): ...@@ -41,7 +41,7 @@ def get_majia():
def get_cookies(user_id): def get_cookies(user_id):
try: try:
with open("/srv/apps/physical/majia/user_session.txt", 'r') as f: with open("/srv/apps/cybertron/user_session.txt", 'r') as f:
try: try:
data = f.readlines() data = f.readlines()
except: except:
...@@ -123,7 +123,7 @@ def time_conv(minutest): ...@@ -123,7 +123,7 @@ def time_conv(minutest):
def get_comments(): def get_comments():
try: try:
data = open("/srv/apps/physical/majia/guanshui.txt", "r") data = open("/srv/apps/physical/injection/data_sync/guanshui.txt", "r")
list_guanshui = [] list_guanshui = []
for i in data: for i in data:
list_guanshui.append(i) list_guanshui.append(i)
......
injection/data_sync/majia_email.txt 0 → 100644
View file @ af32cc9d
This diff is collapsed.
injection/data_sync/tasks.py
View file @ af32cc9d
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
import logging import logging
import traceback import traceback
import json import json
import pymysql
import threading
import random
import datetime
from celery import shared_task from celery import shared_task
from django.conf import settings from django.conf import settings
from django.core import serializers from django.core import serializers
...@@ -9,8 +13,14 @@ from trans2es.type_info import get_type_info_map ...@@ -9,8 +13,14 @@ from trans2es.type_info import get_type_info_map
# from rpc.all import get_rpc_remote_invoker # from rpc.all import get_rpc_remote_invoker
from libs.es import ESPerform from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag from linucb.utils.register_user_tag import RegisterUserTag
from injection.data_sync.get_session import get_comments, click, login, reply
from injection.data_sync.get_session import host, user, passwd, db
exec_count_click = 0
exec_count = 0
@shared_task @shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False): def write_to_es(es_type, pk_list, use_batch_query_set=False):
...@@ -67,26 +77,161 @@ def sync_face_similar_data_to_redis(): ...@@ -67,26 +77,161 @@ def sync_face_similar_data_to_redis():
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@shared_task @shared_task
def sync_user_similar_score(): def sync_user_similar_score():
try: try:
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True) results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
"user_id", flat=True)
redis_key_prefix = "physical:user_similar_score:user_id:" redis_key_prefix = "physical:user_similar_score:user_id:"
logging.info("begin sync user_similar_score!") logging.info("begin sync user_similar_score!")
for user_id in results_items: for user_id in results_items:
redis_key = redis_key_prefix + str(user_id) redis_key = redis_key_prefix + str(user_id)
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score") similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
"-score")
item_list = list() item_list = list()
for item in similar_results_items: for item in similar_results_items:
contrast_user_id = item.contrast_user_id contrast_user_id = item.contrast_user_id
score = item.score score = item.score
item_list.append( item_list.append(
[contrast_user_id,score] [contrast_user_id, score]
) )
redis_client.set(redis_key, json.dumps(item_list)) redis_client.set(redis_key, json.dumps(item_list))
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def auto_click(pk_list):
logging.info("get--------auto_click--------------:%s" % pk_list)
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str(
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(
tuple(
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0 ) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
if topic_id_list:
try:
def fun_timer():
cookies = login()
if cookies:
logging.info("get topic_id_list:%s" % topic_id_list)
click(cookies, topic_id_list[0][0])
global timer
global exec_count_click
exec_count_click += 1
if exec_count_click == 1:
logging.info("----------2-----------")
# sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click == 2:
# sleep_time = random.randint(1000, 1900)
logging.info("----------3-----------")
sleep_time = 50
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click > 2:
pass
sleep_time = random.randint(300, 540)
logging.info("----------1-----------")
timer = threading.Timer(10, fun_timer) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
else:
pass
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s" % pk_list)
exec_count = 0
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
try:
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str( #
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%' " % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(tuple( # and create_time LIKE '%%%%%s%%%%'
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0) and and create_time LIKE '%%%%%s%%%%'"%yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
except:
pass
if topic_id_list:
try:
def fun_comment():
cookies = login()
if cookies:
comment_content = get_comments()
comment = comment_content[0]
reply(cookies, topic_id_list[0][0], comment)
global timer
global exec_count
exec_count += 1
if exec_count == 1:
sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
if exec_count == 2:
sleep_time = random.randint(1000, 1900)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
sleep_time = random.randint(300, 540)
timer = threading.Timer(10, fun_comment) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
libs/es.py
View file @ af32cc9d
...@@ -288,25 +288,40 @@ class ESPerform(object): ...@@ -288,25 +288,40 @@ class ESPerform(object):
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100): def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try: try:
functions_list = list() functions_list = list()
for id in tag_id: # for id in tag_id:
functions_list.append( # functions_list.append(
{ # {
"filter": {"term": {"tag_list": id}}, # "filter": {"term": {"tag_list": id}},
"weight": 1 # "weight": 1
} # }
) # )
functions_list += [ functions_list += [
{ {
"filter": {"term": {"content_level": 6}}, "filter": {
"weight": 6000 "constant_score":{
"filter":{
"term": {"content_level": 6}}
}
},
"weight": 60
}, },
{ {
"filter": {"term": {"content_level": 5}}, "filter": {
"weight": 5000 "constant_score":{
"filter":{
"term": {"content_level": 5}}
}
},
"weight": 50
}, },
{ {
"filter": {"term": {"content_level": 4}}, "filter": {
"weight": 4000 "constant_score":{
"filter":{
"term": {"content_level": 4}}
}
},
"weight": 40
} }
] ]
q = { q = {
...@@ -333,7 +348,7 @@ class ESPerform(object): ...@@ -333,7 +348,7 @@ class ESPerform(object):
"sort": [ "sort": [
{"_score": {"order": "desc"}}, {"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}}, {"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}}, # {"language_type": {"order": "asc"}},
] ]
} }
if len(have_read_topic_id_list) > 0: if len(have_read_topic_id_list) > 0:
......
majia/__init__.py deleted 100644 → 0
View file @ ad489dea
default_app_config = 'majia.apps.MaJiaApp'
majia/admin.py deleted 100644 → 0
View file @ ad489dea
from django.contrib import admin
# Register your models here.
majia/apps.py deleted 100644 → 0
View file @ ad489dea
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from django.apps import AppConfig
class MaJiaApp(AppConfig):
name = 'majia'
label = 'majia'
majia/models.py deleted 100644 → 0
View file @ ad489dea
from django.db import models
# Create your models here.
majia/tasks.py deleted 100644 → 0
View file @ ad489dea
# -*- coding: UTF-8 -*-
import pymysql
import random
import threading
import traceback
import logging
from celery import shared_task
from django.conf import settings
from majia.get_session import get_comments, click, login, reply
# from majia.get_session import host, user, passwd, db
exec_count = 0
def auto_click(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_timer():
# cookies = login()
# if cookies:
# click(cookies, topic_id_list[0])
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# # sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# if exec_count == 2:
# # sleep_time = random.randint(1000, 1900)
# sleep_time = 50
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_timer) # 首次启动
# timer.start()
#
# else:
# pass
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_comment():
# cookies = login()
# if cookies:
# comment_content = get_comments()
# comment = comment_content[0]
# reply(cookies, topic_id_list[0], comment)
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# if exec_count == 2:
# sleep_time = random.randint(1000, 1900)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_comment) # 首次启动
# timer.start()
#
#
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
@shared_task
def auto_majia_effect(pk_list):
auto_click(pk_list)
auto_reply(pk_list)
majia/tests.py deleted 100644 → 0
View file @ ad489dea
from django.test import TestCase
# Create your tests here.
majia/views.py deleted 100644 → 0
View file @ ad489dea
from django.shortcuts import render
# Create your views here.
physical/celery_task_router.py
View file @ af32cc9d
...@@ -9,10 +9,7 @@ class CeleryTaskRouter(object): ...@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
queue_task_map = { queue_task_map = {
"tapir-alpha": [ "tapir-alpha": [
'injection.data_sync.tasks.write_to_es', 'injection.data_sync.tasks.write_to_es',
], ]
"majia-alpha": [
'majia.tasks.auto_majia_effect',
],
} }
# Map[TaskName, QueueName] # Map[TaskName, QueueName]
......
search/utils/topic.py
View file @ af32cc9d
...@@ -177,24 +177,28 @@ class TopicUtils(object): ...@@ -177,24 +177,28 @@ class TopicUtils(object):
}, },
{ {
"filter": { "filter": {
"term": { "constant_score":{
"content_level": 6 "filter":{
"term": {
"content_level": 6
}
}
} }
}, },
"weight": 1000 "weight": 600
} }
] ]
if len(user_similar_score_list) > 0: # if len(user_similar_score_list) > 0:
for item in user_similar_score_list[:100]: # for item in user_similar_score_list[:100]:
score_item = 2 + item[1] # score_item = 2 + item[1]
functions_list.append( # functions_list.append(
{ # {
"filter": {"bool": { # "filter": {"bool": {
"should": {"term": {"user_id": item[0]}}}}, # "should": {"term": {"user_id": item[0]}}}},
"weight": score_item, # "weight": score_item,
} # }
) # )
if len(attention_user_id_list) > 0: if len(attention_user_id_list) > 0:
functions_list.append( functions_list.append(
......
search/views/topic.py
View file @ af32cc9d
...@@ -73,10 +73,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -73,10 +73,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if offset>0: # 首次搜索时不需要过滤已读 if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id) # user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key) # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
user_similar_score_redis_list = json.loads( # user_similar_score_redis_list = json.loads(
redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] # redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
attention_tag_list = list() attention_tag_list = list()
recommend_topic_list = list() recommend_topic_list = list()
......
trans2es/mapping/topic-v1.json
View file @ af32cc9d
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment