Skip to content

P
physical
Commit 5f58738a authored by 黄凯's avatar 黄凯
Browse files
Options

Merge branch 'test' into 'hk' 

# Conflicts:
#   linucb/views/collect_data.py
#   search/views/topic.py
parents e11006a6 f4fea10d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 214 additions and 191 deletions
app_conf.xml
View file @ 5f58738a
......@@ -17,6 +17,5 @@
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
<!--<element value="majia..tasks.tasks"/>-->
</config>
</gm_rpcd_config>
majia/get_session.py → injection/data_sync/get_session.py
View file @ 5f58738a
......@@ -8,15 +8,15 @@ import logging
DATABASES = settings.DATABASES
# host = DATABASES['default']['HOST']
#
# user = DATABASES['default']['USER']
#
# port = DATABASES['default']['PORT']
#
# db = DATABASES['default']['NAME']
#
# passwd = DATABASES['default']['PASSWORD']
host = DATABASES['default']['HOST']
user = DATABASES['default']['USER']
port = DATABASES['default']['PORT']
db = DATABASES['default']['NAME']
passwd = DATABASES['default']['PASSWORD']
auto_click_url = "http://earth.igengmei.com/api/v1/like"
auto_reply_url = "http://earth.igengmei.com/api/v1/reply/create"
......@@ -26,7 +26,7 @@ auto_urge_url = "http://earth.igengmei.com/api/v1/user/urge"
def get_majia():
try:
data = open("/srv/apps/physical/majia/majia_email.txt")
data = open("/srv/apps/physical/injection/data_sync/majia_email.txt")
list = []
for i in data:
list.append(i.strip('\n').strip(','))
......@@ -41,7 +41,7 @@ def get_majia():
def get_cookies(user_id):
try:
with open("/srv/apps/physical/majia/user_session.txt", 'r') as f:
with open("/srv/apps/cybertron/user_session.txt", 'r') as f:
try:
data = f.readlines()
except:
......@@ -123,7 +123,7 @@ def time_conv(minutest):
def get_comments():
try:
data = open("/srv/apps/physical/majia/guanshui.txt", "r")
data = open("/srv/apps/physical/injection/data_sync/guanshui.txt", "r")
list_guanshui = []
for i in data:
list_guanshui.append(i)
......
injection/data_sync/majia_email.txt 0 → 100644
View file @ 5f58738a
This diff is collapsed.
injection/data_sync/tasks.py
View file @ 5f58738a
......@@ -2,6 +2,10 @@
import logging
import traceback
import json
import pymysql
import threading
import random
import datetime
from celery import shared_task
from django.conf import settings
from django.core import serializers
......@@ -9,8 +13,14 @@ from trans2es.type_info import get_type_info_map
# from rpc.all import get_rpc_remote_invoker
from libs.es import ESPerform
from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag
from injection.data_sync.get_session import get_comments, click, login, reply
from injection.data_sync.get_session import host, user, passwd, db
exec_count_click = 0
exec_count = 0
@shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False):
......@@ -67,26 +77,161 @@ def sync_face_similar_data_to_redis():
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@shared_task
def sync_user_similar_score():
try:
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list("user_id",flat=True)
results_items = UserSimilarScore.objects.filter(is_deleted=False).distinct().values("user_id").values_list(
"user_id", flat=True)
redis_key_prefix = "physical:user_similar_score:user_id:"
logging.info("begin sync user_similar_score!")
for user_id in results_items:
redis_key = redis_key_prefix + str(user_id)
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False,user_id=user_id).order_by("-score")
similar_results_items = UserSimilarScore.objects.filter(is_deleted=False, user_id=user_id).order_by(
"-score")
item_list = list()
for item in similar_results_items:
contrast_user_id = item.contrast_user_id
score = item.score
item_list.append(
[contrast_user_id,score]
[contrast_user_id, score]
)
redis_client.set(redis_key, json.dumps(item_list))
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def auto_click(pk_list):
logging.info("get--------auto_click--------------:%s" % pk_list)
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str(
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(
tuple(
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0 ) and create_time LIKE '%%%%%s%%%%'" % yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
if topic_id_list:
try:
def fun_timer():
cookies = login()
if cookies:
logging.info("get topic_id_list:%s" % topic_id_list)
click(cookies, topic_id_list[0][0])
global timer
global exec_count_click
exec_count_click += 1
if exec_count_click == 1:
logging.info("----------2-----------")
# sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click == 2:
# sleep_time = random.randint(1000, 1900)
logging.info("----------3-----------")
sleep_time = 50
timer = threading.Timer(sleep_time, fun_timer)
timer.start()
if exec_count_click > 2:
pass
sleep_time = random.randint(300, 540)
logging.info("----------1-----------")
timer = threading.Timer(10, fun_timer) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
else:
pass
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s" % pk_list)
exec_count = 0
try:
now = datetime.datetime.now()
yes_time_str = now.strftime('%Y-%m-%d')
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
topic_id_list = []
try:
if len(pk_list) > 0:
if len(pk_list) == 1:
cursor.execute(
"SELECT id FROM topic WHERE id = " + str( #
pk_list[
0]) + " and user_id in (select user_id from user_extra where is_shadow = 0) and create_time LIKE '%%%%%s%%%%' " % yes_time_str)
else:
cursor.execute(
"SELECT id FROM topic WHERE id in " + str(tuple( # and create_time LIKE '%%%%%s%%%%'
pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0) and and create_time LIKE '%%%%%s%%%%'"%yes_time_str)
data = cursor.fetchall()
topic_id_list = list(data)
logging.info("Database version : %s " % topic_id_list)
pc.close()
except:
pass
if topic_id_list:
try:
def fun_comment():
cookies = login()
if cookies:
comment_content = get_comments()
comment = comment_content[0]
reply(cookies, topic_id_list[0][0], comment)
global timer
global exec_count
exec_count += 1
if exec_count == 1:
sleep_time = random.randint(300, 540)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
if exec_count == 2:
sleep_time = random.randint(1000, 1900)
sleep_time = 30
timer = threading.Timer(sleep_time, fun_comment)
timer.start()
sleep_time = random.randint(300, 540)
timer = threading.Timer(10, fun_comment) # 首次启动
timer.start()
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
except:
logging.error("catch exception,main:%s" % traceback.format_exc())
libs/es.py
View file @ 5f58738a
......@@ -288,25 +288,40 @@ class ESPerform(object):
def get_tag_topic_list(cls, tag_id, have_read_topic_id_list, size=100):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
# for id in tag_id:
# functions_list.append(
# {
# "filter": {"term": {"tag_list": id}},
# "weight": 1
# }
# )
functions_list += [
{
"filter": {"term": {"content_level": 6}},
"weight": 6000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 6}}
}
},
"weight": 60
},
{
"filter": {"term": {"content_level": 5}},
"weight": 5000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 5}}
}
},
"weight": 50
},
{
"filter": {"term": {"content_level": 4}},
"weight": 4000
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 4}}
}
},
"weight": 40
}
]
q = {
......@@ -333,7 +348,7 @@ class ESPerform(object):
"sort": [
{"_score": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
# {"language_type": {"order": "asc"}},
]
}
if len(have_read_topic_id_list) > 0:
......
majia/__init__.py deleted 100644 → 0
View file @ e11006a6
default_app_config = 'majia.apps.MaJiaApp'
majia/admin.py deleted 100644 → 0
View file @ e11006a6
from django.contrib import admin
# Register your models here.
majia/apps.py deleted 100644 → 0
View file @ e11006a6
# coding=utf-8
from __future__ import unicode_literals, print_function, absolute_import
from django.apps import AppConfig
class MaJiaApp(AppConfig):
name = 'majia'
label = 'majia'
majia/models.py deleted 100644 → 0
View file @ e11006a6
from django.db import models
# Create your models here.
majia/tasks.py deleted 100644 → 0
View file @ e11006a6
# -*- coding: UTF-8 -*-
import pymysql
import random
import threading
import traceback
import logging
from celery import shared_task
from django.conf import settings
from majia.get_session import get_comments, click, login, reply
# from majia.get_session import host, user, passwd, db
exec_count = 0
def auto_click(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_timer():
# cookies = login()
# if cookies:
# click(cookies, topic_id_list[0])
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# # sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# if exec_count == 2:
# # sleep_time = random.randint(1000, 1900)
# sleep_time = 50
# timer = threading.Timer(sleep_time, fun_timer)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_timer) # 首次启动
# timer.start()
#
# else:
# pass
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
def auto_reply(pk_list):
logging.info("get----------------------:%s"%pk_list)
# exec_count = 0
# try:
# pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
# cursor = pc.cursor()
# cursor.execute(
# "SELECT id,user_id FROM topic WHERE id in " + str(
# tuple(pk_list)) + " and user_id in (select user_id from user_extra where is_shadow = 0)")
# data = cursor.fetchall()
# topic_id_list = list(data)
# logging.info("Database version : %s " % topic_id_list)
# pc.close()
# exec_count = 0
# if topic_id_list:
#
# def fun_comment():
# cookies = login()
# if cookies:
# comment_content = get_comments()
# comment = comment_content[0]
# reply(cookies, topic_id_list[0], comment)
#
# global timer
# global exec_count
#
# exec_count += 1
# if exec_count == 1:
# sleep_time = random.randint(300, 540)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# if exec_count == 2:
# sleep_time = random.randint(1000, 1900)
# sleep_time = 30
# timer = threading.Timer(sleep_time, fun_comment)
# timer.start()
#
# sleep_time = random.randint(300, 540)
# timer = threading.Timer(10, fun_comment) # 首次启动
# timer.start()
#
#
# except:
# logging.error("catch exception,main:%s" % traceback.format_exc())
@shared_task
def auto_majia_effect(pk_list):
auto_click(pk_list)
auto_reply(pk_list)
majia/tests.py deleted 100644 → 0
View file @ e11006a6
from django.test import TestCase
# Create your tests here.
majia/views.py deleted 100644 → 0
View file @ e11006a6
from django.shortcuts import render
# Create your views here.
physical/celery_task_router.py
View file @ 5f58738a
......@@ -9,10 +9,7 @@ class CeleryTaskRouter(object):
queue_task_map = {
"tapir-alpha": [
'injection.data_sync.tasks.write_to_es',
],
"majia-alpha": [
'majia.tasks.auto_majia_effect',
],
]
}
# Map[TaskName, QueueName]
......
search/utils/topic.py
View file @ 5f58738a
......@@ -177,24 +177,28 @@ class TopicUtils(object):
},
{
"filter": {
"term": {
"content_level": 6
"constant_score":{
"filter":{
"term": {
"content_level": 6
}
}
}
},
"weight": 1000
"weight": 600
}
]
if len(user_similar_score_list) > 0:
for item in user_similar_score_list[:100]:
score_item = 2 + item[1]
functions_list.append(
{
"filter": {"bool": {
"should": {"term": {"user_id": item[0]}}}},
"weight": score_item,
}
)
# if len(user_similar_score_list) > 0:
# for item in user_similar_score_list[:100]:
# score_item = 2 + item[1]
# functions_list.append(
# {
# "filter": {"bool": {
# "should": {"term": {"user_id": item[0]}}}},
# "weight": score_item,
# }
# )
if len(attention_user_id_list) > 0:
functions_list.append(
......
search/views/topic.py
View file @ 5f58738a
......@@ -73,10 +73,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
user_similar_score_redis_list = json.loads(
redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
attention_tag_list = list()
recommend_topic_list = list()
......
trans2es/mapping/topic-v1.json
View file @ 5f58738a
......@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment