Skip to content

P
physical
Commit 37c1e29a authored by 吴升宇's avatar 吴升宇
Browse files
Options

fix 冲突

parents 5ec79551 696c2cae
Show whitespace changes
Inline Side-by-side
Showing with 183 additions and 67 deletions
crontab.py
View file @ 37c1e29a
......@@ -10,9 +10,9 @@ ontime_list = [
# "00,10,20,30,40,50 * * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m true_click_three",
# "02,12,22,32,42,52 * * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es-m true_click_four",
# "06,16,26,36,46,56 * * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m true_click_five",
"0 14 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_star_urge",
"0 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge1",
"30 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge2",
# "0 14 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_star_urge",
# "0 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge1",
# "30 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge2",
"0 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app",
"30 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app2",
# "*/5 * * * 1 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_5m_by_followed",
......@@ -22,7 +22,8 @@ ontime_list = [
# "0 14,18,22 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m principal_online_comment1",
"25 */1 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_2h_to_topic",
"0 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_1d_by_post",
"1 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
# "1 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
"0 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge_per_1d_by_regist",
"2 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_post",
"3 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
"4 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_topic"
......
libs/es.py
View file @ 37c1e29a
......@@ -422,7 +422,7 @@ class ESPerform(object):
"_script": {
"order": "desc",
"script": {
"inline": "10*doc['topic_ctr_30'].value+doc['like_rate_30'].value+2*doc['topic_ctr_all'].value+doc['like_rate_all'].value"
"inline": "3*doc['topic_ctr_30'].value+2*doc['topic_ctr_all'].value"
},
"type": "number"
}
......@@ -452,7 +452,7 @@ class ESPerform(object):
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("get_tag_topic_list_dict:gyz" + str(q) + str(result_dict))
logging.info("get_tag_topic_list_dict:zw" + str(q) + str(result_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -546,7 +546,7 @@ class ESPerform(object):
"_script": {
"order": "desc",
"script": {
"inline": "10*doc['pictorial_ctr_30'].value+10*doc['like_rate_30'].value+3*doc['pictorial_ctr_all'].value+2*doc['like_rate_all'].value"
"inline": "3*doc['pictorial_ctr_30'].value+2*doc['pictorial_ctr_all'].value"
},
"type": "number"
}
......@@ -567,7 +567,7 @@ class ESPerform(object):
offset=0, size=size)
pictorial_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("get_tag_pictorial_id_list:gyz" + str(q) + str(result_dict))
logging.info("get_tag_pictorial_id_list:zw" + str(q) + str(result_dict))
return pictorial_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
linucb/views/collect_data.py
View file @ 37c1e29a
# -*- coding: UTF-8 -*-
# !/usr/bin/env python
from kafka import KafkaConsumer
import random
from libs.cache import redis_client
......@@ -636,7 +633,7 @@ class CollectData(object):
logging.info("action=api/v1/cards/topic")
tag_name = data["APP"].get("query", [])
tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
name=tag_name).values_list("id",flat=True))
name=tag_name).values_list("id", flat=True))
device_id = data["SYS"]["cl_id"]
user_id = data['SYS'].get('user_id', None)
self.transfer_update_recommend_tag_list(device_id, user_feature, user_id,
......@@ -655,8 +652,7 @@ class CollectData(object):
tag_ids = list(data["APP"].get("tag_ids", []))
tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=tag_ids, is_online=True, is_deleted=False,
is_category=False).values_list("id",flat=True))
tag_query_results_multi = [i for i in tag_ids if i in tag_list]
is_category=False).values_list("id", flat=True))
device_id = data["SYS"]["cl_id"]
user_id = data['SYS'].get('user_id', None)
self.transfer_update_recommend_tag_list(device_id, user_feature, user_id,
......@@ -673,8 +669,7 @@ class CollectData(object):
tag_ids = list(data["APP"].get("tag_ids", []))
tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=tag_ids, is_online=True, is_deleted=False,
is_category=False).values_list("id",flat=True))
tag_query_results_multi = [i for i in tag_ids if i in tag_list]
is_category=False).values_list("id", flat=True))
device_id = data["SYS"]["cl_id"]
user_id = data['SYS'].get('user_id', None)
self.transfer_update_recommend_tag_list(device_id, user_feature, user_id,
......@@ -692,8 +687,8 @@ class CollectData(object):
logging.info('action:%s,tag_list:%s' % (action, str(tag_ids)))
tag_query_results = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=tag_ids, is_online=True, is_deleted=False,
is_category=False).values_list("id",flat=True))
# tag_query_results = [i[0] for i in tag_query_results]
is_category=False).values_list("id", flat=True))
tag_query_results = [i[0] for i in tag_query_results]
logging.info('action:%s,mysql query taglist:%s' % (action, str(tag_query_results)))
tag_query_results_multi = [i for i in tag_ids if i in tag_query_results]
......
search/views/group.py
View file @ 37c1e29a
......@@ -359,7 +359,7 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL
if sort_type == PICTORIAL_TOPIC_SORT.HOT:
q["sort"] = [
{"related_billboard.total_vote_cnt": {
{"related_billboard.real_vote_cnt": {
"order": "desc",
"nested_path": "related_billboard",
"missing": "_last",
......@@ -403,7 +403,7 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL
# }}
# ]
q["sort"] = [
{"related_billboard.total_vote_cnt": {
{"related_billboard.real_vote_cnt": {
"order": "desc",
"nested_path": "related_billboard",
"missing": "_last",
......
trans2es/management/commands/trans2es_mapping2es.py
View file @ 37c1e29a
......@@ -13,7 +13,7 @@ from vest.click import true_click_five, true_click_two, true_click_four, true_cl
one_seven_star_topic, auto_click_per_1d_by_post, auto_click_per_2h_by_post
from vest.follow import auto_follow, auto_follow_new, auto_follow_per_5m_by_followed, \
auto_follow_per_1d_by_regist, auto_follow_per_1d_by_post, auto_follow_per_2h_by_post_and_regist
from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2
from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2, auto_urge_per_1d_by_regist
from vest.fix import fix_no_comment_click
from vest.reply_answer import reply_comment2, reply_comment3, answer_reply2, answer_reply3, answer_reply1, \
answer_reply5, answer_reply7,reply_comment1,reply_comment5,reply_comment7
......@@ -114,6 +114,8 @@ class Command(BaseCommand):
auto_lunch_app2.auto_lunch_app2()
if options["mvest"] == "auto_star_urge":
auto_star_urge.auto_star_urge()
if options["mvest"] == "auto_urge_per_1d_by_regist":
auto_urge_per_1d_by_regist.auto_urge_per_1d_by_regist()
# 关注
if options["mvest"] == "auto_follow":
......
vest/click/auto_click_per_1d_by_post.py
View file @ 37c1e29a
......@@ -57,7 +57,7 @@ def auto_click_per_1d_by_post():
numtime1, numtime2 = time_convs(1, 1)
topic_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=3)
for topic_id in topic_ids:
click_num = random.randint(6, 12)
click_num = random.randint(1, 3)
for i in range(click_num):
auto_click_list.append(topic_id)
......@@ -65,7 +65,7 @@ def auto_click_per_1d_by_post():
numtime1, numtime2 = time_convs(2, 15)
topic_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=3)
for topic_id in topic_ids:
click_num = random.randint(0, 2)
click_num = random.randint(0, 1)
for i in range(click_num):
auto_click_list.append(topic_id)
......
vest/follow/auto_follow_per_1d_by_post.py
View file @ 37c1e29a
......@@ -45,7 +45,7 @@ def auto_follow_per_1d_by_post():
numtime1, numtime2 = time_convs(1, 1)
user_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=3)
for user_id in user_ids:
follow_num = random.randint(3, 6)
follow_num = random.randint(1, 3)
for i in range(follow_num):
auto_follow_list.append(user_id)
......
vest/follow/auto_follow_per_2h_by_post_and_regist.py
View file @ 37c1e29a
......@@ -8,7 +8,7 @@ from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
def get_commnet_id_by_regist(numtime, numtime2, content_level_low=0, content_level_top=6):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
......@@ -18,20 +18,22 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
"select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res_post = cursor.fetchall()
res = cursor.fetchall()
# 注册后
cursor.execute(
return res and [i for i, in res] or []
def get_commnet_id_by_post(numtime, numtime2, content_level_low=0, content_level_top=6):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
"select a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' " % (numtime, numtime2))
# 发贴后
cursor.execute(
res_regist = cursor.fetchall()
res_regist = res_regist and [(i,) for (i, is_shadow) in res_regist if is_shadow != 1] or []
"select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res = []
res.extend(res_regist)
res.extend(res_post)
res = cursor.fetchall()
return res and [i for i, in res] or []
......@@ -54,12 +56,19 @@ def auto_follow_per_2h_by_post_and_regist():
try:
# 发帖，注册后2小时内：[1-3]个粉丝
numtime1, numtime2 = time_conv_hour(0, 1)
user_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=6)
user_ids = get_commnet_id_by_post(numtime2, numtime1, content_level_low=0, content_level_top=6)
for user_id in user_ids:
follow_num = random.randint(3, 5)
for i in range(follow_num):
auto_follow_list.append(user_id)
# 注册后不加粉
# numtime1, numtime2 = time_conv_hour(0, 1)
# user_ids = get_commnet_id_by_regist(numtime2, numtime1, content_level_low=0, content_level_top=6)
# for user_id in user_ids:
# follow_num = random.randint(6, 10)
# for i in range(follow_num):
# auto_follow_list.append(user_id)
except:
logging_exception()
......
vest/reply/auto_reply_per_1d_to_topic.py
View file @ 37c1e29a
......@@ -62,7 +62,7 @@ def auto_reply_per_1d_to_topic():
numtime1, numtime2 = time_convs(1, 1)
topic_ids = get_data(numtime1, numtime2, 0, 3)
for topic_id in topic_ids:
random_num = random.randint(3, 6)
random_num = random.randint(0, 1)
for num in range(random_num):
topic_id_list.append(topic_id)
......@@ -79,7 +79,7 @@ def auto_reply_per_1d_to_topic():
numtime1, numtime2 = time_convs(1, 1)
topic_ids = get_data(numtime1, numtime2, 4, 6)
for topic_id in topic_ids:
random_num = random.randint(1, 6)
random_num = random.randint(2, 5)
for num in range(random_num):
topic_id_list.append(topic_id)
......@@ -87,7 +87,7 @@ def auto_reply_per_1d_to_topic():
numtime1, numtime2 = time_convs(2, 6)
topic_ids = get_data(numtime1, numtime2, 4, 6)
for topic_id in topic_ids:
random_num = random.randint(1, 3)
random_num = random.randint(0, 3)
for num in range(random_num):
topic_id_list.append(topic_id)
......
vest/request/auto_request.py
View file @ 37c1e29a
......@@ -116,13 +116,14 @@ def logins(user_id):
@shared_task(retry_kwargs={'max_retries': 0})
def click(cookies_get, id):
try:
click_key = 'click topic_id:%s, cookies_get: %s' % (str(cookies_get), str(id))
cache_count = click_cache.get(click_key)
if cache_count:
return
else:
click_cache.set(click_key, 1)
cache_count = click_cache.incr(click_key, 1)
click_cache.expire(click_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
# 点赞
try:
......@@ -158,13 +159,15 @@ def click(cookies_get, id):
@shared_task(retry_kwargs={'max_retries': 0})
def reply(cookies_get, id, content):
reply_key = 'click topic_id:%s, cookies_get: %s, content:%s' % (str(id), str(cookies_get), str(content))
cache_count = reply_cache.get(reply_key)
if cache_count:
return
else:
reply_cache.set(reply_key, 1)
try:
reply_key = 'reply topic_id:%s, cookies_get: %s, content:%s' % (str(id), str(cookies_get), str(content))
cache_count = click_cache.incr(reply_key, 1)
reply_cache.expire(reply_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
try:
post_dict = {
'topic_id': id,
......@@ -289,13 +292,15 @@ def get_comments():
@shared_task(retry_kwargs={'max_retries': 0})
def follow(cookies_get, id):
follow_key = 'click user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
cache_count = follow_cache.get(follow_key)
if cache_count:
return
else:
follow_cache.set(follow_key, 1)
try:
follow_key = 'follow user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
cache_count = follow_cache.incr(follow_key, 1)
follow_cache.expire(follow_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
try:
post_dict = {
'type': 1,
......@@ -313,7 +318,17 @@ def follow(cookies_get, id):
logging.error("catch exception,logins:%s" % traceback.format_exc())
@shared_task(retry_kwargs={'max_retries': 0})
def Urge(cookies_get, id):
try:
urge_key = 'urge user_id:%s, cookies_get: %s' % (str(id), str(cookies_get))
cache_count = click_cache.incr(urge_key, 1)
reply_cache.expire(urge_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
try:
post_dict = {
'id': id
......@@ -462,14 +477,16 @@ def set_reply_to_redis():
@shared_task(retry_kwargs={'max_retries': 0})
def reply2(cookies_get, id, content, replied_id):
reply_key = 'click topic_id:%s, cookies_get: %s, content:%s, replied_id: %s' % \
try:
reply_key = 'reply2 topic_id:%s, cookies_get: %s, content:%s, replied_id: %s' % \
(str(id), str(cookies_get), str(content), str(replied_id))
cache_count = reply_cache.get(reply_key)
if cache_count:
return
else:
reply_cache.set(reply_key, 1)
cache_count = reply_cache.incr(reply_key, 1)
reply_cache.expire(reply_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
try:
post_dict = {
'topic_id': id,
......@@ -491,13 +508,15 @@ def reply2(cookies_get, id, content, replied_id):
@shared_task(retry_kwargs={'max_retries': 0})
def pictorial_reply(cookies_get, id, content):
try:
reply_key = 'click topic_id:%s, cookies_get: %s, content:%s' % (str(id), str(cookies_get), str(content))
cache_count = reply_cache.get(reply_key)
if cache_count:
return
else:
reply_cache.set(reply_key, 1)
cache_count = click_cache.incr(reply_key, 1)
reply_cache.expire(reply_key, settings.CACHE_SECONDS)
if cache_count > 1:
return
except:
return
try:
post_dict = {
'pictorial_id': id,
......@@ -635,6 +654,14 @@ def judge_topic_info_get_comment(topic_id):
if len(tag_list) > 0:
comment = get_hellolike_comment()
# 垃圾需求--》垃圾代码 不管了 先上
if 10329872 in edit_tag_list:
comment = get_category_reply_commtent(10329872)
return comment
if 10329873 in edit_tag_list:
comment = get_category_reply_commtent(10329873)
return comment
elif len(edit_tag_list) > 0:
# 如果有编辑标签
# 去拿他的所属分类 根据分类去拿对应的标签（随机拿一个）
......
vest/urge/auto_urge_per_1d_by_regist.py 0 → 100644
View file @ 37c1e29a
import pymysql
import random
import traceback
import logging
from threading import Thread
from vest.request.auto_request import login, time_convs, Urge
from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
# 注册后
# todo 这个查询太慢了，加一个limit
cursor.execute(
"select a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' limit 10000" % (numtime, numtime2))
res = cursor.fetchall()
return res and [i for (i, is_shadow) in res if is_shadow != 1] or []
def batch_handle(auto_follow_list):
for user_id in auto_follow_list:
try:
cookies = login()
if cookies is not None:
# follow(cookies, user_id)
time = get_rand_time()
Urge.apply_async(args=(cookies, user_id), eta=time)
except:
pass
def auto_urge_per_1d_by_regist():
# 注册事件触发自动催更
auto_urge_list = []
try:
# 1天前注册：[3-6]个催更
# numtime1, numtime2 = time_convs(1, 1)
# user_ids= get_commnet_id(numtime2, numtime1)
# for user_id in user_ids:
# follow_num = random.randint(3, 6)
# for i in range(follow_num):
# auto_urge_list.append(user_id)
# 2-10天前注册：[0-1]个催更
# numtime1, numtime2 = time_convs(2, 10)
# user_ids = get_commnet_id(numtime2, numtime1)
# for user_id in user_ids:
# follow_num = random.randint(0, 1)
# for i in range(follow_num):
# auto_urge_list.append(user_id)
# 每隔9天[0-1]个催更
numtime1, numtime2 = time_convs(1, 1)
user_ids = get_commnet_id('2019-05-11', numtime2)
for user_id in user_ids:
follow_num = random.randint(1, 18)
if follow_num == 1:
auto_urge_list.append(user_id)
except:
logging_exception()
logging.error("catch exception,main:%s" % traceback.format_exc())
logging.info('auto_follow_per_1d_by_regist: len %s' % len(auto_urge_list))
print('auto_follow_per_1d_by_regist: len %s' % len(auto_urge_list))
total = len(auto_urge_list)
limit = (total + 10) // 10
for start in range(0, total, limit):
batch = auto_urge_list[start:start + limit]
t = Thread(target=batch_handle, args=[batch])
t.start()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment