Skip to content

P
physical
Commit 7dea9be1 authored by 吴升宇's avatar 吴升宇
Browse files
Options

Merge branch 'master' of git.wanmeizhensuo.com:alpha/physical into like-pre/r01

parents a7dd5277 f3c60f93
Hide whitespace changes
Inline Side-by-side
Showing with 67 additions and 36 deletions
README.md
View file @ 7dea9be1
......@@ -58,3 +58,22 @@ celery -A physical beat
### physical策略端脚本
在Alp-Test-Cos-test001服务器上，通过crontab -l查看
### es常用命令
```bash
#修改别名：
curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-product-prod-190904001","alias":"gm-dbmw-product-read"}},{"add":{"index":"mv-alpha-product-prod-191018401","alias":"gm-dbmw-product-read"}}]}'
#查看别名
curl -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases?pretty"
#查看所有索引
curl -H "Content-Type: application/json" "http://172.17.32.22:9200/_cat/indices?v"
#简单查询
curl -H "Content-Type: application/json" "http://172.17.32.22:9200/gm-dbmw-topic-high-star-read/_search?pretty" -d '{"query":{"term":{"id":"46710078"}}}'
```
### es同步确认脚本
cd /data/log/physical/app/index_contrast/
crontab.py
View file @ 7dea9be1
import random
ontime_list = [
"0 9 * * * source /srv/envs/physical/bin/activate && python /data/log/physical/app/crontab.py",
"10 9 * * * source /srv/envs/physical/bin/activate && python /data/log/physical/app/crontabs.py",
"0 8 * * * source /srv/envs/physical/bin/activate && python /data/log/physical/app/crontab.py",
"10 8 * * * source /srv/envs/physical/bin/activate && python /data/log/physical/app/crontabs.py",
"0 9 * * * sh /data/log/cybertron/app/statistics_query.sh > /data/log/cybertron/app/statistics_query.log",
"54 */1 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_2h_by_post",
# "*/5 * * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m true_click_one",
......@@ -21,11 +21,11 @@ ontime_list = [
"0 0 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m get_user_id",
# "0 14,18,22 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m principal_online_comment1",
"25 */1 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_2h_to_topic",
"0 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_1d_by_post",
"1 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
"2 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_post",
"3 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
"4 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_topic"
"0 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_1d_by_post",
"1 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
"2 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_post",
"3 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
"4 8 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_topic"
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply1",
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply2",
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply3",
......
libs/es.py
View file @ 7dea9be1
......@@ -422,7 +422,7 @@ class ESPerform(object):
"_script": {
"order": "desc",
"script": {
"inline": "10*doc['topic_ctr_30'].value+doc['like_rate_30'].value+2*doc['topic_ctr_all'].value+doc['like_rate_all'].value"
"inline": "3*doc['topic_ctr_30'].value+2*doc['topic_ctr_all'].value"
},
"type": "number"
}
......@@ -452,7 +452,7 @@ class ESPerform(object):
topic_id_dict = dict()
for item in result_dict["hits"]:
topic_id_dict[str(item["_source"]["id"])] = item["_source"]["user_id"]
logging.info("get_tag_topic_list_dict:gyz" + str(q) + str(result_dict))
logging.info("get_tag_topic_list_dict:zw" + str(q) + str(result_dict))
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -546,7 +546,7 @@ class ESPerform(object):
"_script": {
"order": "desc",
"script": {
"inline": "10*doc['pictorial_ctr_30'].value+10*doc['like_rate_30'].value+3*doc['pictorial_ctr_all'].value+2*doc['like_rate_all'].value"
"inline": "3*doc['pictorial_ctr_30'].value+2*doc['pictorial_ctr_all'].value"
},
"type": "number"
}
......@@ -567,7 +567,7 @@ class ESPerform(object):
offset=0, size=size)
pictorial_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
logging.info("get_tag_pictorial_id_list:gyz" + str(q) + str(result_dict))
logging.info("get_tag_pictorial_id_list:zw" + str(q) + str(result_dict))
return pictorial_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
libs/timelib.py
View file @ 7dea9be1
......@@ -23,7 +23,7 @@ def eta_2_push_time(eta):
return push_time
def get_rand_time(hourlow=0, hourup=13, minutelow=0, minuteup=60):
def get_rand_time(hourlow=0, hourup=15, minutelow=0, minuteup=55):
hours = random.randint(hourlow, hourup)
minutes = random.randint(minutelow, minuteup)
# todo redis会自动给加8个小时，所以这边先写死减少8小时
......
script/es_alias.py 0 → 100644
View file @ 7dea9be1
import os
os.system(
"""
source /srv/envs/physical/bin/activate
&& curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-user-prod-190905002","alias":"gm-dbmw-user-read"}},{"add":{"index":"mv-alpha-user-prod-191018401","alias":"gm-dbmw-user-read"}}]}'
&& curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-topic-prod-190928002","alias":"gm-dbmw-topic-read"}},{"add":{"index":"mv-alpha-topic-prod-191018401","alias":"gm-dbmw-topic-read"}}]}'
&& curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-topic-high-star-prod-190928002","alias":"gm-dbmw-topic-high-star-read"}},{"add":{"index":"mv-alpha-topic-high-star-prod-191018401","alias":"gm-dbmw-topic-high-star-read"}}]}'
&& curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-tag-prod-190904001","alias":"gm-dbmw-tag-read"}},{"add":{"index":"mv-alpha-tag-prod-191018401","alias":"gm-dbmw-tag-read"}}]}'
&& curl -X POST -H "Content-Type: application/json" "http://172.17.32.22:9200/_aliases" -d '{"actions":[{"remove":{"index":"mv-alpha-product-prod-190904001","alias":"gm-dbmw-product-read"}},{"add":{"index":"mv-alpha-product-prod-191018401","alias":"gm-dbmw-product-read"}}]}'
"""
)
\ No newline at end of file
vest/click/auto_click_per_1d_by_post.py
View file @ 7dea9be1
......@@ -17,9 +17,8 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
cursor.execute(
"select t.id from topic t left join user_extra u on t.user_id = u.user_id "
"where IFNULL(u.is_shadow,'') != 1 and t.create_time > '%s' and t.create_time < '%s' and t.is_online=1 "
"and t.content_level >= %s and t.content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
"select id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' and is_online=1 "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res = cursor.fetchall()
......@@ -58,7 +57,7 @@ def auto_click_per_1d_by_post():
numtime1, numtime2 = time_convs(1, 1)
topic_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=3)
for topic_id in topic_ids:
click_num = random.randint(2, 6)
click_num = random.randint(6, 12)
for i in range(click_num):
auto_click_list.append(topic_id)
......
vest/click/auto_click_per_2h_by_post.py
View file @ 7dea9be1
......@@ -18,9 +18,8 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
# 发贴后
cursor.execute(
"select t.id from topic t left join user_extra u on t.user_id = u.user_id "
"where IFNULL(u.is_shadow,'') != 1 and t.create_time > '%s' and t.create_time < '%s' and t.is_online=1 "
"and t.content_level >= %s and t.content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
"select id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' and is_online=1 "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res = cursor.fetchall()
......
vest/follow/auto_follow_per_1d_by_post.py
View file @ 7dea9be1
......@@ -16,9 +16,8 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=3):
# 发贴后
cursor.execute(
"select t.user_id from topic t left join user_extra u on t.user_id = u.user_id "
"where IFNULL(u.is_shadow,'') != 1 and t.create_time > '%s' and t.create_time < '%s' "
"and t.content_level >= %s and t.content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
"select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res = cursor.fetchall()
return res and [i for i, in res] or []
......@@ -46,7 +45,7 @@ def auto_follow_per_1d_by_post():
numtime1, numtime2 = time_convs(1, 1)
user_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=3)
for user_id in user_ids:
follow_num = random.randint(2, 6)
follow_num = random.randint(3, 6)
for i in range(follow_num):
auto_follow_list.append(user_id)
......
vest/follow/auto_follow_per_1d_by_regist.py
View file @ 7dea9be1
......@@ -14,13 +14,14 @@ def get_commnet_id(numtime, numtime2):
cursor = pc.cursor()
# 注册后
# todo 这个查询太慢了，加一个limit
cursor.execute(
"select a.user_id from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' and IFNULL(u.is_shadow,'') != 1 " % (numtime, numtime2))
"select a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' limit 10000" % (numtime, numtime2))
res = cursor.fetchall()
return res and [i for i, in res] or []
return res and [i for (i, is_shadow) in res if is_shadow != 1] or []
def batch_handle(auto_follow_list):
......@@ -44,7 +45,7 @@ def auto_follow_per_1d_by_regist():
numtime1, numtime2 = time_convs(1, 1)
user_ids= get_commnet_id(numtime2, numtime1)
for user_id in user_ids:
follow_num = random.randint(1, 3)
follow_num = random.randint(10, 20)
for i in range(follow_num):
auto_follow_list.append(user_id)
......@@ -59,7 +60,8 @@ def auto_follow_per_1d_by_regist():
# 10天前注册：每隔12天[0-1]个赞
numtime1, numtime2 = time_convs(1, 12)
user_ids = get_commnet_id('0', numtime2)
# todo 加一个过滤时间
user_ids = get_commnet_id('2019-05-11', numtime2)
for user_id in user_ids:
follow_num = random.randint(1, 24)
if follow_num == 12:
......
vest/follow/auto_follow_per_2h_by_post_and_regist.py
View file @ 7dea9be1
......@@ -15,19 +15,19 @@ def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
# 发贴后
cursor.execute(
"select t.user_id from topic t left join user_extra u on t.user_id = u.user_id "
"where IFNULL(u.is_shadow,'') != 1 and t.create_time > '%s' and t.create_time < '%s' "
"and t.content_level >= %s and t.content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
"select user_id from topic where is_shadow!= 1 and create_time > '%s' and create_time < '%s' "
"and content_level >= %s and content_level <= %s " % (numtime, numtime2, str(content_level_low), str(content_level_top)))
res_post = cursor.fetchall()
# 注册后
cursor.execute(
"select a.user_id from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' and u.is_shadow = 0 " % (numtime, numtime2))
"select a.user_id, u.is_shadow from account_user a left join user_extra u on a.user_id = u.user_id "
"WHERE a.create_time > '%s' and a.create_time < '%s' " % (numtime, numtime2))
res_regist = cursor.fetchall()
res_regist = res_regist and [(i,) for (i, is_shadow) in res_regist if is_shadow != 1] or []
res = []
res.extend(res_regist)
......@@ -41,7 +41,7 @@ def batch_handle(auto_follow_list):
cookies = login()
if cookies is not None:
# follow(cookies, user_id)
time = get_rand_time(hourup=0)
time = get_rand_time(hourup=1)
follow.apply_async(args=(cookies, user_id), eta=time)
except:
......@@ -56,7 +56,7 @@ def auto_follow_per_2h_by_post_and_regist():
numtime1, numtime2 = time_conv_hour(0, 1)
user_ids = get_commnet_id(numtime2, numtime1, content_level_low=0, content_level_top=6)
for user_id in user_ids:
follow_num = random.randint(1, 3)
follow_num = random.randint(3, 5)
for i in range(follow_num):
auto_follow_list.append(user_id)
......
vest/reply/auto_reply_per_1d_to_topic.py
View file @ 7dea9be1
......@@ -62,7 +62,7 @@ def auto_reply_per_1d_to_topic():
numtime1, numtime2 = time_convs(1, 1)
topic_ids = get_data(numtime1, numtime2, 0, 3)
for topic_id in topic_ids:
random_num = random.randint(1, 3)
random_num = random.randint(3, 6)
for num in range(random_num):
topic_id_list.append(topic_id)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment