Commit fcf10e8b authored by 吴升宇's avatar 吴升宇

Merge branch 'wsy' into 'test'

add readme

See merge request !514
parents 21566502 8e17c112
strategy part like es perform
容器运行命令
docker run -it -p 9999:9999 -d \
--volume=/etc/gm-config:/etc/gm-config \
--volume=/etc/resolv.conf:/etc/resolv.conf \
--volume=/srv/apps/physical/physical/settings_local.py:/srv/apps/physical/physical/settings_local.py \
--entrypoint="/bin/bash" physical-test:1.0 "-c" "source ~/.bashrc && celery -A physical worker -Q tapir-alpha --loglevel=DEBUG --maxtasksperchild 500 -c 62"
strategy part like es perform
容器运行命令
docker run -it -p 9999:9999 -d \
--volume=/etc/gm-config:/etc/gm-config \
--volume=/etc/resolv.conf:/etc/resolv.conf \
--volume=/srv/apps/physical/physical/settings_local.py:/srv/apps/physical/physical/settings_local.py \
--entrypoint="/bin/bash" physical-test:1.0 "-c" "source ~/.bashrc && celery -A physical worker -Q tapir-alpha --loglevel=DEBUG --maxtasksperchild 500 -c 62"
## 启动方式
### 方式1:[program:physical-linucb] 消费kafka数据
一个while True服务,去消费kafka数据,维护redis中device的tag_map
```python
source /srv/envs/physical/bin/activate && python manage.py trans2es_data2es_parallel -S linucb
```
### 方式2:[program:physical] rpc服务
启动rpc服务供后端调用, 接口都在search文件夹下的bind函数定义
```python
gunicorn gm_rpcd.wsgi:application --workers=1 --worker-class=gevent
--worker-connections=1024
--bind=0.0.0.0:9999
--user=gmuser
--chdir /srv/apps/physical/
--timeout 600
--log-level=debug
--error-logfile=/data/log/physical/app/gunicorn_error.log
--access-logfile=/data/log/physical/app/gunicorn_access.log
```
### 方式3 [program:physical-celery] celery 消息队列服务
mysql数据同步到es,入口文件:injection.data_sync.tasks.write_to_es
数据同步工作大部分在数据管道,只有小部分在这,暂时先不用管这个服务,需求用不到
```python
celery -A physical worker -Q tapir-alpha --loglevel=DEBUG --maxtasksperchild 500 -c 62
```
### 方式4 [program:physical-beat-celery] celery 消息队列定时(作用未知)
```python
celery -A physical beat
```
# 自动运行脚本
### venus服务端脚本
通过celery beat 运行
配置说明文件:venus/setting/base.py
具体脚本:venus/communityity/task/xxx.py
目前共5个脚本
1:每天保存用户扫脸图片:community.tasks.save_image_task.save_user_face_image
2:每天3点0分评论自动点赞:community.tasks.crontab_reply_vote_task.crontab_reply_vote
3:向用户推送兴趣标签:community.tasks.push_task.push_interest_tag_pictorial_to_user
4:向用户推送经验标签:community.tasks.push_task.push_topic_draft_task
5:推送主题草稿任务:community.tasks.push_task.push_topic_draft_task
### physical策略端脚本
在Alp-Test-Cos-test001服务器上,通过crontab -l查看
import datetime
def time_conv(minutest, minutest2):
try:
now = datetime.datetime.now()
yes_time = now - datetime.timedelta(minutes=minutest)
yes_time2 = now - datetime.timedelta(minutes=minutest2)
return yes_time, yes_time2
except:
return None
numtime1, numtime2 = time_conv(1, 5)
print(numtime1, numtime2)
......@@ -10,7 +10,7 @@ from trans2es.type_info import get_type_info_map, TypeInfo
from vest.reply import true_comment_one, true_comment_two, true_comment_three, one_seven_topic_comment
from vest.click import true_click_five, true_click_two, true_click_four, true_click_one, true_click_three, \
one_seven_star_topic
from vest.follow import auto_follow, auto_follow_new
from vest.follow import auto_follow, auto_follow_new, auto_follow_per_5m_by_followed
from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2
from vest.fix import fix_no_comment_click
from vest.reply_answer import reply_comment2, reply_comment3, answer_reply2, answer_reply3, answer_reply1, \
......@@ -114,6 +114,9 @@ class Command(BaseCommand):
auto_follow.auto_follow()
if options["mvest"] == "auto_follow_new":
auto_follow_new.auto_follow_new()
if options['mvest'] == "auto_follow_per_5m_by_followed":
auto_follow_per_5m_by_followed.auto_follow_per_5m_by_followed()
# 补足
if options["mvest"] == "fix_no_comment_click":
......
majia_user_ids_list = [241407406, 241407033, 241407154, 241407307, 241407284, 241407248, 241407179, 241407428, 241407470, 241407314, 241407385, 241407490, 241407375, 241407318, 241407508, 241407192, 241407131, 241407226, 241407174, 241407505, 241407358, 241407466, 241407452, 241407093, 241407289, 241407495, 241407405, 241407200, 241407330, 241407079, 241407207, 241407100, 241407116, 241407111, 241407302, 241407095, 241407086, 241407499, 241407519, 241407031, 241407288, 241407436, 241407310, 241407404, 241407493, 241407201, 241407084, 241407158, 241407327, 241407178, 241407164, 241407292, 241407387, 241407352, 241407482, 241407325, 241407256, 241407252, 241407485, 241407376, 241407242, 241407219, 241407166, 241407413, 241407343, 241407423, 241407468, 241407198, 241407170, 241407287, 241407138, 241407353, 241407065, 241407262, 241407501, 241407255, 241407512, 241407309, 241407494, 241407319, 241407373, 241407264, 241407471, 241407250, 241407479, 241407456, 241407443, 241407362, 241407157, 241407130, 241407075, 241407390, 241407036, 241407502, 241407092, 241407513, 241407136, 241407048, 241407448, 241407080, 241407459, 241407045, 241407407, 241407186, 241407083, 241407073, 241407191, 241407421, 241407062, 241407275, 241407347, 241407397, 241407052, 241407393, 241407150, 241407063, 241407212, 241407072, 241407500, 241407331, 241407339, 241407474, 241407378, 241407147, 241407316, 241407301, 241407516, 241407527, 241407233, 241407526, 241407365, 241407384, 241407349, 241407210, 241407162, 241407042, 241407057, 241407514, 241407398, 241407103, 241407509, 241407336, 241407335, 241407145, 241407429, 241407144, 241407068, 241407074, 241407077, 241407238, 241407112, 241407236, 241407245, 241407364, 241407101, 241407108, 241407355, 241407431, 241407247, 241407167, 241407050, 241407064, 241407276, 241407139, 241407120, 241407249, 241407097, 241407437, 241407377, 241407140, 241407475, 241407469, 241407168, 241407486, 241407263, 241407379, 241407243, 241407081, 241407114, 241407230, 241407169, 241407059, 241407354, 241407253, 241407106, 241407467, 241407206, 241407221, 241407481, 241407326, 241407185, 241407322, 241407411, 241407044, 241407285, 241407324, 241407522, 241407175, 241407151, 241407246, 241407433, 241407496, 241407234, 241407183, 241407524, 241407265, 241407146, 241407043, 241407217, 241407046, 241407229, 241407417, 241407124, 241407280, 241407137, 241407091, 241407132, 241407261, 241407115, 241407071, 241407165, 241407369, 241407418, 241407176, 241407040, 241407143, 241407058, 241407491, 241407434, 241407329, 241407462, 241407135, 241407338, 241407283, 241407094, 241407117, 241407239, 241407410, 241407435, 241407060, 241407420, 241407424, 241407308, 241407489, 241407286, 241407268, 241407311, 241407328, 241407232, 241407463, 241407304, 241407346, 241407076, 241407126, 241407177, 241407382, 241407438, 241407032, 241407515, 241407363, 241407189, 241407360, 241407282, 241407295, 241407109, 241407340, 241407333, 241407240, 241407457, 241407231, 241407163, 241407492, 241407220, 241407125, 241407182, 241407070, 241407293, 241407356, 241407190, 241407128, 241407039, 241407241, 241407451, 241407334, 241407270, 241407090, 241407412, 241407465, 241407119, 241407244, 241407487, 241407511, 241407196, 241407415, 241407054, 241407504, 241407300, 241407320, 241407447, 241407205, 241407460, 241407216, 241407389, 241407188, 241407313, 241407156, 241407113, 241407350, 241407525, 241407497, 241407211, 241407345, 241407394, 241407107, 241407266, 241407332, 241407197, 241407337, 241407085, 241407366, 241407305, 241407401, 241407315, 241407089, 241407281, 241407386, 241407038, 241407351, 241407507, 241407484, 241407171, 241407082, 241407214, 241407419, 241407321, 241407409, 241407251, 241407426, 241407444, 241407432, 241407391, 241407357, 241407213, 241407368, 241407416, 241407342, 241407312, 241407303, 241407121, 241407099, 241407521, 241407259, 241407510, 241407049, 241407087, 241407279, 241407441, 241407388, 241407105, 241407478, 241407317, 241407142, 241407096, 241407422, 241407277, 241407141, 241407488, 241407297, 241407473, 241407110, 241407361, 241407306, 241407402, 241407215, 241407273, 241407155, 241407257, 241407454, 241407274, 241407123, 241407104, 241407208, 241407298, 241407399, 241407030, 241407069, 241407153, 241407453, 241407449, 241407440, 241407455, 241407223, 241407278, 241407400, 241407359, 241407088, 241407202, 241407254, 241407194, 241407159, 241407161, 241407061, 241407267, 241407372, 241407381, 241407374, 241407173, 241407392, 241407271, 241407341, 241407299, 241407446, 241407439, 241407523, 241407396, 241407227, 241407041, 241407067, 241407181, 241407344, 241407199, 241407203, 241407193, 241407127, 241407348, 241407118, 241407260, 241407209, 241407370, 241407291, 241407367, 241407380, 241407503, 241407037, 241407269, 241407450, 241407133, 241407371, 241407395, 241407122, 241407204, 241407430, 241407152, 241407148, 241407425, 241407160, 241407427, 241407296, 241407518, 241407480, 241407180, 241407272, 241407403, 241407102, 241407225, 241407195, 241407472, 241407056, 241407498, 241407218, 241407055, 241407323, 241407461, 241407098, 241407235, 241407187, 241407290, 241407520, 241407483, 241407172, 241407134, 241407506, 241407129, 241407476, 241407517, 241407237, 241407442, 241407184, 241407464, 241407529, 241407047, 241407053, 241407458, 241407078, 241407066, 241407294, 241407258, 241407035, 241407445, 241407149, 241407414, 241407222, 241407383, 241407408, 241407034, 241407051, 241407477, 241407224, 241407228, 241407528]
\ No newline at end of file
import pymysql
import random
import traceback
import logging
from vest.request.auto_request import login, time_convs_hours, follow
from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception
def get_commnet_id(numtime):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
cursor.execute(
"SELECT user_id FROM reply WHERE create_time > '%s' " % numtime)
data_comment = cursor.fetchall()
cursor.execute(
"SELECT t.user_id from topic t left join reply r on r.topic_id = t.id WHERE r.create_time > '%s'" % numtime)
data_reply_topic = cursor.fetchall()
cursor.execute(
"SELECT r.user_id from reply r left join reply p on r.replied_id = p.replied_id WHERE p.create_time > '%s'" % numtime)
data_reply_reply = cursor.fetchall()
cursor.execute(
"SELECT user_id FROM group_user_role WHERE create_time > '%s' " % numtime)
data_group_follow = cursor.fetchall()
cursor.execute(
"select user_id from account_user WHERE create_time > '%s'" % numtime)
data_new_user = cursor.fetchall()
data_id = list(data_comment)
data1_id = list(data_reply_topic)
data2_id = list(data_group_follow)
data3_id = list(data_new_user)
data_reply = list(data_reply_reply)
all_data = []
all_data.extend(data_id)
all_data.extend(data2_id)
all_data.extend(data3_id)
all_data.extend(data_reply)
all_data.extend(data1_id)
topic_id_list = []
for i in all_data and all_data:
cursor.execute(
"select user_id from user_extra where user_id =" + str(i[0]) + " and is_shadow =0 ")
data = cursor.fetchall()
user_id = list(data)
if (user_id and user_id[0][1] == 0) or len(user_id) == 0:
topic_id_list.append(i)
logging.info("Database version : %s " % topic_id_list)
pc.close()
return topic_id_list
def auto_follow_per_2h():
# 注册,发帖 2个事件触发自动加粉丝(在2个小时内加1-3个粉)
try:
numtime1, numtime2 = time_convs_hours(1, 2)
user_id = get_commnet_id(numtime1)
try:
for j in user_id and user_id:
id = int(j[0])
follow_num = random.randint(0, 1)
for i in range(follow_num):
cookies = login()
if cookies is not None:
follow(cookies, id)
except:
pass
except:
logging_exception()
logging.error("catch exception,main:%s" % traceback.format_exc())
import pymysql
import traceback
import logging
from vest.request.auto_request import login, follow, time_conv
from vest.request.auto_request import host, user, db, passwd
from vest.data.majia_user_ids import majia_user_ids_list
from libs.error import logging_exception
def get_user_id(numtime):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor()
majia_user_ids_tuple = tuple(majia_user_ids_list)
cursor.execute(
"SELECT user_id FROM user_follow WHERE create_time > '%s' and follow_id in %s " % (numtime, str(majia_user_ids_tuple))
)
data_comment = cursor.fetchall()
return data_comment and [i for i in data_comment] or []
def auto_follow_per_5m_by_followed():
# 如果马甲被用户关注,马甲即反向关注用户, 被关注5分钟后
try:
numtime1, numtime2 = time_conv(1, 5)
user_ids = get_user_id(numtime2)
try:
for user_id in user_ids and user_ids:
cookies = login()
if cookies is not None:
follow(cookies, user_id)
except:
pass
except:
logging_exception()
logging.error("catch exception,main:%s" % traceback.format_exc())
......@@ -112,6 +112,7 @@ def logins(user_id):
def click(cookies_get, id):
# 点赞
try:
post_dict = {
'type': 0,
......@@ -168,6 +169,17 @@ def time_conv_minute(minutest, minutest2):
return None
def time_conv_hour(minutest, minutest2):
try:
now = datetime.datetime.now()
minute = datetime.datetime.now().minute
yes_time = now - datetime.timedelta(hours=minutest)
yes_time2 = now - datetime.timedelta(hours=minutest2)
return yes_time, yes_time2, minute
except:
return None
def time_now(minutest):
try:
now = datetime.datetime.now()
......@@ -178,6 +190,11 @@ def time_now(minutest):
def time_convs(numtime, numtime2):
'''
params: 1, 2
datetime.now(): 2019-10-09 10:35:50.231463
return: 2019-10-08 23:59:59.000000 2019-10-07 00:00:00.000000
'''
try:
now = datetime.datetime.now()
yes_time = now - datetime.timedelta(days=numtime)
......@@ -190,7 +207,6 @@ def time_convs(numtime, numtime2):
logging.info("get yes_time_str:%s" % yes_time_str)
logging.info("get yes_time_str2:%s" % yes_time_str2)
return yes_time_str, yes_time_str2
except:
return None
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment