Commit 4f5aedc1 authored by 吴升宇's avatar 吴升宇

Merge branch 'wsy3' into 'test'

Wsy3

See merge request !528
parents 27e3853a 524ee41f
...@@ -20,6 +20,7 @@ RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32 \ ...@@ -20,6 +20,7 @@ RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32 \
&& apt-get -y update && apt-get -y install mysql-client libmysqlclient-dev \ && apt-get -y update && apt-get -y install mysql-client libmysqlclient-dev \
&& apt-get clean \ && apt-get clean \
&& apt-get autoclean \ && apt-get autoclean \
&& apt-get -y install vim \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt
......
...@@ -15,21 +15,17 @@ ontime_list = [ ...@@ -15,21 +15,17 @@ ontime_list = [
"30 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge2", "30 10 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_urge2",
"0 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app", "0 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app",
"30 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app2", "30 10 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_lunch_app2",
"*/5 * * * 1 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_5m_by_followed", # "*/5 * * * 1 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_5m_by_followed",
"1 */2 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_2h_by_post_and_regist", "1 */2 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_2h_by_post_and_regist",
"0 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m get_login_session", "0 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m get_login_session",
"0 0 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m get_user_id", "0 0 * * 3 source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m get_user_id",
# "0 14,18,22 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m principal_online_comment1", # "0 14,18,22 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m principal_online_comment1",
"25 */2 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_2h_to_topic" "25 */2 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_2h_to_topic",
"0 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_1d_by_post",
] "1 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
"2 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_post",
random_time_list = [ "3 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
"* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_click_per_1d_by_post", "4 9 * * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_topic"
"* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_regist",
"* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_follow_per_1d_by_post",
"* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
"* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m auto_reply_per_1d_to_topic"
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply1", # "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply1",
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply2", # "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply2",
# "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply3", # "* * * source /srv/envs/physical/bin/activate && cd /srv/apps/physical && python manage.py trans2es_mapping2es -m answer_reply3",
...@@ -48,16 +44,6 @@ random_time_list = [ ...@@ -48,16 +44,6 @@ random_time_list = [
] ]
###随机生成10-23的一个数字 代表小时
ss1 = random.choices(range(10, 23), k=100)
###随机生成1-60的一个数字 表示分钟
ss2 = random.choices(range(1, 50), k=100)
for i in range(len(random_time_list)):
ontime_list.append(str(ss2[i]) + " " + str(ss1[i]) + " " + str(random_time_list[i]))
data = open("/data/log/physical/app/conf.txt", "w") data = open("/data/log/physical/app/conf.txt", "w")
# data = open("abc.txt", "w") # data = open("abc.txt", "w")
......
import pytz
import random
from datetime import datetime, timedelta
NOW = datetime.now()
def tzlc(dt, truncate_to_sec=True):
if dt is None:
return None
if truncate_to_sec:
dt = dt.replace(microsecond=0)
return pytz.timezone('Asia/Shanghai').localize(dt)
def eta_2_push_time(eta):
if eta:
eta = datetime.strptime(eta, '%Y-%m-%d %H:%M:%S')
eta = tzlc(eta)
return int((eta - datetime.fromtimestamp(0, pytz.timezone("UTC"))).total_seconds())
else:
push_time = None
return push_time
def get_rand_time(hourlow=0, hourup=13, minutelow=0, miniteup=60):
hours = random.randint(hourlow, hourup)
minutes = random.randint(minutelow, miniteup)
now_time = NOW + timedelta(hours=hours, minutes=minutes)
time = eta_2_push_time(now_time.strftime("%Y-%m-%d %H:%M:%S"))
return datetime.fromtimestamp(time)
...@@ -2,6 +2,6 @@ from __future__ import unicode_literals, absolute_import, print_function ...@@ -2,6 +2,6 @@ from __future__ import unicode_literals, absolute_import, print_function
import pymysql import pymysql
from _celery import app as celery_app from _celery import app as celery_app
from vest import *
pymysql.install_as_MySQLdb() pymysql.install_as_MySQLdb()
#__all__ = ('celery_app',) #__all__ = ('celery_app',)
...@@ -9,7 +9,12 @@ class CeleryTaskRouter(object): ...@@ -9,7 +9,12 @@ class CeleryTaskRouter(object):
queue_task_map = { queue_task_map = {
"tapir-alpha": [ "tapir-alpha": [
'injection.data_sync.tasks.write_to_es', 'injection.data_sync.tasks.write_to_es',
] 'vest.request.auto_request.click',
'vest.request.auto_request.reply',
'vest.request.auto_request.follow',
'vest.request.auto_request.reply2',
'vest.request.auto_request.pictorial_reply'
],
} }
# Map[TaskName, QueueName] # Map[TaskName, QueueName]
......
from .request import *
\ No newline at end of file
...@@ -6,6 +6,7 @@ from threading import Thread ...@@ -6,6 +6,7 @@ from threading import Thread
from vest.request.auto_request import login, time_convs, click from vest.request.auto_request import login, time_convs, click
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6): def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
...@@ -41,7 +42,8 @@ def batch_handle(auto_click_list): ...@@ -41,7 +42,8 @@ def batch_handle(auto_click_list):
try: try:
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
click(cookies, topic_id) click.async_apply(args=(cookies, topic_id), eta=get_rand_time())
# click(cookies, topic_id)
except: except:
pass pass
......
...@@ -6,6 +6,7 @@ from threading import Thread ...@@ -6,6 +6,7 @@ from threading import Thread
from vest.request.auto_request import login, time_conv_hour, click from vest.request.auto_request import login, time_conv_hour, click
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6): def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
...@@ -42,7 +43,8 @@ def batch_handle(auto_click_list): ...@@ -42,7 +43,8 @@ def batch_handle(auto_click_list):
try: try:
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
click(cookies, topic_id) # click(cookies, topic_id)
click.async_apply(args=(cookies, topic_id), eta=get_rand_time(hourup=1))
except: except:
pass pass
......
...@@ -6,6 +6,7 @@ from threading import Thread ...@@ -6,6 +6,7 @@ from threading import Thread
from vest.request.auto_request import login, time_convs, follow from vest.request.auto_request import login, time_convs, follow
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=3): def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=3):
...@@ -28,7 +29,9 @@ def batch_handle(auto_follow_list): ...@@ -28,7 +29,9 @@ def batch_handle(auto_follow_list):
try: try:
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
follow(cookies, user_id) # follow(cookies, user_id)
follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
except: except:
pass pass
......
...@@ -6,6 +6,7 @@ from threading import Thread ...@@ -6,6 +6,7 @@ from threading import Thread
from vest.request.auto_request import login, time_convs, follow from vest.request.auto_request import login, time_convs, follow
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2): def get_commnet_id(numtime, numtime2):
...@@ -27,7 +28,9 @@ def batch_handle(auto_follow_list): ...@@ -27,7 +28,9 @@ def batch_handle(auto_follow_list):
try: try:
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
follow(cookies, user_id) # follow(cookies, user_id)
follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
except: except:
pass pass
......
...@@ -6,7 +6,7 @@ from threading import Thread ...@@ -6,7 +6,7 @@ from threading import Thread
from vest.request.auto_request import login, time_conv_hour, follow from vest.request.auto_request import login, time_conv_hour, follow
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6): def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306) pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
...@@ -40,7 +40,9 @@ def batch_handle(auto_follow_list): ...@@ -40,7 +40,9 @@ def batch_handle(auto_follow_list):
try: try:
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
follow(cookies, user_id) # follow(cookies, user_id)
follow.async_apply(args=(cookies, user_id), eta=get_rand_time(hourup=1))
except: except:
pass pass
......
...@@ -5,6 +5,7 @@ from vest.request.auto_request import login, follow, time_conv_minute ...@@ -5,6 +5,7 @@ from vest.request.auto_request import login, follow, time_conv_minute
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from vest.data.majia_user_ids import majia_user_ids_dict from vest.data.majia_user_ids import majia_user_ids_dict
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
def get_user_id(numtime): def get_user_id(numtime):
...@@ -28,7 +29,9 @@ def auto_follow_per_5m_by_followed(): ...@@ -28,7 +29,9 @@ def auto_follow_per_5m_by_followed():
user_id = user[0] user_id = user[0]
cookies = login(str(majia_user_ids_dict.get(str(user[1])))) cookies = login(str(majia_user_ids_dict.get(str(user[1]))))
if cookies is not None: if cookies is not None:
follow(cookies, user_id) # follow(cookies, user_id)
follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
except: except:
pass pass
......
...@@ -8,6 +8,7 @@ from vest.request.auto_request import login, pictorial_reply, get_majia, get_pic ...@@ -8,6 +8,7 @@ from vest.request.auto_request import login, pictorial_reply, get_majia, get_pic
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from vest.data.topic_models import get_pictorial_tag_by_id from vest.data.topic_models import get_pictorial_tag_by_id
from libs.timelib import get_rand_time
""" """
榜单新增内容(仅限在线的) 榜单新增内容(仅限在线的)
...@@ -38,8 +39,8 @@ def get_data(numtime1, numtime2): ...@@ -38,8 +39,8 @@ def get_data(numtime1, numtime2):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306) pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor() cursor = pc.cursor()
cursor.execute( cursor.execute(
"SELECT pictorial_id FROM community_pictorial_topic WHERE is_online=1 and (create_time >= '%s' and create_time < '%s') group by pictorial_id " % ( "SELECT id FROM community_pictorial WHERE is_online=1 and (create_time >= '%s' and create_time < '%s') " % (
numtime1, numtime2)) numtime2, numtime1))
res = cursor.fetchall() res = cursor.fetchall()
return res and [r for r, in res] or [] return res and [r for r, in res] or []
...@@ -55,16 +56,15 @@ def batch_handle(pictorial_id_list): ...@@ -55,16 +56,15 @@ def batch_handle(pictorial_id_list):
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
comment = judge_pictorial_info_get_comment(pictorial_id) comment = judge_pictorial_info_get_comment(pictorial_id)
pictorial_reply(cookies, pictorial_id, comment) # pictorial_reply(cookies, pictorial_id, comment)
pictorial_reply.async_apply(args=(cookies, pictorial_id, comment), eta=get_rand_time())
except: except:
pass pass
def auto_reply_per_1d_to_pictorial(): def auto_reply_per_1d_to_pictorial():
# 问题自动增加回答 # 问题自动增加回答
# todo topic_vote_cnt和community_pictorial_topic
# todo 1天前发的问题:[0-1]个评论 6天前发的问题:[0-1]个评论 这个是什么鬼
# todo 怎么禁止作者评论
pictorial_id_list = [] pictorial_id_list = []
try: try:
logging.info("comment offline pictorial") logging.info("comment offline pictorial")
......
import pymysql import pymysql
import traceback import traceback
import logging import logging
import random
from threading import Thread from threading import Thread
from vest.request.auto_request import login, time_convs, get_answer_data, reply_answer, get_majia, \ from vest.request.auto_request import login, time_convs, get_answer_data, reply_answer, get_majia, \
set_reply_to_redis, judge_topic_info_get_comment, reply set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
import random from libs.timelib import get_rand_time
def get_data(numtime, numtime2, content_level_low, content_level_top): def get_data(numtime, numtime2, content_level_low, content_level_top):
...@@ -31,7 +32,18 @@ def batch_handle(topic_id_list): ...@@ -31,7 +32,18 @@ def batch_handle(topic_id_list):
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
comment = judge_topic_info_get_comment(topic_id) comment = judge_topic_info_get_comment(topic_id)
reply(cookies, topic_id, comment) if comment:
# reply(cookies, topic_id, comment)
reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time())
else:
comment1, comment2 = get_answer_data()
response = reply_answer(cookies, topic_id, comment1)
cookies = login()
reply_id = response["data"]["id"]
reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time())
except: except:
logging_exception() logging_exception()
logging.error("catch exception,main:%s" % traceback.format_exc()) logging.error("catch exception,main:%s" % traceback.format_exc())
......
...@@ -3,9 +3,10 @@ import traceback ...@@ -3,9 +3,10 @@ import traceback
import logging import logging
from threading import Thread from threading import Thread
from vest.request.auto_request import login, time_conv_hour, get_answer_data, reply_answer, get_majia, \ from vest.request.auto_request import login, time_conv_hour, get_answer_data, reply_answer, get_majia, \
set_reply_to_redis, judge_topic_info_get_comment, reply set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
from vest.request.auto_request import host, user, db, passwd from vest.request.auto_request import host, user, db, passwd
from libs.error import logging_exception from libs.error import logging_exception
from libs.timelib import get_rand_time
import random import random
...@@ -14,7 +15,7 @@ def get_data(numtime, numtime2): ...@@ -14,7 +15,7 @@ def get_data(numtime, numtime2):
pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306) pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
cursor = pc.cursor() cursor = pc.cursor()
cursor.execute( cursor.execute(
"SELECT user_id FROM topic WHERE is_online=1 and (create_time >= '%s' and create_time <= '%s' )" % ( "SELECT id FROM topic WHERE is_online=1 and (create_time >= '%s' and create_time <= '%s' )" % (
numtime2, numtime)) numtime2, numtime))
res = cursor.fetchall() res = cursor.fetchall()
return res and [r for r, in res] or [] return res and [r for r, in res] or []
...@@ -30,7 +31,16 @@ def batch_handle(topic_id_list): ...@@ -30,7 +31,16 @@ def batch_handle(topic_id_list):
cookies = login() cookies = login()
if cookies is not None: if cookies is not None:
comment = judge_topic_info_get_comment(topic_id) comment = judge_topic_info_get_comment(topic_id)
reply(cookies, topic_id, comment) if comment:
# reply(cookies, topic_id, comment)
reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time(hourup=1))
else:
comment1, comment2 = get_answer_data()
response = reply_answer(cookies, topic_id, comment1)
cookies = login()
reply_id = response["data"]["id"]
reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time(hourup=1))
except: except:
logging_exception() logging_exception()
logging.error("catch exception,main:%s" % traceback.format_exc()) logging.error("catch exception,main:%s" % traceback.format_exc())
......
...@@ -7,6 +7,7 @@ import logging ...@@ -7,6 +7,7 @@ import logging
import json import json
import redis import redis
import smtplib import smtplib
from celery import shared_task
from libs.cache import redis_client from libs.cache import redis_client
from email.mime.text import MIMEText from email.mime.text import MIMEText
from email.utils import formataddr from email.utils import formataddr
...@@ -113,6 +114,7 @@ def logins(user_id): ...@@ -113,6 +114,7 @@ def logins(user_id):
return None return None
@shared_task
def click(cookies_get, id): def click(cookies_get, id):
# 点赞 # 点赞
try: try:
...@@ -145,6 +147,7 @@ def click(cookies_get, id): ...@@ -145,6 +147,7 @@ def click(cookies_get, id):
logging.error("catch exception,logins:%s" % traceback.format_exc()) logging.error("catch exception,logins:%s" % traceback.format_exc())
@shared_task
def reply(cookies_get, id, content): def reply(cookies_get, id, content):
try: try:
post_dict = { post_dict = {
...@@ -267,6 +270,7 @@ def get_comments(): ...@@ -267,6 +270,7 @@ def get_comments():
return None return None
@shared_task
def follow(cookies_get, id): def follow(cookies_get, id):
try: try:
post_dict = { post_dict = {
...@@ -431,6 +435,7 @@ def set_reply_to_redis(): ...@@ -431,6 +435,7 @@ def set_reply_to_redis():
logging.error("catch exception,logins:%s" % traceback.format_exc()) logging.error("catch exception,logins:%s" % traceback.format_exc())
@shared_task
def reply2(cookies_get, id, content, replied_id): def reply2(cookies_get, id, content, replied_id):
try: try:
post_dict = { post_dict = {
...@@ -450,6 +455,7 @@ def reply2(cookies_get, id, content, replied_id): ...@@ -450,6 +455,7 @@ def reply2(cookies_get, id, content, replied_id):
logging.error("catch exception,logins:%s" % traceback.format_exc()) logging.error("catch exception,logins:%s" % traceback.format_exc())
@shared_task
def pictorial_reply(cookies_get, id, content): def pictorial_reply(cookies_get, id, content):
try: try:
post_dict = { post_dict = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment