Merge branch 'wsy3' into 'test'

Wsy3

See merge request !528

Dockerfile

@@ -20,6 +20,7 @@ RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32 \
     && apt-get -y update && apt-get -y install mysql-client libmysqlclient-dev \
     && apt-get clean \
     && apt-get autoclean \
+    && apt-get  -y install vim \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt

crontab.py

@@ -15,21 +15,17 @@ ontime_list = [
     "30 10 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_urge2",
     "0 10 * * 3 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_lunch_app",
     "30  10 * * 3 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_lunch_app2",
-    "*/5 * * * 1 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_5m_by_followed",
+    # "*/5 * * * 1 source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_5m_by_followed",
     "1 */2 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_2h_by_post_and_regist",
     "0 9  * * *  source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&   python manage.py  trans2es_mapping2es -m  get_login_session",
     "0 0  * * 3  source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m  get_user_id",
     # "0  14,18,22 *  *  *  source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m  principal_online_comment1",
-    "25 */2 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m auto_reply_per_2h_to_topic"
-
-]
-
-random_time_list = [
-    "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_click_per_1d_by_post",
-    "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_regist",
-    "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_post",
-    "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
-    "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_topic"
+    "25 */2 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical &&  python manage.py  trans2es_mapping2es -m auto_reply_per_2h_to_topic",
+    "0 9 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_click_per_1d_by_post",
+    "1 9 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_regist",
+    "2 9 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_follow_per_1d_by_post",
+    "3 9 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_pictorial",
+    "4 9 * * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m auto_reply_per_1d_to_topic"
     # "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m answer_reply1",
     # "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m answer_reply2",
     # "* * * source /srv/envs/physical/bin/activate  && cd /srv/apps/physical && python manage.py  trans2es_mapping2es -m answer_reply3",
@@ -48,16 +44,6 @@ random_time_list = [
 ]
 
 
-
-###随机生成10-23的一个数字 代表小时
-ss1 = random.choices(range(10, 23), k=100)
-###随机生成1-60的一个数字  表示分钟
-ss2 = random.choices(range(1, 50), k=100)
-
-
-for i in range(len(random_time_list)):
-    ontime_list.append(str(ss2[i]) + " " + str(ss1[i]) + " " + str(random_time_list[i]))
-
 data = open("/data/log/physical/app/conf.txt", "w")
 # data = open("abc.txt", "w")
 

libs/timelib.py

+import pytz
+import random
+from datetime import datetime, timedelta
+
+NOW = datetime.now()
+
+
+def tzlc(dt, truncate_to_sec=True):
+    if dt is None:
+        return None
+    if truncate_to_sec:
+        dt = dt.replace(microsecond=0)
+    return pytz.timezone('Asia/Shanghai').localize(dt)
+
+
+def eta_2_push_time(eta):
+    if eta:
+        eta = datetime.strptime(eta, '%Y-%m-%d %H:%M:%S')
+        eta = tzlc(eta)
+        return int((eta - datetime.fromtimestamp(0, pytz.timezone("UTC"))).total_seconds())
+    else:
+        push_time = None
+    return push_time
+
+
+def get_rand_time(hourlow=0, hourup=13, minutelow=0, miniteup=60):
+    hours = random.randint(hourlow, hourup)
+    minutes = random.randint(minutelow, miniteup)
+    now_time = NOW + timedelta(hours=hours, minutes=minutes)
+    time = eta_2_push_time(now_time.strftime("%Y-%m-%d %H:%M:%S"))
+    return datetime.fromtimestamp(time)
+
+
+
+
+
+
+
+
+

physical/__init__.py

@@ -2,6 +2,6 @@ from __future__ import unicode_literals, absolute_import, print_function
 
 import pymysql
 from _celery import app as celery_app
-
+from vest import  *
 pymysql.install_as_MySQLdb()
 #__all__ = ('celery_app',)

physical/celery_task_router.py

@@ -9,7 +9,12 @@ class CeleryTaskRouter(object):
     queue_task_map = {
         "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
-        ]
+            'vest.request.auto_request.click',
+            'vest.request.auto_request.reply',
+            'vest.request.auto_request.follow',
+            'vest.request.auto_request.reply2',
+            'vest.request.auto_request.pictorial_reply'
+        ],
     }
 
     # Map[TaskName, QueueName]

vest/__init__.py

+from .request import *
\ No newline at end of file

vest/click/auto_click_per_1d_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, click
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
@@ -41,7 +42,8 @@ def batch_handle(auto_click_list):
         try:
             cookies = login()
             if cookies is not None:
-                click(cookies, topic_id)
+                click.async_apply(args=(cookies, topic_id), eta=get_rand_time())
+                # click(cookies, topic_id)
         except:
             pass
 

vest/click/auto_click_per_2h_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, click
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
@@ -42,7 +43,8 @@ def batch_handle(auto_click_list):
         try:
             cookies = login()
             if cookies is not None:
-                click(cookies, topic_id)
+                # click(cookies, topic_id)
+                click.async_apply(args=(cookies, topic_id), eta=get_rand_time(hourup=1))
         except:
             pass
 

vest/follow/auto_follow_per_1d_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=3):
@@ -28,7 +29,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/follow/auto_follow_per_1d_by_regist.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2):
@@ -27,7 +28,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/follow/auto_follow_per_2h_by_post_and_regist.py

@@ -6,7 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
-
+from libs.timelib import get_rand_time
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
@@ -40,7 +40,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time(hourup=1))
+
         except:
             pass
 

vest/follow/auto_follow_per_5m_by_followed.py

@@ -5,6 +5,7 @@ from vest.request.auto_request import login, follow, time_conv_minute
 from vest.request.auto_request import host, user, db, passwd
 from vest.data.majia_user_ids import majia_user_ids_dict
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_user_id(numtime):
@@ -28,7 +29,9 @@ def auto_follow_per_5m_by_followed():
                 user_id = user[0]
                 cookies = login(str(majia_user_ids_dict.get(str(user[1]))))
                 if cookies is not None:
-                    follow(cookies, user_id)
+                    # follow(cookies, user_id)
+                    follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/reply/auto_reply_per_1d_to_pictorial.py

@@ -8,6 +8,7 @@ from vest.request.auto_request import login, pictorial_reply, get_majia, get_pic
 
 from vest.request.auto_request import host, user, db, passwd
 from vest.data.topic_models import get_pictorial_tag_by_id
+from libs.timelib import get_rand_time
 
 """
 榜单新增内容(仅限在线的）
@@ -38,8 +39,8 @@ def get_data(numtime1, numtime2):
         pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
         cursor = pc.cursor()
         cursor.execute(
-            "SELECT  pictorial_id FROM community_pictorial_topic WHERE is_online=1 and (create_time >= '%s' and create_time < '%s') group by pictorial_id " % (
-                numtime1, numtime2))
+            "SELECT id FROM community_pictorial WHERE is_online=1 and (create_time >= '%s' and create_time < '%s') " % (
+                numtime2, numtime1))
         res = cursor.fetchall()
 
         return res and [r for r, in res] or []
@@ -55,16 +56,15 @@ def batch_handle(pictorial_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_pictorial_info_get_comment(pictorial_id)
-                pictorial_reply(cookies, pictorial_id, comment)
+                # pictorial_reply(cookies, pictorial_id, comment)
+                pictorial_reply.async_apply(args=(cookies, pictorial_id, comment), eta=get_rand_time())
+
         except:
             pass
 
 
 def auto_reply_per_1d_to_pictorial():
     # 问题自动增加回答
-    # todo topic_vote_cnt和community_pictorial_topic
-    # todo 1天前发的问题：[0-1]个评论  6天前发的问题：[0-1]个评论 这个是什么鬼
-    # todo 怎么禁止作者评论
     pictorial_id_list = []
     try:
         logging.info("comment offline  pictorial")

vest/reply/auto_reply_per_1d_to_topic.py

 import pymysql
 import traceback
 import logging
+import random
 from threading import Thread
 from vest.request.auto_request import login, time_convs, get_answer_data, reply_answer, get_majia, \
-    set_reply_to_redis, judge_topic_info_get_comment, reply
+    set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
-import random
+from libs.timelib import get_rand_time
 
 
 def get_data(numtime, numtime2, content_level_low, content_level_top):
@@ -31,7 +32,18 @@ def batch_handle(topic_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_topic_info_get_comment(topic_id)
-                reply(cookies, topic_id, comment)
+                if comment:
+                    # reply(cookies, topic_id, comment)
+                    reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time())
+                else:
+                    comment1, comment2 = get_answer_data()
+                    response = reply_answer(cookies, topic_id, comment1)
+                    cookies = login()
+                    reply_id = response["data"]["id"]
+                    reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time())
+
+
+
         except:
             logging_exception()
             logging.error("catch exception,main:%s" % traceback.format_exc())

vest/reply/auto_reply_per_2h_to_topic.py

@@ -3,9 +3,10 @@ import traceback
 import logging
 from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, get_answer_data, reply_answer, get_majia, \
-    set_reply_to_redis, judge_topic_info_get_comment, reply
+    set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 import random
 
 
@@ -14,7 +15,7 @@ def get_data(numtime, numtime2):
         pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
         cursor = pc.cursor()
         cursor.execute(
-            "SELECT  user_id FROM topic WHERE is_online=1  and (create_time >= '%s' and create_time <= '%s' )" % (
+            "SELECT id FROM topic WHERE is_online=1  and (create_time >= '%s' and create_time <= '%s' )" % (
                 numtime2, numtime))
         res = cursor.fetchall()
         return res and [r for r, in res] or []
@@ -30,7 +31,16 @@ def batch_handle(topic_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_topic_info_get_comment(topic_id)
-                reply(cookies, topic_id, comment)
+                if comment:
+                    # reply(cookies, topic_id, comment)
+                    reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time(hourup=1))
+                else:
+                    comment1, comment2 = get_answer_data()
+                    response = reply_answer(cookies, topic_id, comment1)
+                    cookies = login()
+                    reply_id = response["data"]["id"]
+                    reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time(hourup=1))
+
         except:
             logging_exception()
             logging.error("catch exception,main:%s" % traceback.format_exc())

vest/request/auto_request.py

@@ -7,6 +7,7 @@ import logging
 import json
 import redis
 import smtplib
+from celery import shared_task
 from libs.cache import redis_client
 from email.mime.text import MIMEText
 from email.utils import formataddr
@@ -113,6 +114,7 @@ def logins(user_id):
         return None
 
 
+@shared_task
 def click(cookies_get, id):
     # 点赞
     try:
@@ -145,6 +147,7 @@ def click(cookies_get, id):
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task
 def reply(cookies_get, id, content):
     try:
         post_dict = {
@@ -267,6 +270,7 @@ def get_comments():
         return None
 
 
+@shared_task
 def follow(cookies_get, id):
     try:
         post_dict = {
@@ -431,6 +435,7 @@ def set_reply_to_redis():
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task
 def reply2(cookies_get, id, content, replied_id):
     try:
         post_dict = {
@@ -450,6 +455,7 @@ def reply2(cookies_get, id, content, replied_id):
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task
 def pictorial_reply(cookies_get, id, content):
     try:
         post_dict = {