add async apply

Dockerfile

@@ -20,6 +20,7 @@ RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32 \
     && apt-get -y update && apt-get -y install mysql-client libmysqlclient-dev \
     && apt-get clean \
     && apt-get autoclean \
+    && apt-get  -y install vim \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt

libs/timelib.py

+import pytz
+import random
+from datetime import datetime, timedelta
+
+NOW = datetime.now()
+
+
+def tzlc(dt, truncate_to_sec=True):
+    if dt is None:
+        return None
+    if truncate_to_sec:
+        dt = dt.replace(microsecond=0)
+    return pytz.timezone('Asia/Shanghai').localize(dt)
+
+
+def eta_2_push_time(eta):
+    if eta:
+        eta = datetime.strptime(eta, '%Y-%m-%d %H:%M:%S')
+        eta = tzlc(eta)
+        return int((eta - datetime.fromtimestamp(0, pytz.timezone("UTC"))).total_seconds())
+    else:
+        push_time = None
+    return push_time
+
+
+def get_rand_time(hourlow=0, hourup=13, minutelow=0, miniteup=60):
+    hours = random.randint(hourlow, hourup)
+    minutes = random.randint(minutelow, miniteup)
+    now_time = NOW + timedelta(hours=hours, minutes=minutes)
+    time = eta_2_push_time(now_time.strftime("%Y-%m-%d %H:%M:%S"))
+    return datetime.fromtimestamp(time)
+
+
+
+
+
+
+
+
+

physical/__init__.py

@@ -2,6 +2,6 @@ from __future__ import unicode_literals, absolute_import, print_function
 
 import pymysql
 from _celery import app as celery_app
-
+from vest import  *
 pymysql.install_as_MySQLdb()
 #__all__ = ('celery_app',)

physical/celery_task_router.py

@@ -9,7 +9,11 @@ class CeleryTaskRouter(object):
     queue_task_map = {
         "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
+        ],
+        "vest": [
+            "vest.tasks.follow",
         ]
+
     }
 
     # Map[TaskName, QueueName]

vest/__init__.py

+from .request import *
\ No newline at end of file

vest/click/auto_click_per_1d_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, click
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
@@ -41,7 +42,8 @@ def batch_handle(auto_click_list):
         try:
             cookies = login()
             if cookies is not None:
-                click(cookies, topic_id)
+                click.async_apply(args=(cookies, topic_id), eta=get_rand_time())
+                # click(cookies, topic_id)
         except:
             pass
 

vest/click/auto_click_per_2h_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, click
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
@@ -42,7 +43,8 @@ def batch_handle(auto_click_list):
         try:
             cookies = login()
             if cookies is not None:
-                click(cookies, topic_id)
+                # click(cookies, topic_id)
+                click.async_apply(args=(cookies, topic_id), eta=get_rand_time())
         except:
             pass
 

vest/follow/auto_follow_per_1d_by_post.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=3):
@@ -28,7 +29,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/follow/auto_follow_per_1d_by_regist.py

@@ -6,6 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_convs, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_commnet_id(numtime, numtime2):
@@ -27,7 +28,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/follow/auto_follow_per_2h_by_post_and_regist.py

@@ -6,7 +6,7 @@ from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, follow
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
-
+from libs.timelib import get_rand_time
 
 def get_commnet_id(numtime, numtime2, content_level_low=0, content_level_top=6):
     pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
@@ -40,7 +40,9 @@ def batch_handle(auto_follow_list):
         try:
             cookies = login()
             if cookies is not None:
-                follow(cookies, user_id)
+                # follow(cookies, user_id)
+                follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/follow/auto_follow_per_5m_by_followed.py

@@ -5,6 +5,7 @@ from vest.request.auto_request import login, follow, time_conv_minute
 from vest.request.auto_request import host, user, db, passwd
 from vest.data.majia_user_ids import majia_user_ids_dict
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 
 
 def get_user_id(numtime):
@@ -28,7 +29,9 @@ def auto_follow_per_5m_by_followed():
                 user_id = user[0]
                 cookies = login(str(majia_user_ids_dict.get(str(user[1]))))
                 if cookies is not None:
-                    follow(cookies, user_id)
+                    # follow(cookies, user_id)
+                    follow.async_apply(args=(cookies, user_id), eta=get_rand_time())
+
         except:
             pass
 

vest/reply/auto_reply_per_1d_to_pictorial.py

@@ -8,6 +8,7 @@ from vest.request.auto_request import login, pictorial_reply, get_majia, get_pic
 
 from vest.request.auto_request import host, user, db, passwd
 from vest.data.topic_models import get_pictorial_tag_by_id
+from libs.timelib import get_rand_time
 
 """
 榜单新增内容(仅限在线的）
@@ -55,7 +56,9 @@ def batch_handle(pictorial_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_pictorial_info_get_comment(pictorial_id)
-                pictorial_reply(cookies, pictorial_id, comment)
+                # pictorial_reply(cookies, pictorial_id, comment)
+                pictorial_reply.async_apply(args=(cookies, pictorial_id, comment), eta=get_rand_time())
+
         except:
             pass
 

vest/reply/auto_reply_per_1d_to_topic.py

 import pymysql
 import traceback
 import logging
+import random
 from threading import Thread
 from vest.request.auto_request import login, time_convs, get_answer_data, reply_answer, get_majia, \
-    set_reply_to_redis, judge_topic_info_get_comment, reply
+    set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
-import random
+from libs.timelib import get_rand_time
 
 
 def get_data(numtime, numtime2, content_level_low, content_level_top):
@@ -31,7 +32,18 @@ def batch_handle(topic_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_topic_info_get_comment(topic_id)
-                reply(cookies, topic_id, comment)
+                if comment:
+                    # reply(cookies, topic_id, comment)
+                    reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time())
+                else:
+                    comment1, comment2 = get_answer_data()
+                    response = reply_answer(cookies, topic_id, comment1)
+                    cookies = login()
+                    reply_id = response["data"]["id"]
+                    reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time())
+
+
+
         except:
             logging_exception()
             logging.error("catch exception,main:%s" % traceback.format_exc())

vest/reply/auto_reply_per_2h_to_topic.py

@@ -3,9 +3,10 @@ import traceback
 import logging
 from threading import Thread
 from vest.request.auto_request import login, time_conv_hour, get_answer_data, reply_answer, get_majia, \
-    set_reply_to_redis, judge_topic_info_get_comment, reply
+    set_reply_to_redis, judge_topic_info_get_comment, reply, reply2
 from vest.request.auto_request import host, user, db, passwd
 from libs.error import logging_exception
+from libs.timelib import get_rand_time
 import random
 
 
@@ -30,7 +31,16 @@ def batch_handle(topic_id_list):
             cookies = login()
             if cookies is not None:
                 comment = judge_topic_info_get_comment(topic_id)
-                reply(cookies, topic_id, comment)
+                if comment:
+                    # reply(cookies, topic_id, comment)
+                    reply.async_apply(args=(cookies, topic_id, comment), eta=get_rand_time())
+                else:
+                    comment1, comment2 = get_answer_data()
+                    response = reply_answer(cookies, topic_id, comment1)
+                    cookies = login()
+                    reply_id = response["data"]["id"]
+                    reply2.async_apply(args=(cookies, topic_id, comment2, reply_id), eta=get_rand_time())
+
         except:
             logging_exception()
             logging.error("catch exception,main:%s" % traceback.format_exc())

vest/request/auto_request.py

@@ -7,6 +7,7 @@ import logging
 import json
 import redis
 import smtplib
+from celery import shared_task
 from libs.cache import redis_client
 from email.mime.text import MIMEText
 from email.utils import formataddr
@@ -113,6 +114,7 @@ def logins(user_id):
         return None
 
 
+@shared_task
 def click(cookies_get, id):
     # 点赞
     try:
@@ -145,6 +147,7 @@ def click(cookies_get, id):
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task
 def reply(cookies_get, id, content):
     try:
         post_dict = {
@@ -267,6 +270,7 @@ def get_comments():
         return None
 
 
+@shared_task
 def follow(cookies_get, id):
     try:
         post_dict = {
@@ -431,6 +435,7 @@ def set_reply_to_redis():
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task()
 def reply2(cookies_get, id, content, replied_id):
     try:
         post_dict = {
@@ -450,6 +455,7 @@ def reply2(cookies_get, id, content, replied_id):
         logging.error("catch exception,logins:%s" % traceback.format_exc())
 
 
+@shared_task
 def pictorial_reply(cookies_get, id, content):
     try:
         post_dict = {