Merge branch 'wsy' into 'test'

add readme

See merge request !514

README

-strategy part like es perform
-容器运行命令
-docker run -it -p 9999:9999 -d \
---volume=/etc/gm-config:/etc/gm-config \
---volume=/etc/resolv.conf:/etc/resolv.conf  \
---volume=/srv/apps/physical/physical/settings_local.py:/srv/apps/physical/physical/settings_local.py \
---entrypoint="/bin/bash" physical-test:1.0 "-c" "source ~/.bashrc && celery -A physical worker -Q tapir-alpha --loglevel=DEBUG  --maxtasksperchild 500 -c 62"
-

README.md

+strategy part like es perform
+容器运行命令
+docker run -it -p 9999:9999 -d \
+--volume=/etc/gm-config:/etc/gm-config \
+--volume=/etc/resolv.conf:/etc/resolv.conf  \
+--volume=/srv/apps/physical/physical/settings_local.py:/srv/apps/physical/physical/settings_local.py \
+--entrypoint="/bin/bash" physical-test:1.0 "-c" "source ~/.bashrc && celery -A physical worker -Q tapir-alpha --loglevel=DEBUG  --maxtasksperchild 500 -c 62"
+
+
+## 启动方式
+
+### 方式1：[program:physical-linucb] 消费kafka数据 
+一个while True服务，去消费kafka数据，维护redis中device的tag_map
+```python
+source /srv/envs/physical/bin/activate && python manage.py trans2es_data2es_parallel -S linucb
+```
+
+### 方式2：[program:physical] rpc服务
+启动rpc服务供后端调用, 接口都在search文件夹下的bind函数定义
+```python
+gunicorn gm_rpcd.wsgi:application --workers=1 --worker-class=gevent 
+--worker-connections=1024 
+--bind=0.0.0.0:9999 
+--user=gmuser 
+--chdir /srv/apps/physical/ 
+--timeout 600 
+--log-level=debug  
+--error-logfile=/data/log/physical/app/gunicorn_error.log 
+--access-logfile=/data/log/physical/app/gunicorn_access.log
+```
+
+### 方式3 [program:physical-celery] celery 消息队列服务
+mysql数据同步到es，入口文件：injection.data_sync.tasks.write_to_es
+数据同步工作大部分在数据管道，只有小部分在这，暂时先不用管这个服务，需求用不到
+```python
+celery -A physical worker -Q tapir-alpha --loglevel=DEBUG  --maxtasksperchild 500 -c 62
+```
+
+### 方式4 [program:physical-beat-celery] celery 消息队列定时（作用未知）
+```python
+celery -A physical beat
+```
+
+# 自动运行脚本
+### venus服务端脚本
+通过celery beat 运行
+配置说明文件：venus/setting/base.py
+具体脚本：venus/communityity/task/xxx.py
+目前共5个脚本
+1：每天保存用户扫脸图片：community.tasks.save_image_task.save_user_face_image
+2：每天3点0分评论自动点赞：community.tasks.crontab_reply_vote_task.crontab_reply_vote
+3：向用户推送兴趣标签：community.tasks.push_task.push_interest_tag_pictorial_to_user
+4：向用户推送经验标签：community.tasks.push_task.push_topic_draft_task
+5：推送主题草稿任务：community.tasks.push_task.push_topic_draft_task
+
+### physical策略端脚本
+在Alp-Test-Cos-test001服务器上，通过crontab -l查看

test.py

+import datetime
+
+
+def time_conv(minutest, minutest2):
+    try:
+        now = datetime.datetime.now()
+        yes_time = now - datetime.timedelta(minutes=minutest)
+        yes_time2 = now - datetime.timedelta(minutes=minutest2)
+        return yes_time, yes_time2
+    except:
+        return None
+
+numtime1, numtime2 = time_conv(1, 5)
+
+print(numtime1, numtime2)
+

trans2es/management/commands/trans2es_mapping2es.py

@@ -10,7 +10,7 @@ from trans2es.type_info import get_type_info_map, TypeInfo
 from vest.reply import true_comment_one, true_comment_two, true_comment_three, one_seven_topic_comment
 from vest.click import true_click_five, true_click_two, true_click_four, true_click_one, true_click_three, \
     one_seven_star_topic
-from vest.follow import auto_follow, auto_follow_new
+from vest.follow import auto_follow, auto_follow_new, auto_follow_per_5m_by_followed
 from vest.urge import auto_star_urge, auto_lunch_app, auto_lunch_app2, auto_urge1, auto_urge2
 from vest.fix import fix_no_comment_click
 from vest.reply_answer import reply_comment2, reply_comment3, answer_reply2, answer_reply3, answer_reply1, \
@@ -114,6 +114,9 @@ class Command(BaseCommand):
                 auto_follow.auto_follow()
             if options["mvest"] == "auto_follow_new":
                 auto_follow_new.auto_follow_new()
+            if options['mvest'] == "auto_follow_per_5m_by_followed":
+                auto_follow_per_5m_by_followed.auto_follow_per_5m_by_followed()
+
 
             # 补足
             if options["mvest"] == "fix_no_comment_click":

vest/data/majia_user_ids.py

+majia_user_ids_list = [241407406, 241407033, 241407154, 241407307, 241407284, 241407248, 241407179, 241407428, 241407470, 241407314, 241407385, 241407490, 241407375, 241407318, 241407508, 241407192, 241407131, 241407226, 241407174, 241407505, 241407358, 241407466, 241407452, 241407093, 241407289, 241407495, 241407405, 241407200, 241407330, 241407079, 241407207, 241407100, 241407116, 241407111, 241407302, 241407095, 241407086, 241407499, 241407519, 241407031, 241407288, 241407436, 241407310, 241407404, 241407493, 241407201, 241407084, 241407158, 241407327, 241407178, 241407164, 241407292, 241407387, 241407352, 241407482, 241407325, 241407256, 241407252, 241407485, 241407376, 241407242, 241407219, 241407166, 241407413, 241407343, 241407423, 241407468, 241407198, 241407170, 241407287, 241407138, 241407353, 241407065, 241407262, 241407501, 241407255, 241407512, 241407309, 241407494, 241407319, 241407373, 241407264, 241407471, 241407250, 241407479, 241407456, 241407443, 241407362, 241407157, 241407130, 241407075, 241407390, 241407036, 241407502, 241407092, 241407513, 241407136, 241407048, 241407448, 241407080, 241407459, 241407045, 241407407, 241407186, 241407083, 241407073, 241407191, 241407421, 241407062, 241407275, 241407347, 241407397, 241407052, 241407393, 241407150, 241407063, 241407212, 241407072, 241407500, 241407331, 241407339, 241407474, 241407378, 241407147, 241407316, 241407301, 241407516, 241407527, 241407233, 241407526, 241407365, 241407384, 241407349, 241407210, 241407162, 241407042, 241407057, 241407514, 241407398, 241407103, 241407509, 241407336, 241407335, 241407145, 241407429, 241407144, 241407068, 241407074, 241407077, 241407238, 241407112, 241407236, 241407245, 241407364, 241407101, 241407108, 241407355, 241407431, 241407247, 241407167, 241407050, 241407064, 241407276, 241407139, 241407120, 241407249, 241407097, 241407437, 241407377, 241407140, 241407475, 241407469, 241407168, 241407486, 241407263, 241407379, 241407243, 241407081, 241407114, 241407230, 241407169, 241407059, 241407354, 241407253, 241407106, 241407467, 241407206, 241407221, 241407481, 241407326, 241407185, 241407322, 241407411, 241407044, 241407285, 241407324, 241407522, 241407175, 241407151, 241407246, 241407433, 241407496, 241407234, 241407183, 241407524, 241407265, 241407146, 241407043, 241407217, 241407046, 241407229, 241407417, 241407124, 241407280, 241407137, 241407091, 241407132, 241407261, 241407115, 241407071, 241407165, 241407369, 241407418, 241407176, 241407040, 241407143, 241407058, 241407491, 241407434, 241407329, 241407462, 241407135, 241407338, 241407283, 241407094, 241407117, 241407239, 241407410, 241407435, 241407060, 241407420, 241407424, 241407308, 241407489, 241407286, 241407268, 241407311, 241407328, 241407232, 241407463, 241407304, 241407346, 241407076, 241407126, 241407177, 241407382, 241407438, 241407032, 241407515, 241407363, 241407189, 241407360, 241407282, 241407295, 241407109, 241407340, 241407333, 241407240, 241407457, 241407231, 241407163, 241407492, 241407220, 241407125, 241407182, 241407070, 241407293, 241407356, 241407190, 241407128, 241407039, 241407241, 241407451, 241407334, 241407270, 241407090, 241407412, 241407465, 241407119, 241407244, 241407487, 241407511, 241407196, 241407415, 241407054, 241407504, 241407300, 241407320, 241407447, 241407205, 241407460, 241407216, 241407389, 241407188, 241407313, 241407156, 241407113, 241407350, 241407525, 241407497, 241407211, 241407345, 241407394, 241407107, 241407266, 241407332, 241407197, 241407337, 241407085, 241407366, 241407305, 241407401, 241407315, 241407089, 241407281, 241407386, 241407038, 241407351, 241407507, 241407484, 241407171, 241407082, 241407214, 241407419, 241407321, 241407409, 241407251, 241407426, 241407444, 241407432, 241407391, 241407357, 241407213, 241407368, 241407416, 241407342, 241407312, 241407303, 241407121, 241407099, 241407521, 241407259, 241407510, 241407049, 241407087, 241407279, 241407441, 241407388, 241407105, 241407478, 241407317, 241407142, 241407096, 241407422, 241407277, 241407141, 241407488, 241407297, 241407473, 241407110, 241407361, 241407306, 241407402, 241407215, 241407273, 241407155, 241407257, 241407454, 241407274, 241407123, 241407104, 241407208, 241407298, 241407399, 241407030, 241407069, 241407153, 241407453, 241407449, 241407440, 241407455, 241407223, 241407278, 241407400, 241407359, 241407088, 241407202, 241407254, 241407194, 241407159, 241407161, 241407061, 241407267, 241407372, 241407381, 241407374, 241407173, 241407392, 241407271, 241407341, 241407299, 241407446, 241407439, 241407523, 241407396, 241407227, 241407041, 241407067, 241407181, 241407344, 241407199, 241407203, 241407193, 241407127, 241407348, 241407118, 241407260, 241407209, 241407370, 241407291, 241407367, 241407380, 241407503, 241407037, 241407269, 241407450, 241407133, 241407371, 241407395, 241407122, 241407204, 241407430, 241407152, 241407148, 241407425, 241407160, 241407427, 241407296, 241407518, 241407480, 241407180, 241407272, 241407403, 241407102, 241407225, 241407195, 241407472, 241407056, 241407498, 241407218, 241407055, 241407323, 241407461, 241407098, 241407235, 241407187, 241407290, 241407520, 241407483, 241407172, 241407134, 241407506, 241407129, 241407476, 241407517, 241407237, 241407442, 241407184, 241407464, 241407529, 241407047, 241407053, 241407458, 241407078, 241407066, 241407294, 241407258, 241407035, 241407445, 241407149, 241407414, 241407222, 241407383, 241407408, 241407034, 241407051, 241407477, 241407224, 241407228, 241407528]
\ No newline at end of file

vest/follow/auto_follow_per_2h.py

+import pymysql
+import random
+import traceback
+import logging
+from vest.request.auto_request import login, time_convs_hours, follow
+from vest.request.auto_request import host, user, db, passwd
+from libs.error import logging_exception
+
+
+def get_commnet_id(numtime):
+    pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    cursor = pc.cursor()
+
+    cursor.execute(
+        "SELECT  user_id FROM reply WHERE  create_time > '%s' " % numtime)
+
+    data_comment = cursor.fetchall()
+
+    cursor.execute(
+
+        "SELECT  t.user_id  from topic t left join reply r on  r.topic_id = t.id WHERE  r.create_time > '%s'" % numtime)
+
+    data_reply_topic = cursor.fetchall()
+
+    cursor.execute(
+
+        "SELECT  r.user_id  from reply r left join reply p on  r.replied_id = p.replied_id  WHERE  p.create_time > '%s'" % numtime)
+
+    data_reply_reply = cursor.fetchall()
+
+    cursor.execute(
+
+        "SELECT  user_id FROM group_user_role  WHERE  create_time > '%s' " % numtime)
+
+    data_group_follow = cursor.fetchall()
+
+    cursor.execute(
+
+        "select  user_id from account_user  WHERE  create_time > '%s'" % numtime)
+
+    data_new_user = cursor.fetchall()
+
+    data_id = list(data_comment)
+    data1_id = list(data_reply_topic)
+    data2_id = list(data_group_follow)
+    data3_id = list(data_new_user)
+    data_reply = list(data_reply_reply)
+    all_data = []
+    all_data.extend(data_id)
+    all_data.extend(data2_id)
+    all_data.extend(data3_id)
+    all_data.extend(data_reply)
+    all_data.extend(data1_id)
+
+    topic_id_list = []
+    for i in all_data and all_data:
+        cursor.execute(
+            "select user_id from user_extra where user_id =" + str(i[0]) + " and is_shadow =0 ")
+        data = cursor.fetchall()
+        user_id = list(data)
+        if (user_id and user_id[0][1] == 0) or len(user_id) == 0:
+            topic_id_list.append(i)
+
+    logging.info("Database version : %s " % topic_id_list)
+    pc.close()
+    return topic_id_list
+
+
+def auto_follow_per_2h():
+    # 注册，发帖 2个事件触发自动加粉丝(在2个小时内加1-3个粉)
+
+    try:
+        numtime1, numtime2 = time_convs_hours(1, 2)
+        user_id = get_commnet_id(numtime1)
+        try:
+            for j in user_id and user_id:
+                id = int(j[0])
+                follow_num = random.randint(0, 1)
+                for i in range(follow_num):
+                    cookies = login()
+                    if cookies is not None:
+                        follow(cookies, id)
+        except:
+            pass
+
+    except:
+        logging_exception()
+        logging.error("catch exception,main:%s" % traceback.format_exc())

vest/follow/auto_follow_per_5m_by_followed.py

+import pymysql
+import traceback
+import logging
+from vest.request.auto_request import login, follow, time_conv
+from vest.request.auto_request import host, user, db, passwd
+from vest.data.majia_user_ids import majia_user_ids_list
+from libs.error import logging_exception
+
+
+def get_user_id(numtime):
+    pc = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=3306)
+    cursor = pc.cursor()
+    majia_user_ids_tuple = tuple(majia_user_ids_list)
+    cursor.execute(
+        "SELECT  user_id FROM user_follow WHERE  create_time > '%s' and follow_id in %s " % (numtime, str(majia_user_ids_tuple))
+    )
+    data_comment = cursor.fetchall()
+    return data_comment and [i for i in data_comment] or []
+
+
+def auto_follow_per_5m_by_followed():
+    # 如果马甲被用户关注，马甲即反向关注用户, 被关注5分钟后
+    try:
+        numtime1, numtime2 = time_conv(1, 5)
+        user_ids = get_user_id(numtime2)
+        try:
+            for user_id in user_ids and user_ids:
+                cookies = login()
+                if cookies is not None:
+                    follow(cookies, user_id)
+        except:
+            pass
+
+    except:
+        logging_exception()
+        logging.error("catch exception,main:%s" % traceback.format_exc())

vest/request/auto_request.py

@@ -112,6 +112,7 @@ def logins(user_id):
 
 
 def click(cookies_get, id):
+    # 点赞
     try:
         post_dict = {
             'type': 0,
@@ -168,6 +169,17 @@ def time_conv_minute(minutest, minutest2):
         return None
 
 
+def time_conv_hour(minutest, minutest2):
+    try:
+        now = datetime.datetime.now()
+        minute = datetime.datetime.now().minute
+        yes_time = now - datetime.timedelta(hours=minutest)
+        yes_time2 = now - datetime.timedelta(hours=minutest2)
+        return yes_time, yes_time2, minute
+    except:
+        return None
+
+
 def time_now(minutest):
     try:
         now = datetime.datetime.now()
@@ -178,6 +190,11 @@ def time_now(minutest):
 
 
 def time_convs(numtime, numtime2):
+    '''
+    params: 1, 2
+    datetime.now(): 2019-10-09 10:35:50.231463
+    return: 2019-10-08 23:59:59.000000 2019-10-07 00:00:00.000000
+    '''
     try:
         now = datetime.datetime.now()
         yes_time = now - datetime.timedelta(days=numtime)
@@ -190,7 +207,6 @@ def time_convs(numtime, numtime2):
 
         logging.info("get yes_time_str:%s" % yes_time_str)
         logging.info("get yes_time_str2:%s" % yes_time_str2)
-
         return yes_time_str, yes_time_str2
     except:
         return None