Merge branch 'mr/develop/add_adviser_es_monitor' into 'master'

新增转诊派单策略监控

See merge request !3

.gitignore

 /venv
 .idea
+.settings_local.py

requirements.txt

 psycopg2==2.8.6
 pymysql
 DingtalkChatbot==1.3.0
+elasticsearch>=7.0.0,<8.0.0

scripies/adveriser_es_monitor.py

+# coding:utf-8
+"""
+实现转诊数据监控
+airflow定时监控，每隔10分钟监控一次
+mysql 里面的数据和es里面的数据是否一致
+"""
+import pymysql
+import datetime
+from dingtalkchatbot.chatbot import DingtalkChatbot
+import settings_local 
+from elasticsearch import Elasticsearch
+
+
+def main():
+    es = Elasticsearch(settings_local.ES_CONFIG, http_auth=settings_local.ES_HTTP_AUTH, timeout=3600)
+    artemis = settings_local.DATABASES["artemis"]
+    artemis_db = pymysql.connect(
+        host=artemis["HOST"],
+        user=artemis["USER"],
+        passwd=artemis["PASSWORD"],
+        database=artemis["NAME"],
+        port=int(artemis["PORT"]))
+    artemis_cursor = artemis_db.cursor()
+    limit = 100
+    account_sql = "select doctor_id, (rechange_amount+cashback_amount) as amount from account_account order by modify_time desc limit %s"
+    
+    try:
+        artemis_cursor.execute(account_sql, (limit))
+        accounts = artemis_cursor.fetchall()
+        doctor_ids = [item[0] for item in accounts]
+        amount_doctors = {item[0]: item[1] for item in accounts}
+        # 批量获取es merchant_translate_treatment_left_consume 、merchant_id、merchant_left_money、is_merchant_priority_send_order
+        query = {
+            "query": {"bool": {
+                "filter": [
+                    {"terms": {"id": doctor_ids}}]}}}
+        es_result = es.search(query)
+        budgetSql = """
+            select budget, is_open from bdtransfer_prioritybdtransfer
+            where merchant_id = {}
+            """
+
+        consumeSql = """
+            select amount from bdtransfer_bdtransferconsume
+            where merchant_id = {}
+            and date = {} """
+
+        today = str(datetime.date.today())
+        num = 0.0
+        error_doctors = []
+        for item in es_result["hits"]["hits"]:
+            mysql_is_merchant_priority_send_order = False
+            budget = 0
+            consume = 0
+            doctor_id = item['_source']["id"]
+            merchant_id = item['_source']["merchant_id"]
+            left_consume = item['_source']["merchant_translate_treatment_left_consume"]
+            merchant_left_money = item['_source']["merchant_left_money"]
+            is_merchant_priority_send_order = item['_source']["is_merchant_priority_send_order"]
+
+            mysql_merchant_left_money = amount_doctors.get(doctor_id, 0)
+            budget_sql = budgetSql.format(merchant_id)
+            consume_sql = consumeSql.format(merchant_id, today)
+            artemis_cursor.execute(budget_sql)
+            budget_res = artemis_cursor.fetchone()
+            artemis_cursor.execute(consume_sql)
+            consume_res = artemis_cursor.fetchone()
+          
+            if budget_res:
+                budget = budget_res[0]
+                mysql_is_merchant_priority_send_order = budget_res[1]
+            if consume_res:
+                consume = consume_res[0]
+            mysql_left_consume = consume - budget
+            print("""
+                merchant_left_money: {}, mysql_merchant_left_money: {},
+                left_consume: {}, mysql_left_consume:{},
+                is_merchant_priority_send_order: {},
+                mysql_is_merchant_priority_send_order:{}""".format(
+                    merchant_left_money, mysql_merchant_left_money,
+                    left_consume, mysql_left_consume,
+                    is_merchant_priority_send_order,
+                    mysql_is_merchant_priority_send_order
+                ))
+            if (
+                    merchant_left_money == mysql_merchant_left_money and
+                    left_consume == mysql_left_consume and
+                    is_merchant_priority_send_order == mysql_is_merchant_priority_send_order):
+                pass
+            else:
+                num = num + 1
+                error_doctors.append(doctor_id)
+        print(num)
+        if num / limit > 0.02:
+            msg = "数据同步监控,总共{}个医生id，有{}个数据存在差异，具体医生id:{}".format(limit, len(error_doctors), error_doctors)
+            webhook = 'https://oapi.dingtalk.com/robot/send?access_token={}'.format(
+                "1475fb9e6beff63126fef5464a378b6b77fa8655933a407d8159a2a2b1b8c869")
+            xiaoding = DingtalkChatbot(webhook)
+            at_mobiles = ["17794411132"]
+            # at_mobiles = []
+            xiaoding.send_text(msg=msg, at_mobiles=at_mobiles)
+        
+    except Exception as e:
+        msg = "数据同步监控,数据diff出现问题"
+        webhook = 'https://oapi.dingtalk.com/robot/send?access_token={}'.format(
+            "1475fb9e6beff63126fef5464a378b6b77fa8655933a407d8159a2a2b1b8c869")
+        xiaoding = DingtalkChatbot(webhook)
+        at_mobiles = ["17794411132"]
+        # at_mobiles = []
+        xiaoding.send_text(msg=msg, at_mobiles=at_mobiles)
+
+
+if __name__ == '__main__':
+    main()

settings_local.py

+# mysql config
+DATABASES = {
+    'artemis': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'artemis_hotfix2',
+        'USER': 'work',
+        'PASSWORD': 'Gengmei1',
+        'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
+        'PORT': '62120',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
+        },
+    },
+}
+
+ES_CONFIG = [
+    {'host': '172.18.52.14', 'port': 9200},
+    {'host': '172.18.52.133', 'port': 9200},
+    {'host': '172.18.52.7', 'port': 9200}
+    ]
+
+ES_HTTP_AUTH = ('elastic', 'gm_test')
+
+ES_INDEX = 'zhuanzhen'

settings_local.py.template

+# mysql config
+DATABASES = {
+    'artemis': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'artemis_hotfix2',
+        'USER': 'work',
+        'PASSWORD': 'Gengmei1',
+        'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
+        'PORT': '62120',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
+        },
+    },
+}
+
+ES_CONFIG = [
+    {'host': '172.18.52.14', 'port': 9200},
+    {'host': '172.18.52.133', 'port': 9200},
+    {'host': '172.18.52.7', 'port': 9200}
+    ]
+
+ES_HTTP_AUTH = ('elastic', 'gm_test')
+
+ES_INDEX = 'zhuanzhen'