fix

.gitignore

@@ -63,4 +63,6 @@ target/
 .vscode/
 .env/
 .env2/
-.python-version
+.python-version
\ No newline at end of file
+
+.config.yaml
\ No newline at end of file

README.md

+### GM-MYSQL_EXPORTER
\ No newline at end of file

config.yaml → config.yaml.tpl

+############################ aliyun configuration ############################################
 # 要监控的数据库实例
-rds_instances:
-  rdsfewzdmf0jfjp9un8xj:
-    name: prod.zhengxing
+aliyun_instances:
+  instance_id:
+    name: instance_name
     # 连接数利用率因子，连接数利用率*该因子得到一个较为统一的标准，默认1
     connection_usage_factor: 1
     # iops利用率因子, 默认 1
@@ -10,55 +11,14 @@ rds_instances:
     cpu_usage_factor: 1
     # 内存利用率因子
     memory_usage_factor: 1
-    # 数据库类型，主库or从库, 默认master
+    # 数据库类型，主库(master) or 从库(slave), 默认master
     type: master
 
-  rdsg2mpi623x6wioo796k:
-    name: prod.readonly2.zhengxing
-    type: slave
-
-  rm-m5e63m5z6o4re1p49:
-    name: mars.prod
-    connection_usage_factor: 0.1
-
-  rm-m5e842126ng59jrv6:
-    name: data.prod
-    connection_usage_factor: 0.1
-
-  rm-m5eu1wmu0zt6j5n72:
-    name: sms.prod
-
-  rr-m5e2vy0ke9q35kst5:
-    name: prod.readonly.zhengxing
-    type: slave
-
-  rm-m5e90y907sw3bnz88:
-    name: common.prod
-
-  rr-m5e2n024w0z293r5f:
-    name: common.readonly
-    type: slave
-
-  rr-m5efx2m67mi002jq9:
-    name: prod.readonly3.zhengxing
-    type: slave
-
-  rm-m5emg41za2w7l6au3:
-    name: mimas_prod
-
-  rm-m5ey2s823bq0lc616:
-    name: bran_prod
-
-  rr-m5et21lafq1677pid:
-    name: mimas_readonly
-    type: slave
-
-  rr-m5e3036a5l18ym4t6:
-    name: mimas_readonly01
-    type: slave
+  instance_id2:
+    name: instance_name2
 
 # 要监控的数据库指标
-metrics:
+aliyun_metrics:
   # 阿里云SDK查询字段
   CpuUsage:
     # prometheus metric name
@@ -80,7 +40,55 @@ metrics:
 
 # aliyun key and secret
 aliyun_settings:
-  key: 
-  secret: 
+  key: xxx
+  secret: xxx
+
+############################ aliyun configuration ############################################
+
+
+############################ tencent configuration ############################################
+# 要监控的数据库实例
+tencent_instances:
+  instance_id:
+    name: instance_name
+    # 连接数利用率因子，连接数利用率*该因子得到一个较为统一的标准，默认1
+    connection_usage_factor: 1
+    # iops利用率因子, 默认 1
+    iops_usage_factor: 1
+    # cpu利用率因子
+    cpu_usage_factor: 1
+    # 内存利用率因子
+    memory_usage_factor: 1
+    # 数据库类型，主库(master)or从库(slave), 默认master
+    type: master
+
+  instance_id2:
+    name: instance_name2
+
+# 要监控的数据库指标
+tencent_metrics:
+  # 阿里云SDK查询字段
+  CPUUseRate:
+    # prometheus metric name
+    name: rds_cpu_usage_ratio
+    # CPU利用率因子，CPU利用率*该因子得到一个较为统一的标准
+    factor: cpu_usage_factor
+
+  MemoryUse:
+    name: rds_memory_usage_ratio
+    factor: memory_usage_factor
+
+  ThreadsConnected:
+    name: rds_connection_usage_ratio
+    factor: connection_usage_factor
 
+  # IOPSUsage:
+  #   name: rds_iops_usage_ratio
+  #   factor: iops_usage_factor
 
+# tencent key and secret
+tencent_settings:
+  secret_id: xxx
+  secret_key: xxx
+  data_interval: 60
+############################ tencent configuration ############################################
\ No newline at end of file

exporter.py

@@ -23,6 +23,7 @@ class TencentExporter(Exporter):
         self.monitor = Monitor()
         self.instances = config['tencent_instances']
         self.metrics = config['tencent_metrics']
+        self.data_interval = config['tencent_settings']['data_interval']
 
     def export_prometheus_data(self):
         data = self._fetch_monitor_data()
@@ -38,7 +39,7 @@ class TencentExporter(Exporter):
             for i in range(0, int(math.ceil(len(instance_id_list)/10.0))):
                 instance_to_get = instance_id_list[10*i : 10*(i+1)]
                 try:
-                    res = self.monitor.get_cdb_metric(instance_to_get, label)
+                    res = self.monitor.get_cdb_metric(instance_to_get, label, self.data_interval)
                     # print(res)
                     data_from_tencent[label].extend(res['DataPoints'])
                 except Exception as e:
@@ -100,7 +101,7 @@ class AliyunExporter(Exporter):
             for i in range(0, int(math.ceil(len(all_rds_id)/10.0))):
                 instance_id_list = all_rds_id[10*i : 10*(i+1)]
                 try:
-                    res = self.monitor.get_rds_metric(instance_id_list, label, duration=30)
+                    res = self.monitor.get_rds_metric(instance_id_list, label)
                     metrics_from_aliyun[label].extend(res['Datapoints'])
                 except Exception as e:
                     print(e)
@@ -136,15 +137,15 @@ class AliyunExporter(Exporter):
                         point['Maximum']/100.0, point['timestamp']/1000)
                 with_factor_metric_family.add_metric(['{}*{}'.format(rds_name,rds_factor), rds_type, service_provider],
                         point['Average']*rds_factor/100.0, point['timestamp']/1000)
-        
+
             metrics_to_export.extend([avg_metric_family, max_metric_family, with_factor_metric_family])
 
         return metrics_to_export
 
 if __name__ == "__main__":
-    tencent_exporter = TencentExporter()
-    data = tencent_exporter.export_prometheus_data()
-    # print(data)
-    # aliyun_exporter = AliyunExporter()
-    # data = aliyun_exporter.export_prometheus_data()
+    # tencent_exporter = TencentExporter()
+    # data = tencent_exporter.export_prometheus_data()
+    # # print(data)
+    aliyun_exporter = AliyunExporter()
+    data = aliyun_exporter._fetch_monitor_data()
     # print(data)

requirements.txt

 aliyun-python-sdk-cms==5.0.0
 aliyun-python-sdk-core==2.9.5
 aliyuncli==2.1.9
+appnope==0.1.0
+arrow==0.13.1
 astroid==1.6.5
 backports.functools-lru-cache==1.5
+backports.shutil-get-terminal-size==1.0.0
 colorama==0.3.3
 configparser==3.5.0
+decorator==4.4.0
 enum34==1.1.6
 futures==3.2.0
+ipython==5.8.0
+ipython-genutils==0.2.0
 isort==4.3.4
 jmespath==0.7.1
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
+pathlib2==2.3.3
+pexpect==4.6.0
+pickleshare==0.7.5
 prometheus-client==0.4.1
+prompt-toolkit==1.0.15
+ptyprocess==0.6.0
 pycryptodome==3.6.6
+Pygments==2.3.1
 pylint==1.9.3
+python-dateutil==2.8.0
 PyYAML==3.13
+scandir==1.10.0
+simplegeneric==0.8.1
 singledispatch==3.4.0.3
 six==1.11.0
+tencentcloud-sdk-python==3.0.60
+traitlets==4.3.2
+wcwidth==0.1.7
 wrapt==1.10.11

utils/tencent.py

 # -*- coding: utf-8 -*-
 import abc
 import json
+import time
 import pprint
 
 import yaml
@@ -12,6 +13,7 @@ from tencentcloud.common.profile.http_profile import HttpProfile
 from tencentcloud.monitor.v20180724 import models, monitor_client
 
 from conf import config
+from utils.time_tools import timestamp2isoformat
 
 class Tencent(object):
     def __init__(self):
@@ -57,7 +59,18 @@ class Monitor(Tencent):
         resp = self.client.GetMonitorData(req)
         return self.json_format(resp.to_json_string())
 
-    def get_cdb_metric(self, instance_id_list, metric, duration=60*6):
+    def get_cdb_metric(self, instance_id_list, metric, duration=60*5):
+        """ 批量获取腾讯云mysql实例监控指标
+        :param instance_id_list: mysql实例列表
+        :type instance_id_list: list
+        :param metric: 指标名称，详见腾讯云文档 https://cloud.tencent.com/document/api/248/30386#2.2-.E6.8C.87.E6.A0.87.E5.90.8D.E7.A7.B0
+        :type metric: string
+        :param duration: 过去多长时间监控数据的时间段时长, 单位(s)
+        :type duration: int
+        """
+        now = int(time.time())
+        end_time_str = timestamp2isoformat(now)
+        start_time_str = timestamp2isoformat(now - duration)
         params_dict = {
             "Namespace": "QCE/CDB",
             "MetricName": metric,
@@ -72,8 +85,8 @@ class Monitor(Tencent):
                     ]
                 } for instance_id in instance_id_list
             ],
-            "StartTime": "2019-04-02T12:30:00+08:00",
-            "EndTime": "2019-04-02T12:35:00+08:00"
+            "StartTime": start_time_str,
+            "EndTime": end_time_str
         }
         resp = self._get_monitor_data(params_dict)
         return resp

utils/time_tools.py

+# coding=utf-8
+
+import arrow
+
+def timestamp2isoformat(timestamp, tz="local"):
+    return arrow.get(timestamp).to(tz).isoformat()
\ No newline at end of file