re construct

aliyun.py

@@ -3,15 +3,17 @@ import time
 import pprint
 import json
 
+import yaml
 from aliyunsdkcore import client
 from aliyunsdkcms.request.v20170301 import QueryMetricListRequest, QueryMetricLastRequest
 
-import settings
+with open('config.yaml', 'r') as f:
+    config = yaml.load(f)
 
 class ALiYun(object):
     def __init__(self):
-        self.key_id = settings.ALIYUN_KEY
-        self.key_secret = settings.ALIYUN_KEY_SECRET
+        self.key_id = config['aliyun_settings']['key']
+        self.key_secret = config['aliyun_settings']['secret']
 
     def get_client(self, region):
         clt = client.AcsClient(self.key_id, self.key_secret, region)
@@ -49,13 +51,3 @@ class CMS(ALiYun):
         # pprint.pprint(self.json_format(result))
         return self.json_format(result)
 
-
-def main():
-    aliyun_cms = CMS()
-    rds_instance_list = settings.RDS_INSTANCES
-    for rds_instance in rds_instance_list.keys():
-        aliyun_cms.get_rds_metric(rds_instance, 'CpuUsage')
-        aliyun_cms.get_rds_metric(rds_instance, 'DiskUsage')
-
-if __name__ == '__main__':
-    main()

config_local.yaml

+# 要监控的数据库实例
+rds_instances:
+  rdsfewzdmf0jfjp9un8xj:
+    name: prod.zhengxing
+    # 连接数利用率因子，连接数利用率*该因子得到一个较为统一的标准，默认1
+    connection_usage_factor: 1
+    # iops利用率因子, 默认 1
+    iops_usage_factor: 1
+    # cpu利用率因子
+    cpu_usage_factor: 1
+    # 内存利用率因子
+    memory_usage_factor: 1
+    # 数据库类型，主库or从库, 默认master
+    type: master
+
+  rdsg2mpi623x6wioo796k:
+    name: prod.readonly2.zhengxing
+    type: slave
+
+  rm-m5e63m5z6o4re1p49:
+    name: mars.prod
+
+  rm-m5e842126ng59jrv6:
+    name: data.prod
+
+  rm-m5eu1wmu0zt6j5n72:
+    name: sms.prod
+
+  rr-m5e2vy0ke9q35kst5:
+    name: prod.readonly.zhengxing
+    type: slave
+
+  rm-m5e90y907sw3bnz88:
+    name: common.prod
+
+  rr-m5e2n024w0z293r5f:
+    name: common.readonly
+    type: slave
+
+  rr-m5efx2m67mi002jq9:
+    name: prod.readonly3.zhengxing
+    type: slave
+
+  rm-m5emg41za2w7l6au3:
+    name: mimas_prod
+
+  rm-m5ey2s823bq0lc616:
+    name: bran_prod
+
+  rr-m5et21lafq1677pid:
+    name: mimas_readonly
+    type: slave
+
+  rr-m5e3036a5l18ym4t6:
+    name: mimas_readonly01
+    type: slave
+
+# 要监控的数据库指标
+metrics:
+  # 阿里云SDK查询字段
+  CpuUsage:
+    # prometheus metric name
+    name: rds_cpu_usage_ratio
+    # CPU利用率因子，CPU利用率*该因子得到一个较为统一的标准
+    factor: cpu_usage_factor
+
+  MemoryUsage:
+    name: rds_memory_usage_ratio
+    factor: memory_usage_factor
+
+  ConnectionUsage:
+    name: rds_connection_usage_ratio
+    factor: connection_usage_factor
+
+  IOPSUsage:
+    name: rds_iops_usage_ratio
+    factor: iops_usage_factor
+
+# aliyun key and secret
+aliyun_settings:
+  key: LTAIfYsppxRX2Q5m
+  secret: lz7o7pf24zFqQ1JDesHYDTJ99wS058
+
+

main.py

@@ -4,30 +4,37 @@ import json
 import random
 import math
 
+import yaml
 from prometheus_client import start_http_server
 from prometheus_client.core import GaugeMetricFamily, REGISTRY
 
-import settings
 from aliyun import CMS
 
 aliyun_cms = CMS()
-rds_instances = settings.RDS_INSTANCES
+with open('config.yaml', 'r') as f:
+    config = yaml.load(f)
+
+# rds_instances = settings.RDS_INSTANCES
+rds_instances = config['rds_instances']
+queryname_metricname = config['metrics']
 
 class RDSCollector(object):
     def collect(self):
         metrics = []
         # 阿里云SDK查询字段与prometheus metric name的映射
-        queryname_metricname = {
-            'CpuUsage': 'rds_cpu_usage_ratio',
-            # 'DiskUsage': 'rds_disk_usage_ratio',
-            'MemoryUsage': 'rds_memory_usage_radio',
-            'ConnectionUsage': 'rds_connection_usage_radio',
-            'IOPSUsage': 'rds_iops_usage_radio',
-        }
+        # queryname_metricname = {
+        #     'CpuUsage': 'rds_cpu_usage_ratio',
+        #     # 'DiskUsage': 'rds_disk_usage_ratio',
+        #     'MemoryUsage': 'rds_memory_usage_radio',
+        #     'ConnectionUsage': 'rds_connection_usage_radio',
+        #     'IOPSUsage': 'rds_iops_usage_radio',
+        # }
         metrics_from_aliyun = {}
-        all_rds_id = list(rds_instances.keys())
+        # all_rds_id = list(rds_instances.keys())
+        all_rds_id = rds_instances.keys()
 
         # get metrics from aliyun
+        print('get from aliyun')
         for label in queryname_metricname.keys():
             metrics_from_aliyun[label] = []
 
@@ -41,17 +48,34 @@ class RDSCollector(object):
 
         # transform to prometheus metric format
         for queryname, datapoints in metrics_from_aliyun.items():
-            metricname = queryname_metricname[queryname]
-            avg_metric_family = GaugeMetricFamily(metricname, metricname, labels=['rds_name'])
+            metric = queryname_metricname[queryname]
+            metricname = metric['name']
+            factor = metric.get('factor', None)
+            avg_metric_family = GaugeMetricFamily(metricname, metricname,
+                    labels=['rds_name', 'type'])
             max_metricname = '{}_max'.format(metricname)
-            max_metric_family = GaugeMetricFamily(max_metricname, max_metricname, labels=['rds_name'])
+            max_metric_family = GaugeMetricFamily(max_metricname,
+                    max_metricname, labels=['rds_name', 'type'])
+            with_factor_metricname = '{}_with_factor'.format(metricname)
+            with_factor_metric_family = GaugeMetricFamily(with_factor_metricname, 
+                    with_factor_metricname, labels=['rds_name', 'type'])
 
             for point in datapoints:
-                rds_name = rds_instances[point['instanceId']]
-                avg_metric_family.add_metric([rds_name], point['Average']/100.0, point['timestamp']/1000)
-                max_metric_family.add_metric([rds_name], point['Maximum']/100.0, point['timestamp']/1000)
+                # rds_name = rds_instances[point['instanceId']]
+                rds = rds_instances[point['instanceId']]
+                rds_name = rds['name']
+                rds_type = rds.get('type', 'master')
+                # 当前queryname，当前rds实例的因子，若没有配置，默认为1
+                rds_factor = rds.get(factor, 1)
+                avg_metric_family.add_metric([rds_name, rds_type],
+                        point['Average']/100.0, point['timestamp']/1000)
+                max_metric_family.add_metric([rds_name, rds_type],
+                        point['Maximum']/100.0, point['timestamp']/1000)
+                with_factor_metric_family.add_metric([rds_name, rds_type],
+                        point['Average']*rds_factor/100.0, point['timestamp']/1000)
 
-            metrics.extend([avg_metric_family, max_metric_family])
+            metrics.extend([avg_metric_family, max_metric_family,
+                with_factor_metric_family])
 
         for m in metrics:
             yield m

settings.py

-# -*- coding:utf-8 -*-
-
-ALIYUN_KEY = ''
-ALIYUN_KEY_SECRET = ''
-
-# 要监控的rds的实例ID与实例描述的映射
-RDS_INSTANCES = {
-    
-}