add multi platform support

conf.py

+# coding=utf-8
+import yaml
+
+with open('config.yaml', 'r') as f:
+    config = yaml.load(f)

exporter.py

+# coding=utf-8
+import abc
+import math
+import pprint
+
+from prometheus_client.core import GaugeMetricFamily
+
+from conf import config
+from utils.tencent import Monitor
+from utils.aliyun import CMS
+
+
+class Exporter(object):
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def export_prometheus_data(self):
+        pass
+
+
+class TencentExporter(Exporter):
+    def __init__(self):
+        self.monitor = Monitor()
+        self.instances = config['tencent_instances']
+        self.metrics = config['tencent_metrics']
+
+    def export_prometheus_data(self):
+        data = self._fetch_monitor_data()
+        return self._transfer_to_prometheus_format(data)
+
+    def _fetch_monitor_data(self):
+        data_from_tencent = {}
+        instance_id_list = self.instances.keys()
+        # 遍历所有指标
+        for label in self.metrics.keys():
+            data_from_tencent[label] = []
+            # 一次批量取 10 个实例的监控数据
+            for i in range(0, int(math.ceil(len(instance_id_list)/10.0))):
+                instance_to_get = instance_id_list[10*i : 10*(i+1)]
+                try:
+                    res = self.monitor.get_cdb_metric(instance_to_get, label)
+                    # print(res)
+                    data_from_tencent[label].extend(res['DataPoints'])
+                except Exception as e:
+                    print(e)
+        return data_from_tencent
+
+    def _transfer_to_prometheus_format(self, data):
+        # print(data)
+        metrics_to_export = []
+        service_provider = "tencent"
+
+        for label, instance_data_list in data.items():
+            metric = self.metrics[label]
+            metricname = metric['name']
+            factor = metric.get('factor', None)
+            avg_metric_family = GaugeMetricFamily(metricname, metricname, labels=[
+                'rds_name', 'type', 'service_provider'
+            ])
+            with_factor_metricname = '{}_with_factor'.format(metricname)
+            with_factor_metric_family = GaugeMetricFamily(with_factor_metricname, with_factor_metricname, labels=[
+                'rds_name', 'type', 'service_provider'
+            ])
+
+            for instance_data in instance_data_list:
+                timestamp_value_dict = dict(zip(instance_data['Timestamps'], instance_data['Values']))
+                instance_id = instance_data['Dimensions'][0]['Value']
+                instance = self.instances[instance_id]
+                instance_name = instance['name']
+                instance_type = instance.get('type', 'master')
+                # 当前queryname，当前rds实例的因子，若没有配置，默认为1
+                rds_factor = instance.get(factor, 1)
+                for timestamp, value in timestamp_value_dict.items():
+                    avg_metric_family.add_metric([instance_name, instance_type, service_provider], value, timestamp)
+                    with_factor_metric_family.add_metric(
+                        ['{}*{}'.format(instance_name,rds_factor), instance_type, service_provider], 
+                        value*rds_factor, timestamp)
+            metrics_to_export.extend([avg_metric_family, with_factor_metricname])
+
+        return metrics_to_export
+
+
+class AliyunExporter(Exporter):
+    def __init__(self):
+        self.monitor = CMS()
+        self.instances = config['aliyun_instances']
+        self.metrics = config['aliyun_metrics']
+
+    def export_prometheus_data(self):
+        data = self._fetch_monitor_data()
+        return self._transfer_to_prometheus_format(data)
+
+    def _fetch_monitor_data(self):
+        metrics_from_aliyun = {}
+        all_rds_id = self.instances.keys()
+
+        for label in self.metrics.keys():
+            metrics_from_aliyun[label] = []
+
+            for i in range(0, int(math.ceil(len(all_rds_id)/10.0))):
+                instance_id_list = all_rds_id[10*i : 10*(i+1)]
+                try:
+                    res = self.monitor.get_rds_metric(instance_id_list, label, duration=30)
+                    metrics_from_aliyun[label].extend(res['Datapoints'])
+                except Exception as e:
+                    print(e)
+
+        return metrics_from_aliyun
+
+    def _transfer_to_prometheus_format(self, data):
+        metrics_to_export = []
+        service_provider = "aliyun"
+
+        for queryname, datapoints in data.items():
+            metric = self.metrics[queryname]
+            metricname = metric['name']
+            factor = metric.get('factor', None)
+            avg_metric_family = GaugeMetricFamily(metricname, metricname,
+                    labels=['rds_name', 'type', 'service_provider'])
+            max_metricname = '{}_max'.format(metricname)
+            max_metric_family = GaugeMetricFamily(max_metricname,
+                    max_metricname, labels=['rds_name', 'type', 'service_provider'])
+            with_factor_metricname = '{}_with_factor'.format(metricname)
+            with_factor_metric_family = GaugeMetricFamily(with_factor_metricname, 
+                    with_factor_metricname, labels=['rds_name', 'type', 'service_provider'])
+
+            for point in datapoints:
+                rds = self.instances[point['instanceId']]
+                rds_name = rds['name']
+                rds_type = rds.get('type', 'master')
+                # 当前queryname，当前rds实例的因子，若没有配置，默认为1
+                rds_factor = rds.get(factor, 1)
+                avg_metric_family.add_metric([rds_name, rds_type, service_provider],
+                        point['Average']/100.0, point['timestamp']/1000)
+                max_metric_family.add_metric([rds_name, rds_type, service_provider],
+                        point['Maximum']/100.0, point['timestamp']/1000)
+                with_factor_metric_family.add_metric(['{}*{}'.format(rds_name,rds_factor), rds_type, service_provider],
+                        point['Average']*rds_factor/100.0, point['timestamp']/1000)
+        
+            metrics_to_export.extend([avg_metric_family, max_metric_family, with_factor_metric_family])
+
+        print(metrics_to_export)
+        return metrics_to_export
+
+if __name__ == "__main__":
+    tencent_exporter = TencentExporter()
+    data = tencent_exporter.export_prometheus_data()
+    # print(data)
+    # aliyun_exporter = AliyunExporter()
+    # data = aliyun_exporter.export_prometheus_data()
+    # print(data)

main2.py

+# -*- coding:utf-8 -*-
+import time
+import json
+import random
+import math
+
+import yaml
+from prometheus_client import start_http_server
+from prometheus_client.core import REGISTRY
+
+from exporter import TencentExporter, AliyunExporter
+
+class RDSCollector(object):
+    def __init__(self):
+        self.exporter_list = [
+            # TencentExporter(),
+            AliyunExporter(),
+        ]
+
+    def collect(self):
+        metrics = []
+        for exporter in self.exporter_list:
+            metrics.extend(exporter.export_prometheus_data())
+        for m in metrics:
+            yield m
+
+def main():
+    REGISTRY.register(RDSCollector())
+    start_http_server(7086)
+    while True:
+        time.sleep(1)
+
+if __name__ == '__main__':
+    main()

utils/tencent.py

+# -*- coding: utf-8 -*-
+import abc
+import json
+import pprint
+
+import yaml
+from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import \
+    TencentCloudSDKException
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+from tencentcloud.monitor.v20180724 import models, monitor_client
+
+from conf import config
+
+class Tencent(object):
+    def __init__(self):
+        self.secret_id = config['tencent_settings']['secret_id']
+        self.secret_key = config['tencent_settings']['secret_key']
+        self.cred = credential.Credential(self.secret_id, self.secret_key)
+
+    @abc.abstractmethod
+    def _get_client(self):
+        """ get client
+        """
+
+    def json_format(self, response):
+        if isinstance(response, bytes):
+            response = response.decode()
+        return json.loads(response)
+
+
+class Monitor(Tencent):
+    def __init__(self, region="ap-beijing", endpoint="monitor.tencentcloudapi.com"):
+        super(Monitor, self).__init__()
+        self.client = self._get_client(region, endpoint)
+
+    def _get_client(self, region, endpoint):
+        httpProfile = HttpProfile()
+        httpProfile.endpoint = endpoint
+        clientProfile = ClientProfile()
+        clientProfile.httpProfile = httpProfile
+        client = monitor_client.MonitorClient(self.cred, region, clientProfile)
+        return client
+
+    def _describe_base_metrics(self, params_dict):
+        assert isinstance(params_dict, dict)
+        req = models.DescribeBaseMetricsRequest()
+        req.from_json_string(json.dumps(params_dict))
+        resp = self.client.DescribeBaseMetrics(req)
+        return self.json_format(resp.to_json_string())
+
+    def _get_monitor_data(self, params_dict):
+        assert isinstance(params_dict, dict)
+        req = models.GetMonitorDataRequest()
+        req.from_json_string(json.dumps(params_dict))
+        resp = self.client.GetMonitorData(req)
+        return self.json_format(resp.to_json_string())
+
+    def get_cdb_metric(self, instance_id_list, metric, duration=60*6):
+        params_dict = {
+            "Namespace": "QCE/CDB",
+            "MetricName": metric,
+            "Period": 5,
+            "Instances": [
+                {
+                    "Dimensions": [
+                        {
+                            "Name": "InstanceId",
+                            "Value": instance_id
+                        }
+                    ]
+                } for instance_id in instance_id_list
+            ],
+            "StartTime": "2019-04-02T12:30:00+08:00",
+            "EndTime": "2019-04-02T12:35:00+08:00"
+        }
+        # print(params_dict)
+        resp = self._get_monitor_data(params_dict)
+        return resp
+
+# try:
+#     cred = credential.Credential(
+#         "AKIDYdekdHJkhP2Qs5w8t89jKbhiJg4ELvDO", "OZGtpiP1NZ100y66sEE0ZtyUlmQo57yd")
+#     httpProfile = HttpProfile()
+#     httpProfile.endpoint = "monitor.tencentcloudapi.com"
+
+#     clientProfile = ClientProfile()
+#     clientProfile.httpProfile = httpProfile
+#     client = monitor_client.MonitorClient(cred, "ap-beijing", clientProfile)
+
+#     req = models.GetMonitorDataRequest()
+#     params_dict = {
+#         "Namespace": "QCE/CDB",
+#         "MetricName": "CPUUseRate",
+#         "Period": 5,
+#         "Instances": [
+#             {
+#                 "Dimensions": [
+#                     {
+#                         "Name": "InstanceId",
+#                         "Value": "cdb-iaq6uk88"
+#                     }
+#                 ]
+#             }
+#         ],
+#         "StartTime": "2019-03-28T18:30:00+08:00",
+#         "EndTime": "2019-03-28T18:31:00+08:00"
+#     }
+
+#     req.from_json_string(json.dumps(params_dict))
+
+#     resp = client.GetMonitorData(req)
+#     pprint.pprint(json.loads(resp.to_json_string()))
+
+# except TencentCloudSDKException as err:
+#     print(err)