update

celery_stats/celery_stats_monitor.py

@@ -3,7 +3,7 @@
 # @File  : celery_stats_monitor.py
 # @email : litao@igengmei.com
 # @author : litao
-
+# 索引同步redis队列监测
 import redis
 from send_msg_to_dingding.send_msg import send_msg_to_dingtalk
 import datetime, time

celery_stats/es_status.py

@@ -3,7 +3,7 @@
 # @File  : es_status.py
 # @email : litao@igengmei.com
 # @author : litao
-
+# es状态监测
 # from elasticsearch_7 import Elasticsearch
 
 # es = Elasticsearch([
@@ -74,8 +74,12 @@ def get_node_status(ip):
         cpu_15m = data["os"]["cpu"]["load_average"]['15m']
         young_gc = data["jvm"]["gc"]["collectors"]['young']['collection_count']
         young_gc_ms = data["jvm"]["gc"]["collectors"]['young']['collection_time_in_millis']
+        old_gc = data["jvm"]["gc"]["collectors"]['young']['collection_count']
+        old_gc_ms = data["jvm"]["gc"]["collectors"]['young']['collection_time_in_millis']
         young_gc_per_ms = int(young_gc_ms/young_gc)
-        yield node_name, ip, query_time, query_current, fetch_current, cpu_percent, cpu_1m, cpu_5m, cpu_15m, young_gc_per_ms
+        old_gc_ms = int(old_gc_ms/old_gc)
+        yield node_name, ip, query_time, query_current, fetch_current, cpu_percent, cpu_1m, cpu_5m, cpu_15m,\
+              young_gc_per_ms, old_gc, old_gc_ms
 
 
 def parse_cluster_stats(data):
@@ -90,7 +94,14 @@ def parse_cluster_stats(data):
 
 def parse_node_stats(data):
     res_str = ""
-    node_name, ip, query_time, query_current, fetch_current, cpu_percent, cpu_1m, cpu_5m, cpu_15m, young_gc = data
+    node_name, ip, query_time, query_current, fetch_current, cpu_percent, cpu_1m, cpu_5m, cpu_15m, young_gc, \
+    old_gc, old_gc_ms = data
+    old_gc_last = data_dict.get('old_gc',old_gc)
+    old_gc_ms_last = data_dict.get('old_gc_ms',old_gc_ms)
+    if old_gc - old_gc_last > 0:
+        res_str += "{name}节点 old_gc增长中 old_gc为{old_gc} 平均{old_gc_ms}ms\n".format(name=node_name, old_gc=str(old_gc),
+                                                                               old_gc_ms=str(old_gc_ms))
+
     if cpu_1m >= cpu_top or cpu_5m >= cpu_top or cpu_15m >= cpu_top:
         res_str += "{name}节点CPU使用率预警 1m {cpu_1m} 5m {cpu_5m} 15m {cpu_15m} \n" \
                    "目前 query_current为{query_current} fetch_current为{fetch_current} young_gc为{young_gc}ms".format(cpu_1m=str(int(cpu_1m*100))+"%",name=node_name,