spark streaming 监控

eda/smart_rank/spark_streaming_monitor.py

 import time
 import requests
 import datetime
+import webbrowser
+import sys
 from tool import send_email
 
 while True:
@@ -28,3 +30,25 @@ while True:
         send_email("画像增量更新堆积时间", "fail", "")
         break
 
+
+result = {}
+GMT_FORMAT = '%Y-%m-%dT%H:%M:%S.000GMT'
+log_date = sys.argv[1]
+with open("/Users/apple/Desktop/spark_streaming_log/user_portrait_tag_name_increment_update_" + log_date + ".log", "r") as f:
+    lines = f.readlines()
+    lines = list(set(lines)) # 去重
+    for line in lines:
+        line = eval(line)
+        if 'processingTime' in line:
+            batchTime = line['batchTime']
+            batchTime_beijing = datetime.datetime.strptime(batchTime,GMT_FORMAT)+datetime.timedelta(hours=8)
+            result.update({batchTime_beijing: line['processingTime']})
+
+
+from pyecharts import Bar
+bar = Bar("画像增量更新耗时监控")
+bar.add("耗时(ms)", list(result.keys()), list(result.values()), mark_point=['max','min'], mark_line=['average'],
+        is_more_utils=True, is_datazoom_show=True,)
+bar.render()
+
+webbrowser.open("file:///Users/apple/Desktop/spark_streaming_log/render.html")