监控

eda/smart_rank/spark_streaming_monitor.py

 import time
 import requests
-from datetime import date
-
+import datetime
+from tool import send_email
 
 while True:
     result = requests.get('http://192.144.193.19:4042/api/v1/applications/app-20191112140630-0664/streaming/batches')
 
     batchs_info = result.json()
 
-    # batchTime = '2019-11-12T07:53:44.000GMT'
-
     log_path = "/Users/apple/Desktop/spark_streaming_log/"
 
-    today = date.today().strftime('%Y%m%d')
+    today = datetime.date.today().strftime('%Y%m%d')
+
     with open(log_path+"user_portrait_tag_name_increment_update_" + today + ".log", "a") as f:
+        is_email = 0
         for batch in batchs_info:
+            if is_email == 0 and 'totalDelay' in batch and batch['totalDelay'] > 10000:
+                batch_time = batch['totalDelay']
+                GMT_FORMAT = '%Y-%m-%dT%H:%M:%S.000GMT'
+                batchTime_beijing = datetime.datetime.strptime(batchTime, GMT_FORMAT) + datetime.timedelta(hours=8)
+                send_email("画像增量更新堆积时间", str(batchTime_beijing), str(batch['totalDelay']))
+                is_email = 1
             f.write('%s\n' % batch)
     time.sleep(1800)