1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
import time
import requests
import datetime
import webbrowser
import sys
from tool import send_email
while True:
try:
result = requests.get('http://192.144.193.19:4042/api/v1/applications/app-20191118191719-0747/streaming/batches')
batchs_info = result.json()
log_path = "/Users/apple/Desktop/spark_streaming_log/"
today = datetime.date.today().strftime('%Y%m%d')
with open(log_path+"user_portrait_tag_name_increment_update_" + today + ".log", "a") as f:
is_email = 0
for batch in batchs_info:
if is_email == 0 and 'totalDelay' in batch and batch['totalDelay'] > 2000:
batch_time = batch['batchTime']
GMT_FORMAT = '%Y-%m-%dT%H:%M:%S.000GMT'
batch_time_beijing = datetime.datetime.strptime(batch_time, GMT_FORMAT) + datetime.timedelta(hours=8)
send_email("画像增量更新堆积时间", "堆积时刻：" + str(batch_time_beijing) + "堆积时长:" + str(batch['totalDelay']), "")
is_email = 1
f.write('%s\n' % batch)
time.sleep(1800)
except:
send_email("画像增量更新堆积时间", "fail", "")
break
result = {}
GMT_FORMAT = '%Y-%m-%dT%H:%M:%S.000GMT'
log_date = sys.argv[1]
with open("/Users/apple/Desktop/spark_streaming_log/user_portrait_tag_name_increment_update_" + log_date + ".log", "r") as f:
lines = f.readlines()
lines = list(set(lines)) # 去重
for line in lines:
line = eval(line)
if 'processingTime' in line:
batchTime = line['batchTime']
batchTime_beijing = datetime.datetime.strptime(batchTime,GMT_FORMAT)+datetime.timedelta(hours=8)
result.update({batchTime_beijing: line['processingTime']})
from pyecharts import Bar
bar = Bar("画像增量更新耗时监控")
bar.add("耗时(ms)", list(result.keys()), list(result.values()), mark_point=['max','min'], mark_line=['average'],
is_more_utils=True, is_datazoom_show=True,)
bar.render()
webbrowser.open("file:///Users/apple/Desktop/spark_streaming_log/render.html")