# -*- coding: utf-8 -*- """ Created on Thu Feb 1 17:16:16 2018 @author: hanye """ from elasticsearch import Elasticsearch import smtplib from email.message import EmailMessage import datetime import sys video_num_criteria={ 'short-video': { 'observed': { 'miaopai': 400000, 'toutiao': 1300000, 'new_tudou': 900000, 'pearvideo': 20000, "haokan": 200000, 'kwai': 3e6, '腾讯视频': 20e4, '央视新闻+': 12000, '新华社': 300, '人民日报': 300, '看了吗': 3e3, '腾讯新闻': 200000, '抖音': 150000, # '微信': 2000, }, 'new_released': { 'miaopai': 12000, 'toutiao': 30000, 'new_tudou': 30000, 'pearvideo': 500, "haokan":10000, 'kwai': 1e5, '腾讯视频': 5e3, '央视新闻+': 400, '新华社': 10, '人民日报': 10, '看了吗': 100, '腾讯新闻': 1.5e3, '抖音': 1000, # '微信': 1000, } }, 'live-video': { 'observed': { '斗鱼直播': 30e4, '花椒': 270e3, '一直播': 60e3, #'映客': 100000, '新华社': 600, '央视新闻+': 300, '人民日报': 90, '头条直播': 3000, '腾讯新闻直播': 900, }, 'new_released': { '斗鱼直播': 4e4, '花椒': 9000, '一直播': 2000, '新华社': 20, '央视新闻+': 10, # 波动从0到20 '人民日报': 1, # 波动从0到10 '头条直播': 100, # 从0到200 '腾讯新闻直播': 30, } } } email_group={ 'short-video': [ 'zhouyujiang@csm.com.cn', 'hanye@csm.com.cn', 'zhangtianli@csm.com.cn', "litao@csm.com.cn", "gengdi@csm.com.cn" ], 'live-video': [ 'zhouyujiang@csm.com.cn', 'hanye@csm.com.cn', 'yangjinshu@csm.com.cn', 'xinyue@csm.com.cn', 'zhangtianli@csm.com.cn', ], } email_msg_suffix=('\n\n\n' +'-'*80+'\n' +'这是自动发送的邮件，可以不用回复。\n' +'This is an automatically sent message. You do NOT need to reply.\n') es=Elasticsearch(hosts='192.168.17.11', port=9200) index_short_video='maintainance-short-video' doc_type_short_video='daily' index_live_video='maintainance-live-video' doc_type_live_video='daily' idx_dict={ 'short-video': { 'index': index_short_video, 'doc_type': doc_type_short_video, 'data_index': 'short-video-production', 'data_doc_type': 'daily-url', }, 'live-video': { 'index': index_live_video, 'doc_type': doc_type_live_video, 'data_index': 'live-video-production', 'data_doc_type': 'daily-url', } } today=datetime.datetime.now()-datetime.timedelta(days=2) ## test #today=datetime.datetime(2018,1,12) path='/home/hanye/project_data/Python/Projects/proj-short-videos/maintainance/log/' log_fn='email_alert_log_%s' % datetime.datetime.strftime(today, '%b-%Y') f_log=open(path+log_fn, 'a') ## test #import sys #f_log=sys.stdout print('*'*80, file=f_log) print('log timestamp ', datetime.datetime.now(), file=f_log) print('Checking video number for fetch_date', today.isoformat()[:10], file=f_log) fetch_year=today.year fetch_month=today.month fetch_day=today.day alert_msg={} for idx in video_num_criteria: alert_msg[idx]={} for stats_type in ['observed', 'new_released']: alert_msg[idx][stats_type]=[] for platform in video_num_criteria[idx][stats_type]: search_body={ "query": { "bool": { "filter": [ {"term": {"fetch_year": fetch_year}}, {"term": {"fetch_month": fetch_month}}, {"term": {"fetch_day": fetch_day}}, {"term": {"platform.keyword": platform}}, {"term": {"stats_type.keyword": stats_type}}, ] } } } search_resp=es.search(index=idx_dict[idx]['index'], doc_type=idx_dict[idx]['doc_type'], body=search_body, request_timeout=100) if search_resp['hits']['total']>0: video_num=search_resp['hits']['hits'][0]['_source']['video_num'] fetch_date=search_resp['hits']['hits'][0]['_source']['fetch_date'] fetch_date_str=datetime.datetime.fromtimestamp(fetch_date/1e3).isoformat()[:10] chk_source=idx_dict[idx]['index'] else: # if no hits in maintainance index, search directly in data index fetch_time_start_T=datetime.datetime(today.year, today.month, today.day) fetch_time_start=int(fetch_time_start_T.timestamp()*1e3) fetch_time_end=int((fetch_time_start_T+datetime.timedelta(days=1)).timestamp()*1e3) if stats_type=='observed': search_data_idx={ "query": { "bool": { "filter": [ {"term": {"platform.keyword": platform}}, {"range": {"fetch_time": {"gte": fetch_time_start, "lt": fetch_time_end}}} ] } }, "size": 0, } elif stats_type=='new_released': search_data_idx={ "query": { "bool": { "filter": [ {"term": {"platform.keyword": platform}}, {"range": {"release_time": {"gte": fetch_time_start, "lt": fetch_time_end}}}, ] } }, "size": 0, } else: print('Error value with stats_type: %s, system exit' % stats_type, file=f_log) sys.exit(1) search_data_idx_resp=es.search(index=idx_dict[idx]['data_index'], doc_type=idx_dict[idx]['data_doc_type'], body=search_data_idx, request_timeout=100) video_num=search_data_idx_resp['hits']['total'] fetch_date_str=today.isoformat()[:10] chk_source=idx_dict[idx]['data_index'] if video_num0: email_subj='[数据日常运维预警] 数据条目预警 %s %s' % (idx, today.isoformat()[:10]) email_msg_body='' for stats_type in alert_msg[idx]: email_msg_body+=('%s %s 预警：\n' % (idx, stats_type_dict[stats_type])) for ml in alert_msg[idx][stats_type]: msgline='平台 %s 抓取 %s 视频条目数：%d，低于预警值(%d) %.2f%% \n' % (ml['platform'], ml['fetch_date'], ml['video_num'], ml['alert_criteria'], ml['short_perct']) email_msg_body+=msgline if len(alert_msg[idx][stats_type])>0: email_msg_body+='\nchecking data source index name: %s\n\n\n' % alert_msg[idx][stats_type][0]['chk_source'] email_msg_body+=email_msg_suffix print('email_msg_body:\n', email_msg_body, file=f_log) email_msg=EmailMessage() email_msg.set_content(email_msg_body) email_msg['Subject']=email_subj email_msg['From']=sender email_msg['to']=email_group[idx] try: server=smtplib.SMTP(host=csm_mail_service) server.send_message(email_msg) server.quit() print('Successfully sent email to %s for %s' % (email_group[idx], idx), datetime.datetime.now(), file=f_log) print('email_msg:\n', email_msg, file=f_log) except: print('Failed to connect email server.', datetime.datetime.now(), file=f_log) else: print('All platforms is ok for %s on fetch_date %s, %s' % (idx, today.isoformat()[:10], datetime.datetime.now()), file=f_log) print('Alert criteria:\n', video_num_criteria[idx], file=f_log) print('\n\n', file=f_log) f_log.close()