# -*- coding:utf-8 -*-
# @Time : 2019/5/7 10:42 
# @Author : litao
# encoding: utf-8

import sys, os, json, time
import datetime, argparse
import elasticsearch
from elasticsearch.helpers import scan
import pandas as pd
import func_send_email_with_file
try:
    from write_data_into_es.func_get_releaser_id import *
except:
    from write_data_into_es_new.func_get_releaser_id import *

done = False
hosts = '192.168.17.11'
port = 80
user = 'litao'
passwd = 'lQSmSEnGZZxl'
http_auth = (user, passwd)
es = elasticsearch.Elasticsearch(hosts=hosts, port=port, http_auth=http_auth)

parser = argparse.ArgumentParser(description='Specify a platform name.')
parser.add_argument('-c', '--cycle', default="week", type=str,
                    help=('Please input a cycle type.such as week/month/quarter'))
parser.add_argument('-n', '--cycle_num', default=None, type=int,
                    help=('cycle_num. Such as 1/2/3/4/5'))
parser.add_argument('-y', '--year', default=None, type=int,
                    help=('Specify output file path, default None.'))
parser.add_argument('-t', '--compare_type', default="new_released", type=str,
                    help=('Compare with_new released or daily like -t daily'))
parser.add_argument('-m', '--mail', default=None, type=str,
                    help=('send to all or just us. like -m all'))
args = parser.parse_args()
csv_type = {"SMG":[],"an_hui":[],"ronghe":[],"BTV":[],"su_zhou":[]}

def week_num(year=None, cycle=None, cycle_num=None, compare_type=None):
    now = datetime.datetime.now()
    now_canlendar = now.isocalendar()
    if not cycle_num:
        week_canlendar = now_canlendar
    else:
        week_canlendar = (now.year, cycle_num + 1, 0)
    year = week_canlendar[0]
    this_week = week_canlendar[1] - 1
    if this_week == 0:
        last_year = year - 1
        this_week = 1
    else:
        last_year = year
    if this_week == 1:
        last_week = "52"
    else:
        last_week = this_week - 1
    today = datetime.datetime(datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day)
    # today = datetime.datetime(year=2018, month=12, day=25)
    first_day_in_week = today - datetime.timedelta(
        days=now_canlendar[2] + 7 * (now_canlendar[1] - week_canlendar[1] + 1))
    fisrt_day_ts = int(first_day_in_week.timestamp() * 1e3)
    last_day_in_week = first_day_in_week + datetime.timedelta(days=7)
    last_day_ts = int(last_day_in_week.timestamp() * 1e3)

    this_week_index = 'short-video-weekly'
    this_week_doc = 'daily-url-' + str(year) + '_w' + format(this_week, '>02d') + '_s1'
    last_week_index = 'releaser-weekly-short-video'
    last_week_doc = 'doc'
    if compare_type == "new_released":
        this_week_index = last_week_index
        this_week_doc = last_week_doc
    return this_week_index, this_week_doc, last_week_index, last_week_doc, fisrt_day_ts, last_day_ts, this_week, last_week, last_year


def month_num(year=None, cycle=None, cycle_num=None, compare_type=None):
    now = datetime.datetime.now()
    if not year:
        year = now.year
    if not cycle_num:
        this_mon = now.month - 1 if now.month != 1 else 12
        last_mon = this_mon - 1 if this_mon > 1 else this_mon - 1 + 12
        if this_mon == 12:
            last_year = year - 1
        else:
            last_year = year
    else:
        this_mon = cycle_num
        last_mon = cycle_num - 1 if this_mon > 1 else this_mon - 1 + 12
        if this_mon == 1:
            last_year = year - 1
        else:
            last_year = year
    first_day_ts = int(datetime.datetime(year=last_year, month=this_mon, day=1).timestamp() * 1e3)
    if this_mon == 12:
        next_year = last_year + 1
        next_month = 1
    else:
        next_year = year
        next_month = this_mon + 1
    last_day_ts = int(datetime.datetime(year=next_year, month=next_month, day=1).timestamp() * 1e3)
    this_mon_index = "short-video-production-%s" % last_year
    this_mon_doc = "daily-url-%s" % (
            datetime.datetime(year=next_year, month=next_month, day=1) + datetime.timedelta(days=-1)).strftime(
            "%Y-%m-%d")
    last_mon_index = "releaser"
    last_mon_doc = "releasers"
    if compare_type == "new_released":
        this_mon_index = last_mon_index
        this_mon_doc = last_mon_doc
    return this_mon_index, this_mon_doc, last_mon_index, last_mon_doc, first_day_ts, last_day_ts, this_mon, last_mon, last_year


def quarter_num(year=None, cycle=None, cycle_num=None, compare_type=None):
    now = datetime.datetime.now()
    if not cycle_num:
        this_quarter = int(now.month / 3) + 1
    else:
        this_quarter = cycle_num
    last_quarter = this_quarter - 1 if cycle_num > 1 else 4
    if last_quarter == 4:
        last_year = year - 1
    else:
        last_year = year
    first_day_ts = int(datetime.datetime(year=year, month=(this_quarter - 1) * 3 + 1, day=1).timestamp() * 1e3)
    last_day_ts = int(datetime.datetime(year=year, month=this_quarter * 3 + 1, day=1).timestamp() * 1e3)
    this_quarter_index = "short-video-quarter-%s" % year
    this_quarter_doc = "daily-url-2019-Q%s" % this_quarter
    last_quarter_index = "releaser"
    last_quarter_doc = "releasers-%s-Q%s" % (last_year, last_quarter)
    if compare_type == "new_released":
        this_quarter_index = last_quarter_index
        this_quarter_doc = last_quarter_doc
    return this_quarter_index, this_quarter_doc, last_quarter_index, last_quarter_doc, first_day_ts, last_day_ts, this_quarter, last_quarter, last_year


def create_body(platform, releaser, releaserUrl, year=None, cycle="week", cycle_num=None, compare_type=None,project_type=None, **kwargs):
    global this_cycle_index, this_cycle_doc, last_cycle_index, last_cycle_doc, fisrt_day_ts, last_day_ts, this_cycle, last_cycle, last_year
    this_cycle_index = kwargs.get("this_cycle_index")
    this_cycle_doc = kwargs.get("this_cycle_doc")
    last_cycle_index = kwargs.get("last_cycle_index")
    last_cycle_doc = kwargs.get("last_cycle_doc")
    fisrt_day_ts = kwargs.get("fisrt_day_ts")
    last_day_ts = kwargs.get("last_day_ts")
    releaser_id_str = kwargs.get("releaser_id_str")
    if not releaser_id_str:
        releaser_id = get_releaser_id(platform=platform, releaserUrl=releaserUrl)
        if releaser_id:
            releaser_id_str = platform + '_' + releaser_id
        else:
            releaser_id_str = ""
    one_line = {}
    if not this_cycle_index:
        if cycle == "week":
            this_cycle_index, this_cycle_doc, last_cycle_index, last_cycle_doc, fisrt_day_ts, last_day_ts, this_cycle, last_cycle, last_year = week_num(
                year, cycle, cycle_num, compare_type)
        elif cycle == "month":
            this_cycle_index, this_cycle_doc, last_cycle_index, last_cycle_doc, fisrt_day_ts, last_day_ts, this_cycle, last_cycle, last_year = month_num(
                year, cycle, cycle_num, compare_type)
        elif cycle == "quarter":
            this_cycle_index, this_cycle_doc, last_cycle_index, last_cycle_doc, fisrt_day_ts, last_day_ts, this_cycle, last_cycle, last_year = quarter_num(
                year, cycle, cycle_num, compare_type)
        elif cycle == "year":
            pass
    print(this_cycle_index, this_cycle_doc, last_cycle_index, last_cycle_doc, fisrt_day_ts, last_day_ts, this_cycle, last_cycle, last_year)
    if compare_type == "daily":
        search_body1 = {
            "query": {
                "bool": {
                    "filter": [
                        {"term": {"platform.keyword": platform}},
                        {"term": {"releaser.keyword": releaser}},
                        {"range": {"release_time": {"gte": fisrt_day_ts, "lt": last_day_ts}}},
                        {"range": {"duration": {"lte": 600}}},
                    ]
                }
            }, "aggs": {
                "sum_play": {
                    "sum": {
                        "field": "play_count"
                    }
                },
                "sum_favorite": {
                    "sum": {
                        "field": "favorite_count"
                    }
                },
                "sum_comment": {
                    "sum": {
                        "field": "comment_count"
                    }
                }

            }
        }
        if releaser_id_str:
            search_body1["query"]["bool"]["filter"].pop(1)
            search_body1["query"]["bool"]["filter"].append({"term": {"releaser_id_str": releaser_id_str}})
        resp1 = es.search(index=this_cycle_index, doc_type=this_cycle_doc, body=search_body1)
        this_cycle_videonum = resp1['hits']['total']
        this_play_count_sum = int(resp1['aggregations']['sum_play']['value'])
        this_favorite_count_sum = int(resp1['aggregations']['sum_favorite']['value'])
        this_comment_count_sum = int(resp1['aggregations']['sum_comment']['value'])
    elif compare_type == "new_released":
        search_body1 = {
            "query": {
                "bool": {
                    "filter": [
                        {"term": {"platform.keyword": platform}},
                        {"term": {"releaser.keyword": releaser}},
                        {"term": {"stats_type": "new_released"}}
                    ]
                }
            }
        }
        if releaser_id_str:
            search_body1["query"]["bool"]["filter"].pop(1)
            search_body1["query"]["bool"]["filter"].append({"term": {"releaser_id_str.keyword": releaser_id_str}})
        if cycle == "week":

            search_body1["query"]["bool"]["filter"].append({"term": {"data_week_year": year}})
            search_body1["query"]["bool"]["filter"].append({"term": {"data_week_num": this_cycle}})
        elif cycle == "month":
            search_body1["query"]["bool"]["filter"].append({"term": {"data_year": year}})
            search_body1["query"]["bool"]["filter"].append({"term": {"data_month": this_cycle}})
        elif cycle == "quarter":
            search_body1["query"]["bool"]["filter"].append({"term": {"data_year": year}})
            search_body1["query"]["bool"]["filter"].append({"term": {"field_data_quarter": "Q%s" % this_cycle}})
        resp1 = es.search(index=last_cycle_index, doc_type=last_cycle_doc, body=search_body1)
        try:

            this_cycle_videonum = resp1['hits']['hits'][0]['_source']['video_num']
            this_play_count_sum = resp1['hits']['hits'][0]['_source']['play_count_sum']
            this_favorite_count_sum = resp1['hits']['hits'][0]['_source']['favorite_count_sum']
            this_comment_count_sum = resp1['hits']['hits'][0]['_source']['comment_count_sum']

        except:
            print('new_released_releaser_error:', releaser, platform, resp1['hits'])
            this_cycle_videonum = 0
            this_play_count_sum = 0
            this_favorite_count_sum = 0
            this_comment_count_sum = 0

    # 计算上周video_num
    search_body2 = {
        "query": {
            "bool": {
                "filter": [
                    {"term": {"platform.keyword": platform}},
                    {"term": {"releaser.keyword": releaser}},
                    {"term": {"stats_type": "new_released"}}
                ]
            }
        }
    }
    if releaser_id_str:
        search_body2["query"]["bool"]["filter"].pop(1)
        search_body2["query"]["bool"]["filter"].append({"term": {"releaser_id_str.keyword": releaser_id_str}})
    if cycle == "week":
        search_body2["query"]["bool"]["filter"].append({"term": {"data_week_year": last_year}})
        search_body2["query"]["bool"]["filter"].append({"term": {"data_week_num": last_cycle}})
    elif cycle == "month":
        search_body2["query"]["bool"]["filter"].append({"term": {"data_year": last_year}})
        search_body2["query"]["bool"]["filter"].append({"term": {"data_month": last_cycle}})
    elif cycle == "quarter":
        search_body2["query"]["bool"]["filter"].append({"term": {"data_year": last_year}})
        search_body2["query"]["bool"]["filter"].append({"term": {"field_data_quarter": "Q%s" % last_cycle}})

    resp2 = es.search(index=last_cycle_index, doc_type=last_cycle_doc, body=search_body2)
    # 本周observed
    search_body3 = {
        "query": {
            "bool": {
                "filter": [
                    {"term": {"platform.keyword": platform}},
                    {"term": {"releaser.keyword": releaser}},
                    {"term": {"stats_type": "observed"}}
                ]
            }
        }
    }
    if releaser_id_str:
        search_body3["query"]["bool"]["filter"].pop(1)
        search_body3["query"]["bool"]["filter"].append({"term": {"releaser_id_str.keyword": releaser_id_str}})
    if cycle == "week":
        search_body3["query"]["bool"]["filter"].append({"term": {"data_week_year": year}})
        search_body3["query"]["bool"]["filter"].append({"term": {"data_week_num": this_cycle}})
    elif cycle == "month":
        search_body3["query"]["bool"]["filter"].append({"term": {"data_year": year}})
        search_body3["query"]["bool"]["filter"].append({"term": {"data_month": this_cycle}})
    elif cycle == "quarter":
        search_body3["query"]["bool"]["filter"].append({"term": {"data_year": year}})
        search_body3["query"]["bool"]["filter"].append({"term": {"field_data_quarter": "Q%s" % this_cycle}})
    resp3 = es.search(index=last_cycle_index, doc_type=last_cycle_doc, body=search_body3)

    try:
        last_cycle_videonum = resp2['hits']['hits'][0]['_source']['video_num']
        last_play_count_sum = resp2['hits']['hits'][0]['_source']['play_count_sum']
        last_favorite_count_sum = resp2['hits']['hits'][0]['_source']['favorite_count_sum']
        last_comment_count_sum = resp2['hits']['hits'][0]['_source']['comment_count_sum']
    except:
        print('last_week_releaser_error:', releaser, platform, resp2['hits'])
        last_cycle_videonum = 0
        last_play_count_sum = 0
        last_favorite_count_sum = 0
        last_comment_count_sum = 0

    try:
        observed_video_num = resp3['hits']['hits'][0]['_source']['video_num']
        observed_play_count_sum = resp3['hits']['hits'][0]['_source']['play_count_sum']
        observed_favorite_count_sum = resp3['hits']['hits'][0]['_source']['favorite_count_sum']
        observed_comment_count_sum = resp3['hits']['hits'][0]['_source']['comment_count_sum']
    except:
        print('observed_releaser_error:', releaser, platform, resp3['hits'])
        observed_video_num = 0
        observed_play_count_sum = 0
        observed_favorite_count_sum = 0
        observed_comment_count_sum = 0
    try:
        last_fans = resp3['hits']['hits'][0]['_source'].get("last_fans")
        releaser_followers_count = resp3['hits']['hits'][0]['_source'].get("releaser_followers_count")
        fans_change = resp3['hits']['hits'][0]['_source'].get("fans_change")
    except:
        last_fans = ""
        releaser_followers_count = ""
        fans_change = ""


    warning_percent = 10
    # 计算百分比  本周/上一周
    videonum_per = int(
        (this_cycle_videonum - last_cycle_videonum) / last_cycle_videonum * 100) if last_cycle_videonum != 0 else ""
    play_count_per = int(
        (this_play_count_sum - last_play_count_sum) / last_play_count_sum * 100) if last_play_count_sum != 0 else ""
    favorite_count_per = int(
        (
                this_favorite_count_sum - last_favorite_count_sum) / last_favorite_count_sum * 100) if last_favorite_count_sum != 0 else ""
    comment_count_per = int(
        (
                this_comment_count_sum - last_comment_count_sum) / last_comment_count_sum * 100) if last_comment_count_sum != 0 else ""
    one_line.update({
                    "releaser_id_str":releaser_id_str,
                    "releaser": releaser,
                     "platform": platform,
                     "year": year,
                     "cycle": cycle,
                     "cycle_num": cycle_num,
                     "releaserUrl": releaserUrl,
                     "this_%s_videonum" % cycle: this_cycle_videonum,
                     'last_%s_videonum' % cycle: last_cycle_videonum,
                     "video_num_percentage": str(videonum_per) + "%" if type(videonum_per) == int else videonum_per,
                     "this_%s_play_count" % cycle: this_play_count_sum,
                     'last_%s_play_count' % cycle: last_play_count_sum,
                     "play_count_percentage": str(play_count_per) + "%" if type(
                         play_count_per) == int else play_count_per,
                     "this_%s_favorite_count" % cycle: this_favorite_count_sum,
                     'last_%s_favorite_count' % cycle: last_favorite_count_sum,
                     "favorite_count_percentage": str(favorite_count_per) + "%" if type(
                         favorite_count_per) == int else favorite_count_per,
                     "this_%s_comment_count" % cycle: this_comment_count_sum,
                     'last_%s_comment_count' % cycle: last_comment_count_sum,
                     "comment_count_percentage": str(comment_count_per) + "%" if type(
                         comment_count_per) == int else comment_count_per,
                     "observed_video_num": observed_video_num,
                     "observed_play_count_sum": observed_play_count_sum,
                     "observed_favorite_count_sum": observed_favorite_count_sum,
                     "observed_comment_count_sum": observed_comment_count_sum,
                    "last_fans":last_fans,
                     "releaser_followers_count":releaser_followers_count,
                     "fans_change":fans_change
                     })
    try:
        if videonum_per > warning_percent:
            one_line['预警值'] = "above"
        elif videonum_per < -warning_percent:
            one_line['预警值'] = "below"
    except:
        pass
    if this_cycle_videonum > observed_video_num or this_play_count_sum > observed_play_count_sum or this_favorite_count_sum > observed_favorite_count_sum or this_comment_count_sum > observed_comment_count_sum:
        one_line["ob_lt_new"] = "True"
    one_line["project_type"] = project_type
    return one_line


def write_to_es(year=None, cycle=None, cycle_num=None, compare_type="new_released"):
    error_info = ""
    bulk_all_body = ""
    count_true = 0
    count_false = 0
    sreach_body = {
            "query": {
                "bool": {
                    "filter": [
                        {"term": {"key_releaser.keyword": "True"}},
                     {"terms": {"platform.keyword":["new_tudou","toutiao","haokan","腾讯视频","腾讯新闻","网易新闻","抖音","kwai","miaopai"]}}

                    ]
                }
            }
        }


    data_res = scan(index="target_releasers", doc_type="doc",
                    query=sreach_body,
                    request_timeout=300,
                    scroll='15m',
                    client=es,raise_on_error=False)
    for one_data in data_res:
        releaser = one_data["_source"]['releaser']
        platform = one_data["_source"]['platform']
        releaserUrl = one_data["_source"]['releaserUrl']
        releaser_id_str = one_data["_source"]['releaser_id_str']

        project_tags = one_data["_source"].get("project_tags")
        project_type = ""
        if project_tags:
            for i in project_tags:
                if not project_type:
                    project_type += i
                else:
                    project_type += "/" + i
        res = create_body(platform, releaser, releaserUrl, year, cycle, cycle_num, compare_type,project_type=project_type,releaser_id_str=releaser_id_str)

        count_true += 1
        if not res["releaser_id_str"]:
            _id = str(res["year"]) + res["cycle"] + str(res["cycle_num"]) + res["platform"] + res["releaser"]
        else:
            _id = str(res["year"]) + res["cycle"] + str(res["cycle_num"]) + res["releaser_id_str"]
        bulk_head = '{"index": {"_id":"%s"}}' % _id
        bulk_body = json.dumps(res, ensure_ascii=False)
        bulk_one_body = bulk_head + '\n' + bulk_body + '\n'
        bulk_all_body += bulk_one_body
        if count_true % 1000 == 0:
            eror_dic = es.bulk(index="shortvideo_alert", doc_type="doc",
                               body=bulk_all_body, request_timeout=200)
            bulk_all_body = ''
            if eror_dic['errors'] is True:
                count_false = 1
                print(eror_dic['items'])
                print(bulk_all_body)
                error_info += eror_dic['items']
            print(count_true)

    if bulk_all_body != '':
        eror_dic = es.bulk(index="shortvideo_alert", doc_type="doc",
                           body=bulk_all_body, request_timeout=200)
        if eror_dic['errors'] is True:
            count_false = 1
            print(eror_dic)
            error_info += eror_dic['items']
    if count_false == 0:
        return "%s条数据写入成功" % str(count_true + count_true)
    else:
        return error_info


def releaser_video_alert_regular(year, cycle, cycle_num, f_log=None):
    hosts = '192.168.17.11'
    port = 80
    user = 'zhouyujiang'
    passwd = '8tM9JDN2LVxM'
    http_auth = (user, passwd)
    es = elasticsearch.Elasticsearch(hosts=hosts, port=port, http_auth=http_auth)
    warning_percent = 10
    today = datetime.datetime.now()

    if cycle == "week":
        this_cycle_str = "本周"
        last_cycle_str = "上周"
        search_str = cycle
    elif cycle == "month":
        this_cycle_str = "本月"
        last_cycle_str = "上月"
        search_str = cycle
    elif cycle == "quarter":
        this_cycle_str = "本季"
        last_cycle_str = "上季"
        search_str = "quarter"
    elif cycle == "year":
        this_cycle_str = "本年"
        last_cycle_str = "上年"
        search_str = "year"

    #

    # 计算周数和时间
    re_catch_lis = []
    res_lis = []
    one_line = {}
    result = []
    # error_count=0
    count = 0
    # 计算本周video_num
    search_body1 = {
        "query": {
            "bool": {
                "filter": [
                    {"term": {"year": year}},
                    {"term": {"cycle.keyword": cycle}},
                    {"term": {"cycle_num": cycle_num}},
                ]
            }
        }
    }
    resp1 = scan(index="shortvideo_alert", doc_type="doc",
                 query=search_body1,
                 request_timeout=1000,
                 scroll='50m',
                 client=es,raise_on_error=False)
    for res in resp1:
        one_line = {
                    "releaser_id_str":res["_source"].get("releaser_id_str"),
                    "releaser": res["_source"]["releaser"],
                    "platform": res["_source"]["platform"],
                    "releaserUrl": res["_source"]["releaserUrl"],
                    "%s发布量" % this_cycle_str: res["_source"]["this_%s_videonum" % cycle],
                    "%s发布量" % last_cycle_str: res["_source"]["last_%s_videonum" % cycle],
                    "发布量百分比": res["_source"]["video_num_percentage"],
                    "%s播放量" % this_cycle_str: res["_source"]["this_%s_play_count" % cycle],
                    "%s播放量" % last_cycle_str: res["_source"]["last_%s_play_count" % cycle],
                    "播放量百分比": res["_source"]["play_count_percentage"],
                    "%s点赞量" % this_cycle_str: res["_source"]["this_%s_favorite_count" % cycle],
                    "%s点赞量" % last_cycle_str: res["_source"]["last_%s_favorite_count" % cycle],
                    "点赞量百分比": res["_source"]["favorite_count_percentage"],
                    "%s评论量" % this_cycle_str: res["_source"]["this_%s_comment_count" % cycle],
                    "%s评论量" % last_cycle_str: res["_source"]["last_%s_comment_count" % cycle],
                    "评论量百分比": res["_source"]["comment_count_percentage"],
                    "observed_video_num": res["_source"]["observed_video_num"] - res["_source"][
                        "this_%s_videonum" % cycle],
                    "observed_play_count_sum": res["_source"]["observed_play_count_sum"] - res["_source"][
                        "this_%s_play_count" % cycle],
                    "observed_favorite_count_sum": res["_source"]["observed_favorite_count_sum"] - res["_source"][
                        "this_%s_favorite_count" % cycle],
                    "observed_comment_count_sum": res["_source"]["observed_comment_count_sum"] - res["_source"][
                        "this_%s_comment_count" % cycle],
                    "last_fans": res["_source"]["last_fans"],
                    "releaser_followers_count": res["_source"]["releaser_followers_count"],
                    "fans_change": res["_source"]["fans_change"],
                    "预警值": "",
                    "observed小于new_release": "",
                    "project_type":res["_source"].get("project_type")
                    }
        try:
            if int(res["_source"]["video_num_percentage"][:-1]) > warning_percent:
                one_line['预警值'] = "above"
            elif int(res["_source"]["video_num_percentage"][:-1]) < -warning_percent:
                one_line['预警值'] = "below"
                re_catch_lis.append({"platform": res["_source"]["platform"], "releaser": res["_source"]["releaser"]})
        except:
            pass
        if one_line["observed_video_num"] < 0 or one_line["observed_play_count_sum"] < 0 or one_line[
            "observed_favorite_count_sum"] < 0 or one_line["observed_comment_count_sum"] < 0:
            one_line["observed小于new_release"] = "True"
        # if args.mail:
        #     pop_value = ["observed_video_num", "observed_play_count_sum", "observed_favorite_count_sum",
        #                  "observed_comment_count_sum", "observed小于new_release"]
        #     for p in pop_value:
        #         one_line.pop(p)
        result.append(one_line)
        result = sorted(result, key=lambda x: x["%s发布量" % this_cycle_str], reverse=True)
        columns = ["releaser_id_str","releaser", "platform", "releaserUrl", "%s发布量" % this_cycle_str,
                   "%s发布量" % last_cycle_str,
                   "发布量百分比",
                   "%s播放量" % this_cycle_str,
                   "%s播放量" % last_cycle_str,
                   "播放量百分比",
                   "%s点赞量" % this_cycle_str,
                   "%s点赞量" % last_cycle_str,
                   "点赞量百分比",
                   "%s评论量" % this_cycle_str,
                   "%s评论量" % last_cycle_str,
                   "评论量百分比",
                   "observed_video_num",
                   "observed_play_count_sum",
                   "observed_favorite_count_sum",
                   "observed_comment_count_sum",
                   "预警值",
                   "last_fans",
                   "releaser_followers_count",
                    "fans_change",
                   "observed小于new_release",
                   "project_type"
                   ]

        data = pd.DataFrame(result)
        email_subj = cycle + str(cycle_num) + cycle + str(cycle_num - 1) +'_releaser_video_num预警 %s' % (
            today.isoformat()[:10])
        csv_path = 'releaser_video_num预警 %s' % (today.isoformat()[:10])

        try:
            this_path = "/home/hanye/project_data/Python/Projects/proj-short-videos/maintenance/send_recev_email/%s" % csv_path
            try:
                os.makedirs(this_path)
            except:
                pass
        except:
            this_path = os.getcwd() + "/%s/" % csv_path
            os.makedirs(this_path)
            print("fail to create folder %s" % this_path)
        finally:
            data.to_csv("%s/%s.csv" % (this_path, email_subj), encoding='gb18030', columns=columns, mode="w")

    print(this_path)
    return this_path

def send_email(csv_path):
    f_log = None
    # if f_log == None:
    # path = '/home/zhouyujiang/email_log/'
    #     log_fn = 'releaser_video_num_alert_for_%s_log' % datetime.datetime.strftime(today, '%b-%Y')
    #     f_log = open(path + log_fn, 'a')
    #     # pass
    # else:
    #     f_log = sys.stdout
    #
    # print('*' * 80, file=f_log)
    # print('log timestamp ', datetime.datetime.now(), file=f_log)
    # print('Checking task for fetch_date', today.isoformat()[:10], file=f_log)
    email_group = [
        'litao@csm.com.cn',
        'zhouyujiang@csm.com.cn',
        "gengdi@csm.com.cn"

    ]
    today = datetime.datetime.now()
    if args.mail:
        email_group = [
            'zhouyujiang@csm.com.cn',
            'litao@csm.com.cn',
            'hanye@csm.com.cn',
            'zhangtianli@csm.com.cn',
            'zhangminghui@csm.com.cn',
             "luojia@csm.com.cn",
            "gengdi@csm.com.cn",
            "jiangyanhong@csm.com.cn"
        ]

    email_msg_suffix = ('\n\n\n'
                        + '-' * 80 + '\n'
                        + '这是自动发送的邮件，可以不用回复。\n'
                        + 'This is an automatically sent message. You do NOT need to reply.\n')

    # # send the alert email
    csm_mail_service = 'mail.csm.com.cn'
    sender = 'litao@csm.com.cn'
    email_subj = 'releaser_video_num预警 %s' % (today.isoformat()[:10])
    email_msg_body = '问好:\n    附件为'
    email_msg_body += '发布者数据预警数据\n\n'

    email_msg_body += "此邮件为数据预警邮件 如有任何问题联系"

    if email_msg_body != '':
        email_msg_body += email_msg_suffix
        print('email_msg_body:\n', email_msg_body, file=f_log)


    # if args.mail:
    #     pop_value = ["observed_video_num", "observed_play_count_sum", "observed_favorite_count_sum",
    #                  "observed_comment_count_sum", "observed小于new_release"]
    #     for p in pop_value:
    #         i = columns.index(p)
    #         columns.pop(i)

    today_str = today.strftime("%Y-%m-%d")
    func_send_email_with_file.send_file_email(
        file_path=csv_path
        , email_msg_body_str=email_msg_body, email_group=email_group, cc_group=[], title_str=email_subj,
        data_str=today_str, sender="litao@csm.com.cn")
    print('\n\n', file=f_log)
    # f_log.close()

def check_mni_real(year=None, cycle=None, cycle_num=None,this_path = None):
    if not year:
        year = datetime.datetime.now().year
    if not cycle and cycle_num:
        cycle = "week"
        cycle_num = this_cycle

    if cycle == "week":
        cycle_str = "weekly"
    elif cycle == "month":
        cycle_str = "monthly"
    all_list = []
    search_body = {
          "query": {
            "bool":{
              "filter": [
                ],"must": [
                {"exists":{"field":"%s_net_inc_play_count_real"% cycle_str}}
              ]
            }
          }
        }

    scan_re = scan(index=this_cycle_index, doc_type=this_cycle_doc,
                 query=search_body,
                 request_timeout=100,
                 scroll='5m',
                 client=es)
    for one_scan in scan_re:
        try:
            re_time = datetime.datetime.fromtimestamp(one_scan['_source'].get("release_time") / 1000)
            time_str = re_time.strftime('%Y-%m-%d %H:%M:%S')
            fc_time = datetime.datetime.fromtimestamp(one_scan['_source'].get("fetch_time") / 1000)
            fc_str = fc_time.strftime('%Y-%m-%d %H:%M:%S')
        except:
            time_str = ""
            fc_str = ""
        line_dict = ({"id": one_scan['_id'],
                      'releaser': one_scan['_source'].get("releaser"),
                      'title': one_scan['_source'].get("title"),
                      'platform': one_scan['_source'].get("platform"),
                      'url': one_scan['_source'].get("url"),
                      "release_time": time_str,
                      "releaserUrl": one_scan['_source'].get("releaserUrl"),
                      "fetch_time": fc_str,
                      "play_count": one_scan['_source'].get("play_count"),
                      "favorite_count": one_scan['_source'].get("favorite_count"),
                      "comment_count": one_scan['_source'].get("comment_count"),
                      "duration": one_scan['_source'].get("duration"),
                      "%s_inc_play_count_real"% cycle_str: one_scan['_source'].get("%s_net_inc_play_count"% cycle_str),
                      "%s_inc_play_count_real"% cycle_str: one_scan['_source'].get("%s_net_inc_comment_count"% cycle_str),
                      "%s_inc_play_count_real"% cycle_str: one_scan['_source'].get("5s_net_inc_favorite_count"% cycle_str),
                      "project_type":one_scan['_source'].get("project_type")
                      })
        all_list.append(line_dict)


    columns = ["id", 'releaser', 'platform', 'title', "releaserUrl", 'url',
           "release_time",
           "fetch_time",
           "play_count",
           "favorite_count",
           "comment_count", "duration",
           "%s_net_inc_play_count"% cycle_str, "%s_net_inc_comment_count"% cycle_str,
           "%s_net_inc_favorite_count" % cycle_str,
        "project_type"
           ]
    if all_list:
        data = pd.DataFrame(all_list)
        s = datetime.datetime.now()
        ss = str(s)[0:19].replace(' ', '-').replace(':', '-')
        data.to_csv('./%s/%s%s.csv' % (this_path,this_cycle_doc, "MNI为负值的账号汇总"), encoding='gb18030', columns=columns, mode="w")


if args.cycle_num:
    print("write to es")
    write_to_es(year=args.year,cycle=args.cycle,cycle_num=args.cycle_num,compare_type=args.compare_type)
    time.sleep(5)
    csv_path = releaser_video_alert_regular(year=args.year, cycle=args.cycle, cycle_num=args.cycle_num)
    # check_mni_real(year=args.year, cycle=args.cycle, cycle_num=args.cycle_num,this_path=csv_path)
    send_email(csv_path)
    done = True

if __name__ == '__main__':
    now = datetime.datetime.now()
    csv_path = None
    if not done:
        # write_to_es(year=2019, cycle="month", cycle_num=8)
        # time.sleep(5)
        # csv_path = releaser_video_alert_regular(year=2019, cycle="month", cycle_num=8)
        cycle_type = args.cycle
        if cycle_type == "month":
            res_lis = month_num(year=now.year)
        elif cycle_type == "quarter":
            res_lis = quarter_num(year=now.year)
        else:
            res_lis = week_num(year=now.year)
            cycle_type = "week"
        print(res_lis)
        write_to_es(year=res_lis[8], cycle=cycle_type,cycle_num=res_lis[6],compare_type=args.compare_type)
        time.sleep(5)
        csv_path = releaser_video_alert_regular(year=res_lis[8], cycle=cycle_type,cycle_num=res_lis[6])
        # check_mni_real(year=args.year, cycle=args.cycle, cycle_num=args.cycle_num,this_path=csv_path)
        print(csv_path)
        send_email(csv_path)
        print(month_num(year=2020))