# -*- coding:utf-8 -*-
# @Time : 2019/7/26 14:33 
# @Author : litao
# -*- coding: utf-8 -*-
"""
Created on Mon Dec 17 10:05:18 2018

@author: zhouyujiang

从csv 写入 target_releaser索引

"""
import json, re
import datetime, copy
from elasticsearch import Elasticsearch

from crawler.write_data_into_es.func_get_releaser_id import get_releaser_id
import redis
import hashlib

hosts = '172.16.32.37'
port = 9200
es = Elasticsearch(hosts=hosts, port=port)
# pool = redis.ConnectionPool(host='192.168.17.60', port=6379, db=2, decode_responses=True)
# rds = redis.Redis(connection_pool=pool)

today = datetime.datetime.now()
first_day = datetime.datetime(today.year, today.month, 1)
day_before_first_day = first_day - datetime.timedelta(1)
l_month = day_before_first_day.month
l_year = day_before_first_day.year
count = 0


def parse_line_dict(line, line_dict, blank_space_error, new_line_error, err_id_line):
    for k in line_dict:
        try:
            if " " in line_dict[k]:
                blank_space_error = blank_space_error + str(line + 2) + ","
            if "\r" in line_dict[k]:
                new_line_error = new_line_error + str(line + 2) + ","
            if "\n" in line_dict[k]:
                new_line_error = new_line_error + str(line + 2) + ","
            if "\t" in line_dict[k]:
                new_line_error = new_line_error + str(line + 2) + ","
            line_dict[k] = line_dict[k].replace("\r", "").replace("\n", "").replace("\t", "").replace(" ", "")
        except Exception as e:
            # print(e)
            continue
    return line_dict, blank_space_error, new_line_error, err_id_line


def write_to_es(file, push_to_redis=True, update=True, key_releaser=False, update_dic={}, extra_dic={}, **kwargs):
    """

    :param file:
    :param kwargs: not_push_to_redis = True 不push到redis中
                   department : Str 所属部门
                   key_releaser: bool 用于判断是否重点发布者
                   add_departments: list 用于增加部门
                   del_departments: list 用于删除部门
                   add_project_tags: list 用于增加项目标签
                   del_project_tags: list 用于删除项目标签
                   kwargs: extra_dic 用于添加额外的信息

                   导入的csv中添加 purchase_end_time 和 is_purchased 字段
                    用于表示是否采购 purchase_end_time (%Y-%m-%d)
                    is_purchased (0/1)
    :return:
    """

    bulk_all_body = ""
    err_id_line = ""
    blank_space_error = ""
    new_line_error = ""
    error_msg_list = []
    bluk_purchase_list = []
    count = 0

    try:
        f = open(file, 'r', encoding="gb18030")
        head = f.readline()
        head_list = head.strip().split(',')
    except:
        f = file
    for line, i in enumerate(f):
        if type(file) != list:
            try:
                line_list = i.strip().split(',')
                line_dict = dict(zip(head_list, line_list))
            except:
                line_dict = f
        else:
            line_dict = i
        print(i)

        try:
            platform = line_dict['platform']
            if platform == "short_video":
                line_dict['platform'] = line_dict['releaser_platform']
                platform = line_dict['releaser_platform']
        except:
            new_line_error += str(line + 2) + ","
            continue
        line_dict, blank_space_error, new_line_error, err_id_line = parse_line_dict(line, line_dict, blank_space_error,
                                                                                    new_line_error, err_id_line)
        if "" in line_dict:
            line_dict.pop("")
        try:
            releaserUrl = line_dict['releaserUrl']
        except:
            releaserUrl = line_dict['releaserUrl']

        if extra_dic:
            line_dict.update(extra_dic)
        # print(str(get_releaser_id(platform=platform, releaserUrl=releaserUrl)))
        line_dict["releaser_id"] = get_releaser_id(platform=platform, releaserUrl=releaserUrl)

        if line_dict["releaser_id"]:
            doc_id = platform + '_' + line_dict['releaser_id']
        else:
            doc_id = platform + '_' + line_dict['releaser']
            err_id_line += str(line + 2) + ","
        find_exist = {
                "query": {
                        "bool": {
                                "filter": [
                                        {"term": {"_id": doc_id}}
                                ]
                        }
                }
        }
        if not extra_dic.get("project_tags"):
            extra_dic.pop("project_tags", 0)
        if not extra_dic.get("department_tags"):
            extra_dic.pop("department_tags", 0)

        # search_re = es.search(index='target_releasers', doc_type='doc', body=find_exist)
        # if search_re['hits']['total'] > 0:
        #     search_source = search_re['hits']['hits'][0]['_source']
        #     # print(search_source)
        #     if search_source.get("project_tags"):
        #         try:
        #             # print(kwargs.get("extra_dic"))
        #             line_dict["project_tags"].extend(search_source.get("project_tags"))
        #             line_dict["project_tags"] = list(set(line_dict["project_tags"]))
        #             search_source.pop("project_tags", 0)
        #         except Exception as e:
        #             pass
        #             # print("project_tags error", e)
        #     if search_source.get("department_tags"):
        #         try:
        #             # print(kwargs.get("extra_dic"))
        #             line_dict["department_tags"].extend(search_source.get("department_tags"))
        #             line_dict["department_tags"] = list(set(line_dict["department_tags"]))
        #             search_source.pop("department_tags", 0)
        #         except Exception as e:
        #             pass
        #             # print("project_tags error", e)
        #     if update:
        #         line_dict.update(search_source)
        #     line_dict["post_time"] = search_source.get("post_time")

        if line_dict.get("post_time"):
            pass
        else:
            line_dict['post_time'] = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000)

        try:
            line_dict["releaser_id"] = get_releaser_id(platform=platform, releaserUrl=releaserUrl)
            line_dict["releaser_id_str"] = platform + "_" + line_dict["releaser_id"]
            line_dict["is_valid"] = "true"
        except:
            line_dict["releaser_id"] = ""
            line_dict["releaser_id_str"] = ""
            line_dict["is_valid"] = "false"
        if kwargs.get("post_by"):
            line_dict["post_by"] = kwargs.get("post_by")
        if not line_dict.get("project_tags"):
            line_dict["project_tags"] = []
        if not line_dict.get("department_tags"):
            line_dict["department_tags"] = []
        if line_dict.get("add_departments"):
            line_dict["department_tags"].extend(line_dict.get("add_departments"))
            line_dict["department_tags"] = list(set(line_dict["department_tags"]))
        if line_dict.get("del_departments"):
            for key in line_dict.get("del_departments"):
                try:
                    line_dict["department_tags"].remove(key)
                except:
                    continue
        if line_dict.get("add_project_tags"):
            line_dict["project_tags"].extend(line_dict.get("add_project_tags"))
            line_dict["project_tags"] = list(set(line_dict["project_tags"]))
        if line_dict.get("del_project_tags"):
            for key in line_dict.get("del_project_tags"):
                try:
                    line_dict["project_tags"].remove(key)
                except:
                    continue
        bulk_dic = {
                "releaser": line_dict.get("releaser"),
                "releaserUrl": line_dict.get("releaserUrl"),
                "platform": line_dict.get("platform"),
                "releaser_id": line_dict.get("releaser_id"),
                "releaser_id_str": line_dict.get("releaser_id_str"),
                "post_by": line_dict.get("post_by"),
                "post_time": line_dict.get("post_time"),
                "frequency": 3 if line_dict.get("project_tags") else 1,
                "key_releaser": line_dict.get("key_releaser"),
                "is_valid": line_dict.get("is_valid"),
                "has_data": line_dict.get("has_data") if line_dict.get("has_data") else 0,
                "project_tags": line_dict.get("project_tags"),
                "department_tags": line_dict.get("department_tags"),
                'timestamp': int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000),
                'media_type': line_dict.get("media_type") if line_dict.get("media_type") else "",
                'releaser_type': line_dict.get("releaser_type") if line_dict.get("releaser_type") else "",

        }

        bulk_head = '{"index": {"_id":"%s"}}' % doc_id
        # if push_to_redis:
        #     rds.lpush("releaser_doc_id_list", doc_id)
        data_str = json.dumps(bulk_dic, ensure_ascii=False)
        bulk_one_body = bulk_head + '\n' + data_str + '\n'
        #        print(bulk_one_body)
        bulk_all_body += bulk_one_body
        count = count + 1
        if count % 500 == 0:
            eror_dic = es.bulk(index='target_releasers', doc_type='doc',
                               body=bulk_all_body)
            bulk_all_body = ''
            if eror_dic['errors'] is True:
                print(eror_dic)
    if bulk_all_body != '':
        eror_dic = es.bulk(body=bulk_all_body,
                           index='target_releasers',
                           doc_type='doc',
                           )
        if eror_dic['errors'] is True:
            print(eror_dic)
    error_msg_list.append("%s条 写入成功" % count)
    if err_id_line:
        error_msg_list.append("第%s行 releaserUrl错误" % err_id_line[:-1])
    if blank_space_error:
        error_msg_list.append("第%s行 发现存在空格" % blank_space_error[:-1])
    if new_line_error:
        error_msg_list.append("第%s行 发现存在换行符" % new_line_error[:-1])
    return error_msg_list


if __name__ == "__main__":
    data_list =  [{"releaserUrl": "https://weibo.com/u/1764615662", "releaser": "娱乐圈贵妃", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/3662247177", "releaser": "捞娱君", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/2378564111", "releaser": "娱乐扒皮", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/2983578965", "releaser": "娱乐圈小青年", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/3938976579", "releaser": "娱乐捞饭", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/6511177474", "releaser": "小组吃瓜蜀黍", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/6343916471", "releaser": "圈内老顽童", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/6511177474", "releaser": "八组吃瓜蜀黍", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/2921603920", "releaser": "娱乐圈新鲜事", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/6470919752", "releaser": "伊丽莎白骨精啊", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/2653906910?refer_flag=1001030103_&is_hot=1", "releaser": "娱乐榜姐",
          "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/3115996363?is_hot=1", "releaser": "娱乐星事", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/p/1005053212093237/home?from=page_100505&mod=TAB#place", "releaser": "星探扒皮",
          "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/3926129482", "releaser": "星闻追踪", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/5509337969?is_hot=1", "releaser": "卦哥娱乐", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/5477320351", "releaser": "圈内扒爷", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/p/1005055634795408/home?from=page_100505&mod=TAB#place", "releaser": "圈八戒 ",
          "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/u/6511173721", "releaser": "圈内课代表", "platform": "weibo"},
         {"releaserUrl": "https://weibo.com/p/1005055471534537/home?from=page_100505&mod=TAB&is_hot=1#place",
          "releaser": "娱闻少女", "platform": "weibo"}]
    extra_dic = {
            "department_tags":["策略组"],
             'key_releaser': True,
             'frequency': 3,
    }

    # csv_type = {"SMG": [], "an_hui": [], "ronghe": [], "su_zhou": []}
    #ronghe_releaser_write_es(file, post_by="litao")
    write_to_es(data_list, post_by="litao", extra_dic=extra_dic, push_to_redis=False)
