crawler_v_qq.py

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 14:19:00 2018

independent function:
1 list page
2 search page
3 video page
4 releaser page

All function take parameters and return dict list.

parameters:
1 list page: url
2 search page: keyword
3 video page: url
4 releaser page: url


@author: fangyucheng
"""

import os
import asyncio
import copy
import requests
import re
import datetime
import json
# import aiohttp
import random
from bs4 import BeautifulSoup
from multiprocessing import Pool
# from multiprocessing import Process
from crawler.crawler_sys.framework.video_fields_std import Std_fields_video
from crawler.crawler_sys.utils.output_results import retry_get_url
from crawler.crawler_sys.utils.output_results import output_result
# from crawler.crawler_sys.utils import output_log
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.trans_duration_str_to_second import trans_duration
from crawler.crawler_sys.utils.util_logging import logged
import urllib
from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_proxy
try:
    from crawler_sys.framework.func_get_releaser_id import *
except:
    from write_data_into_es.func_get_releaser_id import *


class Crawler_v_qq():
    def __init__(self, timeout=None, platform='腾讯视频'):
        if timeout == None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        # remove fields that crawled data don't have
        pop_key_Lst = ['describe', 'repost_count', 'isOriginal',
                       'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)

        self.list_page_url_dict = {
            '电影': 'http://v.qq.com/x/list/movie',
            '电视剧': 'http://v.qq.com/x/list/tv',
            '综艺': 'http://v.qq.com/x/list/variety',
            '动漫': 'http://v.qq.com/x/list/cartoon',
            '少儿': 'http://v.qq.com/x/list/children',
            '音乐': 'http://v.qq.com/x/list/music',
            '纪录片': 'http://v.qq.com/x/list/doco',
            '新闻': 'http://v.qq.com/x/list/news',
            '军事': 'http://v.qq.com/x/list/military',
            '娱乐': 'http://v.qq.com/x/list/ent',
            '体育': 'http://v.qq.com/x/list/sports',
            '游戏': 'http://v.qq.com/x/list/games',
            '搞笑': 'http://v.qq.com/x/list/fun',
            '微电影': 'http://v.qq.com/x/list/dv',
            '时尚': 'http://v.qq.com/x/list/fashion',
            '生活': 'http://v.qq.com/x/list/life',
            '母婴': 'http://v.qq.com/x/list/baby',
            '汽车': 'http://v.qq.com/x/list/auto',
            '科技': 'http://v.qq.com/x/list/tech',
            '教育': 'http://v.qq.com/x/list/education',
            '财经': 'http://v.qq.com/x/list/finance',
            '房产': 'http://v.qq.com/x/list/house',
            '旅游': 'http://v.qq.com/x/list/travel',
        }
        self.legal_list_page_urls = []
        self.legal_channels = []
        for ch in self.list_page_url_dict:
            list_page_url = self.list_page_url_dict[ch]
            self.legal_list_page_urls.append(list_page_url)
            self.legal_channels.append(ch)

    #        log_path = '/home/hanye/crawlersNew/crawler/crawler_log'
    #        current_day = str(datetime.datetime.now())[:10]
    #        info_log_file = log_path + '/all_' + current_day + '.log'
    #        info_name = self.platform + '_info'
    #        self.loggerinfo = output_log.init_logger(name=info_name, log_file=info_log_file)

    def video_page(self, url, channel=None):
        if 'm.v.qq.com' in url:
            vid_str = ' '.join(re.findall('o/d/y/.*.html', url))
            vid = vid_str.replace('o/d/y/', '').replace('.html', '')
            url = 'https://v.qq.com/x/page/' + vid + '.html'
        get_page = retry_get_url(url, timeout=self.timeout)
        if get_page is None:
            return None
        else:
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            try:
                title = soup.find('h1', {'class': 'video_title _video_title'}).text
                title = title.replace('\n', '')
                title = title.replace('\t', '')
                # remove leading and trailing spaces
                title = re.sub('(^\s+)|(\s+$)', '', title)
            except:
                try:
                    title = soup.find('h1', {'class': 'video_title'}).text
                    title = title.replace('\n', '')
                    title = title.replace('\t', '')
                    title = re.sub('(^\s+)|(\s+$)', '', title)
                except:
                    title = None
            try:
                releaser = soup.find('span', {'class': 'user_name'}).text
            except:
                releaser = None
            try:
                releaserUrl = soup.find('a', {'class': 'user_info'})['href']
            except:
                releaserUrl = None
            try:
                video_intro = soup.find('meta', {'itemprop': 'description'})['content']
            except:
                video_intro = None
            soup_find = soup.find("script", {"r-notemplate": "true"})
            if soup_find != None:
                midstep = soup_find.text
            else:
                print('Failed to get correct html text with soup')
                return None
            video_info_var_Lst = re.findall('var\s+VIDEO_INFO\s+=\s*{.+}', midstep)
            if video_info_var_Lst != []:
                video_info_var = video_info_var_Lst[0]
                video_info_json = re.sub('var\s+VIDEO_INFO\s+=\s*', '', video_info_var)
                try:
                    video_info_dict = json.loads(video_info_json)
                except:
                    print('Failed to transfer video_info_json to dict '
                          'for url: ' % url)
                    video_info_dict = {}
                if video_info_dict != {}:
                    if 'duration' in video_info_dict:
                        duration_str = video_info_dict['duration']
                        duration = int(duration_str)
                    else:
                        duration = None
                    if 'title' in video_info_dict:
                        title = video_info_dict['title']
                    if 'view_all_count' in video_info_dict:
                        play_count = video_info_dict['view_all_count']
                        data_source = 'video_info'
                    else:
                        try:
                            play_count_str = re.findall('interactionCount.*', page)[0]
                            play_count = re.findall('\d+', play_count_str)[0]
                            play_count = int(play_count)
                            data_source = 'interactioncount'
                        except:
                            play_count = None
                    if 'video_checkup_time' in video_info_dict:
                        release_time_str = video_info_dict['video_checkup_time']
                        try:
                            release_time_ts = int(datetime.datetime.strptime(
                                release_time_str, '%Y-%m-%d %H:%M:%S'
                            ).timestamp() * 1e3)
                        except:
                            release_time_ts = None
                    else:
                        release_time_ts = None
            else:
                try:
                    play_count_str = re.findall('interactionCount.*', page)[0]
                    play_count = re.findall('\d+', play_count_str)[0]
                    play_count = int(play_count)
                    data_source = 'interactioncount'
                except:
                    play_count = None
                try:
                    release_time_str = soup.find('span', {'class': 'tag_item'}).text
                    re_lst = re.findall('\d+', release_time_str)
                    release_time_raw = re_lst[0] + '-' + re_lst[1] + '-' + re_lst[2]
                    release_time_ts = int(datetime.datetime.strptime(release_time_raw, '%Y-%m-%d').timestamp() * 1e3)
                except:
                    release_time_ts = None
                try:
                    duration_str = re.findall('duration.*', page)[0]
                    duration = int(re.findall('\d+', duration_str)[0])
                except:
                    duration = None
            fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
            video_dict = copy.deepcopy(self.video_data)
            if play_count != None:
                video_dict['title'] = title
                video_dict['data_source'] = data_source
                if channel is not None:
                    video_dict['channel'] = channel
                video_dict['releaser'] = releaser
                video_dict['play_count'] = play_count
                video_dict['release_time'] = release_time_ts
                video_dict['duration'] = duration
                video_dict['url'] = url
                video_dict['fetch_time'] = fetch_time
                if releaserUrl is not None:
                    video_dict['releaserUrl'] = releaserUrl
                if video_intro is not None:
                    video_dict['video_intro'] = video_intro
            return video_dict

    def search_page(self, keyword, search_pages_max=30,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None):
        search_page_Lst = []

        def process_one_line(data_line):
            url = data_line.h2.a['href']
            dicdicdic = self.video_page(url)
            return dicdicdic

        search_url = ('https://v.qq.com/x/search?q='
                      + keyword
                      + '&cur={}'.format(str(i)) for i in range(1, search_pages_max + 1))
        for urls in search_url:
            get_page = requests.get(urls, timeout=self.timeout)
            print(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            tencent = soup.find_all("div", {"class": "result_item result_item_h _quickopen"})
            for data_line in tencent:
                one_line_dic = process_one_line(data_line)
                print('get one line done')
                print(one_line_dic)
                search_page_Lst.append(one_line_dic)

            if len(search_page_Lst) >= 100:
                output_result(result_Lst=search_page_Lst,
                              platform=self.platform,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              es_index=es_index,
                              doc_type=doc_type)
                search_page_Lst.clear()

        if search_page_Lst != []:
            output_result(result_Lst=search_page_Lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)
        return search_page_Lst

    # list page synchronous
    def list_page_sync(self, listurl, channel=None,
                       output_to_file=False, filepath=None,
                       output_to_es_raw=False,
                       output_to_es_register=False,
                       push_to_redis=False,
                       page_num_max=34,
                       output_es_index=None,
                       output_doc_type=None,
                       ):
        if channel is None:
            channel = listurl.split('list/')[-1]
        # listurl=http://v.qq.com/x/list/fashion/
        list_data_Lst = []
        listnum = []
        videos_in_one_page = 30
        for i in range(0, page_num_max):
            list_num = i * videos_in_one_page
            listnum.append(list_num)
        # 最近热播
        listpage = [listurl + '/?sort=40&offset={}'.format(str(i)) for i in listnum]
        # 最近上架
        # listpage=[listurl+'?sort=5&offset={}'.format(str(i)) for i in listnum]
        for listurls in listpage:
            get_page = retry_get_url(listurls, timeout=self.timeout)
            if get_page is None:
                print('Failed to get page for list page url: %s'
                      % listurls)
                return None
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(listurls)
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('li', {'class': 'list_item'})
            for line in midstep:
                one_video_dic = {}
                url = line.a['href']
                try:
                    one_video_dic = self.video_page(url, channel)
                    find_play_count = BeautifulSoup(list(line)[-2], 'html.parser')
                    play_count_str1 = find_play_count.find('span', {'class': 'num'}).text
                    play_count_str2 = play_count_str1.replace(' ', '')
                    try:
                        play_count = trans_play_count(play_count_str2)
                    except:
                        play_count = 0
                    one_video_dic['play_count'] = play_count
                    list_data_Lst.append(one_video_dic)
                    if len(list_data_Lst) >= 100:
                        if output_es_index != None and output_doc_type != None:
                            output_result(list_data_Lst,
                                          self.platform,
                                          output_to_file=output_to_file,
                                          filepath=filepath,
                                          output_to_es_raw=output_to_es_raw,
                                          output_to_es_register=output_to_es_register,
                                          push_to_redis=push_to_redis,
                                          es_index=output_es_index,
                                          doc_type=output_doc_type)
                            list_data_Lst.clear()
                        else:
                            output_result(list_data_Lst,
                                          self.platform,
                                          output_to_file=output_to_file,
                                          filepath=filepath,
                                          output_to_es_raw=output_to_es_raw,
                                          output_to_es_register=output_to_es_register,
                                          push_to_redis=push_to_redis)
                            list_data_Lst.clear()
                except:
                    print('failed to get data from %s' % url)
        if list_data_Lst != []:
            if output_es_index != None and output_doc_type != None:
                output_result(list_data_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis,
                              es_index=output_es_index,
                              doc_type=output_doc_type)
                list_data_Lst.clear()
            else:
                output_result(list_data_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis)
                list_data_Lst.clear()
        return list_data_Lst


    def get_releaser_image(self,releaserUrl=None,data=None):
        if releaserUrl:
            get_page = requests.get(releaserUrl)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                image_url = re.findall('class="top_avatar_pic"\n          src="(.*)"', page)[0]
                # follower_str = soup.find('span', {'class': 'value _follow_number'}).text
                print(image_url)
                return "http:" + image_url
            except:
                print("can't get image_url")
        else:
            image_url = re.findall('class="top_avatar_pic"\n          src="(.*)"', data)[0]
            # follower_str = soup.find('span', {'class': 'value _follow_number'}).text
            print(image_url)
            return "http:" + image_url

    @staticmethod
    def get_video_image(data):
        video_photo_url = data["pic_496x280"]
        return video_photo_url

    def doc_list_page(self, listurl):
        # listurl=http://v.qq.com/x/list/fashion/
        done = open('done_qq', 'a')
        result = open('result_qq', 'a')
        error = open('error_qq', 'a')
        list_data_Lst = []
        listnum = []
        for i in range(0, 93):
            list_num = i * 30
            listnum.append(list_num)
        # 最近热播
        listpage = [listurl + '?&offset={}'.format(str(i)) for i in listnum]
        # 最近上架
        # listpage=[listurl+'?sort=5&offset={}'.format(str(i)) for i in listnum]
        for listurl in listpage:
            get_page = requests.get(listurl, timeout=self.timeout)
            get_page.encoding = 'utf-8'
            page = get_page.text
            print(listurl)
            done.write(listurl)
            done.write('\n')
            done.flush()
            soup = BeautifulSoup(page, 'html.parser')
            midstep = soup.find_all('strong', {'class': 'figure_title'})
            for line in midstep:
                album_name = line.text
                url = line.a['href']
                get_page = requests.get(url, timeout=self.timeout)
                get_page.encoding = 'utf-8'
                page = get_page.text
                soup = BeautifulSoup(page, 'html.parser')
                try:
                    get_all_url = soup.find('ul', {'class': 'figure_list _hot_wrapper'})
                    url_agg = get_all_url.find_all('a', {'class': 'figure_detail'})
                    urllist = []
                    for line in url_agg:
                        url_part = line['href']
                        url = 'https://v.qq.com' + url_part
                        urllist.append(url)
                    for url in urllist:
                        try:
                            one_video = self.video_page(url)
                            one_video['album_name'] = album_name
                            print(url)
                            list_data_Lst.append(one_video)
                            one_video_json = json.dumps(one_video)
                            result.write(one_video_json)
                            result.write('\n')
                            result.flush()
                        except AttributeError:
                            D0 = {'url': url, 'album_name': album_name}
                            print('there is an error')
                            json_D0 = json.dumps(D0)
                            error.write(json_D0)
                            error.write('\n')
                            error.flush()
                except:
                    one_video = self.video_page(url)
                    one_video['album_name'] = album_name
                    print(url)
                    list_data_Lst.append(one_video)
                    one_video_json = json.dumps(one_video)
                    result.write(one_video_json)
                    result.write('\n')
                    result.flush()
        done.close()
        result.close()
        error.close()
        return list_data_Lst

    def get_releaser_id(self, releaserUrl):
        return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl,is_qq=True)


    def get_release_time_from_str(self, rt_str):
        minute = '分钟'
        hour = '小时'
        day = '天'
        if minute in rt_str:
            rt_int = int(re.findall('\d+', rt_str)[0])
            rt = datetime.datetime.timestamp(datetime.datetime.now()) - rt_int * 60
            release_time = int(rt * 1e3)
        elif hour in rt_str:
            rt_int = int(re.findall('\d+', rt_str)[0])
            rt = datetime.datetime.timestamp(datetime.datetime.now()) - rt_int * 3600
            release_time = int(rt * 1e3)
        elif day in rt_str:
            rt_int = int(re.findall('\d+', rt_str)[0])
            rt = datetime.datetime.timestamp(datetime.datetime.now()) - rt_int * 3600 * 60
            release_time = int(rt * 1e3)
        else:
            release_time = int(datetime.datetime.strptime(rt_str, '%Y-%m-%d').timestamp() * 1e3)
        return release_time

    # @logged
    def releaser_page(self, releaserUrl,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=10000,
                      es_index=None,
                      doc_type=None,proxies_num=None):
        print('Processing releaserUrl %s' % releaserUrl)
        result_Lst = []
        releaser_info = self.get_releaser_id(releaserUrl)
        number_id = releaser_info['number_id']
        releaser_id = releaser_info['releaser_id']
        releaser = releaser_info['releaser']
        pagenum = 0
        has_more = True
        if releaser_id != None:
            while pagenum <= releaser_page_num_max and has_more:
                pagenum += 1
                url_dic = {
                    "vappid": "50662744",
                    "vsecret": "64b037e091deae75d3840dbc5d565c58abe9ea733743bbaf",
                    "iSortType": "0",
                    "page_index": pagenum,
                    "hasMore": "true",
                    "stUserId": number_id,
                    "page_size": "20",
                    "_": datetime.datetime.now().timestamp()
                }
                releaser_page_url = ('http://access.video.qq.com/pc_client/GetUserVidListPage?%s' % urllib.parse.urlencode(url_dic))
                print('Page number: %d'% pagenum)
                try:
                    if proxies_num:
                        get_page = retry_get_url(releaser_page_url, timeout=self.timeout,proxies=proxies_num)
                    else:
                        get_page = retry_get_url(releaser_page_url,timeout=self.timeout)
                except:
                    get_page = None
                    has_more = False
                if get_page != None and get_page.status_code == 200:
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    real_page = page[5:]
                    real_page = real_page.replace('null', 'None')
                    try:
                        get_page_dic = eval(real_page)
                        page_dic = get_page_dic["data"]['vecVidInfo']
                    except:
                        page_dic = None
                        has_more = False
                    if page_dic != None:
                        for a_video in page_dic:
                            try:
                                video_dic = copy.deepcopy(self.video_data)
                                vid = a_video.get("vid")
                                video_info = a_video.get("mapKeyValue")
                                title = video_info['title']
                                try:
                                    play_count = int(float(video_info['view_all_count']))
                                except:
                                    play_count = 0
                                rt_str = video_info['create_time']
                                release_time = datetime.datetime.strptime(rt_str,"%Y-%m-%d %H:%M")
                                url = "https://v.qq.com/x/page/%s.html" % vid
                                duration = int(video_info['duration'])
                                fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
                                video_dic['title'] = title
                                video_dic['duration'] = duration
                                video_dic['url'] = url
                                video_dic['play_count'] = play_count
                                if play_count <= 300:
                                    video_dic['comment_count'] = 0
                                else:
                                    video_dic['comment_count'] = self.func_get_comment_count(url)
                                    if play_count >= 10000 and video_dic['comment_count'] == 0:
                                        video_dic['comment_count'] = self.func_get_comment_count(url)
                                video_dic['releaser'] = releaser
                                video_dic['release_time'] = int(release_time.timestamp()*1e3)
                                video_dic['fetch_time'] = fetch_time
                                video_dic["video_id"] = vid
                                video_dic["releaser_id_str"] = "腾讯视频_%s"%(releaser_id)
                                video_dic['releaserUrl'] = "http://v.qq.com/vplus/%s" % releaser_id
                                video_dic['video_img'] = self.get_video_image(video_info)
                                yield video_dic
                            except Exception as e:
                                print(e)
                                continue

    def get_releaser_follower_num(self, releaserUrl):
        proxies = get_proxy(1)
        if proxies:
            get_page = requests.get(releaserUrl, proxies=proxies,timeout=3)
        else:
            get_page = requests.get(releaserUrl,timeout=3)
        get_page.encoding = 'utf-8'
        page = get_page.text

        # soup = BeautifulSoup(page, 'html.parser')
        try:
            follower_str = re.findall('data-number="(\d*)"', page)[0]
            # follower_str = soup.find('span', {'class': 'value _follow_number'}).text
            follower_num = trans_play_count(follower_str)
            print('%s follower number is %s' % (releaserUrl, follower_num))
            releaser_img_url = self.get_releaser_image(data=page)
            return follower_num, releaser_img_url
        except:
            print("can't can followers")

    # list page asynchronous which is main stream in the future
    def start_list_page(self, task_list):
        self.list_page_task(task_list)
        self.run_list_page_asyncio()

    def list_page_task(self, task_list,
                       page_num_max=34):
        lst_page_task_lst = []
        for list_url in task_list:
            videos_in_one_page = 30
            num_lst = []
            for i in range(0, page_num_max):
                num = i * videos_in_one_page
                num_lst.append(num)
            task_url_lst_new = [list_url + '/?sort=5&offset=' + str(num) for num in num_lst]
            lst_page_task_lst.extend(task_url_lst_new)
            task_url_lst_hot = [list_url + '/?sort=40&offset=' + str(num) for num in num_lst]
            lst_page_task_lst.extend(task_url_lst_hot)
            task_url_lst_praise = [list_url + '/?sort=48&offset=' + str(num) for num in num_lst]
            lst_page_task_lst.extend(task_url_lst_praise)
        random.shuffle(lst_page_task_lst)
        connect_with_redis.push_list_url_to_redis(platform=self.platform, result_lst=lst_page_task_lst)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return page

    async def list_page(self, loop, task_lst):
        async with aiohttp.ClientSession() as sess_lst_page:
            task_video_page = [loop.create_task(self.download_page(sess_lst_page,
                                                                   lst_url)) for lst_url in task_lst]
            lst_result, unfinished = await asyncio.wait(task_video_page)
            lst_page_html_lst = [v.result() for v in lst_result]
            connect_with_redis.push_list_page_html_to_redis(platform=self.platform,
                                                            result_lst=lst_page_html_lst)
            print("the length of url list is %s" % len(lst_page_html_lst))

    def run_list_page_asyncio(self):
        """
        get list page asynchronously
        """
        key = 'v_qq_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            task_lst = connect_with_redis.retrieve_list_url_from_redis(platform=self.platform,
                                                                       retrieve_count=20)
            print('the length of task list is %s' % len(task_lst))
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.list_page(loop, task_lst=task_lst))

    def parse_list_page_single_process(self):
        key = 'v_qq_list_page_html'
        while connect_with_redis.length_of_lst(key) > 0:
            lst_page_html = connect_with_redis.retrieve_list_page_html_from_redis(platform=self.platform)
            url_lst = self.process_list_page(resp=lst_page_html)
            connect_with_redis.push_url_dict_lst_to_redis_set(platform=self.platform,
                                                              result_lst=url_lst)
            print('push %s url dicts into redis' % len(url_lst))

    def parse_list_page_multi_process(self, process_num=20):
        pool = Pool(processes=process_num)
        for line in range(process_num):
            pool.apply_async(func=self.parse_list_page_single_process)
        pool.close()
        pool.join()
        # self.loggerinfo.info('finish parsing list page on %s' % str(datetime.datetime.now()))

    def process_list_page(self, resp):
        video_lst = []
        soup = BeautifulSoup(resp, 'html.parser')
        channel = soup.find('div', {'class': 'filter_list'}).find('a',
                                                                  {'class': 'filter_item current open'}).text
        midstep = soup.find_all('li', {'class': 'list_item'})
        for line in midstep:
            video_dic = {}
            url = line.a['href']
            find_play_count = BeautifulSoup(list(line)[-2], 'html.parser')
            play_count_str = find_play_count.find('span', {'class': 'num'}).text.replace(' ', '')
            try:
                play_count = trans_play_count(play_count_str)
            except:
                play_count = 0
            video_dic = {"url": url,
                         "play_count": play_count,
                         "channel": channel}
            video_lst.append(video_dic)
        return video_lst

    async def asynchronous_get_video_page(self, session, data_dic):
        channel = data_dic['channel']
        url = data_dic['url']
        play_count = data_dic['play_count']
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return (channel + 'fangyuchenggoalkeeper' + str(play_count) +
                'fangyuchenggoalkeeper' + url + 'fangyuchenggoalkeeper' + page)

    async def get_video_page(self, loop, task_lst):
        async with aiohttp.ClientSession() as sess_video_page:
            task_video_page = [loop.create_task(self.asynchronous_get_video_page(sess_video_page,
                                                                                 data_dic)) for data_dic in task_lst]
            video_result, unfinished = await asyncio.wait(task_video_page)
            video_page_download_result_lst = [v.result() for v in video_result]
            connect_with_redis.push_video_page_html_to_redis(platform=self.platform,
                                                             result_lst=video_page_download_result_lst)

    def process_video_page_helper(self, soup):
        try:
            title = soup.find('h1', {'class': 'video_title _video_title'}).text
            title = title.replace('\n', '')
            title = title.replace('\t', '')
            title = re.sub('(^\s+)|(\s+$)', '', title)
        except:
            try:
                title = soup.find('h1', {'class': 'video_title'}).text
                title = title.replace('\n', '')
                title = title.replace('\t', '')
                title = re.sub('(^\s+)|(\s+$)', '', title)
            except:
                title = None
        try:
            video_intro = soup.find('meta', {'itemprop': 'description'})['content']
        except:
            video_intro = None
        try:
            release_time_str = soup.find('span', {'class': 'tag_item'}).text
            re_lst = re.findall('\d+', release_time_str)
            release_time_raw = re_lst[0] + '-' + re_lst[1] + '-' + re_lst[2]
            release_time_ts = int(datetime.datetime.strptime(release_time_raw, '%Y-%m-%d').timestamp() * 1e3)
        except:
            release_time_ts = None
        try:
            duration_str = soup.find('meta', {'itemprop': 'duration'})['content']
            duration_str = duration_str.replace('PT', ':').replace('H', ':').replace('S', ':')
            duration = trans_duration(duration_str)
        except:
            duration = None
        return {"title": title,
                "video_intro": video_intro,
                "release_time": release_time_ts,
                "duration": duration}

    def process_video_page(self, resp_str):
        video_dict = {}
        resp_lst = resp_str.split('fangyuchenggoalkeeper')
        channel = resp_lst[0]
        play_count = int(resp_lst[1])
        url = resp_lst[2]
        page = resp_lst[3]
        soup = BeautifulSoup(page, 'html.parser')
        try:
            soup_find = soup.find("script", {"r-notemplate": "true"})
            midstep = soup_find.text
            video_info_var_Lst = re.findall('var\s+VIDEO_INFO\s+=\s*{.+}', midstep)
            video_info_var = video_info_var_Lst[0]
            video_info_json = re.sub('var\s+VIDEO_INFO\s+=\s*', '', video_info_var)
            video_info_dict = json.loads(video_info_json)
            if video_info_dict != {}:
                try:
                    duration_str = video_info_dict['duration']
                    duration = int(duration_str)
                except:
                    duration = None
                try:
                    title = video_info_dict['title']
                except:
                    title = None
                try:
                    release_time_str = video_info_dict['video_checkup_time']
                    release_time_ts = int(datetime.datetime.strptime(release_time_str,
                                                                     '%Y-%m-%d %H:%M:%S').timestamp() * 1e3)
                except:
                    release_time_ts = None
            else:
                video_dict = self.process_video_page_helper(soup)
        except:
            video_dict = self.process_video_page_helper(soup)
        try:
            releaser = soup.find('span', {'class': 'user_name'}).text
        except:
            releaser = None
            releaserUrl = None
        else:
            try:
                releaserUrl = soup.find('a', {'class': 'user_info'})['href']
            except:
                releaserUrl = None
        try:
            video_intro = soup.find('meta', {'itemprop': 'description'})['content']
        except:
            video_intro = None
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
        if video_dict == {}:
            video_dict['title'] = title
            video_dict['video_intro'] = video_intro
            video_dict['duration'] = duration
            video_dict['release_time'] = release_time_ts
        video_dict['channel'] = channel
        video_dict['platform'] = self.platform
        video_dict['url'] = url
        video_dict['releaser'] = releaser
        video_dict['play_count'] = play_count
        video_dict['fetch_time'] = fetch_time
        video_dict['releaserUrl'] = releaserUrl
        video_dict['data_from'] = 'list_page'
        return video_dict

    def download_video_page_async_single_process(self):

        """get video page asynchronously in single process"""

        key = 'v_qq_url_dict'
        while connect_with_redis.length_of_set(key) > 0:
            task_lst = connect_with_redis.retrieve_url_dict_from_redis_set(platform=self.platform)
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.get_video_page(loop, task_lst=task_lst))

    def download_video_page_async_multi_process(self,
                                                process_num=10):
        pool = Pool(processes=process_num)
        for line in range(process_num):
            pool.apply_async(self.download_video_page_async_single_process)
        pool.close()
        pool.join()
        # self.loggerinfo.info('finish downloading video page on %s' % str(datetime.datetime.now()))

    def parse_video_page_single_process(self,
                                        output_to_file=False,
                                        filepath=None,
                                        push_to_redis=False,
                                        output_to_es_raw=True,
                                        es_index="crawler-data-raw",
                                        doc_type="doc",
                                        output_to_es_register=False, ):
        """
        parse download video page html in single process
        """
        result_lst = []
        count = 0
        pid = os.getpid()
        while connect_with_redis.length_of_lst(key='v_qq_video_page_html') > 0:
            resp_str = connect_with_redis.retrieve_video_page_html_from_redis(platform=self.platform)
            video_dic = self.process_video_page(resp_str=resp_str)
            if video_dic is not None:
                result_lst.append(video_dic)
                count += 1
                print("platform: %s, action: parse video page, process_id: %s, count number: %s"
                      % (self.platform, pid, count))
                if len(result_lst) >= 1000:
                    output_result(result_Lst=result_lst,
                                  platform=self.platform,
                                  output_to_file=output_to_file,
                                  push_to_redis=push_to_redis,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  es_index=es_index,
                                  doc_type=doc_type)
                    result_lst.clear()
        if result_lst != []:
            output_result(result_Lst=result_lst,
                          platform=self.platform,
                          output_to_file=output_to_file,
                          push_to_redis=push_to_redis,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          es_index=es_index,
                          doc_type=doc_type)

    def parse_video_page_multi_process(self,
                                       para_dict,
                                       process_num=30):
        pool = Pool(processes=process_num)
        for line in range(process_num):
            pool.apply_async(self.parse_video_page_single_process, kwds=para_dict)
        pool.close()
        pool.join()
        # self.loggerinfo.info('finish parsing video page on %s' % str(datetime.datetime.now()))

    # renew video's play_count
    # this is for asynchronous video page crawler in the future due to its input and output
    async def get_video_page_html(self, session, url):
        """
        input video page's url to get url and video page html
        """
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return (url + 'fangyuchenggoalkeeper' + page)

    async def renew_video_play_count(self, loop, task_lst):
        async with aiohttp.ClientSession() as session:
            task_video_page = [loop.create_task(self.get_video_page_html(session,
                                                                         url)) for url in task_lst]
            result, unfinished = await asyncio.wait(task_video_page)
            html_lst = [v.result() for v in result]
            connect_with_redis.push_video_page_html_to_redis_renew(platform=self.platform,
                                                                   result_lst=html_lst)
            print("the length of url list is %s" % len(html_lst))

    def run_renew_video_play_count(self):
        """
        renew video play_count
        """
        task_lst = connect_with_redis.retrieve_video_url_from_redis_set(platform=self.platform)
        loop = asyncio.get_event_loop()
        loop.run_until_complete(self.renew_video_play_count(loop, task_lst=task_lst))

    def run_renew_play_count_single_process(self):
        lst_key = connect_with_redis.platform_redis_set_reg[self.platform]
        while connect_with_redis.length_of_set(lst_key) > 0:
            self.run_renew_video_play_count()

    def run_renew_play_count_multi_process(self,
                                           process_num=10):
        pool = Pool(processes=process_num)
        for line in range(process_num):
            pool.apply_async(self.run_renew_play_count_single_process)
        pool.close()
        pool.join()

    def parse_video_html_to_renew_play_count(self, resp_str):
        video_dict = {}
        resp_lst = resp_str.split('fangyuchenggoalkeeper')
        url = resp_lst[0]
        page = resp_lst[1]
        soup = BeautifulSoup(page, 'html.parser')
        try:
            soup_find = soup.find("script", {"r-notemplate": "true"})
            midstep = soup_find.text
            video_info_var_Lst = re.findall('var\s+VIDEO_INFO\s+=\s*{.+}', midstep)
            video_info_var = video_info_var_Lst[0]
            video_info_json = re.sub('var\s+VIDEO_INFO\s+=\s*', '', video_info_var)
            video_info_dict = json.loads(video_info_json)
            duration_str = video_info_dict['duration']
            duration = int(duration_str)
            title = video_info_dict['title']
            play_count = video_info_dict['view_all_count']
            data_source = 'video_info'
            release_time_str = video_info_dict['video_checkup_time']
            release_time_ts = int(datetime.datetime.strptime(release_time_str,
                                                             '%Y-%m-%d %H:%M:%S').timestamp() * 1e3)
        except:
            video_dict = self.process_video_page_helper(soup)
            try:
                play_count_str = soup.find("meta", {"itemprop": "interactionCount"})["content"]
                play_count = int(play_count_str)
            except:
                play_count = 0
            data_source = 'interactioncount'
        try:
            releaser = soup.find('span', {'class': 'user_name'}).text
        except:
            releaser = None
            releaserUrl = None
        else:
            try:
                releaserUrl = soup.find('a', {'class': 'user_info'})['href']
            except:
                releaserUrl = None
        try:
            video_intro = soup.find('meta', {'itemprop': 'description'})['content']
        except:
            video_intro = None
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
        if video_dict == {}:
            video_dict['title'] = title
            video_dict['video_intro'] = video_intro
            video_dict['duration'] = duration
            video_dict['release_time'] = release_time_ts
        video_dict['platform'] = self.platform
        video_dict['url'] = url
        video_dict['releaser'] = releaser
        video_dict['play_count'] = play_count
        video_dict['fetch_time'] = fetch_time
        video_dict['releaserUrl'] = releaserUrl
        video_dict['data_source'] = data_source
        video_dict['data_from'] = 'video_page'
        return video_dict

    def func_get_comment_count(self, url):
        headers = {
                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "accept-encoding": "gzip, deflate, br",
                "accept-language": "zh,zh-CN;q=0.9",
                "cache-control": "max-age=0",
                # "cookie": 'pgv_pvi=203414528; RK=SCQYJhGMVf; ptcz=5f0818b08a7345580a07bce669e0f0468b64107f4ecfb2c9bebf109cb23cf4fb; pgv_pvid=2754880744; ts_uid=176985184; tvfe_boss_uuid=54e907210062ff55; video_guid=0df27917cdb73abd; video_platform=2; XWINDEXGREY=0; mobileUV=1_16ac3c085a7_484c1; tvfe_search_uid=acc18029-4786-42c4-8f6a-f308777454bc; Qs_lvt_311470=1562066061; Qs_pv_311470=992309958717814400; _ga=GA1.2.1184421010.1562066062; login_remember=qq; ptui_loginuin=593516104; o_cookie=593516104; pac_uid=1_593516104; pgv_info=ssid=s8002196895; ied_qq=o0593516104; bucket_id=9231005; ts_refer=cn.bing.com/; ptag=cn_bing_com|channel; ts_last=v.qq.com/x/cover/mzc00200o70fhrw/r3047j4iuak.html; ad_play_index=83',
                "referer": "https://v.qq.com/channel/tech",
                "sec-fetch-mode": "navigate",
                "sec-fetch-site": "none",
                "sec-fetch-user": "?1",
                "upgrade-insecure-requests": "1",
                "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36",
        }
        try:
            get_page = retry_get_url(url, headers=headers,
                                     timeout=self.timeout)
            try:
                comment_count = re.findall("(\d+)热评", get_page.text)[0]
            except:
                comment_count = re.findall("(\d+)条热评", get_page.text)[0]
            return int(comment_count)
        except:
            return 0

    def parse_video_page_to_renew_play_count_single_process(self,
                                                            output_to_file=False,
                                                            filepath=None,
                                                            output_to_es_raw=True,
                                                            output_to_es_register=False,
                                                            es_index="crawler-data-raw",
                                                            doc_type="doc"):
        """
        parse download video page html in single process
        """
        print(output_to_file, filepath,
              output_to_es_raw, output_to_es_register,
              es_index, doc_type)
        result_lst = []
        count = 0
        pid = os.getpid()
        redis_key = connect_with_redis.platform_redis_lst_reg[self.platform]
        while connect_with_redis.length_of_lst(key=redis_key) > 0:
            resp_str = connect_with_redis.retrieve_video_html_from_redis_renew(self.platform)
            video_dic = self.parse_video_html_to_renew_play_count(resp_str)
            if video_dic is not None:
                result_lst.append(video_dic)
                count += 1
                print("the count number of process %s is %s" % (pid, count))
                if len(result_lst) >= 1000:
                    output_result(result_Lst=result_lst,
                                  platform=self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  es_index=es_index,
                                  doc_type=doc_type)
                    result_lst.clear()
        if result_lst != []:
            output_result(result_Lst=result_lst,
                          platform=self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          es_index=es_index,
                          doc_type=doc_type)

    #    def parse_video_page_to_renew_play_count_multiprocess(self,
    #                                                          output_to_file=False,
    #                                                          filepath=None,
    #                                                          output_to_es_raw=True,
    #                                                          output_to_es_register=False,
    #                                                          es_index="crawler-data-raw",
    #                                                          doc_type="doc",
    #                                                          process_num=30):
    #        print("start to parse video page html in multiprocessing")
    #        PARA_DIC = {'output_to_file': output_to_file,
    #                    'filepath': filepath,=None,
    #                                                          output_to_es_raw=True,
    #                                                          output_to_es_register=False,
    #                                                          es_index="crawler-data-raw",
    #                                                          doc_type="doc"
    #        pool = Pool(processes=process_num)
    #        for line in range(process_num):
    #            pool.apply_async(self.parse_video_page_to_renew_play_count_single_process, kwds=para_dic)
    #        pool.close()
    #        pool.join()

    # releaser_page asynchronous crawler
    def releaser_page_by_time(self, start_time, end_time, url,**kwargs):
        data_lis = []
        count_false = 0
        output_to_file = kwargs.get("output_to_file")
        filepath = kwargs.get("filepath")
        push_to_redis = kwargs.get("push_to_redis")
        output_to_es_register = kwargs.get("output_to_es_register")
        output_to_es_raw = kwargs.get("output_to_es_raw")
        es_index = kwargs.get("es_index")
        doc_type = kwargs.get("doc_type")
        for res in self.releaser_page(url,proxies_num=kwargs.get("proxies_num")):
            video_time = res["release_time"]
            # print(res)
            if video_time:
                if start_time < video_time:
                    if video_time < end_time:
                        data_lis.append(res)

                        if len(data_lis) >= 100:
                            output_result(result_Lst=data_lis,
                                          platform=self.platform,
                                          output_to_file=output_to_file,
                                          filepath=filepath,
                                          push_to_redis=push_to_redis,
                                          output_to_es_register=output_to_es_register,
                                          output_to_es_raw=output_to_es_raw,
                                          es_index=es_index,
                                          doc_type=doc_type)
                            data_lis.clear()

                else:
                    count_false += 1
                    if count_false > 10:
                        break
                    else:
                        continue

        if data_lis != []:
            output_result(result_Lst=data_lis,
                          platform=self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          push_to_redis=push_to_redis,
                          output_to_es_register=output_to_es_register,
                          output_to_es_raw=output_to_es_raw,
                          es_index=es_index)


if __name__ == '__main__':
    test = Crawler_v_qq()
    url = 'http://v.qq.com/vplus/e03b859272d662175d9ec4604703c74d'
    # releaserUrl = 'http://v.qq.com/vplus/cfa34d96d1b6609f1dccdea65b26b83d'
    # nnn = test.video_page(url)
    # kw = '任正非 BBC'
    # #sr = test.search_page(kw, search_pages_max=2)
    res = test.releaser_page(url, output_to_es_raw=True,
                        es_index='crawler-data-raw',
                          doc_type='doc',
                         releaser_page_num_max=400)
    for re in res:
        continue
    # releaserUrl=url,)
    #test.get_releaser_follower_num("http://v.qq.com/vplus/cfa34d96d1b6609f1dccdea65b26b83d")
    test.releaser_page_by_time(1558972800000, 1562603029917, releaserUrl, output_to_es_raw=True,
                                      es_index='crawler-data-raw',
                                      doc_type='doc', releaser_page_num_max=4000)
    #test.get_releaser_image(releaserUrl)