crawler_youku.py

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 14:19:00 2018

@author: fangyucheng
"""

import os
import time
import aiohttp
import asyncio
import requests
from bs4 import BeautifulSoup
import datetime
import re
import json
import copy
import random
# from selenium import webdriver
from multiprocessing import Pool
from multiprocessing import Process
from crawler_sys.framework.video_fields_std import Std_fields_video
from crawler_sys.utils.output_results import retry_get_url
from crawler_sys.utils.output_results import output_result
from crawler_sys.utils import trans_strtime_to_timestamp
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.util_logging import logged


class Crawler_youku():

    def __init__(self, timeout=None, platform='youku'):
        if timeout==None:
            self.timeout = 10
        else:
            self.timeout = timeout
        self.platform = platform
        std_fields = Std_fields_video()
        self.video_data = std_fields.video_data
        self.video_data['platform'] = self.platform
        # remove fields that crawled data don't have
        pop_key_Lst = ['channel', 'describe', 'repost_count', 'isOriginal',
                       'video_id']
        for popk in pop_key_Lst:
            self.video_data.pop(popk)

        # with channel == '全部', url field is hard coded in list_page_all method,
        # input url will be ignored
        self.legal_channels = ['全部']
        self.list_page_url_dict = {
            '全部': '_',
            }

    def video_page(self, url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
        releaser = releaser.replace('\n', '')
        releaser = releaser.replace(' ', '')
        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
        if releaserUrl[0:4] == 'http':
            pass
        else:
            releaserUrl = 'http:' + releaserUrl
        try:
            rt_str = soup.find('span', {'class':'video-status'}).text
            rt_lst = re.findall('\d+', rt_str)
            rt_to_timestamp = str(rt_lst[0]) + '-' + str(rt_lst[1]) + '-' + str(rt_lst[2])
            release_time = int(datetime.datetime.strptime(rt_to_timestamp, '%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        D0 = {'releaser':releaser,'releaserUrl':releaserUrl,'release_time':release_time}
        return D0

    def get_video_infor_by_network(self,url_Lst):
        video_id_Lst = []
        for url in url_Lst:
            video_id = ' '.join(re.findall('id.*html',url)).split('_')[1].split('.')[0]
            video_id_Lst.append(video_id)


    def get_video_title_play_count_duration(self, url):
        video_id = ' '.join(re.findall('id.*html', url))
        mobile_url = 'https://m.youku.com/video/' + video_id
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        title = browser.find_element_by_class_name('online').text
        dura = browser.find_element_by_class_name('v-label').text.split('\n')[1]
        dura_lst = ' '.join(dura).split[':']
        if len(dura_lst) == 2:
            duration = dura_lst[0]*60 + dura_lst[1]
        elif len(dura_lst) == 3:
            duration = dura_lst[0]*3600 + dura_lst[1]*60 + dura_lst[2]
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+',pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'duration': duration,
              'play_count': play_count,
              'url': url,
              'fetch_time': fetch_time}
        browser.close()
        return D0

    def get_video_play_count_by_vid(self, vid):
        mobile_url = 'https://m.youku.com/video/id_' + vid + '==.html'
        browser = webdriver.Chrome()
        browser.get(mobile_url)
        time.sleep(10)
        title = browser.find_element_by_class_name('online').text
        pc_midstep1 = browser.find_element_by_class_name('video-num')
        pc_midstep2 = pc_midstep1.text
        pc_midstep3 = pc_midstep2.replace(',','')
        pc_midstep4 = re.findall('[0-9]+', pc_midstep3)
        if '万' in pc_midstep2:
            play_count = pc_midstep4[0]*1e4+pc_midstep4[1]*1e3
        else:
            play_count = ' '.join(pc_midstep4)
        browser.close()
        D0 = {'title': title,
              'play_count': play_count}
        return D0

    def get_video_title_releaser_release_time(self, url):
        """An example of selenium
        """
        video_id = ' '.join(re.findall('id.*html', url))
        browser = webdriver.Chrome()
        browser.get(url)
        title = browser.find_element_by_id('subtitle').text
        releaser = browser.find_element_by_id('module_basic_sub').text
        releaser = releaser.replace('+订阅','')
        releaser = releaser.replace(' ','')
        try:
            rt_midstep = browser.find_element_by_class_name('video-status').text
            rt_midstep = rt_midstep.replace('上传于','')
            rt_midstep = rt_midstep.replace(' ','')
            release_time = int(datetime.datetime.strptime(rt_midstep,'%Y-%m-%d').timestamp()*1e3)
        except:
            release_time = 0
        fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
        D0 = {'video_id': video_id,
              'title': title,
              'release_time': release_time,
              'url': url,
              'fetch_time': fetch_time}
        return D0

    def list_page_sync(self, list_page_url, channel=None,
                  output_to_file=False, filepath=None,
                  output_to_es_raw=False,
                  output_to_es_register=False,
                  push_to_redis=False,
                  page_num_max=30):
        if channel == '全部':
            self.list_page_all(output_to_file=output_to_file, filepath=filepath,
                               output_to_es_raw=output_to_es_raw,
                               output_to_es_register=output_to_es_register,
                               push_to_redis=push_to_redis,
                               page_num_max=page_num_max)
        else:
            # need to be tested
            get_page = requests.get(list_page_url)
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            video = soup.find_all('div',{'class':'v-link'})
            dict_url = []
            for ttt in video:
                midstepurl=ttt.find('a')['href']
                if ('https:' in midstepurl):
                    midstepurl=midstepurl[6:]
                url = 'http:'+midstepurl
                dict_url.append(url)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            for xxxx in dict_url:
                if ("id" not in xxxx):
                    num=dict_url.index(xxxx)
                    dict_url.pop(num)
            list_page_Lst = []
            for videourl in dict_url:
                one_video_dic = self.video_page(videourl)
                print ('one page done')
                list_page_Lst.append(one_video_dic)
            return list_page_Lst


    def search_page(self, keyword, search_pages_max=10,
                    output_to_es_raw=False,
                    output_to_es_register=False,
                    es_index=None,
                    doc_type=None):
        def process_one_line(data_line):
            title = data_line.find("div", {"class" : "v-meta-title"}).a['title']
            url = data_line.find("div", {"class" : "v-meta-title"}).a['href']
            releaser = data_line.find("span", {"class" : "username"}).text
            releaser = releaser.strip()
            releaserUrl = data_line.find("span", {"class" : "username"}).a['href']
            releaserUrl = 'http:' + releaserUrl
            play_count = data_line.find("span", {"class" : "pub"}).text
            dura = data_line.find("span", {"class" : "v-time"}).text
            duration_str = dura
            dl = duration_str.split(':')
            dl_int = []
            for v in dl:
                v = int(v)
                dl_int.append(v)
            if len(dl_int) == 2:
                duration = dl_int[0]*60+dl_int[1]
            else:
                duration = dl_int[0]*3660 + dl_int[1]*60+dl_int[2]
            videourl = 'https:' + url
            dict_page = self.video_page(videourl)
            release_time = dict_page['release_time']
            D0 = copy.deepcopy(self.video_data)
            D0['title'] = title
            D0['url'] = videourl
            D0['duration'] = duration
            D0['releaser'] = releaser
            D0['release_time'] = release_time
            D0['releaserUrl'] = releaserUrl
            D0['play_count'] = play_count

            return D0

        search_data_Lst = []
        jsurl = ('http://www.soku.com/search_video_ajax/q_'
                 + keyword
                 + '_orderby_1_limitdate_0?site=14&page={}'.format(
                     str(i)) for i in range (2, 4))
        for urls in jsurl:
            print(urls)
            get_page = requests.get(urls)
            print('get jspage done')
            page = get_page.text
            soup = BeautifulSoup(page,'html.parser')
            youku = soup.find_all("div", { "class" : "v" })
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
        url=('http://www.soku.com/search_video/q_'
             + keyword
             + '_orderby_1_limitdate_0?spm=a2h0k.8191407.0.0&site=14&page={}'.format(
                 str(i)) for i in range(1, search_pages_max+1))
        for urls in url:
            print(urls)
            get_page = requests.get(urls)
            print('get page done')
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            youku = soup.find_all("div", {"class" : "v"})
            for data_line in youku:
                one_video_dic = process_one_line(data_line)
                search_data_Lst.append(one_video_dic)
                if len(search_data_Lst) >= 100:
                    output_result(result_Lst=search_data_Lst,
                                  platform=self.platform,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  es_index=es_index,
                                  doc_type=doc_type)
                    search_data_Lst.clear()

        if search_data_Lst != []:
            output_result(result_Lst=search_data_Lst,
                          platform=self.platform,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          es_index=es_index,
                          doc_type=doc_type)

        return search_data_Lst


    def get_releaser_detail(self, url):
        releaser_data_Lst = []
        get_page = requests.get(url)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser=soup.find('a',{'class':'username'})['title']
        releaser_intro=soup.find('span',{'class':'desc'}).text
        total_play_count=soup.find('li',{'class':'vnum'})['title']
        fans=soup.find('li',{'class':'snum'})['title']
        midstep=soup.find_all('div',{'class':'title'})
        for brother in midstep:
            if brother.h3.text=='视频':
                num=brother.find('span').text
                self.TotalVideo_num=num[1:-1]
        D_releaser_page={"releaser":releaser,"releaser_intro":releaser_intro,"total_play_count":total_play_count,"follow":fans,"TotalVideo_num":self.TotalVideo_num}
        releaser_data_Lst.append(D_releaser_page)
        return D_releaser_page


    def get_releaser_info(self, releaserUrl):
        get_page = retry_get_url(releaserUrl)
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        releaser = soup.find('a',{'class':'username'})['title']
        releaser_id = soup.find('a',{'class':'user-avatar'})['href']
        #total_video_num = soup.find('div',{'class':'title'}).text
        D0 = {'releaser':releaser, 'releaser_id':releaser_id}
        return D0

    @logged
    def releaser_page(self, releaserUrl, error_file=None,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_es_index=None,
                      output_doc_type=None,
                      output_to_es_register=False,
                      push_to_redis=False,
                      releaser_page_num_max=300):
        result_Lst = []
        releaser_info = self.get_releaser_info(releaserUrl)
        releaser = releaser_info['releaser']
        releaser_id = releaser_info['releaser_id']
        count = 1
        while count <= releaser_page_num_max:
            releaser_url = 'https://i.youku.com' + releaser_id + '/videos?&order=1&page=' + str(count)
            count += 1
            get_page = retry_get_url(releaser_url)
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_lst = soup.find_all("div", { "class" : "v va" })
            for one_video in video_lst:
                try:
                    title = one_video.find("div",{"class":"v-meta-title"}).a['title']
                    url = one_video.find('div',{'class':'v-meta-title'}).a['href']
                    if url[0:4] != 'http':
                        url = 'https:' + url
                    try:
                        play_count_text = one_video.find("span",{"class":"v-num"}).text
                        if '万' in play_count_text:
                            play_count_str = play_count_text.replace('万', '')
                            play_count = int(float(play_count_str)*1e4)
                        else:
                            play_count = int(play_count_text.replace(',', ''))
                    except:
                        play_count = 0
                    dura_str = one_video.find("span",{"class":"v-time"}).text
                    dura_lst = dura_str.split(':')
                    if len(dura_lst) == 2:
                        duration = int(dura_lst[0])*60 + int(dura_lst[1])
                    elif len(dura_lst) == 3:
                        duration = int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2])
                    else:
                        duration = 0
                    rt_str = one_video.find('span',{'class':'v-publishtime'}).text
                    release_time = trans_strtime_to_timestamp.trans_strtime_to_timestamp(rt_str, missing_year=True)
                    fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                    self.video_data['title'] = title
                    self.video_data['duration'] = duration
                    self.video_data['url'] = url
                    self.video_data['play_count'] = play_count
                    self.video_data['releaser'] = releaser
                    self.video_data['releaserUrl'] = releaserUrl
                    self.video_data['release_time'] = release_time
                    self.video_data['fetch_time'] = fetch_time

                    get_data = copy.deepcopy(self.video_data)
                    result_Lst.append(get_data)
                    print('get_one_line %s' % url)
                except:
                    print('failed to get data')
                    pass
                if len(result_Lst)%100 == 0:
                    if output_doc_type is None and output_es_index is None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis)
                        result_Lst.clear()

                    elif output_doc_type is not None and output_es_index is not None:
                        output_result(result_Lst,
                                      self.platform,
                                      output_to_file=output_to_file,
                                      filepath=filepath,
                                      output_to_es_raw=output_to_es_raw,
                                      output_to_es_register=output_to_es_register,
                                      push_to_redis=push_to_redis,
                                      es_index=output_es_index,
                                      doc_type=output_doc_type)
                        result_Lst.clear()

        if result_Lst!=[]:
            if output_doc_type is None and output_es_index is None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis)
            elif output_doc_type is not None and output_es_index is not None:
                output_result(result_Lst,
                              self.platform,
                              output_to_file=output_to_file,
                              filepath=filepath,
                              output_to_es_raw=output_to_es_raw,
                              output_to_es_register=output_to_es_register,
                              push_to_redis=push_to_redis,
                              es_index=output_es_index,
                              doc_type=output_doc_type)


    def doc_list(self, resultfile,donefile,doneurl,errorfile,area):
        list_data_lst=[]
        result=open(resultfile,'a')
        done=open(donefile,'a')
        donepage=open(doneurl,'a')
        error=open(errorfile,'a')
        count=1
        video_count=30
        while video_count==30 and count<=30:
            url='http://list.youku.com/category/show/c_84_a_'+area+'_s_1_d_1_p_'+str(count)+'.html'
            get_page=requests.get(url)
            get_page.encoding='utf-8'
            page = get_page.text
            print(url)
            soup = BeautifulSoup(page,'html.parser')
            midstep = soup.find_all('div',{'class':'p-thumb'})
            video_count=len(midstep)
            done.write(area)
            done.write(str(count))
            done.write('\n')
            done.flush()
            count+=1
            for line in midstep:
                url_step=line.a['href']
                if url_step[:4]!='http':
                    url='http:'+url_step
                else:
                    url=url_step
                try:
                    get_page=requests.get(url)
                    get_page.encoding='utf-8'
                    page = get_page.text
                    soup = BeautifulSoup(page,'html.parser')
                    print(url)
                    try:
                        donepage.write(url)
                        donepage.write('\n')
                        donepage.flush()
                        album_name=soup.find('h2').a.text
                        album_rating=soup.find('span',{'class':'score'}).text
                        sumvideo=soup.find('p',{'class':'tvintro'}).text
                        sumvideo_int=int(re.findall('\d+',sumvideo)[0])
                        if sumvideo_int<=100:
                            midstep=soup.find('div',{'id':'listitem_page1'})
                            first_video=midstep.find('div',{'class':'item item-cover current'})
                            url='http:'+first_video.a['href']
                            title=first_video['title']
                            dura=first_video.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            playcount_str=first_video.find('div',{'class':'status'}).text
                            playcount_str=playcount_str.replace('次播放','')
                            playcount_str=playcount_str.replace(',','')
                            thousand_10='万'
                            if thousand_10 in playcount_str:
                                playcount_str=playcount_str.replace('万','')
                                play_count=int((float(playcount_str))*1e4)
                            else:
                                play_count=int(playcount_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                            D1={'url':url,'video_infor':D0}
                            list_data_lst.append(D1)
                            json_D1=json.dumps(D1)
                            result.write(json_D1)
                            result.write('\n')
                            result.flush()
                            print('get one line')
                            other_video=midstep.find_all('div',{'class':'item item-cover'})
                            for line in other_video:
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                'play_count':play_count,'album_name':album_name,
                                'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                print('get one line')
                            result.flush()
                        if sumvideo_int>100:
                            if sumvideo_int%50==0:
                                page_num=int((sumvideo_int/50))
                            else:
                                page_num=int((sumvideo_int)/50)+1
                            videoid=re.findall('\d+',(re.findall('videoId:"[0-9]{1,10}"',page)[0]))[0]
                            showid=re.findall('\d+',(re.findall('showid:"[0-9]{1,10}"',page)[0]))[0]
                            url_lst=['http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoid+
                                     '&fid=0&showid='+showid+'&sid=0&page={}&callback=tuijsonp23'
                                     .format(str(i)) for i in range(1,page_num+1)]
                            for line in url_lst:
                                get_page = requests.get(line)
                                get_page.encoding = 'utf-8'
                                page = get_page.text
                                page_dic = eval(page[34:-2])
                                videolab = page_dic['html']
                                videolab = videolab.replace('\\','')
                                soup = BeautifulSoup(videolab,'html.parser')
                                first_video=soup.find('div',{'class':'item item-cover current'})
                                url='http:'+first_video.a['href']
                                title=first_video['title']
                                dura=first_video.find('span',{'class':'c-time'}).text
                                dura_lst=dura.split(':')
                                if len(dura_lst)==1:
                                    duration=int(dura_lst[0])
                                elif len(dura_lst)==2:
                                    duration=int(dura_lst[0])*60+int(dura_lst[1])
                                elif len(dura_lst)==3:
                                    duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                playcount_str=first_video.find('div',{'class':'status'}).text
                                playcount_str=playcount_str.replace('次播放','')
                                playcount_str=playcount_str.replace(',','')
                                thousand_10='万'
                                if thousand_10 in playcount_str:
                                    playcount_str=playcount_str.replace('万','')
                                    play_count=int((float(playcount_str))*1e4)
                                else:
                                    play_count=int(playcount_str)
                                fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                D0={'url':url,'tilte':title,'duartion':duration,
                                    'play_count':play_count,'album_name':album_name,
                                    'album_rating':album_rating,'fetch_time':fetch_time}
                                D1={'url':url,'video_infor':D0}
                                list_data_lst.append(D1)
                                json_D1=json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print('get one line')
                                other_video=midstep.find_all('div',{'class':'item item-cover'})
                                for line in other_video:
                                    url='http:'+first_video.a['href']
                                    title=first_video['title']
                                    dura=first_video.find('span',{'class':'c-time'}).text
                                    dura_lst=dura.split(':')
                                    if len(dura_lst)==1:
                                        duration=int(dura_lst[0])
                                    elif len(dura_lst)==2:
                                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                                    elif len(dura_lst)==3:
                                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                                    playcount_str=first_video.find('div',{'class':'status'}).text
                                    playcount_str=playcount_str.replace('次播放','')
                                    playcount_str=playcount_str.replace(',','')
                                    thousand_10='万'
                                    if thousand_10 in playcount_str:
                                        playcount_str=playcount_str.replace('万','')
                                        play_count=int((float(playcount_str))*1e4)
                                    else:
                                        play_count=int(playcount_str)
                                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                                    D0={'url':url,'tilte':title,'duartion':duration,'play_count':play_count,
                                        'album_name':album_name,'album_rating':album_rating,'fetch_time':fetch_time}
                                    D1={'url':url,'video_infor':D0}
                                    list_data_lst.append(D1)
                                    json_D1=json.dumps(D1)
                                    result.write(json_D1)
                                    result.write('\n')
                                    print('get one line')
                                result.flush()
                    except:
                        error.write(url)
                        error.write('\n')
                        error.flush()
                        print('there is an error')
                except:
                    error.write(url)
                    error.write('\n')
                    error.flush()
                    print('there is an error')
        done.close()
        result.close()
        return list_data_lst

    def add_infor(self,taskname,resultname):
        task=open(taskname,'a')
        result=open(resultname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic=eval(line)
            video_infor=line_dic['video_infor']
            task_lst.append(video_infor)
        for line in task_lst:
            url=line['url']
            try:
                video_dic=self.video_page(url)
                line['releaser']=video_dic['releaser']
                line['release_time']=video_dic['release_time']
                line['releaserUrl']=video_dic['releaserUrl']
                line['tags']=video_dic['tags']
                json_line=json.dumps(line)
                result_lst.append(line)
                result.write(json_line)
                result.write('\n')
                result.flush()
            except:
                print('no video infor')
        result.close()
        return result_lst

    def get_doc(self,cluename,taskname,resultname,donename):
        task=open(taskname)
        result=open(resultname,'a')
        clue=open(cluename,'a')
        done=open(donename,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            album_name=line['album_name']
            album_rating=line['album_rating']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                videoId = (" ".join(re.findall('videoId: &#\d+;\d+',page))).split(';')[1]
                videoCategoryId = (" ".join(re.findall('videoCategoryId: &#\d+;\d+',page))).split(';')[1]
                componentid = (" ".join(re.findall('"componentId":\d+',page))).split(':')[1]
                D0={'showid':showid ,'videoid':videoId,'videocategoryid':videoCategoryId,'componentid':componentid}
                json_D0 = json.dumps(D0)
                clue.write(json_D0)
                clue.write('\n')
                clue.flush()
                count=1
                thousand_10='万'
                url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                get_page = requests.get(url)
                get_page.encoding = 'utf-8'
                page = get_page.text
                page_dic = eval(page)
                video_infor = page_dic['html']
                soup=BeautifulSoup(video_infor,'html.parser')
                video_dic = soup.find_all('div',{'class':'item item-cover'})
                sumofvideo = len(video_dic)
                for line in video_dic:
                    title=line['title']
                    url = line.a['href']
                    if 'http:' not in url:
                        url='http:'+url
                    dura=line.find('span',{'class':'c-time'}).text
                    dura_lst=dura.split(':')
                    if len(dura_lst)==1:
                        duration=int(dura_lst[0])
                    elif len(dura_lst)==2:
                        duration=int(dura_lst[0])*60+int(dura_lst[1])
                    elif len(dura_lst)==3:
                        duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                    play_count_str=line.find('div',{'class':'status'}).text
                    play_count_str=play_count_str.replace('次播放','')
                    play_count_str=play_count_str.replace(',','')
                    if thousand_10 in play_count_str:
                        play_count_str=play_count_str.replace('万','')
                        play_count=int((float(play_count_str))*1e4)
                    else:
                        play_count=int(play_count_str)
                    fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                    D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                        'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                    if D1 not in result_lst:
                        result_lst.append(D1)
                        json_D1 = json.dumps(D1)
                        result.write(json_D1)
                        result.write('\n')
                        result.flush()
                        print(url)
                while sumofvideo == 100:
                    count+=1
                    url='http://v.youku.com/page/playlist?&l=debug&pm=3&vid='+videoId+'&fid=0&showid='+showid+'&sid=0&componentid='+componentid+'&videoCategoryId='+videoCategoryId+'&isSimple=false&page='+str(count)
                    get_page = requests.get(url)
                    get_page.encoding = 'utf-8'
                    page = get_page.text
                    page_dic = eval(page)
                    video_infor = page_dic['html']
                    if video_infor =="\n":
                        sumofvideo=0
                    else:
                        soup=BeautifulSoup(video_infor,'html.parser')
                        video_dic = soup.find_all('div',{'class':'item item-cover'})
                        sumofvideo = len(video_dic)
                        for line in video_dic:
                            title=line['title']
                            url = line.a['href']
                            if 'http:' not in url:
                                url='http:'+url
                            dura=line.find('span',{'class':'c-time'}).text
                            dura_lst=dura.split(':')
                            if len(dura_lst)==1:
                                duration=int(dura_lst[0])
                            elif len(dura_lst)==2:
                                duration=int(dura_lst[0])*60+int(dura_lst[1])
                            elif len(dura_lst)==3:
                                duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
                            play_count_str=line.find('div',{'class':'status'}).text
                            play_count_str=play_count_str.replace('次播放','')
                            play_count_str=play_count_str.replace(',','')
                            if thousand_10 in play_count_str:
                                play_count_str=play_count_str.replace('万','')
                                play_count=int((float(play_count_str))*1e4)
                            else:
                                play_count=int(play_count_str)
                            fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
                            D1={'album_name':album_name,'album_rating':album_rating,'title':title,
                                'duration':duration,'play_count':play_count,'url':url,'fetch_time':fetch_time}
                            if D1 not in result_lst:
                                result_lst.append(D1)
                                json_D1 = json.dumps(D1)
                                result.write(json_D1)
                                result.write('\n')
                                result.flush()
                                print(url)
                done.write(urls)
                done.write('\n')
                done.flush()
            except:
                pass
        done.close()
        task.close()
        result.close()
        clue.close()
        return result_lst


    def one_video(self,taskname):
        task=open(taskname)
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            urls=urls.replace('\n','')
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            try:
                showid = (" ".join(re.findall('showid: &#\d+;\d+',page))).split(';')[1]
                print('more video')
            except:
                result_lst.append(urls)
                print('one video')
        task.close()
        return result_lst



    def get_releaser(self,taskname,resultname,errorname):
        task=open(taskname)
        result=open(resultname,'a')
        error=open(errorname,'a')
        task_lst=[]
        result_lst=[]
        for line in task:
            line_dic = eval(line)
            task_lst.append(line_dic)
        for line in task_lst:
            urls=line['url']
            get_page = requests.get(urls)
            get_page.encoding = 'utf-8'
            page = get_page.text
            page2 = page.replace('\\','')
            try:
                releaser=re.findall("[\u4e00-\u9fa5]+","".join(re.findall('class="" title="[\u4e00-\u9fa5]+"',page2)))[0]
                print('get releaser')
            except:
                releaser=None
                error.write(urls)
                error.write('\n')
                error.flush()
                print('no releaser')
            line['releaser']=releaser
            result_lst.append(line)
            json_line=json.dumps(line)
            result.write(json_line)
            result.write('\n')
            result.flush()
        result.close()
        return result_lst


    def get_release_time(self,url):
        get_page=requests.get(url)
        page=get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            time_one=soup.find('span',{'class':'bold mr3'}).text
        except AttributeError:
            time_one=None
        try:
            time_two=time_one[4:]
        except  TypeError:
            time_two=None
        try:
            midsteptime=time_two.strip()
        except  AttributeError:
            midsteptime=None
        try:
            release_time = int(datetime.datetime.strptime(midsteptime, '%Y-%m-%d').timestamp()*1e3)
        except  (TypeError,ValueError):
            release_time = None
        D_video_page={"release_time":release_time,'url':url}
        print('get one line')
        return D_video_page


    def get_video_play_count_releaser_info(self,url):
        get_page = requests.get(url)
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page,'html.parser')
        try:
            video_info = soup.find('div',{'class':'item item-cover current'})
            play_count_str = video_info.find('div',{'class':'status'}).text
            play_count_str = play_count_str.replace('次播放','')
            if '万' in play_count_str:
                play_count_str = play_count_str.replace('万','')
                play_count = int(float(play_count_str)*1e4)
            else:
                play_count = int(play_count_str.replace(',',''))
        except:
            play_count = 0
        try:
            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
            releaser = releaser.replace('\n','')
            releaser = releaser.replace(' ','')
            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
            if releaserUrl[0:4] == 'http':
                pass
            else:
                releaserUrl = 'http:' + releaserUrl
        except:
            releaser = None
            releaserUrl = None
        D0 = {'play_count':play_count, 'releaser':releaser, 'releaserUrl':releaserUrl}
        return D0


    def get_release_time_from_str(self,rt_str):
        minute = '分钟'
        hour = '小时'
        yesterday = '昨天'
        the_day_before_yesterday = '前天'
        the_day_before_many_days = '天前'
        if minute in rt_str or hour in rt_str :
            year = datetime.date.today().year
            month = datetime.date.today().month
            day = datetime.date.today().day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 1)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_yesterday in rt_str:
            raw_data = datetime.date.today() - datetime.timedelta(days = 2)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        elif the_day_before_many_days in rt_str:
            day_shift = int(re.findall('\d+',rt_str)[0])
            raw_data = datetime.date.today() - datetime.timedelta(days = day_shift)
            year = raw_data.year
            month = raw_data.month
            day = raw_data.day
            connect_sign = '-'
            date = str(year) + connect_sign + str(month) + connect_sign + str(day)
            release_time = int(datetime.datetime.strptime(date,'%Y-%m-%d').timestamp()*1e3)
        else:
            release_time = 0
        return release_time



    def list_page_all(self,
                      output_to_file=False, filepath=None,
                      output_to_es_raw=False,
                      output_to_es_register=False,
                      push_to_redis=False,
                      page_num_max=25):
        """
        Get all videos in youku.com '片库' in the home page bottom and the '全部'
        category with '最新发布' filter.
        At present (2018-06-28) the url is http://list.youku.com/category/video/c_0_d_1_s_2.html?spm=a2h1n.8251846.selectID.5!2~5~5!2~1~3!2~A
        This url changes to the following format
        'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(count) + '.html'
        where count is page number.
        """
        # if page_num_max argument is passed in a value greater than 25, will use 25,
        # at present (2018-06-28) there is only 25 pages at most.
        if page_num_max > 25:
            page_num_max = 25
        result_Lst = []
        page_num = 1
        while page_num <= page_num_max:
            lst_url = 'http://list.youku.com/category/video/c_0_d_1_s_2_p_' + str(page_num) + '.html'
            print('Getting data on page %d: %s, %s'
                  % (page_num, lst_url, datetime.datetime.now()))
            get_page = retry_get_url(lst_url)
            if get_page is None:
                print('Failed to get page url: %s, will continue to next page.'
                      % lst_url)
                continue
            get_page.encoding = 'utf-8'
            page = get_page.text
            soup = BeautifulSoup(page, 'html.parser')
            video_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_agg:
                title = line.find('li', {'class': 'title'}).text
                dura_str = line.find('li', {'class': 'status'}).text
                dura_lst = dura_str.split(':')
                if len(dura_lst) == 3:
                    duration = int(int(dura_lst[0])*3600 + int(dura_lst[1])*60 + int(dura_lst[2]))
                elif len(dura_lst) == 2:
                    duration = int(int(dura_lst[0])*60 + int(dura_lst[1]))
                else:
                    duration = 0
                url = line.a['href']
                if url[0:4] != 'http':
                    url = 'https:' + url
                video_info_dic = self.get_video_play_count_releaser_info(url)
                fetch_time = int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)

                rt_str = line.find('li', {'class': ' '}).text
                release_time = self.get_release_time_from_str(rt_str)

                self.video_data['title'] = title
                self.video_data['duration'] = duration
                self.video_data['url'] = url
                self.video_data['play_count'] = video_info_dic['play_count']
                self.video_data['releaser'] = video_info_dic['releaser']
                self.video_data['releaserUrl'] = video_info_dic['releaserUrl']
                self.video_data['release_time'] = release_time
                self.video_data['fetch_time'] = fetch_time

                get_data = copy.deepcopy(self.video_data)
                result_Lst.append(get_data)

                if len(result_Lst) >= 100:
                    print('Got %d lines, %s' % (len(result_Lst),
                                                datetime.datetime.now()))
                    output_result(result_Lst, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    result_Lst.clear()

            page_num += 1

        if result_Lst != []:
            print('Got %d lines, %s' % (len(result_Lst),
                                        datetime.datetime.now()))
            output_result(result_Lst, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            result_Lst.clear()

    def get_title_subtitle(self, url):
        get_page = retry_get_url(url)
        if get_page is None:
            print('Failed to get url: %s' % url)
            return
        get_page.encoding = 'utf-8'
        page = get_page.text
        soup = BeautifulSoup(page, 'html.parser')
        try:
            subtitle = soup.find('span', {'id': 'subtitle'}).text
        except:
            subtitle = None
        try:
            title = soup.h1.find('a', {'target':'_blank'}).text
            title_str = title + subtitle
        except:
            title_str = subtitle
        return title_str

    def select_tv_series_video(self, sp_video_album_name,
                               result_file=None, # modify to from es
                               output_to_file=False,
                               filepath=None,
                               output_to_es_raw=False,
                               output_to_es_register=False,
                               push_to_redis=False):
        select_result = []
        meta = Meta()
        result_lst = meta.dic_file_to_lst(result_file)
        for line in result_lst:
            url = line['url']
            title_str = self.get_title_subtitle(url)
            if sp_video_album_name in title_str:
                line.update({'title': title_str})
                select_result.append(line)
                print('it_belongs_to_what_we_want')
                if len(select_result) >= 100:
                    print('Got %d lines, %s' % (len(select_result), datetime.datetime.now()))
                    output_result(select_result, self.platform,
                                  output_to_file=output_to_file,
                                  filepath=filepath,
                                  output_to_es_raw=output_to_es_raw,
                                  output_to_es_register=output_to_es_register,
                                  push_to_redis=push_to_redis)
                    select_result.clear()
            else:
                print('not_belongs_to')
        if select_result != []:
            print('Got %d lines, %s' % (len(select_result),datetime.datetime.now()))
            output_result(select_result, self.platform,
                          output_to_file=output_to_file,
                          filepath=filepath,
                          output_to_es_raw=output_to_es_raw,
                          output_to_es_register=output_to_es_register,
                          push_to_redis=push_to_redis)
            select_result.clear()



#list page asynchronous crawler
    def start_list_page(self, task_list,
                        output_to_file=False,
                        filepath=None,
                        output_to_es_raw=False,
                        push_to_redis=False,
                        es_index='crawler-data-raw',
                        doc_type='doc',
                        output_to_es_register=False,):
        self.list_page_task(task_list)
        self.download_list_page_single_process()


    def list_page_task(self, task_list, page_num_max=25):
        TASK_LIST = []
        for line in task_list:
            for num in range(1, page_num_max+1):
                url = line.replace('fangyucheng', str(num))
                TASK_LIST.append(url)
        random.shuffle(TASK_LIST)
        connect_with_redis.push_list_url_to_redis(platform=self.platform,
                                                  result_lst=TASK_LIST)

    async def download_page(self, session, url):
        get_page = await session.get(url)
        page = await get_page.text("utf-8", errors="ignore")
        return page

    async def download_list_page(self, loop):
        task_list = connect_with_redis.retrieve_list_url_from_redis(platform=self.platform)
        async with aiohttp.ClientSession() as session:
            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
            done, pending = await asyncio.wait(task)
            RESULT_LIST = [d.result() for d in done]
            connect_with_redis.push_list_page_html_to_redis(platform=self.platform,
                                                            result_lst=RESULT_LIST)

    def download_list_page_single_process(self):
        key = 'youku_list_url'
        while connect_with_redis.length_of_lst(key) > 0:
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.download_list_page(loop))

    def parse_list_page_single_process(self):
        pid = os.getpid()
        URL_LIST = []
        key = 'youku_list_page_html'
        while connect_with_redis.length_of_lst(key) > 0:
            PAGE = connect_with_redis.retrieve_list_page_html_from_redis(platform=self.platform)
            soup = BeautifulSoup(PAGE, 'html.parser')
            video_url_agg = soup.find_all('div', {'class': 'yk-col4 '})
            for line in video_url_agg:
                url = line.a['href']
                video_id_str = ' '.join(re.findall('id_.*html', url))
                video_id = video_id_str.replace('id_', '')
                tudou_url = 'https://video.tudou.com/v/%s' % video_id
                URL_LIST.append(tudou_url)
                connect_with_redis.push_video_url_to_redis_set(platform='new_tudou',
                                                               url_lst=URL_LIST)
            print("platform: %s, action: parse list page, process_id: %s, get %s urls"
                  % (self.platform, pid, len(URL_LIST)))

    def parse_list_page_multi_process(self, processes_num=20):
        pool = Pool(processes=processes_num)
        for num in range(processes_num):
            pool.apply_async(self.parse_list_page_single_process)
        pool.close()
        pool.join()


"""
due to the video play_count in new_tudou is equal to youku, and it is earier to get
play_count in new_tudou, I rebuild the youku video url so that I can use new_tudou
crawler to get data
"""
#    async def download_video_page(self, loop):
#        task_list = connect_with_redis.retrieve_video_url_from_redis_set(platform=self.platform)
#        async with aiohttp.ClientSession() as session:
#            task = [loop.create_task(self.download_page(session, url)) for url in task_list]
#            done, pending = await asyncio.wait(task)
#            RESULT_LIST = [d.result() for d in done]
#            connect_with_redis.push_video_page_html_to_redis(platform=self.platform,
#                                                             result_lst=RESULT_LIST)
#    def download_video_page_async_single_process(self):
#        key = 'youku_video_url'
#        pid = os.getpid()
#        while connect_with_redis.length_of_set(key) > 0:
#            loop = asyncio.get_event_loop()
#            loop.run_until_complete(self.download_video_page(loop))
#            print("platform: %s, action: download video page, process_id: %s"
#                  % (self.platform, pid))
#
#    def download_video_page_async_multi_process(self, processes_num=10):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.download_video_page_async_single_process)
#        pool.close()
#        pool.join()
#
#    def parse_video_page(self, resp_str):
#        soup = BeautifulSoup(resp_str, 'html.parser')
#        title = soup.find('meta', {'name': 'title'})['content']
#        url = soup.find('meta', {'itemprop': 'url'})['content']
#        release_time_str = soup.find('meta', {'itemprop': 'datePublished'})['content']
#        release_time = int(datetime.datetime.strptime(release_time_str,
#                                                      "%Y-%m-%d %H:%M:%S").timestamp()*1e3)
#        releaser = soup.find('span',{'id': "module_basic_sub"}).a.text
#        releaser = releaser.replace('\n', '')
#        releaser = releaser.replace(' ', '')
#        duration_str =  ' '.join(re.findall("seconds: '.*'", resp_str))
#        if '.' in duration_str:
#            duration_str = ' '.join(re.findall("seconds: '\d+.", duration_str))
#            duration = int(' '.join(re.findall('\d+', duration_str))) + 1
#        else:
#            duration = int(' '.join(re.findall('\d+', duration_str)))
#        releaserUrl = soup.find('span',{'id': "module_basic_sub"}).a['href']
#        if releaserUrl[0:4] != 'http':
#            releaserUrl = 'http:' + releaserUrl
#        try:
#            video_info = soup.find('div',{'class': 'item item-cover current'})
#            play_count_str = video_info.find('div',{'class': 'status'}).text
#            play_count = trans_play_count(play_count_str)
#        except:
#            print("can't find play count at %s" % url)
#            play_count = 0
#        try:
#            releaser = soup.find('span',{'id':"module_basic_sub"}).a.text
#            releaserUrl = soup.find('span',{'id':"module_basic_sub"}).a['href']
#        except:
#            releaser = None
#            releaserUrl = None
#        else:
#            releaser = releaser.replace('\n','')
#            releaser = releaser.replace(' ','')
#            if releaserUrl[0:4] != 'http':
#                releaserUrl = 'http:' + releaserUrl
#        fetch_time = int(time.time()*1e3)
#        video_info = {"title": title,
#                      'url': url,
#                      'duration': duration,
#                      'releaser': releaser,
#                      'play_count': play_count,
#                      'releaserUrl': releaserUrl,
#                      'release_time': release_time,
#                      'fetch_time': fetch_time}
#        return video_info
#
#    def parse_video_page_single_process(self, output_to_file=False,
#                                        filepath=None,
#                                        push_to_redis=False,
#                                        output_to_es_raw=False,
#                                        es_index=None,
#                                        doc_type=None,
#                                        output_to_es_register=False):
#        pid = os.getpid()
#        RESULT_LIST = []
#        key = 'youku_video_page_html'
#        while connect_with_redis.length_of_lst(key) > 0:
#            VIDEO_PAGE_HTML = connect_with_redis.retrieve_video_page_html_from_redis(platform=self.platform)
#            try:
#                video_info = self.parse_video_page(resp_str=VIDEO_PAGE_HTML)
#            except:
#                video_info = None
#                print("failed to parse video info")
#            if video_info is not None:
#                RESULT_LIST.append(video_info)
#                print("platform: %s, action: parse video page, process_id: %s, has done %s"
#                      % (self.platform, pid, len(RESULT_LIST)))
#                if len(RESULT_LIST) >= 100:
#                    output_result(result_Lst=RESULT_LIST,
#                                  platform=self.platform,
#                                  output_to_file=output_to_file,
#                                  filepath=filepath,
#                                  push_to_redis=push_to_redis,
#                                  output_to_es_raw=output_to_es_raw,
#                                  es_index=es_index,
#                                  doc_type=doc_type,
#                                  output_to_es_register=output_to_es_register)
#                    RESULT_LIST.clear()
#        if len(RESULT_LIST) != []:
#            output_result(result_Lst=RESULT_LIST,
#                          platform=self.platform,
#                          output_to_file=output_to_file,
#                          filepath=filepath,
#                          push_to_redis=push_to_redis,
#                          output_to_es_raw=output_to_es_raw,
#                          es_index=es_index,
#                          doc_type=doc_type,
#                          output_to_es_register=output_to_es_register)
#            RESULT_LIST.clear()
#
#    def parse_video_page_multi_process(self, para_dic,
#                                       processes_num=30):
#        pool = Pool(processes=processes_num)
#        for num in range(processes_num):
#            pool.apply_async(self.parse_video_page_single_process, kwds=para_dic)
#        pool.close()
#        pool.join()

# test
if __name__=='__main__':
    test = Crawler_youku()
    vid = 'XMjc5MTQyNjg0OA'
    test_selenium = test.get_video_play_count_by_vid(vid)
    sr_yk = test.search_page(keyword='任正非 BBC', search_pages_max=2)