iqiyi.py

# -*- coding:utf-8 -*-
# @Time : 2019/12/19 12:01 
# @Author : litao

import numpy as np
import random
import json, redis, re, requests
from selenium.webdriver import ActionChains
import time, datetime, copy
from selenium import webdriver
from PIL import Image
import os
from selenium.webdriver.support.ui import WebDriverWait
import cv2
from fontTools.ttLib import *
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.trans_duration_str_to_second import trans_duration
from concurrent.futures import ProcessPoolExecutor
import urllib

# rds_list = redis.StrictRedis(host='127.0.0.1', port=6379, db=1, decode_responses=True)
# rds_single = redis.StrictRedis(host='127.0.0.1', port=6379, db=0, decode_responses=True)
# rds_get = redis.StrictRedis(host='127.0.0.1', port=6379, db=15, decode_responses=True)


# rds_copy = redis.StrictRedis(host='127.0.0.1', port=6379, db=1, decode_responses=True)
rds_list = redis.StrictRedis(host='192.168.17.60', port=6379, db=1, decode_responses=True)
rds_single = redis.StrictRedis(host='192.168.17.60', port=6379, db=0, decode_responses=True)
rds_get = redis.StrictRedis(host='192.168.17.60', port=6379, db=15, decode_responses=True)


def revise_data():
    scan_re = rds_single.scan_iter()
    for one_scan in scan_re:
        # print(one_scan)
        data = rds_single.hgetall(one_scan)
        # data["title"] = data["title"].replace("\r", "").replace("\n", "")
        # data["describe"] = data["describe"].replace("\r", "").replace("\n", "")
        rds_get.hmset(one_scan, data)
        # rds_list.hmset(one_scan,data)


class CrawlerMain(object):
    def __init__(self):
        self.chrome_options = webdriver.ChromeOptions()
        # self.chrome_options.add_argument('--headless')
        self.chrome_options.add_argument('--disable-gpu')
        # self.chrome_options.add_argument("--start-maximized")
        self.chrome_options.add_argument("--no-sandbox")
        self.chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
        self.timestamp = str(datetime.datetime.now().timestamp() * 1e3)
        # prefs = {"profile.managed_default_content_settings.images": 2}
        # self.chrome_options.add_experimental_option("prefs", prefs)
        # self.driver = webdriver.Chrome(options=self.chrome_options)
        self.one_video_dic = {
                "platform": "iqiyi_video",
                "ID": "",
                "title": "",
                "url": "",
                "describe": "",
                "video_count": "",
                "sum_duration": "",
                "year": "",
                "provider": "",
                "style_tags": "",
                "project_tags": "",
                "language": "",
                "area": "",
                "if_pay": "",
                "play_count_sum": "",
                "play_heat": "",
                "rate": "",
                "favorite_count_sum": "",
                "comment_count_sum": "",
                "barrage_count": "",
        }
        self.single_video_dic = {
                "platform": "iqiyi_video",
                "vid": "",
                "title": "",
                "video_title": "",
                "url": "",
                "video_url": "",
                "describe": "",
                "video_count": "",
                "duration": "",
                "year": "",
                "provider": "",
                "language": "",
                "area": "",
                "if_pay": "",
                "play_count": "",
                "play_heat": "",
                "rate": "",
                "favorite_count": "",
                "comment_count": "",
                "barrage_count": "",
        }

    def __exit__(self):
        self.driver.close()

    def list_page(self, releaserUrl=None,
                  video_list_xpath=None,
                  play_count_xpath=None,
                  if_free=None,
                  title=None,
                  describe=None,
                  project_tag_xpath=None,
                  provider=None,
                  year=None,
                  next_page_xpath=None,
                  roll=None,
                  project_tag=None,
                  style_tags=None,
                  category_id=""
                  ):
        page = 1
        total_page = 10
        while True:
            headers = {
                    "accept": "*/*",
                    "accept-encoding": "gzip, deflate, br",
                    "accept-language": "zh,zh-CN;q=0.9",
                    "content-type": "application/x-www-form-urlencoded",
                    "cookie": "QP001=1; QP0017=100; QP0018=100; QC005=c48ede3bf6d027f313486d3475d317b7; QC006=d6c2a542ed01ba7953ae094d36a40083; T00404=c8c154e05c0e54572a5eb20047dd6129; QC173=0; QP0013=; Hm_lvt_53b7374a63c37483e5dd97d78d9bb36e=1574822745,1575886435,1576549484; QC007=DIRECT; QC008=1556242615.1576549484.1576549484.9; nu=0; QC175=%7B%22upd%22%3Atrue%2C%22ct%22%3A%22%22%7D; QILINPUSH=1; T00700=EgcI77-tIRAF; QC159=%7B%22color%22%3A%22FFFFFF%22%2C%22channelConfig%22%3A1%2C%22isOpen%22%3A1%2C%22speed%22%3A13%2C%22density%22%3A30%2C%22opacity%22%3A86%2C%22isFilterColorFont%22%3A1%2C%22proofShield%22%3A0%2C%22forcedFontSize%22%3A24%2C%22isFilterImage%22%3A1%2C%22hadTip%22%3A1%7D; QP007=720; Hm_lpvt_53b7374a63c37483e5dd97d78d9bb36e=1576736529; IMS=IggQCRj_vO7vBSokCiBiYWNlNmIxMWExZjJmOTE4NzFmMjc3M2FmMTE0ZDUyYRAA; QC010=14783026; __dfp=a0c95636e67e384d24a08f486f2e6af20530557a862431f4441f2f23083da385e8@1577845483549@1576549484549",
                    "origin": "https://list.iqiyi.com",
                    # "referer": "https://list.iqiyi.com/www/3/-------------24-1-1-iqiyi-1-.html",
                    "sec-fetch-mode": "cors",
                    "sec-fetch-site": "same-site",
                    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",

            }
            quary = {
                    "access_play_control_platform": "14",
                    "channel_id": "3",
                    "data_type": "1",
                    "from": "pcw_list",
                    "is_album_finished": "",
                    "is_purchase": "",
                    # "is_purchase": "0",
                    # "is_purchase": "2",
                    "key": "",
                    "market_release_date_level": "",
                    # "mode": "24",
                    # "mode": "4",
                    "mode": "11",
                    "pageNum": str(page),
                    "pageSize": "1500",
                    "site": "iqiyi",
                    "source_type": "",
                    # "three_category_id": "70;must",
                    # "without_qipu": "1",
                    "three_category_id": "%s;must" % category_id,
                    "without_qipu": "1",
            }
            if page >= 1 and page <= total_page:
                requests_url = "https://pcw-api.iqiyi.com/search/video/videolists?%s" % urllib.parse.urlencode(quary)
                res = requests.get(requests_url, headers=headers)
                print("get data at page %s" % page)
                page += 1
            else:
                break
            res_json = res.json()
            # print(res_json)
            vidoe_list_obj = res_json["data"]["list"]
            total_page = res_json["data"]["pageTotal"] + 1
            try:
                for count, one_video in enumerate(vidoe_list_obj):
                    print(count)
                    area = ""
                    style_tag = ""
                    if_pay = one_video.get("payMark")
                    id = one_video["albumId"]
                    title = one_video["name"]
                    project_name = "iqiyi_%s" % id
                    url = one_video["playUrl"]
                    video_count = one_video["videoCount"]
                    try:
                        describe = one_video["description"]
                    except:
                        describe = one_video["focus"]

                    score = one_video.get("score")
                    style_tag_list = one_video["categories"]
                    if style_tag_list:
                        for l in style_tag_list:
                            try:
                                subName = l["subName"]
                                if subName == "地区":
                                    area = l["name"]
                                elif subName == "类型":
                                    style_tag += l["name"] + ","
                                elif subName == "出品方":
                                    provider = l["name"] + ","
                            except:
                                style_tag = l["name"] + ","
                    formatPeriod = one_video.get("formatPeriod")
                    issueTime = one_video.get("issueTime")
                    if issueTime:
                        issueTime = datetime.datetime.fromtimestamp(issueTime / 1000).strftime("%Y-%m-%d %H:%M:%S")
                    data_dic = {
                            "id": id,
                            "url": url,
                            "title": title,
                            "video_count": video_count,
                            "if_pay": if_pay,
                            "describe": describe,
                            "issueTime": issueTime,
                            "formatPeriod": formatPeriod,
                            "score": score,
                            "style_tags": style_tag,
                            "area": area,
                            "provider": provider

                    }
                    if style_tags:
                        temp_dic = {}
                        try:
                            temp_dic["style_tags"] = temp_dic["style_tags"] + style_tags
                        except:
                            temp_dic["style_tags"] = style_tags
                        data_dic.update(temp_dic)
                    if project_tag:
                        temp_dic = {}
                        temp_dic["project_tags"] = project_tag
                        data_dic.update(temp_dic)
                    if year:
                        temp_dic = {}
                        temp_dic["year"] = year
                        data_dic.update(temp_dic)
                    if provider:
                        temp_dic = {}
                        temp_dic["provider"] = provider
                        data_dic.update(temp_dic)
                    self.parse_data(data_dic, project_name)

            except Exception as e:
                print(e)
                # self.driver.close()

    def style_tags(self, releaserUrl=None,
                   video_list_xpath=None,
                   play_count_xpath=None,
                   if_free=None,
                   title=None,
                   describe=None,
                   style_tags_xpath=None,
                   style=None,
                   project_tag=None,
                   provider=None,
                   year=None,
                   next_page_xpath=None,
                   roll=None, ):
        # self.driver.get(releaserUrl)
        time.sleep(0.2)
        style_tags_obj = self.driver.find_elements_by_xpath(style_tags_xpath)
        for style_tag in style_tags_obj:
            style_tags = style_tag.text
            print(style_tags)
            if style_tags == "全部":
                continue
            action = ActionChains(self.driver)
            action.click(style_tag).perform()
            provider = style_tags
            # if True:
            self.list_page(releaserUrl=self.driver.current_url,
                           video_list_xpath=video_list_xpath,
                           play_count_xpath=play_count_xpath,
                           if_free=if_free,
                           title=title,
                           describe=describe,
                           project_tag_xpath=style_tags_xpath,
                           project_tag=project_tag,
                           style_tags=style,
                           provider=provider,
                           year=year,
                           next_page_xpath=next_page_xpath,
                           roll=roll, )

    def parse_data(self, data_dic, project_name):
        res = rds_list.hgetall(project_name)
        if res:
            if not res.get("project_tags"):
                res["project_tags"] = ""
            if data_dic.get("style_tags"):
                if data_dic.get("style_tags") not in res["style_tags"]:
                    data_dic["style_tags"] = res["style_tags"] + "," + data_dic.get("style_tags")
            if data_dic.get("project_tags"):
                if data_dic.get("project_tags") not in res["project_tags"]:
                    data_dic["project_tags"] = res["project_tags"] + "," + data_dic.get("project_tags")
            if data_dic.get("provider"):
                if data_dic.get("provider") not in res["provider"]:
                    data_dic["provider"] = res["provider"] + "," + data_dic.get("provider")
            for k in data_dic:
                if not data_dic[k]:
                    data_dic[k] = ""
            res.update(data_dic)
            rds_list.hmset(project_name, res)
        else:
            data = copy.deepcopy(self.one_video_dic)
            for k in data_dic:
                if not data_dic[k]:
                    data_dic[k] = ""
            data.update(data_dic)
            rds_list.hmset(project_name, data)

    def parse_single_data(self, data_dic, project_name):
        res = rds_single.hgetall(project_name)
        if res:
            if not res.get("project_tags"):
                res["project_tags"] = ""
            if data_dic.get("style_tags"):
                if data_dic.get("style_tags") not in res["style_tags"]:
                    data_dic["style_tags"] = res["style_tags"] + "," + data_dic.get("style_tags")
            if data_dic.get("project_tags"):
                if data_dic.get("project_tags") not in res["project_tags"]:
                    data_dic["project_tags"] = res["project_tags"] + "," + data_dic.get("project_tags")
            if data_dic.get("provider"):
                if data_dic.get("provider") not in res["provider"]:
                    data_dic["provider"] = res["provider"] + "," + data_dic.get("provider")
            for k in data_dic:
                if not data_dic[k]:
                    data_dic[k] = " "
            res.update(data_dic)
            try:
                print(res)
                rds_single.hmset(project_name, res)
            except:
                pass
        else:
            data = copy.deepcopy(self.single_video_dic)
            for k in data_dic:
                if not data_dic[k]:
                    data_dic[k] = ""
            data.update(data_dic)
            rds_single.hmset(project_name, data)

    def video_page_api(self, task=0):
        has_data = rds_get.dbsize()
        while has_data:
            keys = rds_get.randomkey()
            res = rds_get.hgetall(keys)
            # res = rds_get.hgetall("iqiyi_100010201")
            has_data = rds_get.dbsize()
            # time.sleep(0.2)
            try:
                print(res["url"])
                self.detail_page_api(res)
                # self.detail_page_api(res)
                rds_get.delete(keys)
            except Exception as e:
                print(e, res["url"])

    def video_page(self, task=0):
        # 获取专辑页标签分类
        self.driver = webdriver.Chrome(options=self.chrome_options)
        self.driver.maximize_window()
        has_data = rds_get.dbsize()
        while has_data:
            keys = rds_get.randomkey()
            res = rds_get.hgetall(keys)
            has_data = rds_get.dbsize()
            # time.sleep(0.2)
            try:
                self.driver.get(res["url"])
                time.sleep(5)
                # self.driver.implicitly_wait(10)
                title = res["title"]
                # WebDriverWait(self.driver, 10,2).until(lambda x: x.find_element_by_xpath("/html[1]/body[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/article[1]"))
                playcount_obj = self.driver.find_element_by_xpath("//span[@class='basic-title']")
                action = ActionChains(self.driver)
                action.click(playcount_obj).perform()
                play_conut_obj_lsit = self.driver.find_elements_by_xpath("//div[@class='hot-chart-tab']//li")
                action = ActionChains(self.driver)
                action.click(play_conut_obj_lsit[-1]).perform()
                for x in range(11):
                    # print(x)
                    ActionChains(self.driver).move_by_offset(10 * x, 20).perform()
                play_count_sum_obj = self.driver.find_element_by_xpath("//div[@class='hot-chart']//div[2]")
                print(play_count_sum_obj.location)
                print(play_count_sum_obj.text)
                print("task ", task)
                action = ActionChains(self.driver)
                # action.move_by_offset(1406,1048)
                # action.move_to_element_with_offset(play_count_sum_obj,500,-300).perform()
                # time.sleep(0.5)
                play_count_re = re.findall('指数：(\d+)', play_count_sum_obj.text)
                if play_count_re:
                    play_count = play_count_re[0]
                else:
                    play_count = ""
                print(play_count_re)
                language = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[1]")
                if language:
                    language = language[0].text
                else:
                    language = ""
                style_tags = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[2]")
                if style_tags:
                    style_tags = style_tags[0].text
                else:
                    style_tags = ""
                project_tags = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[3]")
                if project_tags:
                    project_tags = project_tags[0].text
                else:
                    project_tags = ""
                # faverate_count_sum = self.driver.find_element_by_xpath("//span[@class='like-icon-box']")
                one_video_dic = {
                        "play_count_sum": play_count,
                        "language": language,
                        "style_tags": style_tags,
                        "project_tags": project_tags,
                }
                self.parse_data(one_video_dic, keys)
                # self.detail_page_api(res)
                rds_get.delete(keys)
            except Exception as e:
                print(e, res["url"])

    def one_video_page(self, title, one_video_dic, type="list"):
        video_list = self.driver.find_elements_by_xpath("//ul[@class='qy-episode-txt']//li")
        # for video_obj in video_list:
        #     title = ""
        #     video_url = ""
        #     barrage_count = ""
        #     self.parse_single_data(dic, project_name)

    def detail_page_api(self, data):
        # 获取专辑的单条视频ID
        aid = data.get("id")
        headers = {
                "Host": "iface2.iqiyi.com",
                "t": "873713668",
                "X-B3-TraceId": "2B2E20501CEF485382A9B6FD5405861D",
                "Accept-Language": "zh-cn",
                "X-B3-SpanId": "82A9B6FD5405861D",
                "Connection": "keep-alive",
                "Accept": "*/*",
                "User-Agent": "QIYIVideo/10.11.0 (iOS;com.qiyi.hd;iOS13.3;iPad6,11) Corejar",
                "sign": "a900abd6cbfb9e012dbdffa1c72f4a41",
                "Accept-Encoding": "gzip",
                "X-B3-Sampled": "0"
        }

        headers = {
                "Host": "cards.iqiyi.com",
                "t": "873713829",
                "X-B3-TraceId": "DD4CB3B31E544C02AA5BC3340AE2ABC2",
                "Accept-Language": "zh-cn",
                "X-B3-SpanId": "AA5BC3340AE2ABC2",
                "Connection": "keep-alive",
                "Accept": "*/*",
                "User-Agent": "QIYIVideo/10.11.0 (iOS;com.qiyi.hd;iOS13.3;iPad6,11) Corejar",
                "sign": "292c33bc4620256820ea5e1cdf92fcbb",
                "Accept-Encoding": "gzip",
                "X-B3-Sampled": "0",
        }
        page = 0
        has_more = True
        if has_more:
            get_dic = {
                    "api_v": "7.7",
                    "app_k": "f0f6c3ee5709615310c0f053dc9c65f2",
                    "app_v": "10.11.0",
                    "app_gv": "",
                    "app_t": "0",
                    "platform_id": "13",
                    "dev_os": "13.3",
                    "dev_ua": "iPad6,11",
                    "dev_hw": '{"cpu":"","mem":"1634"}',
                    "net_sts": "1",
                    "scrn_sts": "2",
                    "scrn_res": "2048*1536",
                    "scrn_dpi": "786432",
                    "qyid": "7AED33DD-0F97-418D-AFAA-72ED0578A75E",
                    "cupid_uid": "7AED33DD-0F97-418D-AFAA-72ED0578A75E",
                    "cupid_v": "3.37.007",
                    "psp_uid": "2345294915",
                    "psp_cki": "12Q4kR4QaQDO0H4MuUPGay47DxLxzZF7IGAqOm1Vleux5dtgo2AazHR4K5l9SLRjKQY18",
                    "secure_p": "iPad",
                    "secure_v": "1",
                    "lang": "zh_CN",
                    "app_lm": "cn",
                    "upd": "0",
                    "psp_vip": "1",
                    "core": "1",
                    "req_sn": "1577108056510",
                    "net_ip": "%7B%22country%22:%22%E4%B8%AD%E5%9B%BD%22,%22province%22:%22%E5%8C%97%E4%BA%AC%22,%22city%22:%22%E5%8C%97%E4%BA%AC%22,%22cc%22:%22%E8%81%94%E9%80%9A%22,%22area%22:%22%E5%8D%8E%E5%8C%97%22,%22timeout%22:0,%22respcode%22:0%7D",
                    "psp_status": "3",
                    "profile": '{"group":"2","counter":"1","hy_id":"","recall_firstdate":"-1","first_time":"20180813"}',
                    "gps": "",
                    "cust_count": "",
                    "idfa": "7AED33DD-0F97-418D-AFAA-72ED0578A75E",
                    "scrn_scale": "2",
                    "ouid": "3b8797b0d649aa05ece4c769a6d3d745c897ddc2",
                    "youth_model": "0",
                    "bi_params": "",
                    "from_rpage": "Y2F0ZWdvcnlfaG9tZS4z",
                    "player_rpage": "half_ply",
                    "player_block": "Ym9mYW5ncWkx",
                    "page": "player_tabs",
                    "fake_ids": "choose_set",
                    "album_id": aid,
                    # "tv_id": aid,
                    "plt_full": "1",
                    "full": "1",
                    "block_mode": "2",
                    "pps": "0",
                    "req_times": "1",

            }
            page_res = requests.get(data.get("url"), headers=headers)
            # print(page_res.text)
            # get_url = "https://iface2.iqiyi.com/views/3.0/card_view?%s" % urllib.parse.urlencode(get_dic)
            get_url = "https://cards.iqiyi.com/views_plt/3.0/card_view?api_v=7.7&app_k=f0f6c3ee5709615310c0f053dc9c65f2&app_v=10.11.0&app_gv=&app_t=0&platform_id=13&dev_os=13.3&dev_ua=iPad6,11&net_sts=1&qyid=7AED33DD-0F97-418D-AFAA-72ED0578A44E&cupid_uid=7AED33DD-0F97-418D-AFAA-72ED0578A44E&cupid_v=3.37.007&psp_uid=2345294915&psp_cki=12Q4kR4QaQDO0H4MuUPGay47DxLxzZF7IGAqOm1Vleux5dtgo2AazHR4K5l9SLRjKQY18&psp_status=3&secure_p=iPad&secure_v=1&core=1&req_sn=1577009664125&lang=zh_CN&upd=0&app_lm=cn&profile=%7B%22group%22%3A%222%22%2C%22counter%22%3A1%2C%22hy_id%22%3A%22%22%2C%22recall_firstdate%22%3A%22-1%22%2C%22first_time%22%3A%2220180813%22%7D&gps=&idfa=7AED33DD-0F97-418D-AFAA-72ED0578A44E&ouid=3b8797b0d649aa05ece4c769a6d3d745c897ddc2&scrn_scale=2&init_type=0&youth_model=0&bi_params=&wifi_mac=&layout_v=49.1&layout_name=base_layout_pad&page=player_tabs&fake_ids=spoiler_list&album_id={0}&tv_id={1}&video_tab=1&ad_play_source=0&scrn_res=2048*1536&req_times=1".format(
                    aid, aid)

            page += 1
            res = requests.get(get_url, headers=headers)
            # print(res.text)
            res_json = res.json()
            # data_list = res_json["cards"][0]["data"]
            data_list_lis = res_json["cards"][0]["blocks"]
            data_list = {}
            for k, d in enumerate(data_list_lis):
                data_list[k] = d
            for count, data_dic in enumerate(data_list):
                # vid = data_list[data_dic]["_id"]
                vid = data_list[data_dic]["block_id"]
                # description = data_list[data_dic]["other"]["_t"]
                duration = data_list[data_dic]["other"]["duration"]
                issueTime = data_list[data_dic]["other"].get("year")
                # video_title = data_list[data_dic]["meta"][0]["text"]
                video_title = data_list[data_dic]["metas"][0]["text"]
                # video_count = data_list[data_dic]["marks"]["br"]["t"]
                # print(data.get("url"))
                one_video_dic = {
                        "url": data.get("url"),
                        "vid": vid,
                        # "description": description,
                        "issueTime": issueTime,
                        "duration": duration,
                        "album": data.get("title"),
                        "video_title": video_title,
                        # "video_count":video_count
                }
                self.single_page_api(one_video_dic)

    def single_page_api(self, one_video_dic):
        # 获取单集视频url
        vid = one_video_dic["vid"]
        headers = {
                "Host": "iface2.iqiyi.com",
                "t": "938224906",
                "X-B3-TraceId": "D3E5A60235FA4EA59816008A391098B8",
                "Accept-Language": "zh-cn",
                "X-B3-SpanId": "9816008A391098B8",
                "Connection": "keep-alive",
                "Accept": "*/*",
                "User-Agent": "QIYIVideo/10.11.0 (iOS;com.qiyi.hd;iOS13.3;iPad6,11) Corejar",
                "sign": "64b4cfe0a12109fa4b011deef366fa95",
                "Accept-Encoding": "gzip",
                "X-B3-Sampled": "0",
        }
        get_head = {
                "app_k": "f0f6c3ee5709615310c0f053dc9c65f2",
                "api_v": "7.7",
                "app_v": "10.11.0",
                # "qyid": "7AED33DD-0F97-418D-AFAA-72ED0578A44E",
                # "cupid_uid": "7AED33DD-0F97-418D-AFAA-72ED0578A44E",
                "secure_p": "iPad",
                "secure_v": "1",
                "psp_uid": "2345294915",
                "psp_cki": "12Q4kR4QaQDO0H4MuUPGay47DxLxzZF7IGAqOm1Vleux5dtgo2AazHR4K5l9SLRjKQY18",
                "dev_hw": "%7B%22cpu%22:%22%22,%22mem%22:%221634%22%7D",
                "net_sts": "1",
                "scrn_sts": "2",
                "dev_os": "13.3",
                "dev_ua": "iPad6,11",
                "scrn_res": "2048*1536",
                "scrn_dpi": "786432",
                "req_sn": "1577107375313",
                "core": "1",
                "lang": "zh_CN",
                "upd": "0",
                "app_lm": "cn",
                "net_ip": "%7B%22country%22:%22%E4%B8%AD%E5%9B%BD%22,%22province%22:%22%E5%8C%97%E4%BA%AC%22,%22city%22:%22%E5%8C%97%E4%BA%AC%22,%22cc%22:%22%E8%81%94%E9%80%9A%22,%22area%22:%22%E5%8D%8E%E5%8C%97%22,%22timeout%22:0,%22respcode%22:0%7D",
                "cupid_v": "3.37.007",
                "youth_model": "0",
                "bi_params": "",
                "album_id": vid,
                "tv_id": vid,
                "play_res": "512",
                "play_retry": "0",
                "play_core": "1",
                "plugin_id": "",
                "platform_id": "13",
                "app_p": "ipad",
                "app_t": "0",
                "ctl_dubi": "0",
                "req_sn": "1577107375313240000",
                "req_times": "1",
                "scrn_scale": "2",
                "rate": "512,16,8,1024,2048",
                "rpage": "half_ply",
                "block": "bofangqi1",
                "s2": "player_tabs",
                "uid": "2345294915",
                "version": "10.11.0",
                "pps": "0",

        }
        try:
            res = requests.get("https://iface2.iqiyi.com/video/3.0/v_play?%s" % urllib.parse.urlencode(get_head),
                               headers=headers)
            res_json = res.json()
            if_pay = res_json["video"]["pay_mark"]
            glb_year = res_json["video"]["glb_year"]
            video_rul = res_json["video"]["web_url"]
            description = res_json["video"]["subtitle"]
            project_tags = res_json["album"]["tag"]
            video_title = res_json["album"]["_t"]
            year = res_json["album"]["year"]
            one_video_dic["if_pay"] = if_pay
            one_video_dic["year"] = glb_year
            one_video_dic["video_rul"] = video_rul
            one_video_dic["description"] = description
            one_video_dic["project_tags"] = project_tags
            one_video_dic["video_title"] = video_title
            one_video_dic["year"] = year
            self.parse_single_data(one_video_dic, video_rul)
        except Exception as e:
            print(e)
            pass

    def video_page_seleium(self, task=0):
        # seleium 获取单集视频 缺少vid
        self.driver = webdriver.Chrome(options=self.chrome_options)
        self.driver.maximize_window()
        has_data = rds_get.dbsize()
        while has_data:
            keys = rds_get.randomkey()
            res = rds_get.hgetall(keys)
            has_data = rds_get.dbsize()
            # time.sleep(0.2)
            try:
                self.driver.get(res["url"])
                time.sleep(5)
                video_list = self.driver.find_elements_by_xpath("//div[@id='rightPlayList']//li")
                for count, video_obj in enumerate(video_list):
                    # if count <= 1:
                    #     continue
                    try:
                        ActionChains(self.driver).click(video_obj).perform()
                        time.sleep(2)
                    except:
                        continue
                    self.driver.implicitly_wait(10)
                    title = self.driver.find_element_by_xpath("//h1[@class='player-title']").text
                    # WebDriverWait(self.driver, 10,2).until(lambda x: x.find_element_by_xpath("/html[1]/body[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[3]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/article[1]"))
                    # playcount_obj = self.driver.find_element_by_xpath("//span[@class='basic-title']")
                    duration_str = self.driver.find_element_by_xpath("//iqpspan[@class='iqp-time-dur']").text
                    duration = trans_duration(duration_str)
                    # action = ActionChains(self.driver)
                    # action.click(playcount_obj).perform()
                    # play_conut_obj_lsit = self.driver.find_elements_by_xpath("//div[@class='hot-chart-tab']//li")
                    # action = ActionChains(self.driver)
                    # action.click(play_conut_obj_lsit[-1]).perform()
                    # for x in range(11):
                    #     # print(x)
                    #     ActionChains(self.driver).move_by_offset(10 * x, 20).perform()
                    # play_count_sum_obj = self.driver.find_element_by_xpath("//div[@class='hot-chart']//div[2]")
                    # print(play_count_sum_obj.location)
                    # print(play_count_sum_obj.text)
                    print("task ", task)
                    action = ActionChains(self.driver)
                    # action.move_by_offset(1406,1048)
                    # action.move_to_element_with_offset(play_count_sum_obj,500,-300).perform()
                    # time.sleep(0.5)
                    # play_count_re = re.findall('指数：(\d+)', play_count_sum_obj.text)
                    # if play_count_re:
                    #     play_count = play_count_re[0]
                    # else:
                    #     play_count = ""
                    # print(play_count_re)
                    # language = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[1]")
                    # if language:
                    #     language = language[0].text
                    # else:
                    #     language = ""
                    # style_tags = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[2]")
                    # if style_tags:
                    #     style_tags = style_tags[0].text
                    # else:
                    #     style_tags = ""
                    # project_tags = self.driver.find_elements_by_xpath("//div[@class='intro-mn']//a[3]")
                    # if project_tags:
                    #     project_tags = project_tags[0].text
                    # else:
                    #     project_tags = ""
                    # faverate_count_sum = self.driver.find_element_by_xpath("//span[@class='like-icon-box']")
                    one_video_dic = {
                            # "play_count_sum": play_count,
                            "url": res["url"],
                            "video_url": self.driver.current_url,
                            "video_title": title,
                            "album": res["title"],
                            "duration": duration
                    }
                    self.parse_single_data(one_video_dic, one_video_dic["video_url"])
                # self.detail_page_api(res)
                rds_get.delete(keys)
            except Exception as e:
                print(e, res["url"])

    def comment_like_api(self,task=0):
        # 获取评论和点赞
        has_data = rds_get.dbsize()
        while has_data:
            keys = rds_get.randomkey()
            res = rds_get.hgetall(keys)
            has_data = rds_get.dbsize()

            try:
                vid = res.get("vid")
                if not vid:
                    rds_get.delete(keys)
                comment_headers = {
                        "Host": "cards.iqiyi.com",
                        "t": "938224906",
                        "X-B3-TraceId": "4AF7FFCBC6A9460CB396CC59D1754D3B",
                        "Accept-Language": "zh-cn",
                        "X-B3-SpanId": "B396CC59D1754D3B",
                        "Connection": "keep-alive",
                        "Accept": "*/*",
                        "User-Agent": "QIYIVideo/10.11.0 (iOS;com.qiyi.hd;iOS13.3;iPad6,11) Corejar",
                        "sign": "64b4cfe0a12109fa4b011deef366fa95",
                        "Accept-Encoding": "gzip",
                        "X-B3-Sampled": "0",

                }
                like_headers = {
                        "Host": "iface2.iqiyi.com",
                        "t": "938306077",
                        "X-B3-TraceId": "7FABCAD44B0249AE85A86B71BCE6DFDB",
                        "Accept-Language": "zh-cn",
                        "X-B3-SpanId": "85A86B71BCE6DFDB",
                        "Connection": "keep-alive",
                        "Accept": "*/*",
                        "User-Agent": "QIYIVideo/10.11.0 (iOS;com.qiyi.hd;iOS13.3;iPad6,11) Corejar",
                        "sign": "6fd76e1f02b7817dfd32ae5591d8b987",
                        "Accept-Encoding": "gzip",
                        "X-B3-Sampled": "0",
                }

                comment_url = "https://cards.iqiyi.com/views_sns/3.0/short_video_comments?app_gv=&core=1&from_subtype=-1&upd=0&profile=%257B%2522group%2522%253A%25222%2522%252C%2522counter%2522%253A2%252C%2522hy_id%2522%253A%2522%2522%252C%2522recall_firstdate%2522%253A%2522-1%2522%252C%2522first_time%2522%253A%252220180813%2522%257D&app_lm=cn&secure_p=iPad&lang=zh_CN&cupid_v=3.37.007&scrn_scale=2&gps=&phone_operator=0&dev_os=13.3&ouid=3b8797b0d649aa05ece4c769a6d3d745c897ddc2&layout_v=49.1&content_uid=0&psp_cki=12Q4kR4QaQDO0H4MuUPGay47DxLxzZF7IGAqOm1Vleux5dtgo2AazHR4K5l9SLRjKQY18&card_v=3.0&youth_model=0&page_st=common&app_k=f0f6c3ee5709615310c0f053dc9c65f2&dev_ua=iPad6,11&net_sts=1&cupid_uid=7AED33DD-0F97-418D-AFAA-72ED0578A64E&from_type=-1&init_type=0&app_v=10.11.0&idfa=7AED33DD-0F97-418D-AFAA-72ED0578A64E&app_t=0&platform_id=13&content_id={0}&layout_name=base_layout_pad&req_sn=1577107375553&api_v=7.7&nav_search_v2=0&psp_status=3&psp_uid=2345294915&wifi_mac=&qyid=7AED33DD-0F97-418D-AFAA-72ED0578A64E&wall_id=0&bi_params=&secure_v=1&content_pic=http://pic7.iqiyipic.com/image/20180103/bc/6f/v_114479444_m_601_m1.jpg&identity=1&albumid={1}&req_times=1".format(
                    vid, vid)
                like_url = "https://iface2.iqiyi.com/views/3.0/card_view?api_v=7.7&app_k=f0f6c3ee5709615310c0f053dc9c65f2&app_v=10.11.0&app_gv=&app_t=0&platform_id=13&dev_os=13.3&dev_ua=iPad6,11&dev_hw=%7B%22cpu%22:%22%22,%22mem%22:%221634%22%7D&net_sts=1&scrn_sts=2&scrn_res=2048*1536&scrn_dpi=786432&qyid=7AED33DD-0F97-418D-AFAA-72ED0578A54E&cupid_uid=7AED33DD-0F97-418D-AFAA-72ED0578A54E&cupid_v=3.37.007&psp_uid=2345294915&psp_cki=12Q4kR4QaQDO0H4MuUPGay47DxLxzZF7IGAqOm1Vleux5dtgo2AazHR4K5l9SLRjKQY18&secure_p=iPad&secure_v=1&lang=zh_CN&app_lm=cn&upd=0&psp_vip=1&core=1&req_sn=1577187512020&net_ip=%7B%22country%22:%22%E4%B8%AD%E5%9B%BD%22,%22province%22:%22%E5%8C%97%E4%BA%AC%22,%22city%22:%22%E5%8C%97%E4%BA%AC%22,%22cc%22:%22%E8%81%94%E9%80%9A%22,%22area%22:%22%E5%8D%8E%E5%8C%97%22,%22timeout%22:0,%22respcode%22:0%7D&psp_status=3&profile=%257B%2522group%2522%253A%25222%2522%252C%2522counter%2522%253A1%252C%2522hy_id%2522%253A%2522%2522%252C%2522recall_firstdate%2522%253A%2522-1%2522%252C%2522first_time%2522%253A%252220180813%2522%257D&gps=&cust_count=&idfa=7AED33DD-0F97-418D-AFAA-72ED0578A54E&scrn_scale=2&ouid=3b8797b0d649aa05ece4c769a6d3d745c897ddc2&youth_model=0&bi_params=&from_rpage=Y2F0ZWdvcnlfaG9tZS4z&player_rpage=half_ply&player_block=Ym9mYW5ncWkx&page=player_tabs&fake_ids=choose_set&album_id={0}&tv_id={1}&plt_full=1&full=1&block_mode=2&pps=0&req_times=1".format(
                    vid, vid)
                comment_res = requests.get(comment_url, headers=comment_headers)
                like_res = requests.get(like_url, headers=like_headers)
                comment_res_json = comment_res.json()
                like_res_json = like_res.json()
                comment_count = comment_res_json["kv_pair"]["totalCount"]
                try:
                    favorite_count = like_res_json["kvpairs"]["likeCount"]
                except:
                    favorite_count = " "
                dic = {
                        "comment_count":comment_count,
                        "favorite_count":favorite_count
                }
                print(dic,task)
                # print(res)
                self.parse_single_data(dic,keys)
                rds_get.delete(keys)
            except Exception as e:
                print(e)
                continue

    def get_vid(self):
        has_data = rds_get.dbsize()
        while has_data:
            keys = rds_get.randomkey()
            res = rds_get.hgetall(keys)
            has_data = rds_get.dbsize()
            vid = res.get("vid")

            if not vid:
                print(vid,keys)
                headers = {

                        "accept": "*/*",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh,zh-CN;q=0.9",
"cookie": 'QP001=1; QP0017=100; QP0018=100; QC005=c48ede3bf6d027f313486d3475d317b7; QC006=d6c2a542ed01ba7953ae094d36a40083; T00404=c8c154e05c0e54572a5eb20047dd6129; QC173=0; Hm_lvt_53b7374a63c37483e5dd97d78d9bb36e=1574822745,1575886435,1576549484; QC007=DIRECT; QC008=1556242615.1576549484.1576549484.9; nu=0; P00004=.1576737946.c60f13e3da; P00001=87BSsO9M7Lkm1M5cGXEBptdVHC3f39m3vCo3kPoLxm1PbUJm1dlpDUm21zbVGaUrgcWkPNy0e; P00003=2345294915; P00010=2345294915; P00007=87BSsO9M7Lkm1M5cGXEBptdVHC3f39m3vCo3kPoLxm1PbUJm1dlpDUm21zbVGaUrgcWkPNy0e; P00PRU=2345294915; P000email=litaolemo%40foxmail.com; QC160=%7B%22u%22%3A%2215652340574%22%2C%22lang%22%3A%22%22%2C%22local%22%3A%7B%22name%22%3A%22%E4%B8%AD%E5%9B%BD%E5%A4%A7%E9%99%86%22%2C%22init%22%3A%22Z%22%2C%22rcode%22%3A48%2C%22acode%22%3A%2286%22%7D%2C%22type%22%3A%22p1%22%7D; QC170=1; QC175=%7B%22upd%22%3Atrue%2C%22ct%22%3A1576737959108%7D; QP0013=1; QP007=0; QC179=%7B%22userIcon%22%3A%22//www.iqiyipic.com/common/fix/headicons/male-130.png%22%2C%22vipTypes%22%3A%221%22%7D; QY_PUSHMSG_ID=c48ede3bf6d027f313486d3475d317b7; P00002=%7B%22uid%22%3A2345294915%2C%22pru%22%3A2345294915%2C%22user_name%22%3A%2215652340574%22%2C%22nickname%22%3A%22litaolemo001%22%2C%22pnickname%22%3A%22litaolemo001%22%2C%22type%22%3A11%2C%22email%22%3A%22litaolemo%40foxmail.com%22%7D; QC163=1; QC001=1; c242345294915=1576742527454; TQC022=%7B%22au%22%3A%2287BSsO9M7Lkm1M5cGXEBptdVHC3f39m3vCo3kPoLxm1PbUJm1dlpDUm21zbVGaUrgcWkPNy0e%22%2C%22ak%22%3A%228ffff92ffm33w8CqFm13T6zA4tKtsnjA5rL2CEa4Qsva91f6xsPMsQm4%22%7D; PCAU=0; QYABEX={"pcw_home_movie":{"value":"new","abtest":"146_B"}}; T00700=EgcI77-tIRAFEgcIz7-tIRAB; QY00001=2345294915; H_RENEW_COUNT=1576810189038; IQIYIH5_WEBP=1; P01010=1576857600; ut={%22ut%22:[%221%22]%2C%22uid%22:2345294915}; QILINPUSH=1; QC176=%7B%22state%22%3A0%2C%22ct%22%3A1576811187124%7D; Hm_lpvt_53b7374a63c37483e5dd97d78d9bb36e=1576811226; QC159=%7B%22color%22%3A%22FFFFFF%22%2C%22channelConfig%22%3A1%2C%22isOpen%22%3A1%2C%22speed%22%3A13%2C%22density%22%3A30%2C%22opacity%22%3A86%2C%22isFilterColorFont%22%3A1%2C%22proofShield%22%3A0%2C%22forcedFontSize%22%3A24%2C%22isFilterImage%22%3A1%2C%22hadTip%22%3A1%7D; app_server_fail_num=2; app_server_retry_time=1; QP008=3000; IMS=IggQBhj_3_PvBSokCiA3ZjNlMjJmMjUxYzkzYTY1N2I2ZGZlMmFiMGQzOTY1YRAA; QC010=214521235; __dfp=a0c95636e67e384d24a08f486f2e6af20530557a862431f4441f2f23083da385e8@1577845483549@1576549484549',
"referer": "https://www.iqiyi.com/a_19rriflrba.html",
"sec-fetch-mode": "no-cors",
"sec-fetch-site": "same-site",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",

                }
                requests_re = requests.get(res.get("video_url"),headers=headers)
                try:
                    vid = re.findall("tvid=(\d+)",requests_re.text)[0]
                    if not vid:
                        continue
                    dic = {
                            "vid":vid
                    }
                    self.parse_single_data(dic, keys)
                    print(1)
                except:
                    continue
if __name__ == "__main__":
    # crlawler_iqiyi = CrawlerMain()
    # provider_dic = {
    #     # "ALL":"http://list.iqiyi.com/www/3/-------------24-1-1-iqiyi--.html"
    #         "人物":70,
    #         "军事":72,
    #         "历史":74,
    #         "探索":73,
    #         "文化":77,
    #         "社会":71,
    #         "科学":28119,
    #         "旅游":310,
    #         "真人秀":28137,
    #         "政治":28138,
    # }
    # for provider in provider_dic:
    #     crlawler_iqiyi.list_page(releaserUrl=provider_dic[provider],
    #                              video_list_xpath="/html[1]/body[1]/div[6]/div[1]/div[2]/div",
    #                              roll=1000,
    #                              project_tag=provider,
    #                              category_id=str(provider_dic[provider])
    #                              # provider=provider
    #                              )
    # provider_dic = {
    #         # "ALL":"http://list.iqiyi.com/www/3/-------------24-1-1-iqiyi--.html"
    #         "BBC": 28468,
    #         "美国历史频道": 28470,
    #         "探索频道": 28471,
    #         "央视记录": 28472,
    #         "北京纪实频道": 28473,
    #         "上海纪实频道": 28474,
    #         "朗思文化": 28476,
    #         "CNEX": 28477,
    #         "五星传奇": 28478,
    #         "IMG": 28479,
    #         "NHK": 28480,
    #         "爱奇艺出品": 31283,
    #         "Netflix": 31286,
    # }
    # for provider in provider_dic:
    #     crlawler_iqiyi.list_page(releaserUrl=provider_dic[provider],
    #                              video_list_xpath="/html[1]/body[1]/div[6]/div[1]/div[2]/div",
    #                              roll=1000,
    #                              provider=provider,
    #                              category_id=str(provider_dic[provider])
    #                              # provider=provider
    #                              )
    # provider_dic = {
    #         # "ALL":"http://list.iqiyi.com/www/3/-------------24-1-1-iqiyi--.html"
    #         "纪录电影": 29077,
    #         "系列纪录片": 29078,
    #         "网络纪录片": 29082,
    #         "纪实栏目": 29083,
    #
    # }
    # for provider in provider_dic:
    #     crlawler_iqiyi.list_page(releaserUrl=provider_dic[provider],
    #                              video_list_xpath="/html[1]/body[1]/div[6]/div[1]/div[2]/div",
    #                              roll=1000,
    #                              style_tags=provider,
    #                              category_id=str(provider_dic[provider])
    #                              # provider=provider
    #                              )
    # executor = ProcessPoolExecutor(max_workers=4)
    # futures = []
    # # crlawler_tecnt = CrawlerMain()
    # # process_task = crlawler_bilbili.video_page
    # for one_scan in range(4):
    #     crlawler_tecnt = CrawlerMain()
    #     future = executor.submit(crlawler_tecnt.video_page, task=one_scan)
    #     futures.append(future)
    #
    #     # crlawler_bilbili.video_page(one_scan)
    # executor.shutdown(True)
    # crlawler_iqiyi = CrawlerMain()
    # crlawler_iqiyi.video_page()
    # revise_data()

    # executor = ProcessPoolExecutor(max_workers=4)
    # futures = []
    # for one_scan in range(4):
    #     crlawler_tecnt = CrawlerMain()
    #     future = executor.submit(crlawler_tecnt.video_page_api, task=one_scan)
    #     futures.append(future)
    #
    #     # crlawler_bilbili.video_page(one_scan)
    # executor.shutdown(True)
    # crlawler_iqiyi = CrawlerMain()
    # crlawler_iqiyi.video_page_api()

    # crlawler_iqiyi = CrawlerMain()
    # crlawler_iqiyi.get_vid()
    # crlawler_iqiyi.video_page_seleium()


    # executor = ProcessPoolExecutor(max_workers=4)
    # futures = []
    # for one_scan in range(4):
    #     crlawler_tecnt = CrawlerMain()
    #     future = executor.submit(crlawler_tecnt.comment_like_api, task=one_scan)
    #     futures.append(future)
    # # crlawler_bilbili.video_page(one_scan)
    # executor.shutdown(True)
    crlawler_iqiyi = CrawlerMain()
    crlawler_iqiyi.comment_like_api()