# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 14:19:00 2018
@author: fangyucheng
"""
import os
import re
import asyncio
import time
import copy
import requests
import datetime
import aiohttp
import urllib
try:
from crawler_sys.framework.func_get_releaser_id import *
except:
from func_get_releaser_id import *
from bs4 import BeautifulSoup
from multiprocessing import Pool
from multiprocessing import Process
from crawler.crawler_sys.framework.video_fields_std import Std_fields_video
from crawler.crawler_sys.utils.output_results import retry_get_url
from crawler.crawler_sys.utils.output_results import output_result
from crawler.crawler_sys.utils.trans_str_play_count_to_int import trans_play_count
from crawler.crawler_sys.utils.trans_strtime_to_timestamp import trans_strtime_to_timestamp
from crawler.crawler_sys.utils.trans_duration_str_to_second import trans_duration
from crawler.crawler_sys.utils import connect_with_redis
from crawler.crawler_sys.utils.util_logging import logged
from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_proxy
#
#logging = output_log(page_category='crawler',
# program_info='tudou',)
class Crawler_tudou():
def __init__(self, timeout=None, platform='new_tudou'):
if timeout == None:
self.timeout = 10
else:
self.timeout = timeout
self.platform = platform
self.TotalVideo_num = None
self.midstepurl = None
std_fields = Std_fields_video()
self.video_data = std_fields.video_data
self.video_data['platform'] = self.platform
unused_key_list = ['channel', 'describe', 'repost_count', 'isOriginal']
for key in unused_key_list:
self.video_data.pop(key)
self.list_page_url_lst = ["http://www.tudou.com/api/getfeeds?secCateId=10016&utdid=T8v9EQPOimUCAXL%2FAz0YrDOB&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=10195&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=622736331&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=622769673&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=10116&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=622621940&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=10198&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=622336449&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24",
"http://www.tudou.com/api/getfeeds?secCateId=10051&utdid=XA2EFIGslWoCAWp4y3KXcZh7&page_size=24"]
def video_page(self, url):
video_info = copy.deepcopy(self.video_data)
get_page = requests.get(url)
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
video_info['title'] = soup.find('h1', {'class': 'td-playbase__title'}).span.text
except:
video_info['title'] = None
try:
video_info['releaser'] = soup.find('a',{'class':'td-play__userinfo__name'}).text
except:
video_info['releaser'] = None
try:
midsteptime = soup.find('div',{'class':
'td-play__videoinfo__details-box__time'}).text[:-2]
video_info['release_time'] = int(datetime.datetime.strptime(midsteptime,
'%Y-%m-%d %H:%M:%S').timestamp()*1e3)
except:
video_info['release_time'] = None
try:
video_info['releaserUrl'] = soup.find("a", {"class": "td-play__userinfo__name"})['href']
except:
video_info['releaserUrl'] = None
try:
find_play_count = ' '.join(re.findall('total_vv.*stripe_bottom', page))
replace_comma_pcnt = find_play_count.replace(',', '')
play_count_str = ' '.join(re.findall('total_vv":"\d+', replace_comma_pcnt))
video_info['play_count'] = int(' '.join(re.findall('\d+', play_count_str)))
except:
video_info['play_count'] = 0
try:
find_comment_count = ' '.join(re.findall('total_comment.*recommend', page))
replace_comma_ccnt = find_comment_count.replace(',', '')
comment_count_str = ' '.join(re.findall('total_comment":"\d+', replace_comma_ccnt))
video_info['comment_count'] = int(' '.join(re.findall('\d+', comment_count_str)))
except:
video_info['comment_count'] = 0
try:
find_dura = re.findall('stripe_bottom":"\d+:\d+', page)
dura_str = ' '.join(find_dura).split('":"')[-1]
video_info['duration'] = trans_duration(dura_str)
except:
video_info['duration'] = 0
video_info['fetch_time'] = int(time.time()*1e3)
video_info['url'] = url
print("get video data at %s" % url)
return video_info
def search_page(self, keyword, search_pages_max=30,
output_to_es_raw=False,
output_to_es_register=False,
es_index=None,
doc_type=None):
search_data_Lst=[]
url=('http://www.soku.com/nt/search/q_'
+ keyword
+ '_orderby_2_limitdate_0?spm=a2h0k.8191414.0.0&site=14&page={}'
.format(str(i)) for i in range(1, search_pages_max+1))
for urls in url:
print(urls)
get_page = requests.get(urls)
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
potato = soup.find_all("div", { "class" : "v" })
for data_line in potato:
dura = data_line.find("span", { "class" : "v-time"}).text
duration_str=dura
dl = duration_str.split(':')
dl_int = []
for v in dl:
v = int(v)
dl_int.append(v)
if len(dl_int) == 2:
duration = dl_int[0]*60 + dl_int[1]
else:
duration = dl_int[0]*3660+dl_int[1]*60+dl_int[2]
url = data_line.find('div',{'class':'v-meta-title'}).a['href']
url = 'http:'+url
one_video_dic = self.video_page(url)
one_video_dic['url'] = url
one_video_dic['duration']=duration
search_data_Lst.append(one_video_dic)
print('get page done')
if len(search_data_Lst) >= 100:
output_result(result_Lst=search_data_Lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
search_data_Lst.clear()
if search_data_Lst != []:
output_result(result_Lst=search_data_Lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
return search_data_Lst
def process_one_video(self, line):
video_info = copy.deepcopy(self.video_data)
try:
video_info['title'] = line.find('a', {'target':'video'})['title']
except:
video_info['title'] = None
try:
url = line.find('a', {'target':'video'})['href']
video_info['url'] = 'https:' + url
except:
video_info['url'] = None
try:
play_count_str = line.find('span', {'class': 'v-num'}).text
video_info['play_count'] = trans_play_count(play_count_str)
except:
video_info['play_count'] = 0
# logging.warning("can't get play_count at page %s" % video_info['url'])
try:
release_time_str = line.find('span', {'class': 'v-publishtime'}).text
video_info['release_time'] = trans_strtime_to_timestamp(input_time=release_time_str,
missing_year=True)
except:
release_time_str = 0
# logging.warning("can't get release_time at page %s" % video_info['url'])
try:
dura_str = line.find('span', {'class': 'v-time'}).text
video_info['duration'] = trans_duration(dura_str)
except:
video_info['duration'] = 0
# logging.warning("can't get duration at page %s" % video_info['url'])
fetch_time = int(time.time()*1e3)
video_info['fetch_time'] = fetch_time
return video_info
def get_releaser_id(self, releaserUrl):
return get_releaser_id(platform=self.platform,releaserUrl=releaserUrl)
def releaser_page_via_pc(self, releaserUrl,
output_to_file=False,
filepath=None,
releaser_page_num_max=1000,
output_to_es_raw=False,
output_to_es_register=False,
push_to_redis=False,
es_index=None,
doc_type=None):
"""
input releaserUrl must be strict as https://id.tudou.com/i/UMjc5MzI5NDA==/videos?
end with /videos otherwise when scrolling it will make mistakes
"""
releaser_id = self.get_releaser_id(releaserUrl)
print("working on releaser: %s" % releaser_id)
releaserUrl = 'https://id.tudou.com/i/%s/videos' % releaser_id
result_lst = []
get_page = retry_get_url(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
releaser = soup.find('div', {'class': 'user-name'}).a.text
except:
releaser = None
try:
total_video_num_str = soup.find('div', {'class': 'title'}).span.text
total_video_num = total_video_num_str.replace('(', '').replace(')', '').replace(',', '')
total_video_num = trans_play_count(total_video_num)
except:
print(releaserUrl)
if total_video_num%50 == 0:
total_page_num = int(total_video_num/50)
else:
total_page_num = int(total_video_num/50) + 1
if releaser_page_num_max > total_page_num:
releaser_page_num_max = total_page_num
print("releaser page num max is %s" % releaser_page_num_max)
video_lst = soup.find_all('div', {'class': 'v'})
for line in video_lst:
video_info = self.process_one_video(line)
video_info['releaserUrl'] = releaserUrl
video_info['releaser'] = releaser
result_lst.append(video_info)
if releaser_page_num_max >= 2:
page_num = 2
try:
partial_releaserUrl = soup.find('li',{'class':'next'}).a['href']
new_releaserUrl = 'https://id.tudou.com%s' % partial_releaserUrl
except:
print(new_releaserUrl)
while page_num <= releaser_page_num_max:
get_page = retry_get_url(new_releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
if page_num != releaser_page_num_max:
try:
new_releaserUrl = 'https://id.tudou.com' + soup.find('li',{'class':'next'}).a['href']
except:
new_releaserUrl = ('https://id.tudou.com/i/%s/videos?order=1&page=%s' % (releaser_id, page_num))
video_lst = soup.find_all('div', {'class': 'v'})
for line in video_lst:
video_info = self.process_one_video(line)
video_info['releaserUrl'] = releaserUrl
video_info['releaser'] = releaser
result_lst.append(video_info)
print('get page %s list length is %s' % (page_num, len(result_lst)))
page_num += 1
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis)
result_lst.clear()
if result_lst != []:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
es_index=es_index,
doc_type=doc_type)
def get_releaser_follower_num(self, releaserUrl):
get_page = requests.get(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
follower_str = soup.find('li',{'class':'snum'}).em.text
follower_num = trans_play_count(follower_str)
print('%s follower number is %s' % (releaserUrl, follower_num))
releaser_img = self.get_releaser_image(data=soup)
return follower_num,releaser_img
except:
print("can't can followers")
def releaser_video_sum(self, releaserUrl):
get_page = retry_get_url(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
total_video_num_str = soup.find('div', {'class':'title'}).span.text
total_video_num = total_video_num_str.replace('(', '').replace(')', '').replace(',', '')
total_video_num = trans_play_count(total_video_num)
return total_video_num
def start_list_page(self, task_list,
retry_times=20):
url_lst = []
count = 0
while count < retry_times:
count += 1
for list_url in task_list:
get_page = requests.get(list_url)
try:
video_lst = get_page.json()
except:
continue
else:
for video_dic in video_lst:
video_id = video_dic['codeId']
url = 'http://video.tudou.com/v/%s.html' % video_id
url_lst.append(url)
print('push %s video url into redis set' % len(url_lst))
connect_with_redis.push_video_url_to_redis_set(platform=self.platform, url_lst=url_lst)
async def download_html(self, session, url):
get_page = await session.get(url)
page = await get_page.text("utf-8", errors="ignore")
return url + "fangyuchenggoalkeeper" + page
async def get_video_page(self, loop, task_lst):
async with aiohttp.ClientSession() as session:
video_page = [loop.create_task(self.download_html(session, url))
for url in task_lst]
done, pending = await asyncio.wait(video_page)
download_video_page_lst = [d.result() for d in done]
connect_with_redis.push_video_page_html_to_redis(platform=self.platform,
result_lst=download_video_page_lst)
def download_video_page_async(self):
task_lst = connect_with_redis.retrieve_video_url_from_redis_set(platform=self.platform)
loop = asyncio.get_event_loop()
loop.run_until_complete(self.get_video_page(loop, task_lst))
def download_video_page_async_single_process(self):
key = 'new_tudou_video_url'
pid = os.getpid()
while connect_with_redis.length_of_set(key) > 0:
self.download_video_page_async()
print("platform: %s, action: download video page, pid: %s"
% (self.platform, pid))
def download_video_page_async_multi_process(self, process_num=10):
pool = Pool(processes=process_num)
for line in range(process_num):
pool.apply_async(self.download_video_page_async_single_process)
pool.close()
pool.join()
def parse_video_page_html(self, html):
page_lst = html.split('fangyuchenggoalkeeper')
url = page_lst[0]
page = page_lst[1]
soup = BeautifulSoup(page, 'html.parser')
try:
title = soup.find('h1', {'class': 'td-playbase__title'}).span.text
except:
title = None
try:
releaser = soup.find('a',{'class':'td-play__userinfo__name'}).text
except:
releaser = None
try:
midsteptime = soup.find('div',{'class':
'td-play__videoinfo__details-box__time'}).text[:-2]
release_time = int(datetime.datetime.strptime(midsteptime,
'%Y-%m-%d %H:%M:%S').timestamp()*1e3)
except:
release_time = None
try:
releaserUrl = soup.find("a", {"class": "td-play__userinfo__name"})['href']
except:
releaserUrl = None
try:
find_play_count = ' '.join(re.findall('total_vv.*stripe_bottom', page))
replace_comma = find_play_count.replace(',', '')
play_count_str = ' '.join(re.findall('total_vv":"\d+', replace_comma))
play_count = int(' '.join(re.findall('\d+', play_count_str)))
except:
play_count = 0
try:
find_dura = re.findall('stripe_bottom":"\d+:\d+', page)
dura_str = ' '.join(find_dura).split('":"')[-1]
duration = trans_duration(dura_str)
except:
duration = 0
fetch_time = int(time.time()*1e3)
info_dic = {'platform': self.platform,
"title": title,
'url': url,
'duration': duration,
"releaser": releaser,
"release_time": release_time,
"releaserUrl": releaserUrl,
'play_count': play_count,
'fetch_time': fetch_time}
return info_dic
def parse_video_page_single_process(self,
output_to_file=False,
filepath=None,
push_to_redis=False,
output_to_es_raw=False,
es_index=None,
doc_type=None,
output_to_es_register=False):
error_url_list = []
pid = os.getpid()
key = 'new_tudou_video_page_html'
result_lst = []
while connect_with_redis.length_of_lst(key) > 0:
video_page_html = connect_with_redis.retrieve_video_page_html_from_redis(platform=self.platform)
video_info = self.parse_video_page_html(video_page_html)
try:
video_info['title']
except:
continue
if video_info['title'] is not None:
result_lst.append(video_info)
print("platform: %s, action: parse video page, process_id: %s,"
"count number: %s" % (self.platform, pid, len(result_lst)))
if len(result_lst) >= 1000:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
push_to_redis=push_to_redis,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type,
output_to_es_register=output_to_es_register)
result_lst.clear()
else:
try:
error_url_list.append(video_info['url'])
except:
pass
if result_lst != []:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
push_to_redis=push_to_redis,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type,
output_to_es_register=output_to_es_register)
if error_url_list != []:
connect_with_redis.push_video_url_to_redis_set(platform=self.platform,
url_lst=error_url_list)
def parse_video_page_multi_process(self,
para_dict,
process_num=30):
pool = Pool(processes=process_num)
for line in range(process_num):
pool.apply_async(self.parse_video_page_single_process, kwds=para_dict)
pool.close()
pool.join()
def key_customer(self, releaserUrl,
releaser_page_num_max=1000,
output_to_es_raw=False,
es_index='crawler-data-raw',
doc_type='doc'):
"""
input releaserUrl must be strict as https://id.tudou.com/i/UMjc5MzI5NDA==/videos?
end with /videos otherwise when scrolling it will make mistakes
"""
releaser_id = self.get_releaser_id(releaserUrl)
print("working on releaser: %s" % releaser_id)
releaserUrl = 'https://id.tudou.com/i/%s/videos' % releaser_id
result_lst = []
get_page = retry_get_url(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
releaser = soup.find('div', {'class': 'user-name'}).a.text
except:
releaser = None
try:
total_video_num_str = soup.find('div', {'class': 'title'}).span.text
total_video_num = total_video_num_str.replace('(', '').replace(')', '').replace(',', '')
total_video_num = trans_play_count(total_video_num)
except:
print(releaserUrl)
if total_video_num % 50 == 0:
total_page_num = int(total_video_num/50)
else:
total_page_num = int(total_video_num/50) + 1
if releaser_page_num_max > total_page_num:
releaser_page_num_max = total_page_num
print("releaser page num max is %s" % releaser_page_num_max)
video_lst = soup.find_all('div', {'class': 'v'})
for line in video_lst:
video_info = self.process_one_video(line)
video_info['releaserUrl'] = releaserUrl
video_info['releaser'] = releaser
result_lst.append(video_info)
if releaser_page_num_max >= 2:
page_num = 2
try:
partial_releaserUrl = soup.find('li',{'class':'next'}).a['href']
new_releaserUrl = 'https://id.tudou.com%s' % partial_releaserUrl
except:
print(new_releaserUrl)
while page_num <= releaser_page_num_max:
get_page = retry_get_url(new_releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
if page_num != releaser_page_num_max:
try:
new_releaserUrl = 'https://id.tudou.com' + soup.find('li',{'class':'next'}).a['href']
except:
new_releaserUrl = ('https://id.tudou.com/i/%s/videos?order=1&page=%s' % (releaser_id, page_num))
video_lst = soup.find_all('div', {'class': 'v'})
for line in video_lst:
video_info = self.process_one_video(line)
video_info['releaserUrl'] = releaserUrl
video_info['releaser'] = releaser
result_lst.append(video_info)
print('get page %s list length is %s' % (page_num, len(result_lst)))
page_num += 1
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type)
result_lst.clear()
if result_lst != []:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type)
result_lst.clear()
def get_releaser_image(self,releaserUrl=None,data=None):
if releaserUrl:
get_page = requests.get(releaserUrl)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
image_url = soup.find('a', {'class': 'user-avatar'}).next_element["src"]
print(image_url)
return image_url
except:
print("can't get image")
else:
image_url = data.find('a', {'class': 'user-avatar'}).next_element["src"]
print(image_url)
return image_url
@staticmethod
def get_video_image(data):
video_photo_url = data["cover"]["big"]["url"]
return video_photo_url
def get_releaser_name(self, releaserUrl):
"""
Due to the function releaser_page can't get releaser name from api,
I add a function to get it from web page
posted by yucheng fang
"""
get_page = requests.get(releaserUrl,proxies=get_proxy(1))
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
try:
releaser = soup.find('div', {'class': 'user-name'}).a.text
except:
print("can't get releaser name at soup.find('div', {'class': 'user-name'}).a.text")
if releaser is not None:
print("get releaser name at soup.find('div', {'class': 'user-name'}).a.text")
return releaser
else:
print("get releaser name at soup.find('div', {'class': 'user-name'}).a.text")
try:
releaser = soup.find('div', {'class': 'head-avatar'}).a['title']
except:
print("can't get releaser name at soup.find('div', {'class': 'head-avatar'}).a['title']")
if releaser is not None:
return releaser
else:
print("can't get releaser name at soup.find('div', {'class': 'head-avatar'}).a['title']")
return None
def rebuild_releaserUrl(self, releaserUrl):
get_page = requests.get(releaserUrl)
page = get_page.text
soup = BeautifulSoup(page, 'html.parser')
releaser_id = soup.find('div', {'class': 'user-name'}).a['href']
url = 'https://id.tudou.com' + releaser_id + '/videos'
return url
def releaser_page_web(self, releaserUrl,
output_to_file=False,
filepath=None,
releaser_page_num_max=30,
output_to_es_raw=False,
es_index=None,
doc_type=None,
output_to_es_register=False,
push_to_redis=False, proxies_num=None):
releaser_id = self.get_releaser_id(releaserUrl)
# releaser = self.get_releaser_name(releaserUrl)
releaserUrl = 'https://id.tudou.com/i/%s/videos' % releaser_id
json_headers = {
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh,zh-CN;q=0.9",
# "cookie": "cna=W99aFOvX+QACAXL4fBJI3rAw; __ysuid=1541219939103JPW; ykss=e93bad5ef9c26af71c8e7ee5; P_ck_ctl=47F163FE35A5B1B2E479B158A12376A7; __ayvstp=16; __aysvstp=16; _zpdtk=ecd18a6d5d86a28b786b653356133cfb606dd1dc; isg=BOzsOnpUnhIGhYq8YxHgZ36EvcoepZBPH_JJJ0Yt-Rc6UY5bbrVJ3rr3dxdpWcin",
"referer": releaserUrl,
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
"x-csrf-token": "ecd18a6d5d86a28b786b653356133cfb606dd1dc",
"x-requested-with": "XMLHttpRequest",
}
json_cookies = {
"cna": "W99aFOvX+QACAXL4fBJI3rAw",
"__ysuid": "1541219939103JPW",
"ykss": "e93bad5ef9c26af71c8e7ee5",
"P_ck_ctl": "47F163FE35A5B1B2E479B158A12376A7",
"__ayvstp": "16",
"__aysvstp": "16",
"_zpdtk": "ecd18a6d5d86a28b786b653356133cfb606dd1dc",
"isg": "BOzsOnpUnhIGhYq8YxHgZ36EvcoepZBPH_JJJ0Yt-Rc6UY5bbrVJ3rr3dxdpWcin",
}
firsh_page_headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh,zh-CN;q=0.9",
#"cookie": "cna=W99aFOvX+QACAXL4fBJI3rAw; __ysuid=1541219939103JPW; ykss=e93bad5ef9c26af71c8e7ee5; P_ck_ctl=47F163FE35A5B1B2E479B158A12376A7; __ayvstp=16; __aysvstp=16; _zpdtk=9053e5d58ee0c51b1f3da8008dd4bda164ecd846; isg=BHl5FRo0A8WDkd_DnlItMBsXiOVThm042sF8-Juu9KAfIpu049ZUCb80oCjUmgVw",
"referer": releaserUrl,
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "same-origin",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1",
}
first_page_res = retry_get_url(releaserUrl, headers=firsh_page_headers, proxies=proxies_num)
json_cookies.update(dict(first_page_res.cookies))
user_id = re.findall('uid="(\d+)"', first_page_res.text)[0]
zptk_url = "https://id.tudou.com/i/h5/id_%s/playlisttab?uid=%s" % (user_id, user_id)
playlisttab_res = retry_get_url(zptk_url, headers=json_headers, proxies=proxies_num,cookies=json_cookies)
# print(dict(playlisttab_res.cookies))
json_cookies.update(dict(playlisttab_res.cookies))
json_headers["x-csrf-token"] = dict(playlisttab_res.cookies)["_zpdtk"]
count = 1
retry_time = 0
result_list = []
self.video_data['releaserUrl'] = releaserUrl
print("working on releaser_id: %s" % (releaser_id))
while count <= releaser_page_num_max and retry_time < 5:
proxies = get_proxy(proxies_num)
api_url = 'https://id.tudou.com/i/h5/id_%s/videos?ajax=1&pn=%s&pl=20' % (user_id,count)
print(api_url)
if proxies:
get_page = requests.get(api_url, headers=json_headers, proxies=proxies, timeout=3,cookies=json_cookies)
else:
get_page = requests.get(api_url, headers=json_headers, timeout=3,cookies=json_cookies)
_zpdtk = dict(get_page.cookies)
json_cookies.update(_zpdtk)
# print(dict(get_page.cookies))
json_headers["x-csrf-token"] = _zpdtk["_zpdtk"]
page_dic = get_page.json()
releaser_page_num_max = page_dic["page"]["pz"]
releaser = page_dic['channelOwnerInfo']["data"]["nickname"]
# has_more = page_dic.get('has_more')
try:
data_list = page_dic['data']["data"]
time.sleep(0.25)
except:
retry_time += 1
time.sleep(0.25)
print("no more data at page: %s try_time: %s" % (count, retry_time))
continue
if data_list == []:
retry_time += 1
time.sleep(0.25)
print("no more data at page: %s try_time: %s" % (count, retry_time))
continue
else:
retry_time = 0
print("get data at page: %s" % (count))
count += 1
for info_dic in data_list:
video_info = copy.deepcopy(self.video_data)
if type(info_dic) == str:
info_dic = data_list[info_dic]
video_info['video_id'] = info_dic["videoid"]
video_info['title'] = info_dic["title"]
video_info['releaser'] = releaser
video_info['url'] = 'https://video.tudou.com/v/%s.html' % info_dic["videoid"]
video_info['duration'] = int(info_dic.get('seconds')/1e3)
video_info['releaser_id_str'] = "new_tudou_%s" % (releaser_id)
video_info['comment_count'] = int(info_dic.get('total_comment'))
video_info['favorite_count'] = int(info_dic.get('total_up'))
# favorite_count in database means 点赞数, while in web page the factor
# named praiseNumber
# in web page facorite_count means 收藏数
video_info['video_img'] = info_dic.get('thumburl')
video_info['play_count'] = info_dic.get('total_vv')
video_info['release_time'] = int(info_dic.get('publishtime')*1e3)
# print(video_info['release_time'])
# if '天前' in release_time_str:
# video_info['release_time'] = self.video_page(video_info['url'])['release_time']
# else:
# video_info['release_time'] = trans_strtime_to_timestamp(input_time=release_time_str,
# missing_year=True)
video_info['fetch_time'] = int(time.time() * 1e3)
yield video_info
# @logged
def releaser_page(self, releaserUrl,
output_to_file=False,
filepath=None,
releaser_page_num_max=4000,
output_to_es_raw=False,
es_index=None,
doc_type=None,
output_to_es_register=False,
push_to_redis=False,
proxies_num=None
):
"""
get video info from api instead of web page html
the most scroll page is 1000
"""
headers = {'Host': 'apis.tudou.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate',
'Connection': 'keep-alive',
'Cookie': ('isg=BIeH6gcJlwZw_xQESm9jlG-vFTuRJGXxikf0g1l0mJY9yKeKYVuAvzKJbkgzOzPm;'
'cna=XA2EFIGslWoCAWp4y3KXcZh7; ykss=cdbd115c102a68710215ad93;'
'__ysuid=1543316262167mjE; P_ck_ctl=62DE1D55DFE1C0F4F27A8662E6575F08;'
'__ayvstp=32'),
'Upgrade-Insecure-Requests': '1',
'Cache-Control': 'max-age=0'}
count = 1
# has_more = True
retry_time = 0
result_list = []
releaser_id = self.get_releaser_id(releaserUrl)
releaser = self.get_releaser_name(releaserUrl)
releaserUrl = 'https://id.tudou.com/i/%s/videos' % releaser_id
self.video_data['releaser'] = releaser
self.video_data['releaserUrl'] = releaserUrl
url_dic = {"uid": releaser_id,
"pL": "20"}
print("working on releaser: %s releaser_id: %s" % (releaser, releaser_id))
while count <= releaser_page_num_max and retry_time < 5:
proxies = get_proxy(proxies_num)
url_dic['pg'] = str(count)
url_dic['pn'] = str(count)
api_url = 'http://apis.tudou.com/subscribe/v1/video?%s' % urllib.parse.urlencode(url_dic)
# print(api_url)
if proxies:
get_page = requests.get(api_url, headers=headers,proxies=proxies,timeout=5)
else:
get_page = requests.get(api_url, headers=headers,timeout=5)
page_dic = get_page.json()
# has_more = page_dic.get('has_more')
try:
data_list = page_dic['entity']
except:
retry_time += 1
proxies = get_proxy(1)
time.sleep(0.25)
print("no more data at releaser: %s page: %s try_time: %s" % (releaser, count, retry_time))
continue
if data_list == []:
retry_time += 1
proxies = get_proxy(1)
time.sleep(0.25)
print("no more data at releaser: %s page: %s try_time: %s" % (releaser, count, retry_time))
continue
else:
retry_time = 0
print("get data at releaser: %s page: %s" % (releaser, count))
count += 1
for info_dic in data_list:
video_info = copy.deepcopy(self.video_data)
one_video = info_dic.get('detail')
if one_video is not None:
get_title = one_video.get('base_detail')
if get_title is not None:
video_info['title'] = get_title.get('title')
detail_info = one_video.get('video_detail')
if detail_info is not None:
video_id = detail_info.get('video_id')
if video_id is not None:
video_info['video_id'] = video_id
video_info['url'] = 'https://video.tudou.com/v/%s.html' % video_id
video_info['duration'] = detail_info.get('duration')
video_info['releaser_id_str'] = "new_tudou_%s" % (releaser_id)
video_info['comment_count'] = int(detail_info.get('comment_count'))
video_info['favorite_count'] = int(detail_info.get('praiseNumber'))
#favorite_count in database means 点赞数, while in web page the factor
#named praiseNumber
#in web page facorite_count means 收藏数
video_info['shoucang_count'] = (detail_info.get('favorite_count'))
# print('play_count', detail_info.get('vv_count'))
video_info['play_count'] = detail_info.get('vv_count')
video_info['video_img'] = self.get_video_image(detail_info)
release_time_str = detail_info.get('publish_time')
print(release_time_str)
# print(video_info['release_time'])
if '天前' in release_time_str:
video_info['release_time'] = self.video_page(video_info['url'])['release_time']
else:
video_info['release_time'] = trans_strtime_to_timestamp(input_time=release_time_str,
missing_year=True)
video_info['fetch_time'] = int(time.time() * 1e3)
yield video_info
else:
continue
def releaser_page_by_time(self, start_time, end_time, url,**kwargs):
data_lis = []
count_false = 0
output_to_file = kwargs.get("output_to_file")
filepath = kwargs.get("filepath")
push_to_redis = kwargs.get("push_to_redis")
output_to_es_register = kwargs.get("output_to_es_register")
output_to_es_raw = kwargs.get("output_to_es_raw")
es_index = kwargs.get("es_index")
doc_type = kwargs.get("doc_type")
for res in self.releaser_page_web(url,proxies_num=kwargs.get("proxies_num")):
video_time = res["release_time"]
# print(res)
if video_time:
if start_time < video_time:
if video_time < end_time:
data_lis.append(res)
if len(data_lis) >= 100:
output_result(result_Lst=data_lis,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
push_to_redis=push_to_redis,
output_to_es_register=output_to_es_register,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type)
data_lis.clear()
else:
count_false += 1
if count_false > 200:
break
else:
continue
if data_lis != []:
output_result(result_Lst=data_lis,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
push_to_redis=push_to_redis,
output_to_es_register=output_to_es_register,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type)
# test
if __name__=='__main__':
test = Crawler_tudou()
# url = 'https://video.tudou.com/v/XNDExNjcyNTI0MA==.html'
releaser_url = "https://id.tudou.com/i/UNTUzMTU1ODg2OA=="
# ttt = test.video_page("https://video.tudou.com/v/XNDExNjcyNTI0MA==.html")
#releaserUrl=url, output_to_es_raw=True,
# es_index='crawler-data-raw',
# doc_type='doc',
# releaser_page_num_max=100)
test.releaser_page_by_time(1569081600000,1588421876480 ,releaser_url,output_to_es_raw=True,es_index='crawler-data-raw',
doc_type='doc',releaser_page_num_max=4000)
# test.get_releaser_image(releaser_url)
# test.get_releaser_follower_num()