# -*- coding: utf-8 -*-
"""
Created on Wed Mar 28 14:13:32 2018
#cid 可以用来抓取弹幕
#search_page 抓不到 releaserurl
@author: fangyucheng
"""
import requests
import re
import json
import datetime
import time
import copy
from bs4 import BeautifulSoup
from crawler.crawler_sys.framework.video_fields_std import Std_fields_video
from crawler.crawler_sys.utils.output_results import bulk_write_into_es
from crawler.crawler_sys.utils.output_results import retry_get_url
from crawler.crawler_sys.utils.output_results import output_result
from crawler.crawler_sys.utils.trans_format import str_lst_to_file
from crawler.crawler_sys.proxy_pool.connect_with_database import extract_data_to_use
from crawler.crawler_sys.proxy_pool.connect_with_database import update_status
from crawler.crawler_sys.proxy_pool.build_proxy_dic import build_proxy_dic
from crawler.crawler_sys.utils.util_logging import logged
class Crawler_bilibili(Std_fields_video):
def __init__(self, timeout=None, platform='bilibili'):
if timeout==None:
self.timeout = 10
else:
self.timeout = timeout
self.platform = platform
std_fields = Std_fields_video()
self.video_data = std_fields.video_data
self.video_data['platform'] = self.platform
# remove fields that crawled data don't have
pop_key_Lst = ['repost_count', 'isOriginal',]
for popk in pop_key_Lst:
self.video_data.pop(popk)
self.lst_name_rid_dict = {'国产动画': '153',
'搞笑': '138',
'影视杂谈': '182',
'纪录片': {'人文历史': '37',
'科学探索自然': '178',
'军事': '179',
'社会美食旅行': '180'},
'游戏': {'单机游戏': '17'}}
self.headers = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Cookie':'fts=1502427559; buvid3=E8FDA203-70E1-48A6-BE29-E2B833F92DB314456infoc; biliMzIsnew=1; biliMzTs=null; sid=534m3oqx; CNZZDATA2724999=cnzz_eid%3D1621908673-1502776053-https%253A%252F%252Fwww.baidu.com%252F%26ntime%3D1521001760; pgv_pvi=2734144512; rpdid=olilkosokkdoswsqmikqw; LIVE_BUVID=c552bf4415d1fba581d231647ba7b1bf; LIVE_BUVID__ckMd5=d8118a88b8f0fa8b; UM_distinctid=161e545d99a9-07c0b73fa83f93-17357940-1fa400-161e545d99b224; DedeUserID=114627314; DedeUserID__ckMd5=073268b15392f951; SESSDATA=4b30a63b%2C1524982595%2Cd78acc24; bili_jct=06e47d618fff20d978b968f15b3271c5; finger=c650951b; BANGUMI_SS_24014_REC=202051; _dfcaptcha=ca1c709bb04bda0240e4771eb8d90871',
'Host': 'www.bilibili.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0'}
def video_page(self, url):
aid = url.split('av')[1]
if len(aid) != 7:
aid = aid[0:7]
get_page = requests.get(url, headers=self.headers)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page,'html.parser')
title = soup.find('h1').text
try:
video_intro = soup.find('div',{'class':'info open'}).text
except:
video_intro = None
try:
releaser = soup.find('div',{'class':"user clearfix"}).a.text
releaserUrl = soup.find('div',{'class':"user clearfix"}).a['href']
except:
releaser = None
releaserUrl = None
duration = re.findall(r'"duration":[0-9]{1,10}',
' '.join(re.findall(r'videoData.*"duration":[0-9]{1,10}', page)))[0].split(':')[1]
getcid = re.findall(r'cid=.*&aid='+aid,page)[0]
cid = getcid.split('&')[0].split('=')[1]
interfaceurl = ('https://interface.bilibili.com/player?id=cid:'
+ cid + '&aid=' + aid)
get_api_page = requests.get(interfaceurl)
page = get_api_page.text
soup = BeautifulSoup(page,'html.parser')
play_count = soup.click.text
release_time=soup.time.text
fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now()) * 1e3)
D0 = copy.deepcopy(self.video_data)
D0['title'] = title
D0['play_count'] = play_count
D0['releaser'] = releaser
D0['fetch_time'] = fetch_time
D0['describe'] = video_intro
D0['release_time'] = release_time
D0['duration'] = duration
D0['releaserUrl'] = releaserUrl
D0['url'] = url
return D0
def search_page(self, keyword, search_pages_max=30,
output_to_es_raw=False,
output_to_es_register=False,
es_index=None,
doc_type=None):
search_lst = []
def get_one_video(video_dict):
title = video_dict['title']
title = title.replace('<em class="keyword">', '')
title = title.replace('</em>', '')
aid = video_dict['aid']
url = video_dict['arcurl']
releaser = video_dict['author']
video_intro = video_dict['description']
dura = video_dict['duration']
dura_lst = dura.split(':')
duration = int(dura_lst[0])*60 + int(dura_lst[1])
play_count = video_dict['play']
comment_count = video_dict['video_review']
favorite_count = video_dict['favorites']
release_time = int(video_dict['pubdate'] * 1e3)
tag = video_dict['tag']
D0 = copy.deepcopy(self.video_data)
D0['title'] = title
D0['play_count'] = play_count
D0['favorite_count'] = favorite_count
D0['comment_count'] = comment_count
D0['releaser'] = releaser
D0['describe'] = video_intro
D0['release_time'] = release_time
D0['duration'] = duration
D0['url'] = url
return D0
first_url = ('https://api.bilibili.com/x/web-interface/search/type?'
'jsonp=jsonp&search_type=video'
'&keyword=%s' % keyword)
if search_pages_max == 1:
search_urls = [first_url]
else:
search_gen = ('https://api.bilibili.com/x/web-interface/search/type?'
'jsonp=jsonp&search_type=video&keyword='
+ keyword
+ '&page={}'.format(
str(i)) for i in range(2, search_pages_max+1))
search_urls = [first_url]
search_urls.extend(list(search_gen))
for s_url in search_urls:
print(s_url)
get_page = requests.get(s_url)
page_dict = get_page.json()
video_dicts = page_dict['data']['result']
for video_dict in video_dicts:
one_video_dict = get_one_video(video_dict)
search_lst.append(one_video_dict)
if len(search_lst) >= 100:
output_result(result_Lst=search_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
search_lst.clear()
if search_lst != []:
output_result(result_Lst=search_lst,
platform=self.platform,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
es_index=es_index,
doc_type=doc_type)
return search_lst
def write_into_file(self, data_dict, file_obj):
json_str=json.dumps(data_dict)
file_obj.write(json_str)
file_obj.write('\n')
file_obj.flush()
def feed_url_into_redis(self, dict_Lst):
pass
def output_result(self, result_Lst, output_to_file=False, filepath=None):
# write data into es crawler-raw index
bulk_write_into_es(result_Lst)
# feed url into redis
self.feed_url_into_redis(result_Lst)
# output into file according to passed in parameters
if output_to_file==True and filepath!=None:
output_fn='crawler_toutiao_%s_json' % datetime.datetime.now().isoformat()[:10]
output_f=open(filepath+'/'+output_fn, 'a', encoding='utf-8')
self.write_into_file(result_Lst, output_f)
else:
pass
def get_releaser_id(self,releaserUrl):
rid = ' '.join(re.findall('[0-9]+',releaserUrl))
return rid
@logged
def releaser_page(self, releaserUrl, release_time_upper_bdry=None,
release_time_lower_bdry=None, output_to_file=False, filepath=None):
if release_time_upper_bdry == None:
select_max = int((datetime.datetime.now()+datetime.timedelta(day=7)).timestamp()*1e3)
else:
select_max = int(datetime.datetime.strptime(release_time_upper_bdry,'%y-%m-%d').timestamp()*1e3)
if release_time_lower_bdry == None:
select_min = 0
else:
select_min = int(datetime.datetime.strptime(release_time_lower_bdry,'%y-%m-%d').timestamp()*1e3)
rid = self.get_releaser_id(releaserUrl)
result_Lst=[]
count = 0
firsturl='https://space.bilibili.com/ajax/member/getSubmitVideos?mid='+rid
get_page = requests.get(firsturl)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('false','False')
page = page.replace('true','True')
page = page.replace('null','None')
page_dic = eval(page)
totalvideo = int(page_dic['data']['count'])
if int(totalvideo/20)==totalvideo/20:
pagenum=int(totalvideo/20)
else:
pagenum=int(totalvideo/20)+1
while count <= 30:
for i in range(1,pagenum+1):
url='https://space.bilibili.com/ajax/member/getSubmitVideos?mid='+rid+'&page='+str(i)
get_page = requests.get(url)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('false','False')
page = page.replace('true','True')
page = page.replace('null','None')
page_dic = eval(page)
video_dic = page_dic['data']['vlist']
for one_video in video_dic:
release_time = one_video['created']*1e3
if release_time >= select_min and release_time < select_max:
title = one_video['title']
aid = one_video['aid']
url = 'https://www.bilibili.com/video/av'+str(aid)
releaser = one_video['author']
#video_intro = one_video['description']
dura = one_video['length']
dura_lst = dura.split(':')
duration = int(dura_lst[0])*60+int(dura_lst[1])
play_count = one_video['play']
comment_count = one_video['video_review']
favorite_count = one_video['favorites']
fetch_time = int(datetime.datetime.now().timestamp()*1e3)
self.video_data['platform']='bilibili'
self.video_data['title']=title
self.video_data['duration']=duration
self.video_data['url']=url
self.video_data['play_count']=play_count
self.video_data['comment_count']=comment_count
self.video_data['favorite_count']=favorite_count
self.video_data['video_id']=aid
self.video_data['releaser']=releaser
self.video_data['releaser_id']=int(rid)
self.video_data['release_time']=release_time
self.video_data['fetch_time']=fetch_time
print('get one video')
result_Lst.append(self.video_data)
if len(result_Lst)%100==0:
self.output_result(result_Lst,output_to_file=output_to_file,
filepath=filepath)
result_Lst.clear()
elif release_time > select_max:
print('publish after requirement')
elif release_time <select_min:
count+=1
print('publish before requirement')
self.output_result(result_Lst)
def list_page(self, rid,
page_num=1,
channel=None,
output_to_file=False,
filepath=None,
output_to_es_raw=False,
output_to_es_register=False,
push_to_redis=False,
page_num_max=34,
output_es_index=None,
output_doc_type=None,
proxy_dic=None):
result_lst = []
fail_time = 0
while page_num <= page_num_max and fail_time < 5:
lst_url = ('https://api.bilibili.com/x/web-interface/newlist?rid='
+ rid + '&type=0&pn=' + str(page_num) + '&ps=20')
if proxy_dic is not None:
raw_proxy_dic = extract_data_to_use()
record_id = raw_proxy_dic['id']
proxy_dic = build_proxy_dic(raw_proxy_dic=raw_proxy_dic)
print('get proxy_dic %s' % proxy_dic)
try:
get_page = retry_get_url(lst_url, proxies=proxy_dic, timeout=15)
fail_time = 0
page_num += 1
except:
update_status(record_id=record_id,
availability=0)
fail_time += 1
print('%s has failed %s times' % (lst_url, fail_time))
continue
print('get page at %s' % (page_num-1))
page_dic = get_page.json()
total_video = int(page_dic['data']['page']['count'])
if page_num == 1:
if int(total_video/20) == total_video/20:
total_page_num = int(total_video/20)
else:
total_page_num = int(total_video/20) + 1
if total_page_num <= page_num_max:
page_num_max = total_page_num
video_dic = page_dic['data']['archives']
for one_video in video_dic:
video_dic = copy.deepcopy(self.video_data)
video_dic['title'] = one_video['title']
aid = one_video['aid']
video_dic['aid'] = one_video['aid']
try:
attribute = one_video['attribute']
except:
attribute=0
video_dic['attribute'] = attribute
video_dic['url'] = 'https://www.bilibili.com/video/av' + str(aid)
video_dic['releaser'] = one_video['owner']['name']
video_dic['releaser_id'] = one_video['owner']['mid']
video_dic['video_intro'] = one_video['desc']
video_dic['duration'] = one_video['duration']
video_dic['play_count'] = one_video['stat']['view']
video_dic['danmuku'] = one_video['stat']['danmaku']
video_dic['release_time'] = (one_video['pubdate'])*1e3
fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
video_dic['fetch_time'] = fetch_time
result_lst.append(video_dic)
if len(result_lst) >= 100:
if output_es_index is None and output_doc_type is None:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis)
result_lst.clear()
elif output_es_index is not None and output_doc_type is not None:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
es_index=output_es_index,
doc_type=output_doc_type)
result_lst.clear()
time.sleep(0)
if result_lst != []:
if output_es_index is None and output_doc_type is None:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis)
elif output_es_index is not None and output_doc_type is not None:
output_result(result_Lst=result_lst,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
output_to_es_register=output_to_es_register,
push_to_redis=push_to_redis,
es_index=output_es_index,
doc_type=output_doc_type)
return result_lst
def fromdeathtoborn(self,rid,result_doc,done_doc,pn):
list_page=[]
result=open(result_doc,'a')
done=open(done_doc,'a')
firsturl='https://api.bilibili.com/x/web-interface/newlist?rid='+rid+'&type=0&pn=1&ps=20'
get_page = requests.get(firsturl)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('false','False')
page = page.replace('true','True')
page = page.replace('null','None')
page_dic = eval(page)
totalvideo = int(page_dic['data']['page']['count'])
if int(totalvideo/20)==totalvideo/20:
pagenum=int(totalvideo/20)
else:
pagenum=int(totalvideo/20)+1
for i in range(pn,pagenum+1):
url='https://api.bilibili.com/x/web-interface/newlist?rid='+rid+'&type=0&pn='+str(i)+'&ps=20'
get_page = requests.get(url)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('false','False')
page = page.replace('true','True')
page = page.replace('null','None')
page_dic = eval(page)
video_dic = page_dic['data']['archives']
print(url)
done.write(url)
done.write('\n')
done.flush()
for one_video in video_dic:
title = one_video['title']
aid = one_video['aid']
try:
attribute = one_video['attribute']
except:
attribute=0
url = 'https://www.bilibili.com/video/av'+str(aid)
releaser = one_video['owner']['name']
releaserid = one_video['owner']['mid']
video_intro = one_video['desc']
duration = one_video['duration']
play_count = one_video['stat']['view']
danmuku = one_video['stat']['danmaku']
release_time = (one_video['pubdate'])*1e3
fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
D0={'title':title,'play_count':play_count,'releaser':releaser,'releaserid':releaserid,'video_intro':video_intro,'release_time':release_time,'duration':duration,'url':url,'aid':aid,'fetch_time':fetch_time,'subdomain':rid,'attribute':attribute,'danmuku':danmuku}
json_dic=json.dumps(D0)
result.write(json_dic)
result.write('\n')
list_page.append(D0)
print('get one line')
result.flush()
time.sleep(3)
result.close()
done.close()
return list_page
def get_album(self,count,result_doc,done_doc):
album_lst=[]
result=open(result_doc,'a')
done=open(done_doc,'a')
while count<=26:
url = ('https://bangumi.bilibili.com/media/web_api/search/result?season_type=3&page='+str(count)
+'&pagesize=20&style_id=0&order=0&sort=1&producer_id=0')
get_page = requests.get(url)
get_page.encoding = 'utf-8'
page = get_page.text
page = page.replace('false','False')
page = page.replace('true','True')
page = page.replace('null','None')
page_dic = eval(page)['result']['data']
print(count)
count+=1
done.write(str(count))
done.write('\n')
done.flush()
for line in page_dic:
album_name=line['title']
album_infor=line['index_show']
url=line['link']
get_page = requests.get(url)
get_page.encoding = 'utf-8'
page = get_page.text
soup = BeautifulSoup(page,'html.parser')
try:
releaser = soup.find('div',{'class':"user clearfix"}).a.text
releaserurl = soup.find('div',{'class':"user clearfix"}).a['href']
except:
releaser = 'bilibili纪录片'
releaserurl = 'https://space.bilibili.com/7584632'
getid=re.findall(r'"epList".*"newestEp"',page)
id_lst=getid[0]
id_lst=id_lst.replace('"epList":','')
id_lst=id_lst.replace(',"newestEp"','')
id_lst=id_lst.replace('[','')
id_lst=id_lst.replace(']','')
id_lst=id_lst.replace('},{','},,,{')
real_id_lst=id_lst.split(',,,')
for line in real_id_lst:
try:
line=eval(line)
aid=line['aid']
cid=line['cid']
epid=line['ep_id']
title=line['index_title']
if title=='':
title=album_name
interfaceurl='https://interface.bilibili.com/player?id=cid:'+str(cid)+'&aid='+str(aid)
get_page2=requests.get(interfaceurl)
page2 = get_page2.text
soup2 = BeautifulSoup(page2,'html.parser')
play_count=soup2.click.text
release_time=soup2.time.text
release_time=int(release_time*1e3)
fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
dura=soup2.duration.text
dura_lst=dura.split(':')
if len(dura_lst)==1:
duration=int(dura_lst[0])
elif len(dura_lst)==2:
duration=int(dura_lst[0])*60+int(dura_lst[1])
elif len(dura_lst)==3:
duration=int(dura_lst[0])*3600+int(dura_lst[1])*60+int(dura_lst[2])
else:
duration=0
url='https://www.bilibili.com/bangumi/play/ep'+str(epid)
D0={'album_name':album_name,'album_infor':album_infor,'title':title,
'play_count':play_count,'releaser':releaser,'fetch_time':fetch_time,
'release_time':release_time,'duration':duration,'url':url,
'releaserurl':releaserurl,'aid':aid,'cid':cid}
album_lst.append(D0)
json_D0=json.dumps(D0)
result.write(json_D0)
result.write('\n')
print(url)
except:
print('thereisanerror'+url)
result.flush()
result.close()
done.close()
return album_lst
def renew_playcount(self,taskname,resultname):
result=open(resultname,'a')
task=open(taskname)
task_lst=[]
result_lst=[]
for line in task:
line_dic=eval(line)
task_lst.append(line_dic)
for line in task_lst:
aid=line['aid']
cid=line['cid']
interfaceurl='https://interface.bilibili.com/player?id=cid:'+str(cid)+'&aid='+str(aid)
get_page=requests.get(interfaceurl)
page = get_page.text
soup = BeautifulSoup(page,'html.parser')
play_count=soup.click.text
fetch_time=int(datetime.datetime.timestamp(datetime.datetime.now())*1e3)
line['play_count']=play_count
line['fetch_time']=fetch_time
result_lst.append(line)
json_line=json.dumps(line)
result.write(json_line)
result.write('\n')
result.flush()
print(aid)
task.close()
result.close()
return result_lst
def get_video_danmuku(self, url):
result_lst = []
video_dic = self.video_page(url)
cid = video_dic['cid']
danmu_url = 'https://comment.bilibili.com/%s.xml' % cid
get_page = requests.get(danmu_url)
get_page.encoding = "utf-8"
page = get_page.text
soup = BeautifulSoup(page,'html.parser')
danmu_lst = soup.find_all("d")
for line in danmu_lst:
danmu = line.text
result_lst.append(danmu)
return result_lst
if __name__=='__main__':
test = Crawler_bilibili()
# task_lst = ["https://www.bilibili.com/video/av9062434",
# "https://www.bilibili.com/video/av7934136",
# "https://www.bilibili.com/video/av8882049",
# "https://www.bilibili.com/video/av7495100"]
# for url in task_lst:
# aid = url.split('av')[1]
# result = test.get_video_danmuku(url)
# str_lst_to_file(listname=result, filename=aid)
#video_dic = test.video_page("https://www.bilibili.com/video/av9062434")
sr_bb = test.search_page(keyword='任正非 BBC', search_pages_max=3)
"""
https://www.bilibili.com/video/av9062434
https://www.bilibili.com/video/av7934136
https://www.bilibili.com/video/av8882049
https://www.bilibili.com/video/av7495100
"""