# -*- coding:utf-8 -*-
# @Time : 2019/9/4 18:19
# @Author : litao
import os
import re
import time
import copy
import requests
import datetime
import json, random, urllib
from bs4 import BeautifulSoup
# from selenium import webdriver
# from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from crawler.crawler_sys.framework.video_fields_std import Std_fields_video
from crawler.crawler_sys.utils.output_results import output_result
from crawler.crawler_sys.utils.util_logging import logged
from crawler.crawler_sys.proxy_pool.func_get_proxy_form_kuaidaili import get_proxy
try:
from crawler_sys.framework.func_get_releaser_id import *
except:
from func_get_releaser_id import *
from crawler.crawler_sys.utils.output_results import retry_get_url
class Crawler_douyin():
def __init__(self, timeout=None, platform='抖音'):
if timeout == None:
self.timeout = 10
else:
self.timeout = timeout
self.platform = platform
self.TotalVideo_num = None
self.midstepurl = None
std_fields = Std_fields_video()
self.video_data = std_fields.video_data
self.video_data['platform'] = self.platform
unused_key_list = ['channel', 'describe', 'repost_count', 'isOriginal']
for key in unused_key_list:
self.video_data.pop(key)
# self.headers = {
# # 抖音
# # "Host": "aweme.snssdk.com",
# "Connection": "keep-alive",
# # "x-Tt-Token": "%sc05aa0c030d8872y921e4198d7d0e537c6t6684c162a890f23pb73e8f73c716vvc78cff1e65e0iu7e3c%s" % (random.randint(100000,999999),random.randint(100000000,999999999)),
# "sdk-version": "1",
# "User-Agent": "Aweme 9.2.1 rv:92113 (iPhone; iOS 13.3; zh_CN) Cronet",
# # "x-tt-trace-id": "00-0a1eeba%sb4ea508f44a29d%s-0a1ppwc%sb4ea50-01" % (random.randint(100,999),random.randint(10000000,99999999),random.randint(100,999)),
# "Accept-Encoding": "gzip, deflate",
# "X-SS-DP": "1128",
# # "Cookie": 'tt_webid=6636348501838333443; __tea_sdk__user_unique_id=6636348501838333443; _ga=GA1.3.400580664.1533338821; sid_guard=7055c05aa0c030d8872df921e4198d7d%7C1573351823%7C5184000%7CThu%2C+09-Jan-2020+02%3A10%3A23+GMT; uid_tt=cce96b02e4a2ad0dcc2ce9c2edad93a3; sid_tt=7055c05aa0c030d8872df921e4198d7d; sessionid=7055c05aa0c030d8872df921e4198d7d; install_id=93682014831; ttreq=1$9164e2a9cc97597197bec6e03559f7e9d05982c5; odin_tt=cc424af4c816fe1492acb2b887cef5fae585799744a9168f8204fce3d0f011e4694ff93a6a6f3e83bc3e126418aafa54',
# # "X-Khronos": "1576422795",
# # "X-Gorgon": "83009990000046140d8188c11cfdc1dd7b3f0507077b39112481",
#
# }
self.headers = {
# 抖音极速版
# "Host": "api3-normal-c-hl.amemv.com",
"Connection": "keep-alive",
# "X-SS-TC": "0",
"User-Agent": "Linux; U; Android 8.1.0; zh-CN; EML-AL00 Build/HUAWEIEML-AL00",
# "x-tt-trace-id": "00-0a1eeba%sb4ea508f44a29d%s-0a1ppwc%sb4ea50-01" % (random.randint(100,999),random.randint(10000000,99999999),random.randint(100,999)),
"Accept-Encoding": "gzip",
"X-SS-REQ-TICKET": str(int(datetime.datetime.now().timestamp()*1e3)),
# "Cookie": 'tt_webid=6636348501838333443; __tea_sdk__user_unique_id=6636348501838333443; _ga=GA1.3.400580664.1533338821; sid_guard=7055c05aa0c030d8872df921e4198d7d%7C1573351823%7C5184000%7CThu%2C+09-Jan-2020+02%3A10%3A23+GMT; uid_tt=cce96b02e4a2ad0dcc2ce9c2edad93a3; sid_tt=7055c05aa0c030d8872df921e4198d7d; sessionid=7055c05aa0c030d8872df921e4198d7d; install_id=93682014831; ttreq=1$9164e2a9cc97597197bec6e03559f7e9d05982c5; odin_tt=cc424af4c816fe1492acb2b887cef5fae585799744a9168f8204fce3d0f011e4694ff93a6a6f3e83bc3e126418aafa54',
"X-Khronos": str(int(datetime.datetime.now().timestamp())),
"sdk-version": "1",
# "X-Gorgon": "83009990000046140d8188c11cfdc1dd7b3f0507077b39112481",
}
self.api_list = [
"api3-normal-c-hl.amemv.com",
"api3-normal-c-lf.amemv.com",
"api3-normal-c-lf.amemv.com",
"aweme.snssdk.com",
]
def get_releaser_id(self, releaserUrl):
return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl)
def get_releaser_follower_num(self, releaserUrl):
headers = {
"Accept-Encoding": "gzip",
# "sdk-version": "1",
"User-Agent": "ttnet okhttp/3.10.0.2",
"Host": "aweme.snssdk.com",
"Connection": "Keep-Alive",
}
releaser_id = self.get_releaser_id(releaserUrl)
time.sleep(1)
releaserUrl = 'https://{2}/aweme/v1/user/?ac=WIFI&device_id={1}&os_api=18&app_name=aweme&channel=App Store&device_platform=ipad&device_type=iPad6,11&app_version=8.7.1&js_sdk_version=1.17.2.0&version_code=8.7.1&os_version=13.2.3&screen_width=1536&user_id={0}'.format(
releaser_id, str(random.randint(40000000000, 90000000000)),random.choice(self.api_list))
count = 0
while count < 3:
try:
count += 1
time.sleep(random.random())
get_page = retry_get_url(releaserUrl, headers=headers, proxies=10)
page = get_page.json()
follower_num = page["user"].get("follower_count")
print('%s follower number is %s' % (releaserUrl, follower_num))
releaser_img = page["user"].get("avatar_thumb").get("url_list")[0]
return follower_num, releaser_img
except:
print("can't find followers")
continue
else:
return None, None
def get_releaser_page(self,releaserUrl):
headers = {
"Accept-Encoding": "gzip",
# "sdk-version": "1",
"User-Agent": "Linux; U; Android 8.1.0; zh-CN; EML-AL00 Build/HUAWEIEML-AL00",
"Host": "aweme.snssdk.com",
"Connection": "Keep-Alive",
}
releaser_id = self.get_releaser_id(releaserUrl)
time.sleep(1)
releaserUrl = 'https://{2}/aweme/v1/user/?ac=WIFI&device_id={1}&os_api=18&app_name=aweme&channel=App Store&device_platform=ipad&device_type=iPad6,11&app_version=8.9.1&js_sdk_version=1.17.2.0&version_code=8.7.1&os_version=13.2.3&screen_width=1536&user_id={0}'.format(
releaser_id, str(random.randint(40000000000, 90000000000)), random.choice(self.api_list))
count = 0
while count < 3:
try:
count += 1
time.sleep(random.random())
get_page = retry_get_url(releaserUrl, headers=headers, proxies=0)
page = get_page.json()
follower_num = page["user"].get("follower_count")
age = page["user"].get('birthday')
location = page["user"].get('location')
province = page["user"].get('province')
huozan = page["user"].get("total_favorited")
sex = page["user"].get("gender")
verify = page["user"].get("enterprise_verify_reason")
signature = page["user"].get('signature').replace("\r","").replace("\n","")
if page["user"].get('is_gov_media_vip'):
signature_type = "gov"
elif page["user"].get('is_star'):
signature_type = "star"
elif page["user"].get('is_effect_artist'):
signature_type = "artist"
elif page["user"].get('is_verified'):
signature_type = "已认证"
else:
signature_type = ""
dic = {
"age":age,
"location":location,
"huozan":huozan,
"sex":sex,
"verify":verify,
"signature":signature,
"province":province,
"follower_num":follower_num,
"signature_type":signature_type,
}
print(dic)
return dic
except:
print("can't find followers")
continue
else:
return None
def find_releaser_id(self, releaserUrl):
return get_releaser_id(platform=self.platform, releaserUrl=releaserUrl)
def releaser_page(self, releaserUrl,
output_to_file=False,
filepath=None,
releaser_page_num_max=30,
output_to_es_raw=False,
es_index=None,
doc_type=None,
output_to_es_register=False,
push_to_redis=False, proxies_num=None, **kwargs):
"""
get video info from api instead of web page html
the most scroll page is 1000
"""
result_list = []
has_more = True
count = 1
count_false = 0
releaser_id = self.find_releaser_id(releaserUrl)
offset = "0"
#vid = "AB5483CA-FCDC-42F1-AFB1-077A1%sDA" % random.randint(100000, 999999)
ccid = "F153594D-1310-4984-A4C3-A679D4D%s" % random.randint(10000, 99999)
openudid = "5d44f2ea1b74e3731b27e5ed8039ac29f%s" % random.randint(1000000, 9999999)
idfa = "E3FC9054-384B-485F-9B4C-936F33D7D%s" % random.randint(100, 999)
iid = str(random.randint(100000000000, 103000000000))
device_id = str(random.randint(66800000000, 66990000000))
proxies = get_proxy(proxies_num)
while has_more and count <= releaser_page_num_max:
# print(str(releaser_id)+str(max_behot_time))
# js_head = json.loads(get_js(str(releaser_id)+str(max_behot_time)))
print("get %s video on page %s" % (releaser_id, count))
time.sleep(random.randint(1,2))
url_dic = {
"source": "0",
"max_cursor": offset,
"user_id": releaser_id,
"count": "21",
"os_api": "23",
"device_type": "Huawei P20",
"ssmix": "a",
"manifest_version_code": "985",
"dpi": "429",
# "uuid": "440000000189785",
"app_name": "douyin",
"version_name": "9.8.5",
"ts": "1585532172",
"app_type": "normal",
"ac": "wifi",
"update_version_code": "9852",
"channel": "baidu",
"_rticket": str(int(datetime.datetime.now().timestamp())*1e3),
"device_platform": "android",
# "iid": iid,
"version_code": "985",
#"cdid": "87cc1c77-cc3c-41a1-8df6-1e060b9c510b",
#"openudid": "e44cc0264b92bcbf",
"device_id": device_id,
"resolution": "1080*2244",
"os_version": "9.0.1",
"language": "zh",
"device_brand": "Huawei",
"aid": "2329",
"mcc_mnc": "46005",
}
# url_dic = {
# "ac": "WIFI",
# # "iid": iid,
# "device_id": device_id,
# "os_api": "18",
# "app_name": "aweme",
# "channel": "App Store",
# # "idfa": "7AED33DD-0F97-418D-AFAA-72ED0578A44E",
# # "idfa": idfa,
# "device_platform": "iphone",
# "build_number": "92113",
# # "vid": "21B39A50-8C28-4E7E-AEB8-A67B12B1A82B",
# # "vid": vid,
# # "openudid": "b1021c76124449e0e9f0e43bdf51f3314aac263b",
# # "openudid": openudid,
# "device_type": "iPhone9,4",
# "app_version": "9.2.1",
# "js_sdk_version": "1.43.0.1",
# "version_code": "9.2.1",
# "os_version": "13.3",
# "screen_width": "1242",
# "aid": "1128",
# "mcc_mnc": "",
# "user_id": releaser_id,
# "max_cursor": offset,
# "count": "21",
# "source": "0",
# }
# 其中aid 可能为获取数据的关键字段 处理方法 尝试＋1
# host = random.choice(self.api_list)
# self.headers["Host"] = host
url = "https://{1}/aweme/v1/aweme/post/?{0}".format(urllib.parse.urlencode(url_dic),
random.choice(self.api_list))
# url = "https://aweme.snssdk.com/aweme/v1/aweme/post/?source=0&max_cursor=1584105171000&user_id=100027325090&count=10&os_api=23&device_type=MI%205s&ssmix=a&manifest_version_code=940&dpi=270&uuid=440000000189785&app_name=douyin_lite&version_name=9.4.0&ts=1585532172&app_type=normal&ac=wifi&update_version_code=9402&channel=xiaomi&_rticket=1585532172572&device_platform=android&iid=109688778422&version_code=940&cdid=87cc1c77-cc3c-41a1-8df6-1e060b9c510b&openudid=e44cc0264b92bcbf&device_id=69418894872&resolution=810*1440&os_version=6.0.1&language=zh&device_brand=Xiaomi&aid=2329&mcc_mnc=46005"
try:
# proxies = get_proxy(proxies_num)
if proxies_num:
get_page = requests.get(url, headers=self.headers, proxies=proxies, timeout=10)
# get_page = retry_get_url(url, headers=self.headers, proxies=proxies_num, timeout=10)
else:
get_page = requests.get(url, headers=self.headers, timeout=10)
except Exception as e:
proxies = get_proxy(1)
print(e)
continue
page_dic = {}
# print(get_page.text)
try:
page_dic = get_page.json()
# print(get_page)
# print(page_dic)
data_list = page_dic.get('aweme_list')
if not data_list:
get_page = requests.get(url, headers=self.headers, timeout=10)
page_dic = get_page.json()
data_list = page_dic.get('aweme_list')
if not data_list:
raise ValueError
has_more = page_dic.get('has_more')
offset = str(page_dic.get("max_cursor"))
except:
if not data_list:
proxies = get_proxy(1)
count_false += 1
if count_false >= 5:
break
else:
continue
if has_more is None:
has_more = False
if data_list == []:
print("no data in releaser %s page %s" % (releaser_id, count))
# print(page_dic)
# print(url)
proxies = get_proxy(1)
count_false += 1
if count_false >= 5:
has_more = False
continue
else:
count_false = 0
count += 1
for one_video in data_list:
# info_str = one_video.get('content')
video_dic = copy.deepcopy(self.video_data)
video_dic['title'] = one_video.get('desc')
video_dic['url'] = one_video.get('share_url')
video_dic['releaser'] = one_video.get('author').get("nickname")
video_dic['releaserUrl'] = releaserUrl
release_time = one_video.get('create_time')
video_dic['release_time'] = int(release_time * 1e3)
video_dic['duration'] = int(one_video.get('duration') / 1000)
video_dic['play_count'] = 0
video_dic['repost_count'] = one_video.get('statistics').get('share_count')
video_dic['comment_count'] = one_video.get('statistics').get('comment_count')
video_dic['favorite_count'] = one_video.get('statistics').get('digg_count')
video_dic['video_id'] = one_video.get('aweme_id')
video_dic['fetch_time'] = int(datetime.datetime.now().timestamp() * 1e3)
video_dic['releaser_id_str'] = "抖音_%s" % releaser_id
video_dic['video_img'] = one_video.get('video').get('cover').get('url_list')[0]
result_list.append(video_dic)
if len(result_list) >= 100:
output_result(result_Lst=result_list,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type,
output_to_es_register=output_to_es_register)
result_list.clear()
if result_list != []:
# data_count += len(result_list)
# print(result_list)
# print(data_count)
output_result(result_Lst=result_list,
platform=self.platform,
output_to_file=output_to_file,
filepath=filepath,
output_to_es_raw=output_to_es_raw,
es_index=es_index,
doc_type=doc_type,
output_to_es_register=output_to_es_register)
# test
if __name__ == '__main__':
test = Crawler_douyin()
user_lis = [
"https://www.iesdouyin.com/share/user/1235497011657230",
"https://www.iesdouyin.com/share/user/106904794358",
"https://www.iesdouyin.com/share/user/99535335790",
"https://www.iesdouyin.com/share/user/4358120339413549",
"https://www.iesdouyin.com/share/user/1011140572494243",
"https://www.iesdouyin.com/share/user/3368491398726589",
"https://www.iesdouyin.com/share/user/105473533501",
"https://www.iesdouyin.com/share/user/104163951894",
"https://www.iesdouyin.com/share/user/3720349961173368",
"https://www.iesdouyin.com/share/user/1662057182141620",
]
for u in user_lis:
ttt = test.releaser_page(releaserUrl=u, output_to_es_raw=True,
es_index='crawler-data-raw',
doc_type='doc',
releaser_page_num_max=3, proxies_num=0)
# test.get_releaser_page(u)
# break