Commit faa5455c authored by 李小芳's avatar 李小芳

add

parent fa1d3fcd
......@@ -91,8 +91,11 @@ class CrawlerMain(object):
try:
requests_res = requests.get(url, headers=self.headers, allow_redirects=False, timeout=10)
# 打印状态码
page_obj = etree.HTML(requests_res.text)
return page_obj, url
if requests_res:
page_obj = etree.HTML(requests_res.text)
return page_obj, url
else:
return None
except ReadTimeout:
print('timeout')
return None, None
......@@ -119,8 +122,11 @@ class CrawlerMain(object):
.format(query, page)
try:
requests_res = requests.get(url, headers=self.headers, allow_redirects=False, timeout=10)
res_json = requests_res.json()
return res_json
if requests_res:
res_json = requests_res.json()
return res_json
else:
return None
except ReadTimeout:
print('超时,执行下一个请求')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment