Skip to content

C
crawler
Commit fa09f776 authored by 李小芳's avatar 李小芳
Browse files
Options

add

parent b852488d
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 3 deletions
dev/xinyang_ask_tag/crawler_xinyang_ask_service_city.py
View file @ fa09f776
...@@ -95,7 +95,7 @@ class CrawlerMain(object): ...@@ -95,7 +95,7 @@ class CrawlerMain(object):
page_obj = etree.HTML(requests_res.text) page_obj = etree.HTML(requests_res.text)
return page_obj, url return page_obj, url
else: else:
return None,None return None, None
except ReadTimeout: except ReadTimeout:
print('timeout') print('timeout')
return None, None return None, None
...@@ -319,7 +319,6 @@ class SoYongSpider(object): ...@@ -319,7 +319,6 @@ class SoYongSpider(object):
get_data_file.write(json.dumps(data)) get_data_file.write(json.dumps(data))
get_data_file.write("\n") get_data_file.write("\n")
if current_end_flat == True: if current_end_flat == True:
get_lasted_data.append(page)
break break
else: else:
pass pass
...@@ -328,7 +327,7 @@ class SoYongSpider(object): ...@@ -328,7 +327,7 @@ class SoYongSpider(object):
get_data_file.close() get_data_file.close()
print("get_lasted_data:", get_lasted_data) print("get_lasted_data:", get_lasted_data)
if len(get_lasted_data) == 3: if len(get_lasted_data) == 2:
return True return True
else: else:
return False return False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment