Commit 0b418401 authored by litaolemo's avatar litaolemo

update

parent 45055631
...@@ -40,8 +40,9 @@ from write_data_into_es.func_cal_doc_id import cal_doc_id ...@@ -40,8 +40,9 @@ from write_data_into_es.func_cal_doc_id import cal_doc_id
class Crawler_zhihu(): class Crawler_zhihu():
def __init__(self, timeout=None, platform='weibo'): def __init__(self, timeout=5,):
self.platform = "zhihu" self.platform = "zhihu"
self.timeout = timeout
self.session = requests.Session() self.session = requests.Session()
std_fields = Std_fields_video() std_fields = Std_fields_video()
self.video_data = std_fields.video_data self.video_data = std_fields.video_data
...@@ -55,6 +56,7 @@ class Crawler_zhihu(): ...@@ -55,6 +56,7 @@ class Crawler_zhihu():
except: except:
with open('/srv/apps/crawler/crawler_sys/site_crawler/zhihu.js', 'r', encoding='utf-8') as f: with open('/srv/apps/crawler/crawler_sys/site_crawler/zhihu.js', 'r', encoding='utf-8') as f:
js = f.read() js = f.read()
print(js)
self.exec_js = execjs.compile(js) self.exec_js = execjs.compile(js)
for popk in pop_key_Lst: for popk in pop_key_Lst:
self.video_data.pop(popk) self.video_data.pop(popk)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment