Skip to content

C
crawler
Commit 07c010d6 authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent 32ad8f2c
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 3 deletions
crawler_sys/framework/search_page_multi_process.py
View file @ 07c010d6
...@@ -33,8 +33,7 @@ es_framework = Elasticsearch(hosts='192.168.17.11', port=80, ...@@ -33,8 +33,7 @@ es_framework = Elasticsearch(hosts='192.168.17.11', port=80,
# doc_type_target_releaser = 'keywrod' # doc_type_target_releaser = 'keywrod'
if ARGS.platform != []: PLATFORM_LIST = ARGS.platform
PLATFORM_LIST = ARGS.platform
OUTPUT_TO_ES_RAW = ARGS.output_to_es_raw OUTPUT_TO_ES_RAW = ARGS.output_to_es_raw
OUTPUT_TO_ES_REGISTER = ARGS.output_to_es_register OUTPUT_TO_ES_REGISTER = ARGS.output_to_es_register
...@@ -2713,7 +2712,7 @@ kwargs_dict = { ...@@ -2713,7 +2712,7 @@ kwargs_dict = {
'es_index': ES_INDEX, 'es_index': ES_INDEX,
} }
pool = Pool(processes=4) pool = Pool(processes=3)
for platform in PLATFORM_LIST: for platform in PLATFORM_LIST:
search_page_task(platform, OUTPUT_TO_ES_RAW, OUTPUT_TO_ES_REGISTER, ES_INDEX) search_page_task(platform, OUTPUT_TO_ES_RAW, OUTPUT_TO_ES_REGISTER, ES_INDEX)
res = pool.apply_async(func=search_page_task, res = pool.apply_async(func=search_page_task,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment