update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -60,11 +60,11 @@ parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str
                     help=('assign a es_index to write into, default to crawler-data-raw'))
 parser.add_argument('-g', '--output_to_es_register', default='True', type=str,
                     help=('Write data into es or not, default to True'))
-parser.add_argument('-processes', '--processes_num', default=8, type=int,
+parser.add_argument('-processes', '--processes_num', default=4, type=int,
                     help=('Processes number to be used in multiprocessing'))
 parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int,
                     help=('Processes number to be used in multiprocessing'))
-parser.add_argument('-thead', '--thead_num', default=5, type=int,
+parser.add_argument('-thead', '--thead_num', default=4, type=int,
                     help=('thead_num number to be used in multiprocessing'))
 parser.add_argument('-v', '--video', default="False", type=str,
                     help=('Is or not run video_page_crawler'))
@@ -251,7 +251,7 @@ def start_crawler(processe,name):
     for count in range(args.thead_num):
         # single_thead(processe,name)
         t = threading.Thread(target=single_thead, name=str(count),args=(str(processe),name))
-        t.setDaemon(False)
+        # t.setDaemon(False)
         t.start()
 
 
@@ -267,6 +267,7 @@ if __name__ == "__main__":
         future = executor.submit(start_crawler,processe,name)
         futures.append(future)
         print('Processe %s start' % processe)
+    executor.shutdown(True)
     # while True:
     #     now = datetime.datetime.now()
     #     if now.hour >= exit_hour: