update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -63,7 +63,7 @@ parser.add_argument('-processes', '--processes_num', default=2, type=int,
                     help=('Processes number to be used in multiprocessing'))
 parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int,
                     help=('Processes number to be used in multiprocessing'))
-parser.add_argument('-thead', '--thead_num', default=2, type=int,
+parser.add_argument('-thead', '--thead_num', default=8, type=int,
                     help=('thead_num number to be used in multiprocessing'))
 parser.add_argument('-v', '--video', default="False", type=str,
                     help=('Is or not run video_page_crawler'))
@@ -183,8 +183,8 @@ def single_thead(processe,name):
     data_list = []
 
     while True:
-        import pdb
-        pdb.set_trace()
+        # import pdb
+        # pdb.set_trace()
         now = datetime.datetime.now()
         # if now.hour >= exit_hour:
         #     sys.exit()
@@ -284,8 +284,8 @@ if __name__ == "__main__":
         print(kwargs_dict)
         # future = executor.submit(start_crawler,processe,name)
         # futures.append(future)
-        print('Processe %s start' % processe)
-    executor.shutdown(True)
+        # print('Processe %s start' % processe)
+    # executor.shutdown(True)
 
     for future in futures:
         print(future)