update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -45,7 +45,6 @@ import threading
 # rds_1 = sentinel.master_for('ida_redis_master', socket_timeout=1, db=1, decode_responses=True)
 
 rds_1 = redis.StrictRedis(host='172.16.40.164', port=6379, db=19, decode_responses=True, password='ReDis!GmTx*0aN12')
-
 parser = argparse.ArgumentParser(description='Specify a platform name.')
 parser.add_argument('-n', '--max_page', default=30, type=int,
                     help=('The max page numbers to be scroll for each releaser url, '
@@ -262,12 +261,12 @@ if __name__ == "__main__":
         platform_str = platform + "_process"
         rds_1.set(platform_str, 0)
     for processe in range(processes_num):
-        start_crawler(processe,name)
+        # start_crawler(processe,name)
         # print(kwargs_dict)
-    #     future = executor.submit(start_crawler,processe,name)
-    #     futures.append(future)
-    #     print('Processe %s start' % processe)
-    # executor.shutdown(True)
+        future = executor.submit(start_crawler,processe,name)
+        futures.append(future)
+        print('Processe %s start' % processe)
+    executor.shutdown(True)
     # while True:
     #     now = datetime.datetime.now()
     #     if now.hour >= exit_hour: