diff --git a/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py b/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py index 951518e45424ea09e856947c8d4881bdc4412696..96a30a5a10bd1a0e4f32d7e2e08dd0935830ee15 100644 --- a/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py +++ b/crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py @@ -45,7 +45,6 @@ import threading # rds_1 = sentinel.master_for('ida_redis_master', socket_timeout=1, db=1, decode_responses=True) rds_1 = redis.StrictRedis(host='172.16.40.164', port=6379, db=19, decode_responses=True, password='ReDis!GmTx*0aN12') - parser = argparse.ArgumentParser(description='Specify a platform name.') parser.add_argument('-n', '--max_page', default=30, type=int, help=('The max page numbers to be scroll for each releaser url, ' @@ -262,12 +261,12 @@ if __name__ == "__main__": platform_str = platform + "_process" rds_1.set(platform_str, 0) for processe in range(processes_num): - start_crawler(processe,name) + # start_crawler(processe,name) # print(kwargs_dict) - # future = executor.submit(start_crawler,processe,name) - # futures.append(future) - # print('Processe %s start' % processe) - # executor.shutdown(True) + future = executor.submit(start_crawler,processe,name) + futures.append(future) + print('Processe %s start' % processe) + executor.shutdown(True) # while True: # now = datetime.datetime.now() # if now.hour >= exit_hour: