update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -59,11 +59,11 @@ parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str
                     help=('assign a es_index to write into, default to crawler-data-raw'))
 parser.add_argument('-g', '--output_to_es_register', default='True', type=str,
                     help=('Write data into es or not, default to True'))
-parser.add_argument('-processes', '--processes_num', default=1, type=int,
+parser.add_argument('-processes', '--processes_num', default=2, type=int,
                     help=('Processes number to be used in multiprocessing'))
 parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int,
                     help=('Processes number to be used in multiprocessing'))
-parser.add_argument('-thead', '--thead_num', default=1, type=int,
+parser.add_argument('-thead', '--thead_num', default=2, type=int,
                     help=('thead_num number to be used in multiprocessing'))
 parser.add_argument('-v', '--video', default="False", type=str,
                     help=('Is or not run video_page_crawler'))
@@ -123,7 +123,6 @@ output_to_es_register = parse_bool_for_args(args.output_to_es_register)
 
 processes_num = args.processes_num
 name = args.name
-lock = threading.Lock()
 
 es_index = args.es_index
 end_time = int(datetime.datetime.now().timestamp() * 1e3)
@@ -182,8 +181,7 @@ def get_task_from_redis():
 def single_thead(processe,name):
     allow = 10
     data_list = []
-    rds_1 = redis.StrictRedis(host='172.16.40.164', port=6379, db=19, decode_responses=True,
-                              password='ReDis!GmTx*0aN12')
+
     while True:
         import pdb
         pdb.set_trace()