update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -59,11 +59,11 @@ parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str
                     help=('assign a es_index to write into, default to crawler-data-raw'))
 parser.add_argument('-g', '--output_to_es_register', default='True', type=str,
                     help=('Write data into es or not, default to True'))
-parser.add_argument('-processes', '--processes_num', default=4, type=int,
+parser.add_argument('-processes', '--processes_num', default=1, type=int,
                     help=('Processes number to be used in multiprocessing'))
 parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int,
                     help=('Processes number to be used in multiprocessing'))
-parser.add_argument('-thead', '--thead_num', default=4, type=int,
+parser.add_argument('-thead', '--thead_num', default=1, type=int,
                     help=('thead_num number to be used in multiprocessing'))
 parser.add_argument('-v', '--video', default="False", type=str,
                     help=('Is or not run video_page_crawler'))
@@ -139,6 +139,8 @@ kwargs_dict = {
 def get_task_from_redis():
     while True:
         try:
+            import pdb
+            pdb.set_trace()
             print('----------------------')
             # now = datetime.datetime.now()
             # if now.hour >= exit_hour: