Commit ddc8b80a authored by litaolemo's avatar litaolemo

update

parent 221787d7
...@@ -59,11 +59,11 @@ parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str ...@@ -59,11 +59,11 @@ parser.add_argument('-index', '--es_index', default='crawler-data-raw', type=str
help=('assign a es_index to write into, default to crawler-data-raw')) help=('assign a es_index to write into, default to crawler-data-raw'))
parser.add_argument('-g', '--output_to_es_register', default='True', type=str, parser.add_argument('-g', '--output_to_es_register', default='True', type=str,
help=('Write data into es or not, default to True')) help=('Write data into es or not, default to True'))
parser.add_argument('-processes', '--processes_num', default=4, type=int, parser.add_argument('-processes', '--processes_num', default=1, type=int,
help=('Processes number to be used in multiprocessing')) help=('Processes number to be used in multiprocessing'))
parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int, parser.add_argument('-exit_hour', '--exit_hour', default=22, type=int,
help=('Processes number to be used in multiprocessing')) help=('Processes number to be used in multiprocessing'))
parser.add_argument('-thead', '--thead_num', default=4, type=int, parser.add_argument('-thead', '--thead_num', default=1, type=int,
help=('thead_num number to be used in multiprocessing')) help=('thead_num number to be used in multiprocessing'))
parser.add_argument('-v', '--video', default="False", type=str, parser.add_argument('-v', '--video', default="False", type=str,
help=('Is or not run video_page_crawler')) help=('Is or not run video_page_crawler'))
...@@ -139,6 +139,8 @@ kwargs_dict = { ...@@ -139,6 +139,8 @@ kwargs_dict = {
def get_task_from_redis(): def get_task_from_redis():
while True: while True:
try: try:
import pdb
pdb.set_trace()
print('----------------------') print('----------------------')
# now = datetime.datetime.now() # now = datetime.datetime.now()
# if now.hour >= exit_hour: # if now.hour >= exit_hour:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment