Skip to content

C
crawler
Commit 603cf85d authored by litaolemo's avatar litaolemo
Browse files
Options

update

parent 29d2c7f3
Show whitespace changes
Inline Side-by-side
Showing with 10 additions and 10 deletions
crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py
View file @ 603cf85d
...@@ -141,30 +141,30 @@ def get_task_from_redis(): ...@@ -141,30 +141,30 @@ def get_task_from_redis():
while True: while True:
try: try:
print('----------------------') # print('----------------------')
# now = datetime.datetime.now() # now = datetime.datetime.now()
# if now.hour >= exit_hour: # if now.hour >= exit_hour:
# sys.exit() # sys.exit()
res = rds_1.hgetall("process_num") res = rds_1.hgetall("process_num")
print(res) # print(res)
for platform in platform_crawler_reg: for platform in platform_crawler_reg:
platform_str = platform + "_process" platform_str = platform + "_process"
print(platform_str) # print(platform_str)
try: try:
process_num = rds_1.get(platform_str) process_num = rds_1.get(platform_str)
print(process_num) # print(process_num)
except Exception as e: except Exception as e:
print("line 152 error",e) print("line 152 error",e)
continue continue
try: try:
print("rds_1.llen(platform)" ,rds_1.llen(platform)) # print("rds_1.llen(platform)" ,rds_1.llen(platform))
if rds_1.llen(platform) > 0: if rds_1.llen(platform) > 0:
print("process_num",process_num) # print("process_num",process_num)
if process_num: if process_num:
if res.get(platform): if res.get(platform):
print(res.get(platform)) # print(res.get(platform))
if int(res.get(platform)) > int(process_num): if int(res.get(platform)) > int(process_num):
print("get platfrom %s" % platform) # print("get platfrom %s" % platform)
rds_1.incr(platform_str) rds_1.incr(platform_str)
yield platform yield platform
rds_1.decr(platform_str) rds_1.decr(platform_str)
...@@ -191,7 +191,7 @@ def single_thead(processe,name): ...@@ -191,7 +191,7 @@ def single_thead(processe,name):
end_time = int(now.timestamp() * 1e3) end_time = int(now.timestamp() * 1e3)
for count, platform in enumerate(get_task_from_redis()): for count, platform in enumerate(get_task_from_redis()):
now = datetime.datetime.now() now = datetime.datetime.now()
print(count,platform) # print(count,platform)
try: try:
count_false = 0 count_false = 0
releaser_dic_str = rds_1.blpop(platform) releaser_dic_str = rds_1.blpop(platform)
...@@ -267,7 +267,7 @@ def single_thead(processe,name): ...@@ -267,7 +267,7 @@ def single_thead(processe,name):
def start_crawler(processe,name): def start_crawler(processe,name):
for count in range(args.thead_num): for count in range(args.thead_num):
# single_thead(processe,name) # single_thead(processe,name)
print("processe,name" ,processe,name) # print("processe,name" ,processe,name)
t = threading.Thread(target=single_thead, name=str(count),args=(str(processe),name)) t = threading.Thread(target=single_thead, name=str(count),args=(str(processe),name))
# t.setDaemon(False) # t.setDaemon(False)
t.start() t.start()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment