udpate

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -26,7 +26,7 @@ from crawler.crawler_sys.site_crawler_by_redis import crawler_weibo
 import sys
 from crawler.crawler_sys.utils.output_results import output_result
 import argparse, copy, datetime, time
-from crawler.crawler_sys.framework.es_target_releasers import get_releaserUrls_from_es
+# from crawler.crawler_sys.framework.es_target_releasers import get_releaserUrls_from_es
 from crawler.crawler_sys.utils.parse_bool_for_args import parse_bool_for_args
 import redis,json
 from concurrent.futures import ProcessPoolExecutor
@@ -262,12 +262,12 @@ if __name__ == "__main__":
         platform_str = platform + "_process"
         rds_1.set(platform_str, 0)
     for processe in range(processes_num):
-        # start_crawler(processe,name)
+        start_crawler(processe,name)
         # print(kwargs_dict)
-        future = executor.submit(start_crawler,processe,name)
-        futures.append(future)
-        print('Processe %s start' % processe)
-    executor.shutdown(True)
+    #     future = executor.submit(start_crawler,processe,name)
+    #     futures.append(future)
+    #     print('Processe %s start' % processe)
+    # executor.shutdown(True)
     # while True:
     #     now = datetime.datetime.now()
     #     if now.hour >= exit_hour: