update

crawler_sys/framework/update_data_in_target_releasers_multi_process_by_date_from_redis.py

@@ -71,7 +71,7 @@ parser.add_argument('-name', '--name', default="crawler01", type=str,
                     help=('this computer name'))
 args = parser.parse_args()
 
-rds_save = redis.StrictRedis(host='172.16.40.164', port=6379, db=20, decode_responses=True, password='ReDis!GmTx*0aN12')
+rds_save = redis.StrictRedis(host='172.18.51.10', port=6379, db=20, decode_responses=True, password='ReDis!GmTx*0aN12')
 rds_1 = redis.StrictRedis(host='172.18.51.10', port=6379, db=19, decode_responses=True)
 rds_17 = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
 platform_crawler_reg = {

crawler_sys/site_crawler_by_redis/crawler_weibo.py

@@ -127,6 +127,7 @@ class Crawler_weibo():
                 except:
                     get_page = None
                     has_more = False
+                print(get_page.text)
                 if get_page and get_page.status_code == 200:
                     try:
                         page_json = get_page.json()
@@ -330,7 +331,7 @@ if __name__ == '__main__':
 
     ]
     import redis
-    rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=17, decode_responses=True)
+    rds = redis.StrictRedis(host='172.18.51.10', port=6379, db=20, decode_responses=True)
     for url in url_list:
         res = test.releaser_page(url, output_to_es_raw=True,
                             es_index='crawler-data-raw',