update

crawler_sys/proxy_pool/func_get_proxy_form_kuaidaili.py

@@ -42,13 +42,7 @@ rds = func_get_redis()
 def get_proxy_from_redis():
     try:
         one_proxy = rds.randomkey()
-        username = "hanye"
-        password = "i9mmu0a3"
-
-        proxies = {
-            "http": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy},
-            "https": "http://%(user)s:%(pwd)s@%(ip)s/" % {'user': username, 'pwd': password, 'ip': one_proxy}
-        }
+        proxies = {"http": "http://{}".format(one_proxy), "https": "https://{}".format(one_proxy)}
         return proxies
     except Exception as e:
         print(e)
@@ -57,7 +51,7 @@ def get_proxy_from_redis():
 
 def func_get_proxy_to_redis():
     # chance = random.random()
-    auth = kdl.Auth("990866563045611", "quxguz4hwm9cxnx6wpjhkokx04klpr8v")
+    auth = kdl.Auth("981538205840388", 'p7wrheahef20800z0zh4bedu4005ofsb')
     client = kdl.Client(auth)
 
     # 获取订单到期时间, 返回时间字符串
@@ -89,15 +83,24 @@ def func_get_proxy_to_redis():
     # print("valids: ", valids)
 
     # 获取私密代理剩余时间: 返回 ip: seconds(剩余秒数) 组成的dict
-    ips = client.get_dps(1, format='json', dedup=1)
-    seconds = client.get_dps_valid_time(ips)
-    # print("seconds: ", seconds)
-    for key in seconds:
-        rds.set(key, key, ex=int(seconds[key]) - 3)
+    # ips = client.get_dps(1, format='json', dedup=1)
+    # seconds = client.get_dps_valid_time(ips)
+    # # print("seconds: ", seconds)
+    # for key in seconds:
+    #     rds.set(key, key, ex=int(seconds[key]) - 3)
 
     # 获取计数版ip余额（仅私密代理计数版）
     # balance = client.get_ip_balance(sign_type='hmacsha1')
     # print("balance: ", balance)
+    ips = client.get_dps(1, sign_type='hmacsha1', format='json')
+    seconds = client.get_dps_valid_time(ips)
+    print("dps proxy: ", ips, client.get_proxy_authorization())
+    try:
+        for key in seconds:
+            rds.set(key, key, ex=int(seconds[key]) - 3)
+    except Exception as e:
+        print(e)
+    return ips[0]
 
 
 def proxy_test(proxies):
@@ -123,8 +126,8 @@ def get_proxy_dic(max_proxies=None):
     if res is None:
         return None
     if res < max_proxies:
-        func_get_proxy_to_redis()
-        return get_proxy_from_redis()
+        return func_get_proxy_to_redis()
+        # return get_proxy_from_redis()
     else:
         return get_proxy_from_redis()
 

crawler_sys/utils/rpc_data_to_answer.py

@@ -37,7 +37,7 @@ data_type_dict = {
 }
 
 
-def post_data(data_dict:typing.Dict,rpc_type:str) -> typing.Dict:
+def post_single_data(data_dict:typing.Dict,rpc_type:str) -> typing.Dict:
     headers = {
         'X-GAIA-HELIOS-VERSION': '0.7.5',
     }
@@ -58,4 +58,5 @@ def post_data(data_dict:typing.Dict,rpc_type:str) -> typing.Dict:
 
     # response = requests.post('http://cims-qa.paas-develop.env/v1/batch', headers=headers, data=data)
     # print(response.text)
-    # return response.json()
\ No newline at end of file
+    # return response.json()
+