update

crawler_sys/proxy_pool/func_get_proxy_form_kuaidaili.py

@@ -139,7 +139,7 @@ def get_proxy(proxies_num=None):
 
 
 if __name__ == "__main__":
-    proxy_pool_dic = get_proxy(11)
+    proxy_pool_dic = get_proxy(1)
     print(proxy_pool_dic)
     proxy_test(proxy_pool_dic)
     print(get_proxy_from_redis())

crawler_sys/site_crawler/crawler_tudou.py

@@ -958,5 +958,5 @@ if __name__ == '__main__':
     #sr_tud = test.search_page(keyword='任正非', search_pages_max=2)
     # pass
     # test.get_releaser_page(releaser_url)
-    res = requests.get("https://www.baidu.com",proxies={'http': 'http://114.99.116.173:18833/', 'https': 'http://114.99.116.173:18833/'})
-    print(res)
\ No newline at end of file
+    res = requests.get("https://www.baidu.com",proxies={'http': 'http://222.249.238.138:8080', 'https': 'http://222.249.238.138:8080'})
+    print(res.text)
\ No newline at end of file

crawler_sys/site_crawler/crawler_xiaohongshu.py

@@ -205,7 +205,7 @@ class Crawler_xiaohongshu():
                     time_ts = datetime.datetime.strptime(info_dic["time"],'%Y-%m-%d %H:%M').timestamp()
                     if info_dic["type"] != "normal":
                         continue
-                    page_data = self.get_one_page(page_id,proxies=proxies_num,cookies=cookie)
+                    page_data = self.get_one_page(page_id,proxies=proxies_num,cookies=cookie_dic)
                     print(page_data)
                     title = title
                     anwser = desc
@@ -223,4 +223,4 @@ class Crawler_xiaohongshu():
 if __name__ == '__main__':
     test = Crawler_xiaohongshu()
     releaserurl = 'https://www.xiaohongshu.com/user/profile/5abbb57211be1027a0c880ae'
-    res = test.releaser_page(releaserurl)
+    res = test.releaser_page(releaserurl,proxies_num=0)

crawler_sys/utils/rpc.py

+from django.conf import settings
+from helios.rpc import create_default_invoker
+import traceback
+import logging
+
+_rpc_invoker = create_default_invoker(debug=settings.DEBUG)
+
+
+def get_rpc_invoker():
+    return _rpc_invoker
+
+
+rpc_invoker = create_default_invoker(debug=False)
+
+
+def device_is_gray(device_id, cache_key):
+    try:
+        grey_codes = [cache_key]
+        rpc_client = rpc_invoker['platform/grey_control/info_by_names'](grey_codes=grey_codes, device_id=device_id)
+        ret = rpc_client.unwrap()
+        return ret["codes"][cache_key]
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return False

crawler_sys/utils/rpc_data_to_answer.py

@@ -6,6 +6,7 @@
 
 import requests
 import typing
+from crawler.crawler_sys.utils.rpc import rpc_invoker,get_rpc_invoker
 """
 https://www.yuque.com/docs/share/f4abe44b-6593-46b4-b280-5c87e4db2c85?#
 rpc: cims/question/batch_create 创建问题
@@ -42,6 +43,12 @@ def post_data(data_dict:typing.Dict,rpc_type:str) -> typing.Dict:
     data = {
         'requests': '[{"params": {"replies": [{%s}]}, "method": "%s, "timeout": 120}]' % (str(data_dict),rpc_type)
     }
-    response = requests.post('http://cims-qa.paas-develop.env/v1/batch', headers=headers, data=data)
-    print(response.text)
-    return response.json()
\ No newline at end of file
+    rpc_client = get_rpc_invoker()
+    res = rpc_client[rpc_type]().unwrap()
+    print(res)
+
+    return res
+
+    # response = requests.post('http://cims-qa.paas-develop.env/v1/batch', headers=headers, data=data)
+    # print(response.text)
+    # return response.json()
\ No newline at end of file

test/test_sink_rpc.py

@@ -41,4 +41,4 @@ time_ts = datetime.datetime.strptime(page_data["NoteView"]["content"]['time'],'%
 video_dic["user_id"] = random.choice(user_id_list)
 video_dic["create_time"] = time_ts
 rpc_res = post_data(video_dic, "cims/question/batch_create")
-print(rpc_res)
\ No newline at end of file
+print(rpc_res)