Merge branch 'offic' into 'master'

Offic

See merge request !63

eda/smart_rank/tag3_portrait_to_es.py

+import datetime
+
+import pytz
+import redis
+
+from es_tool import es_insert_device_info
+from tool import get_user_portrait_tag3_from_redis
+
+redis_client = redis.StrictRedis.from_url("redis://:ReDis!GmTx*0aN6@172.16.40.133:6379")
+redis_client2 = redis.StrictRedis.from_url("redis://:ReDis!GmTx*0aN9@172.16.40.173:6379")
+redis_client3 = redis.StrictRedis.from_url("redis://:ReDis!GmTx*0aN12@172.16.40.164:6379")
+redis_client4 = redis.StrictRedis.from_url("redis://:XfkMCCdWDIU%ls$h@172.16.50.145:6379")
+
+
+def user_portrait_scan_info():
+    try:
+        round = 0
+        all_count = 0
+        just_projects_count = 0
+        portrait_no_candidates_count = 0
+        keys = "doris:user_portrait:tag3:device_id:*"
+        cur, results = redis_client2.scan(0, keys, 3000)
+        while cur != 0:
+            round += 1
+            print("round: " + str(round))
+            cur, results = redis_client2.scan(cur, keys, 3000)
+            for key in results:
+                key = str(key, "utf-8")
+                device_id = key.split(":")[-1]
+                all_count += 1
+
+                print(str(all_count) + ": " + device_id)
+                write_to_es(device_id)
+
+        print("all count: " + str(all_count))
+        print("just projects portrait: " + str(just_projects_count))
+        print("just portrait no candidates: " + str(portrait_no_candidates_count))
+        print("scan done " + str(datetime.datetime.now()))
+    except Exception as e:
+        print(e)
+
+
+def write_to_es(device_id):
+    body = get_user_portrait_tag3_from_redis(device_id)
+    if body:
+        body["device_id"] = device_id
+        body["last_modified"] = datetime.datetime.strftime(datetime.datetime.now(pytz.timezone("Asia/Shanghai")),
+                                                           "%Y-%m-%dT%H:%M:%S.%f")[:-7] + "Z"
+        es_insert_device_info(device_id, body)
+
+
+if __name__ == "__main__":
+    # device_id = "androidid_a25a1129c0b38f7b"
+    # write_to_es(device_id)
+
+    user_portrait_scan_info()
+
+    print("done: " + str(datetime.datetime.now()))

eda/smart_rank/tool.py

@@ -517,7 +517,7 @@ def get_user_portrait_tag3_from_redis(device_id, limit_score=0):
     portrait_key = "doris:user_portrait:tag3:device_id:" + str(device_id)
     redis_client = get_redis_client()
     if redis_client.exists(portrait_key):
-        user_portrait = json.loads(redis_client.get(portrait_key))
+        user_portrait = json.loads(redis_client.get(portrait_key).decode("utf-8"))
         first_demands = items_gt_score(user_portrait.get("first_demands", {}))
         second_demands = items_gt_score(user_portrait.get("second_demands", {}))
         first_solutions = items_gt_score(user_portrait.get("first_solutions", {}))