use gmkv read ts ctr and zstd compress

eda/smart_rank/dist_periodic_update_smart_rank.py

@@ -12,15 +12,26 @@ import datetime
 from pyspark import SparkConf
 import time
 from pyspark.sql import SparkSession
+import json
+import zstd
 
+def zstd_compress(x):
+    if x:
+        return zstd.compress(bytes(json.dumps(x), encoding='utf-8'), 22)
 
-# ctr变化(更新用户的所有美购smart rank分_
+
+def zstd_decompress(x):
+    if x:
+        result = zstd.decompress(x)
+        return json.loads(result)
+
+
+# ctr变化(更新用户的所有美购smart rank分
 def update_device_smart_rank(device_id, result_all_dict, service_detail_view_count_30_dict, result_smart_rank_score_dict):
     device_meigou_ctr_key = 'device_meigou_ctr:device_id:' + str(device_id)
-    REDIS_URL = 'redis://:ReDis!GmTx*0aN6@172.16.40.133:6379'
-    cli_ins = redis.StrictRedis.from_url(REDIS_URL)
-    if cli_ins.exists(device_meigou_ctr_key):
-        ts_device_meigou_ctr = cli_ins.hgetall(device_meigou_ctr_key)
+    gm_kv_cli = redis.Redis(host="172.16.40.135", port=5379, db=2, socket_timeout=2000)
+    if gm_kv_cli.exists(device_meigou_ctr_key):
+        ts_device_meigou_ctr = gm_kv_cli.hgetall(device_meigou_ctr_key)
         device_meigou_smart_rank = dict()
         for i in ts_device_meigou_ctr:
             ts_ctr = float(ts_device_meigou_ctr[i])
@@ -28,50 +39,15 @@ def update_device_smart_rank(device_id, result_all_dict, service_detail_view_cou
             meigou_smart_rank_score = get_meigou_smart_rank(service_id, result_all_dict, service_detail_view_count_30_dict, ts_ctr, result_smart_rank_score_dict)
             device_meigou_smart_rank.update({service_id: meigou_smart_rank_score})
 
-        device_meigou_smart_rank_key = 'device_meigou_smart_rank:device_id:' + str(device_id)
-        cli_ins.hmset(device_meigou_smart_rank_key, device_meigou_smart_rank)
+        device_meigou_smart_rank_key = 'device_meigou_smart_rank_zstd:device_id:' + str(device_id)
+        REDIS_URL = 'redis://:ReDis!GmTx*0aN6@172.16.40.133:6379'
+        cli_ins = redis.StrictRedis.from_url(REDIS_URL)
+        cli_ins.set(device_meigou_smart_rank_key, zstd_compress(device_meigou_smart_rank))
         cli_ins.expire(device_meigou_smart_rank_key, time=24 * 60 * 60)
         return meigou_smart_rank_score
     return "periodic update fail"
 
 
-# cpc变化(更新每个用户cpc发生变化的美购smart rank分)
-def update_all_device_meigou_smart_rank(service_id, result_all_dict, service_detail_view_count_30_dict, result_smart_rank_score_dict):
-    #cpc价格
-    db_artemis_prod = pymysql.connect(host="172.16.30.137", port=3306, user="artemis", password="gaehau9shai9Ae",
-                                      db="artemis_prod", cursorclass=pymysql.cursors.DictCursor)
-    cur_artemis_prod = db_artemis_prod.cursor()
-    sql_cpc = "select service_id,is_promote,click_price from cpc_promote where service_id={}".format(service_id)
-    cur_artemis_prod.execute(sql_cpc)
-    result_cpc = cur_artemis_prod.fetchall()
-    if len(result_cpc) > 0:
-        if result_cpc[0]["is_promote"]:
-            click_price = result_cpc[0]["click_price"]
-        else:
-            click_price = 0
-    else:
-        click_price = 0
-    db_artemis_prod.close()
-
-    Today_active_device_key = 'Today_active_device:device_id:' + str(datetime.datetime.now().strftime('%Y-%m-%d'))
-    REDIS_URL = 'redis://:ReDis!GmTx*0aN6@172.16.40.133:6379'
-    cli_ins = redis.StrictRedis.from_url(REDIS_URL)
-    if cli_ins.exists(Today_active_device_key):
-        Today_active_device = cli_ins.smembers(Today_active_device_key)
-        for i in Today_active_device:
-            device_meigou_smart_rank_key = 'device_meigou_smart_rank:device_id:' + str(i, encoding='utf-8')
-            device_meigou_ctr_key = 'device_meigou_ctr:device_id:' + str(i, encoding='utf-8')
-            if cli_ins.exists(device_meigou_ctr_key):
-                ts_device_meigou_ctr = cli_ins.hget(device_meigou_ctr_key, service_id)
-                if not ts_device_meigou_ctr:
-                    ts_device_meigou_ctr = 1.0
-            else:
-                ts_device_meigou_ctr = 1.0
-            score = get_meigou_smart_rank(service_id, result_all_dict, service_detail_view_count_30_dict, float(ts_device_meigou_ctr), result_smart_rank_score_dict, click_price)
-            if cli_ins.exists(device_meigou_smart_rank_key):
-                cli_ins.hmset(device_meigou_smart_rank_key, {service_id : score})
-
-
 # 获取美购的smart rank分
 def get_meigou_smart_rank(service_id, result_all_dict, service_detail_view_count_30_dict, meigou_ctr, result_smart_rank_score_dict, table_cpc_price=-1):
     if service_id in result_all_dict: