Commit 50e152fe authored by 高雅喆's avatar 高雅喆

update

parent 1752671c
......@@ -110,6 +110,7 @@ def get_user_tag_score(cl_id, all_log_df, gm_kv_cli, size=10):
finally_score_lst = finally_score[["tag_id","tag_score"]].to_dict('record')
tag_id_list = tag_list2dict(finally_score_lst, size)
# 写gmkv
gm_kv_cli = redis.Redis(host="172.16.40.135", port=5379, db=2, socket_timeout=2000)
cl_id_portrait_key = "user:portrait_tags:cl_id:" + str(cl_id)
tag_id_list_json = json.dumps(tag_id_list)
gm_kv_cli.set(cl_id_portrait_key, tag_id_list_json)
......@@ -154,7 +155,6 @@ if __name__ == '__main__':
spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
spark.sparkContext.setLogLevel("WARN")
gm_kv_cli = redis.Redis(host="172.16.40.135", port=5379, db=2, socket_timeout=2000)
device_ids_lst_rdd = spark.sparkContext.parallelize(device_ids_lst)
result = device_ids_lst_rdd.repartition(100).map(lambda x: get_user_tag_score(x, all_log_df, gm_kv_cli))
result.collect()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment