Skip to content

S
serviceRec
Commit 088b2164 authored by 郭羽's avatar 郭羽
Browse files
Options

update feature

parent 7c8d6595
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
spark/featureEng.py
View file @ 088b2164
...@@ -346,7 +346,7 @@ def featuresToRedis(samples,columns,prefix,redisKey): ...@@ -346,7 +346,7 @@ def featuresToRedis(samples,columns,prefix,redisKey):
prefixSamples = samples.groupBy(idCol).agg(F.max("timestamp").alias(timestampCol)) prefixSamples = samples.groupBy(idCol).agg(F.max("timestamp").alias(timestampCol))
resDatas = samples.join(prefixSamples, on=[idCol], how='left').where(F.col("timestamp") == F.col(timestampCol)) resDatas = samples.join(prefixSamples, on=[idCol], how='left').where(F.col("timestamp") == F.col(timestampCol))
resDatas = resDatas.select(*columns).distinct() resDatas = resDatas.select(*columns).distinct()
print(prefix, len(resDatas)) print(prefix, resDatas.count())
resDatas.repartition(8).foreachPartition(toRedis) resDatas.repartition(8).foreachPartition(toRedis)
""" """
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment