ctr 平滑

spark/featureEng.py

@@ -83,7 +83,7 @@ numberToBucketUdf = F.udf(numberToBucket, StringType())
 priceToBucketUdf = F.udf(priceToBucket, StringType())
 
 def addItemStaticFeatures(samples,itemDF,dataVocab):
-    ctrUdf = F.udf(wilson_ctr, ArrayType(float()))
+    ctrUdf = F.udf(wilson_ctr, FloatType())
     # item不设置over窗口，原因：item可能一直存在，统计数据按照最新即可
     print("item统计特征处理...")
     staticFeatures = samples.groupBy('item_id').agg(F.count(F.lit(1)).alias('itemRatingCount'),
@@ -244,7 +244,7 @@ def addUserFeatures(samples,dataVocab,multiVocab):
 
     extractTagsUdf = F.udf(extractTags, ArrayType(StringType()))
     arrayReverseUdf = F.udf(arrayReverse, ArrayType(StringType()))
-    ctrUdf = F.udf(wilson_ctr, ArrayType(float()))
+    ctrUdf = F.udf(wilson_ctr, FloatType())
     print("user历史数据处理...")
     # user历史记录
     samples = samples.withColumn('userPositiveHistory',F.collect_list(when(F.col('label') == 1, F.col('item_id')).otherwise(F.lit(None))).over(sql.Window.partitionBy("userid").orderBy(F.col("timestamp")).rowsBetween(-100, -1)))