ctr 平滑

spark/featureEng.py

@@ -83,6 +83,7 @@ numberToBucketUdf = F.udf(numberToBucket, StringType())
 priceToBucketUdf = F.udf(priceToBucket, StringType())
 
 def addItemStaticFeatures(samples,itemDF,dataVocab):
+    ctrUdf = F.udf(wilson_ctr, ArrayType(float()))
     # item不设置over窗口，原因：item可能一直存在，统计数据按照最新即可
     print("item统计特征处理...")
     staticFeatures = samples.groupBy('item_id').agg(F.count(F.lit(1)).alias('itemRatingCount'),
@@ -93,7 +94,7 @@ def addItemStaticFeatures(samples,itemDF,dataVocab):
                                                     ).fillna(0) \
         .withColumn('itemRatingStddev', F.format_number(F.col('itemRatingStddev'), NUMBER_PRECISION).cast("float")) \
         .withColumn('itemRatingAvg', F.format_number(F.col('itemRatingAvg'), NUMBER_PRECISION).cast("float")) \
-        .withColumn('itemCtr',F.format_number(F.col("itemClickCount") / (F.col("itemExpCount") + 1), NUMBER_PRECISION).cast("float"))
+        .withColumn('itemCtr',F.format_number(ctrUdf(F.col("itemClickCount"),(F.col("itemExpCount"))), NUMBER_PRECISION).cast("float"))
 
     staticFeatures.show(20, truncate=False)