service model 优化

spark/featureEng2.py

@@ -205,9 +205,12 @@ def addUserFeatures(samples,dataVocab,multiVocab):
     # arrayReverseUdf = F.udf(arrayReverse, ArrayType(StringType()))
     print("user历史数据处理...")
     # user历史记录
-    samples = samples\
-        .withColumn('userPositiveHistory',F.collect_list(when(F.col('label') == 1, F.col('item_id')).otherwise(F.lit(None))).over(sql.Window.partitionBy("userid").orderBy(F.col("timestamp")).rowsBetween(-100, -1))) \
-        .withColumn("userPositiveHistory", F.reverse(F.col("userPositiveHistory")))
+    samples = samples.withColumn('userPositiveHistory',F.collect_list(when(F.col('label') == 1, F.col('item_id')).otherwise(F.lit(None))).over(sql.Window.partitionBy("userid").orderBy(F.col("timestamp")).rowsBetween(-100, -1)))
+    samples.select("userPositiveHistory").show(10,truncate=False)
+
+    samples = samples.withColumn("userPositiveHistory", arrayReverse(F.col("userPositiveHistory")))
+    samples.select("userPositiveHistory").show(10,truncate=False)
+    
     for i in range(1,11):
         samples = samples.withColumn("userRatedHistory"+str(i), F.when(F.col("userPositiveHistory")[i-1].isNotNull(),F.col("userPositiveHistory")[i-1]).otherwise("-1"))
         dataVocab["userRatedHistory"+str(i)] = dataVocab["item_id"]