模型调试

spark/featureEngSk.py

@@ -294,12 +294,12 @@ def getUserProfileFeature(spark, startDay, endDay):
 
     addOneDay_UDF = F.udf(addOneDay, StringType())
     userProfileFeatureDF = userProfileFeatureDF.withColumn('partition_date', addOneDay_UDF('dt'))\
-        .withColumn("os", USER_PREFIX + CATEGORY_PREFIX + "os")\
-        .withColumn("user_city_id", USER_PREFIX + CATEGORY_PREFIX + "user_city_id")\
-        .withColumn("second_solutions", USER_PREFIX + CATEGORY_PREFIX + "second_solutions")\
-        .withColumn("second_demands", USER_PREFIX + CATEGORY_PREFIX + "second_demands")\
-        .withColumn("second_positions", USER_PREFIX + CATEGORY_PREFIX + "second_positions")\
-        .withColumn("projects", USER_PREFIX + CATEGORY_PREFIX + "projects")\
+        .withColumnRenamed("os", USER_PREFIX + CATEGORY_PREFIX + "os")\
+        .withColumnRenamed("user_city_id", USER_PREFIX + CATEGORY_PREFIX + "user_city_id")\
+        .withColumnRenamed("second_solutions", USER_PREFIX + CATEGORY_PREFIX + "second_solutions")\
+        .withColumnRenamed("second_demands", USER_PREFIX + CATEGORY_PREFIX + "second_demands")\
+        .withColumnRenamed("second_positions", USER_PREFIX + CATEGORY_PREFIX + "second_positions")\
+        .withColumnRenamed("projects", USER_PREFIX + CATEGORY_PREFIX + "projects")\
         .drop('dt')
     userProfileFeatureDF.cache()
     userProfileFeatureDF.show(20, False)
@@ -954,8 +954,8 @@ if __name__ == '__main__':
         .join(clickStaticFeatures, on = ["card_id", "partition_date"], how = 'left')\
         .join(expStaticFeatures, on = ["card_id", "partition_date"], how = 'left')\
         .join(itemEsFeatureDF, on = ["card_id"], how = 'left')
-    samples = samples.withColumn("card_id", ITEM_PREFIX + CATEGORY_PREFIX + "card_id")\
-        .withColumn("device_id", USER_PREFIX + CATEGORY_PREFIX + "device_id")\
+    samples = samples.withColumnRenamed("card_id", ITEM_PREFIX + CATEGORY_PREFIX + "card_id")\
+        .withColwithColumnRenamedumn("device_id", USER_PREFIX + CATEGORY_PREFIX + "device_id")\
         .drop("partition_date", "timestamp")
 
     # | -- card_id: string(nullable=true)