特征工程优化

spark/featureEng.py

@@ -129,7 +129,7 @@ def addItemFeatures(samples,itemDF,dataVocab,multiVocab):
 
     # 转string
     for c in bucketColumns:
-        samples = samples.withColumn(c + "Bucket",F.col(c + "Bucket").cast("string"))
+        samples = samples.withColumn(c + "Bucket",F.col(c + "Bucket").cast("string")).drop(c)
 
         dataVocab[c + "Bucket"] = [str(float(i)) for i in range(11)]
 
@@ -854,6 +854,6 @@ if __name__ == '__main__':
     dataVocabStr = json.dumps(dataVocab, ensure_ascii=False)
     dataVocabToRedis(dataVocabStr)
     timestmp9 = int(round(time.time()))
-    print("总耗时s:{}".format(timestmp9 - timestmp7))
+    print("总耗时m:{}".format((timestmp9 - start)/60))
 
     spark.stop()
\ No newline at end of file