美购精排模型

mlp/train.py

@@ -2,7 +2,10 @@ import tensorflow as tf
 import json
 import pandas as pd
 import time
+import sys
+import os
 
+sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
 import utils.connUtils as connUtils
 
 ITEM_NUMBER_COLUMNS = ["smart_rank2"]
@@ -37,7 +40,6 @@ def getDataVocabFromRedis(version):
 
 # 数据类型转换
 def csvTypeConvert(df,data_vocab):
-    df = df.fillna("-1")
     # 离散na值填充
     for k, v in data_vocab.items():
         df[k] = df[k].fillna("-1")

spark/featureEng.py

@@ -186,7 +186,7 @@ def splitAndSaveTrainingTestSamplesByTimeStamp(samples,splitTimestamp, file_path
     # splitTimestamp = quantile[0]
     train = samples.where(F.col("timestampLong") <= splitTimestamp).drop("timestampLong")
     test = samples.where(F.col("timestampLong") > splitTimestamp).drop("timestampLong")
-    print("split train size:{},test size:".format(str(train.count()),str(test.count())))
+    print("split train size:{},test size:{}".format(str(train.count()),str(test.count())))
     trainingSavePath = file_path + '_train'
     testSavePath = file_path + '_test'
     train.write.option("header", "true").option("delimiter", "|").mode('overwrite').csv(trainingSavePath)