特征工程优化

spark/featureEng.py

@@ -834,7 +834,7 @@ if __name__ == '__main__':
     train_columns = model_columns + ["label", "timestamp"]
     trainSamples = samplesWithUserFeatures.select(*train_columns)
     print("write to hdfs start...")
-    splitTimestamp = int(time.mktime(time.strptime(addDays(0), "%Y%m%d")))
+    splitTimestamp = int(time.mktime(time.strptime(addDays(-1), "%Y%m%d")))
     splitAndSaveTrainingTestSamplesByTimeStamp(trainSamples, splitTimestamp, TRAIN_FILE_PATH)
     print("write to hdfs success...")
     timestmp7 = int(round(time.time()))

train/train_service.py

@@ -185,7 +185,7 @@ if __name__ == '__main__':
     print("读取数据...")
     timestmp1 = int(round(time.time()))
     df_train = loadData(data_path_train)
-    # df_test = loadData(data_path_test)
+    df_test = loadData(data_path_test)
     timestmp2 = int(round(time.time()))
     print("读取数据耗时s:{}".format(timestmp2 - timestmp1))
 
@@ -198,20 +198,20 @@ if __name__ == '__main__':
     print(datasColumns)
 
     df_train = df_train[datasColumns + ["label"]]
-    # df_test = df_test[datasColumns + ["label"]]
+    df_test = df_test[datasColumns + ["label"]]
 
     trainSize = df_train["label"].count()
     print("trainSize:{}".format(trainSize))
-    # testSize = df_test["label"].count()
-    # print("trainSize:{},testSize{}".format(trainSize,testSize))
+    testSize = df_test["label"].count()
+    print("trainSize:{},testSize{}".format(trainSize,testSize))
 
     # 数据类型转换
     df_train = csvTypeConvert(datasColumns,df_train,data_vocab)
-    # df_test = csvTypeConvert(datasColumns,df_test,data_vocab)
+    df_test = csvTypeConvert(datasColumns,df_test,data_vocab)
 
     # 获取训练数据
     train_data = getDataSet(df_train,shuffleSize=trainSize,)
-    # test_data = getDataSet(df_test,shuffleSize=testSize)
+    test_data = getDataSet(df_test,shuffleSize=testSize)
 
 
     print("train start...")