retrain diary

src/main.py

@@ -28,46 +28,44 @@ def main():
 
     # data_path = Path("~/data/cvr_data").expanduser()  # local
     data_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
-    diary_df, click_df, conversion_df = diary_fe.read_csv_data(data_path)
+    diary_df, diary_click_df, diary_conversion_df = diary_fe.read_csv_data(data_path)
     device_df = device_fe.read_csv_data(data_path)
     # print(diary_df.sample(1))
     device_df = device_fe.device_feature_engineering(device_df)
     # print(device_df.sample(1))
     diary_df = diary_fe.diary_feature_engineering(diary_df)
     # print(diary_df.sample(1))
-    cc_df = diary_fe.click_feature_engineering(click_df, conversion_df)
+    cc_df = diary_fe.click_feature_engineering(diary_click_df, diary_conversion_df)
     # print(cc_df.sample(1))
     df = diary_fe.join_features(device_df, diary_df, cc_df)
     # print(df.sample(1))
-    print(df.dtypes)
+    # print(df.dtypes)
 
-    # train_df, test_df = train_test_split(df, test_size=0.2)
-    # train_df, val_df = train_test_split(train_df, test_size=0.2)
+    train_df, test_df = train_test_split(df, test_size=0.2)
+    train_df, val_df = train_test_split(train_df, test_size=0.2)
 
-    # all_features = build_features(df)
-    # params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
-    # model_path = str(Path("~/data/model_tmp/").expanduser())
-    # # if os.path.exists(model_path):
-    # #     shutil.rmtree(model_path)
+    all_features = build_features(df)
+    params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
+    model_path = str(Path("~/data/model_tmp/").expanduser())
+    # if os.path.exists(model_path):
+    #     shutil.rmtree(model_path)
 
-    # session_config = tf.compat.v1.ConfigProto()
-    # session_config.gpu_options.allow_growth = True
-    # session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
-    # estimator_config = tf.estimator.RunConfig(session_config=session_config)
+    session_config = tf.compat.v1.ConfigProto()
+    session_config.gpu_options.allow_growth = True
+    session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
+    estimator_config = tf.estimator.RunConfig(session_config=session_config)
 
-    # model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
-    # # TODO 50000
-    # train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=20000)
-    # eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
-    # tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
+    model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
+    train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=50000)
+    eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
+    tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
 
-    # model_export_path = str(Path("~/data/models/").expanduser())
-    # save_path = model_export(model, all_features, model_export_path)
-    # print("save to: " + save_path)
+    model_export_path = str(Path("~/data/models/").expanduser())
+    save_path = model_export(model, all_features, model_export_path)
+    print("save to: " + save_path)
 
-    save_path = "/home/gmuser/data/models/1596012827"
-
-    # save_path = str(Path("~/Desktop/models/1596012827").expanduser())
+    # save_path = str(Path("~/Desktop/models/1596012827").expanduser())  # local
+    # save_path = "/home/gmuser/data/models/1596012827"  # server
 
     # tf.saved_model.load