get col

src/models/esmm/diary_model.py

@@ -148,6 +148,7 @@ def model_predict_diary(device_id, diary_ids, device_dict, diary_dict, predict_f
             for col in _int_columns:
                 features[col] = _int64_feature(int(tmp[col]))
             for col in _float_columns:
+                print(col)
                 features[col] = _float_feature(float(tmp[col]))
             for col in _categorical_columns:
                 features[col] = _bytes_feature(str(tmp[col]).encode(encoding="utf-8"))

src/train_diary.py

@@ -19,64 +19,64 @@ from utils.cache import get_essm_model_save_path, set_essm_model_save_path
 def main():
     time_begin = time.time()
 
-    tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
-
-    # os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-    diary_train_columns = set(diary_fe.INT_COLUMNS + diary_fe.FLOAT_COLUMNS + diary_fe.CATEGORICAL_COLUMNS)
-    print("features: " + str(len(diary_train_columns)))
-    diary_predict_columns = set(PREDICTION_ALL_COLUMNS)
-    print(diary_predict_columns.difference(diary_train_columns))
-    print(diary_train_columns.difference(diary_predict_columns))
-    assert diary_predict_columns == diary_train_columns
-
-    # dataset_path = Path("~/data/cvr_data").expanduser()  # local
-    dataset_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
-    diary_df, diary_click_df, diary_conversion_df = diary_fe.read_csv_data(dataset_path)
-    # print(diary_df.sample(1))
-    diary_df = diary_fe.diary_feature_engineering(diary_df)
-    # print(diary_df.sample(1))
-    device_df = device_fe.read_csv_data(dataset_path)
-    # print(diary_df.sample(1))
-    device_df = device_fe.device_feature_engineering(device_df, "diary")
-    # print(device_df.sample(1))
-    cc_df = click_fe.click_feature_engineering(diary_click_df, diary_conversion_df)
-    # print(cc_df.sample(1))
-    df = diary_fe.join_features(device_df, diary_df, cc_df)
-    # print(df.sample(1))
-    # print(df.dtypes)
-
-    train_df, test_df = train_test_split(df, test_size=0.2)
-    train_df, val_df = train_test_split(train_df, test_size=0.2)
-
-    all_features = fe.build_features(df, diary_fe.INT_COLUMNS, diary_fe.FLOAT_COLUMNS, diary_fe.CATEGORICAL_COLUMNS)
-    params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
-    model_path = str(Path("/data/files/model_tmp/diary/").expanduser())
-    if os.path.exists(model_path):
-        shutil.rmtree(model_path)
-
-    session_config = tf.compat.v1.ConfigProto()
-    session_config.gpu_options.allow_growth = True
-    session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
-    estimator_config = tf.estimator.RunConfig(session_config=session_config)
-
-    model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
-    # TODO 50000
-    train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=15000)
-    eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
-    res = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-    print(res[0])
-    print("ctr_auc: " + str(res[0]["ctr_auc"]))
-    print("ctcvr_auc: " + str(res[0]["ctcvr_auc"]))
-    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
-
-    model_export_path = str(Path("/data/files/models/diary").expanduser())
-    save_path = model_export(model, all_features, model_export_path)
-    print("save to: " + save_path)
-    # TODO save
-    # set_essm_model_save_path("diary", save_path)
-    print("============================================================")
+    # tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
+
+    # # os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+
+    # diary_train_columns = set(diary_fe.INT_COLUMNS + diary_fe.FLOAT_COLUMNS + diary_fe.CATEGORICAL_COLUMNS)
+    # print("features: " + str(len(diary_train_columns)))
+    # diary_predict_columns = set(PREDICTION_ALL_COLUMNS)
+    # print(diary_predict_columns.difference(diary_train_columns))
+    # print(diary_train_columns.difference(diary_predict_columns))
+    # assert diary_predict_columns == diary_train_columns
+
+    # # dataset_path = Path("~/data/cvr_data").expanduser()  # local
+    # dataset_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
+    # diary_df, diary_click_df, diary_conversion_df = diary_fe.read_csv_data(dataset_path)
+    # # print(diary_df.sample(1))
+    # diary_df = diary_fe.diary_feature_engineering(diary_df)
+    # # print(diary_df.sample(1))
+    # device_df = device_fe.read_csv_data(dataset_path)
+    # # print(diary_df.sample(1))
+    # device_df = device_fe.device_feature_engineering(device_df, "diary")
+    # # print(device_df.sample(1))
+    # cc_df = click_fe.click_feature_engineering(diary_click_df, diary_conversion_df)
+    # # print(cc_df.sample(1))
+    # df = diary_fe.join_features(device_df, diary_df, cc_df)
+    # # print(df.sample(1))
+    # # print(df.dtypes)
+
+    # train_df, test_df = train_test_split(df, test_size=0.2)
+    # train_df, val_df = train_test_split(train_df, test_size=0.2)
+
+    # all_features = fe.build_features(df, diary_fe.INT_COLUMNS, diary_fe.FLOAT_COLUMNS, diary_fe.CATEGORICAL_COLUMNS)
+    # params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
+    # model_path = str(Path("/data/files/model_tmp/diary/").expanduser())
+    # if os.path.exists(model_path):
+    #     shutil.rmtree(model_path)
+
+    # session_config = tf.compat.v1.ConfigProto()
+    # session_config.gpu_options.allow_growth = True
+    # session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
+    # estimator_config = tf.estimator.RunConfig(session_config=session_config)
+
+    # model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
+    # # TODO 50000
+    # train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=15000)
+    # eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
+    # res = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
+    # print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+    # print(res[0])
+    # print("ctr_auc: " + str(res[0]["ctr_auc"]))
+    # print("ctcvr_auc: " + str(res[0]["ctcvr_auc"]))
+    # print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+
+    # model_export_path = str(Path("/data/files/models/diary").expanduser())
+    # save_path = model_export(model, all_features, model_export_path)
+    # print("save to: " + save_path)
+    # # TODO save
+    # # set_essm_model_save_path("diary", save_path)
+    # print("============================================================")
 
     # save_path = str(Path("~/Desktop/models/1596012827").expanduser())  # local
     # save_path = "/data/files/models/diary/1597390452"  # server