update save function

src/main_portrait.py

@@ -5,9 +5,9 @@ import time
 
 import tensorflow as tf
 
-from models.esmm.fe import device_fe as device_fe
-from models.esmm.fe import diary_fe as diary_fe
 from models.esmm.diary_model import model_predict_diary
+from models.esmm.fe import device_fe, diary_fe, tractate_fe
+from models.esmm.tractate_model import model_predict_tractate
 from utils.cache import redis_client2
 from utils.grey import recommed_service_category_device_id_by_tail
 from utils.portrait import (user_portrait_tag3_get_candidate_dict, user_portrait_tag3_get_candidate_unread_list,
@@ -32,7 +32,7 @@ def user_portrait_scan_info(device_dict, diary_dict, predict_fn, tail_number):
                     if (user_portrait_tag3_get_candidate_dict(device_id, "diary")):
                         all_count += 1
                         print(str(all_count) + ": " + device_id)
-                        offline_predict(device_id, device_dict, diary_dict, predict_fn)
+                        offline_predict_diary(device_id, device_dict, diary_dict, predict_fn)
 
         print("all count: " + str(all_count))
         print("scan done " + str(datetime.datetime.now()))
@@ -40,7 +40,7 @@ def user_portrait_scan_info(device_dict, diary_dict, predict_fn, tail_number):
         print(e)
 
 
-def offline_predict(device_id, device_dict, diary_dict, predict_fn):
+def offline_predict_diary(device_id, device_dict, diary_dict, predict_fn):
     time_begin = time.time()
 
     diary_ids = user_portrait_tag3_get_candidate_unread_list(device_id, "diary")
@@ -61,11 +61,11 @@ def main():
     diary_dict = diary_fe.get_diary_dict_from_redis()
     print("redis data: " + str(len(device_dict)) + " " + str(len(diary_dict)))
 
-    save_path = "/home/gmuser/data/models/diary/1596083349"
-    predict_fn = tf.contrib.predictor.from_saved_model(save_path)
+    diary_save_path = "/home/gmuser/data/models/diary/1596083349"
+    diary_predict_fn = tf.contrib.predictor.from_saved_model(diary_save_path)
 
     # device_id = "androidid_a25a1129c0b38f7b"
-    # offline_predict(device_id, device_dict, diary_dict, predict_fn)
+    # offline_predict_diary(device_id, device_dict, diary_dict, diary_predict_fn)
 
     # res = user_portrait_tag3_get_candidate_unread_list(device_id, "diary")
     # print(len(res))
@@ -73,7 +73,7 @@ def main():
 
     tail_number = sys.argv[1]
     # "c", "d", "e", "f"
-    user_portrait_scan_info(device_dict, diary_dict, predict_fn, tail_number)
+    user_portrait_scan_info(device_dict, diary_dict, diary_predict_fn, tail_number)
 
 
 if __name__ == "__main__":

src/train_diary.py

@@ -22,44 +22,44 @@ def main():
 
     # os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
 
-    # # data_path = Path("~/data/cvr_data").expanduser()  # local
-    # data_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
-    # diary_df, diary_click_df, diary_conversion_df = diary_fe.read_csv_data(data_path)
-    # # print(diary_df.sample(1))
-    # diary_df = diary_fe.diary_feature_engineering(diary_df)
-    # # print(diary_df.sample(1))
-    # device_df = device_fe.read_csv_data(data_path)
-    # # print(diary_df.sample(1))
-    # device_df = device_fe.device_feature_engineering(device_df, "diary")
-    # # print(device_df.sample(1))
-    # cc_df = click_fe.click_feature_engineering(diary_click_df, diary_conversion_df)
-    # # print(cc_df.sample(1))
-    # df = diary_fe.join_features(device_df, diary_df, cc_df)
-    # # print(df.sample(1))
-    # # print(df.dtypes)
-
-    # train_df, test_df = train_test_split(df, test_size=0.2)
-    # train_df, val_df = train_test_split(train_df, test_size=0.2)
-
-    # all_features = fe.build_features(df, diary_fe.INT_COLUMNS, diary_fe.FLOAT_COLUMNS, diary_fe.CATEGORICAL_COLUMNS)
-    # params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
-    # model_path = str(Path("~/data/model_tmp/").expanduser())
-    # # if os.path.exists(model_path):
-    # #     shutil.rmtree(model_path)
-
-    # session_config = tf.compat.v1.ConfigProto()
-    # session_config.gpu_options.allow_growth = True
-    # session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
-    # estimator_config = tf.estimator.RunConfig(session_config=session_config)
-
-    # model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
-    # train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=50000)
-    # eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
-    # tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
-
-    # model_export_path = str(Path("~/data/models/diary").expanduser())
-    # save_path = model_export(model, all_features, model_export_path)
-    # print("save to: " + save_path)
+    # data_path = Path("~/data/cvr_data").expanduser()  # local
+    data_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
+    diary_df, diary_click_df, diary_conversion_df = diary_fe.read_csv_data(data_path)
+    # print(diary_df.sample(1))
+    diary_df = diary_fe.diary_feature_engineering(diary_df)
+    # print(diary_df.sample(1))
+    device_df = device_fe.read_csv_data(data_path)
+    # print(diary_df.sample(1))
+    device_df = device_fe.device_feature_engineering(device_df, "diary")
+    # print(device_df.sample(1))
+    cc_df = click_fe.click_feature_engineering(diary_click_df, diary_conversion_df)
+    # print(cc_df.sample(1))
+    df = diary_fe.join_features(device_df, diary_df, cc_df)
+    # print(df.sample(1))
+    # print(df.dtypes)
+
+    train_df, test_df = train_test_split(df, test_size=0.2)
+    train_df, val_df = train_test_split(train_df, test_size=0.2)
+
+    all_features = fe.build_features(df, diary_fe.INT_COLUMNS, diary_fe.FLOAT_COLUMNS, diary_fe.CATEGORICAL_COLUMNS)
+    params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
+    model_path = str(Path("~/data/model_tmp/").expanduser())
+    # if os.path.exists(model_path):
+    #     shutil.rmtree(model_path)
+
+    session_config = tf.compat.v1.ConfigProto()
+    session_config.gpu_options.allow_growth = True
+    session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
+    estimator_config = tf.estimator.RunConfig(session_config=session_config)
+
+    model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
+    train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=50000)
+    eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
+    tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
+
+    model_export_path = str(Path("~/data/models/diary").expanduser())
+    save_path = model_export(model, all_features, model_export_path)
+    print("save to: " + save_path)
 
     diary_train_columns = set(diary_fe.INT_COLUMNS + diary_fe.FLOAT_COLUMNS + diary_fe.CATEGORICAL_COLUMNS)
     diary_predict_columns = set(PREDICTION_ALL_COLUMNS)
@@ -69,7 +69,7 @@ def main():
 
     print("============================================================")
     # save_path = str(Path("~/Desktop/models/1596012827").expanduser())  # local
-    save_path = "/home/gmuser/data/models/diary/1596083349"  # server
+    # save_path = "/home/gmuser/data/models/diary/1596083349"  # server
 
     # tf.saved_model.load
 

src/train_tractate.py

@@ -20,43 +20,43 @@ def main():
 
     tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
 
-    # # data_path = Path("~/data/cvr_data").expanduser()  # local
-    # data_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
-    # tractate_df, tractate_click_df, tractate_conversion_df = tractate_fe.read_csv_data(data_path)
-    # tractate_df = tractate_fe.tractate_feature_engineering(tractate_df)
-    # device_df = device_fe.read_csv_data(data_path)
-    # device_df = device_fe.device_feature_engineering(device_df, "tractate")
-    # # print(device_df.columns)
-    # # print(device_df.dtypes, "\n")
-
-    # cc_df = click_fe.click_feature_engineering(tractate_click_df, tractate_conversion_df)
-    # df = tractate_fe.join_features(device_df, tractate_df, cc_df)
-    # # for i in df.columns:
-    # #     print(i)
-    # # print(df.dtypes)
-
-    # train_df, test_df = train_test_split(df, test_size=0.2)
-    # train_df, val_df = train_test_split(train_df, test_size=0.2)
-
-    # all_features = fe.build_features(df, tractate_fe.INT_COLUMNS, tractate_fe.FLOAT_COLUMNS, tractate_fe.CATEGORICAL_COLUMNS)
-    # params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
-    # model_path = str(Path("~/data/model_tmp/").expanduser())
-    # # if os.path.exists(model_path):
-    # #     shutil.rmtree(model_path)
-
-    # session_config = tf.compat.v1.ConfigProto()
-    # session_config.gpu_options.allow_growth = True
-    # session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
-    # estimator_config = tf.estimator.RunConfig(session_config=session_config)
-
-    # model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
-    # train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=50000)
-    # eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
-    # tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
-
-    # model_export_path = str(Path("~/data/models/tractate/").expanduser())
-    # save_path = model_export(model, all_features, model_export_path)
-    # print("save to: " + save_path)
+    # data_path = Path("~/data/cvr_data").expanduser()  # local
+    data_path = Path("/srv/apps/node2vec_git/cvr_data/")  # server
+    tractate_df, tractate_click_df, tractate_conversion_df = tractate_fe.read_csv_data(data_path)
+    tractate_df = tractate_fe.tractate_feature_engineering(tractate_df)
+    device_df = device_fe.read_csv_data(data_path)
+    device_df = device_fe.device_feature_engineering(device_df, "tractate")
+    # print(device_df.columns)
+    # print(device_df.dtypes, "\n")
+
+    cc_df = click_fe.click_feature_engineering(tractate_click_df, tractate_conversion_df)
+    df = tractate_fe.join_features(device_df, tractate_df, cc_df)
+    # for i in df.columns:
+    #     print(i)
+    # print(df.dtypes)
+
+    train_df, test_df = train_test_split(df, test_size=0.2)
+    train_df, val_df = train_test_split(train_df, test_size=0.2)
+
+    all_features = fe.build_features(df, tractate_fe.INT_COLUMNS, tractate_fe.FLOAT_COLUMNS, tractate_fe.CATEGORICAL_COLUMNS)
+    params = {"feature_columns": all_features, "hidden_units": [64, 32], "learning_rate": 0.1}
+    model_path = str(Path("~/data/model_tmp/").expanduser())
+    # if os.path.exists(model_path):
+    #     shutil.rmtree(model_path)
+
+    session_config = tf.compat.v1.ConfigProto()
+    session_config.gpu_options.allow_growth = True
+    session_config.gpu_options.per_process_gpu_memory_fraction = 0.9
+    estimator_config = tf.estimator.RunConfig(session_config=session_config)
+
+    model = tf.estimator.Estimator(model_fn=esmm_model_fn, params=params, model_dir=model_path, config=estimator_config)
+    train_spec = tf.estimator.TrainSpec(input_fn=lambda: esmm_input_fn(train_df, shuffle=True), max_steps=50000)
+    eval_spec = tf.estimator.EvalSpec(input_fn=lambda: esmm_input_fn(val_df, shuffle=False))
+    tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
+
+    model_export_path = str(Path("~/data/models/tractate/").expanduser())
+    save_path = model_export(model, all_features, model_export_path)
+    print("save to: " + save_path)
 
     tractate_train_columns = set(tractate_fe.INT_COLUMNS + tractate_fe.FLOAT_COLUMNS + tractate_fe.CATEGORICAL_COLUMNS)
     tractate_predict_columns = set(PREDICTION_ALL_COLUMNS)
@@ -67,7 +67,7 @@ def main():
     print("============================================================")
 
     # # save_path = str(Path("~/data/models/tractate/1596089465").expanduser())  # local
-    save_path = "/home/gmuser/data/models/tractate/1596092061"  # server
+    # save_path = "/home/gmuser/data/models/tractate/1596092061"  # server
     predict_fn = tf.contrib.predictor.from_saved_model(save_path)
 
     device_dict = device_fe.get_device_dict_from_redis()