Merge branch 'fe' into 'offic'

Fe

See merge request !12

requirements.txt

@@ -7,6 +7,8 @@ numba==0.50.1
 
 redis==2.10.5
 
+pymysql==0.10.0
+
 kafka-python
 python-snappy
 lz4

src/train_diary.py

@@ -14,6 +14,7 @@ from models.esmm.fe import click_fe, device_fe, diary_fe, fe
 from models.esmm.input_fn import esmm_input_fn
 from models.esmm.model import esmm_model_fn, model_export
 from utils.cache import get_essm_model_save_path, set_essm_model_save_path
+from utils.db import record_esmm_auc_to_db
 
 
 def main():
@@ -67,14 +68,18 @@ def main():
     res = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
     print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
     print(res[0])
-    print("ctr_auc: " + str(res[0]["ctr_auc"]))
-    print("ctcvr_auc: " + str(res[0]["ctcvr_auc"]))
+    ctr_auc = str(res[0]["ctr_auc"])
+    ctcvr_auc = str(res[0]["ctcvr_auc"])
+    print("ctr_auc: " + ctr_auc)
+    print("ctcvr_auc: " + ctcvr_auc)
     print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+    total_time = "{:.2f}".format((time.time() - time_begin) / 60)
 
     model_export_path = str(Path("/data/files/models/diary").expanduser())
     save_path = model_export(model, all_features, model_export_path)
     print("save to: " + save_path)
     set_essm_model_save_path("diary", save_path)
+    record_esmm_auc_to_db("diary", ctr_auc, ctcvr_auc, total_time, save_path)
     print("============================================================")
 
     # save_path = str(Path("~/Desktop/models/1596012827").expanduser())  # local
@@ -101,8 +106,7 @@ def main():
         total_1 = (timeit.default_timer() - time_1)
         print("total prediction cost {:.5f}s".format(total_1), "\n")
 
-    total_time = (time.time() - time_begin) / 60
-    print("total cost {:.2f} mins at {}".format(total_time, datetime.now()))
+    print("total cost {} mins at {}".format(total_time, datetime.now()))
 
 
 if __name__ == "__main__":

src/train_tractate.py

@@ -14,6 +14,7 @@ from models.esmm.input_fn import esmm_input_fn
 from models.esmm.model import esmm_model_fn, model_export
 from models.esmm.tractate_model import (PREDICTION_ALL_COLUMNS, model_predict_tractate)
 from utils.cache import get_essm_model_save_path, set_essm_model_save_path
+from utils.db import record_esmm_auc_to_db
 
 
 def main():
@@ -63,18 +64,22 @@ def main():
     res = tf.estimator.train_and_evaluate(model, train_spec, eval_spec)
     print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
     print(res[0])
-    print("ctr_auc: " + str(res[0]["ctr_auc"]))
-    print("ctcvr_auc: " + str(res[0]["ctcvr_auc"]))
+    ctr_auc = str(res[0]["ctr_auc"])
+    ctcvr_auc = str(res[0]["ctcvr_auc"])
+    print("ctr_auc: " + ctr_auc)
+    print("ctcvr_auc: " + ctcvr_auc)
     print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+    total_time = "{:.2f}".format((time.time() - time_begin) / 60)
 
     model_export_path = str(Path("/data/files/models/tractate/").expanduser())
     save_path = model_export(model, all_features, model_export_path)
     print("save to: " + save_path)
     set_essm_model_save_path("tractate", save_path)
+    record_esmm_auc_to_db("tractate", ctr_auc, ctcvr_auc, total_time, save_path)
     print("============================================================")
 
-    save_path = get_essm_model_save_path("tractate")
-    print("load path: " + save_path)
+    # save_path = get_essm_model_save_path("tractate")
+    # print("load path: " + save_path)
 
     # save_path = str(Path("~/data/models/tractate/1598236893").expanduser())  # local
     # save_path = "/data/files/models/tractate/1598254242"  # server
@@ -98,8 +103,7 @@ def main():
         total_1 = (timeit.default_timer() - time_1)
         print("total prediction cost {:.5f}s".format(total_1), "\n")
 
-    total_time = (time.time() - time_begin) / 60
-    print("total cost {:.2f} mins at {}".format(total_time, datetime.now()))
+    print("total cost {} mins at {}".format(total_time, datetime.now()))
 
 
 if __name__ == "__main__":

src/utils/db.py

+import datetime
+
+import pymysql
+
+
+def _write_data_to_mysql(host, port, user, passwd, db, sql):
+    try:
+        db = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, cursorclass=pymysql.cursors.DictCursor)
+        cursor = db.cursor()
+        cursor.execute(sql)
+        db.commit()
+        db.close()
+    except Exception as e:
+        print(e)
+
+
+# CREATE TABLE `esmm_auc_log` (
+#     `id` int(11) NOT NULL AUTO_INCREMENT,
+#     `content_type` text NOT NULL,
+#     `ctr_auc` text NOT NULL,
+#     `ctcvr_auc` text NOT NULL,
+#     `cost` text NOT NULL,
+#     `save_path` text NOT NULL,
+#     `date` datetime,
+#     PRIMARY KEY(`id`)
+# )
+def record_esmm_auc_to_db(content_type, ctr_auc, ctcvr_auc, cost, save_path):
+    dt = datetime.datetime.now()
+    t = dt.strftime("%Y-%m-%d %H:%M:%S")
+    sql = """
+    INSERT INTO esmm_auc_log(content_type, ctr_auc, ctcvr_auc, cost, save_path, date) VALUES ('{}', '{}', '{}', '{}', '{}', '{}')
+    """.format(content_type, ctr_auc, ctcvr_auc, cost, save_path, t)
+    return _write_data_to_mysql("172.16.40.170", 4000, "st_user", "aqpuBLYzEV7tML5RPsN1pntUzFy", "jerry_test", sql)