add check

eda/esmm/Model_pipline/submit.sh

@@ -59,6 +59,8 @@ ${PYTHON_PATH} ${MODEL_PATH}/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --le
 
 echo "train time"
 current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
 echo $current
 
 echo "infer native..."

tensnsorflow/ffm.py

@@ -136,6 +136,49 @@ class multiFFMFormatPandas:
         else:
             return False
 
+def check():
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select max(stat_date) from esmm_train_data"
+    validate_date = con_sql(db, sql)[0].values.tolist()[0]
+    print("validate_date:" + validate_date)
+    temp = datetime.datetime.strptime(validate_date, "%Y-%m-%d")
+    start = (temp - datetime.timedelta(days=30)).strftime("%Y-%m-%d")
+    print(start)
+    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
+          "u.device_type,u.manufacturer,u.channel,c.top,df.level2_ids,e.device_id " \
+          "from esmm_train_data e left join user_feature u on e.device_id = u.device_id " \
+          "left join cid_type_top c on e.device_id = c.device_id " \
+          "left join diary_feat df on e.cid_id = df.diary_id " \
+          "where e.stat_date >= '{}'".format(start)
+    df = con_sql(db, sql)
+    print(df.shape)
+    df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id",4: "clevel1_id", 5: "ccity_name",
+                            6:"device_type",7:"manufacturer",8:"channel",9:"top",10:"level2_ids",11:"device_id"})
+    print("esmm data ok")
+    df = df.fillna("na")
+    # print(df.head(2)
+    df["y"] = df["y"]+df["z"]
+    df["clevel1_id"] = df["clevel1_id"].astype("str")
+    df["top"] = df["top"].astype("str")
+    df["feat"] = df["ucity_id"].str.cat([df["clevel1_id"].values.tolist(),df["ccity_name"].values.tolist(),
+                                         df["device_type"].values.tolist(),df["manufacturer"].values.tolist(),
+                                         df["channel"].values.tolist(), df["top"].values.tolist(),
+                                         df["level2_ids"].values.tolist()], sep=",")
+
+    df = df.drop(["z","stat_date","ucity_id","clevel1_id",  "ccity_name","device_type",
+                  "manufacturer","channel","top","level2_ids","device_id"], axis=1)
+    print(df.head(2))
+
+    print("\n")
+    print("before drop duplicate")
+    print(df.shape[0])
+    print("after drop duplicate")
+    df = df.drop_duplicates()
+    print(df.shape[0])
+    print("after group by")
+    print(len(df.groupby("feat")))
+
 
 
 def get_data():
@@ -297,10 +340,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids):
 
 if __name__ == "__main__":
     path = "/home/gmuser/esmm_data/"
-    a = time.time()
-    temp, validate_date, ucity_id,ccity_name,manufacturer,channel,level2_ids = get_data()
-    model = transform(temp, validate_date)
-    get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids)
-    b = time.time()
-    print("cost(分钟)")
-    print((b-a)/60)
+    # a = time.time()
+    # temp, validate_date, ucity_id,ccity_name,manufacturer,channel,level2_ids = get_data()
+    # model = transform(temp, validate_date)
+    # get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids)
+    # b = time.time()
+    # print("cost(分钟)")
+    # print((b-a)/60)
+
+    check()