change test file

tensnsorflow/feature_engineering.py

@@ -133,7 +133,7 @@ def feature_engineer():
     validate_date = con_sql(db, sql)[0].values.tolist()[0]
     print("validate_date:" + validate_date)
     temp = datetime.datetime.strptime(validate_date, "%Y-%m-%d")
-    start = (temp - datetime.timedelta(days=300)).strftime("%Y-%m-%d")
+    start = (temp - datetime.timedelta(days=3)).strftime("%Y-%m-%d")
     print(start)
 
     db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC')
@@ -188,33 +188,34 @@ def feature_engineer():
                     "tag1","tag2","tag3","tag4","tag5","tag6","tag7",
                      "ucity_id", "ccity_name","device_type", "manufacturer", "channel", "top", "time",
                     "hospital_id","treatment_method", "price_min", "price_max", "treatment_time",
-                    "maintain_time","recover_time").rdd.coalesce(200).map(lambda x: (x[0],float(x[1]),float(x[2]),
+                    "maintain_time","recover_time").rdd.coalesce(200)\
+        .map(lambda x: (x[0],float(x[1]),float(x[2]),
                         app_list_func(x[3], app_list_map), app_list_func(x[4], level2_map),
-                        app_list_func(x[5], level3_map), app_list_func(x[6], level2_map),app_list_func(x[7], level2_map),
-                        app_list_func(x[8], level2_map), app_list_func(x[9], level2_map),app_list_func(x[10], level2_map),
-                        app_list_func(x[11], level2_map),app_list_func(x[12], level2_map),
-                        [value_map[x[0]], value_map[x[13]],value_map[x[14]], value_map[x[15]], value_map[x[16]],
-                         value_map[x[17]],value_map[x[18]], value_map[x[19]], value_map[x[20]],value_map[x[21]],
-                         value_map[x[22]], value_map[x[23]], value_map[x[24]],value_map[x[25]],value_map[x[26]]]))
+                        app_list_func(x[5], level3_map), app_list_func(x[6], level2_map),
+                        app_list_func(x[7], level2_map),app_list_func(x[8], level2_map),
+                        app_list_func(x[9], level2_map),app_list_func(x[10], level2_map),
+                        app_list_func(x[11],level2_map),app_list_func(x[12], level2_map),
+                        [value_map[x[0]], value_map[x[13]],value_map[x[14]], value_map[x[15]],
+                         value_map[x[16]],value_map[x[17]],value_map[x[18]], value_map[x[19]],
+                         value_map[x[20]],value_map[x[21]],value_map[x[22]], value_map[x[23]],
+                         value_map[x[24]],value_map[x[25]],value_map[x[26]]]))
     d = time.time()
     rdd.persist()
-    print("rdd")
-    print((d - c) / 60)
+
     # TODO 上线后把下面train fliter 删除，因为最近一天的数据也要作为训练集
 
-    train = rdd.filter(lambda x: x[0] != validate_date).map(lambda x:(x[1],x[2],x[3],x[4],x[5],x[6],x[7],x[8],x[9],
-                                                                      x[10],x[11],x[12],x[13]))
-    f = time.time()
+    train = rdd.filter(lambda x: x[0] != validate_date)\
+        .map(lambda x:(x[1],x[2],x[3],x[4],x[5],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13]))
     spark.createDataFrame(train).toDF("y","z","app_list","level2_list","level3_list",
                                       "tag1_list","tag2_list","tag3_list","tag4_list",
                                       "tag5_list","tag6_list","tag7_list","ids") \
         .write.format("tfrecords").save(path=path + "tr/", mode="overwrite")
     h = time.time()
     print("train tfrecord done")
-    print((h-f)/60)
+    print((h-d)/60)
 
-    test = rdd.filter(lambda x: x[0] == validate_date).map(lambda x:(x[1],x[2],x[3],x[4],x[5],x[6],x[7],x[8],x[9],
-                                                                      x[10],x[11],x[12],x[13]))
+    test = rdd.filter(lambda x: x[0] == validate_date)\
+        .map(lambda x:(x[1],x[2],x[3],x[4],x[5],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13]))
 
     spark.createDataFrame(test).toDF("y", "z", "app_list", "level2_list", "level3_list",
                                       "tag1_list", "tag2_list", "tag3_list", "tag4_list",