增加特征

eda/esmm/Model_pipline/feature.py

@@ -54,7 +54,7 @@ def get_data():
 
 # 上面order_tag 表不要简称为order，因为order是mysql的保留字，例如order by
     df = con_sql(db, sql)
-    print(df.shape)
+    # print(df.shape)
     df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id", 4: "clevel2_id", 5: "ccity_name",
                             6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "device_id",
                             11: "time", 12: "app_list", 13: "service_id", 14: "level3_ids", 15: "level2",
@@ -103,7 +103,6 @@ def get_data():
         df[i] = df[i].fillna("lost_na")
         df[i] = df[i].apply(app_list_func, args=(level2_map,))
 
-
     unique_values = []
     features = ["ucity_id", "ccity_name", "device_type", "manufacturer",
                 "channel", "top", "time", "stat_date", "hospital_id",

tensnsorflow/multi.py

@@ -6,6 +6,7 @@ import pytispark.pytispark as pti
 from pyspark.sql import SparkSession
 import datetime
 import pandas as pd
+import avro.schema
 
 
 def app_list_func(x,l):
@@ -37,7 +38,7 @@ def feature_engineer():
     validate_date = con_sql(db, sql)[0].values.tolist()[0]
     print("validate_date:" + validate_date)
     temp = datetime.datetime.strptime(validate_date, "%Y-%m-%d")
-    start = (temp - datetime.timedelta(days=3)).strftime("%Y-%m-%d")
+    start = (temp - datetime.timedelta(days=2)).strftime("%Y-%m-%d")
     print(start)
 
     sql = "select e.y,e.z,e.stat_date,e.ucity_id,feat.level2_ids,e.ccity_name,u.device_type,u.manufacturer," \
@@ -53,6 +54,8 @@ def feature_engineer():
 
     df = spark.sql(sql)
 
+    df.write.format("avro").save(path=path + "tr", mode="overwrite")
+
     url = "jdbc:mysql://172.16.30.143:3306/zhengxing"
     jdbcDF = spark.read.format("jdbc").option("driver", "com.mysql.jdbc.Driver").option("url", url) \
         .option("dbtable", "api_service").option("user", 'work').option("password", 'BJQaT9VzDcuPBqkd').load()