change test file

tensnsorflow/multi.py

@@ -135,16 +135,18 @@ if __name__ == '__main__':
         .set("spark.driver.maxResultSize", "8g").set("spark.sql.avro.compression.codec","snappy")
 
     spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
-    ti = pti.TiContext(spark)
-    ti.tidbMapDatabase("jerry_test")
-    # ti.tidbMapDatabase("eagle")
-    spark.sparkContext.setLogLevel("WARN")
-    path = "hdfs:///strategy/esmm/"
-    local_path = "/home/gmuser/esmm/"
-
-    validate_date, value_map, app_list_map = feature_engineer()
-    get_predict(validate_date, value_map, app_list_map)
-
+    # ti = pti.TiContext(spark)
+    # ti.tidbMapDatabase("jerry_test")
+    # # ti.tidbMapDatabase("eagle")
+    # spark.sparkContext.setLogLevel("WARN")
+    # path = "hdfs:///strategy/esmm/"
+    # local_path = "/home/gmuser/esmm/"
+    #
+    # validate_date, value_map, app_list_map = feature_engineer()
+    # get_predict(validate_date, value_map, app_list_map)
+
+    df = spark.read.format("tfrecords").option("recordType", "Example").load("/home/gmuser/va.tfrecord")
+    df.show()
 
 
 

tensnsorflow/train_multi.py

@@ -47,12 +47,11 @@ def input_fn(filenames, batch_size=32, num_epochs=1, perform_shuffle=False):
     print('Parsing', filenames)
     def _parse_fn(record):
         features = {
-            "y": tf.VarLenFeature(tf.int64),
-            "z": tf.VarLenFeature(tf.int64),
-            "ids": tf.VarLenFeature(tf.int64),
+            "y": tf.FixedLenFeature([], tf.float32),
+            "z": tf.FixedLenFeature([], tf.float32),
+            "ids": tf.FixedLenFeature([FLAGS.field_size], tf.int64),
             "level2_ids": tf.VarLenFeature(tf.int64)
         }
-
         parsed = tf.parse_single_example(record, features)
         y = parsed.pop('y')
         z = parsed.pop('z')