并行数据转换

eda/esmm/Model_pipline/train.py

@@ -81,7 +81,8 @@ def input_fn(filenames, batch_size=32, num_epochs=1, perform_shuffle=False):
     # dataset = dataset.repeat(num_epochs)
     # dataset = dataset.batch(batch_size) # Batch size to use
 
-    dataset = tf.data.TFRecordDataset(filenames).apply(tf.contrib.map_and_batch(map_func=_parse_fn, batch_size=batch_size))
+    dataset = tf.data.TFRecordDataset(filenames).apply(tf.contrib.data.map_and_batch(map_func=_parse_fn, batch_size=batch_size, num_parallel_calls=8))
+    dataset = dataset.prefetch(500000)
 
     # dataset = dataset.padded_batch(batch_size, padded_shapes=({"feeds_ids": [None], "feeds_vals": [None], "title_ids": [None]}, [None]))   #不定长补齐