Split .tfrecords file into many .tfrecords files
Is there any way to split .tfrecords file into many .tfrecords files directly, without writing back each Dataset example ?
In tensorflow 2.0.0, this will work:
import tensorflow as tf
raw_dataset = tf.data.TFRecordDataset("input_file.tfrecord")
shards = 10
for i in range(shards):
writer = tf.data.experimental.TFRecordWriter(f"output_file-part-{i}.tfrecord")
writer.write(raw_dataset.shard(shards, i))
You can use a function like this:
import tensorflow as tf
def split_tfrecord(tfrecord_path, split_size):
with tf.Graph().as_default(), tf.Session() as sess:
ds = tf.data.TFRecordDataset(tfrecord_path).batch(split_size)
batch = ds.make_one_shot_iterator().get_next()
part_num = 0
while True:
try:
records = sess.run(batch)
part_path = tfrecord_path + '.{:03d}'.format(part_num)
with tf.python_io.TFRecordWriter(part_path) as writer:
for record in records:
writer.write(record)
part_num += 1
except tf.errors.OutOfRangeError: break
For example, to split the file my_records.tfrecord
into parts of 100 records each, you would do:
split_tfrecord(my_records.tfrecord, 100)
This would create multiple smaller record files my_records.tfrecord.000
, my_records.tfrecord.001
, etc.