How to use the datasets.dataset_utils function in datasets

To help you get started, we’ve selected a few datasets examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github wenwei202 / terngrad / slim / datasets / download_and_convert_mnist.py View on Github external
# First, process the training data:
  with tf.python_io.TFRecordWriter(training_filename) as tfrecord_writer:
    data_filename = os.path.join(dataset_dir, _TRAIN_DATA_FILENAME)
    labels_filename = os.path.join(dataset_dir, _TRAIN_LABELS_FILENAME)
    _add_to_tfrecord(data_filename, labels_filename, 60000, tfrecord_writer)

  # Next, process the testing data:
  with tf.python_io.TFRecordWriter(testing_filename) as tfrecord_writer:
    data_filename = os.path.join(dataset_dir, _TEST_DATA_FILENAME)
    labels_filename = os.path.join(dataset_dir, _TEST_LABELS_FILENAME)
    _add_to_tfrecord(data_filename, labels_filename, 10000, tfrecord_writer)

  # Finally, write the labels file:
  labels_to_class_names = dict(zip(range(len(_CLASS_NAMES)), _CLASS_NAMES))
  dataset_utils.write_label_file(labels_to_class_names, dataset_dir)

  _clean_up_temporary_files(dataset_dir)
  print('\nFinished converting the MNIST dataset!')
github wenwei202 / terngrad / slim / datasets / download_convert_and_shard_cifar10.py View on Github external
filename, offset + 1))
          sys.stdout.flush()

          if ('train' == name) and ( math.floor(offset / images_per_shard) > shard) :
            tfrecord_writer.close()
            shard = shard + 1
            record_filename = _get_output_filename(dataset_dir, name, shard, FLAGS.train_shards)
            tfrecord_writer = tf.python_io.TFRecordWriter(record_filename)

          image = np.squeeze(images[j]).transpose((1, 2, 0))
          label = labels[j]

          png_string = sess.run(encoded_image,
                                feed_dict={image_placeholder: image})

          example = dataset_utils.image_to_tfexample(
              png_string, 'png', _IMAGE_SIZE, _IMAGE_SIZE, label, _CLASS_NAMES[label])
          tfrecord_writer.write(example.SerializeToString())
          offset = offset + 1

  tfrecord_writer.close()
  return offset
github charliememory / Pose-Guided-Person-Image-Generation / datasets / market1501.py View on Github external
'pose_peaks_1': slim.tfexample_decoder.Tensor('pose_peaks_1',shape=[16*8*18]),
      'pose_mask_r4_0': slim.tfexample_decoder.Tensor('pose_mask_r4_0',shape=[128*64*1]),
      'pose_mask_r4_1': slim.tfexample_decoder.Tensor('pose_mask_r4_1',shape=[128*64*1]),

      'pose_sparse_r4_0': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_0', values_key='values_r4_0', shape_key='shape', densify=False),
      'pose_sparse_r4_1': slim.tfexample_decoder.SparseTensor(indices_key='indices_r4_1', values_key='values_r4_1', shape_key='shape', densify=False),
      
      'pose_subs_0': slim.tfexample_decoder.Tensor('pose_subs_0',shape=[20]),
      'pose_subs_1': slim.tfexample_decoder.Tensor('pose_subs_1',shape=[20]),
  }

  decoder = slim.tfexample_decoder.TFExampleDecoder(
      keys_to_features, items_to_handlers)

  labels_to_names = None
  if dataset_utils.has_labels(dataset_dir):
    labels_to_names = dataset_utils.read_label_file(dataset_dir)

  print('load pn_pairs_num ......')
  fpath = os.path.join(dataset_dir, 'pn_pairs_num_'+split_name+'.p')
  with open(fpath,'r') as f:
    pn_pairs_num = pickle.load(f)

  return slim.dataset.Dataset(
      data_sources=file_pattern,
      reader=reader,
      decoder=decoder,
      num_samples=pn_pairs_num,
      items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
      num_classes=_NUM_CLASSES,
      labels_to_names=labels_to_names)
github wenwei202 / terngrad / datasets / imagenet.py View on Github external
}

  items_to_handlers = {
      'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
      'label': slim.tfexample_decoder.Tensor('image/class/label'),
      'label_text': slim.tfexample_decoder.Tensor('image/class/text'),
      'object/bbox': slim.tfexample_decoder.BoundingBox(
          ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
      'object/label': slim.tfexample_decoder.Tensor('image/object/class/label'),
  }

  decoder = slim.tfexample_decoder.TFExampleDecoder(
      keys_to_features, items_to_handlers)

  labels_to_names = None
  if dataset_utils.has_labels(dataset_dir):
    labels_to_names = dataset_utils.read_label_file(dataset_dir)
  else:
    labels_to_names = create_readable_names_for_imagenet_labels()
    dataset_utils.write_label_file(labels_to_names, dataset_dir)

  return slim.dataset.Dataset(
      data_sources=file_pattern,
      reader=reader,
      decoder=decoder,
      num_samples=_SPLITS_TO_SIZES[split_name],
      items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
      num_classes=_NUM_CLASSES,
      labels_to_names=labels_to_names)