How to use the h5py.check_dtype function in h5py

To help you get started, we’ve selected a few h5py examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github frejanordsiek / hdf5storage / hdf5storage / Marshallers.py View on Github external
# Dataset then), that needs to be handled. Otherwise, it is
        # simply written as is to a Dataset. As HDF5 Reference types do
        # look like a structured object array, those have to be excluded
        # explicitly. Complex types may have been converted so that they
        # can have different field names as an HDF5 COMPOUND type, so
        # those have to be excluded too. Also, if any of its fields are
        # an object time (no matter how nested), then rather than
        # converting that field to a HDF5 Reference types, it will just
        # be written as a Group instead (dtypes have a useful hasobject
        # method). The same goes for if there is a null inside its
        # dtype.
        #
        # A flag, wrote_as_struct, is set depending on which path is
        # taken, which is then passed onto write_metadata.
        if data_to_store.dtype.fields is not None \
                and h5py.check_dtype(ref=data_to_store.dtype) \
                is not h5py.Reference \
                and not np.iscomplexobj(data) \
                and (options.structured_numpy_ndarray_as_struct \
                or (data_to_store.dtype.hasobject \
                or '\\x00' in str(data_to_store.dtype)) \
                or does_dtype_have_a_zero_shape(data_to_store.dtype)):
            wrote_as_struct = True
            # Grab the list of fields and properly escape them.
            field_names = [n for n in data_to_store.dtype.names]
            escaped_field_names = [escape_path(n) for n in field_names]

            # If the group doesn't exist, it needs to be created. If it
            # already exists but is not a group, it needs to be deleted
            # before being created.

            try:
github HDFGroup / h5pyd / h5pyd / _apps / utillib.py View on Github external
def dump_dtype(dt):
    if not isinstance(dt, np.dtype):
        raise TypeError("expected np.dtype, but got: {}".format(type(dt)))
    if len(dt) > 0:
        out = "{"
        for name in dt.fields:
            subdt = dt.fields[name][0]
            out += "{}: {} |".format(name, dump_dtype(subdt))
        out = out[:-1] + "}"
    else:
        ref = h5py.check_dtype(ref=dt)
        if ref:
            out = str(ref)
        else:
            vlen = h5py.check_dtype(vlen=dt)
            if vlen:
                out = "VLEN: " + dump_dtype(vlen)
            else:
                out = str(dt)
    return out
github ADicksonLab / wepy / wepy / hdf5.py View on Github external
"""

        records_grp = self.h5['runs/{}/{}'.format(run_idx, run_record_key)]
        field = records_grp[field_name]

        # make sure this is a feature vector
        assert len(field_data.shape) > 1, \
            "field_data must be a feature vector with the same number of dimensions as the number"

        # of datase new frames
        n_new_frames = field_data.shape[0]

        # check whether it is a variable length record, by getting the
        # record dataset dtype and using the checker to see if it is
        # the vlen special type in h5py
        if h5py.check_dtype(vlen=field.dtype) is not None:

            # if it is we have to treat it differently, since it
            # cannot be multidimensional

            # if the dataset has no data in it we need to reshape it
            if all([i == 0 for i in field.shape]):
                # initialize this array
                # if it is empty resize it to make an array the size of
                # the new field_data with the maxshape for the feature
                # dimensions
                field.resize( (n_new_frames,) )

                # set the new data to this
                for i, row in enumerate(field_data):
                    field[i] = row
github ADicksonLab / wepy / wepy / hdf5.py View on Github external
def _convert_record_field_to_table_column(self, run_idx, run_record_key, record_field):

        # get the field dataset
        rec_grp = self.records_grp(run_idx, run_record_key)
        dset = rec_grp[record_field]

        # if it is variable length or if it has more than one element
        # cast all elements to tuples
        if h5py.check_dtype(vlen=dset.dtype) is not None:
            rec_dset = [tuple(value) for value in dset[:]]

        # if it is not variable length make sure it is not more than a
        # 1D feature vector
        elif len(dset.shape) > 2:
            raise TypeError(
                "cannot convert fields with feature vectors more than 1 dimension,"
                " was given {} for {}/{}".format(
                    dset.shape[1:], run_record_key, record_field))

        # if it is only a rank 1 feature vector and it has more than
        # one element make a tuple out of it
        elif dset.shape[1] > 1:
            rec_dset = [tuple(value) for value in dset[:]]

        # otherwise just get the single value instead of keeping it as
github MolSSI / QCPortal / collections / dataset_view.py View on Github external
def _write_dataset(dataset, column, entry_dset):
            assert column.shape[1] == 1
            for i, name in enumerate(entry_dset):
                element = column.loc[name][0]
                if not h5py.check_dtype(vlen=dataset.dtype):
                    dataset[i] = element
                # Variable length datatypes require flattening of the array and special handling of missing values
                else:
                    try:
                        dataset[i] = element.ravel()
                    except AttributeError:
                        if np.isnan(element):
                            pass
                        else:
                            raise
github delmic / odemis / src / odemis / dataio / hdf5.py View on Github external
ST_ESTIMATED = 113
ST_REPORTED = 114
ST_VERIFIED = 115
_dtstate = h5py.special_dtype(enum=('i', {
     "Invalid": ST_INVALID, "Default": ST_DEFAULT, "Estimated": ST_ESTIMATED,
     "Reported": ST_REPORTED, "Verified": ST_VERIFIED}))

# MicroscopeMode
MM_NONE = 0
MM_TRANSMISSION = 1
MM_REFLECTION = 2
MM_FLUORESCENCE = 3
_dtmm = h5py.special_dtype(enum=('i', {
     "None": MM_NONE, "Transmission": MM_TRANSMISSION ,
     "Reflection": MM_REFLECTION, "Fluorescence": MM_FLUORESCENCE}))
_dictmm = h5py.check_dtype(enum=_dtmm)
# MicroscopeType
MT_NONE = 111
MT_WIDEFIELD = 112
MT_CONFOCAL = 113
MT_4PIEXCITATION = 114
MT_NIPKOWDISKCONFOCAL = 115
MT_GENERICSENSOR = 118
_dtmt = h5py.special_dtype(enum=('i', {
    "None": MT_NONE, "WideField": MT_WIDEFIELD, "Confocal": MT_CONFOCAL,
    "4PiExcitation": MT_4PIEXCITATION, "NipkowDiskConfocal": MT_NIPKOWDISKCONFOCAL,
    "GenericSensor": MT_GENERICSENSOR}))
_dictmt = h5py.check_dtype(enum=_dtmt)
# ImagingDirection
ID_UPWARD = 0
ID_DOWNWARD = 1
ID_BOTH = 2
github NeurodataWithoutBorders / pynwb / src / pynwb / form / backends / hdf5 / h5tools.py View on Github external
kwargs['data'] = ReferenceBuilder(target_builder)
            else:
                kwargs["data"] = scalar
        elif ndims == 1:
            d = None
            if h5obj.dtype.kind == 'O':    # read list of strings or list of references
                elem1 = h5obj[0]
                if isinstance(elem1, (text_type, binary_type)):
                    d = h5obj[()]
                elif isinstance(elem1, RegionReference):
                    d = H5RegionDataset(h5obj, self)
                elif isinstance(elem1, Reference):
                    d = H5ReferenceDataset(h5obj, self)
            elif h5obj.dtype.kind == 'V':    # table
                cpd_dt = h5obj.dtype
                ref_cols = [check_dtype(ref=cpd_dt[i]) for i in range(len(cpd_dt))]
                d = H5TableDataset(h5obj, self, ref_cols)
            else:
                d = h5obj
            kwargs["data"] = d
        else:
            kwargs["data"] = h5obj
        ret = DatasetBuilder(name, **kwargs)
        ret.written = True
        return ret