Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Dataset then), that needs to be handled. Otherwise, it is
# simply written as is to a Dataset. As HDF5 Reference types do
# look like a structured object array, those have to be excluded
# explicitly. Complex types may have been converted so that they
# can have different field names as an HDF5 COMPOUND type, so
# those have to be excluded too. Also, if any of its fields are
# an object time (no matter how nested), then rather than
# converting that field to a HDF5 Reference types, it will just
# be written as a Group instead (dtypes have a useful hasobject
# method). The same goes for if there is a null inside its
# dtype.
#
# A flag, wrote_as_struct, is set depending on which path is
# taken, which is then passed onto write_metadata.
if data_to_store.dtype.fields is not None \
and h5py.check_dtype(ref=data_to_store.dtype) \
is not h5py.Reference \
and not np.iscomplexobj(data) \
and (options.structured_numpy_ndarray_as_struct \
or (data_to_store.dtype.hasobject \
or '\\x00' in str(data_to_store.dtype)) \
or does_dtype_have_a_zero_shape(data_to_store.dtype)):
wrote_as_struct = True
# Grab the list of fields and properly escape them.
field_names = [n for n in data_to_store.dtype.names]
escaped_field_names = [escape_path(n) for n in field_names]
# If the group doesn't exist, it needs to be created. If it
# already exists but is not a group, it needs to be deleted
# before being created.
try:
def dump_dtype(dt):
if not isinstance(dt, np.dtype):
raise TypeError("expected np.dtype, but got: {}".format(type(dt)))
if len(dt) > 0:
out = "{"
for name in dt.fields:
subdt = dt.fields[name][0]
out += "{}: {} |".format(name, dump_dtype(subdt))
out = out[:-1] + "}"
else:
ref = h5py.check_dtype(ref=dt)
if ref:
out = str(ref)
else:
vlen = h5py.check_dtype(vlen=dt)
if vlen:
out = "VLEN: " + dump_dtype(vlen)
else:
out = str(dt)
return out
"""
records_grp = self.h5['runs/{}/{}'.format(run_idx, run_record_key)]
field = records_grp[field_name]
# make sure this is a feature vector
assert len(field_data.shape) > 1, \
"field_data must be a feature vector with the same number of dimensions as the number"
# of datase new frames
n_new_frames = field_data.shape[0]
# check whether it is a variable length record, by getting the
# record dataset dtype and using the checker to see if it is
# the vlen special type in h5py
if h5py.check_dtype(vlen=field.dtype) is not None:
# if it is we have to treat it differently, since it
# cannot be multidimensional
# if the dataset has no data in it we need to reshape it
if all([i == 0 for i in field.shape]):
# initialize this array
# if it is empty resize it to make an array the size of
# the new field_data with the maxshape for the feature
# dimensions
field.resize( (n_new_frames,) )
# set the new data to this
for i, row in enumerate(field_data):
field[i] = row
def _convert_record_field_to_table_column(self, run_idx, run_record_key, record_field):
# get the field dataset
rec_grp = self.records_grp(run_idx, run_record_key)
dset = rec_grp[record_field]
# if it is variable length or if it has more than one element
# cast all elements to tuples
if h5py.check_dtype(vlen=dset.dtype) is not None:
rec_dset = [tuple(value) for value in dset[:]]
# if it is not variable length make sure it is not more than a
# 1D feature vector
elif len(dset.shape) > 2:
raise TypeError(
"cannot convert fields with feature vectors more than 1 dimension,"
" was given {} for {}/{}".format(
dset.shape[1:], run_record_key, record_field))
# if it is only a rank 1 feature vector and it has more than
# one element make a tuple out of it
elif dset.shape[1] > 1:
rec_dset = [tuple(value) for value in dset[:]]
# otherwise just get the single value instead of keeping it as
def _write_dataset(dataset, column, entry_dset):
assert column.shape[1] == 1
for i, name in enumerate(entry_dset):
element = column.loc[name][0]
if not h5py.check_dtype(vlen=dataset.dtype):
dataset[i] = element
# Variable length datatypes require flattening of the array and special handling of missing values
else:
try:
dataset[i] = element.ravel()
except AttributeError:
if np.isnan(element):
pass
else:
raise
ST_ESTIMATED = 113
ST_REPORTED = 114
ST_VERIFIED = 115
_dtstate = h5py.special_dtype(enum=('i', {
"Invalid": ST_INVALID, "Default": ST_DEFAULT, "Estimated": ST_ESTIMATED,
"Reported": ST_REPORTED, "Verified": ST_VERIFIED}))
# MicroscopeMode
MM_NONE = 0
MM_TRANSMISSION = 1
MM_REFLECTION = 2
MM_FLUORESCENCE = 3
_dtmm = h5py.special_dtype(enum=('i', {
"None": MM_NONE, "Transmission": MM_TRANSMISSION ,
"Reflection": MM_REFLECTION, "Fluorescence": MM_FLUORESCENCE}))
_dictmm = h5py.check_dtype(enum=_dtmm)
# MicroscopeType
MT_NONE = 111
MT_WIDEFIELD = 112
MT_CONFOCAL = 113
MT_4PIEXCITATION = 114
MT_NIPKOWDISKCONFOCAL = 115
MT_GENERICSENSOR = 118
_dtmt = h5py.special_dtype(enum=('i', {
"None": MT_NONE, "WideField": MT_WIDEFIELD, "Confocal": MT_CONFOCAL,
"4PiExcitation": MT_4PIEXCITATION, "NipkowDiskConfocal": MT_NIPKOWDISKCONFOCAL,
"GenericSensor": MT_GENERICSENSOR}))
_dictmt = h5py.check_dtype(enum=_dtmt)
# ImagingDirection
ID_UPWARD = 0
ID_DOWNWARD = 1
ID_BOTH = 2
kwargs['data'] = ReferenceBuilder(target_builder)
else:
kwargs["data"] = scalar
elif ndims == 1:
d = None
if h5obj.dtype.kind == 'O': # read list of strings or list of references
elem1 = h5obj[0]
if isinstance(elem1, (text_type, binary_type)):
d = h5obj[()]
elif isinstance(elem1, RegionReference):
d = H5RegionDataset(h5obj, self)
elif isinstance(elem1, Reference):
d = H5ReferenceDataset(h5obj, self)
elif h5obj.dtype.kind == 'V': # table
cpd_dt = h5obj.dtype
ref_cols = [check_dtype(ref=cpd_dt[i]) for i in range(len(cpd_dt))]
d = H5TableDataset(h5obj, self, ref_cols)
else:
d = h5obj
kwargs["data"] = d
else:
kwargs["data"] = h5obj
ret = DatasetBuilder(name, **kwargs)
ret.written = True
return ret