Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
print('{0} from {1}'.format(msg, full_path))
mmap_mode = (None if not hasattr(self, 'mmap_mode')
else self.mmap_mode)
filename = os.path.join(full_path, 'output.pkl')
if not self._item_exists(filename):
raise KeyError("Non-existing item (may have been "
"cleared).\nFile %s does not exist" % filename)
# file-like object cannot be used when mmap_mode is set
if mmap_mode is None:
with self._open_item(filename, "rb") as f:
item = numpy_pickle.load(f)
else:
item = numpy_pickle.load(filename, mmap_mode=mmap_mode)
return item
"""
if self._verbose > 1:
t = time.time() - self.timestamp
if self._verbose < 10:
print '[Memory]% 16s: Loading %s...' % (
format_time(t),
self.format_signature(self.func)[0]
)
else:
print '[Memory]% 16s: Loading %s from %s' % (
format_time(t),
self.format_signature(self.func)[0],
output_dir
)
filename = os.path.join(output_dir, 'output.pkl')
return numpy_pickle.load(filename,
mmap_mode=self.mmap_mode)
print('{0}...'.format(msg))
else:
print('{0} from {1}'.format(msg, full_path))
mmap_mode = (None if not hasattr(self, 'mmap_mode')
else self.mmap_mode)
filename = os.path.join(full_path, 'output.pkl')
if not self._item_exists(filename):
raise KeyError("Non-existing item (may have been "
"cleared).\nFile %s does not exist" % filename)
# file-like object cannot be used when mmap_mode is set
if mmap_mode is None:
with self._open_item(filename, "rb") as f:
item = numpy_pickle.load(f)
else:
item = numpy_pickle.load(filename, mmap_mode=mmap_mode)
return item
return func(*args, **kwargs)
hashed_key = hashlib.sha1(key).hexdigest()[:8]
# We construct the filename using the cache key. If the
# file exists, unpickle and return the value.
filename = os.path.join(
cache_path or CACHE_PATH,
'{}.{}-cache-{}'.format(
func.__module__, func.__name__, hashed_key))
if os.path.exists(filename):
filesize = os.path.getsize(filename)
size = "%0.1f MB" % (filesize / (1024 * 1024.0))
logger.debug(" * cache hit: {} ({})".format(filename, size))
return numpy_pickle.load(filename)
else:
logger.debug(" * cache miss: {}".format(filename))
value = func(*args, **kwargs)
tmp_filename = '{}-{}.tmp'.format(
filename,
''.join(random.sample(string.ascii_letters, 4)),
)
try:
numpy_pickle.dump(value, tmp_filename, compress=9)
os.rename(tmp_filename, filename)
except Exception:
logger.exception(
"Saving pickle {} resulted in Exception".format(
filename))
return value
# possible to delete temporary files as soon as the workers are
# done processing this data.
if not os.path.exists(filename):
if self.verbose > 0:
print("Memmapping (shape={}, dtype={}) to new file {}"
.format(a.shape, a.dtype, filename))
for dumped_filename in dump(a, filename):
os.chmod(dumped_filename, FILE_PERMISSIONS)
if self._prewarm:
# Warm up the data by accessing it. This operation ensures
# that the disk access required to create the memmapping
# file are performed in the reducing process and avoids
# concurrent memmap creation in multiple children
# processes.
load(filename, mmap_mode=self._mmap_mode).max()
elif self.verbose > 1:
print("Memmapping (shape={}, dtype={}) to old file {}"
.format(a.shape, a.dtype, filename))
# The worker process will use joblib.load to memmap the data
return (load, (filename, self._mmap_mode))
else:
# do not convert a into memmap, let pickler do its usual copy with
# the default system pickler
if self.verbose > 1:
print("Pickling array (shape={}, dtype={})."
.format(a.shape, a.dtype))
return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),))
for dumped_filename in dump(a, filename):
os.chmod(dumped_filename, FILE_PERMISSIONS)
if self._prewarm:
# Warm up the data by accessing it. This operation ensures
# that the disk access required to create the memmapping
# file are performed in the reducing process and avoids
# concurrent memmap creation in multiple children
# processes.
load(filename, mmap_mode=self._mmap_mode).max()
elif self.verbose > 1:
print("Memmapping (shape={}, dtype={}) to old file {}"
.format(a.shape, a.dtype, filename))
# The worker process will use joblib.load to memmap the data
return (load, (filename, self._mmap_mode))
else:
# do not convert a into memmap, let pickler do its usual copy with
# the default system pickler
if self.verbose > 1:
print("Pickling array (shape={}, dtype={})."
.format(a.shape, a.dtype))
return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),))