How to use the joblib.numpy_pickle.load function in joblib

To help you get started, we’ve selected a few joblib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github joblib / joblib / joblib / _store_backends.py View on Github external
print('{0} from {1}'.format(msg, full_path))

        mmap_mode = (None if not hasattr(self, 'mmap_mode')
                     else self.mmap_mode)

        filename = os.path.join(full_path, 'output.pkl')
        if not self._item_exists(filename):
            raise KeyError("Non-existing item (may have been "
                           "cleared).\nFile %s does not exist" % filename)

        # file-like object cannot be used when mmap_mode is set
        if mmap_mode is None:
            with self._open_item(filename, "rb") as f:
                item = numpy_pickle.load(f)
        else:
            item = numpy_pickle.load(filename, mmap_mode=mmap_mode)
        return item
github chrodan / tdlearn / joblib / memory.py View on Github external
"""
        if self._verbose > 1:
            t = time.time() - self.timestamp
            if self._verbose < 10:
                print '[Memory]% 16s: Loading %s...' % (
                                    format_time(t),
                                    self.format_signature(self.func)[0]
                                    )
            else:
                print '[Memory]% 16s: Loading %s from %s' % (
                                    format_time(t),
                                    self.format_signature(self.func)[0],
                                    output_dir
                                    )
        filename = os.path.join(output_dir, 'output.pkl')
        return numpy_pickle.load(filename,
                                 mmap_mode=self.mmap_mode)
github joblib / joblib / joblib / _store_backends.py View on Github external
print('{0}...'.format(msg))
            else:
                print('{0} from {1}'.format(msg, full_path))

        mmap_mode = (None if not hasattr(self, 'mmap_mode')
                     else self.mmap_mode)

        filename = os.path.join(full_path, 'output.pkl')
        if not self._item_exists(filename):
            raise KeyError("Non-existing item (may have been "
                           "cleared).\nFile %s does not exist" % filename)

        # file-like object cannot be used when mmap_mode is set
        if mmap_mode is None:
            with self._open_item(filename, "rb") as f:
                item = numpy_pickle.load(f)
        else:
            item = numpy_pickle.load(filename, mmap_mode=mmap_mode)
        return item
github dnouri / nolearn / nolearn / cache.py View on Github external
return func(*args, **kwargs)

            hashed_key = hashlib.sha1(key).hexdigest()[:8]

            # We construct the filename using the cache key.  If the
            # file exists, unpickle and return the value.
            filename = os.path.join(
                cache_path or CACHE_PATH,
                '{}.{}-cache-{}'.format(
                    func.__module__, func.__name__, hashed_key))

            if os.path.exists(filename):
                filesize = os.path.getsize(filename)
                size = "%0.1f MB" % (filesize / (1024 * 1024.0))
                logger.debug(" * cache hit: {} ({})".format(filename, size))
                return numpy_pickle.load(filename)
            else:
                logger.debug(" * cache miss: {}".format(filename))
                value = func(*args, **kwargs)
                tmp_filename = '{}-{}.tmp'.format(
                    filename,
                    ''.join(random.sample(string.ascii_letters, 4)),
                    )
                try:
                    numpy_pickle.dump(value, tmp_filename, compress=9)
                    os.rename(tmp_filename, filename)
                except Exception:
                    logger.exception(
                        "Saving pickle {} resulted in Exception".format(
                        filename))
                return value
github joblib / joblib / joblib / _memmapping_reducer.py View on Github external
# possible to delete temporary files as soon as the workers are
            # done processing this data.
            if not os.path.exists(filename):
                if self.verbose > 0:
                    print("Memmapping (shape={}, dtype={}) to new file {}"
                          .format(a.shape, a.dtype, filename))
                for dumped_filename in dump(a, filename):
                    os.chmod(dumped_filename, FILE_PERMISSIONS)

                if self._prewarm:
                    # Warm up the data by accessing it. This operation ensures
                    # that the disk access required to create the memmapping
                    # file are performed in the reducing process and avoids
                    # concurrent memmap creation in multiple children
                    # processes.
                    load(filename, mmap_mode=self._mmap_mode).max()
            elif self.verbose > 1:
                print("Memmapping (shape={}, dtype={}) to old file {}"
                      .format(a.shape, a.dtype, filename))

            # The worker process will use joblib.load to memmap the data
            return (load, (filename, self._mmap_mode))
        else:
            # do not convert a into memmap, let pickler do its usual copy with
            # the default system pickler
            if self.verbose > 1:
                print("Pickling array (shape={}, dtype={})."
                      .format(a.shape, a.dtype))
            return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),))
github joblib / joblib / joblib / _memmapping_reducer.py View on Github external
for dumped_filename in dump(a, filename):
                    os.chmod(dumped_filename, FILE_PERMISSIONS)

                if self._prewarm:
                    # Warm up the data by accessing it. This operation ensures
                    # that the disk access required to create the memmapping
                    # file are performed in the reducing process and avoids
                    # concurrent memmap creation in multiple children
                    # processes.
                    load(filename, mmap_mode=self._mmap_mode).max()
            elif self.verbose > 1:
                print("Memmapping (shape={}, dtype={}) to old file {}"
                      .format(a.shape, a.dtype, filename))

            # The worker process will use joblib.load to memmap the data
            return (load, (filename, self._mmap_mode))
        else:
            # do not convert a into memmap, let pickler do its usual copy with
            # the default system pickler
            if self.verbose > 1:
                print("Pickling array (shape={}, dtype={})."
                      .format(a.shape, a.dtype))
            return (loads, (dumps(a, protocol=HIGHEST_PROTOCOL),))