How to use the joblib.disk.mkdirp function in joblib

To help you get started, we’ve selected a few joblib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github chrodan / tdlearn / joblib / memory.py View on Github external
def clear(self, warn=True):
        """ Empty the function's cache.
        """
        func_dir = self._get_func_dir(mkdir=False)
        if self._verbose and warn:
            self.warn("Clearing cache %s" % func_dir)
        if os.path.exists(func_dir):
            shutil.rmtree(func_dir, ignore_errors=True)
        mkdirp(func_dir)
        func_code, _, first_line = get_func_code(self.func)
        func_code_file = os.path.join(func_dir, 'func_code.py')
        self._write_func_code(func_code_file, func_code, first_line)
github chrodan / tdlearn / joblib / memory.py View on Github external
def _persist_output(self, output, dir):
        """ Persist the given output tuple in the directory.
        """
        try:
            mkdirp(dir)
            filename = os.path.join(dir, 'output.pkl')
            numpy_pickle.dump(output, filename, compress=self.compress)
            if self._verbose > 10:
                print 'Persisting in %s' % dir
        except OSError:
            " Race condition in the creation of the directory "
github chrodan / tdlearn / joblib / memory.py View on Github external
def _get_func_dir(self, mkdir=True):
        """ Get the directory corresponding to the cache for the
            function.
        """
        module, name = get_func_name(self.func)
        module.append(name)
        func_dir = os.path.join(self.cachedir, *module)
        if mkdir:
            mkdirp(func_dir)
        return func_dir
github chrodan / tdlearn / joblib / memory.py View on Github external
as functions are revaluated.
        """
        # XXX: Bad explaination of the None value of cachedir
        Logger.__init__(self)
        self._verbose = verbose
        self.mmap_mode = mmap_mode
        self.timestamp = time.time()
        self.compress = compress
        if compress and mmap_mode is not None:
            warnings.warn('Compressed results cannot be memmapped',
                          stacklevel=2)
        if cachedir is None:
            self.cachedir = None
        else:
            self.cachedir = os.path.join(cachedir, 'joblib')
            mkdirp(self.cachedir)
github joblib / joblib / joblib / logger.py View on Github external
def __init__(self, logfile=None, logdir=None):
        if logfile is not None and logdir is not None:
            raise ValueError('Cannot specify both logfile and logdir')
        # XXX: Need argument docstring
        self.last_time = time.time()
        self.start_time = self.last_time
        if logdir is not None:
            logfile = os.path.join(logdir, 'joblib.log')
        self.logfile = logfile
        if logfile is not None:
            mkdirp(os.path.dirname(logfile))
            if os.path.exists(logfile):
                # Rotate the logs
                for i in range(1, 9):
                    try:
                        shutil.move(logfile + '.%i' % i,
                                    logfile + '.%i' % (i + 1))
                    except:
                        "No reason failing here"
                # Use a copy rather than a move, so that a process
                # monitoring this file does not get lost.
                try:
                    shutil.copy(logfile, logfile + '.1')
                except:
                    "No reason failing here"
            try:
                with open(logfile, 'w') as logfile:
github joblib / joblib / joblib / _store_backends.py View on Github external
def configure(self, location, verbose=1, backend_options=None):
        """Configure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        """
        if backend_options is None:
            backend_options = {}

        # setup location directory
        self.location = location
        if not os.path.exists(self.location):
            mkdirp(self.location)

        # item can be stored compressed for faster I/O
        self.compress = backend_options.get('compress', False)

        # FileSystemStoreBackend can be used with mmap_mode options under
        # certain conditions.
        mmap_mode = backend_options.get('mmap_mode')
        if self.compress and mmap_mode is not None:
            warnings.warn('Compressed items cannot be memmapped in a '
                          'filesystem store. Option will be ignored.',
                          stacklevel=2)

        self.mmap_mode = mmap_mode
        self.verbose = verbose
github bmabey / provenance / examples / sftp / sftp_example.py View on Github external
#!/usr/bin/env python

from joblib.disk import mkdirp

import provenance as p

mkdirp('./remote-machine/sftp-artifacts')

p.load_config(
    {
        'blobstores':
            {
                'sftp':
                    {
                        'type':
                            'sftp',
                        'cachedir':
                            '',
                        'basepath':
                            '<            ""           remote machine                ""                       , ex. /home/me/artifacts>, you need to make sure that path directory exists.',
                        'read':
                            True,
                        'write':