How to use the joblib.dump function in joblib

To help you get started, we’ve selected a few joblib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Neuraxio / Neuraxle / testing / test_step_saving.py View on Github external
pipeline_2 = ResumablePipeline([], cache_folder=tmpdir)
    pipeline_2.name = 'pipeline2'
    pipeline_2.sub_steps_savers = [
        (SOME_STEP_2, []),
        (CHECKPOINT, []),
        (SOME_STEP_3, []),
    ]
    dump(pipeline_2, create_pipeline2_path(tmpdir, True))

    given_saved_some_step(multiply_by=2, name=SOME_STEP_1, path=create_some_step1_path(tmpdir, True))
    given_saved_some_step(multiply_by=4, name=SOME_STEP_2, path=create_some_step2_path(tmpdir, True))
    given_saved_some_step(multiply_by=6, name=SOME_STEP_3, path=create_some_step3_path(tmpdir, True))

    checkpoint = DefaultCheckpoint()
    checkpoint.name = CHECKPOINT
    dump(checkpoint, create_some_checkpoint_path(tmpdir, True))

    p = ResumablePipeline([
        (SOME_STEP_1, MultiplyByN(multiply_by=1)),
        (PIPELINE_2, ResumablePipeline([
            (SOME_STEP_2, MultiplyByN(multiply_by=1)),
            (CHECKPOINT, DefaultCheckpoint()),
            (SOME_STEP_3, MultiplyByN(multiply_by=1))
        ]))
    ], cache_folder=tmpdir)
    p.name = ROOT

    return p
github csc-training / geocomputing / machineLearning / 02_shallows / EXTRA_classification_template.py View on Github external
def trainModel(x_train, y_train, clf, classifierName):
    print("Training the model")
    start_time = time.time()
    clf.fit(x_train, y_train)
    print('Model training took: ', round((time.time() - start_time), 2), ' seconds')

    # Save the model to a file
    modelFilePath = os.path.join(base_folder, ('model_' + classifierName + '.sav'))
    dump(clf, modelFilePath)
    return clf
github arthurmensch / cogspaces / exps / train.py View on Github external
**logistic)
        callback = None

    print("Training model")
    estimator.fit(train_data, train_targets, callback=callback)

    print("Evaluating model")
    test_preds = estimator.predict(test_data)
    metrics = compute_metrics(test_preds, test_targets, target_encoder)
    print(metrics['accuracy'])

    print("Saving model")
    # Save model for further analysis
    dump(target_encoder, join(output_dir, 'target_encoder.pkl'))
    if model['normalize']:
        dump(standard_scaler, join(output_dir, 'standard_scaler.pkl'))
    dump(estimator, join(output_dir, 'estimator.pkl'))
    with open(join(output_dir, 'metrics.json'), 'w+') as f:
        json.dump(metrics, f)
    with open(join(output_dir, 'info.json'), 'w+') as f:
        json.dump(info, f)

    if config['system']['plot']:
        from utils.plotting import make_plots, prepare_plots
        print('Preparing plots')
        prepare_plots(output_dir)
        print("Plotting model")
        plot_components = config['model']['estimator'] in ['multi_study',
                                                           'ensemble']
        make_plots(output_dir, plot_classifs=True,
                   plot_components=plot_components,
                   plot_surface=False, plot_wordclouds=True,
github activityMonitoring / biobankAccelerometerAnalysis / accelerometer / accClassification.py View on Github external
:param str rfModelFile: Intermediate output random forest pickle model
    :param str hmmPriors: Intermediate output HMM priors npy
    :param str hmmEmissions: Intermediate output HMM emissions npy
    :param str hmmTransitions: Intermediate output HMM transitions npy
    :param str hmmMETs: Intermediate output HMM METs npy

    :return: tar file of RF + HMM written to tarArchive
    :rtype: void
    """

    wristListToTxtFile(featureCols, featuresTxt)
    np.save(hmmPriors, priors)
    np.save(hmmEmissions, emissions)
    np.save(hmmTransitions, transitions)
    np.save(hmmMETs, METs)
    joblib.dump(rfModel, rfModelFile, compress=9)

    # Create single .tar file...
    tarOut = tarfile.open(tarArchive, mode='w')
    tarOut.add(featuresTxt)
    tarOut.add(hmmPriors)
    tarOut.add(hmmEmissions)
    tarOut.add(hmmTransitions)
    tarOut.add(hmmMETs)
    tarOut.add(rfModelFile)
    tarOut.close()

    # Remove intermediate files
    os.remove(featuresTxt)
    os.remove(hmmPriors)
    os.remove(hmmEmissions)
    os.remove(hmmTransitions)
github CogStack / CogStack-SemEHR / trans_anns / sentence_pattern.py View on Github external
    @staticmethod
    def save_minimal_component_dictionary(mini_dic, mini_save_file):
        jl.dump(mini_dic, mini_save_file)
github alan-turing-institute / sktime / sktime / benchmarking / base.py View on Github external
def save(self):
        """Save results object as master file"""
        file = os.path.join(self.path, "results.pickle")

        # if file does not exist already, create a new one
        if not os.path.isfile(file):
            dump(self, file)

        # if file already exists, update file adding new datasets, strategies and/or cv_folds
        else:
            results = load(file)
            self.strategy_names = list(set(self.strategy_names + results.strategy_names))
            self.dataset_names = list(set(self.dataset_names + results.dataset_names))
            dump(self, file)
github clcr / pyeo / pyeo / terrain_correction.py View on Github external
def correct_reflectance(band, band_indicies, i, ic_array, ref_array, zenith_array):
    import joblib
    ic_for_linregress = ic_array.T[band_indicies[0], band_indicies[1]].ravel()
    band_for_linregress = band[band_indicies[0], band_indicies[1]].ravel()
    slope, _, _, _, _ = stats.linregress(ic_for_linregress, band_for_linregress)
    corrected_band = (band - (slope * (ic_array.T - _deg_cos(zenith_array.T))))
    joblib.dump(ic_for_linregress, f"{i}_ic")
    joblib.dump(band_for_linregress, f"{i}_band")
    return np.where(band > 0, corrected_band, ref_array[i, ...])
github arthurmensch / cogspaces / sandbox / maps_old.py View on Github external
def introspect(output_dir, baseline=False):
    introspect_dir = join(output_dir, 'maps')
    plot_dir = join(introspect_dir, 'plot')
    if not os.path.exists(plot_dir):
        os.makedirs(plot_dir)
    if baseline:
        lr1, lr3s = analyse_baseline(output_dir)
        dump((lr1, lr3s), join(introspect_dir, 'transformers.pkl'))
        (lr1, lr3s) = load(join(introspect_dir, 'transformers.pkl'))
        baseline_imgs, _ = make_level3_imgs(lr3s)
        baseline_imgs.to_filename(join(introspect_dir, 'classif.nii.gz'))
    else:
        lr1, lr2, lr3s = analyse(output_dir)
        # dump((lr1, lr2, lr3s), join(introspect_dir, 'transformers.pkl'))
        # (lr1, lr2, lr3s) = load(join(introspect_dir, 'transformers.pkl'))
        # imgs, names = make_level3_imgs(lr3s)
        # imgs.to_filename(join(introspect_dir, 'classif.nii.gz'))
        # dump(names, join(introspect_dir, 'names.pkl'))
        imgs2, snrs2 = make_level12_imgs(lr2)
        snrs2.to_filename(join(introspect_dir, 'snr.nii.gz'))
        imgs2.to_filename(join(introspect_dir, 'components.nii.gz'))
github hackalog / cookiecutter-easydata / {{ cookiecutter.repo_name }} / {{ cookiecutter.module_name }} / data / datasets.py View on Github external
'`file_base`')

        if create_dirs:
            os.makedirs(metadata_fq.parent, exist_ok=True)

        if dump_metadata:
            with open(metadata_fq, 'wb') as fo:
                joblib.dump(metadata, fo)
            logger.debug(f'Wrote Dataset Metadata: {metadata_filename}')

        if update_catalog:
            self.update_catalog(catalog_path=catalog_path, catalog_file=catalog_file)

        dataset_fq = dump_path / dataset_filename
        with open(dataset_fq, 'wb') as fo:
            joblib.dump(self, fo)
        logger.debug(f'Wrote Dataset: {dataset_filename}')