Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
pipeline_2 = ResumablePipeline([], cache_folder=tmpdir)
pipeline_2.name = 'pipeline2'
pipeline_2.sub_steps_savers = [
(SOME_STEP_2, []),
(CHECKPOINT, []),
(SOME_STEP_3, []),
]
dump(pipeline_2, create_pipeline2_path(tmpdir, True))
given_saved_some_step(multiply_by=2, name=SOME_STEP_1, path=create_some_step1_path(tmpdir, True))
given_saved_some_step(multiply_by=4, name=SOME_STEP_2, path=create_some_step2_path(tmpdir, True))
given_saved_some_step(multiply_by=6, name=SOME_STEP_3, path=create_some_step3_path(tmpdir, True))
checkpoint = DefaultCheckpoint()
checkpoint.name = CHECKPOINT
dump(checkpoint, create_some_checkpoint_path(tmpdir, True))
p = ResumablePipeline([
(SOME_STEP_1, MultiplyByN(multiply_by=1)),
(PIPELINE_2, ResumablePipeline([
(SOME_STEP_2, MultiplyByN(multiply_by=1)),
(CHECKPOINT, DefaultCheckpoint()),
(SOME_STEP_3, MultiplyByN(multiply_by=1))
]))
], cache_folder=tmpdir)
p.name = ROOT
return p
def trainModel(x_train, y_train, clf, classifierName):
print("Training the model")
start_time = time.time()
clf.fit(x_train, y_train)
print('Model training took: ', round((time.time() - start_time), 2), ' seconds')
# Save the model to a file
modelFilePath = os.path.join(base_folder, ('model_' + classifierName + '.sav'))
dump(clf, modelFilePath)
return clf
**logistic)
callback = None
print("Training model")
estimator.fit(train_data, train_targets, callback=callback)
print("Evaluating model")
test_preds = estimator.predict(test_data)
metrics = compute_metrics(test_preds, test_targets, target_encoder)
print(metrics['accuracy'])
print("Saving model")
# Save model for further analysis
dump(target_encoder, join(output_dir, 'target_encoder.pkl'))
if model['normalize']:
dump(standard_scaler, join(output_dir, 'standard_scaler.pkl'))
dump(estimator, join(output_dir, 'estimator.pkl'))
with open(join(output_dir, 'metrics.json'), 'w+') as f:
json.dump(metrics, f)
with open(join(output_dir, 'info.json'), 'w+') as f:
json.dump(info, f)
if config['system']['plot']:
from utils.plotting import make_plots, prepare_plots
print('Preparing plots')
prepare_plots(output_dir)
print("Plotting model")
plot_components = config['model']['estimator'] in ['multi_study',
'ensemble']
make_plots(output_dir, plot_classifs=True,
plot_components=plot_components,
plot_surface=False, plot_wordclouds=True,
:param str rfModelFile: Intermediate output random forest pickle model
:param str hmmPriors: Intermediate output HMM priors npy
:param str hmmEmissions: Intermediate output HMM emissions npy
:param str hmmTransitions: Intermediate output HMM transitions npy
:param str hmmMETs: Intermediate output HMM METs npy
:return: tar file of RF + HMM written to tarArchive
:rtype: void
"""
wristListToTxtFile(featureCols, featuresTxt)
np.save(hmmPriors, priors)
np.save(hmmEmissions, emissions)
np.save(hmmTransitions, transitions)
np.save(hmmMETs, METs)
joblib.dump(rfModel, rfModelFile, compress=9)
# Create single .tar file...
tarOut = tarfile.open(tarArchive, mode='w')
tarOut.add(featuresTxt)
tarOut.add(hmmPriors)
tarOut.add(hmmEmissions)
tarOut.add(hmmTransitions)
tarOut.add(hmmMETs)
tarOut.add(rfModelFile)
tarOut.close()
# Remove intermediate files
os.remove(featuresTxt)
os.remove(hmmPriors)
os.remove(hmmEmissions)
os.remove(hmmTransitions)
@staticmethod
def save_minimal_component_dictionary(mini_dic, mini_save_file):
jl.dump(mini_dic, mini_save_file)
def save(self):
"""Save results object as master file"""
file = os.path.join(self.path, "results.pickle")
# if file does not exist already, create a new one
if not os.path.isfile(file):
dump(self, file)
# if file already exists, update file adding new datasets, strategies and/or cv_folds
else:
results = load(file)
self.strategy_names = list(set(self.strategy_names + results.strategy_names))
self.dataset_names = list(set(self.dataset_names + results.dataset_names))
dump(self, file)
def correct_reflectance(band, band_indicies, i, ic_array, ref_array, zenith_array):
import joblib
ic_for_linregress = ic_array.T[band_indicies[0], band_indicies[1]].ravel()
band_for_linregress = band[band_indicies[0], band_indicies[1]].ravel()
slope, _, _, _, _ = stats.linregress(ic_for_linregress, band_for_linregress)
corrected_band = (band - (slope * (ic_array.T - _deg_cos(zenith_array.T))))
joblib.dump(ic_for_linregress, f"{i}_ic")
joblib.dump(band_for_linregress, f"{i}_band")
return np.where(band > 0, corrected_band, ref_array[i, ...])
def introspect(output_dir, baseline=False):
introspect_dir = join(output_dir, 'maps')
plot_dir = join(introspect_dir, 'plot')
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
if baseline:
lr1, lr3s = analyse_baseline(output_dir)
dump((lr1, lr3s), join(introspect_dir, 'transformers.pkl'))
(lr1, lr3s) = load(join(introspect_dir, 'transformers.pkl'))
baseline_imgs, _ = make_level3_imgs(lr3s)
baseline_imgs.to_filename(join(introspect_dir, 'classif.nii.gz'))
else:
lr1, lr2, lr3s = analyse(output_dir)
# dump((lr1, lr2, lr3s), join(introspect_dir, 'transformers.pkl'))
# (lr1, lr2, lr3s) = load(join(introspect_dir, 'transformers.pkl'))
# imgs, names = make_level3_imgs(lr3s)
# imgs.to_filename(join(introspect_dir, 'classif.nii.gz'))
# dump(names, join(introspect_dir, 'names.pkl'))
imgs2, snrs2 = make_level12_imgs(lr2)
snrs2.to_filename(join(introspect_dir, 'snr.nii.gz'))
imgs2.to_filename(join(introspect_dir, 'components.nii.gz'))
'`file_base`')
if create_dirs:
os.makedirs(metadata_fq.parent, exist_ok=True)
if dump_metadata:
with open(metadata_fq, 'wb') as fo:
joblib.dump(metadata, fo)
logger.debug(f'Wrote Dataset Metadata: {metadata_filename}')
if update_catalog:
self.update_catalog(catalog_path=catalog_path, catalog_file=catalog_file)
dataset_fq = dump_path / dataset_filename
with open(dataset_fq, 'wb') as fo:
joblib.dump(self, fo)
logger.debug(f'Wrote Dataset: {dataset_filename}')