Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
@property
def state(self):
"""Get the frozen (pickled) model state."""
return self._state(exclude=['last_model'])
@classmethod
def is_test_class(cls, other_cls):
"""Return whether `other_cls` is a subclass of this test class."""
return inspect.isclass(other_cls) and issubclass(other_cls, cls)
def __str__(self):
"""Return the string representation of the test's name."""
return '%s' % self.name
class TestM2M(Test):
"""Abstract class for handling tests involving multiple models.
Enables comparison of model to model predictions, and also against
experimental reference data (optional).
Note: 'TestM2M' would typically be used when handling mutliple (>2)
models, with/without experimental reference data. For single model
tests, you can use the 'Test' class.
"""
def __init__(self, observation=None, name=None, **params):
super(TestM2M, self).__init__(observation, name=name, **params)
def validate_observation(self, observation):
"""Validate the observation provided to the constructor.
# if there is a conflict.
self.params = dict_combine(self.default_params, params)
self.verbose = self.params.pop('verbose', 1)
self.validate_params(self.params)
# Compute possible new params from existing params
self.compute_params()
self.observation = observation
if settings['PREVALIDATE']:
self.validate_observation(self.observation)
if self.score_type is None or not issubclass(self.score_type, Score):
raise Error(("The score type '%s' specified for Test '%s' "
"is not valid.") % (self.score_type, self.name))
super(Test, self).__init__()
raise scores[i][j].score # An exception.
# 9.
from sciunit.scores.collections_m2m import ScoreMatrixM2M
sm = ScoreMatrixM2M(self, models, scores=scores)
return sm
"""
# TODO: see if this needs to be updated and provided:
def optimize(self, model):
raise NotImplementedError(("Optimization not implemented "
"for Test '%s'" % self))
"""
class RangeTest(Test):
"""Test if the model generates a number within a certain range"""
def __init__(self, observation, name=None):
super(RangeTest, self).__init__(observation, name=name)
required_capabilities = (ProducesNumber,)
score_type = BooleanScore
def validate_observation(self, observation):
assert type(observation) in (tuple, list, set)
assert len(observation) == 2
assert observation[1] > observation[0]
def generate_prediction(self, model):
return model.produce_number()
def validate_observation(self, observation):
assert type(observation) in (tuple, list, set)
assert len(observation) == 2
assert observation[1] > observation[0]
def generate_prediction(self, model):
return model.produce_number()
def compute_score(self, observation, prediction):
low = observation[0]
high = observation[1]
return self.score_type(low < prediction < high)
class ProtocolToFeaturesTest(Test):
"""Assume that generating a prediction consists of:
1) Setting up a simulation experiment protocol.
Depending on the backend, this could include editing simulation parameters
in memory or editing a model file. It could include any kind of
experimental protocol, such as a perturbation.
2) Running a model (using e.g. RunnableModel)
3) Extract features from the results
Developers should not need to manually implement `generate_prediction`, and
instead should focus on the other three methods here.
"""
def generate_prediction(self, model):
run_method = getattr(model, "run", None)
assert callable(run_method), \
"Model must have a `run` method to use a ProtocolToFeaturesTest"
def get_group(self, x):
t = int(bool(self.transposed))
if isinstance(x[0], Test) and isinstance(x[1], Model):
result = self.loc[x[1-t], x[t]]
elif isinstance(x[1], Test) and isinstance(x[0], Model):
result = self.loc[x[t], x[1-t]]
elif isinstance(x[0], str):
result = self.__getitem__(x[t]).__getitem__(x[1-t])
else:
raise TypeError("Expected test,model or model,test")
return result
def check_tests_and_models(self, tests_or_models):
assert all([isinstance(tom, Test) for tom in tests_or_models]) or \
all([isinstance(tom, Model) for tom in tests_or_models]), \
"A ScoreArray may be indexed by only test or models"
return tests_or_models
def assert_tests(self, tests):
"""Check and in some cases fixes the list of tests."""
if isinstance(tests, Test):
# Turn singleton test into a sequence
tests = (tests,)
else:
try:
for test in tests:
if not isinstance(test, Test):
raise TypeError(("Test suite provided an iterable "
"containing a non-Test."))
except TypeError:
raise TypeError(("Test suite was not provided with "
"a test or iterable."))
return tests
def __init__(self, tests, name=None, weights=None, include_models=None,
skip_models=None, hooks=None, optimizer=None):
"""
optimizer: a function to bind to self.optimize (first argument must be a testsuite)
"""
self.name = name if name else "Suite_%d" % random.randint(0, 1e12)
if isinstance(tests, dict):
for key, value in tests.items():
if not isinstance(value, Test):
setattr(self, key, value)
tests = [test for test in tests.values() if isinstance(test, Test)]
self.tests = self.assert_tests(tests)
self.weights_ = [] if not weights else list(weights)
self.include_models = include_models if include_models else []
self.skip_models = skip_models if skip_models else []
self.hooks = hooks
super(TestSuite, self).__init__()
if optimizer:
self.optimize = MethodType(optimizer, self)
def __init__(self, tests, name=None, weights=None, include_models=None,
skip_models=None, hooks=None, optimizer=None):
"""
optimizer: a function to bind to self.optimize (first argument must be a testsuite)
"""
self.name = name if name else "Suite_%d" % random.randint(0, 1e12)
if isinstance(tests, dict):
for key, value in tests.items():
if not isinstance(value, Test):
setattr(self, key, value)
tests = [test for test in tests.values() if isinstance(test, Test)]
self.tests = self.assert_tests(tests)
self.weights_ = [] if not weights else list(weights)
self.include_models = include_models if include_models else []
self.skip_models = skip_models if skip_models else []
self.hooks = hooks
super(TestSuite, self).__init__()
if optimizer:
self.optimize = MethodType(optimizer, self)
def __getitem__(self, item):
if isinstance(item, (Test, Model)):
result = ScoreArrayM2M(self.test, self.models,
scores=self.loc[item, :])
elif isinstance(item, str):
result = self.get_by_name(item)
elif isinstance(item, (list, tuple)) and len(item) == 2:
result = self.get_group(item)
else:
raise TypeError(("Expected test/'observation'; model; "
"test/'observation',model; "
"model,test/'observation'; or model,model"))
return result