How to use the sciunit.tests.Test function in sciunit

To help you get started, we’ve selected a few sciunit examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github scidash / sciunit / sciunit / tests.py View on Github external
    @property
    def state(self):
        """Get the frozen (pickled) model state."""
        return self._state(exclude=['last_model'])

    @classmethod
    def is_test_class(cls, other_cls):
        """Return whether `other_cls` is a subclass of this test class."""
        return inspect.isclass(other_cls) and issubclass(other_cls, cls)

    def __str__(self):
        """Return the string representation of the test's name."""
        return '%s' % self.name


class TestM2M(Test):
    """Abstract class for handling tests involving multiple models.

    Enables comparison of model to model predictions, and also against
    experimental reference data (optional).

    Note: 'TestM2M' would typically be used when handling mutliple (>2)
    models, with/without experimental reference data. For single model
    tests, you can use the 'Test' class.
    """

    def __init__(self, observation=None, name=None, **params):
        super(TestM2M, self).__init__(observation, name=name, **params)

    def validate_observation(self, observation):
        """Validate the observation provided to the constructor.
github scidash / sciunit / sciunit / tests.py View on Github external
# if there is a conflict.
        self.params = dict_combine(self.default_params, params)
        self.verbose = self.params.pop('verbose', 1)
        self.validate_params(self.params)
        # Compute possible new params from existing params
        self.compute_params()

        self.observation = observation
        if settings['PREVALIDATE']:
            self.validate_observation(self.observation)

        if self.score_type is None or not issubclass(self.score_type, Score):
            raise Error(("The score type '%s' specified for Test '%s' "
                         "is not valid.") % (self.score_type, self.name))

        super(Test, self).__init__()
github scidash / sciunit / sciunit / tests.py View on Github external
raise scores[i][j].score  # An exception.

        # 9.
        from sciunit.scores.collections_m2m import ScoreMatrixM2M
        sm = ScoreMatrixM2M(self, models, scores=scores)
        return sm

    """
    # TODO: see if this needs to be updated and provided:
    def optimize(self, model):
        raise NotImplementedError(("Optimization not implemented "
                                   "for Test '%s'" % self))
    """


class RangeTest(Test):
    """Test if the model generates a number within a certain range"""

    def __init__(self, observation, name=None):
        super(RangeTest, self).__init__(observation, name=name)

    required_capabilities = (ProducesNumber,)
    score_type = BooleanScore

    def validate_observation(self, observation):
        assert type(observation) in (tuple, list, set)
        assert len(observation) == 2
        assert observation[1] > observation[0]

    def generate_prediction(self, model):
        return model.produce_number()
github scidash / sciunit / sciunit / tests.py View on Github external
def validate_observation(self, observation):
        assert type(observation) in (tuple, list, set)
        assert len(observation) == 2
        assert observation[1] > observation[0]

    def generate_prediction(self, model):
        return model.produce_number()

    def compute_score(self, observation, prediction):
        low = observation[0]
        high = observation[1]
        return self.score_type(low < prediction < high)


class ProtocolToFeaturesTest(Test):
    """Assume that generating a prediction consists of:
    1) Setting up a simulation experiment protocol.
    Depending on the backend, this could include editing simulation parameters
    in memory or editing a model file.  It could include any kind of
    experimental protocol, such as a perturbation.
    2) Running a model (using e.g. RunnableModel)
    3) Extract features from the results

    Developers should not need to manually implement `generate_prediction`, and
    instead should focus on the other three methods here.
    """

    def generate_prediction(self, model):
        run_method = getattr(model, "run", None)
        assert callable(run_method), \
            "Model must have a `run` method to use a ProtocolToFeaturesTest"
github scidash / sciunit / sciunit / scores / collections.py View on Github external
def get_group(self, x):
        t = int(bool(self.transposed))
        if isinstance(x[0], Test) and isinstance(x[1], Model):
            result = self.loc[x[1-t], x[t]]
        elif isinstance(x[1], Test) and isinstance(x[0], Model):
            result = self.loc[x[t], x[1-t]]
        elif isinstance(x[0], str):
            result = self.__getitem__(x[t]).__getitem__(x[1-t])
        else:
            raise TypeError("Expected test,model or model,test")
        return result
github scidash / sciunit / sciunit / scores / collections.py View on Github external
def check_tests_and_models(self, tests_or_models):
        assert all([isinstance(tom, Test) for tom in tests_or_models]) or \
               all([isinstance(tom, Model) for tom in tests_or_models]), \
               "A ScoreArray may be indexed by only test or models"
        return tests_or_models
github scidash / sciunit / sciunit / suites.py View on Github external
def assert_tests(self, tests):
        """Check and in some cases fixes the list of tests."""

        if isinstance(tests, Test):
            # Turn singleton test into a sequence
            tests = (tests,)
        else:
            try:
                for test in tests:
                    if not isinstance(test, Test):
                        raise TypeError(("Test suite provided an iterable "
                                         "containing a non-Test."))
            except TypeError:
                raise TypeError(("Test suite was not provided with "
                                 "a test or iterable."))
        return tests
github scidash / sciunit / sciunit / suites.py View on Github external
def __init__(self, tests, name=None, weights=None, include_models=None,
                 skip_models=None, hooks=None, optimizer=None):
        """
        optimizer: a function to bind to self.optimize (first argument must be a testsuite)
        """
        self.name = name if name else "Suite_%d" % random.randint(0, 1e12)
        if isinstance(tests, dict):
            for key, value in tests.items():
                if not isinstance(value, Test):
                    setattr(self, key, value)
            tests = [test for test in tests.values() if isinstance(test, Test)]
        self.tests = self.assert_tests(tests)
        self.weights_ = [] if not weights else list(weights)
        self.include_models = include_models if include_models else []
        self.skip_models = skip_models if skip_models else []
        self.hooks = hooks
        super(TestSuite, self).__init__()
        if optimizer:
            self.optimize = MethodType(optimizer, self)
github scidash / sciunit / sciunit / suites.py View on Github external
def __init__(self, tests, name=None, weights=None, include_models=None,
                 skip_models=None, hooks=None, optimizer=None):
        """
        optimizer: a function to bind to self.optimize (first argument must be a testsuite)
        """
        self.name = name if name else "Suite_%d" % random.randint(0, 1e12)
        if isinstance(tests, dict):
            for key, value in tests.items():
                if not isinstance(value, Test):
                    setattr(self, key, value)
            tests = [test for test in tests.values() if isinstance(test, Test)]
        self.tests = self.assert_tests(tests)
        self.weights_ = [] if not weights else list(weights)
        self.include_models = include_models if include_models else []
        self.skip_models = skip_models if skip_models else []
        self.hooks = hooks
        super(TestSuite, self).__init__()
        if optimizer:
            self.optimize = MethodType(optimizer, self)
github scidash / sciunit / sciunit / scores / collections_m2m.py View on Github external
def __getitem__(self, item):
        if isinstance(item, (Test, Model)):
            result = ScoreArrayM2M(self.test, self.models,
                                   scores=self.loc[item, :])
        elif isinstance(item, str):
            result = self.get_by_name(item)
        elif isinstance(item, (list, tuple)) and len(item) == 2:
            result = self.get_group(item)
        else:
            raise TypeError(("Expected test/'observation'; model; "
                             "test/'observation',model; "
                             "model,test/'observation'; or model,model"))
        return result