Skip to content

test_suite_validator

Class in charge of validating a TestSuite.

TestSuiteValidator

Helper class to validate a test suite.

Source code in mlte/validation/test_suite_validator.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
class TestSuiteValidator:
    """
    Helper class to validate a test suite.
    """

    def __init__(
        self, test_suite: Optional[TestSuite] = None, test_suite_id: str = ""
    ):
        """
        Initialize a TestSuiteValidator instance.

        :param test_suite: The test suite to be validated
        """

        # If no test suite is provided, load it from its id or its default one.
        if test_suite is None:
            if test_suite_id != "":
                test_suite = TestSuite.load(test_suite_id)
            else:
                test_suite = TestSuite.load()

        self.test_suite = test_suite
        """The test suite to be validated."""

        self.evidence: dict[str, Evidence] = {}
        """Where evidence will be gathered for validation."""

    def add_evidence_list(self, evidence_list: list[Evidence]):
        """
        Adds multiple evidence values.

        :param evidence_list: The list of evidence to add to the internal list.
        """
        for evidence in evidence_list:
            self.add_evidence(evidence)

    def add_evidence(self, evidence: Evidence):
        """
        Adds Evidence associated to a test case.

        :param evidence: The evidence to add to the internal list.
        """
        if not evidence.metadata:
            raise RuntimeError("Provided evidence has no metadata.")

        if evidence.metadata.test_case_id in self.evidence:
            raise RuntimeError(
                f"Can't have two values with the same id: {evidence.metadata.test_case_id}"
            )
        self.evidence[evidence.metadata.test_case_id] = evidence

    def load_and_validate(self) -> TestResults:
        """Loads all Evidence for the model/context and store in session, and validates."""
        self.add_evidence_list(Evidence.load_all())
        return self.validate()

    def validate(self) -> TestResults:
        """
        Validates the internal Evidence given its validators, and generates a TestResults from it.

        :return: The results of the test validation.
        """
        # Check that all test cases have evidence to be validated.
        for test_case_id, test_case in self.test_suite.test_cases.items():
            # Make exception for info cases, where there is no bool exp to validate, but just info to be manually validated later.
            is_info_case = (
                test_case.validator
                and not test_case.validator.bool_exp
                and test_case.validator.info
            )
            if test_case_id not in self.evidence and not is_info_case:
                raise RuntimeError(
                    f"Test Case '{test_case_id}' will be automatically validated, and does not have evidence that can be validated."
                )

        # Validate and aggregate the results.
        results: dict[str, Result] = {}
        for test_case_id, test_case in self.test_suite.test_cases.items():
            # Manage case where there is no evidence, typically for an info validator.
            evidence = (
                self.evidence[test_case_id]
                if test_case_id in self.evidence
                else None
            )
            results[test_case_id] = test_case.validate(evidence)

        return TestResults(test_suite=self.test_suite, results=results)

evidence = {} instance-attribute

Where evidence will be gathered for validation.

test_suite = test_suite instance-attribute

The test suite to be validated.

__init__(test_suite=None, test_suite_id='')

Initialize a TestSuiteValidator instance.

Parameters:

Name Type Description Default
test_suite Optional[TestSuite]

The test suite to be validated

 None
Source code in mlte/validation/test_suite_validator.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def __init__(
    self, test_suite: Optional[TestSuite] = None, test_suite_id: str = ""
):
    """
    Initialize a TestSuiteValidator instance.

    :param test_suite: The test suite to be validated
    """

    # If no test suite is provided, load it from its id or its default one.
    if test_suite is None:
        if test_suite_id != "":
            test_suite = TestSuite.load(test_suite_id)
        else:
            test_suite = TestSuite.load()

    self.test_suite = test_suite
    """The test suite to be validated."""

    self.evidence: dict[str, Evidence] = {}
    """Where evidence will be gathered for validation."""

add_evidence(evidence)

Adds Evidence associated to a test case.

Parameters:

Name Type Description Default
evidence Evidence

The evidence to add to the internal list.

 required
Source code in mlte/validation/test_suite_validator.py 
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def add_evidence(self, evidence: Evidence):
    """
    Adds Evidence associated to a test case.

    :param evidence: The evidence to add to the internal list.
    """
    if not evidence.metadata:
        raise RuntimeError("Provided evidence has no metadata.")

    if evidence.metadata.test_case_id in self.evidence:
        raise RuntimeError(
            f"Can't have two values with the same id: {evidence.metadata.test_case_id}"
        )
    self.evidence[evidence.metadata.test_case_id] = evidence

add_evidence_list(evidence_list)

Adds multiple evidence values.

Parameters:

Name Type Description Default
evidence_list list[Evidence]

The list of evidence to add to the internal list.

 required
Source code in mlte/validation/test_suite_validator.py 
46
47
48
49
50
51
52
53
def add_evidence_list(self, evidence_list: list[Evidence]):
    """
    Adds multiple evidence values.

    :param evidence_list: The list of evidence to add to the internal list.
    """
    for evidence in evidence_list:
        self.add_evidence(evidence)

load_and_validate()

Loads all Evidence for the model/context and store in session, and validates.

Source code in mlte/validation/test_suite_validator.py 
70
71
72
73
def load_and_validate(self) -> TestResults:
    """Loads all Evidence for the model/context and store in session, and validates."""
    self.add_evidence_list(Evidence.load_all())
    return self.validate()

validate()

Validates the internal Evidence given its validators, and generates a TestResults from it.

Returns:

Type Description
TestResults

The results of the test validation.
Source code in mlte/validation/test_suite_validator.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def validate(self) -> TestResults:
    """
    Validates the internal Evidence given its validators, and generates a TestResults from it.

    :return: The results of the test validation.
    """
    # Check that all test cases have evidence to be validated.
    for test_case_id, test_case in self.test_suite.test_cases.items():
        # Make exception for info cases, where there is no bool exp to validate, but just info to be manually validated later.
        is_info_case = (
            test_case.validator
            and not test_case.validator.bool_exp
            and test_case.validator.info
        )
        if test_case_id not in self.evidence and not is_info_case:
            raise RuntimeError(
                f"Test Case '{test_case_id}' will be automatically validated, and does not have evidence that can be validated."
            )

    # Validate and aggregate the results.
    results: dict[str, Result] = {}
    for test_case_id, test_case in self.test_suite.test_cases.items():
        # Manage case where there is no evidence, typically for an info validator.
        evidence = (
            self.evidence[test_case_id]
            if test_case_id in self.evidence
            else None
        )
        results[test_case_id] = test_case.validate(evidence)

    return TestResults(test_suite=self.test_suite, results=results)