Skip to content

test_case

TestCase defines structure for all tests to be defined for a TestSuite.

TestCase

Bases: Serializable

Class that contains all information about a test case.

Source code in mlte/suite/test_case.py 
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
class TestCase(Serializable):
    """Class that contains all information about a test case."""

    def __init__(
        self,
        identifier: str,
        goal: str,
        quality_scenarios: list[str],
        measurement: Optional[Measurement] = None,
        validator: Optional[Validator] = None,
        note: Optional[str] = None,
    ):
        self.identifier = identifier
        """Unique id or name given to the test case."""

        self.goal = goal
        """Goal for the TestCase, reason for it."""

        self.quality_scenarios = quality_scenarios.copy()
        """Quality Attribute Scenario ids that are associated to this test case."""

        self.measurement = measurement
        """Used to measure and get a value for this test case."""

        self.validator = validator
        """Used to validate this test case."""

        self.note = note
        """Additional information to go along with this test case."""

        if self.measurement:
            # If we got a Measurement, ensure its test case id matches ours.
            self.measurement.set_test_case_id(self.identifier)

    def measure(self, *args, **kwargs) -> Evidence:
        """Executes the configured measurement with the given params."""
        if self.measurement is None:
            raise RuntimeError(
                "Can't evaluate measurement, no measurement has been configured."
            )

        return self.measurement.evaluate(*args, **kwargs)

    def validate(self, evidence: Optional[Evidence]) -> Result:
        """Executes the configured validator with the given Evidence."""
        if self.validator is None:
            raise RuntimeError(
                "Can't validate, no validator has been configured."
            )

        result = self.validator.validate(evidence)

        # If evidence was received, add its metadata.
        if evidence:
            result = result._with_evidence_metadata(evidence.metadata)

        return result

    # -------------------------------------------------------------------------
    # Model conversion.
    # -------------------------------------------------------------------------

    def to_model(self) -> TestCaseModel:
        """
        Returns this test case as a model.

        :return: The serialized model object.
        """
        return TestCaseModel(
            identifier=self.identifier,
            goal=self.goal,
            qas_list=self.quality_scenarios,
            measurement=(
                self.measurement.generate_metadata()
                if self.measurement
                else None
            ),
            validator=self.validator.to_model() if self.validator else None,
            note=self.note,
        )

    @classmethod
    def from_model(cls, model: BaseModel) -> TestCase:
        """
        Deserialize a TestCase from a model.

        :param model: The model.

        :return: The deserialized TestCase
        """
        model = typing.cast(TestCaseModel, model)
        test_case: TestCase = TestCase(
            identifier=model.identifier,
            goal=model.goal,
            quality_scenarios=model.qas_list,
            measurement=(
                Measurement.from_metadata(
                    model=model.measurement, test_case_id=model.identifier
                )
                if model.measurement
                else None
            ),
            validator=(
                Validator.from_model(model.validator)
                if model.validator
                else None
            ),
            note=model.note,
        )
        return test_case

    # -------------------------------------------------------------------------
    # Templating.
    # -------------------------------------------------------------------------

    def template(self):
        TestCase(identifier="ID", goal="GOAL", quality_scenarios=["QA"]),

    def to_template_str(self) -> str:
        """Convert the test case into a template string."""
        source = get_function_code(self.template)
        source = source.replace("ID", self.identifier)
        source = source.replace("GOAL", self.goal)
        source = source.replace("QA", self.quality_scenarios[0])
        return source

    # -------------------------------------------------------------------------
    # Builtin overloads.
    # -------------------------------------------------------------------------

    def __str__(self) -> str:
        """Return a string representation of TestCase."""
        return f"{self.identifier}: {self.goal} ({self.quality_scenarios}) {self.note}"

    def __eq__(self, other: object) -> bool:
        """Test instance for equality."""
        if not isinstance(other, TestCase):
            return False

        # Validator has to be compared separately due to their storing of serialized code.
        validators_are_equal = self.validator == other.validator
        a_model = self.to_model()
        b_model = other.to_model()
        a_model.validator = None
        b_model.validator = None

        return a_model == b_model and validators_are_equal

goal = goal instance-attribute

Goal for the TestCase, reason for it.

identifier = identifier instance-attribute

Unique id or name given to the test case.

measurement = measurement instance-attribute

Used to measure and get a value for this test case.

note = note instance-attribute

Additional information to go along with this test case.

quality_scenarios = quality_scenarios.copy() instance-attribute

Quality Attribute Scenario ids that are associated to this test case.

validator = validator instance-attribute

Used to validate this test case.

__eq__(other)

Test instance for equality.

Source code in mlte/suite/test_case.py 
152
153
154
155
156
157
158
159
160
161
162
163
164
def __eq__(self, other: object) -> bool:
    """Test instance for equality."""
    if not isinstance(other, TestCase):
        return False

    # Validator has to be compared separately due to their storing of serialized code.
    validators_are_equal = self.validator == other.validator
    a_model = self.to_model()
    b_model = other.to_model()
    a_model.validator = None
    b_model.validator = None

    return a_model == b_model and validators_are_equal

__str__()

Return a string representation of TestCase.

Source code in mlte/suite/test_case.py 
148
149
150
def __str__(self) -> str:
    """Return a string representation of TestCase."""
    return f"{self.identifier}: {self.goal} ({self.quality_scenarios}) {self.note}"

from_model(model) classmethod

Deserialize a TestCase from a model.

Parameters:

Name Type Description Default
model BaseModel

The model.

 required

Returns:

Type Description
TestCase

The deserialized TestCase
Source code in mlte/suite/test_case.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
@classmethod
def from_model(cls, model: BaseModel) -> TestCase:
    """
    Deserialize a TestCase from a model.

    :param model: The model.

    :return: The deserialized TestCase
    """
    model = typing.cast(TestCaseModel, model)
    test_case: TestCase = TestCase(
        identifier=model.identifier,
        goal=model.goal,
        quality_scenarios=model.qas_list,
        measurement=(
            Measurement.from_metadata(
                model=model.measurement, test_case_id=model.identifier
            )
            if model.measurement
            else None
        ),
        validator=(
            Validator.from_model(model.validator)
            if model.validator
            else None
        ),
        note=model.note,
    )
    return test_case

measure(*args, **kwargs)

Executes the configured measurement with the given params.

Source code in mlte/suite/test_case.py 
52
53
54
55
56
57
58
59
def measure(self, *args, **kwargs) -> Evidence:
    """Executes the configured measurement with the given params."""
    if self.measurement is None:
        raise RuntimeError(
            "Can't evaluate measurement, no measurement has been configured."
        )

    return self.measurement.evaluate(*args, **kwargs)

to_model()

Returns this test case as a model.

Returns:

Type Description
TestCaseModel

The serialized model object.
Source code in mlte/suite/test_case.py 
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def to_model(self) -> TestCaseModel:
    """
    Returns this test case as a model.

    :return: The serialized model object.
    """
    return TestCaseModel(
        identifier=self.identifier,
        goal=self.goal,
        qas_list=self.quality_scenarios,
        measurement=(
            self.measurement.generate_metadata()
            if self.measurement
            else None
        ),
        validator=self.validator.to_model() if self.validator else None,
        note=self.note,
    )

to_template_str()

Convert the test case into a template string.

Source code in mlte/suite/test_case.py 
136
137
138
139
140
141
142
def to_template_str(self) -> str:
    """Convert the test case into a template string."""
    source = get_function_code(self.template)
    source = source.replace("ID", self.identifier)
    source = source.replace("GOAL", self.goal)
    source = source.replace("QA", self.quality_scenarios[0])
    return source

validate(evidence)

Executes the configured validator with the given Evidence.

Source code in mlte/suite/test_case.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def validate(self, evidence: Optional[Evidence]) -> Result:
    """Executes the configured validator with the given Evidence."""
    if self.validator is None:
        raise RuntimeError(
            "Can't validate, no validator has been configured."
        )

    result = self.validator.validate(evidence)

    # If evidence was received, add its metadata.
    if evidence:
        result = result._with_evidence_metadata(evidence.metadata)

    return result