Skip to content

test_results

TestResults class implementation.

TestResults

Bases: Artifact

TestResults represents a the results for a TestSuite.

Source code in mlte/results/test_results.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
class TestResults(Artifact):
    """
    TestResults represents a the results for a TestSuite.
    """

    type = ArtifactType.TEST_RESULTS
    """Class attribute indicating type."""

    def __init__(
        self,
        test_suite: TestSuite,
        identifier: Optional[str] = None,
        results: dict[str, Result] = {},
    ):
        """
        Initialize a TestResults instance.

        :param identifier: An id for this set of test results.
        :param test_suite: The TestSuite
        :param results: The validation Results for the TestSuite
        """
        super().__init__(identifier)

        self.test_suite = test_suite
        """The id of the TestSuite that we validated."""

        self.results = results
        """The validation results for the test_suite, by test case."""

        # Check that all tests have results.
        if test_suite:
            for test_case_id, _ in test_suite.test_cases.items():
                if test_case_id not in results:
                    raise RuntimeError(
                        f"Test Case '{test_case_id}' does not have a result."
                    )

    # -------------------------------------------------------------------------
    # Model serialization.
    # -------------------------------------------------------------------------

    def to_model(self) -> ArtifactModel:
        """
        Generates a model representation of the TestResults.
        :return: The serialized model
        """
        return ArtifactModel(
            header=self.build_artifact_header(),
            body=TestResultsModel(
                test_suite_id=self.test_suite.identifier,
                test_suite=typing.cast(
                    TestSuiteModel, self.test_suite.to_model().body
                ),
                results={
                    test_case_id: result.to_model()
                    for test_case_id, result in self.results.items()
                },
            ),
        )

    @classmethod
    def from_model(cls, model: BaseModel) -> TestResults:
        """
        Deserialize TestResults content from model.
        :param model: The model
        :return: The deserialized specification
        """
        assert isinstance(
            model, ArtifactModel
        ), "Can't create object from non-ArtifactModel model."
        assert (
            model.header.type == ArtifactType.TEST_RESULTS
        ), "Type should be TestResults."
        body = typing.cast(TestResultsModel, model.body)

        # Build the TestSuite and TestResults
        return TestResults(
            identifier=model.header.identifier,
            test_suite=TestSuite(
                identifier=body.test_suite_id,
                test_cases=[
                    TestCase.from_model(test_case_model)
                    for test_case_model in body.test_suite.test_cases
                ],
            ),
            results={
                test_case_id: Result.from_model(test_result_model)
                for test_case_id, test_result_model in body.results.items()
            },
        )

    # Overriden.
    @classmethod
    def load(cls, identifier: typing.Optional[str] = None) -> TestResults:
        """
        Load a TestResults from the configured global session.
        :param identifier: The identifier for the artifact. If None,
        the default id is used.
        """
        suite = super().load(identifier)
        return typing.cast(TestResults, suite)

    # -------------------------------------------------------------------------
    # Helpers.
    # -------------------------------------------------------------------------

    def print_results(self, result_type: str = "all"):
        """Prints the validated results per test case, can be filtered by result type."""
        if result_type not in ["all", "Success", "Failure", "Info"]:
            raise RuntimeError(f"Invalid type: {result_type}")

        for test_case_id, result in self.results.items():
            if result_type == "all" or result_type == str(result):
                print(
                    f" > Test Case: {test_case_id}, result: {result}, details: {result.message}"
                )

    def convert_result(
        self,
        test_case_id: str,
        result_type: Union[Type[Success], Type[Failure]],
        message: str,
    ) -> None:
        """Converts a given Info result into the provided type."""
        if test_case_id not in self.results:
            raise RuntimeError(
                f"Test case {test_case_id} is not in the list of results."
            )

        result = self.results[test_case_id]
        if not isinstance(result, Info):
            raise RuntimeError(
                "Only results of type Info can be manually validated."
            )

        # Create new result based on given type.
        new_result_msg = f"Manually validated: {message} (original message: {result.message})"
        new_result = result_type(new_result_msg)
        self.results[test_case_id] = new_result

    # -------------------------------------------------------------------------
    # Default overriden.
    # -------------------------------------------------------------------------

    def __eq__(self, other: object) -> bool:
        """Test TestResults instance for equality."""
        if not isinstance(other, TestResults):
            return False
        return self._equal(other)

results = results instance-attribute

The validation results for the test_suite, by test case.

test_suite = test_suite instance-attribute

The id of the TestSuite that we validated.

type = ArtifactType.TEST_RESULTS class-attribute instance-attribute

Class attribute indicating type.

__eq__(other)

Test TestResults instance for equality.

Source code in mlte/results/test_results.py 
169
170
171
172
173
def __eq__(self, other: object) -> bool:
    """Test TestResults instance for equality."""
    if not isinstance(other, TestResults):
        return False
    return self._equal(other)

__init__(test_suite, identifier=None, results={})

Initialize a TestResults instance.

Parameters:

Name Type Description Default
identifier Optional[str]

An id for this set of test results.

 None
test_suite TestSuite

The TestSuite

 required
results dict[str, Result]

The validation Results for the TestSuite

 {}
Source code in mlte/results/test_results.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def __init__(
    self,
    test_suite: TestSuite,
    identifier: Optional[str] = None,
    results: dict[str, Result] = {},
):
    """
    Initialize a TestResults instance.

    :param identifier: An id for this set of test results.
    :param test_suite: The TestSuite
    :param results: The validation Results for the TestSuite
    """
    super().__init__(identifier)

    self.test_suite = test_suite
    """The id of the TestSuite that we validated."""

    self.results = results
    """The validation results for the test_suite, by test case."""

    # Check that all tests have results.
    if test_suite:
        for test_case_id, _ in test_suite.test_cases.items():
            if test_case_id not in results:
                raise RuntimeError(
                    f"Test Case '{test_case_id}' does not have a result."
                )

convert_result(test_case_id, result_type, message)

Converts a given Info result into the provided type.

Source code in mlte/results/test_results.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def convert_result(
    self,
    test_case_id: str,
    result_type: Union[Type[Success], Type[Failure]],
    message: str,
) -> None:
    """Converts a given Info result into the provided type."""
    if test_case_id not in self.results:
        raise RuntimeError(
            f"Test case {test_case_id} is not in the list of results."
        )

    result = self.results[test_case_id]
    if not isinstance(result, Info):
        raise RuntimeError(
            "Only results of type Info can be manually validated."
        )

    # Create new result based on given type.
    new_result_msg = f"Manually validated: {message} (original message: {result.message})"
    new_result = result_type(new_result_msg)
    self.results[test_case_id] = new_result

from_model(model) classmethod

Deserialize TestResults content from model.

Parameters:

Name Type Description Default
model BaseModel

The model

 required

Returns:

Type Description
TestResults

The deserialized specification
Source code in mlte/results/test_results.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
@classmethod
def from_model(cls, model: BaseModel) -> TestResults:
    """
    Deserialize TestResults content from model.
    :param model: The model
    :return: The deserialized specification
    """
    assert isinstance(
        model, ArtifactModel
    ), "Can't create object from non-ArtifactModel model."
    assert (
        model.header.type == ArtifactType.TEST_RESULTS
    ), "Type should be TestResults."
    body = typing.cast(TestResultsModel, model.body)

    # Build the TestSuite and TestResults
    return TestResults(
        identifier=model.header.identifier,
        test_suite=TestSuite(
            identifier=body.test_suite_id,
            test_cases=[
                TestCase.from_model(test_case_model)
                for test_case_model in body.test_suite.test_cases
            ],
        ),
        results={
            test_case_id: Result.from_model(test_result_model)
            for test_case_id, test_result_model in body.results.items()
        },
    )

load(identifier=None) classmethod

Load a TestResults from the configured global session.

Parameters:

Name Type Description Default
identifier Optional[str]

The identifier for the artifact. If None, the default id is used.

 None
Source code in mlte/results/test_results.py 
117
118
119
120
121
122
123
124
125
@classmethod
def load(cls, identifier: typing.Optional[str] = None) -> TestResults:
    """
    Load a TestResults from the configured global session.
    :param identifier: The identifier for the artifact. If None,
    the default id is used.
    """
    suite = super().load(identifier)
    return typing.cast(TestResults, suite)

print_results(result_type='all')

Prints the validated results per test case, can be filtered by result type.

Source code in mlte/results/test_results.py 
131
132
133
134
135
136
137
138
139
140
def print_results(self, result_type: str = "all"):
    """Prints the validated results per test case, can be filtered by result type."""
    if result_type not in ["all", "Success", "Failure", "Info"]:
        raise RuntimeError(f"Invalid type: {result_type}")

    for test_case_id, result in self.results.items():
        if result_type == "all" or result_type == str(result):
            print(
                f" > Test Case: {test_case_id}, result: {result}, details: {result.message}"
            )

to_model()

Generates a model representation of the TestResults.

Returns:

Type Description
ArtifactModel

The serialized model
Source code in mlte/results/test_results.py 
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def to_model(self) -> ArtifactModel:
    """
    Generates a model representation of the TestResults.
    :return: The serialized model
    """
    return ArtifactModel(
        header=self.build_artifact_header(),
        body=TestResultsModel(
            test_suite_id=self.test_suite.identifier,
            test_suite=typing.cast(
                TestSuiteModel, self.test_suite.to_model().body
            ),
            results={
                test_case_id: result.to_model()
                for test_case_id, result in self.results.items()
            },
        ),
    )