Skip to content

measurement

Superclass for all measurements.

Measurement

The superclass for all model measurements.

Source code in mlte/measurement/measurement.py 
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
class Measurement:
    """
    The superclass for all model measurements.
    """

    force_stop: bool = True
    """Attribute to force raising an Exception when error is encountered when running a measurement.
    If False, will return an Unavailable Evidence type instead with the error details. Default to true."""

    def __init__(
        self, test_case_id: str | None = None, force_stop: bool = True
    ):
        """Constructor."""

        self.force_stop = force_stop

        # Initialize with no values.
        self.test_case_id: str | None = None
        self.evidence_metadata: EvidenceMetadata | None = None

        if test_case_id is not None:
            # Set our id to the test case id, and generate our evidence metadata.
            self.set_test_case_id(test_case_id)

    def set_test_case_id(self, test_case_id: str):
        # Set the given test case id and update our metadata.

        self.test_case_id = test_case_id
        """The id of the test case this measurement is for."""

        self.set_metadata()
        """Set the evidence metadata."""

    def set_metadata(self):
        """Resets the internal metadata to associate to Evidences generated by this measurement."""
        if self.test_case_id is None:
            raise RuntimeError(
                "Can't generate evidence metadata if test case id has not been set."
            )

        self.evidence_metadata = EvidenceMetadata(
            test_case_id=self.test_case_id, measurement=self.generate_metadata()
        )

    # -------------------------------------------------------------------------
    # Measurement interface definition.
    # -------------------------------------------------------------------------

    @classmethod
    def __subclasshook__(cls, subclass):
        """Define the interface for all concrete measurements."""
        return meta.has_callables(subclass, "__call__")

    @abstractmethod
    def __call__(self, *args, **kwargs) -> Evidence:
        """Evaluate a measurement and return a value semantics."""
        raise NotImplementedError("Cannot evaluate abstract measurement.")

    # -------------------------------------------------------------------------
    # Base methods.
    # -------------------------------------------------------------------------

    def evaluate(self, *args, **kwargs) -> Evidence:
        """
        Evaluate a measurement and return a value with semantics.

        :return: The resulting value of measurement execution, with semantics
        """
        if self.evidence_metadata is None:
            raise RuntimeError(
                "Can't evaluate measurement before setting its id"
            )

        # Evaluate the measurement.
        try:
            return self.__call__(*args, **kwargs).with_metadata(
                self.evidence_metadata
            )
        except Exception as e:
            # If we want to force stop on error, just re-throw, otherwise return as Unavailable evidence.
            if self.force_stop:
                raise e
            else:
                return Unavailable(
                    details=str(e), traceback=traceback.format_exc()
                ).with_metadata(self.evidence_metadata)

    @classmethod
    def output(cls) -> type[Evidence]:
        """Returns the class type object for the Evidence produced by the Measurement."""
        # Opaque is the default Evidence type.
        return Opaque

    # -------------------------------------------------------------------------
    # MeasurementMetadata model handling.
    # -------------------------------------------------------------------------

    def generate_metadata(self) -> MeasurementMetadata:
        """Returns Measurement metadata."""
        return MeasurementMetadata(
            measurement_class=meta.get_qualified_name(self.__class__),
            output_class=meta.get_qualified_name(self.output()),
        )

    @classmethod
    def from_metadata(
        cls, model: MeasurementMetadata, test_case_id: str
    ) -> Measurement:
        """
        Create a Measurement from a model.

        :param model: The MeasurementMetadata model.
        :param test_case_id: The id of the associated test case.
        :return: The Measurement.
        """
        meas_class: type[Measurement] = typing.cast(
            type[Measurement], load_class_or_function(model.measurement_class)
        )
        measurement = meas_class(test_case_id)
        measurement.additional_setup(model)
        return measurement

    def additional_setup(self, model: MeasurementMetadata):
        """Optional method to be overriden by subclasses needing additional setup from metadata."""
        return

    # -------------------------------------------------------------------------
    # Helpers
    # -------------------------------------------------------------------------

    def __str__(self) -> str:
        """Return a string representation of a Measurement."""
        return str(self.generate_metadata())

    def __eq__(self, other: object) -> bool:
        """Test instance for equality."""
        if not isinstance(other, Measurement):
            return False
        return (
            self.test_case_id == other.test_case_id
            and self.evidence_metadata == self.evidence_metadata
        )

force_stop = force_stop class-attribute instance-attribute

Attribute to force raising an Exception when error is encountered when running a measurement. If False, will return an Unavailable Evidence type instead with the error details. Default to true.

__call__(*args, **kwargs) abstractmethod

Evaluate a measurement and return a value semantics.

Source code in mlte/measurement/measurement.py 
73
74
75
76
@abstractmethod
def __call__(self, *args, **kwargs) -> Evidence:
    """Evaluate a measurement and return a value semantics."""
    raise NotImplementedError("Cannot evaluate abstract measurement.")

__eq__(other)

Test instance for equality.

Source code in mlte/measurement/measurement.py 
154
155
156
157
158
159
160
161
def __eq__(self, other: object) -> bool:
    """Test instance for equality."""
    if not isinstance(other, Measurement):
        return False
    return (
        self.test_case_id == other.test_case_id
        and self.evidence_metadata == self.evidence_metadata
    )

__init__(test_case_id=None, force_stop=True)

Constructor.

Source code in mlte/measurement/measurement.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
def __init__(
    self, test_case_id: str | None = None, force_stop: bool = True
):
    """Constructor."""

    self.force_stop = force_stop

    # Initialize with no values.
    self.test_case_id: str | None = None
    self.evidence_metadata: EvidenceMetadata | None = None

    if test_case_id is not None:
        # Set our id to the test case id, and generate our evidence metadata.
        self.set_test_case_id(test_case_id)

__str__()

Return a string representation of a Measurement.

Source code in mlte/measurement/measurement.py 
150
151
152
def __str__(self) -> str:
    """Return a string representation of a Measurement."""
    return str(self.generate_metadata())

__subclasshook__(subclass) classmethod

Define the interface for all concrete measurements.

Source code in mlte/measurement/measurement.py 
68
69
70
71
@classmethod
def __subclasshook__(cls, subclass):
    """Define the interface for all concrete measurements."""
    return meta.has_callables(subclass, "__call__")

additional_setup(model)

Optional method to be overriden by subclasses needing additional setup from metadata.

Source code in mlte/measurement/measurement.py 
142
143
144
def additional_setup(self, model: MeasurementMetadata):
    """Optional method to be overriden by subclasses needing additional setup from metadata."""
    return

evaluate(*args, **kwargs)

Evaluate a measurement and return a value with semantics.

Returns:

Type Description
Evidence

The resulting value of measurement execution, with semantics
Source code in mlte/measurement/measurement.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def evaluate(self, *args, **kwargs) -> Evidence:
    """
    Evaluate a measurement and return a value with semantics.

    :return: The resulting value of measurement execution, with semantics
    """
    if self.evidence_metadata is None:
        raise RuntimeError(
            "Can't evaluate measurement before setting its id"
        )

    # Evaluate the measurement.
    try:
        return self.__call__(*args, **kwargs).with_metadata(
            self.evidence_metadata
        )
    except Exception as e:
        # If we want to force stop on error, just re-throw, otherwise return as Unavailable evidence.
        if self.force_stop:
            raise e
        else:
            return Unavailable(
                details=str(e), traceback=traceback.format_exc()
            ).with_metadata(self.evidence_metadata)

from_metadata(model, test_case_id) classmethod

Create a Measurement from a model.

Parameters:

Name Type Description Default
model MeasurementMetadata

The MeasurementMetadata model.

 required
test_case_id str

The id of the associated test case.

 required

Returns:

Type Description
Measurement

The Measurement.
Source code in mlte/measurement/measurement.py 
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
@classmethod
def from_metadata(
    cls, model: MeasurementMetadata, test_case_id: str
) -> Measurement:
    """
    Create a Measurement from a model.

    :param model: The MeasurementMetadata model.
    :param test_case_id: The id of the associated test case.
    :return: The Measurement.
    """
    meas_class: type[Measurement] = typing.cast(
        type[Measurement], load_class_or_function(model.measurement_class)
    )
    measurement = meas_class(test_case_id)
    measurement.additional_setup(model)
    return measurement

generate_metadata()

Returns Measurement metadata.

Source code in mlte/measurement/measurement.py 
117
118
119
120
121
122
def generate_metadata(self) -> MeasurementMetadata:
    """Returns Measurement metadata."""
    return MeasurementMetadata(
        measurement_class=meta.get_qualified_name(self.__class__),
        output_class=meta.get_qualified_name(self.output()),
    )

output() classmethod

Returns the class type object for the Evidence produced by the Measurement.

Source code in mlte/measurement/measurement.py 
107
108
109
110
111
@classmethod
def output(cls) -> type[Evidence]:
    """Returns the class type object for the Evidence produced by the Measurement."""
    # Opaque is the default Evidence type.
    return Opaque

set_metadata()

Resets the internal metadata to associate to Evidences generated by this measurement.

Source code in mlte/measurement/measurement.py 
53
54
55
56
57
58
59
60
61
62
def set_metadata(self):
    """Resets the internal metadata to associate to Evidences generated by this measurement."""
    if self.test_case_id is None:
        raise RuntimeError(
            "Can't generate evidence metadata if test case id has not been set."
        )

    self.evidence_metadata = EvidenceMetadata(
        test_case_id=self.test_case_id, measurement=self.generate_metadata()
    )