Skip to content

measurement

Superclass for all measurements.

Measurement

The superclass for all model measurements.

Source code in mlte/measurement/measurement.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
class Measurement:
    """
    The superclass for all model measurements.
    """

    def __init__(self, test_case_id: Optional[str] = None):
        """Constructor."""

        # Initialize with no values.
        self.test_case_id: Optional[str] = None
        self.evidence_metadata: Optional[EvidenceMetadata] = None

        if test_case_id is not None:
            # Set our id to the test case id, and generate our evidence metadata.
            self.set_test_case_id(test_case_id)

    def set_test_case_id(self, test_case_id: str):
        # Set the given test case id and update our metadata.

        self.test_case_id = test_case_id
        """The id of the test case this measurement is for."""

        self.set_metadata()
        """Set the evidence metadata."""

    def set_metadata(self):
        """Resets the internal metadata to associate to Evidences generated by this measurement."""
        if self.test_case_id is None:
            raise RuntimeError(
                "Can't generate evidence metadata if test case id has not been set."
            )

        self.evidence_metadata = EvidenceMetadata(
            test_case_id=self.test_case_id, measurement=self.generate_metadata()
        )

    # -------------------------------------------------------------------------
    # Measurement interface definition.
    # -------------------------------------------------------------------------

    @classmethod
    def __subclasshook__(cls, subclass):
        """Define the interface for all concrete measurements."""
        return meta.has_callables(subclass, "__call__")

    @abstractmethod
    def __call__(self, *args, **kwargs) -> Evidence:
        """Evaluate a measurement and return a value semantics."""
        raise NotImplementedError("Cannot evaluate abstract measurement.")

    # -------------------------------------------------------------------------
    # Base methods.
    # -------------------------------------------------------------------------

    def evaluate(self, *args, **kwargs) -> Evidence:
        """
        Evaluate a measurement and return a value with semantics.

        :return: The resulting value of measurement execution, with semantics
        """
        if self.evidence_metadata is None:
            raise RuntimeError(
                "Can't evaluate measurement before setting its id"
            )

        # Evaluate the measurement
        return self.__call__(*args, **kwargs).with_metadata(
            self.evidence_metadata
        )

    @classmethod
    def get_output_type(cls) -> type[Evidence]:
        """Returns the class type object for the Evidence produced by the Measurement."""
        # Opaque is the default Evidence type.
        return Opaque

    # -------------------------------------------------------------------------
    # MeasurementMetadata model handling.
    # -------------------------------------------------------------------------

    def generate_metadata(self) -> MeasurementMetadata:
        """Returns Measurement metadata."""
        return MeasurementMetadata(
            measurement_class=meta.get_qualified_name(self.__class__),
            output_class=meta.get_qualified_name(self.get_output_type()),
        )

    @classmethod
    def from_metadata(
        cls, model: MeasurementMetadata, test_case_id: str
    ) -> Measurement:
        """
        Create a Measurement from a model.

        :param model: The MeasurementMetadata model.
        :param test_case_id: The id of the associated test case.
        :return: The Measurement.
        """
        meas_class: type[Measurement] = typing.cast(
            type[Measurement], load_class_or_function(model.measurement_class)
        )
        measurement = meas_class(test_case_id)
        measurement.additional_setup(model)
        return measurement

    def additional_setup(self, model: MeasurementMetadata):
        """Optional method to be overriden by subclasses needing additional setup from metadata."""
        return

    # -------------------------------------------------------------------------
    # Helpers
    # -------------------------------------------------------------------------

    def __str__(self) -> str:
        """Return a string representation of a Measurement."""
        return str(self.generate_metadata())

    def __eq__(self, other: object) -> bool:
        """Test instance for equality."""
        if not isinstance(other, Measurement):
            return False
        return (
            self.test_case_id == other.test_case_id
            and self.evidence_metadata == self.evidence_metadata
        )

__call__(*args, **kwargs) abstractmethod

Evaluate a measurement and return a value semantics.

Source code in mlte/measurement/measurement.py 
64
65
66
67
@abstractmethod
def __call__(self, *args, **kwargs) -> Evidence:
    """Evaluate a measurement and return a value semantics."""
    raise NotImplementedError("Cannot evaluate abstract measurement.")

__eq__(other)

Test instance for equality.

Source code in mlte/measurement/measurement.py 
136
137
138
139
140
141
142
143
def __eq__(self, other: object) -> bool:
    """Test instance for equality."""
    if not isinstance(other, Measurement):
        return False
    return (
        self.test_case_id == other.test_case_id
        and self.evidence_metadata == self.evidence_metadata
    )

__init__(test_case_id=None)

Constructor.

Source code in mlte/measurement/measurement.py 
24
25
26
27
28
29
30
31
32
33
def __init__(self, test_case_id: Optional[str] = None):
    """Constructor."""

    # Initialize with no values.
    self.test_case_id: Optional[str] = None
    self.evidence_metadata: Optional[EvidenceMetadata] = None

    if test_case_id is not None:
        # Set our id to the test case id, and generate our evidence metadata.
        self.set_test_case_id(test_case_id)

__str__()

Return a string representation of a Measurement.

Source code in mlte/measurement/measurement.py 
132
133
134
def __str__(self) -> str:
    """Return a string representation of a Measurement."""
    return str(self.generate_metadata())

__subclasshook__(subclass) classmethod

Define the interface for all concrete measurements.

Source code in mlte/measurement/measurement.py 
59
60
61
62
@classmethod
def __subclasshook__(cls, subclass):
    """Define the interface for all concrete measurements."""
    return meta.has_callables(subclass, "__call__")

additional_setup(model)

Optional method to be overriden by subclasses needing additional setup from metadata.

Source code in mlte/measurement/measurement.py 
124
125
126
def additional_setup(self, model: MeasurementMetadata):
    """Optional method to be overriden by subclasses needing additional setup from metadata."""
    return

evaluate(*args, **kwargs)

Evaluate a measurement and return a value with semantics.

Returns:

Type Description
Evidence

The resulting value of measurement execution, with semantics
Source code in mlte/measurement/measurement.py 
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def evaluate(self, *args, **kwargs) -> Evidence:
    """
    Evaluate a measurement and return a value with semantics.

    :return: The resulting value of measurement execution, with semantics
    """
    if self.evidence_metadata is None:
        raise RuntimeError(
            "Can't evaluate measurement before setting its id"
        )

    # Evaluate the measurement
    return self.__call__(*args, **kwargs).with_metadata(
        self.evidence_metadata
    )

from_metadata(model, test_case_id) classmethod

Create a Measurement from a model.

Parameters:

Name Type Description Default
model MeasurementMetadata

The MeasurementMetadata model.

 required
test_case_id str

The id of the associated test case.

 required

Returns:

Type Description
Measurement

The Measurement.
Source code in mlte/measurement/measurement.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
@classmethod
def from_metadata(
    cls, model: MeasurementMetadata, test_case_id: str
) -> Measurement:
    """
    Create a Measurement from a model.

    :param model: The MeasurementMetadata model.
    :param test_case_id: The id of the associated test case.
    :return: The Measurement.
    """
    meas_class: type[Measurement] = typing.cast(
        type[Measurement], load_class_or_function(model.measurement_class)
    )
    measurement = meas_class(test_case_id)
    measurement.additional_setup(model)
    return measurement

generate_metadata()

Returns Measurement metadata.

Source code in mlte/measurement/measurement.py 
 99
100
101
102
103
104
def generate_metadata(self) -> MeasurementMetadata:
    """Returns Measurement metadata."""
    return MeasurementMetadata(
        measurement_class=meta.get_qualified_name(self.__class__),
        output_class=meta.get_qualified_name(self.get_output_type()),
    )

get_output_type() classmethod

Returns the class type object for the Evidence produced by the Measurement.

Source code in mlte/measurement/measurement.py 
89
90
91
92
93
@classmethod
def get_output_type(cls) -> type[Evidence]:
    """Returns the class type object for the Evidence produced by the Measurement."""
    # Opaque is the default Evidence type.
    return Opaque

set_metadata()

Resets the internal metadata to associate to Evidences generated by this measurement.

Source code in mlte/measurement/measurement.py 
44
45
46
47
48
49
50
51
52
53
def set_metadata(self):
    """Resets the internal metadata to associate to Evidences generated by this measurement."""
    if self.test_case_id is None:
        raise RuntimeError(
            "Can't generate evidence metadata if test case id has not been set."
        )

    self.evidence_metadata = EvidenceMetadata(
        test_case_id=self.test_case_id, measurement=self.generate_metadata()
    )