Skip to content

external_measurement

Base class for measurements calculated by external functions.

ExternalMeasurement

Bases: Measurement

Base measurement class when the evaluated function is an external function.

Source code in mlte/measurement/external_measurement.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
class ExternalMeasurement(Measurement):
    """Base measurement class when the evaluated function is an external function."""

    EXTERNAL_FUNCTION_KEY = "function"
    """Key to store external function used by this measurement."""

    def __init__(
        self,
        test_case_id: Optional[str] = None,
        output_evidence_type: type[Evidence] = Opaque,
        function: Optional[Callable[..., Any]] = None,
    ):
        """
        Initialize a new ExternalMeasurement measurement.

        :param test_case_id: A unique identifier for the instance
        :param output_evidence_type: The type of the Evidence this measurement will return.
        :param function: The function to be used when evaluating.
        """
        if not issubclass(output_evidence_type, Evidence):
            raise Exception(
                f"Evidence type provided is not a subtype of Evidence: {output_evidence_type}"
            )
        self.output_evidence_type: type = output_evidence_type
        """The output Evidence type that calls to evaluate will return."""

        self.function: Optional[Callable[..., Any]] = function
        """Store the callable function itself."""

        # Call base constructor.
        super().__init__(test_case_id=test_case_id)

    # Overriden.
    def additional_setup(self, model: MeasurementMetadata):
        """Optional method to be overriden by subclasses needing additional setup from metadata."""

        # Set up the output type class.
        self.output_evidence_type = typing.cast(
            type[Evidence], load_class_or_function(model.output_class)
        )

        # Set up the function.
        if self.EXTERNAL_FUNCTION_KEY in model.additional_data:
            self.function = load_class_or_function(
                model.additional_data[self.EXTERNAL_FUNCTION_KEY]
            )

        # Set the metadata.
        self.set_metadata()

    # Overriden.
    def generate_metadata(self) -> MeasurementMetadata:
        """Returns Measurement metadata with additional info."""
        metadata = super().generate_metadata()

        # Override default class with external-measurement specific one.
        metadata.output_class = meta.get_qualified_name(
            self.output_evidence_type
        )

        # Add specific function being used.
        if self.function is not None:
            metadata.additional_data[self.EXTERNAL_FUNCTION_KEY] = (
                meta.get_qualified_name(self.function)
            )
        return metadata

    # Overriden.
    def __call__(self, *args, **kwargs) -> Evidence:
        """Evaluate a measurement and return values without semantics."""
        evidence: Evidence
        if self.function is None:
            # If no function is configured, we just want to wrap the results in the Evidence type.
            evidence = self.output_evidence_type(*args, **kwargs)
        else:
            result = self.function(*args, **kwargs)
            if type(result) is tuple:
                # If we get more than one value from the function, unpack them.
                evidence = self.output_evidence_type(*result)
            else:
                evidence = self.output_evidence_type(result)
        return evidence

    def get_output_type(self) -> type[Evidence]:  # type: ignore
        """Object method with proper results, similar to the class level get_output_type method, which will always return Opaque for this class."""
        return self.output_evidence_type

    def __eq__(self, other: object) -> bool:
        """Test instance for equality."""
        if not isinstance(other, ExternalMeasurement):
            return False
        return (
            self.test_case_id == other.test_case_id
            and self.evidence_metadata == other.evidence_metadata
            and self.output_evidence_type == other.output_evidence_type
            and self.function == other.function
        )

EXTERNAL_FUNCTION_KEY = 'function' class-attribute instance-attribute

Key to store external function used by this measurement.

function = function instance-attribute

Store the callable function itself.

output_evidence_type = output_evidence_type instance-attribute

The output Evidence type that calls to evaluate will return.

__call__(*args, **kwargs)

Evaluate a measurement and return values without semantics.

Source code in mlte/measurement/external_measurement.py 
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def __call__(self, *args, **kwargs) -> Evidence:
    """Evaluate a measurement and return values without semantics."""
    evidence: Evidence
    if self.function is None:
        # If no function is configured, we just want to wrap the results in the Evidence type.
        evidence = self.output_evidence_type(*args, **kwargs)
    else:
        result = self.function(*args, **kwargs)
        if type(result) is tuple:
            # If we get more than one value from the function, unpack them.
            evidence = self.output_evidence_type(*result)
        else:
            evidence = self.output_evidence_type(result)
    return evidence

__eq__(other)

Test instance for equality.

Source code in mlte/measurement/external_measurement.py 
103
104
105
106
107
108
109
110
111
112
def __eq__(self, other: object) -> bool:
    """Test instance for equality."""
    if not isinstance(other, ExternalMeasurement):
        return False
    return (
        self.test_case_id == other.test_case_id
        and self.evidence_metadata == other.evidence_metadata
        and self.output_evidence_type == other.output_evidence_type
        and self.function == other.function
    )

__init__(test_case_id=None, output_evidence_type=Opaque, function=None)

Initialize a new ExternalMeasurement measurement.

Parameters:

Name Type Description Default
test_case_id Optional[str]

A unique identifier for the instance

 None
output_evidence_type type[Evidence]

The type of the Evidence this measurement will return.

 Opaque
function Optional[Callable[..., Any]]

The function to be used when evaluating.

 None
Source code in mlte/measurement/external_measurement.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def __init__(
    self,
    test_case_id: Optional[str] = None,
    output_evidence_type: type[Evidence] = Opaque,
    function: Optional[Callable[..., Any]] = None,
):
    """
    Initialize a new ExternalMeasurement measurement.

    :param test_case_id: A unique identifier for the instance
    :param output_evidence_type: The type of the Evidence this measurement will return.
    :param function: The function to be used when evaluating.
    """
    if not issubclass(output_evidence_type, Evidence):
        raise Exception(
            f"Evidence type provided is not a subtype of Evidence: {output_evidence_type}"
        )
    self.output_evidence_type: type = output_evidence_type
    """The output Evidence type that calls to evaluate will return."""

    self.function: Optional[Callable[..., Any]] = function
    """Store the callable function itself."""

    # Call base constructor.
    super().__init__(test_case_id=test_case_id)

additional_setup(model)

Optional method to be overriden by subclasses needing additional setup from metadata.

Source code in mlte/measurement/external_measurement.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def additional_setup(self, model: MeasurementMetadata):
    """Optional method to be overriden by subclasses needing additional setup from metadata."""

    # Set up the output type class.
    self.output_evidence_type = typing.cast(
        type[Evidence], load_class_or_function(model.output_class)
    )

    # Set up the function.
    if self.EXTERNAL_FUNCTION_KEY in model.additional_data:
        self.function = load_class_or_function(
            model.additional_data[self.EXTERNAL_FUNCTION_KEY]
        )

    # Set the metadata.
    self.set_metadata()

generate_metadata()

Returns Measurement metadata with additional info.

Source code in mlte/measurement/external_measurement.py 
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def generate_metadata(self) -> MeasurementMetadata:
    """Returns Measurement metadata with additional info."""
    metadata = super().generate_metadata()

    # Override default class with external-measurement specific one.
    metadata.output_class = meta.get_qualified_name(
        self.output_evidence_type
    )

    # Add specific function being used.
    if self.function is not None:
        metadata.additional_data[self.EXTERNAL_FUNCTION_KEY] = (
            meta.get_qualified_name(self.function)
        )
    return metadata

get_output_type()

Object method with proper results, similar to the class level get_output_type method, which will always return Opaque for this class.

Source code in mlte/measurement/external_measurement.py 
 99
100
101
def get_output_type(self) -> type[Evidence]:  # type: ignore
    """Object method with proper results, similar to the class level get_output_type method, which will always return Opaque for this class."""
    return self.output_evidence_type