Skip to content

artifact

mlte/report/artifact.py

Artifact implementation for MLTE report.

Report

Bases: Artifact

The report artifact contains the results of MLTE model evaluation.

Source code in mlte/report/artifact.py 
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
class Report(Artifact):
    """The report artifact contains the results of MLTE model evaluation."""

    def __init__(
        self,
        identifier: str = DEFAULT_REPORT_ID,
        system: SystemDescriptor = SystemDescriptor(),
        model: ModelDescriptor = ModelDescriptor(),
        data: List[DataDescriptor] = [],
        system_requirements: List[QASDescriptor] = [],
        validated_spec_id: Optional[str] = None,
        comments: List[CommentDescriptor] = [],
        quantitative_analysis: QuantitiveAnalysisDescriptor = QuantitiveAnalysisDescriptor(),
    ) -> None:
        super().__init__(identifier, ArtifactType.REPORT)

        self.system = system
        """A system requirements."""

        self.data = data
        """A description of the data used during model evaluation."""

        self.model = model
        """The intended use of the model under evaluation."""

        self.system_requirements = system_requirements
        """A description of the system requirements."""

        self.validated_spec_id = validated_spec_id
        """A summary of model performance evaluation."""

        self.comments = comments
        """A collection of comments for the report."""

        self.quantitative_analysis = quantitative_analysis
        """The quantitative analysis for the evaluation."""

    def to_model(self) -> ArtifactModel:
        """Convert a report artifact to its corresponding model."""
        return ArtifactModel(
            header=self.build_artifact_header(),
            body=ReportModel(
                nc_data=NegotiationCardDataModel(
                    system=self.system,
                    data=self.data,
                    model=self.model,
                    system_requirements=self.system_requirements,
                ),
                validated_spec_id=self.validated_spec_id,
                comments=self.comments,
                quantitative_analysis=self.quantitative_analysis,
            ),
        )

    def pre_save_hook(self, context: Context, store: ArtifactStore) -> None:
        """
        Override Artifact.pre_save_hook(). Loads the associated ValidatedSpec details.
        :param context: The context in which to save the artifact
        :param store: The store in which to save the artifact
        :raises RuntimeError: On broken invariant
        """
        if self.validated_spec_id is None:
            return

        with ManagedArtifactSession(store.session()) as handle:
            try:
                artifact = handle.read_artifact(
                    context.model,
                    context.version,
                    self.validated_spec_id,
                )
            except errors.ErrorNotFound:
                raise RuntimeError(
                    f"Validated specification with identifier {self.validated_spec_id} not found."
                )

        if not artifact.header.type == ArtifactType.VALIDATED_SPEC:
            raise RuntimeError(
                f"Validated specification with identifier {self.validated_spec_id} not found."
            )

    def post_load_hook(self, context: Context, store: ArtifactStore) -> None:
        """
        Override Artifact.post_load_hook().
        :param context: The context in which to save the artifact
        :param store: The store in which to save the artifact
        :raises RuntimeError: On broken invariant
        """
        super().post_load_hook(context, store)

    @classmethod
    def from_model(cls, model: ArtifactModel) -> Report:
        """Convert a report model to its corresponding artifact."""
        assert model.header.type == ArtifactType.REPORT, "Broken precondition."
        body = typing.cast(ReportModel, model.body)
        return Report(
            identifier=model.header.identifier,
            system=body.nc_data.system,
            data=body.nc_data.data,
            model=body.nc_data.model,
            system_requirements=body.nc_data.system_requirements,
            validated_spec_id=body.validated_spec_id,
            comments=body.comments,
            quantitative_analysis=body.quantitative_analysis,
        )

    def populate_from(self, artifact: NegotiationCard) -> Report:
        """
        Populate the contents of a report from a negotiation card.
        :param artifact: The artifact to populate from
        :return: The new report artifact with fields populated
        """
        return Report(
            identifier=self.identifier,
            system=deepcopy(artifact.system),
            data=deepcopy(artifact.data),
            model=deepcopy(artifact.model),
            system_requirements=deepcopy(artifact.qas),
            validated_spec_id=self.validated_spec_id,
            comments=deepcopy(self.comments),
            quantitative_analysis=deepcopy(self.quantitative_analysis),
        )

    @staticmethod
    def get_default_id() -> str:
        """Get the default identifier for the artifact."""
        return DEFAULT_REPORT_ID

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, Report):
            return False
        return self._equal(other)

comments = comments instance-attribute

A collection of comments for the report.

data = data instance-attribute

A description of the data used during model evaluation.

model = model instance-attribute

The intended use of the model under evaluation.

quantitative_analysis = quantitative_analysis instance-attribute

The quantitative analysis for the evaluation.

system = system instance-attribute

A system requirements.

system_requirements = system_requirements instance-attribute

A description of the system requirements.

validated_spec_id = validated_spec_id instance-attribute

A summary of model performance evaluation.

from_model(model) classmethod

Convert a report model to its corresponding artifact.

Source code in mlte/report/artifact.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
@classmethod
def from_model(cls, model: ArtifactModel) -> Report:
    """Convert a report model to its corresponding artifact."""
    assert model.header.type == ArtifactType.REPORT, "Broken precondition."
    body = typing.cast(ReportModel, model.body)
    return Report(
        identifier=model.header.identifier,
        system=body.nc_data.system,
        data=body.nc_data.data,
        model=body.nc_data.model,
        system_requirements=body.nc_data.system_requirements,
        validated_spec_id=body.validated_spec_id,
        comments=body.comments,
        quantitative_analysis=body.quantitative_analysis,
    )

get_default_id() staticmethod

Get the default identifier for the artifact.

Source code in mlte/report/artifact.py 
159
160
161
162
@staticmethod
def get_default_id() -> str:
    """Get the default identifier for the artifact."""
    return DEFAULT_REPORT_ID

populate_from(artifact)

Populate the contents of a report from a negotiation card.

Parameters:

Name Type Description Default
artifact NegotiationCard

The artifact to populate from

 required

Returns:

Type Description
Report

The new report artifact with fields populated
Source code in mlte/report/artifact.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def populate_from(self, artifact: NegotiationCard) -> Report:
    """
    Populate the contents of a report from a negotiation card.
    :param artifact: The artifact to populate from
    :return: The new report artifact with fields populated
    """
    return Report(
        identifier=self.identifier,
        system=deepcopy(artifact.system),
        data=deepcopy(artifact.data),
        model=deepcopy(artifact.model),
        system_requirements=deepcopy(artifact.qas),
        validated_spec_id=self.validated_spec_id,
        comments=deepcopy(self.comments),
        quantitative_analysis=deepcopy(self.quantitative_analysis),
    )

post_load_hook(context, store)

Override Artifact.post_load_hook().

Parameters:

Name Type Description Default
context Context

The context in which to save the artifact

 required
store ArtifactStore

The store in which to save the artifact

 required

Raises:

Type Description
RuntimeError

On broken invariant
Source code in mlte/report/artifact.py 
117
118
119
120
121
122
123
124
def post_load_hook(self, context: Context, store: ArtifactStore) -> None:
    """
    Override Artifact.post_load_hook().
    :param context: The context in which to save the artifact
    :param store: The store in which to save the artifact
    :raises RuntimeError: On broken invariant
    """
    super().post_load_hook(context, store)

pre_save_hook(context, store)

Override Artifact.pre_save_hook(). Loads the associated ValidatedSpec details.

Parameters:

Name Type Description Default
context Context

The context in which to save the artifact

 required
store ArtifactStore

The store in which to save the artifact

 required

Raises:

Type Description
RuntimeError

On broken invariant
Source code in mlte/report/artifact.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def pre_save_hook(self, context: Context, store: ArtifactStore) -> None:
    """
    Override Artifact.pre_save_hook(). Loads the associated ValidatedSpec details.
    :param context: The context in which to save the artifact
    :param store: The store in which to save the artifact
    :raises RuntimeError: On broken invariant
    """
    if self.validated_spec_id is None:
        return

    with ManagedArtifactSession(store.session()) as handle:
        try:
            artifact = handle.read_artifact(
                context.model,
                context.version,
                self.validated_spec_id,
            )
        except errors.ErrorNotFound:
            raise RuntimeError(
                f"Validated specification with identifier {self.validated_spec_id} not found."
            )

    if not artifact.header.type == ArtifactType.VALIDATED_SPEC:
        raise RuntimeError(
            f"Validated specification with identifier {self.validated_spec_id} not found."
        )

to_model()

Convert a report artifact to its corresponding model.

Source code in mlte/report/artifact.py 
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def to_model(self) -> ArtifactModel:
    """Convert a report artifact to its corresponding model."""
    return ArtifactModel(
        header=self.build_artifact_header(),
        body=ReportModel(
            nc_data=NegotiationCardDataModel(
                system=self.system,
                data=self.data,
                model=self.model,
                system_requirements=self.system_requirements,
            ),
            validated_spec_id=self.validated_spec_id,
            comments=self.comments,
            quantitative_analysis=self.quantitative_analysis,
        ),
    )