"""A fail-closed release measurement contract for an AI capability. The executable example values and evidence references are explicitly illustrative. They are fixtures for testing contract mechanics, not reported product results, recommended gates, or human-reviewed evidence. """ from __future__ import annotations from dataclasses import asdict, dataclass from hashlib import sha256 import json from math import isfinite import re from typing import Literal, Sequence MetricCategory = Literal["offline", "online", "safety", "latency", "cost"] Comparison = Literal["at_least", "at_most"] REQUIRED_CATEGORIES = frozenset( {"offline", "online", "safety", "latency", "cost"} ) SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") def _required_text(name: str, value: object) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"{name} must be non-empty text") return value def _strict_finite_number(name: str, value: object) -> float: if isinstance(value, bool) or type(value) not in {int, float}: raise ValueError(f"{name} must be a non-boolean number") numeric = float(value) if not isfinite(numeric): raise ValueError(f"{name} must be finite") return numeric def _stable_digest(payload: object) -> str: encoded = json.dumps( payload, allow_nan=False, ensure_ascii=True, separators=(",", ":"), sort_keys=True, ).encode("utf-8") return sha256(encoded).hexdigest() def _require_digest(name: str, value: object) -> str: if not isinstance(value, str) or not SHA256_PATTERN.fullmatch(value): raise ValueError(f"{name} must be a lowercase SHA-256 digest") return value @dataclass(frozen=True) class MetricGate: name: str category: MetricCategory unit: str comparison: Comparison threshold: float slice: str window: str data_source: str denominator: str failure_action: str definition_version: str evidence_version: str evidence_provenance: str def __post_init__(self) -> None: for field in ( "name", "unit", "slice", "window", "data_source", "denominator", "failure_action", "definition_version", "evidence_version", "evidence_provenance", ): _required_text(field, getattr(self, field)) if self.category not in REQUIRED_CATEGORIES: raise ValueError(f"unknown metric category: {self.category}") if self.comparison not in {"at_least", "at_most"}: raise ValueError(f"unknown comparison: {self.comparison}") _strict_finite_number("threshold", self.threshold) @property def definition_digest(self) -> str: return _stable_digest(asdict(self)) def passes(self, observed: float) -> bool: numeric = _strict_finite_number(f"observation for {self.name}", observed) if self.comparison == "at_least": return numeric >= self.threshold return numeric <= self.threshold @property def symbol(self) -> str: return ">=" if self.comparison == "at_least" else "<=" @dataclass(frozen=True) class MetricObservation: metric_name: str value: float slice: str window: str data_source: str denominator: str failure_action: str definition_version: str definition_digest: str contract_version: str contract_digest: str evidence_version: str evidence_provenance: str evidence_manifest: str evidence_digest: str def __post_init__(self) -> None: for field in ( "metric_name", "slice", "window", "data_source", "denominator", "failure_action", "definition_version", "contract_version", "evidence_version", "evidence_provenance", "evidence_manifest", ): _required_text(field, getattr(self, field)) _strict_finite_number(f"observation for {self.metric_name}", self.value) _require_digest("definition_digest", self.definition_digest) _require_digest("contract_digest", self.contract_digest) _require_digest("evidence_digest", self.evidence_digest) expected_evidence_digest = sha256( self.evidence_manifest.encode("utf-8") ).hexdigest() if self.evidence_digest != expected_evidence_digest: raise ValueError("evidence_digest does not match evidence_manifest") @dataclass(frozen=True) class GateResult: gate: MetricGate observation: MetricObservation passed: bool @dataclass(frozen=True) class MeasurementContract: contract_version: str product_claim: str population: str decision_owner: str gates: tuple[MetricGate, ...] def __post_init__(self) -> None: try: gates = tuple(self.gates) except TypeError as error: raise ValueError("gates must be an iterable of MetricGate records") from error if any(not isinstance(gate, MetricGate) for gate in gates): raise ValueError("gates must contain only MetricGate records") # frozen=True does not copy caller-owned containers. Normalize before the # digest can observe later list mutation. object.__setattr__(self, "gates", gates) for field in ( "contract_version", "product_claim", "population", "decision_owner", ): _required_text(field, getattr(self, field)) names = [gate.name for gate in self.gates] if len(names) != len(set(names)): raise ValueError("metric names must be unique") categories = {gate.category for gate in self.gates} missing = REQUIRED_CATEGORIES - categories if missing: raise ValueError( "measurement contract is missing categories: " + ", ".join(sorted(missing)) ) @property def contract_digest(self) -> str: return _stable_digest( { "contract_version": self.contract_version, "decision_owner": self.decision_owner, "gates": [gate.definition_digest for gate in self.gates], "population": self.population, "product_claim": self.product_claim, } ) def evaluate( self, observations: Sequence[MetricObservation], ) -> tuple[GateResult, ...]: if not isinstance(observations, (list, tuple)) or any( not isinstance(observation, MetricObservation) for observation in observations ): raise TypeError( "observations must be a sequence of MetricObservation records; " "bare name-to-value mappings are not accepted" ) observed_by_name: dict[str, MetricObservation] = {} for observation in observations: if observation.metric_name in observed_by_name: raise ValueError( f"duplicate observation: {observation.metric_name}" ) observed_by_name[observation.metric_name] = observation expected = {gate.name for gate in self.gates} received = set(observed_by_name) missing = expected - received unexpected = received - expected if missing or unexpected: details = [] if missing: details.append("missing=" + ",".join(sorted(missing))) if unexpected: details.append("unexpected=" + ",".join(sorted(unexpected))) raise ValueError("observation schema mismatch: " + "; ".join(details)) results = [] for gate in self.gates: observation = observed_by_name[gate.name] expected_scope = { "slice": gate.slice, "window": gate.window, "data_source": gate.data_source, "denominator": gate.denominator, "failure_action": gate.failure_action, "definition_version": gate.definition_version, "definition_digest": gate.definition_digest, "contract_version": self.contract_version, "contract_digest": self.contract_digest, "evidence_version": gate.evidence_version, "evidence_provenance": gate.evidence_provenance, } mismatches = [ field for field, expected_value in expected_scope.items() if getattr(observation, field) != expected_value ] if mismatches: raise ValueError( f"scope/provenance mismatch for {gate.name}: " + ", ".join(mismatches) ) results.append( GateResult(gate, observation, gate.passes(observation.value)) ) return tuple(results) def decision( self, observations: Sequence[MetricObservation], ) -> Literal["SHIP", "HOLD"]: return ( "SHIP" if all(result.passed for result in self.evaluate(observations)) else "HOLD" ) def failed_actions( self, observations: Sequence[MetricObservation], ) -> tuple[str, ...]: return tuple( result.gate.failure_action for result in self.evaluate(observations) if not result.passed ) # All values and evidence references below are illustrative fixtures only. ILLUSTRATIVE_CONTRACT = MeasurementContract( contract_version="measurement-contract-v1", product_claim="Help a user complete a reviewed task with acceptable risk and service cost.", population="Illustrative eligible requests in a controlled release.", decision_owner="Illustrative product and engineering review.", gates=( MetricGate( "offline_task_score", "offline", "ratio", "at_least", 0.82, "eligible_cases/all", "illustrative_case_set_v1", "illustrative_eval_store.offline_cases", "reviewed_eligible_cases", "hold_release_and_investigate_offline_quality", "1", "illustrative_offline_evidence_v1", "illustrative://academy/measurement/offline", ), MetricGate( "online_completion_delta", "online", "percentage_points", "at_least", 1.0, "eligible_requests/all", "illustrative_randomized_window_v1", "illustrative_experiment_store.assignments", "randomized_eligible_users", "hold_release_and_investigate_online_value", "1", "illustrative_online_evidence_v1", "illustrative://academy/measurement/online", ), MetricGate( "unsafe_action_rate", "safety", "ratio", "at_most", 0.01, "eligible_requests/all", "illustrative_canary_window_v1", "illustrative_safety_store.reviewed_actions", "reviewed_eligible_actions", "stop_release_and_escalate_safety_review", "1", "illustrative_safety_evidence_v1", "illustrative://academy/measurement/safety", ), MetricGate( "p95_latency_ms", "latency", "milliseconds", "at_most", 900.0, "eligible_requests/all", "illustrative_canary_window_v1", "illustrative_client_telemetry.completed_requests", "completed_eligible_requests", "hold_release_and_investigate_latency", "1", "illustrative_latency_evidence_v1", "illustrative://academy/measurement/latency", ), MetricGate( "cost_per_completion_usd", "cost", "usd", "at_most", 0.06, "eligible_requests/all", "illustrative_canary_window_v1", "illustrative_cost_store.attributed_usage", "reviewed_successful_completions", "hold_release_and_investigate_unit_cost", "1", "illustrative_cost_evidence_v1", "illustrative://academy/measurement/cost", ), ), ) def _illustrative_observation( gate: MetricGate, value: float, evidence_payload: str, ) -> MetricObservation: return MetricObservation( metric_name=gate.name, value=value, slice=gate.slice, window=gate.window, data_source=gate.data_source, denominator=gate.denominator, failure_action=gate.failure_action, definition_version=gate.definition_version, definition_digest=gate.definition_digest, contract_version=ILLUSTRATIVE_CONTRACT.contract_version, contract_digest=ILLUSTRATIVE_CONTRACT.contract_digest, evidence_version=gate.evidence_version, evidence_provenance=gate.evidence_provenance, evidence_manifest=evidence_payload, evidence_digest=sha256(evidence_payload.encode("utf-8")).hexdigest(), ) ILLUSTRATIVE_VALUES = (0.84, 1.3, 0.008, 940.0, 0.054) ILLUSTRATIVE_OBSERVATIONS = tuple( _illustrative_observation( gate, value, f"illustrative-evidence-payload:{gate.name}:{value}", ) for gate, value in zip( ILLUSTRATIVE_CONTRACT.gates, ILLUSTRATIVE_VALUES, ) ) def format_example() -> str: lines = [ "example=illustrative_only", f"contract={ILLUSTRATIVE_CONTRACT.contract_version}", ] for result in ILLUSTRATIVE_CONTRACT.evaluate(ILLUSTRATIVE_OBSERVATIONS): status = "PASS" if result.passed else "FAIL" lines.append( f"{result.gate.category}:{result.gate.name}={status} " f"({result.observation.value:.3f} {result.gate.symbol} " f"{result.gate.threshold:.3f}) " f"evidence={result.observation.evidence_version}" ) lines.append( f"decision={ILLUSTRATIVE_CONTRACT.decision(ILLUSTRATIVE_OBSERVATIONS)}" ) lines.append( "failure_actions=" + ",".join( ILLUSTRATIVE_CONTRACT.failed_actions(ILLUSTRATIVE_OBSERVATIONS) ) ) return "\n".join(lines) if __name__ == "__main__": print(format_example())