"""Deterministic illustrative evaluation release-decision contract. The fixture demonstrates local contract mechanics. Its cases, labels, thresholds, grader evidence, and interval are invented; they are not product measurements, independent assurance, a safety case, or production certification. """ from dataclasses import asdict, dataclass, fields import hashlib import json import math import re IDENTITY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:/@-]{0,127}") SHA256 = re.compile(r"[a-f0-9]{64}") MAX_CASES = 10_000 def identity(name, value): if type(value) is not str or not IDENTITY.fullmatch(value): raise ValueError(f"{name} must be an exact bounded identity") return value def exact_bool(name, value): if type(value) is not bool: raise ValueError(f"{name} must be an exact boolean") return value def integer(name, value, lower, upper): if type(value) is not int or not lower <= value <= upper: raise ValueError(f"{name} must be an exact integer from {lower} to {upper}") return value def ratio(name, value): if type(value) is not float or not math.isfinite(value): raise ValueError(f"{name} must be an exact finite float") if not 0.0 <= value <= 1.0: raise ValueError(f"{name} must be between zero and one") return value def sequence(name, value, lower, upper): if type(value) not in (tuple, list): raise ValueError(f"{name} must be a bounded tuple or list") integer(f"{name} length", len(value), lower, upper) return tuple(value) def identities(name, value, lower, upper): result = sequence(name, value, lower, upper) for item in result: identity(f"{name} item", item) if len(set(result)) != len(result): raise ValueError(f"{name} must not contain duplicates") return result def digest(value): encoded = json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False, ).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def seal(record): if type(record.content_id) is not str: raise ValueError("content_id must be an exact string") data = asdict(record) data.pop("content_id") return digest(data) def validate_record(record, cls): if type(record) is not cls: raise ValueError(f"{cls.__name__} must be a concrete frozen record") try: rebuilt = cls( **{field.name: getattr(record, field.name) for field in fields(cls)} ) except (AttributeError, TypeError) as error: raise ValueError(f"malformed {cls.__name__}") from error if record != rebuilt or record.content_id != rebuilt.content_id: raise ValueError(f"noncanonical or modified {cls.__name__}") return rebuilt def scope_tuple(value): result = identities("scope", value, 8, 8) return result @dataclass(frozen=True) class EvaluationContract: scope: tuple = ( "release-policy-v1", "candidate-system-v1", "baseline-system-v1", "dataset-v1", "rubric-v1", "grader-policy-v1", "interval-policy-v1", "escalation-policy-v1", ) required_slices: tuple = ("core", "long-tail", "policy-boundary") allowed_graders: tuple = ("deterministic-check-v1", "human-panel-v1") minimum_cases: int = 8 maximum_cases: int = 256 minimum_slice_support: int = 2 minimum_overall_pass_rate: float = 0.85 minimum_slice_pass_rate: float = 0.75 maximum_regression_rate: float = 0.05 maximum_abstention_rate: float = 0.10 maximum_interval_width: float = 0.20 minimum_grader_human_agreement: float = 0.85 minimum_calibration_cases: int = 20 require_critical_pass: bool = True content_id: str = "" def __post_init__(self): object.__setattr__(self, "scope", scope_tuple(self.scope)) object.__setattr__( self, "required_slices", identities("required_slices", self.required_slices, 1, 32), ) object.__setattr__( self, "allowed_graders", identities("allowed_graders", self.allowed_graders, 1, 32), ) integer("minimum_cases", self.minimum_cases, 1, MAX_CASES) integer("maximum_cases", self.maximum_cases, 1, MAX_CASES) if self.minimum_cases > self.maximum_cases: raise ValueError("minimum_cases must not exceed maximum_cases") integer( "minimum_slice_support", self.minimum_slice_support, 1, self.maximum_cases, ) for name in ( "minimum_overall_pass_rate", "minimum_slice_pass_rate", "maximum_regression_rate", "maximum_abstention_rate", "maximum_interval_width", "minimum_grader_human_agreement", ): ratio(name, getattr(self, name)) integer( "minimum_calibration_cases", self.minimum_calibration_cases, 1, MAX_CASES, ) if exact_bool("require_critical_pass", self.require_critical_pass) is not True: raise ValueError("critical cases must fail closed") expected = seal(self) if self.content_id and self.content_id != expected: raise ValueError("evaluation contract digest mismatch") object.__setattr__(self, "content_id", expected) @dataclass(frozen=True) class CaseOutcome: scope: tuple evaluation_id: str case_id: str slice_id: str grader_id: str critical: bool candidate_verdict: str baseline_verdict: str content_id: str = "" def __post_init__(self): object.__setattr__(self, "scope", scope_tuple(self.scope)) for name in ("evaluation_id", "case_id", "slice_id", "grader_id"): identity(name, getattr(self, name)) exact_bool("critical", self.critical) if type(self.candidate_verdict) is not str or self.candidate_verdict not in ( "pass", "fail", "abstain", ): raise ValueError("candidate_verdict must be pass, fail, or abstain") if type(self.baseline_verdict) is not str or self.baseline_verdict not in ( "pass", "fail", ): raise ValueError("baseline_verdict must be pass or fail") expected = seal(self) if self.content_id and self.content_id != expected: raise ValueError("case outcome digest mismatch") object.__setattr__(self, "content_id", expected) @dataclass(frozen=True) class EvaluationEvidence: scope: tuple evaluation_id: str contract_content_id: str evidence_version: str interval_method: str cases: tuple pass_rate_interval_low: float pass_rate_interval_high: float grader_human_agreement: float calibration_cases: int content_id: str = "" def __post_init__(self): object.__setattr__(self, "scope", scope_tuple(self.scope)) for name in ( "evaluation_id", "evidence_version", "interval_method", ): identity(name, getattr(self, name)) if type(self.contract_content_id) is not str or not SHA256.fullmatch( self.contract_content_id ): raise ValueError("contract_content_id must be an exact lowercase SHA-256") cases = sequence("cases", self.cases, 1, MAX_CASES) object.__setattr__( self, "cases", tuple(validate_record(case, CaseOutcome) for case in cases), ) low = ratio("pass_rate_interval_low", self.pass_rate_interval_low) high = ratio("pass_rate_interval_high", self.pass_rate_interval_high) if low > high: raise ValueError("pass-rate interval bounds are reversed") ratio("grader_human_agreement", self.grader_human_agreement) integer("calibration_cases", self.calibration_cases, 0, MAX_CASES) expected = seal(self) if self.content_id and self.content_id != expected: raise ValueError("evaluation evidence digest mismatch") object.__setattr__(self, "content_id", expected) @dataclass(frozen=True) class ReleaseReport: decision: str first_reason: str action: str case_count: int passes: int abstentions: int regressions: int pass_rate: float interval: tuple evidence_id: str claim: str = "LOCAL_EVAL_GATE_NOT_PRODUCTION_CERTIFICATION" def evaluate_release(contract, evidence): """Apply a fixed-order release gate to scope-bound evaluation evidence.""" contract = validate_record(contract, EvaluationContract) evidence = validate_record(evidence, EvaluationEvidence) if evidence.scope != contract.scope: raise ValueError("evidence scope does not match the release contract") if evidence.contract_content_id != contract.content_id: raise ValueError("evidence belongs to another release contract") if evidence.interval_method != "two-sided-score-interval-v1": raise ValueError("unsupported uncertainty interval method") cases = evidence.cases if not contract.minimum_cases <= len(cases) <= contract.maximum_cases: raise ValueError("case count is outside the release contract") if len({case.case_id for case in cases}) != len(cases): raise ValueError("case identities must be unique") if len({case.content_id for case in cases}) != len(cases): raise ValueError("case evidence must be unique") for case in cases: if case.scope != contract.scope or case.evaluation_id != evidence.evaluation_id: raise ValueError("case outcome is outside the evaluation scope") if case.slice_id not in contract.required_slices: raise ValueError("case outcome names an undeclared slice") if case.grader_id not in contract.allowed_graders: raise ValueError("case outcome names an unauthorized grader") by_slice = { slice_id: tuple(case for case in cases if case.slice_id == slice_id) for slice_id in contract.required_slices } if any(len(slice_cases) < contract.minimum_slice_support for slice_cases in by_slice.values()): raise ValueError("required slice has insufficient support") passes = sum(case.candidate_verdict == "pass" for case in cases) abstentions = sum(case.candidate_verdict == "abstain" for case in cases) regressions = sum( case.baseline_verdict == "pass" and case.candidate_verdict != "pass" for case in cases ) pass_rate = passes / len(cases) abstention_rate = abstentions / len(cases) regression_rate = regressions / len(cases) low = evidence.pass_rate_interval_low high = evidence.pass_rate_interval_high if not low <= pass_rate <= high: raise ValueError("declared pass-rate interval does not contain the observed rate") reason = "none" action = "accountable-human-review" decision = "ELIGIBLE_FOR_RELEASE_REVIEW" critical_failure = any( case.critical and case.candidate_verdict != "pass" for case in cases ) slice_failure = any( sum(case.candidate_verdict == "pass" for case in slice_cases) / len(slice_cases) < contract.minimum_slice_pass_rate for slice_cases in by_slice.values() ) if critical_failure: decision, reason, action = "HOLD", "critical-case", "block-release" elif evidence.calibration_cases < contract.minimum_calibration_cases: decision, reason, action = "ESCALATE", "calibration-support", "human-review" elif evidence.grader_human_agreement < contract.minimum_grader_human_agreement: decision, reason, action = "ESCALATE", "grader-disagreement", "human-review" elif high - low > contract.maximum_interval_width: decision, reason, action = "ESCALATE", "wide-uncertainty", "collect-more-evidence" elif abstention_rate > contract.maximum_abstention_rate: decision, reason, action = "ESCALATE", "grader-abstention", "human-review" elif low < contract.minimum_overall_pass_rate: decision, reason, action = "HOLD", "overall-quality", "block-release" elif slice_failure: decision, reason, action = "HOLD", "slice-quality", "block-release" elif regression_rate > contract.maximum_regression_rate: decision, reason, action = "HOLD", "regression-rate", "block-release" evidence_id = digest( { "contract": contract.content_id, "evidence": evidence.content_id, "decision": decision, "reason": reason, "counts": [len(cases), passes, abstentions, regressions], } ) return ReleaseReport( decision, reason, action, len(cases), passes, abstentions, regressions, pass_rate, (low, high), evidence_id, ) def illustrative_fixture(): contract = EvaluationContract() evaluation_id = "eval-2026-09-22" layout = ( ("case-01", "core", "deterministic-check-v1", True, "pass"), ("case-02", "core", "human-panel-v1", False, "pass"), ("case-03", "core", "deterministic-check-v1", False, "pass"), ("case-04", "long-tail", "human-panel-v1", False, "pass"), ("case-05", "long-tail", "deterministic-check-v1", False, "pass"), ("case-06", "policy-boundary", "human-panel-v1", True, "pass"), ("case-07", "policy-boundary", "deterministic-check-v1", True, "pass"), ("case-08", "policy-boundary", "human-panel-v1", False, "pass"), ) cases = tuple( CaseOutcome( contract.scope, evaluation_id, case_id, slice_id, grader_id, critical, verdict, "pass" if case_id != "case-05" else "fail", ) for case_id, slice_id, grader_id, critical, verdict in layout ) evidence = EvaluationEvidence( contract.scope, evaluation_id, contract.content_id, "evidence-v1", "two-sided-score-interval-v1", cases, 0.88, 1.0, 0.92, 40, ) return contract, evidence def main(): report = evaluate_release(*illustrative_fixture()) print("example=illustrative_only") print(f"decision={report.decision}") print(f"first_reason={report.first_reason};action={report.action}") print( f"cases={report.case_count};passes={report.passes};" f"abstentions={report.abstentions};regressions={report.regressions}" ) print( f"pass_rate={report.pass_rate:.3f};" f"interval=[{report.interval[0]:.3f},{report.interval[1]:.3f}]" ) print(f"claim={report.claim}") if __name__ == "__main__": main()