#!/usr/bin/env python3 """Inspect recorded OpenAuditor experiment results with Python's standard library. This checks a recorded case manifest and independently calculates population rates from the supplied patterns. It does not import, execute, or reimplement the private engine. Recorded engine verdicts are inputs to the calculations, not independently reproduced engine outputs. No network or model is used. """ from __future__ import annotations import argparse from collections import Counter, defaultdict from decimal import Decimal, ROUND_CEILING, localcontext from fractions import Fraction import json from pathlib import Path REVISION = "6205721f91352039de5ce861c1c94d15d9903c7d" # A manifest of observed outcomes, not an implementation of the verdict gate. CASE_MANIFEST = { "two_holds_each_0.002": ("true", None), "three_holds_each_0.002": ("uncertain", "combined_bound"), "same_three_one_fails_0.002": ("false", None), "one_qualified_failure_with_large_other_bounds": ("false", None), "hold_sum_exactly_epsilon": ("true", None), "failure_bound_exactly_epsilon": ("false", None), "failure_bound_above_epsilon": ("uncertain", "fails"), "failure_missing_bound": ("uncertain", "fails"), "third_assertion_not_applicable": ("true", None), "third_assertion_caller_skipped": ("true", None), "all_out_of_scope": ("uncertain", "nothing_applicable"), "empty_results": ("uncertain", "nothing_applicable"), "hundred_deterministic_holds": ("true", None), "holds_at_guard": ("uncertain", "guard"), "qualified_failure_at_guard": ("false", None), "deterministic_failure_overrides_missing": ("false", None), "internal_only_hold_missing_bound": ("true", None), } for unresolved in ( "insufficient_evidence", "abstained", "not_asked", "model_unavailable", "context_too_long", "unqualified", ): CASE_MANIFEST[f"holds_plus_{unresolved}"] = ("uncertain", unresolved) CASE_MANIFEST[f"qualified_failure_plus_{unresolved}"] = ("false", None) def require(condition, message): if not condition: raise ValueError(message) def decimal_ratio(numerator, denominator): """Calculate an exact rational, then format at the recording's precision.""" ratio = Fraction(numerator, denominator) with localcontext() as context: context.prec = 28 return str(Decimal(ratio.numerator) / Decimal(ratio.denominator)) def inspect_cases(record): cases = record["cases"] require(len(cases) == record["case_count"] == len(CASE_MANIFEST) == 29, "Recorded case count differs from the 29-case manifest.") by_name = {case["case"]: case for case in cases} require(len(by_name) == len(cases) and set(by_name) == set(CASE_MANIFEST), "Case names are missing, duplicated, or unexpected.") for name, (verdict, reason) in CASE_MANIFEST.items(): case = by_name[name] require(case["verdict"] == verdict, f"Unexpected recorded verdict: {name}") reasons = [item["reason"] for item in case["reasons"]] require(reasons == ([] if reason is None else [reason]), f"Unexpected recorded reasons: {name}") statuses = Counter(item["outcome"] for item in case["assertions"]) require(dict(statuses) == case["status_counts"], f"Outcome counts: {name}") require(sum(statuses.values()) == case["input_count"], f"Input count: {name}") return len(cases) def inspect_population(name, population): """Recount the supplied patterns; gate_verdict is a recorded input label.""" totals = Counter() wrong = Counter() regions = defaultdict(lambda: [0, 0]) population_size = 0 widths = set() for pattern in population["patterns"]: count = pattern["count"] require(type(count) is int and count > 0, f"Invalid count in {name}") predictions, truths = pattern["predictions"], pattern["truths"] require(len(predictions) == len(truths) > 0, f"Pattern dimensions: {name}") widths.add(len(truths)) require(set(truths) <= {"holds", "fails"}, f"Invalid truth label: {name}") require(set(predictions) <= {"holds", "fails", "abstained"}, f"Invalid prediction label: {name}") truth_verdict = "true" if all(value == "holds" for value in truths) else "false" require(pattern["expected_verdict"] == truth_verdict, f"Truth verdict: {name}") recorded_verdict = pattern["gate_verdict"] require(recorded_verdict in {"true", "false", "uncertain"}, f"Invalid recorded verdict: {name}") population_size += count totals[recorded_verdict] += count if recorded_verdict != "uncertain" and recorded_verdict != truth_verdict: wrong[recorded_verdict] += count for index, (prediction, truth) in enumerate(zip(predictions, truths), start=1): if prediction == "abstained": continue region = regions[(f"q{index}", prediction)] region[0] += count region[1] += count if prediction != truth else 0 require(len(widths) == 1, f"Inconsistent question count: {name}") computed_regions = defaultdict(dict) for (question, prediction), (count, errors) in regions.items(): computed_regions[question][prediction] = { "count": count, "errors": errors, "exact_rate_in_constructed_population": decimal_ratio(errors, count), } rates = {verdict: decimal_ratio(wrong[verdict], count) for verdict, count in totals.items() if verdict != "uncertain"} calculated = { "total": population_size, "question_regions": dict(computed_regions), "verdict_counts": dict(totals), "wrong_verdict_counts": {verdict: count for verdict, count in wrong.items() if count}, "decided_coverage": decimal_ratio(population_size - totals["uncertain"], population_size), "conditional_error_by_decided_verdict": rates, } for field, value in calculated.items(): require(value == population[field], f"Recomputed {name}.{field} differs from the record.") return calculated def inspect_zero_error_bound(record): observed = record["published_certainty_lookup_demonstration"]["certainty"] require(observed["errors"] == 0 and observed["measured_on"] == 450, "The published zero-error illustration must contain 0 errors in 450 observations.") # For zero errors, solve (1 - p)**n = alpha. This is the one-sided # 95% binomial upper confidence limit; no engine implementation is used. with localcontext() as context: context.prec = 40 upper = 1 - Decimal("0.05") ** (Decimal(1) / Decimal(450)) rounded = upper.quantize(Decimal("0.00001"), rounding=ROUND_CEILING) display = (1 - rounded).quantize(Decimal("0.0001")) require(str(rounded) == observed["bound"] == "0.00664", "Upper-bound calculation differs.") require(str(display) == observed["value"] == "0.9934", "Displayed certainty differs.") return {"errors": 0, "observations": 450, "confidence_level": "0.95", "upper_error_bound_rounded_up": str(rounded), "displayed_certainty": str(display)} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("results", type=Path, nargs="?", default=Path(__file__).with_name("gate_asymmetry.json")) args = parser.parse_args() record = json.loads(args.results.read_text()) require(record["engine_revision"] == REVISION, "Unexpected recorded engine revision.") require(record["api_certainty_target"] == "0.995" and record["gate_epsilon"] == "0.005", "Unexpected experiment target or budget.") count = inspect_cases(record) populations = {name: inspect_population(name, record["selection_counterexamples"][name]) for name in ("failure_union", "hold_intersection")} for name, population in populations.items(): for regions in population["question_regions"].values(): for region in regions.values(): require(Fraction(region["errors"], region["count"]) <= Fraction(2, 1000), f"A regional error rate exceeds its assumed 0.002 bound: {name}") require(populations["failure_union"]["total"] == 1004, "Unexpected failure population size.") require(populations["hold_intersection"]["total"] == 19999 and populations["hold_intersection"]["verdict_counts"] == {"true": 1, "uncertain": 19998}, "Unexpected hold-intersection population or selected count.") failure_rate = populations["failure_union"]["conditional_error_by_decided_verdict"]["false"] hold_rate = populations["hold_intersection"]["conditional_error_by_decided_verdict"]["true"] require(failure_rate == decimal_ratio(6, 1004), "Failure population must contain 6 errors in 1,004.") require(hold_rate == "1", "The accepted hold population must contain one erroneous decision.") print(json.dumps({ "status": "pass", "recorded_gate_cases_checked": count, "constructed_populations_recomputed": len(populations), "failure_union_conditional_error": failure_rate, "hold_intersection_conditional_error": hold_rate, "zero_error_confidence_bound": inspect_zero_error_bound(record), "scope": "Recorded outcomes and independent arithmetic checked; private engine was not executed.", }, indent=2)) if __name__ == "__main__": main()