"""Synthetic paired evaluation; Python 3.10+, standard library, seed 17.
No API calls. Items, rather than individual judgments, are bootstrap units.
"""
import math
import random
import json
from statistics import mean


def wilson(successes, n, z=1.959963984540054):
    if not 0 <= successes <= n or n <= 0:
        raise ValueError('Require 0 <= successes <= n and n > 0')
    p = successes / n
    den = 1 + z*z/n
    center = (p + z*z/(2*n)) / den
    radius = z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / den
    return center-radius, center+radius


def paired_bootstrap(a, b, repetitions=10000, seed=17):
    if len(a) != len(b) or not a:
        raise ValueError('Require nonempty matched items')
    if not isinstance(repetitions, int) or repetitions < 100:
        raise ValueError('Use at least 100 bootstrap repetitions')
    if any(v not in (0, 1) for v in a + b):
        raise ValueError('This example requires binary outcomes')
    rng = random.Random(seed)
    differences = [y-x for x, y in zip(a, b)]
    means = sorted(mean(rng.choices(differences, k=len(a)))
                   for _ in range(repetitions))
    return mean(differences), means[int(.025*repetitions)], means[int(.975*repetitions)]



def grade_response(raw, allowed_sources, expected_status):
    """Original deterministic contract grader, NOT a semantic truth judge.
    Candidate text is parsed as data; never eval(), execute(), or prompt a judge.
    """
    try:
        response = json.loads(raw)
    except (json.JSONDecodeError, TypeError):
        return {"schema": False, "evidence_ids": False, "status": False}
    schema = (isinstance(response, dict)
              and set(response) == {"answer", "sources", "status"}
              and isinstance(response["answer"], str)
              and isinstance(response["sources"], list)
              and all(isinstance(x, str) for x in response["sources"])
              and response["status"] in ("answer", "abstain"))
    if not schema:
        return {"schema": False, "evidence_ids": False, "status": False}
    return {"schema": True,
            "evidence_ids": (all(x in allowed_sources for x in response["sources"])
                             and (response["status"] != "answer" or bool(response["sources"]))),
            "status": response["status"] == expected_status}


valid = '{"answer":"The policy says 30 days.","sources":["policy-v3"],"status":"answer"}'
assert all(grade_response(valid, {"policy-v3"}, "answer").values())
assert not grade_response('not JSON', set(), 'abstain')['schema']
assert not grade_response(valid, {"policy-v4"}, "answer")['evidence_ids']
assert not grade_response(valid, {"policy-v3"}, "abstain")['status']
# A plausible-looking lie can pass these structural checks. Add human/semantic
# review against the actual source; do not call this contract pass "correctness".
lie = valid.replace('30 days', '300 days')
assert all(grade_response(lie, {"policy-v3"}, "answer").values())

# Same 20 synthetic tasks: 12 both pass, 4 only B, 2 only A, 2 neither.
a = [1]*12 + [0]*4 + [1]*2 + [0]*2
b = [1]*12 + [1]*4 + [0]*2 + [0]*2
assert sum(a) == 14 and sum(b) == 16
low, high = wilson(sum(b), len(b))
delta, lower, upper = paired_bootstrap(a, b)
assert lower <= 0 <= upper  # small sample does not establish superiority
assert paired_bootstrap(a, a) == (0, 0, 0)
assert wilson(0, 20)[0] >= -1e-12
assert math.isclose(1 - .05**(1/100), .029513049607039932)
# Trial independence is an assumption, not a property guaranteed by a model.
assert math.isclose(1-(1-.8)**3, .992)
assert math.isclose(.8**3, .512)
print(f'A: {mean(a):.1%}; B: {mean(b):.1%}')
print(f'B Wilson 95% interval: [{low:.3f}, {high:.3f}]')
print(f'Paired delta: {delta:.1%}; bootstrap 95% interval: [{lower:.3f}, {upper:.3f}]')
print('Zero failures in 100: one-sided 95% upper bound 2.95%')
print('All evaluation checks passed; synthetic data, not model performance.')
