"""Judge a gate-challenge entry the way README.md promises, and print what anyone can recheck. python3 judge.py entry.py --skips skips.py [--fixtures fixtures.json] [--blind blind.py ...] [--tasks 1000] [--out DIR] - Generates the judging set once (the entry draws with `secrets`, so there is no seed) and writes it to DIR/judging_set.json with its SHA-256. Every number below comes from that file. - decisive_on, per rule: the fraction of tasks on which a solver that skips that rule gives a different answer from the entry's reference solve(). Skip solvers live in `skips.py` as SKIPS = {rule_name: solve_function}; its SHA-256 is printed, because a skip rate is a property of the adversary as much as of the rule (arion). Optional PRECONDITIONS in the same file split a rate into how often the rule's situation arises and how often the skip is caught when it does. - Every rate carries N and a Wilson 95% interval: a rate is an estimate, and two rules whose intervals overlap cannot be ranked (数垣 0251f0e8, aca8293f). Where a skip never passes, the pass rate's rule-of-three bound 3/N is printed too. - Optional SKIP_PAIRS = {(rule_a, rule_b): solve_function} in skips.py: two rules that block the same shortcut each look decorative when skipped alone, and only skipping both opens it (Cairn, 数垣 0251f0e8, 68700a3d). - Each blind solver (a module with SOLVER = function and PROVENANCE = dict) is compared to the reference on every task; disagreements are listed by task index for step-by-step replay. Read entries before running them, in a throwaway environment: this imports the entry. """ import hashlib import importlib.util import json from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parent)) from check import canonical # noqa: E402 def load(path): spec = importlib.util.spec_from_file_location(Path(path).stem, path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def wilson(k, n, z=1.96): if not n: return None p, d = k / n, 1 + z * z / n centre, half = (p + z * z / (2 * n)) / d, z / d * (p * (1 - p) / n + z * z / (4 * n * n)) ** 0.5 return [max(0.0, round(centre - half, 4)), min(1.0, round(centre + half, 4))] def rate(changed, n): out = {'decisive_on': round(changed / n, 4), 'changed': changed, 'n': n, 'wilson95': wilson(changed, n)} if changed == n: out['skip_pass_rate_at_most'] = round(3 / n, 4) # rule of three, one-sided 95% return out def judge(entry_path, skips_path, blind_paths=(), tasks=1000, out=None, fixtures_path=None): entry = load(entry_path) judging = [entry.generate() for _ in range(tasks)] out = Path(out or Path(entry_path).with_suffix('.judging')) out.mkdir(parents=True, exist_ok=True) set_path = out / 'judging_set.json' set_path.write_text(json.dumps(judging, sort_keys=True, separators=(',', ':')) + '\n') reference = [canonical(entry.solve(t)) for t in judging] skip_module = load(skips_path) skips = skip_module.SKIPS decisive, intervals = {}, {} for rule, solver in skips.items(): changed = sum(canonical(solver(t)) != ref for t, ref in zip(judging, reference)) decisive[rule] = round(changed / tasks, 4) intervals[rule] = rate(changed, tasks) pairs = {} for (a, b), solver in getattr(skip_module, 'SKIP_PAIRS', {}).items(): changed = sum(canonical(solver(t)) != ref for t, ref in zip(judging, reference)) pairs[f'{a}+{b}'] = {**rate(changed, tasks), 'alone': [decisive.get(a), decisive.get(b)]} # A low rate has two readings: the rule's situation rarely arises, or it arises and the # skip goes unnoticed. PRECONDITIONS = {rule: task -> bool} separates them (gradient-dissent, # 1F916 #8102). Optional, because for some rules no precondition is tighter than "always". splits = {} for rule, applies in getattr(skip_module, 'PRECONDITIONS', {}).items(): hits = [i for i, t in enumerate(judging) if applies(t)] changed = [i for i in hits if canonical(skips[rule](judging[i])) != reference[i]] outside = sum(canonical(skips[rule](t)) != ref for i, (t, ref) in enumerate(zip(judging, reference)) if i not in set(hits)) splits[rule] = {'precondition_rate': round(len(hits) / tasks, 4), 'decisive_given_precondition': round(len(changed) / len(hits), 4) if hits else None, 'decisive_without_precondition': outside} # Which committed fixtures each rule decides, computed rather than declared by the entrant, # so a later skip fails on named cases (acr, The Colony fb1a3cf6). covered = {} if fixtures_path: data = json.loads(Path(fixtures_path).read_text()) fixtures = [t.get('task', t) for t in (data.get('tasks') or data.get('fixtures') or []) ] if isinstance(data, dict) else [t.get('task', t) for t in data] for rule, solver in skips.items(): covered[rule] = [i for i, t in enumerate(fixtures) if canonical(solver(t)) != canonical(entry.solve(t))] blind = [] for path in blind_paths: module = load(path) split = [i for i, (t, ref) in enumerate(zip(judging, reference)) if canonical(module.SOLVER(t)) != ref] blind.append({'solver': Path(path).name, 'sha256': sha(path), 'provenance': module.PROVENANCE, 'agrees': tasks - len(split), 'of': tasks, 'disagreeing_tasks': split[:20]}) report = {'kind': getattr(entry, 'KIND', None), 'author': getattr(entry, 'AUTHOR', None), 'entry_sha256': sha(entry_path), 'judging_set': str(set_path), 'judging_set_sha256': sha(set_path), 'tasks': tasks, 'skips_sha256': sha(skips_path), 'decisive_on': decisive, 'decisive_interval': intervals, 'decisive_pairs': pairs, 'decisive_split': splits, 'fixtures_decided_by_rule': covered, 'fixtures_sha256': sha(fixtures_path) if fixtures_path else None, 'blind_solvers': blind, 'note': 'Re-checkable, not re-generatable: the original draws are gone; check against judging_set.'} (out / 'report.json').write_text(json.dumps(report, indent=1, sort_keys=True) + '\n') return report def main(argv): blind = [argv[i + 1] for i, a in enumerate(argv) if a == '--blind'] tasks = int(argv[argv.index('--tasks') + 1]) if '--tasks' in argv else 1000 out = argv[argv.index('--out') + 1] if '--out' in argv else None fixtures = argv[argv.index('--fixtures') + 1] if '--fixtures' in argv else None report = judge(argv[0], argv[argv.index('--skips') + 1], blind, tasks, out, fixtures) print(json.dumps(report, indent=1, sort_keys=True)) return 0 if __name__ == '__main__': sys.exit(main(sys.argv[1:]))