"""Check a gate-challenge entry: python3 check.py entry.py [--tasks 300] [--fixtures f.json] [--json] Make the required fixtures file: python3 check.py entry.py --write-fixtures fixtures.json Read an entry before running it. This checker screens the source (imports and dangerous names) and runs it in a child process under CPU, memory and wall-clock limits, but a screen is not a sandbox: run untrusted entries on a throwaway machine or account. What it checks is what the live gate needs from a task kind (lockzone/challenge.py): fresh tasks, a stated rule, an exact and deterministic answer, evidence IDs in the answer, and speed. What it cannot check is whether the instructions are complete and whether skipping a rule still passes. Those are judged by hand, with a solver written from the instructions alone and one solver per skipped rule (README.md, "How entries are judged"). """ import ast import hashlib import json from pathlib import Path import resource import statistics import subprocess import sys ALLOWED_IMPORTS = {'secrets', 'json', 'itertools', 'math', 'collections', 'heapq', 'string', 'functools', 'bisect', 're', 'operator'} FORBIDDEN_NAMES = {'open', 'exec', 'eval', 'compile', '__import__', 'input', 'globals', 'locals', 'getattr', 'setattr', 'delattr', 'vars', 'breakpoint', 'memoryview'} MAX_LINES = 150 MAX_TASK_BYTES = 6000 INSTRUCTION_CHARS = (80, 1200) CHILD = r''' import importlib.util, json, sys, time spec = importlib.util.spec_from_file_location('entry', sys.argv[1]) entry = importlib.util.module_from_spec(spec); spec.loader.exec_module(entry) out = {'kind': getattr(entry, 'KIND', None), 'author': getattr(entry, 'AUTHOR', None), 'rows': []} for _ in range(int(sys.argv[2])): t0 = time.perf_counter(); task = entry.generate(); t1 = time.perf_counter() first = entry.solve(task); t2 = time.perf_counter() again = entry.solve(json.loads(json.dumps(task))) out['rows'].append({'task': task, 'answer': first, 'again': again, 'generate_ms': (t1 - t0) * 1000, 'solve_ms': (t2 - t1) * 1000}) print(json.dumps(out)) ''' CHILD_SOLVE = r''' import importlib.util, json, sys spec = importlib.util.spec_from_file_location('entry', sys.argv[1]) entry = importlib.util.module_from_spec(spec); spec.loader.exec_module(entry) tasks = json.load(open(sys.argv[2]))['tasks'] print(json.dumps([entry.solve(t) for t in tasks])) ''' FIXTURE_COUNT, HAND_CHECKED_MIN = 10, 3 def canonical(value): # The live grader's encoding, so "equal" here means equal at the gate. return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True, allow_nan=False) def screen(source): problems = [] if len(source.splitlines()) > MAX_LINES: problems.append(f'longer than {MAX_LINES} lines') tree = ast.parse(source) for node in ast.walk(tree): if isinstance(node, ast.Import): names = [a.name.split('.')[0] for a in node.names] elif isinstance(node, ast.ImportFrom): names = [(node.module or '').split('.')[0]] else: names = [] problems += [f'import {n} is not allowed' for n in names if n not in ALLOWED_IMPORTS] if isinstance(node, ast.Name) and node.id in FORBIDDEN_NAMES: problems.append(f'name {node.id} is not allowed (line {node.lineno})') if isinstance(node, ast.Attribute) and node.attr.startswith('__') and node.attr != '__init__': problems.append(f'dunder attribute {node.attr} is not allowed (line {node.lineno})') return sorted(set(problems)) def limit_child(): resource.setrlimit(resource.RLIMIT_CPU, (60, 60)) try: resource.setrlimit(resource.RLIMIT_AS, (1 << 30, 1 << 30)) except (ValueError, OSError): pass # macOS refuses an address-space cap; CPU and wall clock still apply def run(path, count): done = subprocess.run([sys.executable, '-I', '-c', CHILD, str(path), str(count)], capture_output=True, text=True, timeout=120, preexec_fn=limit_child) if done.returncode != 0: raise RuntimeError(done.stderr.strip()[-2000:] or f'exit {done.returncode}') return json.loads(done.stdout) def strings_in(value): if isinstance(value, str): yield value elif isinstance(value, dict): for v in value.values(): yield from strings_in(v) elif isinstance(value, list): for v in value: yield from strings_in(v) def evaluate(out): rows = out['rows'] results = [] def check(name, ok, detail=''): results.append({'check': name, 'ok': bool(ok), 'detail': detail}) check('KIND is a short lowercase word', isinstance(out['kind'], str) and out['kind'].isidentifier() and out['kind'].islower() and len(out['kind']) <= 24, repr(out['kind'])) check('AUTHOR names who gets the credit', isinstance(out['author'], str) and 0 < len(out['author']) <= 80, repr(out['author'])) tasks = [r['task'] for r in rows] shape_ok = [isinstance(t, dict) and isinstance(t.get('id'), str) and t.get('kind') == out['kind'] and isinstance(t.get('instructions'), str) and isinstance(t.get('answer_shape'), dict) for t in tasks] check('every task has id, kind, instructions, answer_shape', all(shape_ok), f'{shape_ok.count(False)} malformed') lengths = [len(t.get('instructions', '')) for t in tasks] check(f'instructions are {INSTRUCTION_CHARS[0]}-{INSTRUCTION_CHARS[1]} characters', all(INSTRUCTION_CHARS[0] <= n <= INSTRUCTION_CHARS[1] for n in lengths), f'{min(lengths)}-{max(lengths)}') sizes = [len(canonical(t)) for t in tasks] check(f'tasks encode as canonical JSON under {MAX_TASK_BYTES} bytes', max(sizes) <= MAX_TASK_BYTES, f'largest {max(sizes)}') ids = [t['id'] for t in tasks] check('task IDs are fresh per call', len(set(ids)) == len(ids), f'{len(ids) - len(set(ids))} repeats') distinct = len({canonical({k: v for k, v in t.items() if k != 'id'}) for t in tasks}) check('task contents are fresh per call', distinct == len(tasks), f'{distinct}/{len(tasks)} distinct') answers = [canonical(r['answer']) for r in rows] check('the solver is deterministic, also after a JSON round trip', all(canonical(r['answer']) == canonical(r['again']) for r in rows)) check('answers use only answer_shape keys', all(isinstance(r['answer'], dict) and set(r['answer']) <= set(r['task']['answer_shape']) for r in rows)) # evidence-v2: an answer names things in the task, so it cannot be guessed blind. evidence = [any(s in set(strings_in({k: v for k, v in r['task'].items() if k not in ('instructions', 'answer_shape')})) for s in strings_in(r['answer'])) for r in rows] check('answers cite evidence from the task data', sum(evidence) >= 0.99 * len(rows), f'{sum(evidence)}/{len(rows)}') common = max(answers.count(a) for a in set(answers)) check('no answer repeats across tasks', common == 1, f'most common answer seen {common} times') gen, sol = [r['generate_ms'] for r in rows], [r['solve_ms'] for r in rows] check('generate + solve: median under 20 ms, max under 250 ms', statistics.median(g + s for g, s in zip(gen, sol)) < 20 and max(g + s for g, s in zip(gen, sol)) < 250, f'median {statistics.median(gen):.1f}+{statistics.median(sol):.1f} ms, max {max(gen):.1f}+{max(sol):.1f} ms') return results def write_fixtures(path, out_path): """Ten tasks with the entry's own answers. The entrant then works at least three by hand and lists their indices in hand_checked before publishing the file's SHA-256 (rule added 2026-10-07 after arion: it fixes what "correct" means before any judge's solver exists).""" out = run(path, FIXTURE_COUNT) doc = {'kind': out['kind'], 'author': out['author'], 'entry_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'tasks': [r['task'] for r in out['rows']], 'answers': [r['answer'] for r in out['rows']], 'hand_checked': []} Path(out_path).write_text(json.dumps(doc, indent=1, sort_keys=True) + '\n') print(f'wrote {out_path}: work at least {HAND_CHECKED_MIN} by hand, list their indices in hand_checked') def verify_fixtures(path, fixtures_path): doc = json.loads(Path(fixtures_path).read_text()) done = subprocess.run([sys.executable, '-I', '-c', CHILD_SOLVE, str(path), str(fixtures_path)], capture_output=True, text=True, timeout=120, preexec_fn=limit_child) if done.returncode != 0: raise RuntimeError(done.stderr.strip()[-2000:] or f'exit {done.returncode}') solved = json.loads(done.stdout) hand = doc.get('hand_checked') or [] return [ {'check': f'{FIXTURE_COUNT} fixture tasks with committed answers', 'ok': len(doc.get('tasks', [])) == len(doc.get('answers', [])) == FIXTURE_COUNT}, {'check': 'fixtures name this entry', 'ok': doc.get('entry_sha256') == hashlib.sha256(path.read_bytes()).hexdigest()}, {'check': 'the entry reproduces every committed answer', 'ok': [canonical(a) for a in solved] == [canonical(a) for a in doc.get('answers', [])]}, {'check': f'at least {HAND_CHECKED_MIN} answers marked as worked by hand', 'ok': len(set(hand)) >= HAND_CHECKED_MIN and all(isinstance(i, int) and 0 <= i < FIXTURE_COUNT for i in hand)}, ] def main(argv): path = Path(argv[0]) if '--write-fixtures' in argv: write_fixtures(path, argv[argv.index('--write-fixtures') + 1]) return 0 count = int(argv[argv.index('--tasks') + 1]) if '--tasks' in argv else 300 source = path.read_text() problems = screen(source) # Both hashes in every report: which entry was checked, by which checker (arion's point: # agreement is only attributable if the checker's bytes are named too). report = {'entry': path.name, 'entry_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'checker_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'screen': problems, 'checks': []} if not problems: try: out = run(path, count) report['kind'], report['author'] = out['kind'], out['author'] report['checks'] = evaluate(out) report['sample_task'] = out['rows'][0]['task'] if '--fixtures' in argv: fx = Path(argv[argv.index('--fixtures') + 1]) report['fixtures_sha256'] = hashlib.sha256(fx.read_bytes()).hexdigest() report['checks'] += verify_fixtures(path, fx) except (RuntimeError, subprocess.TimeoutExpired, KeyError, ValueError, TypeError) as e: report['error'] = str(e) passed = not problems and 'error' not in report and all(c['ok'] for c in report['checks']) report['passed'] = passed if '--json' in argv: print(json.dumps(report, indent=2)) else: print(f"{report['entry']}: {'PASS' if passed else 'FAIL'} (automatic checks only; judging is by hand)") print(f" entry sha256 {report['entry_sha256']}, checker sha256 {report['checker_sha256']}") if 'fixtures_sha256' in report: print(f" fixtures sha256 {report['fixtures_sha256']}") for p in problems: print(f' screen: {p}') if 'error' in report: print(f" error: {report['error']}") for c in report['checks']: print(f" [{'ok' if c['ok'] else 'NO'}] {c['check']}" + (f" ({c['detail']})" if c.get('detail') else '')) return 0 if passed else 1 if __name__ == '__main__': sys.exit(main(sys.argv[1:]))