"""Results folders as tables: one row per case, one column per engine. A timing folder's repeats (/repeats/*/) count with it, so each cell gives the range over every run. python3 summarize.py [...] Markdown python3 summarize.py --json [...] the same as JSON (the website's release) """ from __future__ import annotations import json import sys from pathlib import Path def runs_of(folder: Path) -> tuple[dict, list[dict]]: data = json.loads((folder / "results.json").read_text()) runs = list(data["runs"]) for repeat in sorted((folder / "repeats").glob("*/results.json")) if (folder / "repeats").is_dir() else []: runs += json.loads(repeat.read_text())["runs"] return data["meta"], runs def span(values: list[int]) -> str: return f"{min(values)}" if min(values) == max(values) else f"{min(values)}–{max(values)}" def cell(runs: list[dict]) -> str: if not runs: return "not run" recorded = [r["recorded"] for r in runs if r.get("recorded")] times = f" ({len(runs)} runs)" if len(runs) > 1 else "" if not recorded: return " / ".join(sorted({r["verdict"] for r in runs})) + times perfect = all(not (r["wrong_note"] or r["no_note"] or r["misses"]) and set(r["judgments"]) == {"W1"} for r in recorded) if perfect: return f"{span([r['steps'] for r in recorded])} W1, none on the wrong note or missed{times}" return (f"{span([r['wrong_note'] for r in recorded])} on the wrong note, " f"{span([r['misses'] for r in recorded])} missed{times}") def summary(folder: Path) -> dict: meta, runs = runs_of(folder) engines = list(meta["engines"]) rows = [] for case, description in meta["cases"].items(): rows.append({ "case": case, "about": description, "cells": {e: cell([r for r in runs if r["engine"] == e and r["case"] == case]) for e in engines}, }) return {"test": folder.name.split("-", 1)[0], "folder": folder.name, "when": meta["when"], "platform": meta["platform"], "engines": engines, "rows": rows} def main() -> None: args = sys.argv[1:] as_json = args[:1] == ["--json"] tables = [summary(Path(a)) for a in (args[1:] if as_json else args)] if as_json: print(json.dumps(tables, indent=2, ensure_ascii=False)) return for t in tables: print(f"### {t['folder']} ({t['platform']}, {t['when']})\n") print("| Case | " + " | ".join(t["engines"]) + " |") print("| --- |" + " --- |" * len(t["engines"])) for row in t["rows"]: print(f"| {row['case']} | " + " | ".join(row["cells"][e] for e in t["engines"]) + " |") print() if __name__ == "__main__": main()