"""Results folders as tables: one row per case, one column per engine. A timing folder's repeats
(<folder>/repeats/*/) count with it, so each cell gives the range over every run.
python3 summarize.py <results folder> [...] Markdown
python3 summarize.py --json <results folder> [...] the same as JSON (the website's release)
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
def runs_of(folder: Path) -> tuple[dict, list[dict]]:
data = json.loads((folder / "results.json").read_text())
runs = list(data["runs"])
for repeat in sorted((folder / "repeats").glob("*/results.json")) if (folder / "repeats").is_dir() else []:
runs += json.loads(repeat.read_text())["runs"]
return data["meta"], runs
def span(values: list[int]) -> str:
return f"{min(values)}" if min(values) == max(values) else f"{min(values)}–{max(values)}"
def cell(runs: list[dict]) -> str:
if not runs:
return "not run"
recorded = [r["recorded"] for r in runs if r.get("recorded")]
times = f" ({len(runs)} runs)" if len(runs) > 1 else ""
if not recorded:
return " / ".join(sorted({r["verdict"] for r in runs})) + times
perfect = all(not (r["wrong_note"] or r["no_note"] or r["misses"]) and set(r["judgments"]) == {"W1"} for r in recorded)
if perfect:
return f"{span([r['steps'] for r in recorded])} W1, none on the wrong note or missed{times}"
return (f"{span([r['wrong_note'] for r in recorded])} on the wrong note, "
f"{span([r['misses'] for r in recorded])} missed{times}")
def summary(folder: Path) -> dict:
meta, runs = runs_of(folder)
engines = list(meta["engines"])
rows = []
for case, description in meta["cases"].items():
rows.append({
"case": case,
"about": description,
"cells": {e: cell([r for r in runs if r["engine"] == e and r["case"] == case]) for e in engines},
})
return {"test": folder.name.split("-", 1)[0], "folder": folder.name, "when": meta["when"],
"platform": meta["platform"], "engines": engines, "rows": rows}
def main() -> None:
args = sys.argv[1:]
as_json = args[:1] == ["--json"]
tables = [summary(Path(a)) for a in (args[1:] if as_json else args)]
if as_json:
print(json.dumps(tables, indent=2, ensure_ascii=False))
return
for t in tables:
print(f"### {t['folder']} ({t['platform']}, {t['when']})\n")
print("| Case | " + " | ".join(t["engines"]) + " |")
print("| --- |" + " --- |" * len(t["engines"]))
for row in t["rows"]:
print(f"| {row['case']} | " + " | ".join(row["cells"][e] for e in t["engines"]) + " |")
print()
if __name__ == "__main__":
main()