"""The judging test: StepManiaY's 653-note test chart played by the timing self-test, which presses the panels through each engine's own input system at exact moments (on the note, or off it by a set offset), on StepManiaY's published build and on ITGmania 1.3.0 with the same self-test and flight recorder added (itgmania-1.3.0-timing-harness.patch; build_itgmania.sh). The recorder writes every judged step and miss with the note it was meant for, so each run counts the steps judged against the wrong note or none, the misses and the judgments. python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out The runs are TIMING.md's in StepManiaY's engine repository. StepManiaY's build starts the stage from its own --smytest=timing --mpsong; ITGmania's starts it from a minimal theme on its _fallback, and the patch starts the self-test with the same flags when gameplay begins. """ from __future__ import annotations import argparse import csv import json import platform import shutil import sys from collections import Counter from datetime import datetime, timezone from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE.parent / "stability")) import make_song # noqa: E402 import run as stability # noqa: E402 SONG = f"{make_song.GROUP}/{make_song.FOLDER}" RECORDED = "Timing self-test:" # name: (offsets ms, late positive; music rate; one frame this long every second, ms) RUNS = { "exact-rate-1.00": ("0", "1.0", ""), "exact-rate-0.75": ("0", "0.75", ""), "exact-60ms-frame-every-second": ("0", "1.0", "60"), "16ms-late-rate-1.50": ("16", "1.5", ""), "16ms-early-rate-1.50": ("-16", "1.5", ""), "10ms-late-60ms-frame-every-second": ("10", "1.0", "60"), } METRICS = """[Global] FallbackTheme=_fallback [Common] InitialScreen="ScreenVerify" [ScreenVerify] Class="ScreenWithMenuElements" Fallback="ScreenWithMenuElements" PrevScreen="ScreenVerify" NextScreen="ScreenGameplay" TimerSeconds=-1 """ START = """-- Opens gameplay on the test song with player 1 joined and failing off. return Def.ActorFrame{ OnCommand=function(self) self:sleep(0.5):queuecommand("Go") end, GoCommand=function(self) local song = SONGMAN:FindSong("%s") if not song then Trace("SMYVERIFY no song") return end GAMESTATE:SetCurrentPlayMode("PlayMode_Regular") GAMESTATE:JoinPlayer(PLAYER_1) GAMESTATE:SetCurrentStyle("single") GAMESTATE:SetCurrentSong(song) GAMESTATE:SetCurrentSteps(PLAYER_1, song:GetAllSteps()[1]) GAMESTATE:GetPlayerState(PLAYER_1):GetPlayerOptions("ModsLevel_Preferred"):FailSetting("FailType_Off") SCREENMAN:SetNewScreen("ScreenGameplay") end, } """ def is_stepmaniay(engine: str) -> bool: return engine.lower().startswith("stepmaniay") def setup_for(engine: str): def setup(game: Path, case: str) -> None: make_song.write(game / "Songs") if not is_stepmaniay(engine): theme = game / "Themes/SMY Verify" (theme / "BGAnimations/ScreenVerify overlay").mkdir(parents=True) (theme / "metrics.ini").write_text(METRICS) (theme / "BGAnimations/ScreenVerify overlay/default.lua").write_text(START % SONG) return setup def arguments(engine: str, offsets: str, rate: str, stutter: str) -> list[str]: args = ["--smytest=timing", f"--timingoffsets={offsets}", f"--timingrate={rate}"] if stutter: args.append(f"--timingstutter={stutter}") if is_stepmaniay(engine): args += [f"--mpsong={SONG}", "--smytestfrom=ScreenInit"] return args def analyze(path: Path) -> dict: """Counts from a flight recorder CSV: the self-test's steps, judged against the note meant, another note or none, their judgments, and the misses.""" steps = wrong = none = misses = 0 judgments: Counter = Counter() errors: list[float] = [] with open(path, newline="") as f: rows = csv.DictReader(line for line in f if not line.startswith("#")) for r in rows: if r["kind"] == "miss": misses += 1 elif r["kind"] == "step" and r["intended_row"]: steps += 1 judgments[r["tns"]] += 1 if r["row"] == "-1": none += 1 elif r["row"] != r["intended_row"]: wrong += 1 else: errors.append(float(r["error_ms"])) return { "steps": steps, "wrong_note": wrong, "no_note": none, "misses": misses, "judgments": dict(judgments.most_common()), "worst_error_ms": round(max(map(abs, errors)), 3) if errors else None, "mean_error_ms": round(sum(errors) / len(errors), 3) if errors else None, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary") parser.add_argument("--out", required=True) parser.add_argument("--work", default="/tmp/runs") parser.add_argument("--case", action="append", help="only these runs") parser.add_argument("--timeout", type=int, default=240) args = parser.parse_args() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) engines = dict(e.split("=", 1) for e in args.engine) cases = args.case or list(RUNS) display = stability.start_display() results = [] try: for case in cases: offsets, rate, stutter = RUNS[case] for engine, binary in engines.items(): preferences = stability.PREFERENCES + "TimingRecorder=1\nStageQuality=Off\n" if not is_stepmaniay(engine): preferences += "\n[Game-dance]\nTheme=SMY Verify\n" result = stability.run_case(engine, Path(binary), case, Path(args.work), out, args.timeout, marker=RECORDED, setup=setup_for(engine), preferences=preferences, done="recorded", args=tuple(arguments(engine, offsets, rate, stutter))) logs = Path(args.work) / engine / case / "game/Logs" recordings = sorted(logs.glob("timing-*.csv")) if recordings: shutil.copy(recordings[-1], out / f"{engine}.{case}.timing.csv") result["recorded"] = analyze(recordings[-1]) results.append(result) rec = result.get("recorded", {}) print(f"{engine:36} {case:34} {result['verdict']:8} steps={rec.get('steps')} wrong={rec.get('wrong_note')} " f"none={rec.get('no_note')} misses={rec.get('misses')} {rec.get('judgments')} worst={rec.get('worst_error_ms')}", flush=True) finally: if display: display.terminate() meta = { "when": datetime.now(timezone.utc).isoformat(timespec="seconds"), "platform": f"{platform.system()} {platform.machine()} ({platform.platform()})", "timeout_s": args.timeout, "engines": engines, "cases": {c: "offsets %s ms, rate %s, %s" % (RUNS[c][0], RUNS[c][1], f"a {RUNS[c][2]} ms frame every second" if RUNS[c][2] else "no long frames") for c in cases}, } (out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2)) print(out / "results.json") if __name__ == "__main__": sys.exit(main())