"""The judging test: StepManiaY's 653-note test chart played by the timing self-test, which presses
the panels through each engine's own input system at exact moments (on the note, or off it by a
set offset), on StepManiaY's published build and on ITGmania 1.3.0 with the same self-test and
flight recorder added (itgmania-1.3.0-timing-harness.patch; build_itgmania.sh). The recorder writes
every judged step and miss with the note it was meant for, so each run counts the steps judged
against the wrong note or none, the misses and the judgments.
python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out <results folder>
The runs are TIMING.md's in StepManiaY's engine repository. StepManiaY's build starts the stage
from its own --smytest=timing --mpsong; ITGmania's starts it from a minimal theme on its _fallback,
and the patch starts the self-test with the same flags when gameplay begins.
"""
from __future__ import annotations
import argparse
import csv
import json
import platform
import shutil
import sys
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent / "stability"))
import make_song # noqa: E402
import run as stability # noqa: E402
SONG = f"{make_song.GROUP}/{make_song.FOLDER}"
RECORDED = "Timing self-test:"
# name: (offsets ms, late positive; music rate; one frame this long every second, ms)
RUNS = {
"exact-rate-1.00": ("0", "1.0", ""),
"exact-rate-0.75": ("0", "0.75", ""),
"exact-60ms-frame-every-second": ("0", "1.0", "60"),
"16ms-late-rate-1.50": ("16", "1.5", ""),
"16ms-early-rate-1.50": ("-16", "1.5", ""),
"10ms-late-60ms-frame-every-second": ("10", "1.0", "60"),
}
METRICS = """[Global]
FallbackTheme=_fallback
[Common]
InitialScreen="ScreenVerify"
[ScreenVerify]
Class="ScreenWithMenuElements"
Fallback="ScreenWithMenuElements"
PrevScreen="ScreenVerify"
NextScreen="ScreenGameplay"
TimerSeconds=-1
"""
START = """-- Opens gameplay on the test song with player 1 joined and failing off.
return Def.ActorFrame{
OnCommand=function(self) self:sleep(0.5):queuecommand("Go") end,
GoCommand=function(self)
local song = SONGMAN:FindSong("%s")
if not song then Trace("SMYVERIFY no song") return end
GAMESTATE:SetCurrentPlayMode("PlayMode_Regular")
GAMESTATE:JoinPlayer(PLAYER_1)
GAMESTATE:SetCurrentStyle("single")
GAMESTATE:SetCurrentSong(song)
GAMESTATE:SetCurrentSteps(PLAYER_1, song:GetAllSteps()[1])
GAMESTATE:GetPlayerState(PLAYER_1):GetPlayerOptions("ModsLevel_Preferred"):FailSetting("FailType_Off")
SCREENMAN:SetNewScreen("ScreenGameplay")
end,
}
"""
def is_stepmaniay(engine: str) -> bool:
return engine.lower().startswith("stepmaniay")
def setup_for(engine: str):
def setup(game: Path, case: str) -> None:
make_song.write(game / "Songs")
if not is_stepmaniay(engine):
theme = game / "Themes/SMY Verify"
(theme / "BGAnimations/ScreenVerify overlay").mkdir(parents=True)
(theme / "metrics.ini").write_text(METRICS)
(theme / "BGAnimations/ScreenVerify overlay/default.lua").write_text(START % SONG)
return setup
def arguments(engine: str, offsets: str, rate: str, stutter: str) -> list[str]:
args = ["--smytest=timing", f"--timingoffsets={offsets}", f"--timingrate={rate}"]
if stutter:
args.append(f"--timingstutter={stutter}")
if is_stepmaniay(engine):
args += [f"--mpsong={SONG}", "--smytestfrom=ScreenInit"]
return args
def analyze(path: Path) -> dict:
"""Counts from a flight recorder CSV: the self-test's steps, judged against the note meant,
another note or none, their judgments, and the misses."""
steps = wrong = none = misses = 0
judgments: Counter = Counter()
errors: list[float] = []
with open(path, newline="") as f:
rows = csv.DictReader(line for line in f if not line.startswith("#"))
for r in rows:
if r["kind"] == "miss":
misses += 1
elif r["kind"] == "step" and r["intended_row"]:
steps += 1
judgments[r["tns"]] += 1
if r["row"] == "-1":
none += 1
elif r["row"] != r["intended_row"]:
wrong += 1
else:
errors.append(float(r["error_ms"]))
return {
"steps": steps,
"wrong_note": wrong,
"no_note": none,
"misses": misses,
"judgments": dict(judgments.most_common()),
"worst_error_ms": round(max(map(abs, errors)), 3) if errors else None,
"mean_error_ms": round(sum(errors) / len(errors), 3) if errors else None,
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary")
parser.add_argument("--out", required=True)
parser.add_argument("--work", default="/tmp/runs")
parser.add_argument("--case", action="append", help="only these runs")
parser.add_argument("--timeout", type=int, default=240)
args = parser.parse_args()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
engines = dict(e.split("=", 1) for e in args.engine)
cases = args.case or list(RUNS)
display = stability.start_display()
results = []
try:
for case in cases:
offsets, rate, stutter = RUNS[case]
for engine, binary in engines.items():
preferences = stability.PREFERENCES + "TimingRecorder=1\nStageQuality=Off\n"
if not is_stepmaniay(engine):
preferences += "\n[Game-dance]\nTheme=SMY Verify\n"
result = stability.run_case(engine, Path(binary), case, Path(args.work), out, args.timeout,
marker=RECORDED, setup=setup_for(engine), preferences=preferences,
done="recorded", args=tuple(arguments(engine, offsets, rate, stutter)))
logs = Path(args.work) / engine / case / "game/Logs"
recordings = sorted(logs.glob("timing-*.csv"))
if recordings:
shutil.copy(recordings[-1], out / f"{engine}.{case}.timing.csv")
result["recorded"] = analyze(recordings[-1])
results.append(result)
rec = result.get("recorded", {})
print(f"{engine:36} {case:34} {result['verdict']:8} steps={rec.get('steps')} wrong={rec.get('wrong_note')} "
f"none={rec.get('no_note')} misses={rec.get('misses')} {rec.get('judgments')} worst={rec.get('worst_error_ms')}", flush=True)
finally:
if display:
display.terminate()
meta = {
"when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"platform": f"{platform.system()} {platform.machine()} ({platform.platform()})",
"timeout_s": args.timeout,
"engines": engines,
"cases": {c: "offsets %s ms, rate %s, %s" % (RUNS[c][0], RUNS[c][1], f"a {RUNS[c][2]} ms frame every second" if RUNS[c][2] else "no long frames") for c in cases},
}
(out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2))
print(out / "results.json")
if __name__ == "__main__":
sys.exit(main())