"""Plays each case's song, for the file bugs that only show once a song is played: a minimal theme on the engines' own _fallback picks the song and its chart and opens gameplay with nobody pressing anything, the same Lua on both engines. A run whose gameplay screen is up and still running 5 s later played; a crash report or an early exit is a crash; neither within the time limit is a hang. python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out """ from __future__ import annotations import argparse import json import platform import sys from datetime import datetime, timezone from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "stability")) import make_songs # noqa: E402 import run as stability # noqa: E402 PLAYING = "SMYVERIFY gameplay running" # What each case reaches in ITGmania 1.3.0 (github.com/itgmania/itgmania, tag v1.3.0). CASES = { "00-control": "A valid song; every engine should play it.", "03-ssc-timesig-1-256": "src/Background.cpp:499 steps through the song a measure of rows at a time to place " "random backgrounds; at 1/256 a measure is 0 rows, so j never advances (and :525 takes % 0).", } METRICS = """[Global] FallbackTheme=_fallback [Common] InitialScreen="ScreenVerify" [ScreenVerify] Class="ScreenWithMenuElements" Fallback="ScreenWithMenuElements" PrevScreen="ScreenVerify" NextScreen="ScreenGameplay" TimerSeconds=-1 """ START = """-- Opens gameplay on one song with player 1 joined, then gets out of the way. local SONG = "{song}" return Def.ActorFrame{{ OnCommand=function(self) self:sleep(0.5):queuecommand("Go") end, GoCommand=function(self) local song = SONGMAN:FindSong(SONG) if not song then Trace("SMYVERIFY no song " .. SONG) return end GAMESTATE:SetCurrentPlayMode("PlayMode_Regular") GAMESTATE:JoinPlayer(PLAYER_1) GAMESTATE:SetCurrentStyle("single") GAMESTATE:SetCurrentSong(song) GAMESTATE:SetCurrentSteps(PLAYER_1, song:GetAllSteps()[1]) Trace("SMYVERIFY gameplay requested") SCREENMAN:SetNewScreen("ScreenGameplay") end, }} """ GAMEPLAY = """return Def.ActorFrame{ OnCommand=function(self) Trace("SMYVERIFY gameplay running") end, } """ def setup(game: Path, case: str) -> None: theme = game / "Themes/SMY Verify" (theme / "BGAnimations/ScreenVerify overlay").mkdir(parents=True) (theme / "BGAnimations/ScreenGameplay overlay").mkdir(parents=True) (theme / "metrics.ini").write_text(METRICS) (theme / "BGAnimations/ScreenVerify overlay/default.lua").write_text(START.format(song=f"{make_songs.GROUP}/{case}")) (theme / "BGAnimations/ScreenGameplay overlay/default.lua").write_text(GAMEPLAY) def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary") parser.add_argument("--out", required=True) parser.add_argument("--work", default="/tmp/runs") parser.add_argument("--case", action="append", help="only these cases") parser.add_argument("--timeout", type=int, default=120) args = parser.parse_args() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) engines = dict(e.split("=", 1) for e in args.engine) cases = args.case or list(CASES) preferences = stability.PREFERENCES + "\n[Game-dance]\nTheme=SMY Verify\n" display = stability.start_display() results = [] try: for case in cases: for engine, binary in engines.items(): result = stability.run_case(engine, Path(binary), case, Path(args.work), out, args.timeout, marker=PLAYING, setup=setup, preferences=preferences, done="played") results.append(result) print(f"{engine:32} {case:24} {result['verdict']:8} {result['seconds']:6}s exit={result['exit']} {result['crash'][:80]}", flush=True) finally: if display: display.terminate() meta = { "when": datetime.now(timezone.utc).isoformat(timespec="seconds"), "platform": f"{platform.system()} {platform.machine()} ({platform.platform()})", "timeout_s": args.timeout, "engines": engines, "cases": {c: CASES[c] for c in cases}, } (out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2)) print(out / "results.json") if __name__ == "__main__": sys.exit(main())