"""Runs every stability case on every engine, each in a fresh portable copy of the engine, and records what the engine did with it: loaded the song list, crashed, or hung. What an engine says about a file it skips goes to Logs/userlog.txt, kept beside each run's log. python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out Run it in the kit's container (../../verify.sh), which has a virtual display and software OpenGL. Each run is a full copy of the engine's folder beside a Portable.ini, with the control song and one case in Songs, the Null sound driver and no network. Both engines log "Loading courses." once every song is loaded; a run that logs it and is still running 5 s later loaded the list. A crash report or an exit before that is a crash; neither within the time limit is a hang. """ from __future__ import annotations import argparse import json import os import platform import re import shutil import signal import subprocess import sys import time from datetime import datetime, timezone from pathlib import Path import make_songs GRACE_S = 5 SCANNED = "Loading courses." # What each case reaches in ITGmania 1.3.0 (github.com/itgmania/itgmania, tag v1.3.0). CASES = { "00-control": "A valid song; every engine should load it.", "01-wav-negative-chunk": "src/RageSoundReader_WAV.cpp:497 adds the chunk's size to the read position without " "checking it; -8 seeks back to the same chunk header forever.", "02-ssc-empty-speeds": "src/NotesLoaderSM.cpp:834 reads vs2[0] before checking vs2.size(); " '"=" splits into no fields.', "03-ssc-timesig-1-256": "Nothing in 1.3.0 rejects a signature under one note row a measure, but nothing in its " "song scan divides by one either: the old engine should load this list. Its hang is in tests/playback.", "04-dwi-lowercase-mode": 'src/NotesLoaderDWI.cpp:278 compares the mode with == "SINGLE"; ' "#single: reaches ASSERT_M(0, ...) at :287.", } PREFERENCES = """[Options] SoundDrivers=Null Windowed=1 DisplayWidth=640 DisplayHeight=360 LogToDisk=1 ForceLogFlush=1 GrooveStatsApiUrl=http://127.0.0.1:9/ GrooveStatsLinkPage=http://127.0.0.1:9/ GrooveStatsLinkUrl=ws://127.0.0.1:9/ GrooveStatsCrossPost=0 GrooveStatsUnlockDownloads=0 ArrowCloudApiUrl=http://127.0.0.1:9/ ArrowCloudCrossPost=0 BoogieStatsApiUrl=http://127.0.0.1:9/ StepManiaYServer=http://127.0.0.1:9 StepManiaYAskToSignIn=0 """ def start_display() -> subprocess.Popen | None: if os.path.exists("/tmp/.X11-unix/X99"): return None xvfb = subprocess.Popen(["Xvfb", ":99", "-screen", "0", "1280x720x24", "-nolisten", "tcp"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) for _ in range(100): if os.path.exists("/tmp/.X11-unix/X99"): return xvfb time.sleep(0.1) raise SystemExit("Xvfb didn't start") def prepare(run_dir: Path, binary: Path, case: str, setup=None, preferences: str = PREFERENCES) -> Path: if run_dir.exists(): shutil.rmtree(run_dir) game = run_dir / "game" shutil.copytree(binary.parent, game, symlinks=True) shutil.rmtree(game / "Songs", ignore_errors=True) (game / "Save").mkdir(parents=True, exist_ok=True) (game / "Portable.ini").touch() (game / "Save/Preferences.ini").write_text(preferences) group = game / "Songs" / make_songs.GROUP for name in {"00-control", case} & set(make_songs.CASES): folder = group / name folder.mkdir(parents=True) for filename, content in make_songs.CASES[name].items(): (folder / filename).write_bytes(content if isinstance(content, bytes) else content.encode()) if setup: setup(game, case) return game / binary.name def crash_reports(game: Path, since: float) -> list[Path]: candidates = [game / "Logs/crashinfo.txt", Path("/tmp/crashinfo.txt")] return [p for p in candidates if p.exists() and p.stat().st_mtime >= since - 1] def run_case(engine: str, binary: Path, case: str, work: Path, out: Path, timeout_s: int, marker: str = SCANNED, setup=None, preferences: str = PREFERENCES, done: str = "loaded", args: tuple = ()) -> dict: binary = prepare(work / engine / case, binary, case, setup, preferences) game = binary.parent log = game / "Logs/log.txt" console = open(out / f"{engine}.{case}.console.txt", "wb") started = time.time() proc = subprocess.Popen([str(binary), *args], cwd=game, stdout=console, stderr=subprocess.STDOUT, start_new_session=True) verdict, scanned_at = None, None while verdict is None: time.sleep(0.25) elapsed = time.time() - started text = log.read_text(errors="replace") if log.exists() else "" if crash_reports(game, started): verdict = "crashed" elif proc.poll() is not None: verdict = "crashed" if proc.returncode != 0 else "exited" elif marker in text: scanned_at = scanned_at or elapsed if elapsed - scanned_at >= GRACE_S: verdict = done elif elapsed >= timeout_s: verdict = "hung" seconds = round(time.time() - started, 1) if proc.poll() is None: os.killpg(proc.pid, signal.SIGKILL) proc.wait() console.close() text = log.read_text(errors="replace") if log.exists() else "" if log.exists(): shutil.copy(log, out / f"{engine}.{case}.log.txt") userlog = game / "Logs/userlog.txt" user_lines = [] if userlog.exists(): shutil.copy(userlog, out / f"{engine}.{case}.userlog.txt") user_lines = [ln.strip() for ln in userlog.read_text(errors="replace").splitlines() if ln.strip()] crash = "" for i, report in enumerate(crash_reports(game, started)): shutil.copy(report, out / f"{engine}.{case}.crashinfo{i or ''}.txt") crash = crash or " ".join(report.read_text(errors="replace").split())[:300] report.unlink() if report.parent == Path("/tmp") else None folder = f"{make_songs.GROUP}/{case}" mentions = [ln.strip() for ln in text.splitlines() if case != "00-control" and case in ln] flagged = [ln.strip() for ln in text.splitlines() if re.search(r"(?i)\b(warn\w*|error|invalid|assert\w*|refus\w*|skipp\w*)\b", ln)] return { "engine": engine, "case": case, "verdict": verdict, "seconds": seconds, "scan_done_at": round(scanned_at, 1) if scanned_at else None, "exit": proc.returncode, "log_mentions_case": mentions[:12], "log_flags": [ln for ln in flagged if folder in ln or make_songs.GROUP in ln or case in ln][:12], "userlog": user_lines[:12], "crash": crash, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary") parser.add_argument("--out", required=True) parser.add_argument("--work", default="/tmp/runs") parser.add_argument("--case", action="append", help="only these cases") parser.add_argument("--timeout", type=int, default=180, help="seconds before a run that never finishes its scan is a hang") args = parser.parse_args() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) engines = dict(e.split("=", 1) for e in args.engine) cases = args.case or list(CASES) display = start_display() results = [] try: for case in cases: for engine, binary in engines.items(): result = run_case(engine, Path(binary), case, Path(args.work), out, args.timeout) results.append(result) print(f"{engine:32} {case:24} {result['verdict']:8} {result['seconds']:6}s exit={result['exit']} {result['crash'][:80]}", flush=True) finally: if display: display.terminate() meta = { "when": datetime.now(timezone.utc).isoformat(timespec="seconds"), "platform": f"{platform.system()} {platform.machine()} ({platform.platform()})", "timeout_s": args.timeout, "engines": engines, "cases": {c: CASES[c] for c in cases}, } (out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2)) print(out / "results.json") if __name__ == "__main__": sys.exit(main())