"""Runs every stability case on every engine, each in a fresh portable copy of the engine, and
records what the engine did with it: loaded the song list, crashed, or hung. What an engine says
about a file it skips goes to Logs/userlog.txt, kept beside each run's log.
python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out <results folder>
Run it in the kit's container (../../verify.sh), which has a virtual display and software OpenGL.
Each run is a full copy of the engine's folder beside a Portable.ini, with the control song and one
case in Songs, the Null sound driver and no network. Both engines log "Loading courses." once every
song is loaded; a run that logs it and is still running 5 s later loaded the list. A crash report
or an exit before that is a crash; neither within the time limit is a hang.
"""
from __future__ import annotations
import argparse
import json
import os
import platform
import re
import shutil
import signal
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import make_songs
GRACE_S = 5
SCANNED = "Loading courses."
# What each case reaches in ITGmania 1.3.0 (github.com/itgmania/itgmania, tag v1.3.0).
CASES = {
"00-control": "A valid song; every engine should load it.",
"01-wav-negative-chunk": "src/RageSoundReader_WAV.cpp:497 adds the chunk's size to the read position without "
"checking it; -8 seeks back to the same chunk header forever.",
"02-ssc-empty-speeds": "src/NotesLoaderSM.cpp:834 reads vs2[0] before checking vs2.size(); "
'"=" splits into no fields.',
"03-ssc-timesig-1-256": "Nothing in 1.3.0 rejects a signature under one note row a measure, but nothing in its "
"song scan divides by one either: the old engine should load this list. Its hang is in tests/playback.",
"04-dwi-lowercase-mode": 'src/NotesLoaderDWI.cpp:278 compares the mode with == "SINGLE"; '
"#single: reaches ASSERT_M(0, ...) at :287.",
}
PREFERENCES = """[Options]
SoundDrivers=Null
Windowed=1
DisplayWidth=640
DisplayHeight=360
LogToDisk=1
ForceLogFlush=1
GrooveStatsApiUrl=http://127.0.0.1:9/
GrooveStatsLinkPage=http://127.0.0.1:9/
GrooveStatsLinkUrl=ws://127.0.0.1:9/
GrooveStatsCrossPost=0
GrooveStatsUnlockDownloads=0
ArrowCloudApiUrl=http://127.0.0.1:9/
ArrowCloudCrossPost=0
BoogieStatsApiUrl=http://127.0.0.1:9/
StepManiaYServer=http://127.0.0.1:9
StepManiaYAskToSignIn=0
"""
def start_display() -> subprocess.Popen | None:
if os.path.exists("/tmp/.X11-unix/X99"):
return None
xvfb = subprocess.Popen(["Xvfb", ":99", "-screen", "0", "1280x720x24", "-nolisten", "tcp"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
for _ in range(100):
if os.path.exists("/tmp/.X11-unix/X99"):
return xvfb
time.sleep(0.1)
raise SystemExit("Xvfb didn't start")
def prepare(run_dir: Path, binary: Path, case: str, setup=None, preferences: str = PREFERENCES) -> Path:
if run_dir.exists():
shutil.rmtree(run_dir)
game = run_dir / "game"
shutil.copytree(binary.parent, game, symlinks=True)
shutil.rmtree(game / "Songs", ignore_errors=True)
(game / "Save").mkdir(parents=True, exist_ok=True)
(game / "Portable.ini").touch()
(game / "Save/Preferences.ini").write_text(preferences)
group = game / "Songs" / make_songs.GROUP
for name in {"00-control", case} & set(make_songs.CASES):
folder = group / name
folder.mkdir(parents=True)
for filename, content in make_songs.CASES[name].items():
(folder / filename).write_bytes(content if isinstance(content, bytes) else content.encode())
if setup:
setup(game, case)
return game / binary.name
def crash_reports(game: Path, since: float) -> list[Path]:
candidates = [game / "Logs/crashinfo.txt", Path("/tmp/crashinfo.txt")]
return [p for p in candidates if p.exists() and p.stat().st_mtime >= since - 1]
def run_case(engine: str, binary: Path, case: str, work: Path, out: Path, timeout_s: int,
marker: str = SCANNED, setup=None, preferences: str = PREFERENCES, done: str = "loaded",
args: tuple = ()) -> dict:
binary = prepare(work / engine / case, binary, case, setup, preferences)
game = binary.parent
log = game / "Logs/log.txt"
console = open(out / f"{engine}.{case}.console.txt", "wb")
started = time.time()
proc = subprocess.Popen([str(binary), *args], cwd=game, stdout=console, stderr=subprocess.STDOUT, start_new_session=True)
verdict, scanned_at = None, None
while verdict is None:
time.sleep(0.25)
elapsed = time.time() - started
text = log.read_text(errors="replace") if log.exists() else ""
if crash_reports(game, started):
verdict = "crashed"
elif proc.poll() is not None:
verdict = "crashed" if proc.returncode != 0 else "exited"
elif marker in text:
scanned_at = scanned_at or elapsed
if elapsed - scanned_at >= GRACE_S:
verdict = done
elif elapsed >= timeout_s:
verdict = "hung"
seconds = round(time.time() - started, 1)
if proc.poll() is None:
os.killpg(proc.pid, signal.SIGKILL)
proc.wait()
console.close()
text = log.read_text(errors="replace") if log.exists() else ""
if log.exists():
shutil.copy(log, out / f"{engine}.{case}.log.txt")
userlog = game / "Logs/userlog.txt"
user_lines = []
if userlog.exists():
shutil.copy(userlog, out / f"{engine}.{case}.userlog.txt")
user_lines = [ln.strip() for ln in userlog.read_text(errors="replace").splitlines() if ln.strip()]
crash = ""
for i, report in enumerate(crash_reports(game, started)):
shutil.copy(report, out / f"{engine}.{case}.crashinfo{i or ''}.txt")
crash = crash or " ".join(report.read_text(errors="replace").split())[:300]
report.unlink() if report.parent == Path("/tmp") else None
folder = f"{make_songs.GROUP}/{case}"
mentions = [ln.strip() for ln in text.splitlines() if case != "00-control" and case in ln]
flagged = [ln.strip() for ln in text.splitlines() if re.search(r"(?i)\b(warn\w*|error|invalid|assert\w*|refus\w*|skipp\w*)\b", ln)]
return {
"engine": engine,
"case": case,
"verdict": verdict,
"seconds": seconds,
"scan_done_at": round(scanned_at, 1) if scanned_at else None,
"exit": proc.returncode,
"log_mentions_case": mentions[:12],
"log_flags": [ln for ln in flagged if folder in ln or make_songs.GROUP in ln or case in ln][:12],
"userlog": user_lines[:12],
"crash": crash,
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary")
parser.add_argument("--out", required=True)
parser.add_argument("--work", default="/tmp/runs")
parser.add_argument("--case", action="append", help="only these cases")
parser.add_argument("--timeout", type=int, default=180, help="seconds before a run that never finishes its scan is a hang")
args = parser.parse_args()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
engines = dict(e.split("=", 1) for e in args.engine)
cases = args.case or list(CASES)
display = start_display()
results = []
try:
for case in cases:
for engine, binary in engines.items():
result = run_case(engine, Path(binary), case, Path(args.work), out, args.timeout)
results.append(result)
print(f"{engine:32} {case:24} {result['verdict']:8} {result['seconds']:6}s exit={result['exit']} {result['crash'][:80]}", flush=True)
finally:
if display:
display.terminate()
meta = {
"when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"platform": f"{platform.system()} {platform.machine()} ({platform.platform()})",
"timeout_s": args.timeout,
"engines": engines,
"cases": {c: CASES[c] for c in cases},
}
(out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2))
print(out / "results.json")
if __name__ == "__main__":
sys.exit(main())