Skip to content

Check it yourself

stepmaniay-verification Public

Tests anyone can run against StepManiaY's published builds and ITGmania 1.3.0, the release it grew from.

Clone

Clone with HTTPS

Plain git over HTTPS: no account, no app, nothing else to install.

Download the source tarball 3.0 MB

199 lines · 8.2 KB

Raw
"""Runs every stability case on every engine, each in a fresh portable copy of the engine, and
records what the engine did with it: loaded the song list, crashed, or hung. What an engine says
about a file it skips goes to Logs/userlog.txt, kept beside each run's log.

    python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out <results folder>

Run it in the kit's container (../../verify.sh), which has a virtual display and software OpenGL.
Each run is a full copy of the engine's folder beside a Portable.ini, with the control song and one
case in Songs, the Null sound driver and no network. Both engines log "Loading courses." once every
song is loaded; a run that logs it and is still running 5 s later loaded the list. A crash report
or an exit before that is a crash; neither within the time limit is a hang.
"""

from __future__ import annotations

import argparse
import json
import os
import platform
import re
import shutil
import signal
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path

import make_songs

GRACE_S = 5
SCANNED = "Loading courses."

# What each case reaches in ITGmania 1.3.0 (github.com/itgmania/itgmania, tag v1.3.0).
CASES = {
    "00-control": "A valid song; every engine should load it.",
    "01-wav-negative-chunk": "src/RageSoundReader_WAV.cpp:497 adds the chunk's size to the read position without "
    "checking it; -8 seeks back to the same chunk header forever.",
    "02-ssc-empty-speeds": "src/NotesLoaderSM.cpp:834 reads vs2[0] before checking vs2.size(); "
    '"=" splits into no fields.',
    "03-ssc-timesig-1-256": "Nothing in 1.3.0 rejects a signature under one note row a measure, but nothing in its "
    "song scan divides by one either: the old engine should load this list. Its hang is in tests/playback.",
    "04-dwi-lowercase-mode": 'src/NotesLoaderDWI.cpp:278 compares the mode with == "SINGLE"; '
    "#single: reaches ASSERT_M(0, ...) at :287.",
}

PREFERENCES = """[Options]
SoundDrivers=Null
Windowed=1
DisplayWidth=640
DisplayHeight=360
LogToDisk=1
ForceLogFlush=1
GrooveStatsApiUrl=http://127.0.0.1:9/
GrooveStatsLinkPage=http://127.0.0.1:9/
GrooveStatsLinkUrl=ws://127.0.0.1:9/
GrooveStatsCrossPost=0
GrooveStatsUnlockDownloads=0
ArrowCloudApiUrl=http://127.0.0.1:9/
ArrowCloudCrossPost=0
BoogieStatsApiUrl=http://127.0.0.1:9/
StepManiaYServer=http://127.0.0.1:9
StepManiaYAskToSignIn=0
"""


def start_display() -> subprocess.Popen | None:
    if os.path.exists("/tmp/.X11-unix/X99"):
        return None
    xvfb = subprocess.Popen(["Xvfb", ":99", "-screen", "0", "1280x720x24", "-nolisten", "tcp"],
                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    for _ in range(100):
        if os.path.exists("/tmp/.X11-unix/X99"):
            return xvfb
        time.sleep(0.1)
    raise SystemExit("Xvfb didn't start")


def prepare(run_dir: Path, binary: Path, case: str, setup=None, preferences: str = PREFERENCES) -> Path:
    if run_dir.exists():
        shutil.rmtree(run_dir)
    game = run_dir / "game"
    shutil.copytree(binary.parent, game, symlinks=True)
    shutil.rmtree(game / "Songs", ignore_errors=True)
    (game / "Save").mkdir(parents=True, exist_ok=True)
    (game / "Portable.ini").touch()
    (game / "Save/Preferences.ini").write_text(preferences)
    group = game / "Songs" / make_songs.GROUP
    for name in {"00-control", case} & set(make_songs.CASES):
        folder = group / name
        folder.mkdir(parents=True)
        for filename, content in make_songs.CASES[name].items():
            (folder / filename).write_bytes(content if isinstance(content, bytes) else content.encode())
    if setup:
        setup(game, case)
    return game / binary.name


def crash_reports(game: Path, since: float) -> list[Path]:
    candidates = [game / "Logs/crashinfo.txt", Path("/tmp/crashinfo.txt")]
    return [p for p in candidates if p.exists() and p.stat().st_mtime >= since - 1]


def run_case(engine: str, binary: Path, case: str, work: Path, out: Path, timeout_s: int,
             marker: str = SCANNED, setup=None, preferences: str = PREFERENCES, done: str = "loaded",
             args: tuple = ()) -> dict:
    binary = prepare(work / engine / case, binary, case, setup, preferences)
    game = binary.parent
    log = game / "Logs/log.txt"
    console = open(out / f"{engine}.{case}.console.txt", "wb")
    started = time.time()
    proc = subprocess.Popen([str(binary), *args], cwd=game, stdout=console, stderr=subprocess.STDOUT, start_new_session=True)
    verdict, scanned_at = None, None
    while verdict is None:
        time.sleep(0.25)
        elapsed = time.time() - started
        text = log.read_text(errors="replace") if log.exists() else ""
        if crash_reports(game, started):
            verdict = "crashed"
        elif proc.poll() is not None:
            verdict = "crashed" if proc.returncode != 0 else "exited"
        elif marker in text:
            scanned_at = scanned_at or elapsed
            if elapsed - scanned_at >= GRACE_S:
                verdict = done
        elif elapsed >= timeout_s:
            verdict = "hung"
    seconds = round(time.time() - started, 1)
    if proc.poll() is None:
        os.killpg(proc.pid, signal.SIGKILL)
    proc.wait()
    console.close()

    text = log.read_text(errors="replace") if log.exists() else ""
    if log.exists():
        shutil.copy(log, out / f"{engine}.{case}.log.txt")
    userlog = game / "Logs/userlog.txt"
    user_lines = []
    if userlog.exists():
        shutil.copy(userlog, out / f"{engine}.{case}.userlog.txt")
        user_lines = [ln.strip() for ln in userlog.read_text(errors="replace").splitlines() if ln.strip()]
    crash = ""
    for i, report in enumerate(crash_reports(game, started)):
        shutil.copy(report, out / f"{engine}.{case}.crashinfo{i or ''}.txt")
        crash = crash or " ".join(report.read_text(errors="replace").split())[:300]
        report.unlink() if report.parent == Path("/tmp") else None
    folder = f"{make_songs.GROUP}/{case}"
    mentions = [ln.strip() for ln in text.splitlines() if case != "00-control" and case in ln]
    flagged = [ln.strip() for ln in text.splitlines() if re.search(r"(?i)\b(warn\w*|error|invalid|assert\w*|refus\w*|skipp\w*)\b", ln)]
    return {
        "engine": engine,
        "case": case,
        "verdict": verdict,
        "seconds": seconds,
        "scan_done_at": round(scanned_at, 1) if scanned_at else None,
        "exit": proc.returncode,
        "log_mentions_case": mentions[:12],
        "log_flags": [ln for ln in flagged if folder in ln or make_songs.GROUP in ln or case in ln][:12],
        "userlog": user_lines[:12],
        "crash": crash,
    }


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary")
    parser.add_argument("--out", required=True)
    parser.add_argument("--work", default="/tmp/runs")
    parser.add_argument("--case", action="append", help="only these cases")
    parser.add_argument("--timeout", type=int, default=180, help="seconds before a run that never finishes its scan is a hang")
    args = parser.parse_args()
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    engines = dict(e.split("=", 1) for e in args.engine)
    cases = args.case or list(CASES)
    display = start_display()
    results = []
    try:
        for case in cases:
            for engine, binary in engines.items():
                result = run_case(engine, Path(binary), case, Path(args.work), out, args.timeout)
                results.append(result)
                print(f"{engine:32} {case:24} {result['verdict']:8} {result['seconds']:6}s exit={result['exit']}  {result['crash'][:80]}", flush=True)
    finally:
        if display:
            display.terminate()
    meta = {
        "when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "platform": f"{platform.system()} {platform.machine()} ({platform.platform()})",
        "timeout_s": args.timeout,
        "engines": engines,
        "cases": {c: CASES[c] for c in cases},
    }
    (out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2))
    print(out / "results.json")


if __name__ == "__main__":
    sys.exit(main())