Skip to content

Check it yourself

stepmaniay-verification Public

Tests anyone can run against StepManiaY's published builds and ITGmania 1.3.0, the release it grew from.

Clone

Clone with HTTPS

Plain git over HTTPS: no account, no app, nothing else to install.

Download the source tarball 3.0 MB

115 lines · 4.4 KB

Raw
"""Plays each case's song, for the file bugs that only show once a song is played: a minimal theme
on the engines' own _fallback picks the song and its chart and opens gameplay with nobody pressing
anything, the same Lua on both engines. A run whose gameplay screen is up and still running 5 s
later played; a crash report or an early exit is a crash; neither within the time limit is a hang.

    python3 run.py --engine NAME=/path/to/engine/binary [--engine ...] --out <results folder>
"""

from __future__ import annotations

import argparse
import json
import platform
import sys
from datetime import datetime, timezone
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "stability"))
import make_songs  # noqa: E402
import run as stability  # noqa: E402

PLAYING = "SMYVERIFY gameplay running"

# What each case reaches in ITGmania 1.3.0 (github.com/itgmania/itgmania, tag v1.3.0).
CASES = {
    "00-control": "A valid song; every engine should play it.",
    "03-ssc-timesig-1-256": "src/Background.cpp:499 steps through the song a measure of rows at a time to place "
    "random backgrounds; at 1/256 a measure is 0 rows, so j never advances (and :525 takes % 0).",
}

METRICS = """[Global]
FallbackTheme=_fallback

[Common]
InitialScreen="ScreenVerify"

[ScreenVerify]
Class="ScreenWithMenuElements"
Fallback="ScreenWithMenuElements"
PrevScreen="ScreenVerify"
NextScreen="ScreenGameplay"
TimerSeconds=-1
"""

START = """-- Opens gameplay on one song with player 1 joined, then gets out of the way.
local SONG = "{song}"
return Def.ActorFrame{{
  OnCommand=function(self) self:sleep(0.5):queuecommand("Go") end,
  GoCommand=function(self)
    local song = SONGMAN:FindSong(SONG)
    if not song then Trace("SMYVERIFY no song " .. SONG) return end
    GAMESTATE:SetCurrentPlayMode("PlayMode_Regular")
    GAMESTATE:JoinPlayer(PLAYER_1)
    GAMESTATE:SetCurrentStyle("single")
    GAMESTATE:SetCurrentSong(song)
    GAMESTATE:SetCurrentSteps(PLAYER_1, song:GetAllSteps()[1])
    Trace("SMYVERIFY gameplay requested")
    SCREENMAN:SetNewScreen("ScreenGameplay")
  end,
}}
"""

GAMEPLAY = """return Def.ActorFrame{
  OnCommand=function(self) Trace("SMYVERIFY gameplay running") end,
}
"""


def setup(game: Path, case: str) -> None:
    theme = game / "Themes/SMY Verify"
    (theme / "BGAnimations/ScreenVerify overlay").mkdir(parents=True)
    (theme / "BGAnimations/ScreenGameplay overlay").mkdir(parents=True)
    (theme / "metrics.ini").write_text(METRICS)
    (theme / "BGAnimations/ScreenVerify overlay/default.lua").write_text(START.format(song=f"{make_songs.GROUP}/{case}"))
    (theme / "BGAnimations/ScreenGameplay overlay/default.lua").write_text(GAMEPLAY)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--engine", action="append", required=True, help="NAME=/path/to/engine/binary")
    parser.add_argument("--out", required=True)
    parser.add_argument("--work", default="/tmp/runs")
    parser.add_argument("--case", action="append", help="only these cases")
    parser.add_argument("--timeout", type=int, default=120)
    args = parser.parse_args()
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    engines = dict(e.split("=", 1) for e in args.engine)
    cases = args.case or list(CASES)
    preferences = stability.PREFERENCES + "\n[Game-dance]\nTheme=SMY Verify\n"
    display = stability.start_display()
    results = []
    try:
        for case in cases:
            for engine, binary in engines.items():
                result = stability.run_case(engine, Path(binary), case, Path(args.work), out, args.timeout,
                                            marker=PLAYING, setup=setup, preferences=preferences, done="played")
                results.append(result)
                print(f"{engine:32} {case:24} {result['verdict']:8} {result['seconds']:6}s exit={result['exit']}  {result['crash'][:80]}", flush=True)
    finally:
        if display:
            display.terminate()
    meta = {
        "when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
        "platform": f"{platform.system()} {platform.machine()} ({platform.platform()})",
        "timeout_s": args.timeout,
        "engines": engines,
        "cases": {c: CASES[c] for c in cases},
    }
    (out / "results.json").write_text(json.dumps({"meta": meta, "runs": results}, indent=2))
    print(out / "results.json")


if __name__ == "__main__":
    sys.exit(main())