bench/bench.py (3121B)
1 #!/usr/bin/env python3
2 """벤치마크 하네스. 사용: bench.py <tool> [scenario...]
3 콜드 규율: 매 회 출력 디렉터리와 도구별 캐시를 지운다. taskset -c 0-3, 3회, 중앙값.
4 결과는 results/<tool>-<scenario>.json 으로 append-safe하게 저장."""
5 import json, os, re, shutil, statistics, subprocess, sys, time
6
7 S = os.path.dirname(os.path.abspath(__file__))
8 RUNS = 3
9 GEM_BIN = subprocess.run(["ruby", "-e", "puts Gem.user_dir"], capture_output=True, text=True).stdout.strip() + "/bin"
10
11 TOOLS = {
12 "sqzass": {
13 "cmd": lambda d: [os.environ.get("SQZASS", "target/release/sqzass"),
14 "build", "-i", d, "-o", f"{d}/out"],
15 "clean": lambda d: [f"{d}/out"],
16 "cwd": None,
17 },
18 "hugo": {
19 "cmd": lambda d: ["hugo", "--quiet", "--source", d, "--destination", f"{d}/out"],
20 "clean": lambda d: [f"{d}/out", f"{d}/resources", f"{d}/.hugo_build.lock"],
21 "cwd": None,
22 },
23 "zola": {
24 "cmd": lambda d: ["zola", "--root", d, "build", "--output-dir", f"{d}/out", "--force"],
25 "clean": lambda d: [f"{d}/out"],
26 "cwd": None,
27 },
28 "jekyll": {
29 "cmd": lambda d: [f"{GEM_BIN}/jekyll", "build", "-s", d, "-d", f"{d}/out", "--disable-disk-cache"],
30 "clean": lambda d: [f"{d}/out", f"{d}/.jekyll-cache"],
31 "cwd": None,
32 },
33 "astro": {
34 "cmd": lambda d: ["./node_modules/.bin/astro", "build"],
35 "clean": lambda d: [f"{d}/dist", f"{d}/.astro-cache"],
36 "cwd": lambda d: d,
37 },
38 }
39
40 def clean(paths):
41 for p in paths:
42 if os.path.isdir(p):
43 shutil.rmtree(p, ignore_errors=True)
44 elif os.path.exists(p):
45 os.remove(p)
46
47 def run_one(tool, scenario):
48 t = TOOLS[tool]
49 d = f"{S}/{tool}/{scenario}"
50 times, rss = [], []
51 for i in range(RUNS):
52 clean(t["clean"](d))
53 cmd = ["taskset", "-c", "0-3", "/usr/bin/time", "-v"] + t["cmd"](d)
54 cwd = t["cwd"](d) if t["cwd"] else None
55 t0 = time.monotonic()
56 r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
57 wall = time.monotonic() - t0
58 if r.returncode != 0:
59 print(f"FAIL {tool}/{scenario}: {r.stderr[-500:]}", file=sys.stderr)
60 sys.exit(1)
61 m = re.search(r"Maximum resident set size \(kbytes\): (\d+)", r.stderr)
62 times.append(wall * 1000)
63 rss.append(int(m.group(1)) if m else 0)
64 time.sleep(2)
65 result = {
66 "tool": tool, "scenario": scenario,
67 "median_ms": round(statistics.median(times)),
68 "runs_ms": [round(x) for x in times],
69 "peak_rss_mb": round(max(rss) / 1024),
70 }
71 os.makedirs(f"{S}/results", exist_ok=True)
72 with open(f"{S}/results/{tool}-{scenario}.json", "w") as f:
73 json.dump(result, f)
74 print(f"{tool:7} {scenario:8} 중앙값 {result['median_ms']:>7} ms "
75 f"(runs: {', '.join(str(x) for x in result['runs_ms'])}) peakRSS {result['peak_rss_mb']} MB")
76
77 tool = sys.argv[1]
78 scenarios = sys.argv[2:] or ["minimal", "blog", "heavyr", "heavyu"]
79 for sc in scenarios:
80 run_one(tool, sc)