scripts/benchmark.py (7690B)
1 #!/usr/bin/env python3
2 """Compare generator scaling and byte-identical output using temporary Git fixtures."""
3
4 import argparse
5 import hashlib
6 import importlib.util
7 import json
8 import os
9 from pathlib import Path
10 import statistics
11 import subprocess
12 import tempfile
13 import time
14
15 ROOT = Path(__file__).resolve().parents[1]
16
17
18 def fixture(path, files, commits=1, images=False):
19 subprocess.run(["git", "init", "-q", "--bare", "--initial-branch=main", path], check=True)
20 payload = "\x89PNG\r\n\x1a\nimage" if images else "text\n"
21 suffix = "png" if images else "txt"
22 data = [f"blob\nmark :1\ndata {len(payload.encode('latin1'))}\n{payload}\n"]
23 for number in range(commits):
24 message = f"Commit {number}\n"
25 data.append(f"commit refs/heads/main\ncommitter Tester <test@example.invalid> "
26 f"{1767225600 + number} +0000\ndata {len(message)}\n{message}")
27 if number == 0:
28 data.extend(f"M 100644 :1 files/{index:06}.{suffix}\n" for index in range(files))
29 else:
30 content = f"value {number}\n"
31 data.append(f"M 100644 inline history.txt\ndata {len(content)}\n{content}")
32 data.append("\n")
33 subprocess.run(["git", "-C", path, "fast-import", "--quiet"],
34 input="".join(data).encode("latin1"), check=True)
35
36
37 def manifest(output):
38 return {str(path.relative_to(output)): hashlib.sha256(path.read_bytes()).hexdigest()
39 for path in output.rglob("*") if path.is_file()}
40
41
42 def timed(function, repeats):
43 samples = []
44 for _ in range(repeats):
45 start = time.perf_counter()
46 function()
47 samples.append(time.perf_counter() - start)
48 return {"median_seconds": statistics.median(samples), "samples": samples}
49
50
51 def assets(path):
52 spec = importlib.util.spec_from_file_location("assets", path)
53 module = importlib.util.module_from_spec(spec)
54 spec.loader.exec_module(module)
55 return module
56
57
58 def git_processes(command, trace):
59 subprocess.run(command, check=True, env=dict(os.environ, GIT_TRACE2_EVENT=str(trace)))
60 starts = [event["argv"] for line in trace.read_text().splitlines()
61 if (event := json.loads(line)).get("event") == "start"]
62 return {"total": len(starts), "diff_tree": sum("diff-tree" in argv for argv in starts)}
63
64
65 def image_benchmark(root, count, paths, repeats):
66 repo = root / "images.git"
67 fixture(repo, count, images=True)
68 results = {"case": f"images-{count}", "images": count}
69 outputs = {}
70 for label, path in paths.items():
71 module = assets(path)
72 sample = 0
73 def copy_images():
74 nonlocal sample
75 output = root / f"images-{label}-{sample}"
76 sample += 1
77 output.mkdir()
78 resolver = module.ReadmeLinks(output, repo, {"full_name": "test/repo", "default_branch": "main"})
79 try:
80 for index in range(count):
81 name = f"files/{index:06}.png"
82 assert resolver.resolve_link(name, True) == "raw/" + name
83 finally:
84 resolver.close()
85 outputs[label] = output
86 copy_images()
87 results[label] = timed(copy_images, repeats)
88 assert manifest(outputs["before"]) == manifest(outputs["after"])
89 results["identical_files"] = count
90 results["speedup"] = results["before"]["median_seconds"] / results["after"]["median_seconds"]
91 return results
92
93
94 def main():
95 parser = argparse.ArgumentParser(description=__doc__)
96 parser.add_argument("--baseline", type=Path, required=True, help="Generator binary before optimization")
97 parser.add_argument("--binary", type=Path, default=ROOT / ".cache/cargo/release/sqzer-stagit")
98 parser.add_argument("--baseline-assets", type=Path, help="Optional previous readme_assets.py")
99 parser.add_argument("--sizes", type=int, nargs="+", default=[1000, 4000])
100 parser.add_argument("--history", type=int, default=300)
101 parser.add_argument("--images", type=int, default=100)
102 parser.add_argument("--repeats", type=int, default=3)
103 parser.add_argument("--report", type=Path)
104 args = parser.parse_args()
105 if args.repeats < 1 or args.history < 1 or args.images < 1 or any(size < 1 for size in args.sizes):
106 parser.error("Sizes, history, images and repeats must be positive")
107 binaries = {"before": args.baseline.resolve(), "after": args.binary.resolve()}
108 report = []
109 with tempfile.TemporaryDirectory(prefix="sqzer-benchmark-") as directory:
110 root = Path(directory)
111 cases = [(f"files-{size}", size, 1) for size in args.sizes]
112 cases.append((f"history-{args.history}", 10, args.history))
113 for name, count, commits in cases:
114 repo = root / (name + ".git")
115 fixture(repo, count, commits)
116 result = {"case": name, "files": count, "commits": commits}
117 outputs = {}
118 indexes = {}
119 for label, binary in binaries.items():
120 output = root / (name + "-" + label)
121 command = [str(binary), "--output", str(output), str(repo)]
122 # Warm code/data caches equally; measured runs reuse the same
123 # output directory. Full-site cold writes are measured separately.
124 subprocess.run(command, check=True)
125 result[label] = timed(lambda: subprocess.run(command, check=True), args.repeats)
126 result[label]["git_processes"] = git_processes(command, root / f"{name}-{label}.jsonl")
127 outputs[label] = manifest(output)
128 index_command = [str(binary), "--index", str(repo)]
129 indexes[label] = subprocess.check_output(index_command)
130 result[label]["index"] = timed(lambda: subprocess.check_output(index_command), args.repeats)
131 assert outputs["before"] == outputs["after"], f"Output changed: {name}"
132 assert indexes["before"] == indexes["after"], f"Catalog changed: {name}"
133 result["identical_files"] = len(outputs["after"])
134 result["speedup"] = result["before"]["median_seconds"] / result["after"]["median_seconds"]
135 if args.baseline_assets and commits == 1:
136 # Missing relative links exercise directory lookup without an
137 # early match. Each run includes construction of the directory index.
138 links = [f"absent-{index}" for index in range(count)]
139 metadata = {"full_name": "test/repo", "default_branch": "main"}
140 asset_results = {}
141 for label, path in (("before", args.baseline_assets),
142 ("after", ROOT / "scripts/readme_assets.py")):
143 module = assets(path)
144 def resolve():
145 resolver = module.ReadmeLinks(root, repo, metadata)
146 try:
147 assert [resolver.resolve_link(link, False) for link in links] == links
148 finally:
149 resolver.close()
150 asset_results[label] = timed(resolve, args.repeats)
151 result["readme_links"] = asset_results
152 report.append(result)
153 print(json.dumps(result), flush=True)
154 if args.baseline_assets:
155 result = image_benchmark(root, args.images,
156 {"before": args.baseline_assets, "after": ROOT / "scripts/readme_assets.py"}, args.repeats)
157 report.append(result)
158 print(json.dumps(result), flush=True)
159 if args.report:
160 args.report.write_text(json.dumps(report, indent=2) + "\n")
161
162
163 if __name__ == "__main__":
164 main()