scripts/build.py (8549B)
1 #!/usr/bin/env python3
2 """Generate a local stagit site from debuman's public repositories."""
3
4 import argparse
5 from concurrent.futures import ThreadPoolExecutor
6 from datetime import datetime, timezone
7 import json
8 import os
9 from pathlib import Path
10 import re
11 import shutil
12 import signal
13 import subprocess
14 from urllib.request import Request, urlopen
15 from readme_assets import link_readme_assets
16
17 ROOT = Path(__file__).resolve().parents[1]
18 CACHE = ROOT / ".cache"
19 OWNER = "debuman"
20 VERSION = "sqzer-stagit 0.1.0"
21 SOURCE = ROOT / "generator"
22 BINARY = CACHE / "cargo" / "release" / "sqzer-stagit"
23 ASSETS = ("style.css",)
24
25
26 def run(*args, **kwargs):
27 timeout = kwargs.pop("timeout", 300)
28 command = [str(arg) for arg in args]
29 process = subprocess.Popen(command, start_new_session=True, **kwargs)
30 try:
31 stdout, stderr = process.communicate(timeout=timeout)
32 except BaseException:
33 # Stop the entire local process group, including generator Git children.
34 try:
35 os.killpg(process.pid, signal.SIGKILL)
36 except ProcessLookupError:
37 pass
38 process.wait()
39 raise
40 if process.returncode:
41 raise subprocess.CalledProcessError(process.returncode, command, stdout, stderr)
42 return subprocess.CompletedProcess(command, process.returncode, stdout, stderr)
43
44
45 def request(url):
46 headers = {"User-Agent": "oni.sqzer.com", "Accept": "application/vnd.github+json"}
47 if url.startswith("https://api.github.com/") and (token := os.environ.get("GITHUB_TOKEN")):
48 headers["Authorization"] = "Bearer " + token
49 req = Request(url, headers=headers)
50 with urlopen(req, timeout=90) as response:
51 data = response.read(8 * 1024 * 1024 + 1)
52 if len(data) > 8 * 1024 * 1024:
53 raise RuntimeError("GitHub API response exceeds the 8 MiB limit")
54 return data
55
56
57 def configure_git():
58 local = CACHE / "secure-git/installed/bin"
59 if (local / "git").is_file():
60 os.environ["PATH"] = str(local) + os.pathsep + os.environ["PATH"]
61 version = subprocess.check_output(["git", "--version"], text=True, timeout=10).strip()
62 match = re.search(r"git version (\d+)\.(\d+)\.(\d+)", version)
63 if not match or tuple(map(int, match.groups())) < (2, 56, 1):
64 raise RuntimeError("Git 2.56.1 or newer is required. Run python3 scripts/bootstrap_git.py inside WSL")
65 print(f"Using {version}", flush=True)
66
67
68 def build_tool():
69 run("cargo", "build", "--release", "--offline", "--locked",
70 "--target-dir", CACHE / "cargo", cwd=SOURCE)
71
72
73 def repositories(offline):
74 metadata = CACHE / "repositories.json"
75 if offline:
76 repos = json.loads(metadata.read_text(encoding="utf-8"))
77 else:
78 repos = []
79 page = 1
80 while True:
81 batch = json.loads(request(
82 f"https://api.github.com/users/{OWNER}/repos?type=owner&per_page=100&page={page}"
83 ))
84 if not batch:
85 break
86 repos.extend(batch)
87 page += 1
88 repos = [repo for repo in repos if not repo["private"]
89 and repo["owner"]["login"].lower() == OWNER.lower()]
90 if not repos:
91 raise RuntimeError("No public repositories found; keeping the existing preview")
92 metadata.write_text(json.dumps(repos, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
93 for repo in repos:
94 name = repo["name"]
95 if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or name in (".", ".."):
96 raise RuntimeError(f"Unsafe repository name: {name!r}")
97 if repo["private"] or repo["owner"]["login"].lower() != OWNER.lower():
98 raise RuntimeError("Only public repositories owned by debuman are allowed")
99 if repo["clone_url"] != f"https://github.com/{OWNER}/{name}.git":
100 raise RuntimeError(f"Unexpected repository URL for {name}")
101 if repo.get("full_name") != f"{OWNER}/{name}":
102 raise RuntimeError(f"Unexpected repository identity for {name}")
103 run("git", "check-ref-format", f"refs/heads/{repo['default_branch']}", stdout=subprocess.DEVNULL)
104 return sorted(repos, key=lambda repo: repo["name"].lower())
105
106
107 def sync_repo(repo, offline):
108 target = CACHE / "repos" / (repo["name"] + ".git")
109 if not target.exists():
110 if offline:
111 raise RuntimeError(f"Missing local repository: {target}")
112 run("git", "-c", "credential.helper=", "-c", "core.hooksPath=/dev/null",
113 "-c", "protocol.allow=never", "-c", "protocol.https.allow=always", "-c", "http.sslVerify=true",
114 "-c", "transfer.fsckObjects=true", "clone", "--bare", "--quiet",
115 "--", repo["clone_url"], target)
116 elif not offline:
117 run("git", "-C", target, "-c", "credential.helper=", "-c", "core.hooksPath=/dev/null",
118 "-c", "protocol.allow=never", "-c", "protocol.https.allow=always", "-c", "http.sslVerify=true",
119 "-c", "fetch.fsckObjects=true", "-c", "fetch.recurseSubmodules=false", "fetch", "--quiet",
120 "--prune", "--prune-tags", repo["clone_url"], "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*")
121 # These local copies are used only as generator input.
122 run("git", "-C", target, "config", "remote.origin.pushurl", "DISABLED")
123 run("git", "-C", target, "symbolic-ref", "HEAD", f"refs/heads/{repo['default_branch']}")
124 for name, value in {
125 "description": repo.get("description") or "",
126 "owner": OWNER,
127 "url": repo["clone_url"],
128 }.items():
129 (target / name).write_text(value.replace("\n", " ").strip() + "\n", encoding="utf-8")
130 print(f"Ready: {repo['name']}", flush=True)
131 return target
132
133
134 def generate(repos, paths, base_url):
135 stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
136 staging = CACHE / ("build-" + stamp)
137 staging.mkdir()
138 for asset in ASSETS:
139 shutil.copyfile(ROOT / "theme" / asset, staging / asset)
140 notices = staging / "licenses"
141 notices.mkdir()
142 shutil.copyfile(ROOT / "LICENSE", notices / "sqzer-stagit.txt")
143 for notice in (ROOT / "third-party").glob("*LICENSE*"):
144 shutil.copyfile(notice, notices / notice.name)
145 for number, (repo, path) in enumerate(zip(repos, paths), 1):
146 output = staging / repo["name"]
147 output.mkdir()
148 print(f"[{number}/{len(repos)}] Generating {repo['name']}", flush=True)
149 run("prlimit", "--as=2147483648", "--cpu=120", "--", BINARY, "-l", "50", "-u",
150 f"{base_url.rstrip('/')}/{repo['name']}/", path, cwd=output, timeout=180)
151 run("git", "-C", path, "archive", "--format=tar.gz",
152 f"--prefix={repo['name']}/", f"--output={output / 'archive.tar.gz'}", "HEAD")
153 link_readme_assets(output, path, repo)
154 for asset in ASSETS:
155 shutil.copyfile(ROOT / "theme" / asset, output / asset)
156 with (staging / "index.html").open("w", encoding="utf-8") as index:
157 run("prlimit", "--as=2147483648", "--cpu=120", "--", BINARY, "--index", *paths, stdout=index, timeout=180)
158 public = ROOT / "public"
159 if public.exists():
160 # Keep the previous preview as a local backup; never delete source files.
161 public.rename(CACHE / ("previous-" + stamp))
162 staging.rename(public)
163 report = {
164 "generated_at": stamp,
165 "stagit_version": VERSION,
166 "base_url": base_url,
167 "intended_domain": "oni.sqzer.com",
168 "repositories": [repo["name"] for repo in repos],
169 }
170 (CACHE / "last-build.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
171 print(f"Generated {len(repos)} repositories in {public}", flush=True)
172
173
174 def main():
175 parser = argparse.ArgumentParser(description=__doc__)
176 parser.add_argument("--offline", action="store_true", help="Rebuild from the local snapshot without fetching GitHub")
177 parser.add_argument("--base-url", default="http://127.0.0.1:4174", help="Absolute URL for Atom feed links")
178 args = parser.parse_args()
179 if os.name == "nt":
180 parser.error("Run build.ps1 to build with WSL Ubuntu on Windows")
181 CACHE.mkdir(exist_ok=True)
182 (CACHE / "repos").mkdir(exist_ok=True)
183 os.environ["GIT_TERMINAL_PROMPT"] = "0"
184 configure_git()
185 build_tool()
186 repos = repositories(args.offline)
187 print(f"Public repositories: {len(repos)}", flush=True)
188 with ThreadPoolExecutor(max_workers=4) as pool:
189 paths = list(pool.map(lambda repo: sync_repo(repo, args.offline), repos))
190 generate(repos, paths, args.base_url)
191
192
193 if __name__ == "__main__":
194 main()