generator/src/repo.rs (19350B)
1 use crate::Result;
2 use std::borrow::Cow;
3 use std::collections::HashSet;
4 use std::fs;
5 use std::io::{BufRead, BufReader, Read, Write};
6 use std::path::{Path, PathBuf};
7 use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
8
9 pub const MAX_BLOB_BYTES: usize = 32 * 1024 * 1024;
10 const MAX_DIFF_BYTES: usize = 64 * 1024 * 1024;
11 const MAX_GIT_OUTPUT_BYTES: u64 = 64 * 1024 * 1024;
12
13 fn object_id(value: &[u8]) -> Result<String> {
14 if !matches!(value.len(), 40 | 64) || !value.iter().all(u8::is_ascii_hexdigit) {
15 return Err("invalid Git object ID".into());
16 }
17 Ok(String::from_utf8(value.to_vec())?)
18 }
19
20 #[derive(Debug)]
21 pub struct Commit {
22 pub oid: String,
23 pub parents: Vec<String>,
24 pub author: String,
25 pub email: String,
26 pub timestamp: i64,
27 pub iso_date: String,
28 pub display_date: String,
29 pub subject: String,
30 pub message: String,
31 }
32
33 pub struct Entry {
34 pub mode: String,
35 pub kind: String,
36 pub oid: String,
37 pub size: usize,
38 pub path: String,
39 }
40
41 pub struct Change {
42 pub old_path: String,
43 pub path: String,
44 pub added: usize,
45 pub removed: usize,
46 pub binary: bool,
47 }
48
49 pub struct Reference {
50 pub name: String,
51 pub tag: bool,
52 pub oid: String,
53 pub timestamp: i64,
54 pub author: String,
55 pub is_commit: bool,
56 }
57
58 pub struct Repository {
59 pub path: PathBuf,
60 pub name: String,
61 pub description: String,
62 pub clone_url: String,
63 pub owner: String,
64 pub commits: Vec<Commit>,
65 pub entries: Vec<Entry>,
66 readme_index: Option<usize>,
67 license_index: Option<usize>,
68 pub has_submodules: bool,
69 }
70
71 fn text(bytes: &[u8]) -> String {
72 String::from_utf8_lossy(bytes).into_owned()
73 }
74
75 pub fn command(path: &Path) -> Command {
76 let mut command = Command::new("git");
77 command.args([
78 "--no-pager",
79 "-c",
80 "color.ui=false",
81 "-c",
82 "core.quotePath=false",
83 "-C",
84 ]);
85 command.arg(path);
86 command.env("GIT_TERMINAL_PROMPT", "0");
87 command.env("TZ", "UTC");
88 command
89 }
90
91 impl Repository {
92 pub fn open(path: &Path) -> Result<Self> {
93 Self::load(path, false)
94 }
95
96 // The catalog only needs metadata and the tip commit, not the entire tree
97 // and history of every repository.
98 pub fn open_index(path: &Path) -> Result<Self> {
99 Self::load(path, true)
100 }
101
102 fn load(path: &Path, index_only: bool) -> Result<Self> {
103 let path = fs::canonicalize(path)?;
104 let directory = path
105 .file_name()
106 .ok_or("repository has no directory name")?
107 .to_string_lossy();
108 let name = directory
109 .strip_suffix(".git")
110 .unwrap_or(&directory)
111 .to_owned();
112 let git_dir = command(&path)
113 .args(["rev-parse", "--absolute-git-dir"])
114 .output()?;
115 if !git_dir.status.success() {
116 return Err("input is not a Git repository".into());
117 }
118 let git_dir = PathBuf::from(String::from_utf8(git_dir.stdout)?.trim_end());
119 let metadata = |name: &str| -> String {
120 let file = git_dir.join(name);
121 // Never read a tracked worktree file or follow a metadata symlink.
122 match fs::symlink_metadata(&file) {
123 Ok(info) if info.is_file() && info.len() <= 16 * 1024 => fs::read_to_string(file)
124 .unwrap_or_default()
125 .trim()
126 .to_owned(),
127 _ => String::new(),
128 }
129 };
130 let mut repo = Self {
131 name,
132 description: metadata("description"),
133 clone_url: metadata("url"),
134 owner: metadata("owner"),
135 path,
136 commits: Vec::new(),
137 entries: Vec::new(),
138 readme_index: None,
139 license_index: None,
140 has_submodules: false,
141 };
142 // Distinguish an empty repository from a missing or invalid input path.
143 if command(&repo.path)
144 .args(["rev-parse", "--verify", "HEAD"])
145 .stdout(Stdio::null())
146 .stderr(Stdio::null())
147 .status()?
148 .success()
149 {
150 if index_only {
151 repo.commits = repo.read_history(&["--max-count=1", "HEAD"], false)?;
152 } else {
153 repo.commits = repo.read_history(&["HEAD"], false)?;
154 repo.entries = repo.read_entries()?;
155 repo.index_navigation();
156 }
157 }
158 Ok(repo)
159 }
160
161 pub fn git(&self, args: &[&str]) -> Result<Vec<u8>> {
162 let mut child = command(&self.path).args(args).stdout(Stdio::piped()).spawn()?;
163 let mut output = Vec::new();
164 let result = child.stdout.take().ok_or("Git stdout is unavailable")?
165 .take(MAX_GIT_OUTPUT_BYTES + 1).read_to_end(&mut output);
166 if result.is_err() || output.len() as u64 > MAX_GIT_OUTPUT_BYTES {
167 let _ = child.kill();
168 let _ = child.wait();
169 result?;
170 return Err("Git metadata exceeds the 64 MiB limit".into());
171 }
172 if !child.wait()?.success() {
173 return Err(format!("git {} failed", args.join(" ")).into());
174 }
175 Ok(output)
176 }
177
178 fn read_history(&self, revisions: &[&str], exclude_head: bool) -> Result<Vec<Commit>> {
179 if revisions.is_empty() {
180 return Ok(Vec::new());
181 }
182 let mut args = vec![
183 "log",
184 "-z",
185 "--no-show-signature",
186 "--format=%H%x00%P%x00%an%x00%ae%x00%at%x00%aI%x00%aD%x00%s%x00%B",
187 ];
188 args.extend_from_slice(revisions);
189 if exclude_head && !self.commits.is_empty() {
190 args.extend(["--not", "HEAD"]);
191 }
192 args.push("--");
193 let data = self.git(&args)?;
194 let mut fields: Vec<&[u8]> = data.split(|byte| *byte == 0).collect();
195 if fields.last() == Some(&&b""[..]) {
196 fields.pop();
197 }
198 if fields.len() % 9 != 0 {
199 return Err("unexpected Git commit record".into());
200 }
201 fields
202 .chunks_exact(9)
203 .map(|f| {
204 Ok(Commit {
205 oid: object_id(f[0])?,
206 parents: f[1].split(|byte| *byte == b' ').filter(|value| !value.is_empty())
207 .map(object_id).collect::<Result<Vec<_>>>()?,
208 author: text(f[2]),
209 email: text(f[3]),
210 timestamp: text(f[4]).parse()?,
211 iso_date: text(f[5]),
212 display_date: text(f[6]),
213 subject: text(f[7]),
214 message: text(f[8]),
215 })
216 })
217 .collect()
218 }
219
220 pub fn tagged_history(&self, references: &[Reference]) -> Result<Vec<Commit>> {
221 let mut known: HashSet<_> = self
222 .commits
223 .iter()
224 .map(|commit| commit.oid.as_str())
225 .collect();
226 let revisions: Vec<_> = references
227 .iter()
228 .filter(|reference| reference.tag && reference.is_commit)
229 .map(|reference| reference.oid.as_str())
230 .filter(|oid| known.insert(*oid))
231 .collect();
232 self.read_history(&revisions, true)
233 }
234
235 fn read_entries(&self) -> Result<Vec<Entry>> {
236 self.git(&["ls-tree", "-r", "-l", "-z", "--full-tree", "HEAD"])?
237 .split(|byte| *byte == 0)
238 .filter(|record| !record.is_empty())
239 .map(|record| {
240 let tab = record
241 .iter()
242 .position(|byte| *byte == b'\t')
243 .ok_or("invalid Git tree entry")?;
244 let header = text(&record[..tab]);
245 let fields: Vec<_> = header.split_whitespace().collect();
246 if fields.len() != 4 {
247 return Err("invalid Git tree metadata".into());
248 }
249 let path = String::from_utf8(record[tab + 1..].to_vec())?;
250 if Path::new(&path).is_absolute()
251 || path.contains(['\\', ':'])
252 || path.split('/').any(|part| part == ".." || part == ".")
253 {
254 return Err("unsafe Git file path".into());
255 }
256 Ok(Entry {
257 mode: fields[0].into(),
258 kind: fields[1].into(),
259 oid: object_id(fields[2].as_bytes())?,
260 size: fields[3].parse().unwrap_or(0),
261 path,
262 })
263 })
264 .collect()
265 }
266
267 pub fn references(&self) -> Result<Vec<Reference>> {
268 let data = self.git(&["for-each-ref", "--sort=-creatordate",
269 "--format=%(refname)%00%(objectname)%00%(*objectname)%00%(authordate:unix)%00%(*authordate:unix)%00%(authorname)%00%(*authorname)%00%(objecttype)%00%(*objecttype)",
270 "refs/heads", "refs/tags"])?;
271 data.split(|byte| *byte == b'\n')
272 .filter(|record| !record.is_empty())
273 .map(|record| {
274 let fields: Vec<_> = record.split(|byte| *byte == 0).collect();
275 if fields.len() != 9 {
276 return Err("invalid Git reference".into());
277 }
278 let name = text(fields[0]);
279 let annotated = !fields[2].is_empty();
280 Ok(Reference {
281 tag: name.starts_with("refs/tags/"),
282 name: name.splitn(3, '/').nth(2).unwrap_or(&name).into(),
283 oid: object_id(fields[if annotated { 2 } else { 1 }])?,
284 timestamp: text(fields[if annotated { 4 } else { 3 }])
285 .parse()
286 .unwrap_or(0),
287 author: text(fields[if annotated { 6 } else { 5 }]),
288 is_commit: fields[if annotated { 8 } else { 7 }] == b"commit",
289 })
290 })
291 .collect()
292 }
293
294 pub fn readme(&self) -> Option<&Entry> {
295 self.readme_index.map(|index| &self.entries[index])
296 }
297
298 pub fn license(&self) -> Option<&Entry> {
299 self.license_index.map(|index| &self.entries[index])
300 }
301
302 fn index_navigation(&mut self) {
303 let find = |names: &[&str]| {
304 names.iter().find_map(|name| {
305 self.entries
306 .iter()
307 .position(|entry| entry.kind == "blob" && entry.path == *name)
308 })
309 };
310 self.readme_index = find(&[
311 "README",
312 "README.md",
313 "README.txt",
314 "readme",
315 "readme.txt",
316 "README.markdown",
317 ]);
318 self.license_index = find(&["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING"]);
319 self.has_submodules = self.entries.iter().any(|entry| entry.path == ".gitmodules");
320 }
321 }
322
323 const DIFF_END: &[u8] = b"oni-end-of-diff\n";
324
325 pub struct Diff {
326 pub changes: Vec<Change>,
327 data: Vec<u8>,
328 patch_start: usize,
329 }
330
331 impl Diff {
332 pub fn patch(&self) -> Cow<'_, str> {
333 String::from_utf8_lossy(&self.data[self.patch_start..])
334 }
335 }
336
337 fn read_diff(output: &mut impl BufRead) -> Result<Diff> {
338 let mut data = Vec::new();
339 let mut patch_start = None;
340 loop {
341 let start = data.len();
342 let remaining = MAX_DIFF_BYTES.saturating_sub(data.len()) + 1;
343 if (&mut *output)
344 .take(remaining as u64)
345 .read_until(b'\n', &mut data)?
346 == 0
347 {
348 return Err("Git diff stream ended before its response marker".into());
349 }
350 if data.len() > MAX_DIFF_BYTES {
351 return Err("Git diff exceeds the 64 MiB limit".into());
352 }
353 // Git echoes non-object stdin lines. In a patch every content line has
354 // a diff prefix, so this marker cannot be a source line. Raw numstat
355 // filenames may contain newlines and even the marker: ignore it there.
356 if &data[start..] == DIFF_END && (start == 0 || patch_start.is_some()) {
357 data.truncate(start);
358 break;
359 }
360 if patch_start.is_none() {
361 // The combined numstat/patch boundary is a double NUL. Rename
362 // records cannot contain it because both paths are nonempty.
363 if let Some(offset) = data[start..].windows(2).position(|b| b == b"\0\0") {
364 patch_start = Some(start + offset + 2);
365 }
366 }
367 }
368 let patch_start = patch_start.unwrap_or(0);
369 if !data.is_empty() && patch_start == 0 {
370 return Err("missing Git diff separator".into());
371 }
372 Ok(Diff {
373 changes: parse_changes(&data[..patch_start])?,
374 data,
375 patch_start,
376 })
377 }
378
379 pub struct Diffs {
380 child: Child,
381 input: Option<ChildStdin>,
382 output: BufReader<ChildStdout>,
383 }
384
385 impl Diffs {
386 pub fn new(repo: &Repository) -> Result<Self> {
387 let mut child = command(&repo.path)
388 .args([
389 "diff-tree",
390 "--stdin",
391 "--root",
392 "--no-commit-id",
393 "-r",
394 "--find-renames=100%",
395 "--find-copies=100%",
396 "--no-ext-diff",
397 "--no-textconv",
398 "--numstat",
399 "-z",
400 "-p",
401 ])
402 .stdin(Stdio::piped())
403 .stdout(Stdio::piped())
404 .spawn()?;
405 let input = child.stdin.take().ok_or("Git stdin is unavailable")?;
406 let output = BufReader::new(child.stdout.take().ok_or("Git stdout is unavailable")?);
407 Ok(Self {
408 child,
409 input: Some(input),
410 output,
411 })
412 }
413
414 pub fn read(&mut self, commit: &Commit) -> Result<Diff> {
415 let input = self.input.as_mut().ok_or("Git diff stream is closed")?;
416 write!(input, "{}", commit.oid)?;
417 // Explicitly select the first parent, including for merge commits.
418 // A root commit has no parents and is handled by --root.
419 if let Some(parent) = commit.parents.first() {
420 write!(input, " {parent}")?;
421 }
422 writeln!(input)?;
423 input.write_all(DIFF_END)?;
424 input.flush()?;
425 read_diff(&mut self.output)
426 }
427
428 pub fn finish(&mut self) -> Result<()> {
429 self.input.take();
430 if !self.child.wait()?.success() {
431 return Err("Git diff process failed".into());
432 }
433 Ok(())
434 }
435 }
436
437 impl Drop for Diffs {
438 fn drop(&mut self) {
439 let _ = self.child.kill();
440 let _ = self.child.wait();
441 }
442 }
443
444 fn parse_changes(data: &[u8]) -> Result<Vec<Change>> {
445 let mut records = data.split(|byte| *byte == 0);
446 let mut changes = Vec::new();
447 while let Some(record) = records.next() {
448 if record.is_empty() {
449 continue;
450 }
451 let fields: Vec<_> = record.splitn(3, |byte| *byte == b'\t').collect();
452 if fields.len() != 3 {
453 return Err("invalid Git diff statistics".into());
454 }
455 // With -z, renamed/copied files use an empty path followed by two
456 // NUL-terminated names. Filenames themselves may contain tabs/newlines.
457 let (old_path, path) = if fields[2].is_empty() {
458 let old = records
459 .next()
460 .filter(|name| !name.is_empty())
461 .ok_or("missing original diff path")?;
462 let new = records
463 .next()
464 .filter(|name| !name.is_empty())
465 .ok_or("missing destination diff path")?;
466 (text(old), text(new))
467 } else {
468 (text(fields[2]), text(fields[2]))
469 };
470 let binary = fields[0] == b"-" && fields[1] == b"-";
471 changes.push(Change {
472 old_path,
473 path,
474 binary,
475 added: if binary { 0 } else { text(fields[0]).parse()? },
476 removed: if binary { 0 } else { text(fields[1]).parse()? },
477 });
478 }
479 Ok(changes)
480 }
481
482 pub struct Objects {
483 child: Child,
484 input: ChildStdin,
485 output: BufReader<ChildStdout>,
486 }
487
488 impl Objects {
489 pub fn new(repo: &Repository) -> Result<Self> {
490 let mut child = command(&repo.path)
491 .args(["cat-file", "--batch"])
492 .stdin(Stdio::piped())
493 .stdout(Stdio::piped())
494 .spawn()?;
495 let input = child.stdin.take().ok_or("Git stdin is unavailable")?;
496 let output = BufReader::new(child.stdout.take().ok_or("Git stdout is unavailable")?);
497 Ok(Self {
498 child,
499 input,
500 output,
501 })
502 }
503
504 pub fn blob(&mut self, oid: &str) -> Result<Vec<u8>> {
505 writeln!(self.input, "{oid}")?;
506 self.input.flush()?;
507 let mut line = String::new();
508 self.output.read_line(&mut line)?;
509 let fields: Vec<_> = line.split_whitespace().collect();
510 if fields.len() != 3 || fields[1] != "blob" {
511 return Err("Git object is not a blob".into());
512 }
513 let size: usize = fields[2].parse()?;
514 if size > MAX_BLOB_BYTES {
515 return Err("Git blob exceeds the 32 MiB limit".into());
516 }
517 let mut bytes = vec![0; size];
518 self.output.read_exact(&mut bytes)?;
519 let mut separator = [0];
520 self.output.read_exact(&mut separator)?;
521 if separator != [b'\n'] {
522 return Err("invalid Git object separator".into());
523 }
524 Ok(bytes)
525 }
526 }
527
528 impl Drop for Objects {
529 fn drop(&mut self) {
530 let _ = self.child.kill();
531 let _ = self.child.wait();
532 }
533 }
534
535 #[cfg(test)]
536 mod tests {
537 use super::*;
538
539 #[test]
540 fn rejects_object_id_record_injection() {
541 assert!(object_id(b"../../outside").is_err());
542 assert!(object_id(b"--output=/tmp/out").is_err());
543 assert!(object_id(&[b'a'; 40]).is_ok());
544 assert!(object_id(&[b'0'; 64]).is_ok());
545 assert!(object_id(&[b'g'; 40]).is_err());
546 }
547
548 #[test]
549 fn separates_combined_statistics_from_patch() {
550 let data = b"0\t0\t\0old\tname.txt\0new\noni-end-of-diff\nname.txt\0-\t-\timage.png\0\0diff --git a/x b/x\n+oni-end-of-diff\noni-end-of-diff\noni-end-of-diff\n";
551 let mut input = &data[..];
552 let diff = read_diff(&mut input).unwrap();
553 assert_eq!(diff.changes.len(), 2);
554 assert_eq!(diff.changes[0].path, "new\noni-end-of-diff\nname.txt");
555 assert!(diff.changes[1].binary);
556 assert_eq!(diff.patch(), "diff --git a/x b/x\n+oni-end-of-diff\n");
557 let empty = read_diff(&mut input).unwrap();
558 assert!(empty.changes.is_empty());
559 assert!(empty.patch().is_empty());
560 assert!(read_diff(&mut &b"1\t0\tx\0"[..]).is_err());
561 assert!(read_diff(&mut &b""[..]).is_err());
562 }
563
564 #[test]
565 fn parses_rename_records_without_splitting_filenames() {
566 let changes =
567 parse_changes(b"0\t0\t\0old\tname.txt\0new\nname.txt\0-\t-\timage.png\0").unwrap();
568 assert_eq!(changes.len(), 2);
569 assert_eq!(changes[0].old_path, "old\tname.txt");
570 assert_eq!(changes[0].path, "new\nname.txt");
571 assert_eq!(changes[0].added + changes[0].removed, 0);
572 assert!(changes[1].binary);
573 assert!(parse_changes(b"0\t0\t\0old.txt\0").is_err());
574 }
575 }