scripts/audit_site.py (2833B)
1 #!/usr/bin/env python3
2 """Check generated pages, local links, feeds, archives, and active-content boundaries."""
3
4 from html.parser import HTMLParser
5 import json
6 from pathlib import Path
7 import tarfile
8 from urllib.parse import unquote, urlsplit
9 import xml.etree.ElementTree as ET
10
11 from readme_assets import image_type
12
13 ROOT = Path(__file__).resolve().parents[1]
14 PUBLIC = ROOT / "public"
15
16
17 class Audit(HTMLParser):
18 def __init__(self, page):
19 super().__init__()
20 self.page = page
21 self.links = 0
22 self.csp = False
23
24 def handle_starttag(self, tag, attributes):
25 attrs = dict(attributes)
26 assert tag not in {"script", "iframe", "object", "embed", "base", "form", "svg", "math"}, (self.page, tag)
27 assert not any(name.startswith("on") or name in {"srcdoc", "style"} for name in attrs), self.page
28 if tag == "meta" and attrs.get("http-equiv", "").lower() == "content-security-policy":
29 assert "script-src 'none'" in attrs.get("content", ""), self.page
30 self.csp = True
31 for key, value in attributes:
32 if key not in ("href", "src") or not value:
33 continue
34 url = urlsplit(value)
35 assert url.scheme.lower() in ("", "http", "https", "mailto"), (self.page, value)
36 if url.scheme or url.netloc or not url.path:
37 continue
38 self.links += 1
39 target = PUBLIC / unquote(url.path).lstrip("/") if url.path.startswith("/") else self.page.parent / unquote(url.path)
40 target.resolve().relative_to(PUBLIC.resolve())
41 assert target.exists(), (self.page, value)
42
43
44 def main():
45 report = json.loads((ROOT / ".cache/last-build.json").read_text())
46 pages = list(PUBLIC.rglob("*.html"))
47 links = 0
48 for page in pages:
49 parser = Audit(page)
50 parser.feed(page.read_text(encoding="utf-8"))
51 assert parser.csp, page
52 links += parser.links
53 for name in report["repositories"]:
54 for feed in ("atom.xml", "tags.xml"):
55 ET.parse(PUBLIC / name / feed)
56 with tarfile.open(PUBLIC / name / "archive.tar.gz") as archive:
57 assert archive.getmembers(), name
58 for asset in (PUBLIC / name / "raw").rglob("*"):
59 if asset.is_file():
60 assert image_type(asset.name, asset.read_bytes()), asset
61 result = {"repositories": len(report["repositories"]), "html_pages": len(pages),
62 "local_links": links, "missing_links": 0, "unsafe_elements": 0,
63 "archives": len(report["repositories"])}
64 destination = ROOT / ".cache/security-audit/site.json"
65 destination.parent.mkdir(parents=True, exist_ok=True)
66 destination.write_text(json.dumps(result, indent=2) + "\n")
67 print(json.dumps(result))
68
69
70 if __name__ == "__main__":
71 main()