tools/update-distro-data.mjs (11194B)
1 #!/usr/bin/env node
2 // Fetches distribution security data and writes it to site/data/distro.json.
3 //
4 // Sources
5 // - OSV: fixed versions per package for Debian and Ubuntu (https://osv.dev)
6 // - Red Hat Security Data API: RHEL advisories (RHSA) and package states
7 // - php-src GitHub security advisories: checks that the fixed versions match upstream.json
8 //
9 // - distro.json is rewritten only when its content changes (commit history = data change history).
10 // - status.json is written on every run (last check time, success per source). It is not committed.
11 // - If one source fails, its previous data is kept as is and the failure is recorded.
12 // - A 404 means "no record yet", which is normal before a distribution publishes anything.
13 // But if the previous distro.json has entries for that CVE, a 404 is treated as a fault
14 // on the source side: the previous entries for that CVE are kept, and the source is
15 // recorded as failed in status.json with the CVEs in kept_previous. To accept a record
16 // that really was withdrawn, delete its entries from distro.json by hand.
17 // - If the fixed versions in the official advisories differ from upstream.json, exit with code 1 (a person must check).
18
19 import { readFile, writeFile } from 'node:fs/promises';
20
21 const ROOT = new URL('..', import.meta.url);
22 const UPSTREAM_FILE = new URL('site/data/upstream.json', ROOT);
23 const DISTRO_FILE = new URL('site/data/distro.json', ROOT);
24 const STATUS_FILE = new URL('site/data/status.json', ROOT);
25
26 const GITHUB_TOKEN = process.env.GITHUB_TOKEN || '';
27
28 // Print a warning, and in GitHub Actions also an annotation on the run.
29 function warn(message) {
30 console.error(message);
31 if (process.env.GITHUB_ACTIONS === 'true') console.log(`::warning::${message.replace(/\s+/g, ' ')}`);
32 }
33
34 // Returns null on 404 (no record).
35 async function getJson(url, headers = {}) {
36 const res = await fetch(url, {
37 headers: { accept: 'application/json', 'user-agent': 'php2shell-scanner-data-updater', ...headers },
38 signal: AbortSignal.timeout(30_000),
39 });
40 if (res.status === 404) return null;
41 if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
42 return res.json();
43 }
44
45 async function readJson(file, fallback) {
46 try {
47 return JSON.parse(await readFile(file, 'utf8'));
48 } catch {
49 return fallback;
50 }
51 }
52
53 function purlCodename(purl) {
54 const query = String(purl || '').split('?')[1] || '';
55 const distro = new URLSearchParams(query).get('distro') || '';
56 return distro.split('/').pop() || null;
57 }
58
59 function addOsv(target, family, cve, record) {
60 const prefix = family === 'debian' ? 'Debian' : 'Ubuntu';
61 for (const a of record?.affected || []) {
62 const parts = String(a.package?.ecosystem || '').split(':');
63 if (parts[0] !== prefix) continue;
64 const pro = parts[1] === 'Pro';
65 const release = pro ? parts[2] : parts[1];
66 if (!release || !a.package?.name) continue;
67 const codename = purlCodename(a.package.purl);
68 const rel = (target[release] ||= { codename, packages: {} });
69 if (!rel.codename && codename) rel.codename = codename;
70 const pkg = (rel.packages[a.package.name] ||= { pro, fixed: {} });
71 pkg.pro = pkg.pro && pro;
72 const fixed = (a.ranges || []).flatMap((range) => range.events || []).find((e) => e.fixed);
73 pkg.fixed[cve] = fixed ? fixed.fixed : null;
74 }
75 }
76
77 // The record for cve returned 404. Copy that CVE's entries from the previous data
78 // into target. Returns true if there were any, that is, the record existed before.
79 function keepOsv(target, previous, cve) {
80 let found = false;
81 for (const [release, prevRel] of Object.entries(previous || {})) {
82 for (const [name, prevPkg] of Object.entries(prevRel?.packages || {})) {
83 if (!Object.hasOwn(prevPkg?.fixed || {}, cve)) continue;
84 found = true;
85 const rel = (target[release] ||= { codename: prevRel.codename ?? null, packages: {} });
86 if (!rel.codename && prevRel.codename) rel.codename = prevRel.codename;
87 const pkg = (rel.packages[name] ||= { pro: prevPkg.pro === true, fixed: {} });
88 pkg.pro = pkg.pro && prevPkg.pro === true;
89 pkg.fixed[cve] = prevPkg.fixed[cve];
90 }
91 }
92 return found;
93 }
94
95 async function fetchOsv(family, list, previous, kept) {
96 const out = {};
97 for (const c of list) {
98 const record = await getJson(`https://api.osv.dev/v1/vulns/${family.toUpperCase()}-${c.id}`);
99 if (record) addOsv(out, family, c.id, record);
100 else if (keepOsv(out, previous, c.id)) kept.push(c.id);
101 }
102 return out;
103 }
104
105 function addRedhat(target, notes, cve, record) {
106 if (!record) return;
107 notes[cve] = {
108 severity: record.threat_severity || null,
109 cvss3: record.cvss3?.cvss3_base_score || null,
110 statement: record.statement || null,
111 };
112 const relOf = (cpe) => /enterprise_linux:(\d+)/.exec(cpe || '')?.[1];
113 for (const a of record.affected_release || []) {
114 const rel = relOf(a.cpe);
115 if (!rel) continue;
116 (target[rel] ||= { advisories: [], states: [] }).advisories.push({ cve, advisory: a.advisory, package: a.package });
117 }
118 for (const s of record.package_state || []) {
119 const rel = relOf(s.cpe);
120 if (!rel) continue;
121 (target[rel] ||= { advisories: [], states: [] }).states.push({ cve, package: s.package_name, state: s.fix_state });
122 }
123 }
124
125 // Same as keepOsv, for the Red Hat releases and notes.
126 function keepRedhat(releases, notes, previous, previousNotes, cve) {
127 let found = false;
128 if (previousNotes && Object.hasOwn(previousNotes, cve)) {
129 notes[cve] = previousNotes[cve];
130 found = true;
131 }
132 for (const [rel, prev] of Object.entries(previous || {})) {
133 const advisories = (prev?.advisories || []).filter((a) => a.cve === cve);
134 const states = (prev?.states || []).filter((s) => s.cve === cve);
135 if (!advisories.length && !states.length) continue;
136 found = true;
137 const target = (releases[rel] ||= { advisories: [], states: [] });
138 target.advisories.push(...advisories);
139 target.states.push(...states);
140 }
141 return found;
142 }
143
144 // fn(kept) returns the source's data, and pushes onto kept every CVE whose record
145 // returned 404 although the previous data had entries for it. Such a source still
146 // counts as failed: part of its data is not current.
147 async function fetchSource(name, fn) {
148 const kept = [];
149 try {
150 const data = await fn(kept);
151 if (!kept.length) return { ok: true, data };
152 const error = `no record any more for ${kept.join(', ')} (HTTP 404); the previous entries were kept`;
153 warn(`[${name}] ${error}`);
154 return { ok: false, data, error, kept };
155 } catch (err) {
156 warn(`[${name}] failed: ${err.message}`);
157 return { ok: false, error: err.message };
158 }
159 }
160
161 function sortDeep(value) {
162 if (Array.isArray(value)) return value.map(sortDeep);
163 if (value && typeof value === 'object') {
164 return Object.fromEntries(Object.keys(value).sort().map((k) => [k, sortDeep(value[k])]));
165 }
166 return value;
167 }
168
169 const byKey = (...keys) => (a, b) => {
170 for (const k of keys) {
171 const c = String(a[k]).localeCompare(String(b[k]));
172 if (c) return c;
173 }
174 return 0;
175 };
176
177 async function main() {
178 const upstream = JSON.parse(await readFile(UPSTREAM_FILE, 'utf8'));
179 const previous = await readJson(DISTRO_FILE, {});
180 const cves = upstream.cves;
181
182 const sources = {
183 'osv-debian': await fetchSource('osv-debian', (kept) => fetchOsv('debian', cves, previous.debian, kept)),
184 'osv-ubuntu': await fetchSource('osv-ubuntu', (kept) => fetchOsv('ubuntu', cves, previous.ubuntu, kept)),
185 redhat: await fetchSource('redhat', async (kept) => {
186 const releases = {};
187 const notes = {};
188 for (const c of cves) {
189 const record = await getJson(`https://access.redhat.com/hydra/rest/securitydata/cve/${c.id}.json`);
190 if (record) addRedhat(releases, notes, c.id, record);
191 else if (keepRedhat(releases, notes, previous.redhat, previous.redhat_notes, c.id)) kept.push(c.id);
192 }
193 for (const rel of Object.values(releases)) {
194 rel.advisories.sort(byKey('cve', 'advisory', 'package'));
195 rel.states.sort(byKey('cve', 'package'));
196 }
197 return { releases, notes };
198 }),
199 ghsa: await fetchSource('ghsa', async () => {
200 const out = {};
201 const auth = GITHUB_TOKEN ? { authorization: `Bearer ${GITHUB_TOKEN}` } : {};
202 for (const c of cves) {
203 const g = await getJson(`https://api.github.com/repos/php/php-src/security-advisories/${c.ghsa}`, auth);
204 if (!g) throw new Error(`advisory ${c.ghsa} not found`);
205 out[c.id] = {};
206 for (const v of g.vulnerabilities || []) {
207 const branch = /^(\d+\.\d+)\./.exec(v.patched_versions || '')?.[1];
208 if (branch) out[c.id][branch] = v.patched_versions;
209 }
210 }
211 return out;
212 }),
213 };
214
215 // A source with data (fully fresh, or fresh with some CVEs kept from before) replaces the
216 // previous data. A source that failed outright leaves the previous data as it was.
217 const pick = (name, fresh, old) => (sources[name].data !== undefined ? fresh : old);
218 const content = sortDeep({
219 schema: 1,
220 debian: pick('osv-debian', sources['osv-debian'].data, previous.debian || {}),
221 ubuntu: pick('osv-ubuntu', sources['osv-ubuntu'].data, previous.ubuntu || {}),
222 redhat: pick('redhat', sources.redhat.data?.releases, previous.redhat || {}),
223 redhat_notes: pick('redhat', sources.redhat.data?.notes, previous.redhat_notes || {}),
224 });
225
226 const { updated_at: _ignored, ...previousContent } = previous;
227 const changed = JSON.stringify(sortDeep(previousContent)) !== JSON.stringify(content);
228 const now = new Date().toISOString();
229 if (changed) {
230 await writeFile(DISTRO_FILE, JSON.stringify({ updated_at: now, ...content }, null, 2) + '\n');
231 console.log('distro.json updated');
232 } else {
233 console.log('distro.json unchanged');
234 }
235
236 let exitCode = 0;
237 const mismatches = [];
238 if (sources.ghsa.ok) {
239 for (const [cve, fixed] of Object.entries(sources.ghsa.data)) {
240 const branches = new Set([...Object.keys(fixed), ...Object.keys(upstream.fixed)]);
241 for (const b of branches) {
242 if (fixed[b] !== upstream.fixed[b]) mismatches.push(`${cve} ${b}: official advisory ${fixed[b] ?? 'none'} / upstream.json ${upstream.fixed[b] ?? 'none'}`);
243 }
244 }
245 }
246 if (mismatches.length) {
247 console.error('The fixed versions in the official advisories differ from upstream.json. Check upstream.json:\n ' + mismatches.join('\n '));
248 exitCode = 1;
249 }
250
251 const status = {
252 checked_at: now,
253 data_updated_at: changed ? now : previous.updated_at || null,
254 upstream_matches_advisory: sources.ghsa.ok ? mismatches.length === 0 : null,
255 sources: Object.fromEntries(Object.entries(sources).map(([k, v]) => [
256 k,
257 v.ok ? { ok: true } : { ok: false, error: v.error, ...(v.kept ? { kept_previous: v.kept } : {}) },
258 ])),
259 };
260 await writeFile(STATUS_FILE, JSON.stringify(status, null, 2) + '\n');
261
262 const failed = Object.entries(sources).filter(([, v]) => v.data === undefined).map(([k]) => k);
263 if (failed.length === Object.keys(sources).length) {
264 console.error('No data could be fetched from any source.');
265 exitCode = 1;
266 }
267 process.exitCode = exitCode;
268 }
269
270 await main();