src/pages/robots.txt.ts (785B)
1 import type { APIContext } from 'astro';
2
3 /**
4 * Written as an endpoint rather than dropped in public/ so the sitemap line is built from
5 * `site` in astro.config.mjs. A robots.txt that names a hard-coded host is one rename away
6 * from pointing at someone else's sitemap, and this domain has already moved once.
7 *
8 * Everything is allowed: the whole site is public and there is nothing here a crawler
9 * should be kept out of. The file exists for the sitemap line, not to forbid anything.
10 */
11 export function GET(context: APIContext) {
12 const body = [
13 'User-agent: *',
14 'Allow: /',
15 '',
16 `Sitemap: ${new URL('sitemap-index.xml', context.site).href}`,
17 '',
18 ].join('\n');
19
20 return new Response(body, { headers: { 'Content-Type': 'text/plain; charset=utf-8' } });
21 }