diff --git a/example.env b/example.env index 0bbb1f7dd69fa28736c9ed795759c460f527c782..aca7de23251b29814861e946ff1fa244adeed468 100644 --- a/example.env +++ b/example.env @@ -3,6 +3,9 @@ # Are advanced & experimental features enabled? # ADVANCED=false +# Are search robots allowed to index this site? +# ALLOW_ROBOTS=false + # Public HTTP(S) URLs of LexImpact Socio-Fiscal API server API_BASE_URLS="https://simulateur-socio-fiscal.leximpact.dev/api/" # API_BASE_URLS="http://localhost:8000" diff --git a/package-lock.json b/package-lock.json index 177429df3019009c894d334a9a6e3c33b32bd750..f202405aa15c3c69184afaf486a8bbf14b1c72ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "autoprefixer": "^10.2.5", "cssnano": "^5.0.5", "d3-scale": "^4.0.0", + "dedent-js": "^1.0.1", "dotenv": "^10.0.0", "eslint": "^7.22.0", "eslint-config-prettier": "^8.1.0", @@ -3171,6 +3172,12 @@ } } }, + "node_modules/dedent-js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dedent-js/-/dedent-js-1.0.1.tgz", + "integrity": "sha1-vuX7fJ5yfYXf+iRZDRDsGrElUwU=", + "dev": true + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -9323,6 +9330,12 @@ "ms": "2.1.2" } }, + "dedent-js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dedent-js/-/dedent-js-1.0.1.tgz", + "integrity": "sha1-vuX7fJ5yfYXf+iRZDRDsGrElUwU=", + "dev": true + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", diff --git a/package.json b/package.json index 4944814fded30700ef0951484c725560a3754539..33c12f43058f735c3159e0f913bb49b96cd66169 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "autoprefixer": "^10.2.5", "cssnano": "^5.0.5", "d3-scale": "^4.0.0", + "dedent-js": "^1.0.1", "dotenv": "^10.0.0", "eslint": "^7.22.0", "eslint-config-prettier": "^8.1.0", diff --git a/src/lib/auditors/config.ts b/src/lib/auditors/config.ts index 0568bf67bc1e5806d307f6471c7eb5ab83adbf2c..b2cb2811d7a23754fdabe8dd9249fd3e7af9137f 100644 --- a/src/lib/auditors/config.ts +++ b/src/lib/auditors/config.ts @@ -31,7 +31,7 @@ export function auditConfig( const errors: { [key: string]: unknown } = {} const remainingKeys = new Set(Object.keys(data)) - for (const key of ["advanced", "proxy"]) { + for (const key of ["advanced", "allowRobots", "proxy"]) { audit.attribute( data, key, diff --git a/src/lib/server/config.ts b/src/lib/server/config.ts index 323a8ac1cb020aae19e6463c9aa480b39120865f..3a2b3d3dc2dfea9af6a6d83bcb86865a9880c8ad 100644 --- a/src/lib/server/config.ts +++ b/src/lib/server/config.ts @@ -4,6 +4,7 @@ import { validateConfig } from "$lib/auditors/config" export interface Config { advanced: boolean + allowRobots: boolean apiBaseUrls: string[] apiWebSocketBaseUrls: string[] baseUrl: string @@ -42,6 +43,7 @@ export interface Config { const [validConfig, error] = validateConfig({ advanced: process.env["ADVANCED"], + allowRobots: process.env["ALLOW_ROBOTS"], apiBaseUrls: process.env["API_BASE_URLS"], baseUrl: process.env["BASE_URL"], childrenKey: process.env["CHILDREN_KEY"], diff --git a/src/routes/robots.txt.ts b/src/routes/robots.txt.ts new file mode 100644 index 0000000000000000000000000000000000000000..b83b51bd209473a2216b2d4feaa11d7e4b565433 --- /dev/null +++ b/src/routes/robots.txt.ts @@ -0,0 +1,17 @@ +import type { RequestHandler } from "@sveltejs/kit" +import dedent from "dedent-js" + +import config from "$lib/server/config" + +const { allowRobots } = config + +export const get: RequestHandler = async ({ params }) => { + return { + body: dedent` + # https://www.robotstxt.org/robotstxt.html + User-agent: * + Disallow:${allowRobots ? "" : " /"} + `, + headers: { "Content-Type": "text/plain; charset=utf-8" }, + } +} diff --git a/static/robots.txt b/static/robots.txt deleted file mode 100644 index e9e57dc4d41b9b46e05112e9f45b7ea6ac0ba15e..0000000000000000000000000000000000000000 --- a/static/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -# https://www.robotstxt.org/robotstxt.html -User-agent: * -Disallow: