chore(workflows/build): use static robots.txt

.github/workflows/prod-build.yml

@@ -301,7 +301,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/prod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

.github/workflows/stage-build.yml

@@ -317,7 +317,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/nonprod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

.github/workflows/test-build.yml

@@ -249,7 +249,8 @@ jobs:
           yarn build:sw
           yarn build:client
           yarn build:ssr
-          yarn tool:legacy build-robots-txt
+
+          cp assets/nonprod/robots.txt client/build/robots.txt
 
           yarn rari content sync-translated-content
           yarn rari git-history

assets/nonprod/robots.txt

@@ -0,0 +1,3 @@
+User-Agent: *
+
+Disallow: /

assets/prod/robots.txt

@@ -0,0 +1,6 @@
+User-agent: *
+Sitemap: https://developer.mozilla.org/sitemap.xml
+
+Disallow: /api/
+Disallow: /*/files/
+Disallow: /media

package.json

@@ -27,8 +27,8 @@
     "build:legacy::curriculum": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/build-curriculum.ts",
     "build:legacy::docs": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/cli.ts -n",
     "build:legacy:blog": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node build/build-blog.ts",
-    "build:legacy:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy popularities && yarn tool:legacy spas && yarn tool:legacy gather-git-history && yarn tool:legacy build-robots-txt",
-    "build:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy build-robots-txt",
+    "build:legacy:prepare": "yarn build:client && yarn build:ssr && yarn tool:legacy popularities && yarn tool:legacy spas && yarn tool:legacy gather-git-history",
+    "build:prepare": "yarn build:client && yarn build:ssr",
     "build:ssr": "cross-env NODE_ENV=production NODE_OPTIONS=\"--no-warnings=ExperimentalWarning --loader ts-node/esm\" node ssr/prepare.ts && webpack --mode=production --config=ssr/webpack.config.js",
     "build:sw": "cd client/pwa && yarn && yarn build:prod",
     "build:sw-dev": "cd client/pwa && yarn && yarn build",

testing/tests/index.test.ts

@@ -1177,17 +1177,6 @@ test("built search-index.json (en-US)", () => {
   expect(urlToTitle.get("/en-US/docs/Web/Foo")).toBe("<foo>: A test tag");
 });
 
-test("the robots.txt file was created", () => {
-  const filePath = path.join(buildRoot, "robots.txt");
-  const text = fs.readFileSync(filePath, "utf-8");
-  // The content of robots file when it's in production mode is
-  // to ONLY say `Disallow: /api/`.
-  // When the robots file is for disallowing everything it
-  // will ONLY say `Disallow: /`.
-  expect(text).toContain("Disallow: /api/");
-  expect(text).not.toContain("Disallow: /\n");
-});
-
 test("duplicate IDs are de-duplicated", () => {
   const builtFolder = path.join(
     buildRoot,

tool/cli.ts

@@ -25,13 +25,11 @@
 import { isValidLocale } from "../libs/locale-utils/index.js";
 import type { Doc } from "../libs/types/document.js";
 import {
-  ALWAYS_ALLOW_ROBOTS,
   BUILD_OUT_ROOT,
   CONTENT_ROOT,
   CONTENT_TRANSLATED_ROOT,
 } from "../libs/env/index.js";
 import { runMakePopularitiesFile } from "./popularities.js";
-import { runBuildRobotsTxt } from "./build-robots-txt.js";
 import { syncAllTranslatedContent } from "./sync-translated-content.js";
 import { macroUsageReport } from "./macro-usage-report.js";
 import * as kumascript from "../kumascript/index.js";
@@ -176,7 +174,7 @@
   logger: Logger;
 }
 
 interface BuildRobotsTxtActionParameters extends ActionParameters {
   options: {
     outfile: string;
     maxUris: number;
@@ -869,28 +867,6 @@
     })
   )
 
-  .command(
-    "build-robots-txt",
-    "Generate a robots.txt in the build root depending ALWAYS_ALLOW_ROBOTS"
-  )
-  .option("--outfile <path>", "name of the generated file", {
-    default: path.join(BUILD_OUT_ROOT, "robots.txt"),
-  })
-  .action(
-    tryOrExit(async ({ options, logger }: BuildRobotsTxtActionParameters) => {
-      const { outfile } = options;
-      await runBuildRobotsTxt(outfile);
-      logger.info(
-        chalk.yellow(
-          `Generated ${path.relative(
-            ".",
-            outfile
-          )} based on ALWAYS_ALLOW_ROBOTS=${ALWAYS_ALLOW_ROBOTS}`
-        )
-      );
-    })
-  )
-
   .command("spas", "Build (SSR) all the skeleton apps for single page apps")
   .action(
     tryOrExit(async ({ options }) => {