fix(robots): remove /search from robots.txt

mdn · May 17, 2024 · f844aab · f844aab
1 parent 4fcadd2
commit f844aab
Showing 1 changed file with 1 addition and 7 deletions.
diff --git a/tool/build-robots-txt.ts b/tool/build-robots-txt.ts
@@ -24,12 +24,6 @@ Disallow: /
 `;
 
 export async function runBuildRobotsTxt(outfile: string) {
-  let content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
-  if (ALWAYS_ALLOW_ROBOTS) {
-    // Append extra lines specifically when we do allow robots.
-    for (const locale of VALID_LOCALES.values()) {
-      content += `Disallow: /${locale}/search\n`;
-    }
-  }
+  const content = ALWAYS_ALLOW_ROBOTS ? ALLOW_TEXT : DISALLOW_TEXT;
   fs.writeFileSync(outfile, `${content.trim()}\n`, "utf-8");
 }