From 7eea14920b8b7cb7018adf037ec37757b80a8425 Mon Sep 17 00:00:00 2001 From: e-moran Date: Tue, 29 Oct 2024 17:16:03 +0000 Subject: [PATCH] update validate.js --- validate.js | 4 ++ well-known-bots.json | 112 +++++++++++++++++++++---------------------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/validate.js b/validate.js index a5b8b0b..e13ad40 100644 --- a/validate.js +++ b/validate.js @@ -53,6 +53,10 @@ if (process.argv[2] === "--check") { console.error("Item has wrong type specified for `url` string field:", item); process.exit(1); } + if (!Array.isArray(item.verification)) { + console.error("Item is missing required `validation` array field:", item); + process.exit(1); + } // TODO: Check `addition_date` is defined properly // TODO: Check or remove `depends_on` field if (typeof item.instances !== "undefined") { diff --git a/well-known-bots.json b/well-known-bots.json index b0731a9..79b9fa1 100644 --- a/well-known-bots.json +++ b/well-known-bots.json @@ -9,10 +9,10 @@ "url": "http://www.google.com/bot.html", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -52,10 +52,10 @@ "pattern": "Googlebot-Image", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -72,10 +72,10 @@ "pattern": "Googlebot-News", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -92,10 +92,10 @@ "pattern": "Googlebot-Video", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -113,9 +113,9 @@ "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { - "type": "dns", - "masks": [ - "rate-limited-proxy-***-***-***-***.google.com" + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" ] } ], @@ -134,9 +134,9 @@ "url": "https://support.google.com/adwords/answer/2404197", "verification": [ { - "type": "dns", - "masks": [ - "rate-limited-proxy-***-***-***-***.google.com" + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" ] } ], @@ -157,10 +157,10 @@ "url": "https://support.google.com/webmasters/answer/178852", "verification": [ { - "type": "dns", - "masks": [ - "***-***-***-***.gae.googleusercontent.com", - "google-proxy-***-***-***-***.google.com" + "type": "dns", + "masks": [ + "***-***-***-***.gae.googleusercontent.com", + "google-proxy-***-***-***-***.google.com" ] } ], @@ -178,9 +178,9 @@ "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { - "type": "dns", - "masks": [ - "rate-limited-proxy-***-***-***-***.google.com" + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" ] } ], @@ -213,9 +213,9 @@ "url": "https://support.google.com/webmasters/answer/1061943?hl=en", "verification": [ { - "type": "dns", - "masks": [ - "rate-limited-proxy-***-***-***-***.google.com" + "type": "dns", + "masks": [ + "rate-limited-proxy-***-***-***-***.google.com" ] } ], @@ -233,10 +233,10 @@ "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -255,10 +255,10 @@ "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -277,10 +277,10 @@ "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", "verification": [ { - "type": "dns", - "masks": [ - "crawl-***-***-***-***.googlebot.com", - "geo-crawl-***-***-***-***.geo.googlebot.com" + "type": "dns", + "masks": [ + "crawl-***-***-***-***.googlebot.com", + "geo-crawl-***-***-***-***.geo.googlebot.com" ] } ], @@ -298,9 +298,9 @@ "url": "http://www.bing.com/bingbot.htm", "verification": [ { - "type": "dns", - "masks": [ - "@.search.msn.com" + "type": "dns", + "masks": [ + "@.search.msn.com" ] } ], @@ -987,11 +987,11 @@ "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", "verification": [ { - "type": "dns", - "masks": [ - "yandex.ru", - "yandex.com", - "yandex.net" + "type": "dns", + "masks": [ + "yandex.ru", + "yandex.com", + "yandex.net" ] } ], @@ -1100,10 +1100,10 @@ "url": "http://www.baidu.jp/spider/", "verification": [ { - "type": "dns", - "masks": [ - "@.baidu.jp", - "@.baidu.com" + "type": "dns", + "masks": [ + "@.baidu.jp", + "@.baidu.com" ] } ],