fix: remove some invalid agents

medama-io · Jan 19, 2025 · e906dd7 · e906dd7
1 parent 1e9e67b
commit e906dd7
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 6 deletions.
diff --git a/agents/final.txt b/agents/final.txt
@@ -66,6 +66,7 @@ MozillaXULinuxxkoKoreUSAppleWebKitKHTMLlikeGeckoChromeSafari
 MozillaWindowsNTAppleWebKitKHTMLlikeGeckoChromeSafariOPR
 MozillaiPhoneCPUiPhoneOSlikeMacOSXAppleWebKitKHTMLlikeGeckoVersionMobileSafaricompatibleGooglebothttpwwwgooglecombot
 MozillaMacintoshIntelMacOSXAppleWebKit
+Pinterest
 Googlebothttpwwwgooglebotcombot
 MozillacompatibleDuckDuckGoFaviconsBot
 MozillaWindowsPhoneAndroidAppleWebKitKHTMLlikeGeckoChromeMobileSafariEdge
@@ -1655,7 +1656,6 @@ UniversalFeedParserhttpfeedparserorgAppEngineGoogle
 MozillaWindowsNTWinxAppleWebKitKHTMLlikeGeckoChromeSafariOPR
 JavaUnfurlBot
 MozillacompatibleSquidbot
-securityJavaScriptcomBot
 yacybotglobalamdWindowsjavaAsiaruhttpyacynetbot
 yacybotglobalamdWindowsjavaAsiaenhttpyacynetbot
 MozillacompatibleMailRUBot
@@ -1870,7 +1870,6 @@ CustomPhoneAndroid
 APPRADIOPR
 Pingdomcombot
 pythonrequestsDalvikLinuxUAndroidAndroid
-JavasCriptwasxsscontextjtvressendWASidsleepMozillaXLinuxxAppleWebKitKHTMLlikeGeckoChromeSafariltMozillaXLinuxxAppleWebKitKHTMLlikeGeckoChromeSafari
 MozillacompatibleYandexAdNethttpyandexcombot
 compatibleYandexBot
 SnapchatLGAndroid
@@ -1884,8 +1883,6 @@ DuckduckgoAndroid
 DuckduckgocomAndroid
 SEODuckduckgocomAndroid
 HostDuckduckgocomAndroid
-HostDuckduckgocomParentAndroid
-PostDuckduckgocomParentAndroid
 BotoPythonLinuxamznxBot
 BotMozillacompatibleYandexBothttpyandexcombotsAppleWebKitKHTMLlikeGeckoChrome
 FeedfetcherGoogle

diff --git a/internal/match.go b/internal/match.go
@@ -83,7 +83,7 @@ var matchMap = map[string][]string{
 	MobileDevice: {"ONEPLUS", "Huawei", "HTC", "Galaxy", "iPhone", "iPod", "Windows Phone", "WindowsPhone", "LG"},
 	Tablet:       {Tablet, "Touch", "iPad", "Nintendo Switch", "NintendoSwitch", "Kindle"},
 	TV:           {TV, "Large Screen", "LargeScreen", "Smart Display", "SmartDisplay", "PLAYSTATION", "PlayStation", "ADT-2", "ADT-1", "CrKey", "Roku", "AFT", "Web0S", "Nexus Player", "Xbox", "XBOX", "Nintendo WiiU", "NintendoWiiU"},
-	Bot:          {Bot, "HeadlessChrome", "bot", "Slurp", "LinkCheck", "QuickLook", "Haosou", "Yahoo Ad", "YahooAd", "Google", "Mediapartners", "Headless", "facebookexternalhit", "facebookcatalog", "Baidu"},
+	Bot:          {Bot, "HeadlessChrome", "bot", "Slurp", "LinkCheck", "QuickLook", "Haosou", "Yahoo Ad", "YahooAd", "Google", "Mediapartners", "Headless", "facebookexternalhit", "facebookcatalog", "Baidu", "Instagram", "Pinterest", "PageSpeedInsights"},
 
 	// Version
 	Version: {Version},

diff --git a/scripts/main.go b/scripts/main.go
@@ -29,7 +29,8 @@ func CleanAgentsFile(filePath string) ([]string, error) {
 			continue
 		}
 
-		if strings.Contains(line, "javascript") || strings.Contains(line, "function") || strings.Contains(line, "quot") || strings.Contains(line, "parent") {
+		lineLower := strings.ToLower(line)
+		if strings.Contains(lineLower, "javascript") || strings.Contains(lineLower, "function") || strings.Contains(lineLower, "quot") || strings.Contains(lineLower, "parent") {
 			continue
 		}