diff --git a/internal/config/config.go b/internal/config/config.go index 26dde7d..f8c7869 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,7 +9,7 @@ import ( type Config struct { Site string `env:"SITE"` - AllowedDomains string `env:"ALLOWED_DOMAINS"` + AllowedDomains []string `env:"ALLOWED_DOMAINS" envSeparator:","` UserAgent string `env:"USER_AGENT" envDefault:"govukbot"` Headers map[string]string `env:"HEADERS"` Concurrency int `env:"CONCURRENCY" envDefault:"10"` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3b93191..9cb7e74 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,7 +25,7 @@ func TestNewConfig(t *testing.T) { name: "env vars", envVars: map[string]string{ "SITE": "example.com", - "ALLOWED_DOMAINS": "example.com", + "ALLOWED_DOMAINS": "example.com,foo.bar", "USER_AGENT": "custom-agent", "HEADERS": "Test-Header:Test-Value", "CONCURRENCY": "20", @@ -33,7 +33,7 @@ func TestNewConfig(t *testing.T) { }, expected: &Config{ Site: "example.com", - AllowedDomains: "example.com", + AllowedDomains: []string{"example.com", "foo.bar"}, UserAgent: "custom-agent", Headers: map[string]string{ "Test-Header": "Test-Value", diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index a4f82fa..445c3c1 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -29,7 +29,7 @@ func NewCrawler(cfg *config.Config) (*Crawler, error) { func newCollector(cfg *config.Config) (*colly.Collector, error) { c := colly.NewCollector( colly.UserAgent(cfg.UserAgent), - colly.AllowedDomains(cfg.AllowedDomains), + colly.AllowedDomains(cfg.AllowedDomains...), colly.DisallowedURLFilters(cfg.DisallowedURLFilters...), colly.Async(true), )