diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index a6b70565fd7..0ba58388a34 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -82,7 +82,7 @@ func (s mappedConfig) postProcess(ctx context.Context, q mappedQuery, attrConfig if attrConfig.hasSplit() { results := attrConfig.splitString(result) // skip cleaning when the query is used for searching - if q.getType() == SearchQuery { + if q.getType() == SearchQuery || attrConfig.hasDuplicate() { return results } results = attrConfig.cleanResults(results) @@ -100,7 +100,7 @@ func (s mappedConfig) postProcess(ctx context.Context, q mappedQuery, attrConfig ret = append(ret, text) } // skip cleaning when the query is used for searching - if q.getType() == SearchQuery { + if q.getType() == SearchQuery || attrConfig.hasDuplicate() { return ret } ret = attrConfig.cleanResults(ret) @@ -660,6 +660,7 @@ type mappedScraperAttrConfig struct { PostProcess []mappedPostProcessAction `yaml:"postProcess"` Concat string `yaml:"concat"` Split string `yaml:"split"` + Duplicate bool `yaml:"duplicate"` postProcessActions []postProcessAction @@ -743,6 +744,10 @@ func (c mappedScraperAttrConfig) hasSplit() bool { return c.Split != "" } +func (c mappedScraperAttrConfig) hasDuplicate() bool { + return c.Duplicate +} + func (c mappedScraperAttrConfig) concatenateResults(nodes []string) string { separator := c.Concat return strings.Join(nodes, separator) diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 06b6ad5b686..0aa3e96f61b 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -464,13 +464,13 @@ const sceneHTML = `