diff --git a/.chloggen/ottl-improve-parsing-errors.yaml b/.chloggen/ottl-improve-parsing-errors.yaml new file mode 100755 index 000000000000..231c9c1123a7 --- /dev/null +++ b/.chloggen/ottl-improve-parsing-errors.yaml @@ -0,0 +1,22 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Improve error reporting for errors during statement parsing + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [23840] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + - Failures are now printed for all statements within a context, and the statements are printed next to the errors. + - Erroneous values found during parsing are now quoted in error logs. diff --git a/.chloggen/support-client-info-metadata.yaml b/.chloggen/support-client-info-metadata.yaml new file mode 100644 index 000000000000..9fa2d42ceb5a --- /dev/null +++ b/.chloggen/support-client-info-metadata.yaml @@ -0,0 +1,17 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: routingprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Enables processor to extract metadata from client.Info + +# One or more tracking issues related to the change +issues: [20913] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + Enables processor to perform context based routing for payloads on the http server of otlp receiver \ No newline at end of file diff --git a/.github/workflows/build-and-test-windows.yml b/.github/workflows/build-and-test-windows.yml index 50e8ee8f40cd..16989ca54fa5 100644 --- a/.github/workflows/build-and-test-windows.yml +++ b/.github/workflows/build-and-test-windows.yml @@ -43,7 +43,7 @@ jobs: run: Install-WindowsFeature -name Web-Server -IncludeManagementTools - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-mod-cache diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2e6cdfd9e7c8..6bab8624a622 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -78,7 +78,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: "1.20" + go-version: "~1.20.6" cache: false - name: Cache Go id: go-cache @@ -138,7 +138,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -160,7 +160,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -308,7 +308,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -328,7 +328,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -353,7 +353,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -416,7 +416,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -547,7 +547,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Mkdir bin and dist run: | diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index e05e054901d9..47fec8c39cb1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 083052f76be3..4d0b3cd0a6b6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/create-dependabot-pr.yml b/.github/workflows/create-dependabot-pr.yml index 911f79dc0ed5..0c72851e90ec 100644 --- a/.github/workflows/create-dependabot-pr.yml +++ b/.github/workflows/create-dependabot-pr.yml @@ -12,7 +12,7 @@ jobs: run: sudo apt-get update; sudo apt-get install zsh - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Run dependabot-pr.sh run: ./.github/workflows/scripts/dependabot-pr.sh diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b9c7b98ab88a..28b8a64c939a 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache diff --git a/.github/workflows/load-tests.yml b/.github/workflows/load-tests.yml index 7e280416c54b..a854e7a14f0a 100644 --- a/.github/workflows/load-tests.yml +++ b/.github/workflows/load-tests.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache @@ -64,7 +64,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 63413bdbbed3..5a5934fba6e0 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -26,7 +26,7 @@ jobs: path: opentelemetry-collector-contrib - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Prepare release for contrib working-directory: opentelemetry-collector-contrib diff --git a/.github/workflows/prometheus-compliance-tests.yml b/.github/workflows/prometheus-compliance-tests.yml index a5d22160d3be..e2f2a3f918ff 100644 --- a/.github/workflows/prometheus-compliance-tests.yml +++ b/.github/workflows/prometheus-compliance-tests.yml @@ -26,7 +26,7 @@ jobs: path: opentelemetry-collector-contrib - uses: actions/setup-go@v4 with: - go-version: ~1.19.10 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache diff --git a/.github/workflows/tidy-dependencies.yml b/.github/workflows/tidy-dependencies.yml index 1699d3c5898b..dc0777251ebc 100644 --- a/.github/workflows/tidy-dependencies.yml +++ b/.github/workflows/tidy-dependencies.yml @@ -21,7 +21,7 @@ jobs: token: ${{ secrets.OPENTELEMETRYBOT_GITHUB_TOKEN }} - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: ~1.19.11 cache: false - name: Cache Go id: go-cache diff --git a/.golangci.yml b/.golangci.yml index f60f2ac7e254..f5fefb7e8a34 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -167,9 +167,6 @@ issues: - path: deltatorateprocessor linters: - exhaustive - - path: groupbyattrsprocessor - linters: - - exhaustive - path: filterprocessor linters: - exhaustive diff --git a/cmd/opampsupervisor/go.mod b/cmd/opampsupervisor/go.mod index ca2afd7e3d8a..75e6dc6d8f57 100644 --- a/cmd/opampsupervisor/go.mod +++ b/cmd/opampsupervisor/go.mod @@ -7,7 +7,7 @@ require ( github.com/knadh/koanf v1.5.0 github.com/oklog/ulid/v2 v2.1.0 github.com/open-telemetry/opamp-go v0.6.0 - go.opentelemetry.io/collector/config/configtls v0.80.1-0.20230629144634-c3f70bd1f8ea + go.opentelemetry.io/collector/config/configtls v0.81.0 go.uber.org/zap v1.24.0 ) @@ -20,7 +20,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.4 // indirect - go.opentelemetry.io/collector/config/configopaque v0.80.1-0.20230629144634-c3f70bd1f8ea // indirect + go.opentelemetry.io/collector/config/configopaque v0.81.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.10.0 // indirect diff --git a/cmd/opampsupervisor/go.sum b/cmd/opampsupervisor/go.sum index 0c2265a821a6..6141cc18c460 100644 --- a/cmd/opampsupervisor/go.sum +++ b/cmd/opampsupervisor/go.sum @@ -250,10 +250,10 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opentelemetry.io/collector/config/configopaque v0.80.1-0.20230629144634-c3f70bd1f8ea h1:14awmqtjlsjo5PuV0Zt/e31vTAsU3CgwYA2eVV2fSL8= -go.opentelemetry.io/collector/config/configopaque v0.80.1-0.20230629144634-c3f70bd1f8ea/go.mod h1:pM1oy6gasukw3H6jAvc9Q9OtFaaY2IbfeuwCPAjOgXc= -go.opentelemetry.io/collector/config/configtls v0.80.1-0.20230629144634-c3f70bd1f8ea h1:QS+urjZ0LFsC3GI9xk9J6V4s3GzkOQ7Ef7+1KFZjFhY= -go.opentelemetry.io/collector/config/configtls v0.80.1-0.20230629144634-c3f70bd1f8ea/go.mod h1:fO1VgdtrcgcVA3Y2vB/YQvTh2tNNFW0R0NjWrtvjTOQ= +go.opentelemetry.io/collector/config/configopaque v0.81.0 h1:MkCAGh0WydRWydETB9FLnuCj9hDPDiz2g4Wxnl53I0w= +go.opentelemetry.io/collector/config/configopaque v0.81.0/go.mod h1:pM1oy6gasukw3H6jAvc9Q9OtFaaY2IbfeuwCPAjOgXc= +go.opentelemetry.io/collector/config/configtls v0.81.0 h1:2vt+yOZUvGq5ADqFAxL5ONm1ACuGXDSs87AWT54Ez4M= +go.opentelemetry.io/collector/config/configtls v0.81.0/go.mod h1:HMHTYBMMgqBpTvnNAhQYmjO7XuoBMe2T4qRHcKluB4Q= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= diff --git a/extension/observer/dockerobserver/integration_test.go b/extension/observer/dockerobserver/integration_test.go index dd431c8e2d64..62729c5ebaf1 100644 --- a/extension/observer/dockerobserver/integration_test.go +++ b/extension/observer/dockerobserver/integration_test.go @@ -36,6 +36,7 @@ func (h *testHost) ReportFatalError(err error) { var _ component.Host = (*testHost)(nil) func TestObserverEmitsEndpointsIntegration(t *testing.T) { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") image := "docker.io/library/nginx" tag := "1.17" @@ -82,6 +83,7 @@ func TestObserverEmitsEndpointsIntegration(t *testing.T) { } func TestObserverUpdatesEndpointsIntegration(t *testing.T) { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") image := "docker.io/library/nginx" tag := "1.17" @@ -143,6 +145,7 @@ func TestObserverUpdatesEndpointsIntegration(t *testing.T) { } func TestObserverRemovesEndpointsIntegration(t *testing.T) { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") image := "docker.io/library/nginx" tag := "1.17" @@ -185,6 +188,7 @@ func TestObserverRemovesEndpointsIntegration(t *testing.T) { } func TestObserverExcludesImagesIntegration(t *testing.T) { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") ctx := context.Background() req := testcontainers.ContainerRequest{ Image: "docker.io/library/nginx:1.17", diff --git a/internal/coreinternal/scraperinttest/scraperint.go b/internal/coreinternal/scraperinttest/scraperint.go index de4def2c89b7..11e991d7aefd 100644 --- a/internal/coreinternal/scraperinttest/scraperint.go +++ b/internal/coreinternal/scraperinttest/scraperint.go @@ -143,6 +143,7 @@ func (it *IntegrationTest) Run(t *testing.T) { } func (it *IntegrationTest) createNetwork(t *testing.T) testcontainers.Network { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") var errs error var network testcontainers.Network @@ -163,6 +164,7 @@ func (it *IntegrationTest) createNetwork(t *testing.T) testcontainers.Network { } func (it *IntegrationTest) createContainers(t *testing.T) *ContainerInfo { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") var wg sync.WaitGroup ci := &ContainerInfo{ containers: make(map[string]testcontainers.Container, len(it.containerRequests)), diff --git a/pkg/ottl/functions.go b/pkg/ottl/functions.go index 58a3695ec79a..5cf65df79f39 100644 --- a/pkg/ottl/functions.go +++ b/pkg/ottl/functions.go @@ -20,7 +20,7 @@ type Enum int64 func (p *Parser[K]) newFunctionCall(ed editor) (Expr[K], error) { f, ok := p.functions[ed.Function] if !ok { - return Expr[K]{}, fmt.Errorf("undefined function %v", ed.Function) + return Expr[K]{}, fmt.Errorf("undefined function %q", ed.Function) } args := f.CreateDefaultArguments() @@ -30,12 +30,12 @@ func (p *Parser[K]) newFunctionCall(ed editor) (Expr[K], error) { // settability requirements. Non-pointer values are not // modifiable through reflection. if reflect.TypeOf(args).Kind() != reflect.Pointer { - return Expr[K]{}, fmt.Errorf("factory for %s must return a pointer to an Arguments value in its CreateDefaultArguments method", ed.Function) + return Expr[K]{}, fmt.Errorf("factory for %q must return a pointer to an Arguments value in its CreateDefaultArguments method", ed.Function) } err := p.buildArgs(ed, reflect.ValueOf(args).Elem()) if err != nil { - return Expr[K]{}, fmt.Errorf("error while parsing arguments for call to '%v': %w", ed.Function, err) + return Expr[K]{}, fmt.Errorf("error while parsing arguments for call to %q: %w", ed.Function, err) } } @@ -61,17 +61,17 @@ func (p *Parser[K]) buildArgs(ed editor, argsVal reflect.Value) error { fieldTag, ok := argsType.Field(i).Tag.Lookup("ottlarg") if !ok { - return fmt.Errorf("no `ottlarg` struct tag on Arguments field '%s'", argsType.Field(i).Name) + return fmt.Errorf("no `ottlarg` struct tag on Arguments field %q", argsType.Field(i).Name) } argNum, err := strconv.Atoi(fieldTag) if err != nil { - return fmt.Errorf("ottlarg struct tag on field '%s' is not a valid integer: %w", argsType.Field(i).Name, err) + return fmt.Errorf("ottlarg struct tag on field %q is not a valid integer: %w", argsType.Field(i).Name, err) } if argNum < 0 || argNum >= len(ed.Arguments) { - return fmt.Errorf("ottlarg struct tag on field '%s' has value %d, but must be between 0 and %d", argsType.Field(i).Name, argNum, len(ed.Arguments)) + return fmt.Errorf("ottlarg struct tag on field %q has value %d, but must be between 0 and %d", argsType.Field(i).Name, argNum, len(ed.Arguments)) } argVal := ed.Arguments[argNum] @@ -167,7 +167,7 @@ func (p *Parser[K]) buildSliceArg(argVal value, argType reflect.Type) (any, erro } return arg, nil default: - return nil, fmt.Errorf("unsupported slice type '%s' for function", argType.Elem().Name()) + return nil, fmt.Errorf("unsupported slice type %q for function", argType.Elem().Name()) } } diff --git a/pkg/ottl/go.mod b/pkg/ottl/go.mod index df44e3dc0fd1..d546e1c2cd67 100644 --- a/pkg/ottl/go.mod +++ b/pkg/ottl/go.mod @@ -13,6 +13,7 @@ require ( go.opentelemetry.io/collector/component v0.81.0 go.opentelemetry.io/collector/pdata v1.0.0-rcv0013 go.opentelemetry.io/otel/trace v1.16.0 + go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea ) @@ -35,7 +36,6 @@ require ( go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect diff --git a/pkg/ottl/parser.go b/pkg/ottl/parser.go index 74caf47862fd..d60626180451 100644 --- a/pkg/ottl/parser.go +++ b/pkg/ottl/parser.go @@ -10,6 +10,7 @@ import ( "github.com/alecthomas/participle/v2" "go.opentelemetry.io/collector/component" + "go.uber.org/multierr" "go.uber.org/zap" ) @@ -96,15 +97,27 @@ func WithEnumParser[K any](parser EnumParser) Option[K] { } } +// ParseStatements parses string statements into ottl.Statement objects ready for execution. +// Returns a slice of statements and a nil error on successful parsing. +// If parsing fails, returns an empty slice with a multierr error containing +// an error per failed statement. func (p *Parser[K]) ParseStatements(statements []string) ([]*Statement[K], error) { var parsedStatements []*Statement[K] + var parseErr error + for _, statement := range statements { ps, err := p.ParseStatement(statement) if err != nil { - return nil, err + parseErr = multierr.Append(parseErr, fmt.Errorf("unable to parse OTTL statement %q: %w", statement, err)) + continue } parsedStatements = append(parsedStatements, ps) } + + if parseErr != nil { + return nil, parseErr + } + return parsedStatements, nil } @@ -134,7 +147,7 @@ func parseStatement(raw string) (*parsedStatement, error) { parsed, err := parser.ParseString("", raw) if err != nil { - return nil, fmt.Errorf("unable to parse OTTL statement: %w", err) + return nil, fmt.Errorf("statement has invalid syntax: %w", err) } err = parsed.checkForCustomError() if err != nil { diff --git a/pkg/ottl/parser_test.go b/pkg/ottl/parser_test.go index cfafc97df823..b760de38e9f2 100644 --- a/pkg/ottl/parser_test.go +++ b/pkg/ottl/parser_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/component/componenttest" + "go.uber.org/multierr" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottltest" ) @@ -1365,6 +1366,33 @@ func testParseEnum(val *EnumSymbol) (*Enum, error) { return nil, fmt.Errorf("enum symbol not provided") } +func Test_ParseStatements_Error(t *testing.T) { + statements := []string{ + `set(`, + `set("foo)`, + `set(name.)`, + } + + p, _ := NewParser( + CreateFactoryMap[any](), + testParsePath, + componenttest.NewNopTelemetrySettings(), + WithEnumParser[any](testParseEnum), + ) + + _, err := p.ParseStatements(statements) + + assert.Error(t, err) + + multiErrs := multierr.Errors(err) + + assert.Len(t, multiErrs, len(statements), "ParseStatements didn't return an error per statement") + + for i, statementErr := range multiErrs { + assert.ErrorContains(t, statementErr, fmt.Sprintf("unable to parse OTTL statement %q", statements[i])) + } +} + // This test doesn't validate parser results, simply checks whether the parse succeeds or not. // It's a fast way to check a large range of possible syntaxes. func Test_parseStatement(t *testing.T) { diff --git a/pkg/stanza/fileconsumer/file_test.go b/pkg/stanza/fileconsumer/file_test.go index c0f4528f4f9c..42c6433c9376 100644 --- a/pkg/stanza/fileconsumer/file_test.go +++ b/pkg/stanza/fileconsumer/file_test.go @@ -218,7 +218,8 @@ func TestFileFieldsUpdatedAfterRestart(t *testing.T) { op1, emitCalls1 := buildTestManager(t, cfg) // Create a file, then start - temp := openTemp(t, tempDir) + temp, err := os.CreateTemp(tempDir, "") + require.NoError(t, err) writeString(t, temp, "testlog1\n") persister := testutil.NewUnscopedMockPersister() @@ -230,10 +231,12 @@ func TestFileFieldsUpdatedAfterRestart(t *testing.T) { assert.Equal(t, temp.Name(), emitCall1.attrs.Path) require.NoError(t, op1.Stop()) + temp.Close() // On windows, we must close the file before renaming it newPath := temp.Name() + "_renamed" require.NoError(t, os.Rename(temp.Name(), newPath)) + temp = openFile(t, newPath) writeString(t, temp, "testlog2\n") op2, emitCalls2 := buildTestManager(t, cfg) diff --git a/processor/filterprocessor/config_test.go b/processor/filterprocessor/config_test.go index 3725043675ff..98ca97765a50 100644 --- a/processor/filterprocessor/config_test.go +++ b/processor/filterprocessor/config_test.go @@ -893,24 +893,19 @@ func TestLoadingConfigOTTL(t *testing.T) { errorMessage: "cannot use ottl conditions and include/exclude for logs at the same time", }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_span"), - errorMessage: "unable to parse OTTL statement: 1:25: unexpected token \"test\" (expected ( | ) \"]\")", + id: component.NewIDWithName(metadata.Type, "bad_syntax_span"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_spanevent"), - errorMessage: "unable to parse OTTL statement: 1:25: unexpected token \"test\" (expected ( | ) \"]\")", + id: component.NewIDWithName(metadata.Type, "bad_syntax_spanevent"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_metric"), - errorMessage: "unable to parse OTTL statement: 1:34: unexpected token \"test\" (expected ( | ) \"]\")", + id: component.NewIDWithName(metadata.Type, "bad_syntax_metric"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_datapoint"), - errorMessage: "unable to parse OTTL statement: 1:25: unexpected token \"test\" (expected ( | ) \"]\")", + id: component.NewIDWithName(metadata.Type, "bad_syntax_datapoint"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_log"), - errorMessage: "unable to parse OTTL statement: 1:25: unexpected token \"test\" (expected ( | ) \"]\")", + id: component.NewIDWithName(metadata.Type, "bad_syntax_log"), }, } @@ -924,7 +919,11 @@ func TestLoadingConfigOTTL(t *testing.T) { require.NoError(t, component.UnmarshalConfig(sub, cfg)) if tt.expected == nil { - assert.EqualError(t, component.ValidateConfig(cfg), tt.errorMessage) + if tt.errorMessage != "" { + assert.EqualError(t, component.ValidateConfig(cfg), tt.errorMessage) + } else { + assert.Error(t, component.ValidateConfig(cfg)) + } } else { assert.NoError(t, component.ValidateConfig(cfg)) assert.Equal(t, tt.expected, cfg) diff --git a/processor/groupbyattrsprocessor/processor.go b/processor/groupbyattrsprocessor/processor.go index 8d8f8b9d66a6..a06200b2ce5e 100644 --- a/processor/groupbyattrsprocessor/processor.go +++ b/processor/groupbyattrsprocessor/processor.go @@ -147,6 +147,7 @@ func (gap *groupByAttrsProcessor) processMetrics(ctx context.Context, md pmetric dataPoint.CopyTo(groupedMetric.ExponentialHistogram().DataPoints().AppendEmpty()) } + case pmetric.MetricTypeEmpty: } } } @@ -222,6 +223,7 @@ func getMetricInInstrumentationLibrary(ilm pmetric.ScopeMetrics, searchedMetric case pmetric.MetricTypeSummary: metric.SetEmptySummary() + case pmetric.MetricTypeEmpty: } return metric diff --git a/processor/routingprocessor/README.md b/processor/routingprocessor/README.md index 4faff8d22821..625621ca1057 100644 --- a/processor/routingprocessor/README.md +++ b/processor/routingprocessor/README.md @@ -104,6 +104,7 @@ It is also possible to mix both the conventional routing configuration and the r - [OTTL] statements can be applied only to resource attributes. - Currently, it is not possible to specify the boolean statements without function invocation as the routing condition. It is required to provide the NOOP `route()` or any other supported function as part of the routing statement, see [#13545](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/13545) for more information. +- If data is received on OTLP http server, `include_metadata` must be set to true in order to use context based routing. - Supported [OTTL] functions: - [IsMatch](../../pkg/ottl/ottlfuncs/README.md#IsMatch) - [delete_key](../../pkg/ottl/ottlfuncs/README.md#delete_key) diff --git a/processor/routingprocessor/extract.go b/processor/routingprocessor/extract.go index 5b1f57866dd8..b96a07d48a42 100644 --- a/processor/routingprocessor/extract.go +++ b/processor/routingprocessor/extract.go @@ -7,6 +7,7 @@ import ( "context" "strings" + "go.opentelemetry.io/collector/client" "go.uber.org/zap" "google.golang.org/grpc/metadata" ) @@ -32,14 +33,12 @@ func (e extractor) extractFromContext(ctx context.Context) string { // right now, we only support looking up attributes from requests that have // gone through the gRPC server in that case, it will add the HTTP headers // as context metadata - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return "" - } + values, ok := e.extractFromGRPCContext(ctx) - // we have gRPC metadata in the context but does it have our key? - values, ok := md[strings.ToLower(e.fromAttr)] if !ok { + values = e.extractFromHTTPContext(ctx) + } + if len(values) == 0 { return "" } @@ -52,3 +51,25 @@ func (e extractor) extractFromContext(ctx context.Context) string { return values[0] } + +func (e extractor) extractFromGRPCContext(ctx context.Context) ([]string, bool) { + + md, ok := metadata.FromIncomingContext(ctx) + + if !ok { + return nil, false + } + + values, ok := md[strings.ToLower(e.fromAttr)] + if !ok { + return nil, false + } + return values, true +} + +func (e extractor) extractFromHTTPContext(ctx context.Context) []string { + info := client.FromContext(ctx) + md := info.Metadata + values := md.Get(e.fromAttr) + return values +} diff --git a/processor/routingprocessor/extract_test.go b/processor/routingprocessor/extract_test.go index 2113e2d8dc65..b5a3136ab23b 100644 --- a/processor/routingprocessor/extract_test.go +++ b/processor/routingprocessor/extract_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/client" "go.uber.org/zap" "google.golang.org/grpc/metadata" ) @@ -55,6 +56,45 @@ func TestExtractorForTraces_FromContext(t *testing.T) { fromAttr: "X-Tenant", expectedValue: "globex", }, + { + name: "value from existing HTTP attribute", + ctxFunc: func() context.Context { + return client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"acme"}, + })}) + }, + fromAttr: "X-Tenant", + expectedValue: "acme", + }, + { + name: "value from existing HTTP attribute: case insensitive", + ctxFunc: func() context.Context { + return client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"acme"}, + })}) + }, + fromAttr: "x-tenant", + expectedValue: "acme", + }, + { + name: "no values from empty context", + ctxFunc: context.Background, + fromAttr: "X-Tenant", + expectedValue: "", + }, + { + name: "no values from existing HTTP attribute", + ctxFunc: func() context.Context { + return client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {""}, + })}) + }, + fromAttr: "X-Tenant", + expectedValue: "", + }, } for _, tc := range testcases { diff --git a/processor/routingprocessor/go.mod b/processor/routingprocessor/go.mod index 9a934b4dce72..1f84b8a476b9 100644 --- a/processor/routingprocessor/go.mod +++ b/processor/routingprocessor/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.81.0 github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/collector v0.81.0 go.opentelemetry.io/collector/component v0.81.0 go.opentelemetry.io/collector/config/configgrpc v0.81.0 go.opentelemetry.io/collector/confmap v0.81.0 @@ -47,7 +48,6 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.81.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/collector v0.81.0 // indirect go.opentelemetry.io/collector/config/configauth v0.81.0 // indirect go.opentelemetry.io/collector/config/configcompression v0.81.0 // indirect go.opentelemetry.io/collector/config/confignet v0.81.0 // indirect diff --git a/processor/routingprocessor/logs_test.go b/processor/routingprocessor/logs_test.go index 396178c3d3ab..28e60338c3cb 100644 --- a/processor/routingprocessor/logs_test.go +++ b/processor/routingprocessor/logs_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/client" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/plog" @@ -64,7 +65,7 @@ func TestLogs_RoutingWorks_Context(t *testing.T) { rl := l.ResourceLogs().AppendEmpty() rl.Resource().Attributes().PutStr("X-Tenant", "acme") - t.Run("non default route is properly used", func(t *testing.T) { + t.Run("grpc metadata: non default route is properly used", func(t *testing.T) { assert.NoError(t, exp.ConsumeLogs( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "acme", @@ -79,7 +80,7 @@ func TestLogs_RoutingWorks_Context(t *testing.T) { ) }) - t.Run("default route is taken when no matching route can be found", func(t *testing.T) { + t.Run("grpc metadata: default route is taken when no matching route can be found", func(t *testing.T) { assert.NoError(t, exp.ConsumeLogs( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "some-custom-value1", @@ -93,6 +94,37 @@ func TestLogs_RoutingWorks_Context(t *testing.T) { "log should not be routed to non default exporter", ) }) + + t.Run("client.Info metadata: non default route is properly used", func(t *testing.T) { + assert.NoError(t, exp.ConsumeLogs( + client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"acme"}, + })}), + l, + )) + assert.Len(t, defaultExp.AllLogs(), 1, + "log should not be routed to default exporter", + ) + assert.Len(t, lExp.AllLogs(), 2, + "log should be routed to non default exporter", + ) + }) + + t.Run("client.Info metadata: default route is taken when no matching route can be found", func(t *testing.T) { + assert.NoError(t, exp.ConsumeLogs(client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"some-custom-value1"}, + })}), + l, + )) + assert.Len(t, defaultExp.AllLogs(), 2, + "log should be routed to default exporter", + ) + assert.Len(t, lExp.AllLogs(), 2, + "log should not be routed to non default exporter", + ) + }) } func TestLogs_RoutingWorks_ResourceAttribute(t *testing.T) { diff --git a/processor/routingprocessor/metrics_test.go b/processor/routingprocessor/metrics_test.go index 87c6ae99df03..5fcc1ca59968 100644 --- a/processor/routingprocessor/metrics_test.go +++ b/processor/routingprocessor/metrics_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/client" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/pmetric" @@ -124,7 +125,7 @@ func TestMetrics_RoutingWorks_Context(t *testing.T) { rm := m.ResourceMetrics().AppendEmpty() rm.Resource().Attributes().PutStr("X-Tenant", "acme") - t.Run("non default route is properly used", func(t *testing.T) { + t.Run("grpc metadata: non default route is properly used", func(t *testing.T) { assert.NoError(t, exp.ConsumeMetrics( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "acme", @@ -139,7 +140,7 @@ func TestMetrics_RoutingWorks_Context(t *testing.T) { ) }) - t.Run("default route is taken when no matching route can be found", func(t *testing.T) { + t.Run("grpc metadata: default route is taken when no matching route can be found", func(t *testing.T) { assert.NoError(t, exp.ConsumeMetrics( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "some-custom-value1", @@ -153,6 +154,39 @@ func TestMetrics_RoutingWorks_Context(t *testing.T) { "metric should not be routed to non default exporter", ) }) + + t.Run("client.Info metadata: non default route is properly used", func(t *testing.T) { + assert.NoError(t, exp.ConsumeMetrics( + client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"acme"}, + })}), + m, + )) + assert.Len(t, defaultExp.AllMetrics(), 1, + "metric should not be routed to default exporter", + ) + assert.Len(t, mExp.AllMetrics(), 2, + "metric should be routed to non default exporter", + ) + }) + + t.Run("client.Info metadata: default route is taken when no matching route can be found", func(t *testing.T) { + assert.NoError(t, exp.ConsumeMetrics( + client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"some-custom-value1"}, + })}), + m, + )) + assert.Len(t, defaultExp.AllMetrics(), 2, + "metric should be routed to default exporter", + ) + assert.Len(t, mExp.AllMetrics(), 2, + "metric should not be routed to non default exporter", + ) + }) + } func TestMetrics_RoutingWorks_ResourceAttribute(t *testing.T) { diff --git a/processor/routingprocessor/traces_test.go b/processor/routingprocessor/traces_test.go index 7d5fbe98de72..d42a71dc89c8 100644 --- a/processor/routingprocessor/traces_test.go +++ b/processor/routingprocessor/traces_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/client" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/consumer/consumertest" @@ -168,7 +169,7 @@ func TestTraces_RoutingWorks_Context(t *testing.T) { rs := tr.ResourceSpans().AppendEmpty() rs.Resource().Attributes().PutStr("X-Tenant", "acme") - t.Run("non default route is properly used", func(t *testing.T) { + t.Run("grpc metadata: non default route is properly used", func(t *testing.T) { assert.NoError(t, exp.ConsumeTraces( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "acme", @@ -183,7 +184,7 @@ func TestTraces_RoutingWorks_Context(t *testing.T) { ) }) - t.Run("default route is taken when no matching route can be found", func(t *testing.T) { + t.Run("grpc metadata: default route is taken when no matching route can be found", func(t *testing.T) { assert.NoError(t, exp.ConsumeTraces( metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{ "X-Tenant": "some-custom-value1", @@ -197,6 +198,38 @@ func TestTraces_RoutingWorks_Context(t *testing.T) { "trace should not be routed to non default exporter", ) }) + + t.Run("client.Info metadata: non default route is properly used", func(t *testing.T) { + assert.NoError(t, exp.ConsumeTraces( + client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"acme"}, + })}), + tr, + )) + assert.Len(t, defaultExp.AllTraces(), 1, + "trace should not be routed to default exporter", + ) + assert.Len(t, tExp.AllTraces(), 2, + "trace should be routed to non default exporter", + ) + }) + + t.Run("client.Info metadata: default route is taken when no matching route can be found", func(t *testing.T) { + assert.NoError(t, exp.ConsumeTraces( + client.NewContext(context.Background(), + client.Info{Metadata: client.NewMetadata(map[string][]string{ + "X-Tenant": {"some-custom-value1"}, + })}), + tr, + )) + assert.Len(t, defaultExp.AllTraces(), 2, + "trace should be routed to default exporter", + ) + assert.Len(t, tExp.AllTraces(), 2, + "trace should not be routed to non default exporter", + ) + }) } func TestTraces_RoutingWorks_ResourceAttribute(t *testing.T) { diff --git a/processor/transformprocessor/config_test.go b/processor/transformprocessor/config_test.go index cbde7b0e489a..add12415f9b0 100644 --- a/processor/transformprocessor/config_test.go +++ b/processor/transformprocessor/config_test.go @@ -20,9 +20,8 @@ func TestLoadConfig(t *testing.T) { t.Parallel() tests := []struct { - id component.ID - expected component.Config - errorMessage string + id component.ID + expected component.Config }{ { id: component.NewIDWithName(metadata.Type, ""), @@ -92,28 +91,22 @@ func TestLoadConfig(t *testing.T) { }, }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_trace"), - errorMessage: "unable to parse OTTL statement: 1:18: unexpected token \"where\" (expected \")\" Key*)", + id: component.NewIDWithName(metadata.Type, "bad_syntax_trace"), }, { - id: component.NewIDWithName(metadata.Type, "unknown_function_trace"), - errorMessage: "undefined function not_a_function", + id: component.NewIDWithName(metadata.Type, "unknown_function_trace"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_metric"), - errorMessage: "unable to parse OTTL statement: 1:18: unexpected token \"where\" (expected \")\" Key*)", + id: component.NewIDWithName(metadata.Type, "bad_syntax_metric"), }, { - id: component.NewIDWithName(metadata.Type, "unknown_function_metric"), - errorMessage: "undefined function not_a_function", + id: component.NewIDWithName(metadata.Type, "unknown_function_metric"), }, { - id: component.NewIDWithName(metadata.Type, "bad_syntax_log"), - errorMessage: "unable to parse OTTL statement: 1:18: unexpected token \"where\" (expected \")\" Key*)", + id: component.NewIDWithName(metadata.Type, "bad_syntax_log"), }, { - id: component.NewIDWithName(metadata.Type, "unknown_function_log"), - errorMessage: "undefined function not_a_function", + id: component.NewIDWithName(metadata.Type, "unknown_function_log"), }, } for _, tt := range tests { @@ -129,7 +122,7 @@ func TestLoadConfig(t *testing.T) { assert.NoError(t, component.UnmarshalConfig(sub, cfg)) if tt.expected == nil { - assert.EqualError(t, component.ValidateConfig(cfg), tt.errorMessage) + assert.Error(t, component.ValidateConfig(cfg)) return } assert.NoError(t, component.ValidateConfig(cfg)) diff --git a/receiver/dockerstatsreceiver/integration_test.go b/receiver/dockerstatsreceiver/integration_test.go index 0feb94bbe155..c81e69c213bf 100644 --- a/receiver/dockerstatsreceiver/integration_test.go +++ b/receiver/dockerstatsreceiver/integration_test.go @@ -53,6 +53,7 @@ func paramsAndContext(t *testing.T) (rcvr.CreateSettings, context.Context, conte } func createNginxContainer(ctx context.Context, t *testing.T) testcontainers.Container { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") req := testcontainers.ContainerRequest{ Image: "docker.io/library/nginx:1.17", ExposedPorts: []string{"80/tcp"}, @@ -85,6 +86,8 @@ func hasResourceScopeMetrics(containerID string, metrics []pmetric.Metrics) bool func TestDefaultMetricsIntegration(t *testing.T) { t.Parallel() + // remove nolint when https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/24240 is resolved + // nolint:staticcheck params, ctx, cancel := paramsAndContext(t) defer cancel() @@ -139,6 +142,8 @@ func TestMonitoringAddedAndRemovedContainerIntegration(t *testing.T) { func TestExcludedImageProducesNoMetricsIntegration(t *testing.T) { t.Parallel() + // remove nolint when https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/24240 is resolved + // nolint:staticcheck params, ctx, cancel := paramsAndContext(t) defer cancel() diff --git a/receiver/snmpreceiver/integration_test.go b/receiver/snmpreceiver/integration_test.go index bf5f225ea10b..95fb887124b7 100644 --- a/receiver/snmpreceiver/integration_test.go +++ b/receiver/snmpreceiver/integration_test.go @@ -26,6 +26,8 @@ import ( ) func TestIntegration(t *testing.T) { + // remove nolint when https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/24240 is resolved + // nolint:staticcheck testCases := []struct { desc string configFilename string @@ -94,6 +96,7 @@ var ( ) func getContainer(t *testing.T, req testcontainers.ContainerRequest) testcontainers.Container { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") require.NoError(t, req.Validate()) container, err := testcontainers.GenericContainer( context.Background(), diff --git a/receiver/sqlqueryreceiver/integration_test.go b/receiver/sqlqueryreceiver/integration_test.go index 4fd470c2f909..0ec0d1fcc488 100644 --- a/receiver/sqlqueryreceiver/integration_test.go +++ b/receiver/sqlqueryreceiver/integration_test.go @@ -248,6 +248,7 @@ func TestPostgresIntegrationLogsTrackingWithStorage(t *testing.T) { } func startPostgresDbContainer(t *testing.T, externalPort string) testcontainers.Container { + t.Skip("See https://github.com/testcontainers/testcontainers-go/issues/1359") req := testcontainers.ContainerRequest{ Image: "postgres:9.6.24", Env: map[string]string{