Move all remaining packages out of lib

.errcheck.txt

@@ -29,37 +29,6 @@
 (*github.com/Jeffail/benthos/v3/internal/impl/pulsar.pulsarReader).disconnect
 (*github.com/Jeffail/benthos/v3/internal/impl/pulsar.pulsarWriter).disconnect
 (*github.com/Jeffail/benthos/v3/internal/transaction.Tracked).Ack
-(*github.com/Jeffail/benthos/v3/lib/api.Type).Shutdown
-(*github.com/Jeffail/benthos/v3/lib/broker.FanIn).WaitForClose
-(*github.com/Jeffail/benthos/v3/lib/buffer/parallel.Memory).PushMessage
-(*github.com/Jeffail/benthos/v3/lib/buffer/single.MmapCache).Delete
-(*github.com/Jeffail/benthos/v3/lib/buffer/single.MmapCache).Remove
-(*github.com/Jeffail/benthos/v3/lib/buffer/single.MmapCache).RemoveAll
-(*github.com/Jeffail/benthos/v3/lib/input/reader.AMQP).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.AMQP09).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.AMQP1).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.Kafka).commit
-(*github.com/Jeffail/benthos/v3/lib/input/reader.Kinesis).commit
-(*github.com/Jeffail/benthos/v3/lib/input/reader.NSQ).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.Preserver).Acknowledge
-(*github.com/Jeffail/benthos/v3/lib/input/reader.RedisList).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.RedisPubSub).disconnect
-(*github.com/Jeffail/benthos/v3/lib/input/reader.RedisStreams).disconnect
-(*github.com/Jeffail/benthos/v3/lib/message.Part).SetJSON
-(*github.com/Jeffail/benthos/v3/lib/message.Type).Iter
-(*github.com/Jeffail/benthos/v3/lib/message/metadata.Type).Iter
-(*github.com/Jeffail/benthos/v3/lib/metrics.CloudWatch).flush
-(*github.com/Jeffail/benthos/v3/lib/output.MockOutputType).Consume
-(*github.com/Jeffail/benthos/v3/lib/output/writer.AMQP).disconnect
-(*github.com/Jeffail/benthos/v3/lib/output/writer.AMQP1).disconnect
-(*github.com/Jeffail/benthos/v3/lib/output/writer.RedisHash).disconnect
-(*github.com/Jeffail/benthos/v3/lib/output/writer.RedisList).disconnect
-(*github.com/Jeffail/benthos/v3/lib/output/writer.RedisPubSub).disconnect
-(*github.com/Jeffail/benthos/v3/lib/output/writer.RedisStreams).disconnect
-(*github.com/Jeffail/benthos/v3/lib/processor.subprocWrapper).start
-(*github.com/Jeffail/benthos/v3/lib/processor.subprocWrapper).stop
-(*github.com/Jeffail/benthos/v3/lib/stream.Type).Stop
-(*github.com/Jeffail/benthos/v3/lib/util.ClosablePool).Close
 (*github.com/Jeffail/benthos/v3/public/bloblang.Environment).RegisterFunction
 (*github.com/Jeffail/benthos/v3/public/bloblang.Environment).RegisterMethod
 (*github.com/Jeffail/gabs/v2.Container).ArrayAppendP
@@ -101,27 +70,6 @@
 (github.com/Jeffail/benthos/v3/internal/codec.Reader).Close
 (github.com/Jeffail/benthos/v3/internal/codec.Writer).Close
 (github.com/Jeffail/benthos/v3/internal/codec.Writer).EndBatch
-(github.com/Jeffail/benthos/v3/lib/buffer.Single).ShiftMessage
-(github.com/Jeffail/benthos/v3/lib/metrics.Flat).Decr
-(github.com/Jeffail/benthos/v3/lib/metrics.Flat).Gauge
-(github.com/Jeffail/benthos/v3/lib/metrics.Flat).Incr
-(github.com/Jeffail/benthos/v3/lib/metrics.Flat).Timing
-(github.com/Jeffail/benthos/v3/lib/metrics.StatCounter).Incr
-(github.com/Jeffail/benthos/v3/lib/metrics.StatGauge).Decr
-(github.com/Jeffail/benthos/v3/lib/metrics.StatGauge).Incr
-(github.com/Jeffail/benthos/v3/lib/metrics.StatGauge).Set
-(github.com/Jeffail/benthos/v3/lib/metrics.StatTimer).Timing
-(github.com/Jeffail/benthos/v3/lib/metrics.Type).Close
-(github.com/Jeffail/benthos/v3/lib/processor.hasher).Write
-(github.com/Jeffail/benthos/v3/lib/tracer.Type).Close
-(github.com/Jeffail/benthos/v3/lib/types.Cache).Set
-(github.com/Jeffail/benthos/v3/lib/types.Closable).WaitForClose
-(github.com/Jeffail/benthos/v3/lib/types.Consumer).Consume
-(github.com/Jeffail/benthos/v3/lib/types.Message).Iter
-(github.com/Jeffail/benthos/v3/lib/types.Metadata).Iter
-(github.com/Jeffail/benthos/v3/lib/types.OutputWriter).WriteTransaction
-(github.com/Jeffail/benthos/v3/lib/types.Part).JSON
-(github.com/Jeffail/benthos/v3/lib/types.Part).SetJSON
 (github.com/Shopify/sarama.Client).Close
 (github.com/Shopify/sarama.Consumer).Close
 (github.com/Shopify/sarama.ConsumerGroup).Close
@@ -131,11 +79,6 @@
 (github.com/influxdata/influxdb1-client/v2.Client).Close
 (github.com/nats-io/stan.go.Conn).Close
 (github.com/nats-io/stan.go.Subscription).Unsubscribe
-(github.com/quipo/statsd.Statsd).Close
-(github.com/quipo/statsd.Statsd).Decr
-(github.com/quipo/statsd.Statsd).Gauge
-(github.com/quipo/statsd.Statsd).Incr
-(github.com/quipo/statsd.Statsd).Timing
 (go.nanomsg.org/mangos/v3.Socket).Close
 (golang.org/x/crypto/ssh.Conn).Close
 (io.Closer).Close
@@ -148,11 +91,8 @@
 (net.PacketConn).SetReadDeadline
 (net/http.ResponseWriter).Write
 fmt.Fprintf
-github.com/Jeffail/benthos/v3/lib/broker.ComplementGenericConfig
-github.com/Jeffail/benthos/v3/lib/message/roundtrip.SetAsResponse
 github.com/urfave/cli/v2.ShowAppHelp
 internal/docs.SanitiseComponentConfig
-lib/test/integration.sendBatch
 os.Setenv
 os.Unsetenv
 syscall.Statfs

.goreleaser.yml

@@ -14,8 +14,8 @@ builds:
       - CGO_ENABLED=0
     ldflags: >
       -s -w
-      -X github.com/Jeffail/benthos/v3/lib/service.Version={{.Version}}
-      -X github.com/Jeffail/benthos/v3/lib/service.DateBuilt={{.Date}}
+      -X github.com/Jeffail/benthos/v3/internal/cli.Version={{.Version}}
+      -X github.com/Jeffail/benthos/v3/internal/cli.DateBuilt={{.Date}}
   - id: benthos-lambda
     main: cmd/serverless/benthos-lambda/main.go
     binary: benthos-lambda

CONTRIBUTING.md

@@ -45,7 +45,7 @@ If your change has an impact on documentation then make sure it is generated wit
 
 ### Adding New Components
 
-The APIs for adding new components (inputs, outputs, processors, caches, etc) has recently been simplified. If you are planning to create a new component you should use the latest implementations within `./internal/impl` as inspiration rather than those found under `./lib`.
+The APIs for adding new components (inputs, outputs, processors, caches, etc) has recently been simplified. If you are planning to create a new component you should use the latest implementations within `./internal/impl` as inspiration.
 
 ### Plugins
 

Makefile

@@ -18,8 +18,8 @@ VER_PATCH := $(shell echo $(VER_CUT) | cut -f3 -d.)
 VER_RC    := $(shell echo $(VER_PATCH) | cut -f2 -d-)
 DATE      := $(shell date +"%Y-%m-%dT%H:%M:%SZ")
 
-VER_FLAGS = -X github.com/Jeffail/benthos/v3/lib/service.Version=$(VERSION) \
-	-X github.com/Jeffail/benthos/v3/lib/service.DateBuilt=$(DATE)
+VER_FLAGS = -X github.com/Jeffail/benthos/v3/internal/cli.Version=$(VERSION) \
+	-X github.com/Jeffail/benthos/v3/internal/cli.DateBuilt=$(DATE)
 
 LD_FLAGS   = -w -s
 GO_FLAGS   =
@@ -35,7 +35,7 @@ install: $(APPS)
 deps:
 	@go mod tidy
 
-SOURCE_FILES = $(shell find lib internal public cmd -type f)
+SOURCE_FILES = $(shell find internal public cmd -type f)
 TEMPLATE_FILES = $(shell find template -path template/test -prune -o -type f -name "*.yaml")
 
 $(PATHINSTBIN)/%: $(SOURCE_FILES) $(TEMPLATE_FILES)
@@ -84,7 +84,7 @@ fmt:
 
 lint:
 	@go vet $(GO_FLAGS) ./...
-	@golangci-lint run --timeout 5m cmd/... lib/... internal/... public/...
+	@golangci-lint run --timeout 5m cmd/... internal/... public/...
 
 test: $(APPS)
 	@go test $(GO_FLAGS) -ldflags "$(LD_FLAGS)" -timeout 3m -race ./...

cmd/tools/benthos_docs_gen/main.go

@@ -10,15 +10,15 @@ import (
 
 	"github.com/Jeffail/benthos/v3/internal/bundle"
 	"github.com/Jeffail/benthos/v3/internal/docs"
+	"github.com/Jeffail/benthos/v3/internal/old/buffer"
+	"github.com/Jeffail/benthos/v3/internal/old/cache"
+	"github.com/Jeffail/benthos/v3/internal/old/input"
+	"github.com/Jeffail/benthos/v3/internal/old/metrics"
+	"github.com/Jeffail/benthos/v3/internal/old/output"
+	"github.com/Jeffail/benthos/v3/internal/old/processor"
+	"github.com/Jeffail/benthos/v3/internal/old/ratelimit"
+	"github.com/Jeffail/benthos/v3/internal/old/tracer"
 	"github.com/Jeffail/benthos/v3/internal/template"
-	"github.com/Jeffail/benthos/v3/lib/buffer"
-	"github.com/Jeffail/benthos/v3/lib/cache"
-	"github.com/Jeffail/benthos/v3/lib/input"
-	"github.com/Jeffail/benthos/v3/lib/metrics"
-	"github.com/Jeffail/benthos/v3/lib/output"
-	"github.com/Jeffail/benthos/v3/lib/processor"
-	"github.com/Jeffail/benthos/v3/lib/ratelimit"
-	"github.com/Jeffail/benthos/v3/lib/tracer"
 
 	_ "github.com/Jeffail/benthos/v3/public/components/all"
 )

internal/api/api.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	httpdocs "github.com/Jeffail/benthos/v3/internal/http/docs"
-	"github.com/Jeffail/benthos/v3/lib/log"
-	"github.com/Jeffail/benthos/v3/lib/metrics"
+	"github.com/Jeffail/benthos/v3/internal/log"
+	"github.com/Jeffail/benthos/v3/internal/old/metrics"
 	"github.com/gorilla/mux"
 	yaml "gopkg.in/yaml.v3"
 )

internal/api/api_test.go

@@ -5,8 +5,8 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/Jeffail/benthos/v3/lib/log"
-	"github.com/Jeffail/benthos/v3/lib/metrics"
+	"github.com/Jeffail/benthos/v3/internal/log"
+	"github.com/Jeffail/benthos/v3/internal/old/metrics"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

internal/batch/count.go

@@ -3,7 +3,7 @@ package batch
 import (
 	"context"
 
-	"github.com/Jeffail/benthos/v3/lib/message"
+	"github.com/Jeffail/benthos/v3/internal/message"
 )
 
 type batchedCountKeyType int

internal/batch/count_test.go

@@ -3,7 +3,7 @@ package batch
 import (
 	"testing"
 
-	"github.com/Jeffail/benthos/v3/lib/message"
+	"github.com/Jeffail/benthos/v3/internal/message"
 	"github.com/stretchr/testify/assert"
 )
 

internal/batch/error.go

@@ -5,7 +5,7 @@ package batch
 import (
 	"errors"
 
-	"github.com/Jeffail/benthos/v3/lib/message"
+	"github.com/Jeffail/benthos/v3/internal/message"
 )
 
 // Error is an error type that also allows storing granular errors for each

lib/message/batch/docs.go → internal/batch/policy/docs.go

@@ -1,4 +1,4 @@
-package batch
+package policy
 
 import "github.com/Jeffail/benthos/v3/internal/docs"
 

lib/message/batch/docs_test.go → internal/batch/policy/docs_test.go

@@ -1,21 +1,21 @@
-package batch_test
+package policy_test
 
 import (
 	"testing"
 
+	"github.com/Jeffail/benthos/v3/internal/batch/policy"
 	"github.com/Jeffail/benthos/v3/internal/docs"
-	"github.com/Jeffail/benthos/v3/lib/message/batch"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
 
 func TestBatchPolicySanit(t *testing.T) {
-	conf := batch.NewPolicyConfig()
+	conf := policy.NewConfig()
 
 	var node yaml.Node
 	require.NoError(t, node.Encode(conf))
-	require.NoError(t, batch.FieldSpec().SanitiseYAML(&node, docs.SanitiseConfig{
+	require.NoError(t, policy.FieldSpec().SanitiseYAML(&node, docs.SanitiseConfig{
 		RemoveTypeField: true,
 	}))
 

internal/batch/policy/package.go

@@ -0,0 +1,3 @@
+// Package policy provides tooling for creating and executing Benthos message
+// batch policies.
+package policy