Merge audit logging feature branch into main

.github/workflows/ci.yml

@@ -86,7 +86,7 @@ jobs:
       - name: Build
         run: go build -v "./..."
       - name: Run Tests
-        run: go test -shuffle=on -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )'
+        run: go test -tags cb_sg_devmode -shuffle=on -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )'
         shell: bash
       - name: Annotate Failures
         if: always()
@@ -104,7 +104,7 @@ jobs:
         with:
           go-version: 1.22.5
       - name: Run Tests
-        run: go test -race -shuffle=on -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )'
+        run: go test -tags cb_sg_devmode -race -shuffle=on -timeout=30m -count=1 -json -v "./..." | tee test.json | jq -s -jr 'sort_by(.Package,.Time) | .[].Output | select (. != null )'
         shell: bash
       - name: Annotate Failures
         if: always()

base/audit_events.go

@@ -0,0 +1,88 @@
+// Copyright 2024-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+
+package base
+
+const (
+	// auditdSyncGatewayStartID is the start of an ID range allocated for Sync Gateway by auditd
+	auditdSyncGatewayStartID AuditID = 53248
+	// auditdSyncGatewayEndID is the maximum ID that can be allocated to a Sync Gateway audit descriptor.
+	auditdSyncGatewayEndID = auditdSyncGatewayStartID + auditdIDBlockSize // 57343
+
+	// auditdIDBlockSize is the number of IDs allocated to each module in auditd.
+	auditdIDBlockSize = 0xFFF
+)
+
+const (
+	AuditIDAuditEnabled AuditID = 53248
+
+	AuditIDPublicUserAuthenticated        AuditID = 53260
+	AuditIDPublicUserAuthenticationFailed AuditID = 53261
+
+	AuditIDReadDatabase AuditID = 53301
+)
+
+// AuditEvents is a table of audit events created by Sync Gateway.
+//
+// This is used to generate:
+//   - events themselves
+//   - a kv-auditd-compatible descriptor with TestGenerateAuditdModuleDescriptor
+//   - CSV output for each event to be used to document
+var AuditEvents = events{
+	AuditIDAuditEnabled: {}, // TODO: unreferenced event - somehow find this mistake via test or lint!
+	AuditIDReadDatabase: {
+		Name:        "Read database",
+		Description: "Information about this database was read.",
+		MandatoryFields: AuditFields{
+			"db": "database_name",
+		},
+		OptionalFields:     AuditFields{},
+		EnabledByDefault:   true,
+		FilteringPermitted: false,
+		EventType:          eventTypeUser,
+	},
+	AuditIDPublicUserAuthenticated: {
+		Name:        "User authenticated",
+		Description: "User successfully authenticated",
+		MandatoryFields: AuditFields{
+			"method": "auth_method", // e.g. "basic", "oidc", "cookie", ...
+		},
+		OptionalFields: AuditFields{
+			"oidc_issuer": "issuer",
+		},
+		EnabledByDefault:   true,
+		FilteringPermitted: false,
+		EventType:          eventTypeUser,
+	},
+	AuditIDPublicUserAuthenticationFailed: {
+		Name:        "User authentication failed",
+		Description: "User authentication failed",
+		MandatoryFields: AuditFields{
+			"method": "auth_method", // e.g. "basic", "oidc", "cookie", ...
+		},
+		OptionalFields: AuditFields{
+			"username": "username",
+		},
+		EnabledByDefault:   true,
+		FilteringPermitted: false,
+		EventType:          eventTypeUser,
+	},
+}
+
+// DefaultAuditEventIDs is a list of audit event IDs that are enabled by default.
+var DefaultAuditEventIDs = buildDefaultAuditIDList(AuditEvents)
+
+func buildDefaultAuditIDList(e events) []uint {
+	var ids []uint
+	for id, event := range e {
+		if event.EnabledByDefault {
+			ids = append(ids, uint(id))
+		}
+	}
+	return ids
+}

base/audit_events_test.go

@@ -0,0 +1,76 @@
+// Copyright 2024-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+
+package base
+
+import (
+	"bytes"
+	"encoding/csv"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
+)
+
+func TestValidateAuditEvents(t *testing.T) {
+	// Ensures that the above audit event IDs are within the allocated range and are valid.
+	require.NoError(t, validateAuditEvents(AuditEvents))
+}
+
+func validateAuditEvents(e events) error {
+	for id, descriptor := range e {
+		if id < auditdSyncGatewayStartID || id > auditdSyncGatewayEndID {
+			return fmt.Errorf("invalid audit event ID: %d %q (allowed range: %d-%d)",
+				id, descriptor.Name, auditdSyncGatewayStartID, auditdSyncGatewayEndID)
+		}
+	}
+	return nil
+}
+
+// TestGenerateAuditDescriptorCSV outputs a CSV of AuditEvents.
+func TestGenerateAuditDescriptorCSV(t *testing.T) {
+	b, err := generateCSVModuleDescriptor(AuditEvents)
+	require.NoError(t, err)
+	fmt.Print(string(b))
+}
+
+// generateCSVModuleDescriptor returns a CSV module descriptor for the given events.
+func generateCSVModuleDescriptor(e events) ([]byte, error) {
+	buf := bytes.NewBuffer(nil)
+	w := csv.NewWriter(buf)
+
+	// Write header
+	if err := w.Write([]string{"ID", "Name", "Description", "DefaultEnabled", "Filterable", "EventType", "MandatoryFields", "OptionalFields"}); err != nil {
+		return nil, err
+	}
+
+	for id, event := range e {
+		if err := w.Write([]string{
+			id.String(),
+			event.Name,
+			event.Description,
+			strconv.FormatBool(event.EnabledByDefault),
+			strconv.FormatBool(event.FilteringPermitted),
+			string(event.EventType),
+			strings.Join(maps.Keys(event.MandatoryFields), ", "),
+			strings.Join(maps.Keys(event.OptionalFields), ", "),
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	w.Flush()
+	if err := w.Error(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

base/audit_types.go

@@ -0,0 +1,90 @@
+// Copyright 2024-Pres	ent Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+
+package base
+
+import (
+	"fmt"
+	"strconv"
+)
+
+// AuditID is a unique identifier for an audit event.
+type AuditID uint
+
+// String implements Stringer for AuditID
+func (i AuditID) String() string {
+	return strconv.FormatUint(uint64(i), 10)
+}
+
+// events is a map of audit event IDs to event descriptors.
+type events map[AuditID]EventDescriptor
+
+// EventDescriptor is an audit event. The fields closely (but not exactly) follows kv_engine's auditd descriptor implementation.
+type EventDescriptor struct {
+	// Name is a short textual Name of the event
+	Name string
+	// Description is a longer Name / Description of the event
+	Description string
+	// EnabledByDefault indicates whether the event should be enabled by default
+	EnabledByDefault bool
+	// FilteringPermitted indicates whether the event can be filtered or not
+	FilteringPermitted bool
+	// MandatoryFields describe field(s) required for a valid instance of the event
+	MandatoryFields AuditFields
+
+	// The following fields are for documentation-use only.
+	// OptionalFields describe optional field(s) valid in an instance of the event
+	OptionalFields AuditFields
+	// EventType represents a type of event
+	EventType eventType
+}
+
+const (
+	eventTypeAdmin eventType = "admin"
+	eventTypeUser  eventType = "user"
+	eventTypeData  eventType = "data"
+)
+
+type eventType string
+
+// AuditFields represents additional data associated with a specific audit event invocation.
+// E.g. Username, IPs, request parameters, etc.
+type AuditFields map[string]any
+
+func (i AuditID) MustValidateFields(f AuditFields) {
+	if err := i.ValidateFields(f); err != nil {
+		panic(fmt.Errorf("audit event %s invalid:\n%v", i, err))
+	}
+}
+
+func (i AuditID) ValidateFields(f AuditFields) error {
+	if i < auditdSyncGatewayStartID || i > auditdSyncGatewayEndID {
+		return fmt.Errorf("invalid audit event ID: %d (allowed range: %d-%d)", i, auditdSyncGatewayStartID, auditdSyncGatewayEndID)
+	}
+	event, ok := AuditEvents[i]
+	if !ok {
+		return fmt.Errorf("unknown audit event ID %d", i)
+	}
+	return mandatoryFieldsPresent(f, event.MandatoryFields)
+}
+
+func mandatoryFieldsPresent(fields, mandatoryFields AuditFields) error {
+	me := &MultiError{}
+	for k, v := range mandatoryFields {
+		// recurse if map
+		if vv, ok := v.(map[string]any); ok {
+			if pv, ok := fields[k].(map[string]any); ok {
+				me = me.Append(mandatoryFieldsPresent(pv, vv))
+			}
+		}
+		if _, ok := fields[k]; !ok {
+			me = me.Append(fmt.Errorf("missing mandatory field %s", k))
+		}
+	}
+	return me.ErrorOrNil()
+}

base/auditd_descriptor.go

@@ -0,0 +1,34 @@
+// Copyright 2024-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+
+package base
+
+import (
+	"encoding/json"
+)
+
+const (
+	// moduleDescriptorName is the name of the module. Must match the name used in the module descriptor file.
+	moduleDescriptorName = "sync_gateway"
+	// auditdFormatVersion is the version of the auditd format to be used. Only version 2 is supported.
+	auditdFormatVersion = 2
+)
+
+// generateAuditdModuleDescriptor returns an auditd-compatible module descriptor for the given events.
+func generateAuditdModuleDescriptor(e events) ([]byte, error) {
+	auditEvents := make([]auditdEventDescriptor, 0, len(e))
+	for id, event := range e {
+		auditEvents = append(auditEvents, toAuditdEventDescriptor(id, event))
+	}
+	m := auditdModuleDescriptor{
+		Version: auditdFormatVersion,
+		Module:  moduleDescriptorName,
+		Events:  auditEvents,
+	}
+	return json.Marshal(m)
+}

base/auditd_descriptor_test.go

@@ -0,0 +1,22 @@
+// Copyright 2024-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+
+package base
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestGenerateAuditdModuleDescriptor outputs a generated auditd module descriptor for AuditEvents.
+func TestGenerateAuditdModuleDescriptor(t *testing.T) {
+	b, err := generateAuditdModuleDescriptor(AuditEvents)
+	require.NoError(t, err)
+	t.Log(string(b))
+}