Merge commit '7e891607bdc09f4c20d713d09a04c2950795de57' into anemone-…

…merge

.golangci-strict.yml

@@ -34,37 +34,36 @@ linters:
     #- unparam # Reports unused function parameters
     - unused # (megacheck) Checks Go code for unused constants, variables, functions and types
   disable:
-    - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
-    - depguard # Go linter that checks if package imports are in a list of acceptable packages
-    - dogsled # Checks assignments with too many blank identifiers # (e.g. x, _, _, _, := f())
-    - err113 # Golang linter to check the errors handling expressions
-    - funlen # Tool for detection of long functions
-    - gochecknoglobals # Checks that no globals are present in Go code
-    - gochecknoinits # Checks that no init functions are present in Go code
-    - gocognit # Computes and checks the cognitive complexity of functions
-    - gocyclo # Computes and checks the cyclomatic complexity of functions
-    - godot # Check if comments end in a period
-    - godox # Tool for detection of FIXME, TODO and other comment keywords
-    - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
-    - gomodguard # Allow and block list linter for direct Go module dependencies.
-    - lll # Reports long lines
-    - nestif # Reports deeply nested if statements
-    - nolintlint # Reports ill-formed or insufficient nolint directives
-    - rowserrcheck # checks whether Err of rows is checked successfully
-    - stylecheck # Stylecheck is a replacement for golint
-    - testpackage # linter that makes you use a separate _test package
-    - whitespace # Tool for detection of leading and trailing whitespace
-    - wsl # Whitespace Linter - Forces you to use empty lines!
+    # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
+    # - depguard # Go linter that checks if package imports are in a list of acceptable packages
+    # - dogsled # Checks assignments with too many blank identifiers # (e.g. x, _, _, _, := f())
+    # - err113 # Golang linter to check the errors handling expressions
+    # - funlen # Tool for detection of long functions
+    # - gochecknoglobals # Checks that no globals are present in Go code
+    # - gochecknoinits # Checks that no init functions are present in Go code
+    # - gocognit # Computes and checks the cognitive complexity of functions
+    # - gocyclo # Computes and checks the cyclomatic complexity of functions
+    # - godot # Check if comments end in a period
+    # - godox # Tool for detection of FIXME, TODO and other comment keywords
+    # - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
+    # - gomodguard # Allow and block list linter for direct Go module dependencies.
+    # - lll # Reports long lines
+    # - nestif # Reports deeply nested if statements
+    # - rowserrcheck # checks whether Err of rows is checked successfully
+    # - stylecheck # Stylecheck is a replacement for golint
+    # - testpackage # linter that makes you use a separate _test package
+    # - whitespace # Tool for detection of leading and trailing whitespace
+    # - wsl # Whitespace Linter - Forces you to use empty lines!
     # Once fixed, should enable
-    - dupl # Tool for code clone detection
-    - goconst # Finds repeated strings that could be replaced by a constant
-    - goprintffuncname # Checks that printf-like functions are named with `f` at the end
-    - gosec # (gas) Inspects source code for security problems
+    # - dupl # Tool for code clone detection
+    # - goconst # Finds repeated strings that could be replaced by a constant
+    # - goprintffuncname # Checks that printf-like functions are named with `f` at the end
+    # - gosec # (gas) Inspects source code for security problems
     - gosimple # (megacheck) Linter for Go source code that specializes in simplifying a code
-    - nakedret # Finds naked returns in functions greater than a specified function length
-    - prealloc # Finds slice declarations that could potentially be preallocated
-    - revive # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes
-    - unparam # Reports unused function parameters
+    # - nakedret # Finds naked returns in functions greater than a specified function length
+    # - prealloc # Finds slice declarations that could potentially be preallocated
+    # - revive # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes
+    # - unparam # Reports unused function parameters
 
 # Don't enable fieldalignment, changing the field alignment requires checking to see if anyone uses constructors
 # without names. If there is a memory issue on a specific field, that is best found with a heap profile.
@@ -76,7 +75,9 @@ linters:
 # Disable goconst in test files, often we have duplicated strings across tests, but don't make sense as constants.
 issues:
   exclude-rules:
-    - path: (_test\.go|utilities_testing\.go)
+    # cover _testing.go (utility testing files) and _test.go files
+    # base/util_testing.go / rest/utilities_testing\.*.go
+    - path: (_test.*\.go)
       linters:
         - goconst
     - path: rest/debug.go
@@ -104,4 +105,5 @@ linters-settings:
       - wrapperFunc
     settings:
       ruleguard:
-        rules: '${configDir}/ruleguard/*.go'
+        rules: '${configDir}/ruleguard/rules-*.go'
+        failOn: all

.golangci.yml

@@ -33,28 +33,27 @@ linters:
     - unconvert # Remove unnecessary type conversions
     - unparam # Reports unused function parameters
     - unused # (megacheck) Checks Go code for unused constants, variables, functions and types
-  disable:
-    - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
-    - depguard # Go linter that checks if package imports are in a list of acceptable packages
-    - dogsled # Checks assignments with too many blank identifiers # (e.g. x, _, _, _, := f())
-    - err113 # Golang linter to check the errors handling expressions
-    - funlen # Tool for detection of long functions
-    - gochecknoglobals # Checks that no globals are present in Go code
-    - gochecknoinits # Checks that no init functions are present in Go code
-    - gocognit # Computes and checks the cognitive complexity of functions
-    - gocyclo # Computes and checks the cyclomatic complexity of functions
-    - godot # Check if comments end in a period
-    - godox # Tool for detection of FIXME, TODO and other comment keywords
-    - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
-    - gomodguard # Allow and block list linter for direct Go module dependencies.
-    - lll # Reports long lines
-    - nestif # Reports deeply nested if statements
-    - nolintlint # Reports ill-formed or insufficient nolint directives
-    - rowserrcheck # checks whether Err of rows is checked successfully
-    - stylecheck # Stylecheck is a replacement for golint
-    - testpackage # linter that makes you use a separate _test package
-    - whitespace # Tool for detection of leading and trailing whitespace
-    - wsl # Whitespace Linter - Forces you to use empty lines!
+    # disable:
+    # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
+    # - depguard # Go linter that checks if package imports are in a list of acceptable packages
+    # - dogsled # Checks assignments with too many blank identifiers # (e.g. x, _, _, _, := f())
+    # - err113 # Golang linter to check the errors handling expressions
+    # - funlen # Tool for detection of long functions
+    # - gochecknoglobals # Checks that no globals are present in Go code
+    # - gochecknoinits # Checks that no init functions are present in Go code
+    # - gocognit # Computes and checks the cognitive complexity of functions
+    # - gocyclo # Computes and checks the cyclomatic complexity of functions
+    # - godot # Check if comments end in a period
+    # - godox # Tool for detection of FIXME, TODO and other comment keywords
+    # - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
+    # - gomodguard # Allow and block list linter for direct Go module dependencies.
+    # - lll # Reports long lines
+    # - nestif # Reports deeply nested if statements
+    # - rowserrcheck # checks whether Err of rows is checked successfully
+    # - stylecheck # Stylecheck is a replacement for golint
+    # - testpackage # linter that makes you use a separate _test package
+    # - whitespace # Tool for detection of leading and trailing whitespace
+    # - wsl # Whitespace Linter - Forces you to use empty lines!
 
 linters-settings:
   govet:
@@ -65,16 +64,19 @@ linters-settings:
       - ruleguard
     settings:
       ruleguard:
-        rules: '${configDir}/ruleguard/*.go'
+        rules: '${configDir}/ruleguard/rules-*.go'
+        failOn: all
 
 # Disable goconst in test files, often we have duplicated strings across tests, but don't make sense as constants.
 issues:
   exclude-rules:
-    - path: (_test\.go|utilities_testing\.go)
+    # cover _testing.go (utility testing files) and _test.go files
+    # base/util_testing.go / rest/utilities_testing\.*.go
+    - path: (_test.*\.go)
       linters:
         - goconst
         - prealloc
-    - path: (_test\.go|utilities_testing\.go)
+    - path: (_test.*\.go)
       linters:
         - govet
       text: fieldalignment

auth/auth.go

@@ -631,6 +631,8 @@ func (auth *Authenticator) UpdateUserEmail(u User, email string) error {
 		if err != nil {
 			return nil, err
 		}
+		currentUser.SetUpdatedAt()
+
 		return currentUser, nil
 	}
 
@@ -662,6 +664,7 @@ func (auth *Authenticator) rehashPassword(user User, password string) error {
 			if err != nil {
 				return nil, err
 			}
+			currentUserImpl.SetUpdatedAt()
 			return currentUserImpl, nil
 		} else {
 			return nil, base.ErrUpdateCancel
@@ -740,6 +743,7 @@ func (auth *Authenticator) DeleteRole(role Role, purge bool, deleteSeq uint64) e
 		}
 		p.setDeleted(true)
 		p.SetSequence(deleteSeq)
+		p.SetUpdatedAt()
 
 		// Update channel history for default collection
 		channelHistory := auth.calculateHistory(p.Name(), deleteSeq, p.Channels(), nil, p.ChannelHistory())
@@ -955,6 +959,8 @@ func (auth *Authenticator) RegisterNewUser(username, email string) (User, error)
 			base.WarnfCtx(auth.LogCtx, "Skipping SetEmail for user %q - Invalid email address provided: %q", base.UD(username), base.UD(email))
 		}
 	}
+	user.SetUpdatedAt()
+	user.SetCreatedAt(time.Now().UTC())
 
 	err = auth.Save(user)
 	if base.IsCasMismatch(err) {

auth/principal.go

@@ -52,6 +52,12 @@ type Principal interface {
 	setDeleted(bool)
 	IsDeleted() bool
 
+	// Sets the updated time for the principal document
+	SetUpdatedAt()
+
+	// Sets the created time for the principal document
+	SetCreatedAt(t time.Time)
+
 	// Principal includes the PrincipalCollectionAccess interface for operations against
 	// the _default._default collection (stored directly on the principal for backward
 	// compatibility)

auth/role.go

@@ -33,6 +33,8 @@ type roleImpl struct {
 	ChannelInvalSeq   uint64                                  `json:"channel_inval_seq,omitempty"` // Sequence at which the channels were invalidated. Data remains in Channels_ for history calculation.
 	Deleted           bool                                    `json:"deleted,omitempty"`
 	CollectionsAccess map[string]map[string]*CollectionAccess `json:"collection_access,omitempty"` // Nested maps of CollectionAccess, indexed by scope and collection name
+	UpdatedAt         time.Time                               `json:"updated_at"`
+	CreatedAt         time.Time                               `json:"created_at"`
 	cas               uint64
 	docID             string // key used to store the roleImpl
 }
@@ -277,6 +279,14 @@ func (role *roleImpl) Name() string {
 	return role.Name_
 }
 
+func (role *roleImpl) SetUpdatedAt() {
+	role.UpdatedAt = time.Now().UTC()
+}
+
+func (role *roleImpl) SetCreatedAt(t time.Time) {
+	role.CreatedAt = t
+}
+
 func (role *roleImpl) Sequence() uint64 {
 	return role.Sequence_
 }

base/collection_xattr_test.go

@@ -1324,11 +1324,8 @@ func TestWriteWithXattrsInsertAndDeleteError(t *testing.T) {
 func requireXattrsEqual(t testing.TB, expected map[string][]byte, actual map[string][]byte) {
 	require.Len(t, actual, len(expected), "Expected xattrs to be the same length %v, got %v", expected, actual)
 	for k, v := range expected {
-		actualV, ok := actual[k]
-		if !ok {
-			require.Fail(t, "Missing expected xattr %s", k)
-		}
-		require.JSONEq(t, string(v), string(actualV))
+		require.Contains(t, actual, k)
+		require.JSONEq(t, string(v), string(actual[k]))
 	}
 }
 

base/dcp_test.go

@@ -466,7 +466,7 @@ func TestConcurrentCBGTIndexCreation(t *testing.T) {
 			case <-terminatorChan:
 				context.Manager.Stop()
 			case <-time.After(20 * time.Second):
-				assert.Fail(t, "manager goroutine not terminated: %v", managerUUID)
+				require.Fail(t, fmt.Sprintf("manager goroutine not terminated: %v", managerUUID))
 			}
 
 		}(i, terminator)

base/leaky_bucket.go

@@ -125,7 +125,12 @@ type LeakyBucketConfig struct {
 
 	// Returns a partial error the first time ViewCustom is called
 	FirstTimeViewCustomPartialError bool
-	PostQueryCallback               func(ddoc, viewName string, params map[string]interface{}) // Issues callback after issuing query when bucket.ViewQuery is called
+
+	// QueryCallback allows tests to set a callback that will be issued prior to issuing a view query
+	QueryCallback     func(ddoc, viewName string, params map[string]any) error
+	PostQueryCallback func(ddoc, viewName string, params map[string]interface{}) // Issues callback after issuing query when bucket.ViewQuery is called
+
+	N1QLQueryCallback func(ctx context.Context, statement string, params map[string]any, consistency ConsistencyMode, adhoc bool) error
 
 	PostN1QLQueryCallback func()
 

base/leaky_datastore.go

@@ -241,6 +241,12 @@ func (lds *LeakyDataStore) ViewQuery(ctx context.Context, ddoc, name string, par
 	if !ok {
 		return nil, errors.New("bucket does not support views")
 	}
+	if lds.config.QueryCallback != nil {
+		err := lds.config.QueryCallback(ddoc, name, params)
+		if err != nil {
+			return nil, err
+		}
+	}
 	iterator, err := vs.ViewQuery(ctx, ddoc, name, params)
 
 	if lds.config.FirstTimeViewCustomPartialError {
@@ -324,10 +330,14 @@ func (lds *LeakyDataStore) SetFirstTimeViewCustomPartialError(val bool) {
 	lds.config.FirstTimeViewCustomPartialError = val
 }
 
-func (lds *LeakyDataStore) SetPostQueryCallback(callback func(ddoc, viewName string, params map[string]interface{})) {
+func (lds *LeakyDataStore) SetPostQueryCallback(callback func(ddoc, viewName string, params map[string]any)) {
 	lds.config.PostQueryCallback = callback
 }
 
+func (lds *LeakyDataStore) SetQueryCallback(fn func(ddoc, viewName string, params map[string]any) error) {
+	lds.config.QueryCallback = fn
+}
+
 func (lds *LeakyDataStore) SetPostN1QLQueryCallback(callback func()) {
 	lds.config.PostN1QLQueryCallback = callback
 }
@@ -447,6 +457,12 @@ func (lds *LeakyDataStore) Query(ctx context.Context, statement string, params m
 	if err != nil {
 		return nil, err
 	}
+	if lds.config.N1QLQueryCallback != nil {
+		err := lds.config.N1QLQueryCallback(ctx, statement, params, consistency, adhoc)
+		if err != nil {
+			return nil, err
+		}
+	}
 	iterator, err := n1qlStore.Query(ctx, statement, params, consistency, adhoc)
 
 	if lds.config.PostN1QLQueryCallback != nil {

base/main_test_bucket_pool.go

@@ -121,6 +121,7 @@ func NewTestBucketPoolWithOptions(ctx context.Context, bucketReadierFunc TBPBuck
 		unclosedBuckets:        make(map[string]map[string]struct{}),
 		useExistingBucket:      TestUseExistingBucket(),
 		useDefaultScope:        options.UseDefaultScope,
+		skipMobileXDCR:         true, // do not set up enableCrossClusterVersioning until Sync Gateway 4.x
 	}
 
 	tbp.cluster = newTestCluster(ctx, UnitTestUrl(), &tbp)

base/main_test_cluster.go

@@ -66,11 +66,11 @@ func newTestCluster(ctx context.Context, server string, tbp *TestBucketPool) *tb
 // getGocbClusterForTest makes cluster connection. Callers must close. Returns the cluster and the connection string used to connect.
 func getGocbClusterForTest(ctx context.Context, server string) (*gocb.Cluster, string) {
 
-	testClusterTimeout := 10 * time.Second
 	spec := BucketSpec{
-		Server:          server,
-		TLSSkipVerify:   true,
-		BucketOpTimeout: &testClusterTimeout,
+		Server:        server,
+		TLSSkipVerify: true,
+		// use longer timeout than DefaultBucketOpTimeout to avoid timeouts in test harness from using buckets after flush, which takes some time to reinitialize
+		BucketOpTimeout: Ptr(time.Duration(30) * time.Second),
 	}
 	connStr, err := spec.GetGoCBConnString()
 	if err != nil {

base/stats.go

@@ -87,6 +87,8 @@ const (
 	StatAddedVersion3dot1dot4     = "3.1.4"
 	StatAddedVersion3dot2dot0     = "3.2.0"
 	StatAddedVersion3dot2dot1     = "3.2.1"
+	StatAddedVersion3dot2dot2     = "3.2.2"
+	StatAddedVersion3dot2dot3     = "3.2.3"
 	StatAddedVersion3dot3dot0     = "3.3.0"
 	StatAddedVersion4dot0dot0     = "4.0.0"
 
@@ -313,6 +315,10 @@ func (g *GlobalStat) initResourceUtilizationStats() error {
 	if err != nil {
 		return err
 	}
+	resUtil.NumIdleQueryOps, err = NewIntStat(SubsystemDatabaseKey, "num_idle_query_ops", StatUnitNoUnits, NumIdleQueryOpsDesc, StatAddedVersion3dot2dot2, StatDeprecatedVersionNotDeprecated, StatStabilityCommitted, nil, nil, prometheus.CounterValue, 0)
+	if err != nil {
+		return err
+	}
 
 	resUtil.Uptime, err = NewDurStat(ResourceUtilizationSubsystem, "uptime", StatUnitNanoseconds, UptimeDesc, StatAddedVersion3dot0dot0, StatDeprecatedVersionNotDeprecated, StatStabilityCommitted, nil, nil, prometheus.CounterValue, time.Now())
 	if err != nil {
@@ -367,8 +373,9 @@ type ResourceUtilization struct {
 	// The node CPU usage calculation based values from /proc of user + system since the last time this function was called.
 	NodeCpuPercentUtil *SgwFloatStat `json:"node_cpu_percent_utilization"`
 
-	// The number of background kv operations.
-	NumIdleKvOps *SgwIntStat `json:"idle_kv_ops"`
+	// The number of background kv/query operations.
+	NumIdleKvOps    *SgwIntStat `json:"idle_kv_ops"`
+	NumIdleQueryOps *SgwIntStat `json:"idle_query_ops"`
 
 	// The memory utilization (Resident Set Size) for the process, in bytes.
 	ProcessMemoryResident *SgwIntStat `json:"process_memory_resident"`

base/stats_descriptions.go

@@ -314,7 +314,8 @@ const (
 
 	SyncProcessComputeDesc = "The compute unit for syncing with clients measured through cpu time and memory used for sync"
 
-	NumIdleKvOpsDesc = "The total number of idle kv operations."
+	NumIdleKvOpsDesc    = "The total number of idle kv operations."
+	NumIdleQueryOpsDesc = "The total number of idle query operations."
 )
 
 // Delta Sync stats descriptions

db/active_replicator.go

@@ -214,8 +214,10 @@ func connect(arc *activeReplicatorCommon, idSuffix string) (blipSender *blip.Sen
 	// - make this configurable for testing mixed-version replications
 	// - if unspecified, default to v2 and v3 until VV is supported with ISGR, then also include v4
 	protocols := []string{CBMobileReplicationV3.SubprotocolString(), CBMobileReplicationV2.SubprotocolString()}
-	blipContext, err := NewSGBlipContextWithProtocols(arc.ctx, arc.config.ID+idSuffix, originPatterns, protocols, nil)
+	cancelCtx, cancelFunc := context.WithCancel(context.Background())
+	blipContext, err := NewSGBlipContextWithProtocols(arc.ctx, arc.config.ID+idSuffix, originPatterns, protocols, cancelCtx)
 	if err != nil {
+		cancelFunc()
 		return nil, nil, err
 	}
 	blipContext.WebsocketPingInterval = arc.config.WebsocketPingInterval
@@ -226,7 +228,10 @@ func connect(arc *activeReplicatorCommon, idSuffix string) (blipSender *blip.Sen
 		}
 	}
 
-	bsc = NewBlipSyncContext(arc.ctx, blipContext, arc.config.ActiveDB, blipContext.ID, arc.replicationStats)
+	bsc, err = NewBlipSyncContext(arc.ctx, blipContext, arc.config.ActiveDB, blipContext.ID, arc.replicationStats, cancelFunc)
+	if err != nil {
+		return nil, nil, err
+	}
 
 	bsc.loggingCtx = base.CorrelationIDLogCtx(
 		arc.config.ActiveDB.AddDatabaseLogContext(base.NewNonCancelCtx().Ctx),