*: add mark function for log desensitization (#8306)

ref #8305

Provide sensitive log redact marker `‹..›` to customize log desensitization.

Signed-off-by: JmPotato <[email protected]>

conf/config.toml

@@ -33,7 +33,10 @@
 # key-path = ""
 ## A CN which must be provided by a client
 # cert-allowed-cn = ["example.com"]
-## Whether or not to enable redact log.
+## Whether to enable the log redaction. It can be the following values:
+##   - false: disable redact log.
+##   - true: enable redact log, which will replace the sensitive information with "?".
+##   - "MARKER": enable redact log, which will use single guillemets ‹› to enclose the sensitive information.
 # redact-info-log = false
 
 [security.encryption]

pkg/core/region.go

@@ -2049,14 +2049,6 @@ func DiffRegionKeyInfo(origin *RegionInfo, other *RegionInfo) string {
 	return strings.Join(ret, ", ")
 }
 
-// String converts slice of bytes to string without copy.
-func String(b []byte) string {
-	if len(b) == 0 {
-		return ""
-	}
-	return unsafe.String(unsafe.SliceData(b), len(b))
-}
-
 // ToUpperASCIIInplace bytes.ToUpper but zero-cost
 func ToUpperASCIIInplace(s []byte) []byte {
 	hasLower := false
@@ -2095,7 +2087,7 @@ func HexRegionKey(key []byte) []byte {
 // HexRegionKeyStr converts region key to hex format. Used for formatting region in
 // logs.
 func HexRegionKeyStr(key []byte) string {
-	return String(HexRegionKey(key))
+	return typeutil.BytesToString(HexRegionKey(key))
 }
 
 // RegionToHexMeta converts a region meta's keys to hex format. Used for formatting

pkg/storage/hot_region_storage.go

@@ -37,6 +37,7 @@ import (
 	"github.com/tikv/pd/pkg/storage/kv"
 	"github.com/tikv/pd/pkg/utils/logutil"
 	"github.com/tikv/pd/pkg/utils/syncutil"
+	"github.com/tikv/pd/pkg/utils/typeutil"
 	"go.uber.org/zap"
 )
 
@@ -266,8 +267,8 @@ func (h *HotRegionStorage) packHistoryHotRegions(historyHotRegions []HistoryHotR
 		if err != nil {
 			return err
 		}
-		historyHotRegions[i].StartKey = core.String(region.StartKey)
-		historyHotRegions[i].EndKey = core.String(region.EndKey)
+		historyHotRegions[i].StartKey = typeutil.BytesToString(region.StartKey)
+		historyHotRegions[i].EndKey = typeutil.BytesToString(region.EndKey)
 		key := HotRegionStorePath(hotRegionType, historyHotRegions[i].UpdateTime, historyHotRegions[i].RegionID)
 		h.batchHotInfo[key] = &historyHotRegions[i]
 	}
@@ -385,8 +386,8 @@ func (it *HotRegionStorageIterator) Next() (*HistoryHotRegion, error) {
 	if err := encryption.DecryptRegion(region, it.encryptionKeyManager); err != nil {
 		return nil, err
 	}
-	message.StartKey = core.String(region.StartKey)
-	message.EndKey = core.String(region.EndKey)
+	message.StartKey = typeutil.BytesToString(region.StartKey)
+	message.EndKey = typeutil.BytesToString(region.EndKey)
 	message.EncryptionMeta = nil
 	return &message, nil
 }

pkg/utils/configutil/configutil.go

@@ -25,6 +25,7 @@ import (
 	"github.com/spf13/pflag"
 	"github.com/tikv/pd/pkg/encryption"
 	"github.com/tikv/pd/pkg/utils/grpcutil"
+	"github.com/tikv/pd/pkg/utils/logutil"
 	"github.com/tikv/pd/pkg/utils/typeutil"
 )
 
@@ -78,9 +79,12 @@ func (m *ConfigMetaData) CheckUndecoded() error {
 // SecurityConfig indicates the security configuration
 type SecurityConfig struct {
 	grpcutil.TLSConfig
-	// RedactInfoLog indicates that whether enabling redact log
-	RedactInfoLog bool              `toml:"redact-info-log" json:"redact-info-log"`
-	Encryption    encryption.Config `toml:"encryption" json:"encryption"`
+	// RedactInfoLog indicates that whether to enable the log redaction. It can be the following values:
+	//   - false: disable redact log.
+	//   - true: enable redact log, which will replace the sensitive information with "?".
+	//   - "MARKER": enable redact log, which will use single guillemets ‹› to enclose the sensitive information.
+	RedactInfoLog logutil.RedactInfoLogType `toml:"redact-info-log" json:"redact-info-log"`
+	Encryption    encryption.Config         `toml:"encryption" json:"encryption"`
 }
 
 // PrintConfigCheckMsg prints the message about configuration checks.

pkg/utils/logutil/log.go

@@ -15,12 +15,15 @@
 package logutil
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 	"sync/atomic"
 
 	"github.com/pingcap/log"
 	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/utils/typeutil"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
@@ -67,16 +70,19 @@ func StringToZapLogLevel(level string) zapcore.Level {
 }
 
 // SetupLogger setup the logger.
-func SetupLogger(logConfig log.Config, logger **zap.Logger, logProps **log.ZapProperties, enabled ...bool) error {
+func SetupLogger(
+	logConfig log.Config,
+	logger **zap.Logger,
+	logProps **log.ZapProperties,
+	redactInfoLogType RedactInfoLogType,
+) error {
 	lg, p, err := log.InitLogger(&logConfig, zap.AddStacktrace(zapcore.FatalLevel))
 	if err != nil {
 		return errs.ErrInitLogger.Wrap(err)
 	}
 	*logger = lg
 	*logProps = p
-	if len(enabled) > 0 {
-		SetRedactLog(enabled[0])
-	}
+	setRedactType(redactInfoLogType)
 	return nil
 }
 
@@ -88,22 +94,111 @@ func LogPanic() {
 	}
 }
 
+// RedactInfoLogType is the behavior of redacting sensitive information in logs.
+type RedactInfoLogType int
+
+const (
+	// RedactInfoLogOFF means log redaction is disabled.
+	RedactInfoLogOFF RedactInfoLogType = iota
+	// RedactInfoLogON means log redaction is enabled, and will replace the sensitive information with "?".
+	RedactInfoLogON
+	// RedactInfoLogMarker means log redaction is enabled, and will use single guillemets ‹› to enclose the sensitive information.
+	RedactInfoLogMarker
+)
+
+// MarshalJSON implements the `json.Marshaler` interface to ensure the compatibility.
+func (t RedactInfoLogType) MarshalJSON() ([]byte, error) {
+	switch t {
+	case RedactInfoLogON:
+		return json.Marshal(true)
+	case RedactInfoLogMarker:
+		return json.Marshal("MARKER")
+	default:
+	}
+	return json.Marshal(false)
+}
+
+const invalidRedactInfoLogTypeErrMsg = `the "redact-info-log" value is invalid; it should be either false, true, or "MARKER"`
+
+// UnmarshalJSON implements the `json.Marshaler` interface to ensure the compatibility.
+func (t *RedactInfoLogType) UnmarshalJSON(data []byte) error {
+	var s string
+	err := json.Unmarshal(data, &s)
+	if err == nil && strings.ToUpper(s) == "MARKER" {
+		*t = RedactInfoLogMarker
+		return nil
+	}
+	var b bool
+	err = json.Unmarshal(data, &b)
+	if err != nil {
+		return errors.New(invalidRedactInfoLogTypeErrMsg)
+	}
+	if b {
+		*t = RedactInfoLogON
+	} else {
+		*t = RedactInfoLogOFF
+	}
+	return nil
+}
+
+// UnmarshalTOML implements the `toml.Unmarshaler` interface to ensure the compatibility.
+func (t *RedactInfoLogType) UnmarshalTOML(data any) error {
+	switch v := data.(type) {
+	case bool:
+		if v {
+			*t = RedactInfoLogON
+		} else {
+			*t = RedactInfoLogOFF
+		}
+		return nil
+	case string:
+		if strings.ToUpper(v) == "MARKER" {
+			*t = RedactInfoLogMarker
+			return nil
+		}
+		return errors.New(invalidRedactInfoLogTypeErrMsg)
+	default:
+	}
+	return errors.New(invalidRedactInfoLogTypeErrMsg)
+}
+
 var (
-	enabledRedactLog atomic.Value
+	curRedactType atomic.Value
 )
 
 func init() {
-	SetRedactLog(false)
+	setRedactType(RedactInfoLogOFF)
+}
+
+func getRedactType() RedactInfoLogType {
+	return curRedactType.Load().(RedactInfoLogType)
 }
 
-// IsRedactLogEnabled indicates whether the log desensitization is enabled
-func IsRedactLogEnabled() bool {
-	return enabledRedactLog.Load().(bool)
+func setRedactType(redactInfoLogType RedactInfoLogType) {
+	curRedactType.Store(redactInfoLogType)
 }
 
-// SetRedactLog sets enabledRedactLog
-func SetRedactLog(enabled bool) {
-	enabledRedactLog.Store(enabled)
+const (
+	leftMark  = '‹'
+	rightMark = '›'
+)
+
+func redactInfo(input string) string {
+	res := &strings.Builder{}
+	res.Grow(len(input) + 2)
+	_, _ = res.WriteRune(leftMark)
+	for _, c := range input {
+		// Double the mark character if it is already in the input string.
+		// to avoid the ambiguity of the redacted content.
+		if c == leftMark || c == rightMark {
+			_, _ = res.WriteRune(c)
+			_, _ = res.WriteRune(c)
+		} else {
+			_, _ = res.WriteRune(c)
+		}
+	}
+	_, _ = res.WriteRune(rightMark)
+	return res.String()
 }
 
 // ZapRedactByteString receives []byte argument and return omitted information zap.Field if redact log enabled
@@ -123,34 +218,48 @@ func ZapRedactStringer(key string, arg fmt.Stringer) zap.Field {
 
 // RedactBytes receives []byte argument and return omitted information if redact log enabled
 func RedactBytes(arg []byte) []byte {
-	if IsRedactLogEnabled() {
+	switch getRedactType() {
+	case RedactInfoLogON:
 		return []byte("?")
+	case RedactInfoLogMarker:
+		// Use unsafe conversion to avoid copy.
+		return typeutil.StringToBytes(redactInfo(typeutil.BytesToString(arg)))
+	default:
 	}
 	return arg
 }
 
 // RedactString receives string argument and return omitted information if redact log enabled
 func RedactString(arg string) string {
-	if IsRedactLogEnabled() {
+	switch getRedactType() {
+	case RedactInfoLogON:
 		return "?"
+	case RedactInfoLogMarker:
+		return redactInfo(arg)
+	default:
 	}
 	return arg
 }
 
 // RedactStringer receives stringer argument and return omitted information if redact log enabled
 func RedactStringer(arg fmt.Stringer) fmt.Stringer {
-	if IsRedactLogEnabled() {
-		return stringer{}
+	switch getRedactType() {
+	case RedactInfoLogON:
+		return &redactedStringer{"?"}
+	case RedactInfoLogMarker:
+		return &redactedStringer{redactInfo(arg.String())}
+	default:
 	}
 	return arg
 }
 
-type stringer struct {
+type redactedStringer struct {
+	content string
 }
 
 // String implement fmt.Stringer
-func (stringer) String() string {
-	return "?"
+func (rs *redactedStringer) String() string {
+	return rs.content
 }
 
 // CondUint32 constructs a field with the given key and value conditionally.