diff --git a/cmd/metal-api/internal/grpc/grpc-server.go b/cmd/metal-api/internal/grpc/grpc-server.go index c7213a10..41abd017 100644 --- a/cmd/metal-api/internal/grpc/grpc-server.go +++ b/cmd/metal-api/internal/grpc/grpc-server.go @@ -52,7 +52,7 @@ type ServerConfig struct { ResponseInterval time.Duration CheckInterval time.Duration BMCSuperUserPasswordFile string - Auditing auditing.Auditing + Auditing []auditing.Auditing IPMISuperUser metal.MachineIPMISuperUser } @@ -121,7 +121,7 @@ func Run(cfg *ServerConfig) error { logging.UnaryServerInterceptor(interceptorLogger(log)), recovery.UnaryServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), } - if cfg.Auditing != nil { + if len(cfg.Auditing) > 0 { shouldAudit := func(fullMethod string) bool { switch fullMethod { case "/api.v1.BootService/Register": @@ -130,16 +130,19 @@ func Run(cfg *ServerConfig) error { return false } } - auditStreamInterceptor, err := auditing.StreamServerInterceptor(cfg.Auditing, log.WithGroup("auditing-grpc"), shouldAudit) - if err != nil { - return err - } - auditUnaryInterceptor, err := auditing.UnaryServerInterceptor(cfg.Auditing, log.WithGroup("auditing-grpc"), shouldAudit) - if err != nil { - return err + + for _, backend := range cfg.Auditing { + auditStreamInterceptor, err := auditing.StreamServerInterceptor(backend, log.WithGroup("auditing-grpc"), shouldAudit) + if err != nil { + return err + } + auditUnaryInterceptor, err := auditing.UnaryServerInterceptor(backend, log.WithGroup("auditing-grpc"), shouldAudit) + if err != nil { + return err + } + streamInterceptors = append(streamInterceptors, auditStreamInterceptor) + unaryInterceptors = append(unaryInterceptors, auditUnaryInterceptor) } - streamInterceptors = append(streamInterceptors, auditStreamInterceptor) - unaryInterceptors = append(unaryInterceptors, auditUnaryInterceptor) } unaryInterceptors = append(unaryInterceptors, metrics.GrpcMetrics, recovery.UnaryServerInterceptor(recoveryOpt)) diff --git a/cmd/metal-api/internal/service/audit-service.go b/cmd/metal-api/internal/service/audit-service.go index 87bc0212..daa7c3bc 100644 --- a/cmd/metal-api/internal/service/audit-service.go +++ b/cmd/metal-api/internal/service/audit-service.go @@ -64,7 +64,7 @@ func (r *auditResource) find(request *restful.Request, response *restful.Respons return } - backendResult, err := r.a.Search(auditing.EntryFilter{ + backendResult, err := r.a.Search(request.Request.Context(), auditing.EntryFilter{ Limit: requestPayload.Limit, From: requestPayload.From, To: requestPayload.To, diff --git a/cmd/metal-api/main.go b/cmd/metal-api/main.go index 31514a80..343e3a4e 100644 --- a/cmd/metal-api/main.go +++ b/cmd/metal-api/main.go @@ -287,11 +287,19 @@ func init() { rootCmd.Flags().StringP("masterdata-certkeypath", "", "", "the tls certificate key to talk to the masterdata-api") rootCmd.Flags().Bool("auditing-enabled", false, "enable auditing") - rootCmd.Flags().String("auditing-url", "http://localhost:7700", "url of the auditing service") - rootCmd.Flags().String("auditing-api-key", "secret", "api key for the auditing service") - rootCmd.Flags().String("auditing-index-prefix", "auditing", "auditing index prefix") - rootCmd.Flags().String("auditing-index-interval", "@daily", "auditing index creation interval, can be one of @hourly|@daily|@monthly") - rootCmd.Flags().Int64("auditing-keep", 14, "the amount of indexes to keep until cleanup") + + rootCmd.Flags().String("auditing-meili-url", "http://localhost:7700", "url of the auditing service") + rootCmd.Flags().String("auditing-meili-api-key", "secret", "api key for the auditing service") + rootCmd.Flags().String("auditing-meili-index-prefix", "auditing", "auditing index prefix") + rootCmd.Flags().String("auditing-meili-index-interval", "@daily", "auditing index creation interval, can be one of @hourly|@daily|@monthly") + rootCmd.Flags().Int64("auditing-meili-keep", 14, "the amount of indexes to keep until cleanup") + + rootCmd.Flags().String("auditing-timescaledb-host", "", "host of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-port", "", "port of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-db", "", "database name of the auditing service") + rootCmd.Flags().String("auditing-timescaledb-user", "", "user for the auditing service") + rootCmd.Flags().String("auditing-timescaledb-password", "", "password for the auditing service") + rootCmd.Flags().String("auditing-timescaledb-retention", "", "the time until audit traces are cleaned up") rootCmd.Flags().String("headscale-addr", "", "address of headscale server") rootCmd.Flags().String("headscale-cp-addr", "", "address of headscale control plane") @@ -691,7 +699,7 @@ func initAuth(lg *slog.Logger) security.UserGetter { return security.NewCreds(auths...) } -func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser metal.MachineIPMISuperUser) *restfulspec.Config { +func initRestServices(audit []auditing.Auditing, withauth bool, ipmiSuperUser metal.MachineIPMISuperUser) *restfulspec.Config { service.BasePath = viper.GetString("base-path") if !strings.HasPrefix(service.BasePath, "/") || !strings.HasSuffix(service.BasePath, "/") { log.Fatal("base path must start and end with a slash") @@ -757,7 +765,7 @@ func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser meta releaseVersion = pointer.Pointer(viper.GetString("release-version")) } - restful.DefaultContainer.Add(service.NewAudit(logger.WithGroup("audit-service"), audit)) + restful.DefaultContainer.Add(service.NewAudit(logger.WithGroup("audit-service"), pointer.FirstOrZero(audit))) restful.DefaultContainer.Add(service.NewPartition(logger.WithGroup("partition-service"), ds, nsqer)) restful.DefaultContainer.Add(service.NewImage(logger.WithGroup("image-service"), ds)) restful.DefaultContainer.Add(service.NewSize(logger.WithGroup("size-service"), ds, mdc)) @@ -790,8 +798,8 @@ func initRestServices(audit auditing.Auditing, withauth bool, ipmiSuperUser meta restful.DefaultContainer.Filter(ensurer.EnsureAllowedTenantFilter) } - if audit != nil { - httpFilter, err := auditing.HttpFilter(audit, logger.WithGroup("audit-middleware")) + for _, backend := range audit { + httpFilter, err := auditing.HttpFilter(backend, logger.WithGroup("audit-middleware")) if err != nil { log.Fatalf("unable to create http filter for auditing: %s", err) } @@ -908,7 +916,7 @@ func evaluateVPNConnected() error { } // might return (nil, nil) if auditing is disabled! -func createAuditingClient(log *slog.Logger) (auditing.Auditing, error) { +func createAuditingClient(log *slog.Logger) ([]auditing.Auditing, error) { isEnabled := viper.GetBool("auditing-enabled") if !isEnabled { log.Warn("auditing is disabled, can be enabled by setting --auditing-enabled=true") @@ -916,15 +924,46 @@ func createAuditingClient(log *slog.Logger) (auditing.Auditing, error) { } c := auditing.Config{ - Component: "metal-api", - URL: viper.GetString("auditing-url"), - APIKey: viper.GetString("auditing-api-key"), - IndexPrefix: viper.GetString("auditing-index-prefix"), - RotationInterval: auditing.Interval(viper.GetString("auditing-index-interval")), - Keep: viper.GetInt64("auditing-keep"), - Log: log, //FIXME - } - return auditing.New(c) + Component: "metal-api", + Log: log, + } + + var backends []auditing.Auditing + + if viper.IsSet("auditing-timescaledb-host") { + backend, err := auditing.NewTimescaleDB(c, auditing.TimescaleDbConfig{ + Host: viper.GetString("auditing-timescaledb-host"), + Port: viper.GetString("auditing-timescaledb-port"), + DB: viper.GetString("auditing-timescaledb-db"), + User: viper.GetString("auditing-timescaledb-user"), + Password: viper.GetString("auditing-timescaledb-password"), + Retention: viper.GetString("auditing-timescaledb-retention"), + }) + + if err != nil { + return nil, err + } + + backends = append(backends, backend) + } + + if viper.IsSet("auditing-meili-api-key") { + backend, err := auditing.NewMeilisearch(c, auditing.MeilisearchConfig{ + URL: viper.GetString("auditing-meili-url"), + APIKey: viper.GetString("auditing-meili-api-key"), + IndexPrefix: viper.GetString("auditing-meili-index-prefix"), + RotationInterval: auditing.Interval(viper.GetString("auditing-meili-index-interval")), + Keep: viper.GetInt64("auditing-meili-keep"), + }) + + if err != nil { + return nil, err + } + + backends = append(backends, backend) + } + + return backends, nil } func run() error { diff --git a/go.mod b/go.mod index 3ffbd58f..72291eab 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,8 @@ require ( github.com/looplab/fsm v1.0.2 github.com/metal-stack/go-ipam v1.14.8 github.com/metal-stack/masterdata-api v0.11.5 - github.com/metal-stack/metal-lib v0.19.0 - github.com/metal-stack/security v0.9.1 + github.com/metal-stack/metal-lib v0.19.3-0.20250206084302-3347c1f4cf98 + github.com/metal-stack/security v0.9.2 github.com/metal-stack/v v1.0.3 github.com/nsqio/go-nsq v1.1.0 github.com/prometheus/client_golang v1.20.5 @@ -110,6 +110,7 @@ require ( github.com/lestrrat-go/jwx/v2 v2.1.3 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/lib/pq v1.10.9 // indirect + github.com/lopezator/migrator v0.3.1 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/magiconair/properties v1.8.9 // indirect github.com/mailru/easyjson v0.9.0 // indirect diff --git a/go.sum b/go.sum index 755d726d..b14aa3be 100644 --- a/go.sum +++ b/go.sum @@ -174,10 +174,22 @@ github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9Ksj github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= @@ -233,6 +245,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/looplab/fsm v1.0.2 h1:f0kdMzr4CRpXtaKKRUxwLYJ7PirTdwrtNumeLN+mDx8= github.com/looplab/fsm v1.0.2/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4= +github.com/lopezator/migrator v0.3.1 h1:ZFPT6aC7+nGWkqhleynABZ6ftycSf6hmHHLOaryq1Og= +github.com/lopezator/migrator v0.3.1/go.mod h1:X+lHDMZ9Ci3/KdbypJcQYFFwipVrJsX4fRCQ4QLauYk= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= @@ -257,10 +271,10 @@ github.com/metal-stack/go-ipam v1.14.8 h1:M840hp1RcM2fHfMjFG5rn02yKFq6hXh+WvmYIS github.com/metal-stack/go-ipam v1.14.8/go.mod h1:LzAKT0X9dGAMs1uoLyJfJZkhwMMDLaSYvx9n6MJ9FI4= github.com/metal-stack/masterdata-api v0.11.5 h1:r7bYdhdVgOjCk6k7K/SCLlHALH23ZuMGY8E4udk4wXQ= github.com/metal-stack/masterdata-api v0.11.5/go.mod h1:Xk8kqxAp3NkAc2BX8yTIWgSlD77T897QSdRSluWvP4w= -github.com/metal-stack/metal-lib v0.19.0 h1:4yBnp/jPGgX9KeCje3A4MFL2oDjgjOjgsIK391LltRI= -github.com/metal-stack/metal-lib v0.19.0/go.mod h1:fCMaWwVGA/xAoGvBk72/nfzqBkHly0iOzrWpc55Fau4= -github.com/metal-stack/security v0.9.1 h1:cx3afSJPSOh03E9gKjdG6mbNU+ox/dqV7q8T9MkrHxo= -github.com/metal-stack/security v0.9.1/go.mod h1:ENm5kPjqh4uYvn79sAIxd6GZBwtF2GSsGdkLELrB/D4= +github.com/metal-stack/metal-lib v0.19.3-0.20250206084302-3347c1f4cf98 h1:ggLbpnpA92UAk8/m5hLq0r4vszLJ5RIkc4VSEjsewqs= +github.com/metal-stack/metal-lib v0.19.3-0.20250206084302-3347c1f4cf98/go.mod h1:t4X78F8Vb9zvXTspeNQ6ERGAeuD73u/s867Y0ygGF74= +github.com/metal-stack/security v0.9.2 h1:WH5LYwKccoEgS532f5qwonS7zX4/PJJO1RQ6R6Md410= +github.com/metal-stack/security v0.9.2/go.mod h1:ENm5kPjqh4uYvn79sAIxd6GZBwtF2GSsGdkLELrB/D4= github.com/metal-stack/v v1.0.3 h1:Sh2oBlnxrCUD+mVpzfC8HiqL045YWkxs0gpTvkjppqs= github.com/metal-stack/v v1.0.3/go.mod h1:YTahEu7/ishwpYKnp/VaW/7nf8+PInogkfGwLcGPdXg= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=