From 98fb552cbd526bc4203a94b74f7bcdb7d1733508 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Mon, 14 Oct 2024 12:25:37 +0200 Subject: [PATCH 01/11] Initial support for devcontainers --- .devcontainer/Dockerfile | 4 +++ .devcontainer/devcontainer.json | 11 +++++++ .devcontainer/docker-compose.yml | 43 ++++++++++++++++++++++++++++ .env-devcontainer | 15 ++++++++++ .gitignore | 4 ++- .vscode/launch.json | 49 ++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml create mode 100644 .env-devcontainer create mode 100644 .vscode/launch.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..4f698fd7 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,4 @@ +FROM mcr.microsoft.com/devcontainers/go + +WORKDIR /go/src/github.com/cloudoperators/heureka +ADD . . diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..ed41eda9 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,11 @@ +{ + "name": "heureka & MariaDB", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "postCreateCommand": "make", + "forwardPorts": [ + 80, + "mariadb:3306" + ] +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..e8ca21f8 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,43 @@ +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + + volumes: + - ../..:/workspaces:cached + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:mariadb + + environment: + # MariaDB Connection + DB_ADDRESS: mariadb + DB_PORT: 3306 + DB_USER: mariadb + DB_PASSWORD: mariadb + DB_ROOT_PASSWORD: mariadb + DB_NAME: heureka + DB_SCHEMA: /app_sqlschema/schema.sql + + + SEED_MODE: true + AUTH_TYPE: token + AUTH_TOKEN_SECRET: xxx + + mariadb: + image: mariadb:latest + container_name: mariadb + restart: unless-stopped + + environment: + MARIADB_USER: mariadb + MARIADB_PASSWORD: mariadb + MARIADB_DATABASE: heureka + MARIADB_ROOT_PASSWORD: mariadb + + volumes: + - ../.mariadb-dev/:/var/lib/mysql diff --git a/.env-devcontainer b/.env-devcontainer new file mode 100644 index 00000000..f52e5102 --- /dev/null +++ b/.env-devcontainer @@ -0,0 +1,15 @@ +DB_USER=mariadb +DB_PASSWORD=mariadb +DB_ROOT_PASSWORD=mariadb +DB_NAME=heureka +DB_ADDRESS=mariadb +DB_PORT=3306 +DB_SCHEMA=internal/database/mariadb/init/schema.sql + +DB_CONTAINER_IMAGE=mariadb:latest + +LOG_PRETTY_PRINT=true + +LOCAL_TEST_DB=true + +SEED_MODE=true diff --git a/.gitignore b/.gitignore index f5070af1..a1a4fa73 100644 --- a/.gitignore +++ b/.gitignore @@ -21,8 +21,10 @@ # vendor/ .vscode .DS_Store +cmd/heureka/__debug_bin* # Heureka specifics build/ .db/ -.env \ No newline at end of file +.env +.mariadb-dev/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..46949909 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,49 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch K8sScanner", + "type": "go", + "request": "launch", + "mode": "auto", + "cwd": "${workspaceFolder}", + "program": "scanner/k8s-assets/main.go", + "envFile": "${workspaceFolder}/scanner/k8s-assets/.env" + }, + { + "name": "Launch NvdScanner", + "type": "go", + "request": "launch", + "mode": "auto", + "cwd": "${workspaceFolder}", + "program": "scanner/nvd/main.go", + "envFile": "${workspaceFolder}/scanner/nvd/.env" + }, + { + "name": "Launch KeppelScanner", + "type": "go", + "request": "launch", + "mode": "auto", + "cwd": "${workspaceFolder}", + "program": "scanner/keppel/main.go", + "envFile": "${workspaceFolder}/scanner/keppel/.env" + }, + { + "name": "Launch Heureka DevContainer", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "cmd/heureka/main.go", + "args": [ + "-mode", + "seed" + ], + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env-devcontainer", + "showLog": true + } + ] +} \ No newline at end of file From e47184cd87e46f2f0da142176c34ebe746999550 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Mon, 14 Oct 2024 14:28:19 +0200 Subject: [PATCH 02/11] Mound additional volume for heureka devcontainer --- .devcontainer/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index e8ca21f8..ab552f7f 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -6,6 +6,7 @@ services: volumes: - ../..:/workspaces:cached + - ../internal/database/mariadb/init/schema.sql:/app_sqlschema/schema.sql # Overrides default command so things don't shut down after the process ends. command: sleep infinity From dc0188e77d855640336716ddb2872e58edf04937 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Mon, 14 Oct 2024 16:01:13 +0200 Subject: [PATCH 03/11] Add licence headers --- .devcontainer/Dockerfile | 3 +++ .devcontainer/docker-compose.yml | 3 +++ .env-devcontainer | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4f698fd7..203f3488 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + FROM mcr.microsoft.com/devcontainers/go WORKDIR /go/src/github.com/cloudoperators/heureka diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index ab552f7f..fc2dcf8e 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + services: app: build: diff --git a/.env-devcontainer b/.env-devcontainer index f52e5102..efab3f87 100644 --- a/.env-devcontainer +++ b/.env-devcontainer @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + DB_USER=mariadb DB_PASSWORD=mariadb DB_ROOT_PASSWORD=mariadb From ff68139018f9abd7bb07fa23def91bd619e6b723 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Mon, 14 Oct 2024 16:50:01 +0200 Subject: [PATCH 04/11] Update README.md with devcontainer setup --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 08922784..c57db86d 100644 --- a/README.md +++ b/README.md @@ -87,12 +87,37 @@ docker-compose --profile db --profile heureka up ### Makefile -Alternatively, the application can be started by using the provided Makefile: +The application can be started by using the provided Makefile: ``` make start-all-heureka ``` +### Devcontainers + +Devcontainers is a new standard for development environments based on (docker) +[devcontainer](./.devcontainer). + +At the moment devcontainers are supported by Visual Studio Code and IDEA IDEs. + +For Microsoft Visual Studio code, install the remote container extension via +Ctrl-P and this command: + + ext install ms-vscode-remote.remote-containers + +When opening the root folder in Visual Studio code a prompt will ask you to +open the project in a dev container, which you should. + +Once inside the devcontainer the provided launch.json is configured to allow +launching heureka and running the unit and integration tests. + +At the moment there is a known issue with the permissions of the .mariadb-dev +folder. This folder has to be deleted every time after using the devcontainers. +Use the following command in the root folder of heureka: + + sudo rm -rf .mariadb-dev + + ### Tests #### Mockery From 5d81a4a28e6dd52f85143aa7fcc47564490a8a04 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Thu, 17 Oct 2024 16:20:45 +0200 Subject: [PATCH 05/11] Add environment test variables for devcontainer --- .testenv-devcontainer | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .testenv-devcontainer diff --git a/.testenv-devcontainer b/.testenv-devcontainer new file mode 100644 index 00000000..52fef971 --- /dev/null +++ b/.testenv-devcontainer @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + +LOCAL_TEST_DB=true + +DB_USER=mariadb +DB_PASSWORD=mariadb +DB_ROOT_PASSWORD=mariadb +DB_NAME=heureka +DB_ADDRESS=mariadb +DB_PORT=3306 +DB_SCHEMA=internal/database/mariadb/init/schema.sql + +LOG_PRETTY_PRINT=true + +SEED_MODE=false \ No newline at end of file From cc5e069a744dd80cd150c8fbb20b2035703d1813 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Thu, 17 Oct 2024 16:21:59 +0200 Subject: [PATCH 06/11] Use devcontainer settings for tests in VSCode --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..6ee10c66 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "go.testEnvFile": "${workspaceFolder}/.testenv-devcontainer" +} From 4904a0d1b11b21b6ebe9a9337437b074aada82b6 Mon Sep 17 00:00:00 2001 From: Wolfram Fischer Date: Fri, 18 Oct 2024 17:06:44 +0200 Subject: [PATCH 07/11] Use string instead of boolean for SEED_MODE variable in docker-compose.yml --- .devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index fc2dcf8e..d391fde6 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -28,7 +28,7 @@ services: DB_SCHEMA: /app_sqlschema/schema.sql - SEED_MODE: true + SEED_MODE: "true" AUTH_TYPE: token AUTH_TOKEN_SECRET: xxx From 17bd65b045d4f84f9c2c1aa74c4195cddbd0ac79 Mon Sep 17 00:00:00 2001 From: David Rochow Date: Tue, 22 Oct 2024 08:09:00 +0200 Subject: [PATCH 08/11] fix(test): added connection test for API Server --- internal/e2e/token_auth_test.go | 2 ++ internal/server/server.go | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/e2e/token_auth_test.go b/internal/e2e/token_auth_test.go index ab2a8ce6..19b424cf 100644 --- a/internal/e2e/token_auth_test.go +++ b/internal/e2e/token_auth_test.go @@ -36,6 +36,8 @@ var _ = Describe("Getting access via API", Label("e2e", "TokenAuthorization"), f queryUrl = fmt.Sprintf("http://localhost:%s/query", cfg.Port) s.NonBlockingStart() + err = s.TestConnection(20) + Expect(err).To(BeNil(), "API Server should be reachable") }) AfterEach(func() { diff --git a/internal/server/server.go b/internal/server/server.go index 5e83f83b..1610ea92 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -12,10 +12,9 @@ import ( "syscall" "time" + graphqlapi "github.com/cloudoperators/heureka/internal/api/graphql" "github.com/cloudoperators/heureka/internal/app" "github.com/cloudoperators/heureka/internal/database/mariadb" - - graphqlapi "github.com/cloudoperators/heureka/internal/api/graphql" "github.com/cloudoperators/heureka/internal/util" "github.com/onuryilmaz/ginprom" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -189,3 +188,21 @@ func (s *Server) NonBlockingStop() { log.Println("Server exiting") } + +func (s *Server) TestConnection(backOff int) error { + if backOff <= 0 { + return fmt.Errorf("Unable to connect to API Server, exceeded backoffs...") + } + //before each try wait 100 milliseconds + time.Sleep(time.Millisecond * 100) + + //timeout after 100 milliseconds + reqCtx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer cancel() + _, err := http.NewRequestWithContext(reqCtx, http.MethodGet, fmt.Sprintf("http://localhost%s/ready", s.config.Port), nil) + if err != nil { + return s.TestConnection(backOff - 1) + } + + return nil +} From 31a5b825cf63e5b05df0c19533e58e272fea95aa Mon Sep 17 00:00:00 2001 From: David Rochow Date: Tue, 22 Oct 2024 08:52:11 +0200 Subject: [PATCH 09/11] fix(test): use wait group for waiting for listening instead of periodic check --- internal/e2e/token_auth_test.go | 2 -- internal/server/server.go | 32 +++++++++++++------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/internal/e2e/token_auth_test.go b/internal/e2e/token_auth_test.go index 19b424cf..ab2a8ce6 100644 --- a/internal/e2e/token_auth_test.go +++ b/internal/e2e/token_auth_test.go @@ -36,8 +36,6 @@ var _ = Describe("Getting access via API", Label("e2e", "TokenAuthorization"), f queryUrl = fmt.Sprintf("http://localhost:%s/query", cfg.Port) s.NonBlockingStart() - err = s.TestConnection(20) - Expect(err).To(BeNil(), "API Server should be reachable") }) AfterEach(func() { diff --git a/internal/server/server.go b/internal/server/server.go index 1610ea92..7cdd6457 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -7,8 +7,10 @@ import ( "context" "fmt" "log" + "net" "net/http" "os/signal" + "sync" "syscall" "time" @@ -144,12 +146,22 @@ func (s *Server) NonBlockingStart() { Handler: s.router.Handler(), } + var waitGroup sync.WaitGroup + waitGroup.Add(1) go func() { logrus.Info("Starting Non Blocking HTTP Server...") - if err := s.nonBlockingSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + ln, err := net.Listen("tcp", s.nonBlockingSrv.Addr) + if err != nil { + logrus.WithError(err).Fatalf("Error while start listening...") + } + go func() { + waitGroup.Done() + }() + if err := s.nonBlockingSrv.Serve(ln); err != nil && err != http.ErrServerClosed { logrus.WithError(err).Fatalf("Error while serving HTTP Server.") } }() + waitGroup.Wait() } func (s *Server) BlockingStop() { @@ -188,21 +200,3 @@ func (s *Server) NonBlockingStop() { log.Println("Server exiting") } - -func (s *Server) TestConnection(backOff int) error { - if backOff <= 0 { - return fmt.Errorf("Unable to connect to API Server, exceeded backoffs...") - } - //before each try wait 100 milliseconds - time.Sleep(time.Millisecond * 100) - - //timeout after 100 milliseconds - reqCtx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) - defer cancel() - _, err := http.NewRequestWithContext(reqCtx, http.MethodGet, fmt.Sprintf("http://localhost%s/ready", s.config.Port), nil) - if err != nil { - return s.TestConnection(backOff - 1) - } - - return nil -} From f91492d08b2ba6846083338891670d25b92b3526 Mon Sep 17 00:00:00 2001 From: David Rochow Date: Tue, 22 Oct 2024 09:06:45 +0200 Subject: [PATCH 10/11] fix(test): removing AUTH_TYPE and AUTH_TOKEN_SECRET from compose yaml and set default AuthType to none --- .devcontainer/docker-compose.yml | 2 -- internal/util/config.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d391fde6..979c6f36 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -29,8 +29,6 @@ services: SEED_MODE: "true" - AUTH_TYPE: token - AUTH_TOKEN_SECRET: xxx mariadb: image: mariadb:latest diff --git a/internal/util/config.go b/internal/util/config.go index c96aac97..1f2e8a42 100644 --- a/internal/util/config.go +++ b/internal/util/config.go @@ -33,7 +33,7 @@ type Config struct { //// https://pkg.go.dev/github.com/robfig/cron#hdr-Predefined_schedules //DiscoverySchedule string `envconfig:"DISOVERY_SCHEDULE" default:"0 0 0 * * *" json:"discoverySchedule"` SeedMode bool `envconfig:"SEED_MODE" required:"false" default:"false" json:"seedMode"` - AuthType string `envconfig:"AUTH_TYPE" required:"false" json:"-"` + AuthType string `envconfig:"AUTH_TYPE" required:"false" json:"-" default:"none"` AuthTokenSecret string `envconfig:"AUTH_TOKEN_SECRET" required:"false" json:"-"` DefaultIssuePriority int64 `envconfig:"DEFAULT_ISSUE_PRIORITY" default:"100" json:"defaultIssuePriority"` DefaultRepositoryName string `envconfig:"DEFAULT_REPOSITORY_NAME" default:"nvd" json:"defaultRepositoryName"` From 0b97ff2a9f87cdfa1918b9a96e55dc5ee3cd405d Mon Sep 17 00:00:00 2001 From: David Rochow Date: Tue, 22 Oct 2024 10:38:13 +0200 Subject: [PATCH 11/11] fix(test): added FirstListenThenServe to utility pkg and added in nonblocking server start and seperate token test --- .../api/graphql/access/token_auth_test.go | 6 +--- internal/server/server.go | 21 ++---------- pkg/util/server.go | 33 +++++++++++++++++++ 3 files changed, 37 insertions(+), 23 deletions(-) create mode 100644 pkg/util/server.go diff --git a/internal/api/graphql/access/token_auth_test.go b/internal/api/graphql/access/token_auth_test.go index 5a2b6cff..a81deed2 100644 --- a/internal/api/graphql/access/token_auth_test.go +++ b/internal/api/graphql/access/token_auth_test.go @@ -57,11 +57,7 @@ func (s *server) startInBackground(port string) { s.ctx, s.cancel = context.WithCancel(context.Background()) s.srv = &http.Server{Addr: fmt.Sprintf(":%s", port), Handler: r} - go func() { - if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - fmt.Println("Unexpected error when running test server") - } - }() + util2.FirstListenThenServe(s.srv) } func (s *server) stop() { diff --git a/internal/server/server.go b/internal/server/server.go index 7cdd6457..966eef88 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -7,10 +7,8 @@ import ( "context" "fmt" "log" - "net" "net/http" "os/signal" - "sync" "syscall" "time" @@ -18,6 +16,7 @@ import ( "github.com/cloudoperators/heureka/internal/app" "github.com/cloudoperators/heureka/internal/database/mariadb" "github.com/cloudoperators/heureka/internal/util" + util2 "github.com/cloudoperators/heureka/pkg/util" "github.com/onuryilmaz/ginprom" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -146,22 +145,7 @@ func (s *Server) NonBlockingStart() { Handler: s.router.Handler(), } - var waitGroup sync.WaitGroup - waitGroup.Add(1) - go func() { - logrus.Info("Starting Non Blocking HTTP Server...") - ln, err := net.Listen("tcp", s.nonBlockingSrv.Addr) - if err != nil { - logrus.WithError(err).Fatalf("Error while start listening...") - } - go func() { - waitGroup.Done() - }() - if err := s.nonBlockingSrv.Serve(ln); err != nil && err != http.ErrServerClosed { - logrus.WithError(err).Fatalf("Error while serving HTTP Server.") - } - }() - waitGroup.Wait() + util2.FirstListenThenServe(s.nonBlockingSrv) } func (s *Server) BlockingStop() { @@ -175,6 +159,7 @@ func (s *Server) BlockingStop() { log.Fatalf("Error while shuting down Heureka App: %s", err) } } + func (s *Server) NonBlockingStop() { ctx := *s.nonBlockingCtx stop := *s.nonBlockingStop diff --git a/pkg/util/server.go b/pkg/util/server.go new file mode 100644 index 00000000..a6eb46cb --- /dev/null +++ b/pkg/util/server.go @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "net" + "net/http" + "sync" + + "github.com/sirupsen/logrus" +) + +// FirstListenThenServe is a utility function that ensures that first a listener is spin up then the http server is setup for serving asynchronously +// this is requried to ensure in tests that the server is spinned up before jumping to tests. +func FirstListenThenServe(srv *http.Server) { + var waitGroup sync.WaitGroup + waitGroup.Add(1) + go func() { + logrus.Info("Starting Non Blocking HTTP Server...") + ln, err := net.Listen("tcp", srv.Addr) + if err != nil { + logrus.WithError(err).Fatalf("Error while start listening...") + } + go func() { + waitGroup.Done() + }() + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + logrus.WithError(err).Fatalf("Error while serving HTTP Server.") + } + }() + waitGroup.Wait() +}