From 8e168aac226fb73cf6fffa1fb68946e83a5b948a Mon Sep 17 00:00:00 2001 From: bupd Date: Mon, 4 Nov 2024 19:10:46 +0530 Subject: [PATCH 1/4] add quickstart guide Signed-off-by: bupd --- QUICKSTART.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 27 ++++++++------ 2 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 QUICKSTART.md diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..e6aac9e --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,100 @@ +## Quickstart: Local Setup of Harbor Satellite from Source + +To set up Harbor Satellite locally, follow these steps: + +### 1. Prerequisites +Ensure you have: +- A Harbor registry instance (or similar OCI-compliant registry). +- Credentials with permission to create robot accounts in the registry. +- The latest version of Dagger installed. + +### 2. Set Up the Registry +In this guide, we'll use a Harbor registry instance. + +- **Registry Login**: Obtain the username and password for your registry, ensuring it has appropriate permissions. + +### 3. Configure Ground Control +Navigate to the `ground-control` directory and set up the following environment variables: + +```bash +HARBOR_USERNAME=admin +HARBOR_PASSWORD=Harbor12345 +HARBOR_URL=https://demo.goharbor.io + +PORT=8080 +APP_ENV=local + +DB_HOST=localhost +DB_PORT=5432 +DB_DATABASE=groundcontrol +DB_USERNAME=postgres # Customize based on your DB config +DB_PASSWORD=password # Customize based on your DB config +``` + +### 4. Run Ground Control +To start the Ground Control service, execute the following Dagger command: + +```bash +dagger call build-dev --component ground-control up +``` + +> **Note:** Ensure you have set up Dagger with the latest version before running this command. Ground Control will run on port 8080. + +### 5. Configure Satellite +Return to the root project directory: + +```bash +cd .. +``` + +Then navigate to the `satellite` directory and verify that `config.toml` is set up correctly: + +```toml +# Whether to use the built-in Zot registry or not +bring_own_registry = false + +# IP address and port of the registry +own_registry_adr = "127.0.0.1" +own_registry_port = "8585" + +# URL of remote registry or local file path +url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" + + + +# Default path for Zot registry config.json +zotConfigPath = "./registry/config.json" + +# Set logging level +log_level = "info" +``` + +### 6. Register the Satellite with Ground Control +Using `curl` or Postman, make a `POST` request to register the Satellite with Ground Control: + +```bash +curl -X POST http://localhost:8080/satellites/register -H "Content-Type: application/json" -d '{ "name": "" }' +``` + +The response will include a token string. Set this token in the Satellite `.env` file: + +```console +TOKEN= +``` + +### 7. Start the Satellite +Run the following Dagger command to start the Satellite service: + +```bash +dagger call build-dev --component satellite up +``` + +The Satellite service will start on port 9090. Ensure that the `ground_control_url` is correctly set in the Satellite configuration before launching. + + +### 8. Finalize and Run +After setting the token, you can now run the Satellite. This setup will launch the Satellite in a container with the following exposed ports: +- **9090** for the Satellite service. +- **8585** for the Zot registry (if configured). + +With this setup, your Harbor Satellite should be up and running! diff --git a/README.md b/README.md index cb92a63..1145b4d 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ Harbor Satellite will synchronize with the central Harbor registry, when Interne Harbor Satellite will also include a toolset enabling the monitoring and management of local decentralized registries. +## QuickStart +Please refer to the latest setup instructions in QUICKSTART.md file for detailed steps on setting up Harbor Satellite locally. + ## Non-Goals T.B.D. @@ -51,29 +54,29 @@ Harbor Satellite, at its most basic, will run in a single container and will be Harbor Satellite may be implemented following 1 or several of 3 different architectures depending on its use cases : -1. **Replicating from a remote registry to a local registry.** +1. **Replicating from a remote registry to a local registry.** In this basic use case, the stateless satellite component will handle pulling images from a remote registry and then pushing them to the local OCI compliant registry. This local registry will then be accessible to other local edge devices who can pull required images directly from it. -_(A direct access from edge device to the remote registry is still possible when network conditions permit it)._ -The satellite component may also handle updating container runtime configurations and fetching image lists from Ground Control, a part of Harbor. -The stateful local regsitry will also need to handle storing and managing data on local volumes. -A typical use case would work as follows : +_(A direct access from edge device to the remote registry is still possible when network conditions permit it)._ +The satellite component may also handle updating container runtime configurations and fetching image lists from Ground Control, a part of Harbor. +The stateful local regsitry will also need to handle storing and managing data on local volumes. +A typical use case would work as follows : _In an edge computing environment where IoT devices are deployed to a location with limited or no internet connnectivity, these devices need to run containerised images but cannot pull from a central Harbor registry. A local Harbor Satellite instance can be deployed and take up this role while Internet connectivity is unreliable and distribute all required images. Once a reliable connection is re-established, the Harbor Satellite instance will be able to pull required images from its central Harbor registry and thus store up to date images locally._ ![Use Case #1](docs/images/satellite_use_case_1.svg)

Use case #1

-2. **Replicating from a remote regsitry to a local Spegel Registry** -The stateless satellite component send pull instructions to Spegel instances running with each node of a Kubernetes cluster. The node will then directly pull images from a remote registry and share it with other local nodes, removing the need for each of them to individually pull an image from a remote registry. -The network interfaces (boundaries) represented in this use case should and will be the same as those represented in use case #1 -A typical use case would work as follows : +2. **Replicating from a remote regsitry to a local Spegel Registry** +The stateless satellite component send pull instructions to Spegel instances running with each node of a Kubernetes cluster. The node will then directly pull images from a remote registry and share it with other local nodes, removing the need for each of them to individually pull an image from a remote registry. +The network interfaces (boundaries) represented in this use case should and will be the same as those represented in use case #1 +A typical use case would work as follows : _In a larger scale edge computing environment with a significant amount of IoT devices needing to run containerised applications, a single local registry in might not be able to handle the increased amount of demands from edge devices. The solution is to deploy several registries to several nodes who are able to automatically replicate images across each other thanks to Spegel instances running together with each node. The Satellite component will use the same interface to instruct each node when, where and how to pull new images that need to be replicated across the cluster._ ![Use Case #2](docs/images/satellite_use_case_2.svg)

Use case #2

-3. **Proxying from a remote regsitry over the local registry** -The stateless satellite component will be in charge of configuring the local OCI compliant registry, which will be running in proxy mode only. This local registry will then handle pulling necessary images from the remote registry and serving them up for use by local edge devices. -A typical use case would work as follows : +3. **Proxying from a remote regsitry over the local registry** +The stateless satellite component will be in charge of configuring the local OCI compliant registry, which will be running in proxy mode only. This local registry will then handle pulling necessary images from the remote registry and serving them up for use by local edge devices. +A typical use case would work as follows : _When, for a number of possible different reasons, the remote registry side of the diagram would not be able to produce a list of images to push down to the Harbor Satellite, the Satellite would then act as a proxy and forward all requests from edge devices to the remote registry. This ensures the availability of necessary images without the need for a pre-compiled list of images_ ![Use Case #3](docs/images/satellite_use_case_3.svg) From 9dcfc664b42ef59a5879cc8a8882e49be6137654 Mon Sep 17 00:00:00 2001 From: bupd Date: Mon, 4 Nov 2024 19:14:58 +0530 Subject: [PATCH 2/4] add build-dev function to dagger - This adds development server for ground control - This commit only focuses on setting the dev server for ground-control - Dagger has a known limitation for hot reloading dev servers. - limitation issue in dagger (https://github.com/dagger/dagger/issues/6990) Signed-off-by: bupd --- .air.toml | 13 +++++++++++++ ci/main.go | 36 ++++++++++++++++++++++++++++++++++++ ground-control/.air.toml | 13 +++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .air.toml create mode 100644 ground-control/.air.toml diff --git a/.air.toml b/.air.toml new file mode 100644 index 0000000..e03ed91 --- /dev/null +++ b/.air.toml @@ -0,0 +1,13 @@ +[build] + bin = "main" + dir = "." + cmd = "go build -o main ." + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + run_after_build = "chmod +x main" + include_ext = ["go"] + +[log] + level = "debug" diff --git a/ci/main.go b/ci/main.go index fd6ca31..21add26 100644 --- a/ci/main.go +++ b/ci/main.go @@ -20,6 +20,42 @@ const ( type HarborSatellite struct{} +// build-dev function would start the dev server for the component provided. +func (m *HarborSatellite) BuildDev( + ctx context.Context, + // +optional + // +defaultPath="./ground-control" + source *dagger.Directory, + component string, +) (*dagger.Service, error) { + if component == "satellite" || component == "ground-control" { + golang := dag.Container(). + From("golang:latest"). + WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod")). + WithEnvVariable("GOMODCACHE", "/go/pkg/mod"). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build")). + WithEnvVariable("GOCACHE", "/go/build-cache"). + WithMountedDirectory(PROJ_MOUNT, source). + WithWorkdir(PROJ_MOUNT). + WithExec([]string{"go", "install", "github.com/air-verse/air@latest"}) + + if component == "ground-control" { + golang = golang. + WithExec([]string{"ls", "-la"}). + // WithWorkdir("./ground-control/"). + // WithExec([]string{"go", "mod", "download"}). + WithExec([]string{"ls", "-la"}). + WithExec([]string{"air", "-c", ".air.toml"}). + WithExposedPort(8080) + } + // to-do: build else part for the satellite + + return golang.AsService(), nil + } + + return nil, fmt.Errorf("error: please provide component as either satellite or ground-control") +} + // Build function would start the build process for the name provided. Source should be the path to the main.go file. func (m *HarborSatellite) Build( ctx context.Context, diff --git a/ground-control/.air.toml b/ground-control/.air.toml new file mode 100644 index 0000000..e03ed91 --- /dev/null +++ b/ground-control/.air.toml @@ -0,0 +1,13 @@ +[build] + bin = "main" + dir = "." + cmd = "go build -o main ." + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + run_after_build = "chmod +x main" + include_ext = ["go"] + +[log] + level = "debug" From 3a82f41cc3303b25f42285bba82797fef5fb8e50 Mon Sep 17 00:00:00 2001 From: bupd Date: Wed, 13 Nov 2024 16:22:38 +0530 Subject: [PATCH 3/4] ground control as stateful service Signed-off-by: bupd --- ci/main.go | 29 +++++++++++++++++++++++++---- dagger.json | 4 ++-- go.mod | 21 ++++++++++++--------- go.sum | 30 ++++++++++++++++++------------ ground-control/.env | 2 +- 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/ci/main.go b/ci/main.go index 21add26..27d19a9 100644 --- a/ci/main.go +++ b/ci/main.go @@ -24,7 +24,7 @@ type HarborSatellite struct{} func (m *HarborSatellite) BuildDev( ctx context.Context, // +optional - // +defaultPath="./ground-control" + // +defaultPath="." source *dagger.Directory, component string, ) (*dagger.Service, error) { @@ -36,15 +36,24 @@ func (m *HarborSatellite) BuildDev( WithMountedCache("/go/build-cache", dag.CacheVolume("go-build")). WithEnvVariable("GOCACHE", "/go/build-cache"). WithMountedDirectory(PROJ_MOUNT, source). - WithWorkdir(PROJ_MOUNT). + WithWorkdir(PROJ_MOUNT). WithExec([]string{"go", "install", "github.com/air-verse/air@latest"}) if component == "ground-control" { + db, err := m.Db(ctx) + if err != nil { + return nil, err + } + golang = golang. + WithWorkdir(PROJ_MOUNT+"/ground-control/sql/schema"). WithExec([]string{"ls", "-la"}). - // WithWorkdir("./ground-control/"). - // WithExec([]string{"go", "mod", "download"}). + WithServiceBinding("pgservice", db). + WithExec([]string{"go", "install", "github.com/pressly/goose/v3/cmd/goose@latest"}). + WithExec([]string{"goose", "postgres", "postgres://postgres:password@pgservice:5432/groundcontrol", "up"}). + WithWorkdir(PROJ_MOUNT + "/ground-control"). WithExec([]string{"ls", "-la"}). + WithExec([]string{"go", "mod", "download"}). WithExec([]string{"air", "-c", ".air.toml"}). WithExposedPort(8080) } @@ -56,6 +65,18 @@ func (m *HarborSatellite) BuildDev( return nil, fmt.Errorf("error: please provide component as either satellite or ground-control") } +// starts postgres DB container for ground-control. +func (m *HarborSatellite) Db(ctx context.Context) (*dagger.Service, error) { + return dag.Container(). + From("postgres:17"). + WithEnvVariable("POSTGRES_USER", "postgres"). + WithEnvVariable("POSTGRES_PASSWORD", "password"). + WithEnvVariable("POSTGRES_HOST_AUTH_METHOD", "trust"). + WithEnvVariable("POSTGRES_DB", "groundcontrol"). + WithExposedPort(5432). + AsService().Start(ctx) +} + // Build function would start the build process for the name provided. Source should be the path to the main.go file. func (m *HarborSatellite) Build( ctx context.Context, diff --git a/dagger.json b/dagger.json index b79d3a0..05d2ccf 100644 --- a/dagger.json +++ b/dagger.json @@ -1,6 +1,6 @@ { "name": "harbor-satellite", + "engineVersion": "v0.14.0", "sdk": "go", - "source": "ci", - "engineVersion": "v0.13.3" + "source": "ci" } diff --git a/go.mod b/go.mod index 6871c9b..666a5f7 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module container-registry.com/harbor-satellite -go 1.22.3 +go 1.22.5 -toolchain go1.22.4 +toolchain go1.23.2 require ( dagger.io/dagger v0.10.3 @@ -28,7 +28,7 @@ require ( cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/storage v1.39.1 // indirect dario.cat/mergo v1.0.0 // indirect - github.com/99designs/gqlgen v0.17.49 + github.com/99designs/gqlgen v0.17.55 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect @@ -64,7 +64,7 @@ require ( github.com/ThalesIgnite/crypto11 v1.2.5 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/adrg/xdg v0.5.0 // indirect + github.com/adrg/xdg v0.5.1 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect @@ -373,7 +373,7 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/vbatts/tar-split v0.11.5 // indirect github.com/vbauerster/mpb/v8 v8.7.3 // indirect - github.com/vektah/gqlparser/v2 v2.5.16 + github.com/vektah/gqlparser/v2 v2.5.17 github.com/xanzy/go-gitlab v0.102.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -393,13 +393,16 @@ require ( go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/log v0.3.0 - go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/sdk/log v0.3.0 + go.opentelemetry.io/otel/sdk/metric v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 go.opentelemetry.io/proto/otlp v1.3.1 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect @@ -407,13 +410,13 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/mod v0.18.0 // indirect + golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.22.0 // indirect + golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.172.0 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect diff --git a/go.sum b/go.sum index 5dbf7b5..7801ef3 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ= -github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0= +github.com/99designs/gqlgen v0.17.55 h1:3vzrNWYyzSZjGDFo68e5j9sSauLxfKvLp+6ioRokVtM= +github.com/99designs/gqlgen v0.17.55/go.mod h1:3Bq768f8hgVPGZxL8aY9MaYmbxa6llPM/qu1IGH1EJo= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= @@ -292,8 +292,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= -github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= +github.com/adrg/xdg v0.5.1 h1:Im8iDbEFARltY09yOJlSGu4Asjk2vF85+3Dyru8uJ0U= +github.com/adrg/xdg v0.5.1/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= @@ -1457,8 +1457,8 @@ github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinC github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0= github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM= -github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= -github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= +github.com/vektah/gqlparser/v2 v2.5.17 h1:9At7WblLV7/36nulgekUgIaqHZWn5hxqluxrxGUhOmI= +github.com/vektah/gqlparser/v2 v2.5.17/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/veraison/go-cose v1.2.1 h1:Gj4x20D0YP79J2+cK3anjGEMwIkg2xX+TKVVGUXwNAc= github.com/veraison/go-cose v1.2.1/go.mod h1:t6V8WJzHm1PD5HNsuDjW3KLv577uWb6UTzbZGvdQHD8= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= @@ -1536,6 +1536,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-2024051809000 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88/go.mod h1:JGG8ebaMO5nXOPnvKEl+DiA4MGwFjCbjsxT1WHIEBPY= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0 h1:ccBrA8nCY5mM0y5uO7FT0ze4S0TuFcWdDB2FxGMTjkI= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0/go.mod h1:/9pb6634zi2Lk8LYg9Q0X8Ar6jka4dkFOylBLbVQPCE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0/go.mod h1:xJntEd2KL6Qdg5lwp97HMLQDVeAhrYxmzFseAMDPQ8I= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= @@ -1550,6 +1554,8 @@ go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kT go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1625,8 +1631,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1825,8 +1831,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1922,8 +1928,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/ground-control/.env b/ground-control/.env index 2cf1faa..2b0e12e 100644 --- a/ground-control/.env +++ b/ground-control/.env @@ -6,7 +6,7 @@ ZOT_URL="127.0.0.1:8585" PORT=8080 APP_ENV=local -DB_HOST=localhost +DB_HOST=pgservice DB_PORT=5432 DB_DATABASE=groundcontrol # Customize user and pass based on your config From b00b5adb6e42dc1ed8cfd6268bca020eeba14dfb Mon Sep 17 00:00:00 2001 From: bupd Date: Wed, 13 Nov 2024 17:39:29 +0530 Subject: [PATCH 4/4] update build dev service Signed-off-by: bupd --- QUICKSTART.md | 27 +++++++++++++--- ci/main.go | 89 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 94 insertions(+), 22 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index e6aac9e..c6af818 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -24,7 +24,7 @@ HARBOR_URL=https://demo.goharbor.io PORT=8080 APP_ENV=local -DB_HOST=localhost +DB_HOST=pgservice DB_PORT=5432 DB_DATABASE=groundcontrol DB_USERNAME=postgres # Customize based on your DB config @@ -35,7 +35,19 @@ DB_PASSWORD=password # Customize based on your DB config To start the Ground Control service, execute the following Dagger command: ```bash -dagger call build-dev --component ground-control up +dagger call run-ground-control up +``` + +You can also build ground-control binary using the below command + +```bash +dagger call build-dev --platform "linux/amd64" --component "ground-control" export --path=./gc-dev +``` + +To Run ground-control binary use + +```bash +./gc-dev ``` > **Note:** Ensure you have set up Dagger with the latest version before running this command. Ground Control will run on port 8080. @@ -82,11 +94,16 @@ The response will include a token string. Set this token in the Satellite `.env` TOKEN= ``` -### 7. Start the Satellite -Run the following Dagger command to start the Satellite service: +### 7. Build the Satellite +Run the following Dagger command to build the Satellite: + +```bash +dagger call build-dev --platform "linux/amd64" --component satellite export --path=./satellite-dev +``` +To Run Satellite: ```bash -dagger call build-dev --component satellite up +./satellite-dev ``` The Satellite service will start on port 9090. Ensure that the `ground_control_url` is correctly set in the Satellite configuration before launching. diff --git a/ci/main.go b/ci/main.go index 27d19a9..4df950c 100644 --- a/ci/main.go +++ b/ci/main.go @@ -3,8 +3,10 @@ package main import ( "context" "fmt" + "log" "log/slog" "os" + "strings" "container-registry.com/harbor-satellite/ci/internal/dagger" ) @@ -20,15 +22,61 @@ const ( type HarborSatellite struct{} -// build-dev function would start the dev server for the component provided. +// start the dev server for ground-control. +func (m *HarborSatellite) RunGroundControl( + ctx context.Context, + // +optional + // +defaultPath="." + source *dagger.Directory, +) (*dagger.Service, error) { + golang := dag.Container(). + From("golang:latest"). + WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod")). + WithEnvVariable("GOMODCACHE", "/go/pkg/mod"). + WithMountedCache("/go/build-cache", dag.CacheVolume("go-build")). + WithEnvVariable("GOCACHE", "/go/build-cache"). + WithMountedDirectory(PROJ_MOUNT, source). + WithWorkdir(PROJ_MOUNT). + WithExec([]string{"go", "install", "github.com/air-verse/air@latest"}) + + db, err := m.Db(ctx) + if err != nil { + return nil, err + } + + golang = golang. + WithWorkdir(PROJ_MOUNT+"/ground-control/sql/schema"). + WithExec([]string{"ls", "-la"}). + WithServiceBinding("pgservice", db). + WithExec([]string{"go", "install", "github.com/pressly/goose/v3/cmd/goose@latest"}). + WithExec([]string{"goose", "postgres", "postgres://postgres:password@pgservice:5432/groundcontrol", "up"}). + WithWorkdir(PROJ_MOUNT + "/ground-control"). + WithExec([]string{"ls", "-la"}). + WithExec([]string{"go", "mod", "download"}). + WithExec([]string{"air", "-c", ".air.toml"}). + WithExposedPort(8080) + + return golang.AsService(), nil +} + +// quickly build binaries for components for given platform. func (m *HarborSatellite) BuildDev( ctx context.Context, // +optional // +defaultPath="." source *dagger.Directory, + platform string, component string, -) (*dagger.Service, error) { +) (*dagger.File, error) { + fmt.Println("🛠️ Building Harbor-Cli with Dagger...") + // Define the path for the binary output + os, arch, err := parsePlatform(platform) + if err != nil { + log.Fatalf("Error parsing platform: %v", err) + } + if component == "satellite" || component == "ground-control" { + var binaryFile *dagger.File golang := dag.Container(). From("golang:latest"). WithMountedCache("/go/pkg/mod", dag.CacheVolume("go-mod")). @@ -37,29 +85,27 @@ func (m *HarborSatellite) BuildDev( WithEnvVariable("GOCACHE", "/go/build-cache"). WithMountedDirectory(PROJ_MOUNT, source). WithWorkdir(PROJ_MOUNT). - WithExec([]string{"go", "install", "github.com/air-verse/air@latest"}) + WithEnvVariable("GOOS", os). + WithEnvVariable("GOARCH", arch) if component == "ground-control" { - db, err := m.Db(ctx) - if err != nil { - return nil, err - } - golang = golang. - WithWorkdir(PROJ_MOUNT+"/ground-control/sql/schema"). - WithExec([]string{"ls", "-la"}). - WithServiceBinding("pgservice", db). - WithExec([]string{"go", "install", "github.com/pressly/goose/v3/cmd/goose@latest"}). - WithExec([]string{"goose", "postgres", "postgres://postgres:password@pgservice:5432/groundcontrol", "up"}). WithWorkdir(PROJ_MOUNT + "/ground-control"). WithExec([]string{"ls", "-la"}). WithExec([]string{"go", "mod", "download"}). - WithExec([]string{"air", "-c", ".air.toml"}). - WithExposedPort(8080) + WithExec([]string{"go", "build", "."}) + + binaryFile = golang.File(PROJ_MOUNT + "/ground-control/ground-control") + } else { + golang = golang. + WithExec([]string{"ls", "-la"}). + WithExec([]string{"go", "mod", "download"}). + WithExec([]string{"go", "build", "."}) + + binaryFile = golang.File(PROJ_MOUNT + "/harbor-satellite") } - // to-do: build else part for the satellite - return golang.AsService(), nil + return binaryFile, nil } return nil, fmt.Errorf("error: please provide component as either satellite or ground-control") @@ -135,3 +181,12 @@ func (m *HarborSatellite) Release(ctx context.Context, directory *dagger.Directo return release_output, nil } + +// Parse the platform string into os and arch +func parsePlatform(platform string) (string, string, error) { + parts := strings.Split(platform, "/") + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid platform format: %s. Should be os/arch. E.g. darwin/amd64", platform) + } + return parts[0], parts[1], nil +}