From 265152f6a048c37a93bda31ae00dda1acfdf5fd8 Mon Sep 17 00:00:00 2001 From: Anna Geller Date: Fri, 4 Aug 2023 17:01:50 +0200 Subject: [PATCH] fix indentation + add minio as Kestra storage --- content/docs/14.troubleshooting.md | 166 +++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 44 deletions(-) diff --git a/content/docs/14.troubleshooting.md b/content/docs/14.troubleshooting.md index 5720fc4573..f3703ec166 100644 --- a/content/docs/14.troubleshooting.md +++ b/content/docs/14.troubleshooting.md @@ -144,53 +144,131 @@ Finally, you can also use the `host` network mode for the `kestra` service. This version: "3" volumes: + postgres-data: + driver: local kestra-data: driver: local +networks: + kestra_net: + services: - kestra: - image: kestra/kestra:latest-full - pull_policy: always - entrypoint: /bin/bash - network_mode: host + postgres: + image: postgres + volumes: + - postgres-data:/var/lib/postgresql/data environment: - JAVA_OPTS: "--add-opens java.base/java.nio=ALL-UNNAMED" - NODE_OPTIONS: "--max-old-space-size=4096" - KESTRA_CONFIGURATION: | - datasources: - postgres: - url: jdbc:postgresql://localhost:5432/kestra - driverClassName: org.postgresql.Driver - username: kestra - password: k3str4 - kestra: - server: - basic-auth: - enabled: false - username: admin - password: kestra - anonymous-usage-report: - enabled: true - repository: - type: postgres - storage: - type: local - local: - base-path: "/app/storage" - queue: - type: postgres - tasks: - tmp-dir: - path: /tmp/kestra-wd/tmp - scripts: - docker: - volume-enabled: true - defaults: # just one example to show global taskDefaults - - type: io.kestra.plugin.airbyte.connections.Sync - url: http://host.docker.internal:8000/ - username: airbyte - password: password - url: http://localhost:8080/ - variables: - env-vars-prefix: "" # to avoid requiring KESTRA_ prefix on env vars + POSTGRES_DB: kestra + POSTGRES_USER: kestra + POSTGRES_PASSWORD: k3str4 + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 30s + timeout: 10s + retries: 10 + networks: + kestra_net: + + iceberg-rest: + image: tabulario/iceberg-rest + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + networks: + kestra_net: + + minio: + image: minio/minio + container_name: minio + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + networks: + kestra_net: + aliases: + - warehouse.minio + ports: + - 9001:9001 + - 9000:9000 + command: ["server", "/data", "--console-address", ":9001"] + + mc: + depends_on: + - minio + image: minio/mc + container_name: mc + networks: + kestra_net: + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " + + kestra: + image: kestra/kestra:latest-full + pull_policy: always + entrypoint: /bin/bash + # Note that this is meant for development only. Refer to the documentation for production deployments of Kestra which runs without a root user. + user: "root" + command: + - -c + - /app/kestra server standalone --worker-thread=128 + volumes: + - kestra-data:/app/storage + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/kestra-wd:/tmp/kestra-wd + environment: + KESTRA_CONFIGURATION: | + datasources: + postgres: + url: jdbc:postgresql://postgres:5432/kestra + driverClassName: org.postgresql.Driver + username: kestra + password: k3str4 + kestra: + server: + basic-auth: + enabled: false + username: admin + password: kestra + repository: + type: postgres + storage: + type: minio + minio: + endpoint: http://minio + port: 9000 + accessKey: admin + secretKey: password + region: us-east-1 + bucket: minio/warehouse + queue: + type: postgres + tasks: + tmp-dir: + path: /tmp/kestra-wd/tmp + url: http://localhost:8080/ + ports: + - "8080:8080" + - "8081:8081" + depends_on: + postgres: + condition: service_started + networks: + kestra_net: ``` \ No newline at end of file