Merge branch 'master' into package-chainspec-with-docker-image

.env

@@ -123,6 +123,12 @@ APP_PRIVATE_KEY=this-is-not-so-secret-change-it
 TRUSTED_REVERSE_PROXIES=1
 # =====================================================================================
 
+## Telemetry
+# yes/no
+TELEMETRY_ENABLED=no
+# TELEMETRY_ENDPOINT=http://apm-server:8200
+TELEMETRY_ENDPOINT=http://collector:4318
+
 # joystream/node docker image tag
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.

.github/workflows/joystream-apps-docker.yml

@@ -24,6 +24,14 @@ jobs:
           echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
           echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
 
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
@@ -74,14 +82,7 @@ jobs:
         run: |
           yarn build:packages
           cp docker-compose-no-bind-volumes.yml docker-compose.yml
-          ./tests/network-tests/run-test-node-docker.sh
-          sleep 15
-          docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
-          sleep 30
-          ./tests/network-tests/start-storage.sh
-          export REUSE_KEYS=true
-          export SKIP_STORAGE_AND_DISTRIBUTION=true
-          IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
+          ./tests/network-tests/run-tests.sh content-directory
 
       - name: Push new versions
         if: github.event_name == 'workflow_dispatch'

README.md

@@ -14,7 +14,7 @@ The following tools are required for building, testing and contributing to this
 - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
 - [nodejs](https://nodejs.org/) v14.x - _required_
 - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
-- [docker](https://www.docker.com/get-started) and docker-compose v.1.29 or higher - _required_
+- [docker](https://www.docker.com/get-started) and docker-compose v2.20.x or higher - _required_
 - [ansible](https://www.ansible.com/) - _optional_
 
 If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.

colossus.Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/x86-64 node:18 as builder
+FROM node:18 as builder
 
 WORKDIR /joystream
 
@@ -29,7 +29,7 @@ RUN yarn workspace @joystream/opentelemetry build
 RUN yarn workspace storage-node build
 RUN yarn cache clean
 
-FROM --platform=linux/x86-64 node:18 as final
+FROM node:18 as final
 WORKDIR /joystream
 COPY --from=builder /joystream /joystream
 RUN yarn --frozen-lockfile --production

devops/ansible/roles/common/tasks/install-tools.yml

@@ -33,6 +33,20 @@
       - lsb-release
     state: latest
 
+# ref: https://docs.docker.com/engine/install/ubuntu/#uninstall-old-versions
+- name: Remove old docker version
+  become: true
+  apt:
+    name:
+      - docker-compose
+      - docker.io
+      - docker-doc
+      - docker-compose
+      - podman-docker
+      - containerd
+      - runc
+    state: absent
+
 - name: Add Docker GPG Key and repo
   become: true
   shell:
@@ -47,7 +61,7 @@
       "
     creates: /etc/apt/sources.list.d/docker.list
 
-- name: Update apt and install docker
+- name: Install Docker
   become: true
   apt:
     name:
@@ -58,30 +72,24 @@
     state: latest
     update_cache: true
 
-- name: Remove old docker-compose
-  become: true
-  apt:
-    name:
-      - docker-compose
-    state: absent
-
-- name: Install newer version of docker-compose
+- name: Install docker-compose
   become: true
   shell:
     cmd: |
       bash -c "
-        curl -L 'https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose
+        curl -L 'https://github.com/docker/compose/releases/download/v2.20.1/docker-compose-$(uname -s)-$(uname -m)' -o /usr/local/bin/docker-compose
         chmod +x /usr/local/bin/docker-compose
         ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
       "
     creates: /usr/local/bin/docker-compose
 
-- name: Install Docker Module for Python
+- name: Install Docker module for Python
   pip:
     name: docker
 
 # usermod -aG docker username
-- name: Add {{ ansible_user }} to docker group
+# To allow non root user to run docker
+- name: Add user to docker group
   become: true
   ansible.builtin.user:
     name: "{{ ansible_user }}"

distributor-node.Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/x86-64 node:18 as builder
+FROM node:18 as builder
 
 WORKDIR /joystream
 
@@ -36,7 +36,7 @@ RUN \
   yarn workspace @joystream/distributor-cli build &&\
   yarn cache clean
 
-FROM --platform=linux/x86-64 node:18 as final
+FROM node:18 as final
 WORKDIR /joystream
 COPY --from=builder /joystream /joystream
 RUN yarn --frozen-lockfile --production

docker-compose.yml

@@ -32,11 +32,11 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
-      # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
-      # - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
-    entrypoint: ['yarn']
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
+      - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+    entrypoint: ['/joystream/entrypoints/storage.sh']
     command: [
-      'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data',
+      'server', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data',
       '--sync', '--syncInterval=1',
       '--queryNodeEndpoint=${COLOSSUS_QUERY_NODE_URL}',
       '--apiUrl=${JOYSTREAM_NODE_WS}',
@@ -74,14 +74,14 @@ services:
       JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: /data
       JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: /cache
       JOYSTREAM_DISTRIBUTOR__LOGS__FILE__PATH: /logs
-      # OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
-      # OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${TELEMETRY_ENDPOINT}
+      OTEL_RESOURCE_ATTRIBUTES: service.name=distributor-1,deployment.environment=production
       #   JOYSTREAM_DISTRIBUTOR__LOGS__CONSOLE: "off"
       #   JOYSTREAM_DISTRIBUTOR__LOGS__FILE: "off"
       # JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[\"1:0\",\"1:1\"]"
-    entrypoint: ['yarn']
+    entrypoint: ['/joystream/entrypoints/distributor.sh']
     command: ['start']
 
   colossus-2:
@@ -200,8 +200,8 @@ services:
       - GRAPHQL_PLAYGROUND_ENDPOINT=${GRAPHQL_PLAYGROUND_ENDPOINT}
       - GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT=${GRAPHQL_PLAYGROUND_SUBSCRIPTION_ENDPOINT}
       - GRAPHQL_PLAYGROUND_CDN_URL=${GRAPHQL_PLAYGROUND_CDN_URL}
-      # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
-      # - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
+      - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
     ports:
       - "${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}"
       - "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}"
@@ -212,7 +212,7 @@ services:
         source: .
         target: /joystream
     working_dir: /joystream
-    command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod']
+    entrypoint: ['/joystream/entrypoints/graphql-server.sh']
 
   processor:
     image: node:14
@@ -228,14 +228,26 @@ services:
       - BATCH_SIZE=100
       - QUEUE_FACTOR=1
       - QUEUE_MAX_CAP_FACTOR=4
+      - WARTHOG_DB_HOST=db
+      - WARTHOG_DB_DATABASE=${DB_NAME}
+      - WARTHOG_DB_USERNAME=${DB_USER}
+      - WARTHOG_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_APP_PORT=4002
     depends_on:
       - db
     volumes:
       - type: bind
         source: .
         target: /joystream
     working_dir: /joystream
-    command: ['yarn', 'workspace', 'query-node-root', 'processor:start']
+    entrypoint: sh
+    command: >
+      -c "
+        yarn workspace query-node config:dev ;
+        yarn workspace query-node-root db:prepare ;
+        yarn workspace query-node-root db:migrate ;
+        yarn workspace query-node-root processor:start
+      "
 
   indexer:
     image: joystream/hydra-indexer:v5.0.0-alpha.1

entrypoints/distributor.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# docker entrypoint fot distributor node, to allow running with telemetry
+if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "start" ]]; then
+    node --require @joystream/opentelemetry /joystream/distributor-node/bin/run $*
+else
+    /joystream/distributor-node/bin/run $*
+fi

entrypoints/graphql-server.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# docker entrypoint fot graphql-server, to allow running with telemetry
+if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
+    yarn workspace query-node-root query-node:start:prod:with-instrumentation $*
+else
+    yarn workspace query-node-root query-node:start:prod $*
+fi

entrypoints/storage.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# docker entrypoint fot storage node, to allow running with telemetry
+if [[ "$TELEMETRY_ENABLED" = "yes" ]] && [[ $1 = "server" ]]; then
+    node --require @joystream/opentelemetry /joystream/storage-node/bin/run $*
+else
+    /joystream/storage-node/bin/run $*
+fi

query-node.Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/x86-64 node:18 as builder
+FROM node:18 as builder
 
 WORKDIR /joystream
 COPY . /joystream

query-node/start.sh

@@ -9,24 +9,17 @@ cd $SCRIPT_PATH
 
 # Bring up db
 docker-compose -f ../docker-compose.yml up -d db
-
-# Wait for the db to be up
-until docker-compose -f ../docker-compose.yml logs db | grep "server started"; do
-  echo "Waiting for the db to be ready..."
-  sleep 1
-done
-
-# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
-docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node config:dev"
-# Migrate the databases
-docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:prepare"
-docker-compose -f ../docker-compose.yml run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:migrate"
+echo "Waiting for the db to be ready..."
+sleep 5
 
 # Start indexer and gateway
 docker-compose -f ../docker-compose.yml up -d indexer
 docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway
 
-# Start processor and graphql server
+# Start processor
 docker-compose -f ../docker-compose.yml up -d processor
-docker-compose -f ../docker-compose.yml up -d graphql-server
+echo "Waiting for processor to be ready..." && sleep 30
 
+# Start graphql-server
+docker-compose -f ../docker-compose.yml up -d graphql-server
+echo "Waiting for graphql-server to be ready..." && sleep 30

setup.sh

@@ -8,11 +8,17 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # code build tools
     sudo apt-get update -y
     sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \
-      unzip libc6-dev make libssl-dev python3 cmake
-    # docker
-    sudo apt-get install -y docker.io containerd runc
-    # docker-compose
-    sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev
+
+    # Docker: do not replace existing installation to avoid distrupting running containers
+    if ! command -v docker &> /dev/null
+    then
+      # Install Docker from linux distro maintaners
+      sudo apt-get install -y docker.io containerd runc
+    fi
+    # Install latest version of docker-compose
+    COMPOSE_VERSION=$(curl -sL https://api.github.com/repos/docker/compose/releases/latest | jq -r ".tag_name")
+    sudo curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
     sudo chmod +x /usr/local/bin/docker-compose
     sudo ln -sf /usr/local/bin/docker-compose /usr/bin/docker-compose
 elif [[ "$OSTYPE" == "darwin"* ]]; then