diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e6dfd1a09..5e13e2e198 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,7 @@ name: CI on: [push] env: - KNORA_GDB_LICENSE: ${{ github.workspace }}/ci/graphdb.license + KNORA_GDB_LICENSE: ${{ github.workspace }}/ci/graphdb/UNIBAS_DEV_GRAPHDB_SE_latest-31-12-2020_1cores.license jobs: compile: @@ -54,9 +54,16 @@ jobs: needs: compile runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout source + uses: actions/checkout@v1 with: fetch-depth: 1 + - name: Checkout private ci-assets + uses: actions/checkout@v2 + with: + repository: dasch-swiss/dsp-ci-assets + token: ${{ secrets.GitHub_PAT }} # `GitHub_PAT` is a secret that contains the PAT. + path: ci - name: cache maven artifacts uses: actions/cache@v1 with: @@ -66,26 +73,28 @@ jobs: ${{ runner.OS }}-build-${{ env.cache-name }}- ${{ runner.OS }}-build- ${{ runner.OS }}- - - name: prepare needed graphdb-se files - run: | - echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg - tar -C ci -xvf ci/secrets.tar - mkdir -p $GITHUB_WORKSPACE/graphdb - cp $GITHUB_WORKSPACE/ci/graphdb.license $GITHUB_WORKSPACE/graphdb/graphdb.license - cp $GITHUB_WORKSPACE/webapi/scripts/KnoraRules.pie $GITHUB_WORKSPACE/graphdb - name: Print env file - run: make print-env-file + run: | + make env-file + make print-env-file - name: Run API unit tests - run: make unit-tests-with-coverage + run: make test-unit-ci api-e2e-tests: name: API E2E Tests needs: compile runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout source + uses: actions/checkout@v1 with: fetch-depth: 1 + - name: Checkout private ci-assets + uses: actions/checkout@v2 + with: + repository: dasch-swiss/dsp-ci-assets + token: ${{ secrets.GitHub_PAT }} # `GitHub_PAT` is a secret that contains the PAT. + path: ci - name: cache maven artifacts uses: actions/cache@v1 with: @@ -95,26 +104,28 @@ jobs: ${{ runner.OS }}-build-${{ env.cache-name }}- ${{ runner.OS }}-build- ${{ runner.OS }}- - - name: prepare needed graphdb-se files - run: | - echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg - tar -C ci -xvf ci/secrets.tar - mkdir -p $GITHUB_WORKSPACE/graphdb - cp $GITHUB_WORKSPACE/ci/graphdb.license $GITHUB_WORKSPACE/graphdb/graphdb.license - cp $GITHUB_WORKSPACE/webapi/scripts/KnoraRules.pie $GITHUB_WORKSPACE/graphdb - name: print env file - run: make print-env-file + run: | + make env-file + make print-env-file - name: run API E2E tests - run: make e2e-tests-with-coverage + run: make test-e2e-ci api-integration-tests: name: API Integration Tests needs: compile runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout source + uses: actions/checkout@v1 with: fetch-depth: 1 + - name: Checkout private ci-assets + uses: actions/checkout@v2 + with: + repository: dasch-swiss/dsp-ci-assets + token: ${{ secrets.GitHub_PAT }} # `GitHub_PAT` is a secret that contains the PAT. + path: ci - name: cache maven artifacts uses: actions/cache@v1 with: @@ -124,17 +135,45 @@ jobs: ${{ runner.OS }}-build-${{ env.cache-name }}- ${{ runner.OS }}-build- ${{ runner.OS }}- - - name: prepare needed graphdb-se files - run: | - echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg - tar -C ci -xvf ci/secrets.tar - mkdir -p $GITHUB_WORKSPACE/graphdb - cp $GITHUB_WORKSPACE/ci/graphdb.license $GITHUB_WORKSPACE/graphdb/graphdb.license - cp $GITHUB_WORKSPACE/webapi/scripts/KnoraRules.pie $GITHUB_WORKSPACE/graphdb - name: print env file - run: make print-env-file + run: | + make env-file + make print-env-file - name: run API integration tests - run: make it-tests-with-coverage + run: make test-it-ci + + js-lib-tests: + name: JS Lib Tests + needs: compile + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: Checkout private ci-assets + uses: actions/checkout@v2 + with: + repository: dasch-swiss/dsp-ci-assets + token: ${{ secrets.GitHub_PAT }} # `GitHub_PAT` is a secret that contains the PAT. + path: ci + - name: cache maven artifacts + uses: actions/cache@v1 + with: + path: ~/.ivy2 + key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }} + restore-keys: | + ${{ runner.OS }}-build-${{ env.cache-name }}- + ${{ runner.OS }}-build- + ${{ runner.OS }}- + - name: install requirements + run: sudo apt-get install expect + - name: print env file + run: | + make env-file + make print-env-file + - name: run JS lib tests + run: make test-js-lib-integration upgrade-unit-tests: name: Upgrade Unit Tests @@ -161,9 +200,16 @@ jobs: needs: compile runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout source + uses: actions/checkout@v1 with: fetch-depth: 1 + - name: Checkout private ci-assets + uses: actions/checkout@v2 + with: + repository: dasch-swiss/dsp-ci-assets + token: ${{ secrets.GitHub_PAT }} # `GitHub_PAT` is a secret that contains the PAT. + path: ci - uses: actions/setup-java@v1 with: java-version: '11.x.x' @@ -182,17 +228,10 @@ jobs: run: | make build-all-scala make build-all-images - - name: prepare needed graphdb-se files - run: | - echo ${{ secrets.license_encryption_key }} | gpg --quiet --batch --yes --decrypt --passphrase-fd 0 --output $GITHUB_WORKSPACE/ci/secrets.tar $GITHUB_WORKSPACE/ci/secrets.tar.gpg - tar -C ci -xvf ci/secrets.tar - mkdir -p $GITHUB_WORKSPACE/graphdb - cp $GITHUB_WORKSPACE/ci/graphdb.license $GITHUB_WORKSPACE/graphdb/graphdb.license - cp $GITHUB_WORKSPACE/webapi/scripts/KnoraRules.pie $GITHUB_WORKSPACE/graphdb - name: start and initialize graphdb-se run: | docker network create knora-net - docker run --rm -d --name=db --network=knora-net -p 127.0.0.1:7200:7200 -v $GITHUB_WORKSPACE/ci/graphdb.license:/graphdb/graphdb.license -e GDB_HEAP_SIZE=$GDB_HEAP_SIZE daschswiss/knora-graphdb-se -Dgraphdb.license.file=/graphdb/graphdb.license + docker run --rm -d --name=db --network=knora-net -p 127.0.0.1:7200:7200 -e GDB_HEAP_SIZE=$GDB_HEAP_SIZE daschswiss/knora-graphdb-se -Dgraphdb.license.file=/graphdb/graphdb.license sleep 5 make init-db-test-minimal - name: unzip test data diff --git a/.gitignore b/.gitignore index 2ce6c86803..f41ad1afe1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ .ensime .ensime_cache .env +/.tmp/ native **/.DS_Store docs/env diff --git a/Makefile b/Makefile index 81c658a82d..ffd99cb937 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Determine this makefile's path. # Be sure to place this BEFORE `include` directives, if any. -THIS_FILE := $(lastword $(MAKEFILE_LIST)) +# THIS_FILE := $(lastword $(MAKEFILE_LIST)) +THIS_FILE := $(abspath $(lastword $(MAKEFILE_LIST))) +CURRENT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) include vars.mk @@ -10,11 +12,11 @@ include vars.mk .PHONY: docs-publish docs-publish: ## build and publish docs - docker run --rm -it -v $(PWD):/knora -v $(HOME)/.ivy2:/root/.ivy2 -v $(HOME)/.ssh:/root/.ssh daschswiss/sbt-paradox /bin/sh -c "cd /knora && git config --global user.email $(GIT_EMAIL) && sbt docs/ghpagesPushSite" + docker run --rm -it -v $(CURRENT_DIR):/knora -v $(HOME)/.ivy2:/root/.ivy2 -v $(HOME)/.ssh:/root/.ssh daschswiss/sbt-paradox /bin/sh -c "cd /knora && git config --global user.email $(GIT_EMAIL) && sbt docs/ghpagesPushSite" .PHONY: docs-build docs-build: ## build the docs - docker run --rm -v $(PWD):/knora -v $(HOME)/.ivy2:/root/.ivy2 daschswiss/sbt-paradox /bin/sh -c "cd /knora && sbt docs/makeSite" + docker run --rm -v $(CURRENT_DIR):/knora -v $(HOME)/.ivy2:/root/.ivy2 daschswiss/sbt-paradox /bin/sh -c "cd /knora && sbt docs/makeSite" ################################# # Docker targets @@ -22,7 +24,7 @@ docs-build: ## build the docs .PHONY: build-all-scala build-all-scala: ## build all scala projects - sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage upgrade/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage + @sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage upgrade/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage ## knora-api .PHONY: build-knora-api-image @@ -38,6 +40,11 @@ publish-knora-api-image: build-knora-api-image ## publish knora-api image to Doc build-knora-graphdb-se-image: build-all-scala ## build and publish knora-graphdb-se docker image locally @mkdir -p .docker @sed -e "s/@GRAPHDB_IMAGE@/daschswiss\/graphdb\:$(GRAPHDB_SE_VERSION)-se/" docker/knora-graphdb.template.dockerfile > .docker/knora-graphdb-se.dockerfile +ifeq ($(KNORA_GDB_LICENSE), unknown) + touch $(CURRENT_DIR)/knora-graphdb-se/target/universal/stage/scripts/graphdb.license +else + cp $(KNORA_GDB_LICENSE) $(CURRENT_DIR)/knora-graphdb-se/target/universal/stage/scripts/graphdb.license +endif docker build -t $(KNORA_GRAPHDB_SE_IMAGE) -t $(REPO_PREFIX)/$(KNORA_GRAPHDB_SE_REPO):latest -f .docker/knora-graphdb-se.dockerfile knora-graphdb-se/target/universal .PHONY: publish-knora-graphdb-se-image @@ -49,6 +56,7 @@ publish-knora-graphdb-se-image: build-knora-graphdb-se-image ## publish knora-gr build-knora-graphdb-free-image: build-all-scala ## build and publish knora-graphdb-free docker image locally @mkdir -p .docker @sed -e "s/@GRAPHDB_IMAGE@/daschswiss\/graphdb\:$(GRAPHDB_FREE_VERSION)-free/" docker/knora-graphdb.template.dockerfile > .docker/knora-graphdb-free.dockerfile + touch $(CURRENT_DIR)/knora-graphdb-free/target/universal/stage/scripts/graphdb.license docker build -t $(KNORA_GRAPHDB_FREE_IMAGE) -f .docker/knora-graphdb-free.dockerfile knora-graphdb-free/target/universal .PHONY: publish-knora-graphdb-free-image @@ -105,7 +113,7 @@ publish-all-images: publish-knora-api-image publish-knora-graphdb-se-image publi ################################# .PHONY: print-env-file -print-env-file: env-file ## prints the env file used by knora-stack +print-env-file: ## prints the env file used by knora-stack @cat .env .PHONY: env-file @@ -113,11 +121,9 @@ env-file: ## write the env file used by knora-stack. ifeq ($(KNORA_GDB_LICENSE), unknown) $(warning No GraphDB-SE license set. Using GraphDB-Free.) @echo KNORA_GRAPHDB_IMAGE=$(KNORA_GRAPHDB_FREE_IMAGE) > .env - @echo KNORA_GDB_LICENSE_FILE=no-license >> .env @echo KNORA_GDB_TYPE=graphdb-free >> .env else @echo KNORA_GRAPHDB_IMAGE=$(KNORA_GRAPHDB_SE_IMAGE) > .env - @echo KNORA_GDB_LICENSE_FILE=$(KNORA_GDB_LICENSE) >> .env @echo KNORA_GDB_TYPE=graphdb-se >> .env endif ifeq ($(KNORA_GDB_IMPORT), unknown) @@ -136,13 +142,21 @@ endif @echo KNORA_SIPI_IMAGE=$(KNORA_SIPI_IMAGE) >> .env @echo KNORA_API_IMAGE=$(KNORA_API_IMAGE) >> .env @echo KNORA_SALSAH1_IMAGE=$(KNORA_SALSAH1_IMAGE) >> .env + @echo DOCKERHOST=$(DOCKERHOST) >> .env @echo KNORA_WEBAPI_DB_CONNECTIONS=$(KNORA_WEBAPI_DB_CONNECTIONS) >> .env + @echo KNORA_GRAPHDB_REPOSITORY_NAME=$(KNORA_GRAPHDB_REPOSITORY_NAME) >> .env + @echo LOCAL_HOME=$(CURRENT_DIR) >> .env ## knora stack .PHONY: stack-up stack-up: build-all-images env-file ## starts the knora-stack: graphdb, sipi, redis, api, salsah1. docker-compose -f docker/knora.docker-compose.yml up -d +.PHONY: stack-up-ci +stack-up-ci: KNORA_GRAPHDB_REPOSITORY_NAME := knora-test-unit +stack-up-ci: build-all-images env-file print-env-file ## starts the knora-stack using 'knora-test-unit' repository: graphdb, sipi, redis, api, salsah1. + docker-compose -f docker/knora.docker-compose.yml up -d + .PHONY: stack-restart stack-restart: stack-up ## re-starts the knora-stack: graphdb, sipi, redis, api, salsah1. docker-compose -f docker/knora.docker-compose.yml restart @@ -175,14 +189,26 @@ stack-logs-redis: ## prints out and follows the logs of the 'redis' container ru stack-logs-api: ## prints out and follows the logs of the 'api' container running in knora-stack. docker-compose -f docker/knora.docker-compose.yml logs -f api +.PHONY: stack-logs-api-no-follow +stack-logs-api-no-follow: ## prints out the logs of the 'api' container running in knora-stack. + docker-compose -f docker/knora.docker-compose.yml logs api + .PHONY: stack-logs-salsah1 stack-logs-salsah1: ## prints out and follows the logs of the 'salsah1' container running in knora-stack. docker-compose -f docker/knora.docker-compose.yml logs -f salsah1 +.PHONY: stack-health +stack-health: + curl 0.0.0.0:3333/health + .PHONY: stack-down stack-down: ## stops the knora-stack. docker-compose -f docker/knora.docker-compose.yml down +.PHONY: stack-config +stack-config: + docker-compose -f docker/knora.docker-compose.yml config + ## stack without api .PHONY: stack-without-api stack-without-api: stack-up ## starts the knora-stack without knora-api: graphdb, sipi, redis, salsah1. @@ -194,11 +220,11 @@ stack-without-api-and-sipi: stack-up ## starts the knora-stack without knora-api docker-compose -f docker/knora.docker-compose.yml stop api docker-compose -f docker/knora.docker-compose.yml stop sipi -.PHONY: unit-tests -unit-tests: stack-without-api init-db-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest'). +.PHONY: test-unit +test-unit: stack-without-api init-db-test-unit ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest'). docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -209,14 +235,14 @@ unit-tests: stack-without-api init-db-test-unit ## runs the unit tests (equivale --network=docker_knora-net \ daschswiss/scala-sbt sbt 'webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest' -.PHONY: unit-tests-with-coverage -unit-tests-with-coverage: stack-without-api ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting. +.PHONY: test-unit-ci +test-unit-ci: stack-without-api ## runs the unit tests (equivalent to 'sbt webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting. @echo $@ # print target name @sleep 5 @$(MAKE) -f $(THIS_FILE) init-db-test-unit docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -227,11 +253,11 @@ unit-tests-with-coverage: stack-without-api ## runs the unit tests (equivalent t --network=docker_knora-net \ daschswiss/scala-sbt sbt coverage 'webapi/testOnly -- -l org.knora.webapi.testing.tags.E2ETest' webapi/coverageReport -.PHONY: e2e-tests -e2e-tests: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest'). +.PHONY: test-e2e +test-e2e: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest'). docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -242,14 +268,14 @@ e2e-tests: stack-without-api init-db-test-unit ## runs the e2e tests (equivalent --network=docker_knora-net \ daschswiss/scala-sbt sbt 'webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest' -.PHONY: e2e-tests-with-coverage -e2e-tests-with-coverage: stack-without-api ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting. +.PHONY: test-e2e-ci +test-e2e-ci: stack-without-api ## runs the e2e tests (equivalent to 'sbt webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest') with code-coverage reporting. @echo $@ # print target name @sleep 5 @$(MAKE) -f $(THIS_FILE) init-db-test-unit docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -260,11 +286,11 @@ e2e-tests-with-coverage: stack-without-api ## runs the e2e tests (equivalent to --network=docker_knora-net \ daschswiss/scala-sbt sbt coverage 'webapi/testOnly -- -n org.knora.webapi.testing.tags.E2ETest' webapi/coverageReport -.PHONY: it-tests -it-tests: stack-without-api init-db-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it'). +.PHONY: test-it +test-it: stack-without-api init-db-test-unit ## runs the integration tests (equivalent to 'sbt webapi/it'). docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -275,14 +301,14 @@ it-tests: stack-without-api init-db-test-unit ## runs the integration tests (equ --network=docker_knora-net \ daschswiss/scala-sbt sbt 'webapi/it:test' -.PHONY: it-tests-with-coverage -it-tests-with-coverage: stack-without-api ## runs the integration tests (equivalent to 'sbt webapi/it:test') with code-coverage reporting. +.PHONY: test-it-ci +test-it-ci: stack-without-api ## runs the integration tests (equivalent to 'sbt webapi/it:test') with code-coverage reporting. @echo $@ # print target name @sleep 5 @$(MAKE) -f $(THIS_FILE) init-db-test-unit docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -293,12 +319,12 @@ it-tests-with-coverage: stack-without-api ## runs the integration tests (equival --network=docker_knora-net \ daschswiss/scala-sbt sbt coverage webapi/it:test webapi/coverageReport -.PHONY: normal-tests -normal-tests: stack-without-api ## runs the normal tests (equivalent to 'sbt webapi/test'). +.PHONY: test-all +test-all: stack-without-api ## runs the all tests (equivalent to 'sbt webapi/test'). # docker build -t webapi-test -f docker/knora-api-test.dockerfile webapi/build/test/target/universal docker run --rm \ -v /tmp:/tmp \ - -v $(PWD):/src \ + -v $(CURRENT_DIR):/src \ -v $(HOME)/.ivy2:/root/.ivy2 \ --name=api \ -e KNORA_WEBAPI_TRIPLESTORE_HOST=db \ @@ -309,34 +335,69 @@ normal-tests: stack-without-api ## runs the normal tests (equivalent to 'sbt web --network=docker_knora-net \ daschswiss/scala-sbt sbt webapi/test +.PHONY: test-js-lib-integration +test-js-lib-integration: clean-local-tmp stack-without-api ## run knora-api-js-lib tests against the knora-stack + $(MAKE) -f $(THIS_FILE) print-env-file + $(MAKE) -f $(THIS_FILE) stack-config + @sleep 15 + @$(MAKE) -f $(THIS_FILE) init-db-test + @sleep 15 + @$(MAKE) -f $(THIS_FILE) stack-restart-api + sleep 15 + @$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow + @git clone --single-branch --depth 1 https://github.com/dasch-swiss/knora-api-js-lib.git $(CURRENT_DIR)/.tmp/js-lib + $(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib npm-install + $(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib test + .PHONY: init-db-test init-db-test: ## initializes the knora-test repository - $(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test .PHONY: init-db-test-minimal init-db-test-minimal: ## initializes the knora-test repository with minimal data - $(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test-minimal + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test-minimal .PHONY: init-db-test-unit init-db-test-unit: ## initializes the knora-test-unit repository - $(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test-unit + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test-unit + +.PHONY: init-db-test-unit-minimal +init-db-test-unit-minimal: ## initializes the knora-test-unit repository with minimal data + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-se-docker-init-knora-test-unit-minimal .PHONY: init-db-test-free init-db-test-free: ## initializes the knora-test repository (for GraphDB-Free) - $(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-free + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-free .PHONY: init-db-test-minimal-free init-db-test-minimal-free: ## initializes the knora-test repository with minimal data (for GraphDB-Free) - $(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-minimal + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-minimal .PHONY: init-db-test-unit-free init-db-test-unit-free: ## initializes the knora-test-unit repository (for GraphDB-Free) - $(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-unit + @echo $@ + @$(MAKE) -C webapi/scripts graphdb-free-docker-init-knora-test-unit + +################################# +# Other +################################# + +.PHONY: clean-local-tmp +clean-local-tmp: + @rm -rf $(CURRENT_DIR)/.tmp + @mkdir $(CURRENT_DIR)/.tmp clean: ## clean build artifacts @rm -rf .docker @rm -rf .env @sbt clean + @rm -rf .tmp clean-docker: ## cleans the docker installation docker system prune -af diff --git a/ci/.gitignore b/ci/.gitignore deleted file mode 100644 index 0d6c6ae576..0000000000 --- a/ci/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory -* -# Except this file and the encrypted graphdb license -!.gitignore -!decrypt_secrets.sh -!secrets.tar.gpg -!README.md diff --git a/ci/README.md b/ci/README.md deleted file mode 100644 index 12d2939100..0000000000 --- a/ci/README.md +++ /dev/null @@ -1,8 +0,0 @@ -How to encrypt and decrypt the license file used in CI: - -```shell script -$ tar -cf secrets.tar graphdb.license -$ gpg --cipher-algo AES256 --symmetric secrets.tar -$ gpg --quiet --batch --yes --decrypt --passphrase SECRET_PASSPHRASE --output=secrets.tar secrets.tar.gpg -$ tar -xvf secrets.tar -``` diff --git a/ci/secrets.tar.gpg b/ci/secrets.tar.gpg deleted file mode 100644 index 7833a4f0de..0000000000 Binary files a/ci/secrets.tar.gpg and /dev/null differ diff --git a/docker/knora-graphdb.template.dockerfile b/docker/knora-graphdb.template.dockerfile index d20cf63797..09f8db0b9f 100644 --- a/docker/knora-graphdb.template.dockerfile +++ b/docker/knora-graphdb.template.dockerfile @@ -1,5 +1,6 @@ FROM @GRAPHDB_IMAGE@ COPY stage/scripts/KnoraRules.pie /graphdb/KnoraRules.pie +COPY stage/scripts/graphdb.license /graphdb/graphdb.license LABEL MAINTAINER="400790+subotic@users.noreply.github.com" diff --git a/docker/knora.docker-compose.yml b/docker/knora.docker-compose.yml index 9e76489647..a436bb1e27 100644 --- a/docker/knora.docker-compose.yml +++ b/docker/knora.docker-compose.yml @@ -7,9 +7,8 @@ services: ports: - "7200:7200" volumes: - - ${KNORA_GDB_LICENSE_FILE}:/graphdb/graphdb.license - - ${KNORA_GDB_IMPORT_DIR}:/root/graphdb-import - - ${KNORA_GDB_HOME_DIR}:/opt/graphdb/home + - ${KNORA_GDB_IMPORT_DIR}:/root/graphdb-import:delegated + - ${KNORA_GDB_HOME_DIR}:/opt/graphdb/home:delegated networks: - knora-net environment: @@ -23,9 +22,9 @@ services: - "1024:1024" volumes: - /tmp:/tmp - - ${PWD}/sipi/config:/sipi/config - - ${PWD}/sipi/images:/sipi/images:delegated - - ${PWD}/sipi/server:/sipi/server:delegated + - ${LOCAL_HOME}/sipi/config:/sipi/config:delegated + - ${LOCAL_HOME}/sipi/images:/sipi/images:delegated + - ${LOCAL_HOME}/sipi/server:/sipi/server:delegated networks: - knora-net restart: unless-stopped @@ -60,7 +59,7 @@ services: - KNORA_WEBAPI_TRIPLESTORE_HOST=db - KNORA_WEBAPI_TRIPLESTORE_DBTYPE=${KNORA_GDB_TYPE} - KNORA_WEBAPI_SIPI_INTERNAL_HOST=sipi - - KNORA_WEBAPI_TRIPLESTORE_GRAPHDB_REPOSITORY_NAME=knora-test + - KNORA_WEBAPI_TRIPLESTORE_GRAPHDB_REPOSITORY_NAME=${KNORA_GRAPHDB_REPOSITORY_NAME} - KNORA_WEBAPI_CACHE_SERVICE_ENABLED=true - KNORA_WEBAPI_CACHE_SERVICE_REDIS_HOST=redis - KNORA_WEBAPI_CACHE_SERVICE_REDIS_PORT=6379 diff --git a/sipi/Makefile b/sipi/Makefile new file mode 100644 index 0000000000..aef4ee50b7 --- /dev/null +++ b/sipi/Makefile @@ -0,0 +1,28 @@ +include ../vars.mk + +.PHONY: print-env-file +print-env-file: env-file ## prints the env file used by sipi + @cat .env + +.PHONY: env-file +env-file: ## write the env file used by sipi + @echo SIPI_VERSION=$(SIPI_VERSION) > .env + @echo DOCKERHOST=$(DOCKERHOST) > .env + +.PHONY: up +up: env-file ## start sipi + docker-compose up -d sipi + +.PHONY: logs +logs: env-file ## follow logs + docker-compose logs -f sipi + +.PHONY: down +down: env-file ## stop sipi + docker-compose down + +.PHONY: help +help: ## this help + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort + +.DEFAULT_GOAL := help diff --git a/sipi/README.md b/sipi/README.md index 90d4baf843..8ff1135961 100644 --- a/sipi/README.md +++ b/sipi/README.md @@ -9,14 +9,10 @@ This directory holds Knora specific scripts and configurations for [Sipi](https: To start Sipi using Docker (with config for integration tests), run from inside this folder: ``` -$ export DOCKERHOST=LOCAL_IP_ADDRESS -$ docker image rm --force dhlabbasel/sipi:develop // deletes cached image and needs only to be used when newer image is available on dockerhub -$ docker-compose up sipi -$ docker-compose down // for cleanup +$ make up +$ make down // for cleanup ``` -where `LOCAL_IP_ADDRESS` is the IP of the host running the `Knora-Service`. - ### Using a Locally-Compiled Sipi Type the following in this directory, assuming that the Sipi source tree is in @@ -36,4 +32,6 @@ $ export DOCKERHOST=LOCAL_IP_ADDRESS $ docker image rm --force dhlabbasel/sipi:develop // deletes cached image and needs only to be used when newer image is available on dockerhub $ docker-compose up sipi-no-auth $ docker-compose down // for cleanup -``` \ No newline at end of file +``` + +where `LOCAL_IP_ADDRESS` is the IP of the host running the `Knora-Service`. \ No newline at end of file diff --git a/sipi/docker-compose.yml b/sipi/docker-compose.yml index 12d3c600c0..305f91830c 100644 --- a/sipi/docker-compose.yml +++ b/sipi/docker-compose.yml @@ -4,7 +4,7 @@ version: '3' services: # sipi using default (production-like) configuration with additional routes for testing sipi: - image: dhlabbasel/sipi:develop + image: dhlabbasel/sipi:${SIPI_VERSION} container_name: sipi ports: - "1024:1024" @@ -16,7 +16,7 @@ services: - $PWD/images:/sipi/images - $PWD/server:/sipi/server extra_hosts: - - api:$DOCKERHOST + - api:${DOCKERHOST} restart: unless-stopped environment: - SIPI_EXTERNAL_PROTOCOL=http @@ -26,7 +26,7 @@ services: # sipi using configuration which disables authentication sipi-no-auth: - image: dhlabbasel/sipi:develop + image: dhlabbasel/sipi:${SIPI_VERSION} container_name: sipi-no-auth ports: - "1024:1024" @@ -38,7 +38,7 @@ services: - $PWD/images:/sipi/images - $PWD/server:/sipi/server extra_hosts: - - api:$DOCKERHOST + - api:${DOCKERHOST} restart: unless-stopped environment: - SIPI_EXTERNAL_PROTOCOL=http diff --git a/vars.mk b/vars.mk index a264d6d876..dedffb0000 100644 --- a/vars.mk +++ b/vars.mk @@ -1,7 +1,7 @@ SIPI_VERSION := 2.0.1 GRAPHDB_SE_VERSION := 9.0.0 GRAPHDB_FREE_VERSION := 9.0.0 -GRAPHDB_HEAP_SIZE := 5G +_GRAPHDB_HEAP_SIZE := 5G REPO_PREFIX := daschswiss KNORA_API_REPO := knora-api @@ -12,6 +12,7 @@ KNORA_ASSETS_REPO := knora-assets KNORA_UPGRADE_REPO := knora-upgrade KNORA_SALSAH1_REPO := knora-salsah1 KNORA_WEBAPI_DB_CONNECTIONS := 2 +KNORA_GRAPHDB_REPOSITORY_NAME := knora-test ifeq ($(BUILD_TAG),) BUILD_TAG := $(shell git describe --tag --dirty --abbrev=7) @@ -64,8 +65,15 @@ ifeq ($(KNORA_GDB_HOME),) KNORA_GDB_HOME := unknown endif -ifeq ($(GDB_HEAP_SIZE),) +ifeq ($(GRAPHDB_HEAP_SIZE),) + KNORA_GDB_HEAP_SIZE := $(_GRAPHDB_HEAP_SIZE) +else KNORA_GDB_HEAP_SIZE := $(GRAPHDB_HEAP_SIZE) +endif + +UNAME := $(shell uname) +ifeq ($(UNAME),Darwin) + DOCKERHOST := $(shell ifconfig en0 | grep inet | grep -v inet6 | cut -d ' ' -f2) else - KNORA_GDB_HEAP_SIZE := $(GDB_HEAP_SIZE) -endif \ No newline at end of file + DOCKERHOST := $(shell ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+') +endif diff --git a/webapi/scripts/Makefile b/webapi/scripts/Makefile index 979803884c..64feb50f74 100644 --- a/webapi/scripts/Makefile +++ b/webapi/scripts/Makefile @@ -10,6 +10,10 @@ graphdb-se-docker-init-knora-test-minimal: ## initializes docker GraphDB-SE with graphdb-se-docker-init-knora-test-unit: ## initializes docker GraphDB-SE with the knora-test-unit repository ./graphdb-se-docker-init-knora-test-unit.sh +.PHONY: graphdb-se-docker-init-knora-test-unit-minimal +graphdb-se-docker-init-knora-test-unit-minimal: ## initializes docker GraphDB-SE with the knora-test-unit repository and minimal data + ./graphdb-se-docker-init-knora-test-unit-minimal.sh + .PHONY: graphdb-free-docker-init-knora-test graphdb-free-docker-init-knora-test: ## initializes docker GraphDB-Free with the knora-test repository ./graphdb-free-docker-init-knora-test.sh diff --git a/webapi/scripts/graphdb-knora-test-unit-data-minimal.expect b/webapi/scripts/graphdb-knora-test-unit-data-minimal.expect new file mode 100755 index 0000000000..00ef6f474a --- /dev/null +++ b/webapi/scripts/graphdb-knora-test-unit-data-minimal.expect @@ -0,0 +1,28 @@ +#!/usr/bin/expect + +set timeout 120 +spawn openrdf-console/bin/console.sh --force --echo --serverURL [lindex $argv 0] +set prompt "> " +expect $prompt +send "open knora-test-unit .\r" +expect $prompt +send "load ../../knora-ontologies/knora-admin.ttl into http://www.knora.org/ontology/knora-admin .\r" +expect $prompt +send "load ../../knora-ontologies/knora-base.ttl into http://www.knora.org/ontology/knora-base .\r" +expect $prompt +send "load ../../knora-ontologies/standoff-onto.ttl into http://www.knora.org/ontology/standoff .\r" +expect $prompt +send "load ../../knora-ontologies/standoff-data.ttl into http://www.knora.org/data/standoff .\r" +expect $prompt +send "load ../../knora-ontologies/salsah-gui.ttl into http://www.knora.org/ontology/salsah-gui .\r" +expect $prompt +send "load ../_test_data/all_data/admin-data.ttl into http://www.knora.org/data/admin .\r" +expect $prompt +send "load ../_test_data/all_data/permissions-data.ttl into http://www.knora.org/data/permissions .\r" +expect $prompt +send "load ../_test_data/all_data/system-data.ttl into http://www.knora.org/data/0000/SystemProject .\r" +expect $prompt + +send "close .\r" +expect $prompt +send "exit .\r" diff --git a/webapi/scripts/graphdb-se-docker-init-knora-test-unit-minimal.sh b/webapi/scripts/graphdb-se-docker-init-knora-test-unit-minimal.sh new file mode 100755 index 0000000000..9612071a23 --- /dev/null +++ b/webapi/scripts/graphdb-se-docker-init-knora-test-unit-minimal.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +GRAPHDB="http://localhost:7200" +CONSOLE="openrdf-console/bin/console.sh --force --echo --serverURL $GRAPHDB" + +GREEN='\033[0;32m' +RED='\033[0;31m' +NO_COLOUR='\033[0m' +DELIMITER="****************************************************************************************************\n* " + +printf "${GREEN}${DELIMITER}Deleting repository${NO_COLOUR}\n\n" + +cat graphdb-drop-knora-test-unit-repository.ttl | $CONSOLE + +printf "\n${GREEN}${DELIMITER}Creating repository${NO_COLOUR}\n\n" + +sed -e 's@PIE_FILE@'"/graphdb/KnoraRules.pie"'@' graphdb-se-knora-test-unit-repository-config.ttl.tmpl > graphdb-se-knora-test-unit-repository-config.ttl + +curl -X POST -H "Content-Type:text/turtle" -T graphdb-se-knora-test-unit-repository-config.ttl "$GRAPHDB/repositories/SYSTEM/rdf-graphs/service?graph=http://www.knora.org/config-test-unit" + +curl -X POST -H "Content-Type:text/turtle" -d " a ." $GRAPHDB/repositories/SYSTEM/statements + +printf "${GREEN}Repository created.\n\n${DELIMITER}Creating Lucene Index${NO_COLOUR}\n\n" + +STATUS=$(curl -s -w '%{http_code}' -S -X POST --data-urlencode 'update@./graphdb-knora-index-create.rq' $GRAPHDB/repositories/knora-test-unit/statements) + +if [ "$STATUS" == "204" ] +then + printf "${GREEN}Lucene index built.${NO_COLOUR}\n\n" +else + printf "${RED}Building of Lucene index failed: ${STATUS}${NO_COLOUR}\n\n" +fi + +printf "${GREEN}Loading Data${NO_COLOUR}\n\n" + +./graphdb-knora-test-unit-data-minimal.expect $GRAPHDB + +printf "${GREEN}Data Loaded.\n\n${DELIMITER}Updating Lucene Index${NO_COLOUR}\n\n" + +curl -X POST --data-urlencode 'update@./graphdb-knora-index-update.rq' $GRAPHDB/repositories/knora-test-unit/statements + +printf "${GREEN}Lucene Index Updated.${NO_COLOUR}" diff --git a/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala b/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala index c467433fa2..c6494f511e 100644 --- a/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala +++ b/webapi/src/main/scala/org/knora/webapi/app/ApplicationActor.scala @@ -142,7 +142,13 @@ class ApplicationActor extends Actor with LazyLogging with AroundDirectives with def receive: PartialFunction[Any, Unit] = { - /* Entry point for startup sequence */ + /* Called from main. Initiates application startup. */ + case AppStart(withOntologies, requiresSipi) => appStart(withOntologies, requiresSipi) + + /* Usually only called from tests */ + case AppStop() => appStop() + + /* Called from the "appStart" method. Entry point for startup sequence. */ case InitStartUp(skipLoadingOfOntologies, requiresIIIFService) => { logger.info("Startup initiated, please wait ...") @@ -316,8 +322,8 @@ class ApplicationActor extends Actor with LazyLogging with AroundDirectives with logger.warn("Redis-Server not running. Please start the Redis-Server.") timers.startSingleTimer("CheckCacheService", CheckCacheService, 5.seconds) - case AppStart(withOntologies, requiresSipi) => appStart(withOntologies, requiresSipi) - case AppStop() => appStop() + + case responderMessage: KnoraRequestV1 => responderManager forward responderMessage case responderMessage: KnoraRequestV2 => responderManager forward responderMessage @@ -330,6 +336,8 @@ class ApplicationActor extends Actor with LazyLogging with AroundDirectives with /** * All routes composed together and CORS activated. + * ALL requests go through each of the routes in ORDER. + * The FIRST matching route is used for handling a request. */ private val apiRoutes: Route = logDuration { addServerHeader { diff --git a/webapi/src/main/scala/org/knora/webapi/routing/HealthRoute.scala b/webapi/src/main/scala/org/knora/webapi/routing/HealthRoute.scala index ed22c6ed83..bc8e492665 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/HealthRoute.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/HealthRoute.scala @@ -43,7 +43,7 @@ case class HealthCheckResult(name: String, trait HealthCheck { this: HealthRoute => - override implicit val timeout: Timeout = 1.second + override implicit val timeout: Timeout = 2997.millis protected def healthcheck(): Future[HttpResponse] = for { diff --git a/webapi/src/main/scala/org/knora/webapi/routing/RejectingRoute.scala b/webapi/src/main/scala/org/knora/webapi/routing/RejectingRoute.scala index 9aae6f73a3..6c6c32b7bc 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/RejectingRoute.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/RejectingRoute.scala @@ -34,7 +34,7 @@ import scala.util.{Failure, Success} trait AppStateAccess { this: RejectingRoute => - override implicit val timeout: Timeout = 1.second + override implicit val timeout: Timeout = 2998.millis protected def getAppState(): Future[AppState] = for {