Merge pull request #54 from bento-platform/releases/v4.0

v4.0

.github/workflows/api.build.yml

@@ -30,7 +30,8 @@ jobs:
           cp ./etc/example.env .env
           source .env
           echo "GOHAN_API_BUILDER_BASE_IMAGE=$GOHAN_API_BUILDER_BASE_IMAGE" >> $GITHUB_ENV
-          echo "GOHAN_API_BASE_IMAGE=$GOHAN_API_BASE_IMAGE" >> $GITHUB_ENV
+          echo "GOHAN_API_DEV_BASE_IMAGE=$GOHAN_API_DEV_BASE_IMAGE" >> $GITHUB_ENV
+          echo "GOHAN_API_PROD_BASE_IMAGE=$GOHAN_API_PROD_BASE_IMAGE" >> $GITHUB_ENV
 
       - name: Load environment variables from .env file
         uses: xom9ikk/dotenv@v2
@@ -41,10 +42,11 @@ jobs:
           context: "{{defaultContext}}:src/api"
           build-args: |
             BUILDER_BASE_IMAGE=${{ env.GOHAN_API_BUILDER_BASE_IMAGE }}
-            BASE_IMAGE=${{ env.GOHAN_API_BASE_IMAGE }}
+            BASE_DEV_IMAGE=${{ env.GOHAN_API_DEV_BASE_IMAGE }}
+            BASE_PROD_IMAGE=${{ env.GOHAN_API_PROD_BASE_IMAGE }}
           registry: ghcr.io
           registry-username: ${{ github.actor }}
           registry-password: ${{ secrets.GITHUB_TOKEN }}
           image-name: ghcr.io/bento-platform/gohan-api
-          development-dockerfile: Dockerfile
+          development-dockerfile: dev.Dockerfile
           dockerfile: Dockerfile

.github/workflows/api.test.yml

@@ -0,0 +1,47 @@
+name: Test gohan-api
+
+on:
+  push:
+    branches:
+      - "**" # TEMP
+  pull_request:
+    branches:
+      - "features/**"  # temp
+
+jobs:
+  build-test:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Gather default environment variables
+        id: envx
+        run: |
+          cp ./etc/example.env .env
+          source .env
+
+      - name: Load environment variables from .env file
+        uses: xom9ikk/dotenv@v2
+
+      # - name: Run Bento build action
+      #   uses: bento-platform/[email protected]
+      #   with:
+      #     context: "{{defaultContext}}:src/api"
+      #     build-args: |
+      #       BASE_IMAGE=${{ env.GOHAN_API_BASE_IMAGE }}
+      #     registry: ghcr.io
+      #     registry-username: ${{ github.actor }}
+      #     registry-password: ${{ secrets.GITHUB_TOKEN }}
+      #     image-name: ghcr.io/bento-platform/gohan-api
+      #     development-dockerfile: Dockerfile
+      #     dockerfile: Dockerfile
+      - name: API Test
+        run: |
+          sudo apt-get install -y tabix
+          make test-api

.gitignore

@@ -1,4 +1,5 @@
-.vscode
+.vscode/*
+!.vscode/launch.json
 .DS_store
 .idea
 
@@ -12,6 +13,7 @@ data/
 data-x/
 
 # vcfs
+vcfs/*
 */vcfs/*.txt
 */vcfs/*.vcf
 */vcfs/*.vcf.gz

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach to PID (Bento)",
+            "type": "go",
+            "request": "attach",
+            "mode": "local",
+            "processId": 0,
+        }
+    ]
+}

Makefile

@@ -51,27 +51,29 @@ init-vendor:
 init-data-dirs:
 	mkdir -p ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
 	chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
-	chmod -R 770 ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
+	chmod -R 777 ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
 
 	mkdir -p ${GOHAN_DRS_DATA_DIR}
+	mkdir -p ${GOHAN_DRS_DATA_DIR}/db
+	mkdir -p ${GOHAN_DRS_DATA_DIR}/obj
 	chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_DRS_DATA_DIR}
-	chmod -R 770 ${GOHAN_DRS_DATA_DIR}
+	chmod -R 777 ${GOHAN_DRS_DATA_DIR}
 
 	mkdir -p ${GOHAN_ES_DATA_DIR}
 	chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_ES_DATA_DIR}
-	chmod -R 770 ${GOHAN_ES_DATA_DIR}
+	chmod -R 777 ${GOHAN_ES_DATA_DIR}
 
 	@# tmp:
 	@# (setup for when gohan needs to preprocess vcf's at ingestion time):
 	mkdir -p ${GOHAN_API_VCF_PATH}
 	mkdir -p ${GOHAN_API_VCF_PATH}/tmp
 	chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_VCF_PATH}
-	chmod -R 770 ${GOHAN_API_VCF_PATH}/tmp
+	chmod -R 777 ${GOHAN_API_VCF_PATH}
 
 	mkdir -p ${GOHAN_API_GTF_PATH}
 	mkdir -p ${GOHAN_API_GTF_PATH}/tmp
 	chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_GTF_PATH}
-	chmod -R 770 ${GOHAN_API_GTF_PATH}/tmp
+	chmod -R 777 ${GOHAN_API_GTF_PATH}
 
 	@echo ".. done!"
 
@@ -196,17 +198,50 @@ clean-api-drs-bridge-data:
 
 
 ## Tests
-test-api-dev: prepare-test-config
+test-api: init prepare-test-config
+	# # @# Run the tests directly from the api source directory
+	# # cd src/api && \
+	# # go clean -cache && \
+	# # go test ./tests/unit/... -v
+
+	@# restart any running containers and print
+	docker compose -f docker-compose.test.yaml down
+	docker compose -f docker-compose.test.yaml up -d
+
+	@# run build tests
+	@# - print api and drs logs in the
+	@#   event of a failued
+	cd src/api && \
+	go clean -cache && \
+	(go test ./tests/build/... -v || ((docker logs gohan-api | tail -n 100) && (docker logs gohan-drs | tail -n 100) && exit 1)) && \
+	cd ../..
+
+	@# shut down the containers and print
+	@# the tail end of the
+	@# api and elasticsearch logs
+	docker compose -f docker-compose.test.yaml stop
+	docker logs gohan-api | tail -n 50
+	docker logs elasticsearch | tail -n 50
+
+
+test-api-dev: prepare-dev-config
 	@# Run the tests
 	cd src/api && \
 	go clean -cache && \
 	go test ./tests/integration/... -v
 
+
 prepare-test-config:
+	@# Prepare environment variables dynamically via a JSON file 
+	@# since xUnit doens't support loading env variables natively
+	envsubst < ./etc/test.yml.tpl > ./src/api/tests/common/test.config.yml
+
+prepare-dev-config:
 	@# Prepare environment variables dynamically via a JSON file 
 	@# since xUnit doens't support loading env variables natively
 	envsubst < ./etc/test.config.yml.tpl > ./src/api/tests/common/test.config.yml
 
+
 clean-tests:
 	@# Clean up
 	rm ./src/api/tests/common/test.config.yml

README.md

@@ -77,25 +77,10 @@
   # view catalogue
   curl -k https://gohan.local/genes/overview
 
-
-  # create table
-  DATA='{
-      "name": "Gohan Box Test Table",
-      "data_type": "variant",
-      "dataset": "00000000-0000-0000-0000-000000000000",
-      "metadata": {}
-  }'
-  curl -k -0 -v -X POST https://gohan.local/tables \
-    -H 'Content-Type:application/json' \
-    --data "$(echo $DATA)" | jq
-
-  # <obtain the table "id">
-
-
   # move vcf.gz files to `$GOHAN_API_VCF_PATH`
 
   # ingest vcf.gz
-  curl -k https://gohan.local/variants/ingestion/run\?fileNames=<filename>\&assemblyId=GRCh37\&filterOutReferences=true\&tableId=<table id>
+  curl -k https://gohan.local/variants/ingestion/run\?fileNames=<filename>\&assemblyId=GRCh37\&filterOutReferences=true\&dataset=00000000-0000-0000-0000-000000000000
   
   # monitor progress:
   curl -k https://gohan.local/variants/ingestion/requests
@@ -483,126 +468,6 @@ Response
 <br />
 
 
-**`/tables`**
-
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables`<br/>
-
-<br/>
-
-Response
->```json  
-> [
->   {
-> 	  "id":             `string`,
->  	  "name":           `string`,
-> 	  "data_type":      `string`,
-> 	  "dataset":        `string`,
-> 	  "assembly_ids": `[]string`,
-> 	  "metadata":        {...},
-> 	  "schema":          {...},
->   },
->   ...
-> ]
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**POST** `/tables`<br/>
->```json
-> {
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "metadata":        {...},
-> }
-> ```
-
-<br/>
-
-Response
->```json
-> {
->    "id":             `string`,
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "assembly_ids": `[]string`,
->    "metadata":        {...},
->    "schema":          {...},
-> }
-> ```
-
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables/:id`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
->```json
-> {
->    "id":             `string`,
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "assembly_ids": `[]string`,
->    "metadata":        {...},
->    "schema":          {...},
-> }
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables/:id/summary`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
->```json
-> {
->    "count":               `int`,
->    "data_type_specific":  {...},
-> }
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**DELETE** `/tables/:id`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
-
-`Status Code:` **204**
-
-<br />
-<br />
-
-
-
 
 ## Deployments :
 
@@ -630,3 +495,11 @@ Once `elasticsearch`, `drs`, the `api`, and the `gateway` are up, run
 make test-api-dev
 ```
 
+## Dev Container debug
+
+Interactive debug in VSCode is only possible When using the development image of gohan-api.
+
+Using the "Attach to PID(Bento)" debug config, select the PID associated with the following path:
+```
+/gohan-api/src/api/tmp/main
+```

docker-compose.test.yaml

@@ -0,0 +1,28 @@
+# # Test-only (!) compose file for the Gohan
+
+networks:
+  bridge-net:
+    external: true
+
+services:
+  api:
+    ports:
+      - "${GOHAN_API_EXTERNAL_PORT}:${GOHAN_API_INTERNAL_PORT}"
+    extends:
+      file: docker-compose.yaml
+      service: api
+
+
+  elasticsearch:
+    ports:
+      - ${GOHAN_ES_EXTERNAL_PORT_1}:${GOHAN_ES_INTERNAL_PORT_1}
+    extends:
+      file: docker-compose.yaml
+      service: elasticsearch
+
+  drs:
+    ports:
+      - "${GOHAN_DRS_EXTERNAL_PORT}:${GOHAN_DRS_INTERNAL_PORT}"
+    extends:
+      file: docker-compose.yaml
+      service: drs