Merge pull request #49 from bento-platform/features/remove-tables

feat!: removed tables

.github/workflows/api.build.yml

@@ -30,7 +30,8 @@ jobs:
           cp ./etc/example.env .env
           source .env
           echo "GOHAN_API_BUILDER_BASE_IMAGE=$GOHAN_API_BUILDER_BASE_IMAGE" >> $GITHUB_ENV
-          echo "GOHAN_API_BASE_IMAGE=$GOHAN_API_BASE_IMAGE" >> $GITHUB_ENV
+          echo "GOHAN_API_DEV_BASE_IMAGE=$GOHAN_API_DEV_BASE_IMAGE" >> $GITHUB_ENV
+          echo "GOHAN_API_PROD_BASE_IMAGE=$GOHAN_API_PROD_BASE_IMAGE" >> $GITHUB_ENV
 
       - name: Load environment variables from .env file
         uses: xom9ikk/dotenv@v2
@@ -41,10 +42,11 @@ jobs:
           context: "{{defaultContext}}:src/api"
           build-args: |
             BUILDER_BASE_IMAGE=${{ env.GOHAN_API_BUILDER_BASE_IMAGE }}
-            BASE_IMAGE=${{ env.GOHAN_API_BASE_IMAGE }}
+            BASE_DEV_IMAGE=${{ env.GOHAN_API_DEV_BASE_IMAGE }}
+            BASE_PROD_IMAGE=${{ env.GOHAN_API_PROD_BASE_IMAGE }}
           registry: ghcr.io
           registry-username: ${{ github.actor }}
           registry-password: ${{ secrets.GITHUB_TOKEN }}
           image-name: ghcr.io/bento-platform/gohan-api
-          development-dockerfile: Dockerfile
+          development-dockerfile: dev.Dockerfile
           dockerfile: Dockerfile

.github/workflows/api.test.yml

@@ -34,7 +34,6 @@ jobs:
       #   with:
       #     context: "{{defaultContext}}:src/api"
       #     build-args: |
-      #       BUILDER_BASE_IMAGE=${{ env.GOHAN_API_BUILDER_BASE_IMAGE }}
       #       BASE_IMAGE=${{ env.GOHAN_API_BASE_IMAGE }}
       #     registry: ghcr.io
       #     registry-username: ${{ github.actor }}
@@ -45,4 +44,4 @@ jobs:
       - name: API Test
         run: |
           sudo apt-get install -y tabix
-          make test-api
+          make test-api

.gitignore

@@ -1,4 +1,5 @@
-.vscode
+.vscode/*
+!.vscode/launch.json
 .DS_store
 .idea
 
@@ -12,6 +13,7 @@ data/
 data-x/
 
 # vcfs
+vcfs/*
 */vcfs/*.txt
 */vcfs/*.vcf
 */vcfs/*.vcf.gz

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach to PID (Bento)",
+            "type": "go",
+            "request": "attach",
+            "mode": "local",
+            "processId": 0,
+        }
+    ]
+}

README.md

@@ -77,25 +77,10 @@
   # view catalogue
   curl -k https://gohan.local/genes/overview
 
-
-  # create table
-  DATA='{
-      "name": "Gohan Box Test Table",
-      "data_type": "variant",
-      "dataset": "00000000-0000-0000-0000-000000000000",
-      "metadata": {}
-  }'
-  curl -k -0 -v -X POST https://gohan.local/tables \
-    -H 'Content-Type:application/json' \
-    --data "$(echo $DATA)" | jq
-
-  # <obtain the table "id">
-
-
   # move vcf.gz files to `$GOHAN_API_VCF_PATH`
 
   # ingest vcf.gz
-  curl -k https://gohan.local/variants/ingestion/run\?fileNames=<filename>\&assemblyId=GRCh37\&filterOutReferences=true\&tableId=<table id>
+  curl -k https://gohan.local/variants/ingestion/run\?fileNames=<filename>\&assemblyId=GRCh37\&filterOutReferences=true\&dataset=00000000-0000-0000-0000-000000000000
   
   # monitor progress:
   curl -k https://gohan.local/variants/ingestion/requests
@@ -483,126 +468,6 @@ Response
 <br />
 
 
-**`/tables`**
-
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables`<br/>
-
-<br/>
-
-Response
->```json  
-> [
->   {
-> 	  "id":             `string`,
->  	  "name":           `string`,
-> 	  "data_type":      `string`,
-> 	  "dataset":        `string`,
-> 	  "assembly_ids": `[]string`,
-> 	  "metadata":        {...},
-> 	  "schema":          {...},
->   },
->   ...
-> ]
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**POST** `/tables`<br/>
->```json
-> {
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "metadata":        {...},
-> }
-> ```
-
-<br/>
-
-Response
->```json
-> {
->    "id":             `string`,
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "assembly_ids": `[]string`,
->    "metadata":        {...},
->    "schema":          {...},
-> }
-> ```
-
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables/:id`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
->```json
-> {
->    "id":             `string`,
->    "name":           `string`,
->    "data_type":      `string`,
->    "dataset":        `string`,
->    "assembly_ids": `[]string`,
->    "metadata":        {...},
->    "schema":          {...},
-> }
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**GET** `/tables/:id/summary`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
->```json
-> {
->    "count":               `int`,
->    "data_type_specific":  {...},
-> }
-> ```
-
-<br />
-<br />
-
-
-Request
-> &nbsp;&nbsp;**DELETE** `/tables/:id`<br/>
-> &nbsp;&nbsp;&nbsp;path params: 
->   - id : **string (UUID)** `(required)`
-
-<br/>
-
-Response
-
-`Status Code:` **204**
-
-<br />
-<br />
-
-
-
 
 ## Deployments :
 
@@ -630,3 +495,11 @@ Once `elasticsearch`, `drs`, the `api`, and the `gateway` are up, run
 make test-api-dev
 ```
 
+## Dev Container debug
+
+Interactive debug in VSCode is only possible When using the development image of gohan-api.
+
+Using the "Attach to PID(Bento)" debug config, select the PID associated with the following path:
+```
+/gohan-api/src/api/tmp/main
+```

docker-compose.yaml

@@ -33,7 +33,8 @@ services:
       context: $PWD/src/api
       args:
         BUILDER_BASE_IMAGE: ${GOHAN_API_BUILDER_BASE_IMAGE}
-        BASE_IMAGE: ${GOHAN_API_BASE_IMAGE}
+        BASE_PROD_IMAGE: ${GOHAN_API_PROD_BASE_IMAGE}
+        BASE_DEV_IMAGE: ${GOHAN_API_PROD_BASE_IMAGE}
         # also passed in as an ENV from within Dockerfile :
         GOHAN_API_INTERNAL_PORT: ${GOHAN_API_INTERNAL_PORT} 
     networks:
@@ -151,6 +152,7 @@ services:
       - DATABASE=/drs/bento_drs/data/db/  # slightly confused naming, folder for database to go in
       - DATA=/drs/bento_drs/data/obj/  # DRS file objects, vs. the database
       - INTERNAL_PORT=${GOHAN_DRS_INTERNAL_PORT}
+      - AUTHZ_ENABLED=False
     volumes:
       - ${GOHAN_DRS_DATA_DIR}:/drs/bento_drs/data
       - ${GOHAN_API_DRS_BRIDGE_HOST_DIR}:${GOHAN_DRS_API_DRS_BRIDGE_DIR_CONTAINERIZED}

etc/example.env

@@ -39,7 +39,8 @@ GOHAN_API_IMAGE=gohan-api
 GOHAN_API_VERSION=latest
 
 GOHAN_API_BUILDER_BASE_IMAGE=golang:1.20-bullseye
-GOHAN_API_BASE_IMAGE=ghcr.io/bento-platform/bento_base_image:plain-debian-2023.03.06
+GOHAN_API_DEV_BASE_IMAGE=ghcr.io/bento-platform/bento_base_image:golang-debian-2023.08.30
+GOHAN_API_PROD_BASE_IMAGE=ghcr.io/bento-platform/bento_base_image:plain-debian-2023.08.30
 
 GOHAN_API_CONTAINER_NAME=gohan-api
 GOHAN_API_SERVICE_HOST=0.0.0.0
@@ -114,7 +115,7 @@ GOHAN_KB_ES_PORT=9200
 
 # DRS
 GOHAN_DRS_IMAGE=ghcr.io/bento-platform/bento_drs
-GOHAN_DRS_VERSION=0.9.0
+GOHAN_DRS_VERSION=0.12.3
 GOHAN_DRS_CONTAINER_NAME=gohan-drs
 GOHAN_DRS_INTERNAL_PORT=5000
 GOHAN_DRS_EXTERNAL_PORT=6000
@@ -164,4 +165,4 @@ GOHAN_PUBLIC_URL=${GOHAN_PUBLIC_PROTO}://${GOHAN_PUBLIC_HOSTNAME}:${GOHAN_PUBLIC
 
 GOHAN_ES_PUBLIC_URL=${GOHAN_PUBLIC_PROTO}://${GOHAN_PUBLIC_ES_SUBDOMAIN}${GOHAN_PUBLIC_HOSTNAME}:${GOHAN_PUBLIC_PORT}
 GOHAN_DRS_PUBLIC_URL=${GOHAN_PUBLIC_PROTO}://${GOHAN_PUBLIC_DRS_SUBDOMAIN}${GOHAN_PUBLIC_HOSTNAME}
-GOHAN_KB_PUBLIC_URL=${GOHAN_PUBLIC_PROTO}://${GOHAN_PUBLIC_KB_SUBDOMAIN}${GOHAN_PUBLIC_HOSTNAME}:${GOHAN_PUBLIC_PORT}
+GOHAN_KB_PUBLIC_URL=${GOHAN_PUBLIC_PROTO}://${GOHAN_PUBLIC_KB_SUBDOMAIN}${GOHAN_PUBLIC_HOSTNAME}:${GOHAN_PUBLIC_PORT}

src/api/Dockerfile

@@ -1,5 +1,5 @@
 ARG BUILDER_BASE_IMAGE
-ARG BASE_IMAGE
+ARG BASE_PROD_IMAGE
 
 # Stage 1 - builder
 FROM $BUILDER_BASE_IMAGE as builder
@@ -15,9 +15,7 @@ COPY . .
 RUN go mod vendor && \
     go build -ldflags="-s -w" -o gohan_api
 
-
-# Stage two - executioner
-FROM $BASE_IMAGE
+FROM $BASE_PROD_IMAGE
 
 # Debian updates
 #  - tabix for indexing VCFs

src/api/contexts/contexts.go

@@ -7,6 +7,7 @@ import (
 	variantsService "gohan/api/services/variants"
 
 	es7 "github.com/elastic/go-elasticsearch/v7"
+	"github.com/google/uuid"
 	"github.com/labstack/echo"
 )
 
@@ -29,6 +30,8 @@ type (
 		Chromosome string
 		Genotype   constants.GenotypeQuery
 		SampleIds  []string
+		Dataset    uuid.UUID
+		DataType   string
 		PositionBounds
 	}
 

src/api/dev.Dockerfile

@@ -0,0 +1,39 @@
+ARG BASE_DEV_IMAGE
+
+FROM $BASE_DEV_IMAGE
+
+LABEL org.opencontainers.image.description="Local development image for Bento Gohan."
+LABEL devcontainer.metadata='[{ \
+  "customizations": { \
+    "vscode": { \
+      "extensions": ["golang.go", "eamodio.gitlens"], \
+      "settings": {"workspaceFolder": "/gohan-api"} \
+    } \
+  } \
+}]'
+
+# Maintainer
+LABEL maintainer="Brennan Brouillette <[email protected]>"
+
+WORKDIR /app
+
+# Debian updates
+#  - tabix for indexing VCFs
+#  - other base dependencies provided by the base image
+RUN apt-get update -y && \
+    apt-get upgrade -y && \
+    apt-get install -y tabix && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN go install github.com/cosmtrek/air@latest
+
+COPY go.mod go.sum ./
+RUN go mod download && go mod vendor
+
+# Repository mounted to the container
+WORKDIR /gohan-api/src/api
+COPY entrypoint.bash .
+COPY run.dev.bash .
+
+ENTRYPOINT [ "bash", "./entrypoint.bash" ]
+CMD [ "air" ]

src/api/entrypoint.bash

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+cd /gohan-api || exit
+
+# Create bento_user and home
+source /create_service_user.bash
+
+# Create dev build directory
+mkdir -p src/api/tmp
+
+# Set permissions / groups
+chown -R bento_user:bento_user ./
+chown -R bento_user:bento_user /app
+chmod -R o-rwx src/api/tmp
+
+# Drop into bento_user from root and execute the CMD specified for the image
+exec gosu bento_user "$@"