Skip to content

Commit

Permalink
Merge pull request #52 from bento-platform/features/test-upgrade
Browse files Browse the repository at this point in the history
[WIP] Test upgrades
  • Loading branch information
brouillette authored Jul 7, 2023
2 parents d66ed4f + 7f1cb52 commit 5c13d59
Show file tree
Hide file tree
Showing 19 changed files with 1,508 additions and 880 deletions.
48 changes: 48 additions & 0 deletions .github/workflows/api.test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Test gohan-api

on:
push:
branches:
- "**" # TEMP
pull_request:
branches:
- "features/**" # temp

jobs:
build-test:
runs-on: ubuntu-latest

permissions:
contents: read
packages: write

steps:
- name: Checkout
uses: actions/checkout@v3

- name: Gather default environment variables
id: envx
run: |
cp ./etc/example.env .env
source .env
- name: Load environment variables from .env file
uses: xom9ikk/dotenv@v2

# - name: Run Bento build action
# uses: bento-platform/[email protected]
# with:
# context: "{{defaultContext}}:src/api"
# build-args: |
# BUILDER_BASE_IMAGE=${{ env.GOHAN_API_BUILDER_BASE_IMAGE }}
# BASE_IMAGE=${{ env.GOHAN_API_BASE_IMAGE }}
# registry: ghcr.io
# registry-username: ${{ github.actor }}
# registry-password: ${{ secrets.GITHUB_TOKEN }}
# image-name: ghcr.io/bento-platform/gohan-api
# development-dockerfile: Dockerfile
# dockerfile: Dockerfile
- name: API Test
run: |
sudo apt-get install -y tabix
make test-api
47 changes: 41 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,27 +51,29 @@ init-vendor:
init-data-dirs:
mkdir -p ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
chmod -R 770 ${GOHAN_API_DRS_BRIDGE_HOST_DIR}
chmod -R 777 ${GOHAN_API_DRS_BRIDGE_HOST_DIR}

mkdir -p ${GOHAN_DRS_DATA_DIR}
mkdir -p ${GOHAN_DRS_DATA_DIR}/db
mkdir -p ${GOHAN_DRS_DATA_DIR}/obj
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_DRS_DATA_DIR}
chmod -R 770 ${GOHAN_DRS_DATA_DIR}
chmod -R 777 ${GOHAN_DRS_DATA_DIR}

mkdir -p ${GOHAN_ES_DATA_DIR}
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_ES_DATA_DIR}
chmod -R 770 ${GOHAN_ES_DATA_DIR}
chmod -R 777 ${GOHAN_ES_DATA_DIR}

@# tmp:
@# (setup for when gohan needs to preprocess vcf's at ingestion time):
mkdir -p ${GOHAN_API_VCF_PATH}
mkdir -p ${GOHAN_API_VCF_PATH}/tmp
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_VCF_PATH}
chmod -R 770 ${GOHAN_API_VCF_PATH}/tmp
chmod -R 777 ${GOHAN_API_VCF_PATH}

mkdir -p ${GOHAN_API_GTF_PATH}
mkdir -p ${GOHAN_API_GTF_PATH}/tmp
chown -R ${HOST_USER_UID}:${HOST_USER_GID} ${GOHAN_API_GTF_PATH}
chmod -R 770 ${GOHAN_API_GTF_PATH}/tmp
chmod -R 777 ${GOHAN_API_GTF_PATH}

@echo ".. done!"

Expand Down Expand Up @@ -196,17 +198,50 @@ clean-api-drs-bridge-data:


## Tests
test-api-dev: prepare-test-config
test-api: init prepare-test-config
# # @# Run the tests directly from the api source directory
# # cd src/api && \
# # go clean -cache && \
# # go test ./tests/unit/... -v

@# restart any running containers and print
docker compose -f docker-compose.test.yaml down
docker compose -f docker-compose.test.yaml up -d

@# run build tests
@# - print api and drs logs in the
@# event of a failued
cd src/api && \
go clean -cache && \
(go test ./tests/build/... -v || ((docker logs gohan-api | tail -n 100) && (docker logs gohan-drs | tail -n 100) && exit 1)) && \
cd ../..

@# shut down the containers and print
@# the tail end of the
@# api and elasticsearch logs
docker compose -f docker-compose.test.yaml stop
docker logs gohan-api | tail -n 50
docker logs elasticsearch | tail -n 50


test-api-dev: prepare-dev-config
@# Run the tests
cd src/api && \
go clean -cache && \
go test ./tests/integration/... -v


prepare-test-config:
@# Prepare environment variables dynamically via a JSON file
@# since xUnit doens't support loading env variables natively
envsubst < ./etc/test.yml.tpl > ./src/api/tests/common/test.config.yml

prepare-dev-config:
@# Prepare environment variables dynamically via a JSON file
@# since xUnit doens't support loading env variables natively
envsubst < ./etc/test.config.yml.tpl > ./src/api/tests/common/test.config.yml


clean-tests:
@# Clean up
rm ./src/api/tests/common/test.config.yml
28 changes: 28 additions & 0 deletions docker-compose.test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# # Test-only (!) compose file for the Gohan

networks:
bridge-net:
external: true

services:
api:
ports:
- "${GOHAN_API_EXTERNAL_PORT}:${GOHAN_API_INTERNAL_PORT}"
extends:
file: docker-compose.yaml
service: api


elasticsearch:
ports:
- ${GOHAN_ES_EXTERNAL_PORT_1}:${GOHAN_ES_INTERNAL_PORT_1}
extends:
file: docker-compose.yaml
service: elasticsearch

drs:
ports:
- "${GOHAN_DRS_EXTERNAL_PORT}:${GOHAN_DRS_INTERNAL_PORT}"
extends:
file: docker-compose.yaml
service: drs
2 changes: 1 addition & 1 deletion docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ services:
- GOHAN_API_BULK_INDEXING_CAP=${GOHAN_API_BULK_INDEXING_CAP}
- GOHAN_API_FILE_PROC_CONC_LVL=${GOHAN_API_FILE_PROC_CONC_LVL}
- GOHAN_API_LINE_PROC_CONC_LVL=${GOHAN_API_LINE_PROC_CONC_LVL}

- GOHAN_API_INTERNAL_PORT

# Elasticsearch
- GOHAN_ES_URL=${GOHAN_PRIVATE_ES_URL}
Expand Down
6 changes: 4 additions & 2 deletions etc/example.env
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,10 @@ GOHAN_ES_EXTERNAL_PORT_2=9300
GOHAN_ES_INTERNAL_PORT_2=9300

GOHAN_ES_DISC_TYP=single-node
GOHAN_ES_MEM_LIM=14G
GOHAN_ES_CPUS=6
GOHAN_ES_MEM_LIM=4G
GOHAN_ES_CPUS=2
#GOHAN_ES_MEM_LIM=14G
#GOHAN_ES_CPUS=6

# allocate a little bit less memory than what is stated in `mem_limit` ^^
GOHAN_ES_JAVA_OPTS="ES_JAVA_OPTS=-Xms13g -Xmx13g"
Expand Down
22 changes: 22 additions & 0 deletions etc/test.yml.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
debug: false
api:
url: http://0.0.0.0:5000
port: ${GOHAN_API_INTERNAL_PORT}
vcfPath: "${GOHAN_API_CONTAINERIZED_VCF_PATH}"
localVcfPath: "${GOHAN_API_VCF_PATH}"

elasticsearch:
url: elasticsearch:${GOHAN_API_ES_PORT}
username: "${GOHAN_ES_USERNAME}"
password: "${GOHAN_ES_PASSWORD}"

drs:
url: gohan-drs:${GOHAN_DRS_INTERNAL_PORT}
username: "${GOHAN_DRS_BASIC_AUTH_USERNAME}"
password: "${GOHAN_DRS_BASIC_AUTH_PASSWORD}"

authX:
isAuthorizationEnabled: false
oidcPublicJwksUrl: ""
opaUrl: ""
requiredHeadersCommaSep: ""
12 changes: 6 additions & 6 deletions src/api/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,16 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/net v0.3.0 // indirect
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
24 changes: 14 additions & 10 deletions src/api/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -46,19 +49,20 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
Expand Down
3 changes: 2 additions & 1 deletion src/api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,8 @@ func main() {
e.GET("/genes/overview", genesMvc.GetGenesOverview)
e.GET("/genes/search", genesMvc.GenesGetByNomenclatureWildcard,
// middleware
gam.ValidateOptionalChromosomeAttribute)
gam.ValidateOptionalChromosomeAttribute,
gam.MandateAssemblyIdAttribute)
e.GET("/genes/ingestion/requests", genesMvc.GetAllGeneIngestionRequests)
e.GET("/genes/ingestion/run", genesMvc.GenesIngest)
e.GET("/genes/ingestion/stats", genesMvc.GenesIngestionStats)
Expand Down
11 changes: 11 additions & 0 deletions src/api/models/ingest/ingest.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,14 @@ type IngestResponseDTO struct {
State State `json:"state"`
Message string `json:"message"`
}

type IngestStatsDto struct {
NumAdded int
NumFlushed int
NumFailed int
NumIndexed int
NumCreated int
NumUpdated int
NumDeleted int
NumRequests int
}
2 changes: 1 addition & 1 deletion src/api/mvc/genes/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ func GenesIngest(c echo.Context) error {
assemblyWg.Wait()
}()

return c.JSON(http.StatusOK, "{\"message\":\"please check in with /genes/overview !\"}")
return c.JSON(http.StatusOK, map[string]interface{}{"message": "please check in with /genes/overview !"})
}

func GetAllGeneIngestionRequests(c echo.Context) error {
Expand Down
Loading

0 comments on commit 5c13d59

Please sign in to comment.