From 06993055897382433437f807e7085fbd7112ab66 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 16:31:23 -0400 Subject: [PATCH 01/15] chore: added QueryParameters to context - assemblyId middleware --- src/api/contexts/contexts.go | 7 +++++ src/api/middleware/assemblyMiddleware.go | 33 +++++++++++++++++++++++- src/api/mvc/genes/main.go | 11 ++++---- src/api/mvc/main.go | 10 +++---- src/api/mvc/variants/main.go | 15 ++++++----- 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index bc829a76..93aabe87 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -2,6 +2,7 @@ package contexts import ( "gohan/api/models" + "gohan/api/models/constants" "gohan/api/services" variantsService "gohan/api/services/variants" @@ -18,5 +19,11 @@ type ( Config *models.Config IngestionService *services.IngestionService VariantService *variantsService.VariantService + QueryParameters + } + + // Convenient storage for relevant http context data + QueryParameters struct { + AssemblyId constants.AssemblyId } ) diff --git a/src/api/middleware/assemblyMiddleware.go b/src/api/middleware/assemblyMiddleware.go index 871fb11d..f3e22895 100644 --- a/src/api/middleware/assemblyMiddleware.go +++ b/src/api/middleware/assemblyMiddleware.go @@ -1,6 +1,8 @@ package middleware import ( + "gohan/api/contexts" + "gohan/api/models/constants" assid "gohan/api/models/constants/assembly-id" "net/http" @@ -19,6 +21,35 @@ func MandateAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc { return echo.NewHTTPError(http.StatusBadRequest, "Missing or unknown assemblyId!") } - return next(c) + // forward a type-safe value down the pipeline + gc := c.(*contexts.GohanContext) + gc.AssemblyId = constants.AssemblyId(assemblyId) + + return next(gc) + } +} + +/* +Echo middleware to ensure the optional `assemblyId` +HTTP query parameter is valid if present +*/ +func OptionalAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + + // check for assemblyId query parameter + assemblyId := c.QueryParam("assemblyId") + if len(assemblyId) >= 0 { + if !assid.IsKnownAssemblyId(assemblyId) { + // if an id was provided and was invalid, return an error + return echo.NewHTTPError(http.StatusBadRequest, "Invalid assemblyId!") + } + gc.AssemblyId = assid.Unknown + } else { + // forward a type-safe value down the pipeline + gc.AssemblyId = constants.AssemblyId(assemblyId) + } + + return next(gc) } } diff --git a/src/api/mvc/genes/main.go b/src/api/mvc/genes/main.go index 8f4e0fa9..76b493a8 100644 --- a/src/api/mvc/genes/main.go +++ b/src/api/mvc/genes/main.go @@ -322,8 +322,9 @@ func GetAllGeneIngestionRequests(c echo.Context) error { func GenesGetByNomenclatureWildcard(c echo.Context) error { fmt.Printf("[%s] - GenesGetByNomenclatureWildcard hit!\n", time.Now()) - cfg := c.(*contexts.GohanContext).Config - es := c.(*contexts.GohanContext).Es7Client + gc := c.(*contexts.GohanContext) + cfg := gc.Config + es := gc.Es7Client // Chromosome search term chromosomeSearchTerm := c.QueryParam("chromosome") @@ -338,10 +339,10 @@ func GenesGetByNomenclatureWildcard(c echo.Context) error { // Assembly ID // perform wildcard search if empty/random parameter is passed // - set to Unknown to trigger it - assId := assemblyId.Unknown - if assemblyId.CastToAssemblyId(c.QueryParam("assemblyId")) != assemblyId.Unknown { + var assId constants.AssemblyId + if gc.AssemblyId != assemblyId.Unknown { // retrieve passed parameter if is valid - assId = assemblyId.CastToAssemblyId(c.QueryParam("assemblyId")) + assId = gc.AssemblyId } // Size diff --git a/src/api/mvc/main.go b/src/api/mvc/main.go index 5d7736a1..6f56dad1 100644 --- a/src/api/mvc/main.go +++ b/src/api/mvc/main.go @@ -3,7 +3,6 @@ package mvc import ( "gohan/api/contexts" "gohan/api/models/constants" - a "gohan/api/models/constants/assembly-id" gq "gohan/api/models/constants/genotype-query" "log" "strconv" @@ -14,7 +13,8 @@ import ( ) func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, int, string, string, []string, constants.GenotypeQuery, constants.AssemblyId, string) { - es := c.(*contexts.GohanContext).Es7Client + gc := c.(*contexts.GohanContext) + es := gc.Es7Client chromosome := c.QueryParam("chromosome") if len(chromosome) == 0 { @@ -73,11 +73,7 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, } } - assemblyId := a.Unknown - assemblyIdQP := c.QueryParam("assemblyId") - if len(assemblyIdQP) > 0 && a.IsKnownAssemblyId(assemblyIdQP) { - assemblyId = a.CastToAssemblyId(assemblyIdQP) - } + assemblyId := gc.AssemblyId tableId := c.QueryParam("tableId") if len(tableId) == 0 { diff --git a/src/api/mvc/variants/main.go b/src/api/mvc/variants/main.go index 0922f0b9..378bde96 100644 --- a/src/api/mvc/variants/main.go +++ b/src/api/mvc/variants/main.go @@ -16,7 +16,6 @@ import ( "time" "gohan/api/contexts" - a "gohan/api/models/constants/assembly-id" s "gohan/api/models/constants/sort" "gohan/api/models/dtos" "gohan/api/models/indexes" @@ -102,13 +101,16 @@ func VariantsCountBySampleId(c echo.Context) error { func VariantsIngest(c echo.Context) error { fmt.Printf("[%s] - VariantsIngest hit!\n", time.Now()) - cfg := c.(*contexts.GohanContext).Config + gc := c.(*contexts.GohanContext) + + cfg := gc.Config vcfPath := cfg.Api.VcfPath drsUrl := cfg.Drs.Url drsUsername := cfg.Drs.Username drsPassword := cfg.Drs.Password - ingestionService := c.(*contexts.GohanContext).IngestionService + // query parameters + assemblyId := gc.AssemblyId // retrieve query parameters (comman separated) var fileNames []string @@ -206,7 +208,6 @@ func VariantsIngest(c echo.Context) error { // ----- } - assemblyId := a.CastToAssemblyId(c.QueryParam("assemblyId")) tableId := c.QueryParam("tableId") // TODO: validate table exists in elasticsearch @@ -228,8 +229,10 @@ func VariantsIngest(c echo.Context) error { fmt.Printf("Ingest Start: %s\n", startTime) // ingest vcf - // ingserviceMux := sync.RWMutex{} - responseDtos := []ingest.IngestResponseDTO{} + var ( + ingestionService = gc.IngestionService + responseDtos = []ingest.IngestResponseDTO{} + ) for _, fileName := range fileNames { // check if there is an already existing ingestion request state From 4f66885093326a8df68ebb82e4f2277a46ae5673 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 16:58:59 -0400 Subject: [PATCH 02/15] chore: - added alleles to context - patched middleware, wildcard handling --- src/api/contexts/contexts.go | 1 + .../middleware/calibratedAllelesMiddleware.go | 19 +++++++++++++++---- src/api/mvc/main.go | 8 +------- .../repositories/elasticsearch/variants.go | 18 ++++++------------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index 93aabe87..7104afa6 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -25,5 +25,6 @@ type ( // Convenient storage for relevant http context data QueryParameters struct { AssemblyId constants.AssemblyId + Alleles []string } ) diff --git a/src/api/middleware/calibratedAllelesMiddleware.go b/src/api/middleware/calibratedAllelesMiddleware.go index 390fc2c0..44124368 100644 --- a/src/api/middleware/calibratedAllelesMiddleware.go +++ b/src/api/middleware/calibratedAllelesMiddleware.go @@ -1,6 +1,7 @@ package middleware import ( + "gohan/api/contexts" "gohan/api/models/dtos/errors" "net/http" "strings" @@ -10,24 +11,34 @@ import ( func MandateCalibratedAlleles(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { - if len(c.QueryParam("alleles")) > 0 { - allelesQP := strings.Split(c.QueryParam("alleles"), ",") + gc := c.(*contexts.GohanContext) + + var alleles []string + var allelesQP string = c.QueryParam("alleles") + if len(allelesQP) > 0 { + alleles = strings.Split(allelesQP, ",") // ensure the allele query is properly formatted - if allelesQP[len(allelesQP)-1] == "" { + if alleles[len(alleles)-1] == "" { return echo.NewHTTPError( http.StatusBadRequest, errors.CreateSimpleBadRequest("Found an empty allele! Please double check your request!")) } // ensure no more than 2 alleles are provided at once - if len(allelesQP) > 2 { + if len(alleles) > 2 { return echo.NewHTTPError( http.StatusBadRequest, errors.CreateSimpleBadRequest("Too many alleles! Please only provide 1 or 2")) } + + for i, a := range alleles { + alleles[i] = strings.Replace(a, "N", "?", -1) + // alleles = append(alleles, strings.Replace(a, "N", "?", -1)) + } } + gc.Alleles = alleles return next(c) } } diff --git a/src/api/mvc/main.go b/src/api/mvc/main.go index 6f56dad1..a2755e3c 100644 --- a/src/api/mvc/main.go +++ b/src/api/mvc/main.go @@ -49,8 +49,7 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, reference := c.QueryParam("reference") alternative := c.QueryParam("alternative") - var alleles []string - allelesQP := strings.Split(c.QueryParam("alleles"), ",") + alleles := gc.Alleles // reference, alternative and alleles can have the // single-wildcard character 'N', which adheres to @@ -59,11 +58,6 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, // swap all 'N's into '?'s for elasticsearch reference = strings.Replace(reference, "N", "?", -1) alternative = strings.Replace(alternative, "N", "?", -1) - if len(allelesQP) > 0 && allelesQP[0] != "" { // check it isn't empty - for _, a := range allelesQP { - alleles = append(alleles, strings.Replace(a, "N", "?", -1)) - } - } genotype := gq.UNCALLED genotypeQP := c.QueryParam("genotype") diff --git a/src/api/repositories/elasticsearch/variants.go b/src/api/repositories/elasticsearch/variants.go index f777a6d3..18f61cfe 100644 --- a/src/api/repositories/elasticsearch/variants.go +++ b/src/api/repositories/elasticsearch/variants.go @@ -665,32 +665,26 @@ func addAllelesToShouldMap(alleles []string, genotype c.GenotypeQuery, allelesSh if genotype == gq.ALTERNATE || genotype == gq.REFERENCE { // haploid case // - queried allele should be present on the left side of the pair with an empty right side - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[0], "AND", "\"\"")}) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[0], "AND", "\"\"")) } else { // assume diploid-type of search as default // - queried allele can be present on either side of the pair - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[0], "OR", alleles[0])}) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[0], "OR", alleles[0])) } case 2: if genotype == gq.ALTERNATE || genotype == gq.REFERENCE { // haploid case // - either queried allele can be present on the left side of the pair with an empty right side - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[0], "AND", "\"\"")}) - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[1], "AND", "\"\"")}) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[0], "AND", "\"\"")) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[1], "AND", "\"\"")) } else { // assume diploid-type of search as default // - treat as a left/right pair; // either queried allele can be present on the left or right side of the pair - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[0], "AND", alleles[1])}) - allelesShouldMap = append(allelesShouldMap, map[string]interface{}{ - "query_string": allelesShouldMapBuilder(alleles[1], "AND", alleles[0])}) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[0], "AND", alleles[1])) + allelesShouldMap = append(allelesShouldMap, allelesShouldMapBuilder(alleles[1], "AND", alleles[0])) } // TODO: triploid ? } From 569c2cefbc559707ccb16519da6c4ac6e2cf4b9b Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 16:59:38 -0400 Subject: [PATCH 03/15] patch: comment --- src/api/middleware/calibratedAllelesMiddleware.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/middleware/calibratedAllelesMiddleware.go b/src/api/middleware/calibratedAllelesMiddleware.go index 44124368..6a7d4dc6 100644 --- a/src/api/middleware/calibratedAllelesMiddleware.go +++ b/src/api/middleware/calibratedAllelesMiddleware.go @@ -34,7 +34,6 @@ func MandateCalibratedAlleles(next echo.HandlerFunc) echo.HandlerFunc { for i, a := range alleles { alleles[i] = strings.Replace(a, "N", "?", -1) - // alleles = append(alleles, strings.Replace(a, "N", "?", -1)) } } From 5566165bfa95913992ba8faefa8878768630df5e Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 17:06:50 -0400 Subject: [PATCH 04/15] chore: added bounds to context --- src/api/contexts/contexts.go | 6 +++++ .../middleware/calibratedBoundsMiddleware.go | 5 ++++ src/api/mvc/main.go | 27 ++----------------- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index 7104afa6..167a345f 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -26,5 +26,11 @@ type ( QueryParameters struct { AssemblyId constants.AssemblyId Alleles []string + PositionBounds + } + + PositionBounds struct { + LowerBound int + UpperBound int } ) diff --git a/src/api/middleware/calibratedBoundsMiddleware.go b/src/api/middleware/calibratedBoundsMiddleware.go index 96c4bef0..60d3fd01 100644 --- a/src/api/middleware/calibratedBoundsMiddleware.go +++ b/src/api/middleware/calibratedBoundsMiddleware.go @@ -1,6 +1,7 @@ package middleware import ( + "gohan/api/contexts" "net/http" "strconv" @@ -9,6 +10,8 @@ import ( func MandateCalibratedBounds(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + var ( lowerBound int upperBound int @@ -50,6 +53,8 @@ func MandateCalibratedBounds(next echo.HandlerFunc) echo.HandlerFunc { return echo.NewHTTPError(http.StatusBadRequest, "Invalid lower and upper bounds!") } + gc.LowerBound = lowerBound + gc.UpperBound = upperBound return next(c) } } diff --git a/src/api/mvc/main.go b/src/api/mvc/main.go index a2755e3c..5d119c61 100644 --- a/src/api/mvc/main.go +++ b/src/api/mvc/main.go @@ -4,8 +4,6 @@ import ( "gohan/api/contexts" "gohan/api/models/constants" gq "gohan/api/models/constants/genotype-query" - "log" - "strconv" "strings" "github.com/elastic/go-elasticsearch/v7" @@ -22,29 +20,8 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, chromosome = "*" } - lowerBoundQP := c.QueryParam("lowerBound") - var ( - lowerBound int - lbErr error - ) - if len(lowerBoundQP) > 0 { - lowerBound, lbErr = strconv.Atoi(lowerBoundQP) - if lbErr != nil { - log.Fatal(lbErr) - } - } - - upperBoundQP := c.QueryParam("upperBound") - var ( - upperBound int - ubErr error - ) - if len(upperBoundQP) > 0 { - upperBound, ubErr = strconv.Atoi(upperBoundQP) - if ubErr != nil { - log.Fatal(ubErr) - } - } + lowerBound := gc.LowerBound + upperBound := gc.UpperBound reference := c.QueryParam("reference") alternative := c.QueryParam("alternative") From 3eefad6e5ffc21966eb11e529f1ea5c4820522b4 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 17:15:22 -0400 Subject: [PATCH 05/15] chore: added chrom to context - validation, wildcard --- src/api/contexts/contexts.go | 3 ++- src/api/middleware/chromosomeMiddleware.go | 12 +++++++++++- src/api/mvc/genes/main.go | 6 +----- src/api/mvc/main.go | 6 +----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index 167a345f..4631a314 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -15,17 +15,18 @@ type ( // an elasticsearch client and other variables GohanContext struct { echo.Context + QueryParameters Es7Client *es7.Client Config *models.Config IngestionService *services.IngestionService VariantService *variantsService.VariantService - QueryParameters } // Convenient storage for relevant http context data QueryParameters struct { AssemblyId constants.AssemblyId Alleles []string + Chromosome string PositionBounds } diff --git a/src/api/middleware/chromosomeMiddleware.go b/src/api/middleware/chromosomeMiddleware.go index 7b134e42..1f800bf7 100644 --- a/src/api/middleware/chromosomeMiddleware.go +++ b/src/api/middleware/chromosomeMiddleware.go @@ -1,6 +1,7 @@ package middleware import ( + "gohan/api/contexts" "gohan/api/models/constants/chromosome" "net/http" @@ -12,6 +13,8 @@ Echo middleware to ensure a valid `chromosome` HTTP query parameter was provided */ func ValidateOptionalChromosomeAttribute(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + // check for chromosome query parameter chromQP := c.QueryParam("chromosome") @@ -22,6 +25,13 @@ func ValidateOptionalChromosomeAttribute(next echo.HandlerFunc) echo.HandlerFunc return echo.NewHTTPError(http.StatusBadRequest, "Please provide a valid 'chromosome' (either 1-23, X, Y, or M)") } - return next(c) + if len(chromQP) == 0 { + // if no chromosome is provided, assume "wildcard" search + gc.Chromosome = "*" + } else { + gc.Chromosome = chromQP + } + + return next(gc) } } diff --git a/src/api/mvc/genes/main.go b/src/api/mvc/genes/main.go index 76b493a8..d5e7cbdd 100644 --- a/src/api/mvc/genes/main.go +++ b/src/api/mvc/genes/main.go @@ -327,11 +327,7 @@ func GenesGetByNomenclatureWildcard(c echo.Context) error { es := gc.Es7Client // Chromosome search term - chromosomeSearchTerm := c.QueryParam("chromosome") - if len(chromosomeSearchTerm) == 0 { - // if no chromosome is provided, assume "wildcard" search - chromosomeSearchTerm = "*" - } + chromosomeSearchTerm := gc.Chromosome // Name search term term := c.QueryParam("term") diff --git a/src/api/mvc/main.go b/src/api/mvc/main.go index 5d119c61..5be32add 100644 --- a/src/api/mvc/main.go +++ b/src/api/mvc/main.go @@ -14,11 +14,7 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, gc := c.(*contexts.GohanContext) es := gc.Es7Client - chromosome := c.QueryParam("chromosome") - if len(chromosome) == 0 { - // if no chromosome is provided, assume "wildcard" search - chromosome = "*" - } + chromosome := gc.Chromosome lowerBound := gc.LowerBound upperBound := gc.UpperBound From c5e302e630c23ad9a73fb22979f9224942b1677e Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 17:30:38 -0400 Subject: [PATCH 06/15] chore: added genotype to context --- src/api/contexts/contexts.go | 1 + src/api/middleware/genotypeMiddleware.go | 11 ++++++++++- src/api/mvc/main.go | 10 +--------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index 4631a314..f7259dac 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -27,6 +27,7 @@ type ( AssemblyId constants.AssemblyId Alleles []string Chromosome string + Genotype constants.GenotypeQuery PositionBounds } diff --git a/src/api/middleware/genotypeMiddleware.go b/src/api/middleware/genotypeMiddleware.go index 3d68493d..91503c3c 100644 --- a/src/api/middleware/genotypeMiddleware.go +++ b/src/api/middleware/genotypeMiddleware.go @@ -4,6 +4,8 @@ import ( "fmt" "net/http" + "gohan/api/contexts" + "gohan/api/models/constants" gq "gohan/api/models/constants/genotype-query" "github.com/labstack/echo" @@ -14,18 +16,25 @@ Echo middleware to ensure the validity of the optionally provided `genotype` HTT */ func ValidatePotentialGenotypeQueryParameter(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + // check for a genotype query parameter + var ( + genotype constants.GenotypeQuery + genotypeErr error + ) genotypeQP := c.QueryParam("genotype") if len(genotypeQP) > 0 { // validate that the genotype string // converts to a valid GenotypeQuery // (skips "UNCALLED" values) - _, genotypeErr := gq.CastToGenoType(genotypeQP) + genotype, genotypeErr = gq.CastToGenoType(genotypeQP) if genotypeErr != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid genotype query %s, %s", genotypeQP, genotypeErr)) } } + gc.Genotype = genotype return next(c) } } diff --git a/src/api/mvc/main.go b/src/api/mvc/main.go index 5be32add..91ccfd10 100644 --- a/src/api/mvc/main.go +++ b/src/api/mvc/main.go @@ -3,7 +3,6 @@ package mvc import ( "gohan/api/contexts" "gohan/api/models/constants" - gq "gohan/api/models/constants/genotype-query" "strings" "github.com/elastic/go-elasticsearch/v7" @@ -32,14 +31,7 @@ func RetrieveCommonElements(c echo.Context) (*elasticsearch.Client, string, int, reference = strings.Replace(reference, "N", "?", -1) alternative = strings.Replace(alternative, "N", "?", -1) - genotype := gq.UNCALLED - genotypeQP := c.QueryParam("genotype") - if len(genotypeQP) > 0 { - if parsedGenotype, gErr := gq.CastToGenoType(genotypeQP); gErr == nil { - genotype = parsedGenotype - } - } - + genotype := gc.Genotype assemblyId := gc.AssemblyId tableId := c.QueryParam("tableId") From 8aa1cda5c79a7a84311dc78438e22ee9f3b74aa0 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Thu, 8 Jun 2023 17:40:31 -0400 Subject: [PATCH 07/15] patch: passing gohan context down echo next --- src/api/middleware/calibratedAllelesMiddleware.go | 2 +- src/api/middleware/calibratedBoundsMiddleware.go | 2 +- src/api/middleware/genotypeMiddleware.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/middleware/calibratedAllelesMiddleware.go b/src/api/middleware/calibratedAllelesMiddleware.go index 6a7d4dc6..42e8baab 100644 --- a/src/api/middleware/calibratedAllelesMiddleware.go +++ b/src/api/middleware/calibratedAllelesMiddleware.go @@ -38,6 +38,6 @@ func MandateCalibratedAlleles(next echo.HandlerFunc) echo.HandlerFunc { } gc.Alleles = alleles - return next(c) + return next(gc) } } diff --git a/src/api/middleware/calibratedBoundsMiddleware.go b/src/api/middleware/calibratedBoundsMiddleware.go index 60d3fd01..6174c1ce 100644 --- a/src/api/middleware/calibratedBoundsMiddleware.go +++ b/src/api/middleware/calibratedBoundsMiddleware.go @@ -55,6 +55,6 @@ func MandateCalibratedBounds(next echo.HandlerFunc) echo.HandlerFunc { gc.LowerBound = lowerBound gc.UpperBound = upperBound - return next(c) + return next(gc) } } diff --git a/src/api/middleware/genotypeMiddleware.go b/src/api/middleware/genotypeMiddleware.go index 91503c3c..c15eaeb9 100644 --- a/src/api/middleware/genotypeMiddleware.go +++ b/src/api/middleware/genotypeMiddleware.go @@ -35,6 +35,6 @@ func ValidatePotentialGenotypeQueryParameter(next echo.HandlerFunc) echo.Handler } gc.Genotype = genotype - return next(c) + return next(gc) } } From 5173a1898459c6ced7cd51b889a5cff5e2d0528f Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 16:45:39 -0400 Subject: [PATCH 08/15] patch: assembly middleware logic --- src/api/middleware/assemblyMiddleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/middleware/assemblyMiddleware.go b/src/api/middleware/assemblyMiddleware.go index f3e22895..8fcf2ef4 100644 --- a/src/api/middleware/assemblyMiddleware.go +++ b/src/api/middleware/assemblyMiddleware.go @@ -39,15 +39,15 @@ func OptionalAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc { // check for assemblyId query parameter assemblyId := c.QueryParam("assemblyId") - if len(assemblyId) >= 0 { + if len(assemblyId) > 0 { if !assid.IsKnownAssemblyId(assemblyId) { // if an id was provided and was invalid, return an error return echo.NewHTTPError(http.StatusBadRequest, "Invalid assemblyId!") } - gc.AssemblyId = assid.Unknown - } else { // forward a type-safe value down the pipeline gc.AssemblyId = constants.AssemblyId(assemblyId) + } else { + gc.AssemblyId = assid.Unknown } return next(gc) From 2f3213dc7ce45a9c49fa6fec38bcd124d9f5be72 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 17:05:29 -0400 Subject: [PATCH 09/15] chore: sampleIds context qp - minor refactoring as well --- src/api/contexts/contexts.go | 1 + src/api/middleware/sampleMiddleware.go | 20 ++++++++++++++------ src/api/mvc/variants/main.go | 21 ++++++++------------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/api/contexts/contexts.go b/src/api/contexts/contexts.go index f7259dac..7919a606 100644 --- a/src/api/contexts/contexts.go +++ b/src/api/contexts/contexts.go @@ -28,6 +28,7 @@ type ( Alleles []string Chromosome string Genotype constants.GenotypeQuery + SampleIds []string PositionBounds } diff --git a/src/api/middleware/sampleMiddleware.go b/src/api/middleware/sampleMiddleware.go index d34d1e2c..81450fb5 100644 --- a/src/api/middleware/sampleMiddleware.go +++ b/src/api/middleware/sampleMiddleware.go @@ -1,6 +1,7 @@ package middleware import ( + "gohan/api/contexts" "net/http" "strings" @@ -8,10 +9,12 @@ import ( ) /* - Echo middleware to ensure a singular `id` HTTP query parameter was provided +Echo middleware to ensure a singular `id` HTTP query parameter was provided */ func MandateSampleIdsSingularAttribute(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + // check for id query parameter sampleId := c.QueryParam("id") if len(sampleId) == 0 { @@ -19,22 +22,27 @@ func MandateSampleIdsSingularAttribute(next echo.HandlerFunc) echo.HandlerFunc { return echo.NewHTTPError(http.StatusBadRequest, "Missing 'id' query parameter for sample id querying!") } - return next(c) + gc.SampleIds = append(gc.SampleIds, sampleId) + return next(gc) } } /* - Echo middleware to ensure a pluralized `id` (spelled `ids`) HTTP query parameter was provided +Echo middleware to ensure a pluralized `id` (spelled `ids`) HTTP query parameter was provided */ func MandateSampleIdsPluralAttribute(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { + gc := c.(*contexts.GohanContext) + // check for id's query parameter - sampleIds := strings.Split(c.QueryParam("ids"), ",") - if len(sampleIds) == 0 { + sampleIdQP := c.QueryParam("ids") + if len(sampleIdQP) == 0 { // if no ids were provided return an error return echo.NewHTTPError(http.StatusBadRequest, "Missing 'ids' query parameter for sample id querying!") } + sampleIds := strings.Split(sampleIdQP, ",") - return next(c) + gc.SampleIds = append(gc.SampleIds, sampleIds...) + return next(gc) } } diff --git a/src/api/mvc/variants/main.go b/src/api/mvc/variants/main.go index 378bde96..6b446c0b 100644 --- a/src/api/mvc/variants/main.go +++ b/src/api/mvc/variants/main.go @@ -53,12 +53,10 @@ func VariantsGetByVariantId(c echo.Context) error { } func VariantsGetBySampleId(c echo.Context) error { fmt.Printf("[%s] - VariantsGetBySampleId hit!\n", time.Now()) + // retrieve sample Ids from query parameter (comma separated) - sampleIds := strings.Split(c.QueryParam("ids"), ",") - if len(sampleIds[0]) == 0 { - // if no ids were provided, assume "wildcard" search - sampleIds = []string{"*"} - } + gc := c.(*contexts.GohanContext) + sampleIds := gc.SampleIds return executeGetByIds(c, sampleIds, false, false) } @@ -88,15 +86,12 @@ func VariantsCountByVariantId(c echo.Context) error { } func VariantsCountBySampleId(c echo.Context) error { fmt.Printf("[%s] - VariantsCountBySampleId hit!\n", time.Now()) - // retrieve single sample id from query parameter and map to a list - // to conform to function signature - singleSampleIdSlice := []string{c.QueryParam("id")} - if len(singleSampleIdSlice[0]) == 0 { - // if no id was provided, assume "wildcard" search - singleSampleIdSlice = []string{"*"} - } + // retrieve single sample id from query parameter already mapped + // to a slice to conform to function signature + gc := c.(*contexts.GohanContext) + expectedSingleSampleIdSlice := gc.SampleIds - return executeCountByIds(c, singleSampleIdSlice, false) + return executeCountByIds(c, expectedSingleSampleIdSlice, false) } func VariantsIngest(c echo.Context) error { From fe07480596c79bd984779e721ca436c464ec3cbe Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 17:10:45 -0400 Subject: [PATCH 10/15] patch: redundant function --- src/api/middleware/assemblyMiddleware.go | 25 ------------------------ 1 file changed, 25 deletions(-) diff --git a/src/api/middleware/assemblyMiddleware.go b/src/api/middleware/assemblyMiddleware.go index 8fcf2ef4..6c4f7fc3 100644 --- a/src/api/middleware/assemblyMiddleware.go +++ b/src/api/middleware/assemblyMiddleware.go @@ -28,28 +28,3 @@ func MandateAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc { return next(gc) } } - -/* -Echo middleware to ensure the optional `assemblyId` -HTTP query parameter is valid if present -*/ -func OptionalAssemblyIdAttribute(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - gc := c.(*contexts.GohanContext) - - // check for assemblyId query parameter - assemblyId := c.QueryParam("assemblyId") - if len(assemblyId) > 0 { - if !assid.IsKnownAssemblyId(assemblyId) { - // if an id was provided and was invalid, return an error - return echo.NewHTTPError(http.StatusBadRequest, "Invalid assemblyId!") - } - // forward a type-safe value down the pipeline - gc.AssemblyId = constants.AssemblyId(assemblyId) - } else { - gc.AssemblyId = assid.Unknown - } - - return next(gc) - } -} From 4ee7a0f74526017ce15cadec6f941ae772ff6503 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 17:33:09 -0400 Subject: [PATCH 11/15] patch: alleles middleware on count endpoints --- src/api/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/main.go b/src/api/main.go index 7477f66f..3c0ff6de 100644 --- a/src/api/main.go +++ b/src/api/main.go @@ -170,12 +170,14 @@ func main() { // middleware gam.ValidateOptionalChromosomeAttribute, gam.MandateCalibratedBounds, + gam.MandateCalibratedAlleles, gam.MandateAssemblyIdAttribute, gam.ValidatePotentialGenotypeQueryParameter) e.GET("/variants/count/by/sampleId", variantsMvc.VariantsCountBySampleId, // middleware gam.ValidateOptionalChromosomeAttribute, gam.MandateCalibratedBounds, + gam.MandateCalibratedAlleles, gam.MandateAssemblyIdAttribute, gam.MandateSampleIdsSingularAttribute, gam.ValidatePotentialGenotypeQueryParameter) From abe6b9f805e00a001c3d15d2c13f526c93481e4a Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 17:50:09 -0400 Subject: [PATCH 12/15] patch: bump semver --- etc/example.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/example.env b/etc/example.env index 6c8aa17c..6a11c3fc 100644 --- a/etc/example.env +++ b/etc/example.env @@ -2,7 +2,7 @@ GOHAN_DEBUG=false GOHAN_SERVICE_CONTACT=someone@somewhere.ca -GOHAN_SEMVER=3.8.0 +GOHAN_SEMVER=3.9.0 GOHAN_SERVICES="gateway api elasticsearch kibana drs authorization" # GOOS=linux From 0b25bbb95b9ee8904fdd9065293c295205f79e5e Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Fri, 9 Jun 2023 17:58:02 -0400 Subject: [PATCH 13/15] patch: handle wildcards in sampleId middleware --- src/api/main.go | 4 ++-- src/api/middleware/sampleMiddleware.go | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/api/main.go b/src/api/main.go index 3c0ff6de..354e7949 100644 --- a/src/api/main.go +++ b/src/api/main.go @@ -162,7 +162,7 @@ func main() { gam.MandateCalibratedBounds, gam.MandateCalibratedAlleles, gam.MandateAssemblyIdAttribute, - gam.MandateSampleIdsPluralAttribute, + gam.CalibrateOptionalSampleIdsPluralAttribute, gam.ValidatePotentialGenotypeQueryParameter) e.GET("/variants/get/by/documentId", variantsMvc.VariantsGetByDocumentId) @@ -179,7 +179,7 @@ func main() { gam.MandateCalibratedBounds, gam.MandateCalibratedAlleles, gam.MandateAssemblyIdAttribute, - gam.MandateSampleIdsSingularAttribute, + gam.CalibrateOptionalSampleIdsSingularAttribute, gam.ValidatePotentialGenotypeQueryParameter) // TODO: refactor (deduplicate) -- diff --git a/src/api/middleware/sampleMiddleware.go b/src/api/middleware/sampleMiddleware.go index 81450fb5..68149061 100644 --- a/src/api/middleware/sampleMiddleware.go +++ b/src/api/middleware/sampleMiddleware.go @@ -2,24 +2,22 @@ package middleware import ( "gohan/api/contexts" - "net/http" "strings" "github.com/labstack/echo" ) /* -Echo middleware to ensure a singular `id` HTTP query parameter was provided +Echo middleware to prepare the context for an optionall provided singular `id` HTTP query parameter */ -func MandateSampleIdsSingularAttribute(next echo.HandlerFunc) echo.HandlerFunc { +func CalibrateOptionalSampleIdsSingularAttribute(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { gc := c.(*contexts.GohanContext) // check for id query parameter sampleId := c.QueryParam("id") if len(sampleId) == 0 { - // if no id was provided return an error - return echo.NewHTTPError(http.StatusBadRequest, "Missing 'id' query parameter for sample id querying!") + sampleId = "*" // wildcard } gc.SampleIds = append(gc.SampleIds, sampleId) @@ -28,19 +26,22 @@ func MandateSampleIdsSingularAttribute(next echo.HandlerFunc) echo.HandlerFunc { } /* -Echo middleware to ensure a pluralized `id` (spelled `ids`) HTTP query parameter was provided +Echo middleware to prepare the context for an optionally provided pluralized `id` (spelled `ids`) HTTP query parameter */ -func MandateSampleIdsPluralAttribute(next echo.HandlerFunc) echo.HandlerFunc { +func CalibrateOptionalSampleIdsPluralAttribute(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { gc := c.(*contexts.GohanContext) // check for id's query parameter - sampleIdQP := c.QueryParam("ids") - if len(sampleIdQP) == 0 { - // if no ids were provided return an error - return echo.NewHTTPError(http.StatusBadRequest, "Missing 'ids' query parameter for sample id querying!") + var ( + sampleIdQP = c.QueryParam("ids") + sampleIds []string + ) + if len(sampleIdQP) > 0 { + sampleIds = strings.Split(sampleIdQP, ",") + } else { + sampleIds = append(sampleIds, "*") // wildcard } - sampleIds := strings.Split(sampleIdQP, ",") gc.SampleIds = append(gc.SampleIds, sampleIds...) return next(gc) From 34a6139a709c5c30ec0dab6932e3bfe37d14a7d8 Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Mon, 12 Jun 2023 16:12:52 -0400 Subject: [PATCH 14/15] patch: improved middleware allele handling - upper case alleles - validation - transform N to ? --- .../middleware/calibratedAllelesMiddleware.go | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/api/middleware/calibratedAllelesMiddleware.go b/src/api/middleware/calibratedAllelesMiddleware.go index 42e8baab..34be90cd 100644 --- a/src/api/middleware/calibratedAllelesMiddleware.go +++ b/src/api/middleware/calibratedAllelesMiddleware.go @@ -1,8 +1,10 @@ package middleware import ( + "fmt" "gohan/api/contexts" "gohan/api/models/dtos/errors" + "gohan/api/utils" "net/http" "strings" @@ -32,8 +34,23 @@ func MandateCalibratedAlleles(next echo.HandlerFunc) echo.HandlerFunc { errors.CreateSimpleBadRequest("Too many alleles! Please only provide 1 or 2")) } - for i, a := range alleles { - alleles[i] = strings.Replace(a, "N", "?", -1) + // check validity of each provided character + for i, allele := range alleles { + // - accept lower cases, but transform to upper for elasticsearch (case sensitive) + upperAllele := strings.ToUpper(allele) + // - check validity of the rest + for _, nuc := range upperAllele { + if !utils.StringInSlice(string(nuc), utils.AcceptedNucleotideCharacters) { + // return status 400 if any allele is incorrect + return echo.NewHTTPError( + http.StatusBadRequest, + errors.CreateSimpleBadRequest(fmt.Sprintf("Nucleotide %s unacceptable! Please double check and try again.", string(nuc)))) + } + } + // - replace 'N' with uppercase '?' for elasticsearch + upperAllele = strings.Replace(upperAllele, "N", "?", -1) + + alleles[i] = upperAllele } } From 883b6026e3eea2099133ab1af474cf8a0a58e96a Mon Sep 17 00:00:00 2001 From: Brennan Brouillette Date: Wed, 14 Jun 2023 15:40:53 -0400 Subject: [PATCH 15/15] patch: table summary - es query crashing missing chromosome --- src/api/mvc/tables/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/mvc/tables/main.go b/src/api/mvc/tables/main.go index 828c4254..35c9dbd9 100644 --- a/src/api/mvc/tables/main.go +++ b/src/api/mvc/tables/main.go @@ -156,7 +156,7 @@ func GetTableSummary(c echo.Context) error { // obtain other potentially relevant parameters from available query parameters // (these should be empty, but utilizing this common function is convenient to set up // the call to the variants index through the repository functions) - var es, chromosome, lowerBound, upperBound, reference, alternative, alleles, genotype, assemblyId, _ = mvc.RetrieveCommonElements(c) + var es, _, lowerBound, upperBound, reference, alternative, alleles, genotype, assemblyId, _ = mvc.RetrieveCommonElements(c) // unused tableId from query parameter set to '_' // table id must be provided @@ -213,7 +213,7 @@ func GetTableSummary(c echo.Context) error { totalVariantsCount := 0.0 docs, countError := esRepo.CountDocumentsContainerVariantOrSampleIdInPositionRange(cfg, es, - chromosome, lowerBound, upperBound, + "*", lowerBound, upperBound, // always wildcard "*" chromosome "", "", // note : both variantId and sampleId are deliberately set to "" reference, alternative, alleles, genotype, assemblyId, tableId) if countError != nil {