From 9aa28aba623ac8dcb5c0008b5d0ae583e10532e8 Mon Sep 17 00:00:00 2001 From: Alexander Taepper Date: Thu, 16 Jan 2025 12:19:53 +0100 Subject: [PATCH] chore(deps): SILO v0.5.0 (#3529) * chore(deps): SILO v0.5.0 (#3529) * refactor: remove remains of pango_lineage from source code (#3529) --- .../kotlin/org/loculus/backend/config/Config.kt | 3 --- .../ProcessedSequenceEntryValidator.kt | 16 ---------------- .../submission/PreparedProcessedData.kt | 9 --------- .../SubmitProcessedDataEndpointTest.kt | 9 --------- backend/src/test/resources/backend_config.json | 4 ++-- .../resources/backend_config_single_segment.json | 2 +- .../for-administrators/setup-with-kubernetes.md | 2 +- kubernetes/loculus/silo_import_job.sh | 6 +++--- .../templates/lapis-silo-database-config.yaml | 4 ---- .../loculus/templates/lapis-silo-deployment.yaml | 9 +++------ kubernetes/loculus/values.yaml | 2 +- preprocessing/specification.md | 1 - .../components/SearchPage/SearchForm.spec.tsx | 2 +- .../components/SearchPage/SearchFullUI.spec.tsx | 2 +- .../components/SearchPage/fields/FieldProps.tsx | 2 +- .../SearchPage/fields/NormalTextField.tsx | 2 +- website/src/types/config.ts | 11 +---------- 17 files changed, 16 insertions(+), 70 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt index 7205e1924e..a2ee9d98f4 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/Config.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/Config.kt @@ -44,9 +44,6 @@ enum class MetadataType { @JsonProperty("date") DATE, - @JsonProperty("pango_lineage") - PANGO_LINEAGE, - @JsonProperty("boolean") BOOLEAN, diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt index ca8e3a18e5..4c23f1cbdd 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/ProcessedSequenceEntryValidator.kt @@ -20,8 +20,6 @@ import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException private const val DATE_FORMAT = "yyyy-MM-dd" -private const val PANGO_LINEAGE_REGEX_PATTERN = "[a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}" -private val pangoLineageRegex = Regex(PANGO_LINEAGE_REGEX_PATTERN) interface Symbol { val symbol: Char @@ -100,9 +98,6 @@ private fun validateNoUnknownInMetaData(data: Map, known: List { - if (!isValidPangoLineage(fieldValue.asText())) { - throw ProcessingValidationException( - "Expected type 'pango_lineage' for field '${metadata.name}', " + - "found value '$fieldValue'. " + - "A pango lineage must be of the form $PANGO_LINEAGE_REGEX_PATTERN, e.g. 'XBB' or 'BA.1.5'.", - ) - } - return - } - else -> {} } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt index f3446306ed..7f4a57a931 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/PreparedProcessedData.kt @@ -212,15 +212,6 @@ object PreparedProcessedData { ), ) - fun withWrongPangoLineageFormat(accession: Accession) = defaultSuccessfulSubmittedData.copy( - accession = accession, - data = defaultProcessedData.copy( - metadata = defaultProcessedData.metadata + mapOf( - "pangoLineage" to TextNode("A.5.invalid"), - ), - ), - ) - fun withWrongBooleanFormat(accession: Accession) = defaultSuccessfulSubmittedData.copy( accession = accession, data = defaultProcessedData.copy( diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt index 31faed8680..9ff2af74f6 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitProcessedDataEndpointTest.kt @@ -506,15 +506,6 @@ class SubmitProcessedDataEndpointTest( expectedErrorMessage = "Expected type 'boolean' for field 'booleanColumn', found value '\"not a boolean\"'.", ), - InvalidDataScenario( - name = "data with wrong pango lineage format", - processedDataThatNeedsAValidAccession = PreparedProcessedData.withWrongPangoLineageFormat( - accession = "DoesNotMatter", - ), - expectedErrorMessage = - "Expected type 'pango_lineage' for field 'pangoLineage', found value '\"A.5.invalid\"'. " + - "A pango lineage must be of the form [a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}, e.g. 'XBB' or 'BA.1.5'.", - ), InvalidDataScenario( name = "data with explicit null for required field", processedDataThatNeedsAValidAccession = PreparedProcessedData.withNullForFields( diff --git a/backend/src/test/resources/backend_config.json b/backend/src/test/resources/backend_config.json index 375a59bb4e..9f8c3a59d5 100644 --- a/backend/src/test/resources/backend_config.json +++ b/backend/src/test/resources/backend_config.json @@ -65,7 +65,7 @@ }, { "name": "pangoLineage", - "type": "pango_lineage", + "type": "string", "autocomplete": true }, { @@ -172,7 +172,7 @@ }, { "name": "pangoLineage", - "type": "pango_lineage", + "type": "string", "autocomplete": true }, { diff --git a/backend/src/test/resources/backend_config_single_segment.json b/backend/src/test/resources/backend_config_single_segment.json index d084daa578..f1d2c5dd6d 100644 --- a/backend/src/test/resources/backend_config_single_segment.json +++ b/backend/src/test/resources/backend_config_single_segment.json @@ -65,7 +65,7 @@ }, { "name": "pangoLineage", - "type": "pango_lineage", + "type": "string", "autocomplete": true }, { diff --git a/docs/src/content/docs/for-administrators/setup-with-kubernetes.md b/docs/src/content/docs/for-administrators/setup-with-kubernetes.md index a9268f6ab2..907ce24c9a 100644 --- a/docs/src/content/docs/for-administrators/setup-with-kubernetes.md +++ b/docs/src/content/docs/for-administrators/setup-with-kubernetes.md @@ -98,7 +98,7 @@ organisms: type: string autocomplete: true - name: pangoLineage - type: pango_lineage + type: string autocomplete: true required: true - name: insdcAccessionFull diff --git a/kubernetes/loculus/silo_import_job.sh b/kubernetes/loculus/silo_import_job.sh index 86206940df..7d1518c8c1 100755 --- a/kubernetes/loculus/silo_import_job.sh +++ b/kubernetes/loculus/silo_import_job.sh @@ -203,17 +203,17 @@ preprocessing() { cp "$new_input_data_path" "$silo_input_data_path" set +e - time /app/siloApi --preprocessing --preprocessingConfig=$preprocessing_config_file_merged + time /app/silo preprocessing --preprocessing-config=$preprocessing_config_file_merged exit_code=$? set -e if [ $exit_code -ne 0 ]; then - echo "SiloApi command failed with exit code $exit_code, cleaning up and exiting." + echo "silo command failed with exit code $exit_code, cleaning up and exiting." delete_all_input # Delete input so that we don't skip preprocessing next time due to hash equality exit $exit_code fi - echo "SiloApi command succeeded" + echo "silo command succeeded" echo "Removing touchfile $new_input_touchfile to indicate successful processing" rm "$new_input_touchfile" diff --git a/kubernetes/loculus/templates/lapis-silo-database-config.yaml b/kubernetes/loculus/templates/lapis-silo-database-config.yaml index fb3029dda1..bc055b89af 100644 --- a/kubernetes/loculus/templates/lapis-silo-database-config.yaml +++ b/kubernetes/loculus/templates/lapis-silo-database-config.yaml @@ -19,7 +19,6 @@ data: preprocessing_config.yaml: | ndjsonInputFilename: data.ndjson.zst - pangoLineageDefinitionFilename: pangolineage_alias.json referenceGenomeFilename: reference_genomes.json reference_genomes.json: | @@ -32,7 +31,4 @@ data: silo_import_wrapper.sh: | {{ range $importScriptWrapperLines }} {{ . }}{{ end }} - - pangolineage_alias.json: | - {{ $instance.pangolineage_alias | default dict | toJson }} {{- end }} \ No newline at end of file diff --git a/kubernetes/loculus/templates/lapis-silo-deployment.yaml b/kubernetes/loculus/templates/lapis-silo-deployment.yaml index 7d31bbf182..5d042cf090 100644 --- a/kubernetes/loculus/templates/lapis-silo-deployment.yaml +++ b/kubernetes/loculus/templates/lapis-silo-deployment.yaml @@ -33,7 +33,7 @@ spec: ports: - containerPort: 8081 args: - - "--api" + - "api" volumeMounts: - name: lapis-silo-shared-data mountPath: /data @@ -94,14 +94,11 @@ spec: mountPath: /preprocessing/input/reference_genomes.json subPath: reference_genomes.json - name: lapis-silo-database-config-processed - mountPath: /preprocessing/input/pangolineage_alias.json - subPath: pangolineage_alias.json + mountPath: /preprocessing/input/database_config.yaml + subPath: database_config.yaml - name: lapis-silo-database-config-processed mountPath: /app/preprocessing_config.yaml subPath: preprocessing_config.yaml - - name: lapis-silo-database-config-processed - mountPath: /app/database_config.yaml - subPath: database_config.yaml - name: lapis-silo-shared-data mountPath: /preprocessing/output - name: lapis-silo-input-data-cache diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 7c0ce9498d..a1e16c28d2 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1527,7 +1527,7 @@ bannerMessage: "This is a demonstration environment. It may contain non-accurate welcomeMessageHTML: null additionalHeadHTML: "" images: - lapisSilo: "ghcr.io/genspectrum/lapis-silo:0.3.2" + lapisSilo: "ghcr.io/genspectrum/lapis-silo:0.5.0" lapis: "ghcr.io/genspectrum/lapis:0.3.10" secrets: smtp-password: diff --git a/preprocessing/specification.md b/preprocessing/specification.md index 605ce8c3be..c12e82d5f7 100644 --- a/preprocessing/specification.md +++ b/preprocessing/specification.md @@ -123,7 +123,6 @@ The `metadata` field should contain a flat object consisting of the fields speci - `int` (integer) - `float` - `date` (supplied as a string with complete ISO-8601 date, e.g., "2023-08-30") -- `pango_lineage` (supplied as a string with a properly formatted SARS-CoV-2 Pango lineage, e.g., "B.1.1.7") - `authors` (comma separated list of authors, treated as a string in the current prepro pipeline) #### Sequences diff --git a/website/src/components/SearchPage/SearchForm.spec.tsx b/website/src/components/SearchPage/SearchForm.spec.tsx index 5852ed2805..5517a682f4 100644 --- a/website/src/components/SearchPage/SearchForm.spec.tsx +++ b/website/src/components/SearchPage/SearchForm.spec.tsx @@ -26,7 +26,7 @@ const defaultSearchFormFilters: MetadataFilter[] = [ }, { name: 'field3', - type: 'pango_lineage', + type: 'string', label: 'Field 3', autocomplete: true, initiallyVisible: true, diff --git a/website/src/components/SearchPage/SearchFullUI.spec.tsx b/website/src/components/SearchPage/SearchFullUI.spec.tsx index 5a1e0f002e..0d6c256007 100644 --- a/website/src/components/SearchPage/SearchFullUI.spec.tsx +++ b/website/src/components/SearchPage/SearchFullUI.spec.tsx @@ -49,7 +49,7 @@ const defaultSearchFormFilters: MetadataFilter[] = [ }, { name: 'field3', - type: 'pango_lineage', + type: 'string', label: 'Field 3', autocomplete: true, initiallyVisible: true, diff --git a/website/src/components/SearchPage/fields/FieldProps.tsx b/website/src/components/SearchPage/fields/FieldProps.tsx index d9d7927558..54adc7afb3 100644 --- a/website/src/components/SearchPage/fields/FieldProps.tsx +++ b/website/src/components/SearchPage/fields/FieldProps.tsx @@ -10,5 +10,5 @@ export type FieldProps = { onFocus?: FocusEventHandler; onBlur?: FocusEventHandler; fieldValue: string; - type?: 'string' | 'boolean' | 'float' | 'int' | 'pango_lineage' | 'authors'; + type?: 'string' | 'boolean' | 'float' | 'int' | 'authors'; }; diff --git a/website/src/components/SearchPage/fields/NormalTextField.tsx b/website/src/components/SearchPage/fields/NormalTextField.tsx index df5dfa1a41..115cc36454 100644 --- a/website/src/components/SearchPage/fields/NormalTextField.tsx +++ b/website/src/components/SearchPage/fields/NormalTextField.tsx @@ -10,7 +10,7 @@ export type NormalFieldProps = { onFocus?: FocusEventHandler; onBlur?: FocusEventHandler; fieldValue: string | number; - type?: 'string' | 'boolean' | 'float' | 'int' | 'pango_lineage' | 'authors'; + type?: 'string' | 'boolean' | 'float' | 'int' | 'authors'; }; export const NormalTextField = forwardRef((props, ref) => { diff --git a/website/src/types/config.ts b/website/src/types/config.ts index 688ac04206..ae069935b4 100644 --- a/website/src/types/config.ts +++ b/website/src/types/config.ts @@ -4,16 +4,7 @@ import { mutationProportionCount, orderByType } from './lapis.ts'; import { referenceGenomes } from './referencesGenomes.ts'; // These metadata types need to be kept in sync with the backend config class `MetadataType` in Config.kt -const metadataPossibleTypes = z.enum([ - 'string', - 'date', - 'int', - 'float', - 'pango_lineage', - 'timestamp', - 'boolean', - 'authors', -] as const); +const metadataPossibleTypes = z.enum(['string', 'date', 'int', 'float', 'timestamp', 'boolean', 'authors'] as const); export const segmentedMutations = z.object({ segment: z.string(),