Increase curl retries to fix CI flakiness

charts/qdrant/README.md

@@ -92,6 +92,19 @@ Some storage providers allow resizing volumes in-place, but most require a pod r
 kubectl rollout restart statefulset qdrant
 ```
 
+### Immutable Pod fields
+
+In addition to immutable fields on StatefulSets, Pods also have some fields which are immutable, which means the above method may not work for some changes, such as setting `snapshotPersistence.enabled: true`. In that case, after following the above method, you'll see an error like this when you `kubectl describe` your StatefulSet:
+
+```
+pod updates may not change fields other than `spec.containers[*].image`,
+`spec.initContainers[*].image`,`spec.activeDeadlineSeconds`,
+`spec.tolerations` (only additions to existing tolerations),
+`spec.terminationGracePeriodSeconds` (allow it to be set to 1 if it was previously negative)
+```
+
+To fix this, you must manually delete all of your Qdrant pods, starting with node-0. This will cause your cluster to go down, but will allow the StatefulSet to recreate your Pods with the correct configuration.
+
 ## Restoring from Snapshots
 
 This helm chart allows you to restore a snapshot into your Qdrant cluster either from an internal or external PersistentVolumeClaim.

charts/qdrant/templates/tests/test-db-interaction.yaml

@@ -41,9 +41,15 @@ data:
   entrypoint.sh: |
     #!/bin/bash
     set -xe
+    # Kind's networking is very flaky
     echo 'connect-timeout = 5' > $HOME/.curlrc
-    echo 'retry = 5' >> $HOME/.curlrc
+    echo 'retry = 60' >> $HOME/.curlrc
+    echo 'retry-delay = 5' >> $HOME/.curlrc
     echo 'retry-all-errors' >> $HOME/.curlrc
+    # Don't clutter the logs with progress bars
+    echo 'no-progress-meter' >> $HOME/.curlrc
+    # Ensure errors cause the script to fail, but show the response body
+    echo 'fail-with-body' >> $HOME/.curlrc
 
     if [ -d /mnt/secrets/certs ]; then
       cp /mnt/secrets/certs/ca.pem /usr/share/pki/trust/anchors/private-ca.pem
@@ -53,11 +59,11 @@ data:
     QDRANT_COLLECTION="test_collection"
     {{- range .Values.service.ports }}
     {{- if eq .name "http" }}
-    echo "Connecting to {{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
-    QDRANT_URL="http://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    echo "Connecting to {{ include "qdrant.fullname" $root }}:{{ .port }}"
+    QDRANT_URL="http://{{ include "qdrant.fullname" $root }}:{{ .port }}"
     {{- if and $root.Values.config.service $root.Values.config.service.enable_tls }}
     echo "Using https"
-    QDRANT_URL="https://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    QDRANT_URL="https://{{ include "qdrant.fullname" $root }}:{{ .port }}"
     {{- end }}
     {{- end }}
     {{- end }}
@@ -76,7 +82,7 @@ data:
     -H 'Content-Type: application-json' \
     -d '{"vectors":{"size":4,"distance":"Dot"}}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION} --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}
 
     # Insert points
     curl -X PUT \
@@ -90,11 +96,11 @@ data:
       {"id":6,"vector":[0.35, 0.08, 0.11, 0.44],"payload":{"city":"Mumbai"}}
     ]}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points
 
     # Run query
     curl -X POST \
     -H 'Content-Type: application-json' \
     -d '{"vector":[0.2, 0.1, 0.9, 0.7],"limit":3}' \
     -H  "${API_KEY_HEADER}" \
-    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search --fail-with-body
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search

test/integration/snapshot_persistence.bats

@@ -32,4 +32,5 @@ setup_file() {
 
 teardown_file() {
     helm delete qdrant -n qdrant-helm-integration || true
+    kubectl wait --for=delete pod/qdrant-0 -n qdrant-helm-integration --timeout=300s
 }