run testing (#4681)

* run testing

* need to break on success

* add a readme

* raise vespa to 6GB

* allow test to retry

* add 20 attempts

* put memory limits back to normal

* restore chart testing on changes only

* increase retries to 40

---------

Co-authored-by: Richard Kuo (Onyx) <rkuo@onyx.app>

.github/workflows/pr-helm-chart-testing.yml

@@ -37,6 +37,11 @@ jobs:
           echo "changed=true" >> "$GITHUB_OUTPUT"
         fi
 
+    # uncomment to force run chart-testing
+#     - name: Force run chart-testing (list-changed)
+#       id: list-changed
+#       run: echo "changed=true" >> $GITHUB_OUTPUT
+        
     # lint all charts if any changes were detected
     - name: Run chart-testing (lint)
       if: steps.list-changed.outputs.changed == 'true'

backend/onyx/db/swap_index.py

@@ -95,6 +95,7 @@ def _perform_index_swap(
 
             logger.notice("Vespa index swap complete.")
             success = True
+            break
         except Exception:
             logger.exception(
                 f"Vespa index swap did not succeed. The Vespa service may not be ready yet. Retrying in {WAIT_SECONDS} seconds."

deployment/helm/README.md

@@ -0,0 +1,13 @@
+# Local testing
+
+- One time setup
+- brew install kind
+- Ensure you have no config at ~/.kube/config
+- kind create cluster
+- mv ~/.kube/config ~/.kube/kind-config
+
+- Command setup
+- export KUBECONFIG=~/.kube/kind-config
+- kubectl config use-context kind-kind
+- from source root run
+- ct install --all --helm-extra-set-args="--set=nginx.enabled=false" --debug --config ct.yaml

deployment/helm/charts/onyx/templates/tests/test-connection.yaml

@@ -10,6 +10,16 @@ spec:
   containers:
     - name: wget
       image: busybox
-      command: ['wget']
-      args: ['{{ include "onyx-stack.fullname" . }}-webserver:{{ .Values.webserver.service.servicePort }}']
+      command:
+        - /bin/sh
+        - -c
+      args:
+        - |
+          for i in $(seq 1 40); do
+            echo "Attempt $i: wget {{ include "onyx-stack.fullname" . }}-webserver:{{ .Values.webserver.service.servicePort }}"
+            wget {{ include "onyx-stack.fullname" . }}-webserver:{{ .Values.webserver.service.servicePort }} && exit 0
+            sleep 15
+          done
+          echo "Service unavailable after 40 attempts"
+          exit 1
   restartPolicy: Never