first cut at redis (#2226)

* first cut at redis

* fix startup dependencies on redis

* kombu cleanup - fail silently

* mypy

* add redis_host environment override

* update REDIS_HOST env var in docker-compose.dev.yml

* update the rest of the docker files

* update contributing guide

* renaming cache to cache_volume

* add redis password to various deployments

* try setting up pr testing for helm

* fix indent

* hopefully this release version actually exists

* fix command line option to --chart-dirs

* fetch-depth 0

* edit values.yaml

* try setting ct working directory

* bypass testing only on change for now

* move files and lint them

* update helm testing

* some issues suggest using --config works

* add vespa repo

* add postgresql repo

* increase timeout

* try amd64 runner

* fix redis password reference

* add comment to helm chart testing workflow

* rename helm testing workflow to disable it

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>

deployment/docker_compose/docker-compose.dev.yml

@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     ports:
@@ -62,6 +63,7 @@ services:
       # Other services
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -107,6 +109,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
@@ -137,6 +140,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
       - POSTGRES_DB=${POSTGRES_DB:-}
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose for OAuth2 connectors
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -330,9 +334,19 @@ services:
     # in order to make this work on both Unix-like systems and windows
     command: > 
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
-      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
-        
+      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"  
+
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache_volume:/data
+
 volumes:
+  cache_volume:
   db_volume:
   vespa_volume: # Created by the container itself
 

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     ports:
@@ -58,6 +59,7 @@ services:
       # Other services
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache      
       - WEB_DOMAIN=${WEB_DOMAIN:-}  # For frontend redirect auth purpose
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -99,6 +101,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
@@ -129,6 +132,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
       - POSTGRES_DB=${POSTGRES_DB:-}
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-}  # For frontend redirect auth purpose for OAuth2 connectors
       # Don't change the NLP model configs unless you know what you're doing
       - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
@@ -341,9 +345,20 @@ services:
     command: > 
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
       && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
-        
+
+
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache_volume:/data
+
 
 volumes:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself

deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml

@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     env_file:
@@ -20,6 +21,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -39,6 +41,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
@@ -48,6 +51,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
     extra_hosts:
@@ -204,7 +208,18 @@ services:
       - .env.nginx
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache_volume:/data
+
+
 volumes:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself

deployment/docker_compose/docker-compose.prod.yml

@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
     restart: always
     env_file:
@@ -20,6 +21,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -39,6 +41,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
       - inference_model_server
       - indexing_model_server
     restart: always
@@ -48,6 +51,7 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
     extra_hosts:
@@ -221,7 +225,18 @@ services:
     entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache_volume:/data
+
+
 volumes:
+  cache_volume:
   db_volume:
   vespa_volume:
   # Created by the container itself

deployment/docker_compose/docker-compose.search-testing.yml

@@ -12,6 +12,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
     restart: always
     ports:
       - "8080"
@@ -21,6 +22,7 @@ services:
       - AUTH_TYPE=disabled
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - ENV_SEED_CONFIGURATION=${ENV_SEED_CONFIGURATION:-}
@@ -43,6 +45,7 @@ services:
     depends_on:
       - relational_db
       - index
+      - cache
     restart: always
     env_file:
       - .env_eval
@@ -50,6 +53,7 @@ services:
       - AUTH_TYPE=disabled
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
@@ -200,7 +204,18 @@ services:
       && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
 
 
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server
+    volumes: 
+      - cache_volume:/data
+
+
 volumes:
+  cache_volume:
   db_volume:
     driver: local
     driver_opts:

deployment/helm/charts/danswer/Chart.yaml

@@ -22,14 +22,11 @@ dependencies:
     version: 14.3.1
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
-  - name: vespa 
+  - name: vespa
     version: 0.2.3
     repository: https://unoplat.github.io/vespa-helm-charts
     condition: vespa.enabled
   - name: nginx
     version: 15.14.0
     repository: oci://registry-1.docker.io/bitnamicharts
-    condition: nginx.enabled  
-    
-
-    
+    condition: nginx.enabled

deployment/helm/charts/danswer/values.yaml

@@ -84,7 +84,7 @@ postgresql:
   auth:
     existingSecret: danswer-secrets
     secretKeys:
-      adminPasswordKey: postgres_password #overwriting as postgres typically expects 'postgres-password'
+      adminPasswordKey: postgres_password  # overwriting as postgres typically expects 'postgres-password'
 
 nginx:
   containerPorts:
@@ -330,7 +330,7 @@ vespa:
   affinity: {}
 
 
-#ingress:
+# ingress:
 #  enabled: false
 #  className: ""
 #  annotations: {}
@@ -358,8 +358,10 @@ persistence:
 auth:
   # for storing smtp, oauth, slack, and other secrets
   # keys are lowercased version of env vars (e.g. SMTP_USER -> smtp_user)
-  existingSecret: "" # danswer-secrets
+  existingSecret: ""  # danswer-secrets
   # optionally override the secret keys to reference in the secret
+  # this is used to populate the env vars in individual deployments
+  # the values here reference the keys in secrets below
   secretKeys:
     postgres_password: "postgres_password"
     smtp_pass: ""
@@ -369,9 +371,11 @@ auth:
     gen_ai_api_key: ""
     danswer_bot_slack_app_token: ""
     danswer_bot_slack_bot_token: ""
+    redis_password: "redis_password"
   # will be overridden by the existingSecret if set
   secretName: "danswer-secrets"
   # set values as strings, they will be base64 encoded
+  # this is used to populate the secrets yaml
   secrets:
     postgres_password: "postgres"
     smtp_pass: ""
@@ -381,13 +385,14 @@ auth:
     gen_ai_api_key: ""
     danswer_bot_slack_app_token: ""
     danswer_bot_slack_bot_token: ""
+    redis_password: "password"
 
 configMap:
   AUTH_TYPE: "disabled"  # Change this for production uses unless Danswer is only accessible behind VPN
   SESSION_EXPIRE_TIME_SECONDS: "86400"  # 1 Day Default
   VALID_EMAIL_DOMAINS: ""  # Can be something like danswer.ai, as an extra double-check
-  SMTP_SERVER: "" # For sending verification emails, if unspecified then defaults to 'smtp.gmail.com'
-  SMTP_PORT: "" # For sending verification emails, if unspecified then defaults to '587'
+  SMTP_SERVER: ""  # For sending verification emails, if unspecified then defaults to 'smtp.gmail.com'
+  SMTP_PORT: ""  # For sending verification emails, if unspecified then defaults to '587'
   SMTP_USER: ""  # 'your-email@company.com'
   # SMTP_PASS: ""  # 'your-gmail-password'
   EMAIL_FROM: ""  # 'your-email@company.com' SMTP_USER missing used instead

deployment/kubernetes/api_server-service-deployment.yaml

@@ -52,6 +52,11 @@ spec:
             secretKeyRef:
               name: danswer-secrets
               key: google_oauth_client_secret
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap

deployment/kubernetes/background-deployment.yaml

@@ -19,6 +19,12 @@ spec:
         command: ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
         # There are some extra values since this is shared between services
         # There are no conflicts though, extra env variables are simply ignored
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap

deployment/kubernetes/env-configmap.yaml

@@ -31,6 +31,7 @@ data:
   # Other Services
   POSTGRES_HOST: "relational-db-service"
   VESPA_HOST: "document-index-service"
+  REDIS_HOST: "redis-service"
   # Internet Search Tool
   BING_API_KEY: ""
   # Don't change the NLP models unless you know what you're doing

deployment/kubernetes/redis-service-deployment.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-service
+spec:
+  selector:
+    app: redis
+  ports:
+    - name: redis
+      protocol: TCP
+      port: 6379
+      targetPort: 6379
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - name: redis
+        image: redis:7.4-alpine
+        ports:
+        - containerPort: 6379
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
+        command: ["redis-server"]
+        args: ["--requirepass", "$(REDIS_PASSWORD)"]

deployment/kubernetes/secrets.yaml

@@ -8,4 +8,6 @@ data:
   postgres_user: cG9zdGdyZXM= # "postgres" base64 encoded
   postgres_password: cGFzc3dvcmQ= # "password" base64 encoded
   google_oauth_client_id: ZXhhbXBsZS1jbGllbnQtaWQ= # "example-client-id" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
-  google_oauth_client_secret: example_google_oauth_secret # "example-client-secret" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
+  google_oauth_client_secret: ZXhhbXBsZV9nb29nbGVfb2F1dGhfc2VjcmV0 # "example-client-secret" base64 encoded. You will need to provide this, use echo -n "your-client-id" | base64
+  redis_password: cGFzc3dvcmQ= # "password" base64 encoded
+  

deployment/kubernetes/web_server-service-deployment.yaml

@@ -33,6 +33,12 @@ spec:
         - containerPort: 3000
         # There are some extra values since this is shared between services
         # There are no conflicts though, extra env variables are simply ignored
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: danswer-secrets
+              key: redis_password
         envFrom:
         - configMapRef:
             name: env-configmap