DAN-81 Improve search round 2 (#82)

Includes:
- Multi vector indexing/search
- Ensemble model reranking
- Keyword Search backend

deployment/.env

@@ -1,5 +1,2 @@
-# For a local deployment, no additional setup is needed
-# Refer to env.dev.template and env.prod.template for additional options
-
-# Setting Auth to false for local setup convenience to avoid setting up Google OAuth app in GPC.
-DISABLE_AUTH=True
+# This empty .env file is provided for compatibility with older Docker/Docker-Compose installations
+# To change default values, check env.dev.template or env.prod.template

deployment/docker-compose.dev.yml

@@ -7,6 +7,7 @@ services:
     depends_on:
       - relational_db
       - vector_db
+      - search_engine
     restart: always
     ports:
       - "8080:8080"
@@ -15,6 +16,9 @@ services:
     environment:
       - POSTGRES_HOST=relational_db
       - QDRANT_HOST=vector_db
+      - TYPESENSE_HOST=search_engine
+      - TYPESENSE_API_KEY=${TYPESENSE_API_KEY:-local_dev_typesense}
+      - DISABLE_AUTH=True
     volumes:
       - local_dynamic_storage:/home/storage
   background:
@@ -43,12 +47,13 @@ services:
       - .env
     environment:
       - INTERNAL_URL=http://api_server:8080
+      - DISABLE_AUTH=True
   relational_db:
     image: postgres:15.2-alpine
     restart: always
     environment:
-      POSTGRES_USER: ${POSTGRES_USER:-postgres}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
     env_file:
       - .env
     ports:
@@ -58,10 +63,24 @@ services:
   vector_db:
     image: qdrant/qdrant:v1.1.3
     restart: always
+    env_file:
+      - .env
     ports:
       - "6333:6333"
     volumes:
       - qdrant_volume:/qdrant/storage
+  search_engine:
+    image: typesense/typesense:0.24.1
+    restart: always
+    environment:
+      - TYPESENSE_API_KEY=${TYPESENSE_API_KEY:-local_dev_typesense}
+      - TYPESENSE_DATA_DIR=/typesense/storage
+    env_file:
+      - .env
+    ports:
+      - "8108:8108"
+    volumes:
+      - typesense_volume:/typesense/storage
   nginx:
     image: nginx:1.23.4-alpine
     restart: always
@@ -82,3 +101,4 @@ volumes:
   local_dynamic_storage:
   db_volume:
   qdrant_volume:
+  typesense_volume:

deployment/docker-compose.prod.yml

@@ -7,12 +7,15 @@ services:
     depends_on:
       - relational_db
       - vector_db
+      - search_engine
     restart: always
     env_file:
       - .env
     environment:
       - POSTGRES_HOST=relational_db
       - QDRANT_HOST=vector_db
+      - TYPESENSE_HOST=search_engine
+      - TYPESENSE_API_KEY=${TYPESENSE_API_KEY:-local_dev_typesense}
     volumes:
       - local_dynamic_storage:/home/storage
   background:
@@ -54,8 +57,22 @@ services:
   vector_db:
     image: qdrant/qdrant:v1.1.3
     restart: always
+    env_file:
+      - .env
     volumes:
       - qdrant_volume:/qdrant/storage
+  search_engine:
+    image: typesense/typesense:0.24.1
+    restart: always
+    # TYPESENSE_API_KEY must be set in .env file
+    environment:
+      - TYPESENSE_DATA_DIR=/typesense/storage
+    env_file:
+      - .env
+    ports:
+      - "8108:8108"
+    volumes:
+      - typesense_volume:/typesense/storage
   nginx:
     image: nginx:1.23.4-alpine
     restart: always
@@ -83,3 +100,4 @@ volumes:
   local_dynamic_storage:
   db_volume:
   qdrant_volume:
+  typesense_volume:

deployment/env.dev.template

@@ -1,13 +1,8 @@
 # Very basic .env file with options that are easy to change. Allows you to deploy everything on a single machine.
-# We don't suggest using these settings for production.
-
+# .env is not required unless you wish to change defaults
 
 # Choose between "openai-chat-completion" and "openai-completion"
 INTERNAL_MODEL_VERSION=openai-chat-completion
 
 # Use a valid model for the choice above, consult https://platform.openai.com/docs/models/model-endpoint-compatibility
 OPENAPI_MODEL_VERSION=gpt-3.5-turbo
-
-
-# Auth not necessary for local
-DISABLE_AUTH=True

deployment/env.prod.template

@@ -13,14 +13,8 @@ OPENAI_MODEL_VERSION=gpt-4
 # Could be something like danswer.companyname.com. Requires additional setup if not localhost
 WEB_DOMAIN=http://localhost:3000
 
-
-# BACKEND DB can leave these as defaults
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=password
-
-
-# AUTH CONFIGS
-DISABLE_AUTH=False
+# Required
+TYPESENSE_API_KEY=
 
 # Currently frontend page doesn't have basic auth, use OAuth if user auth is enabled.
 ENABLE_OAUTH=True