DAN-80 Example Env files (#48)

Also added alembic migrations running automatically

deployment/README.md

@@ -1,7 +1,20 @@
 This serves as an example for how to deploy everything on a single machine. This is
 not optimal, but can get you started easily and cheaply. To run:
 
-1. Set up a `.env` + `.env.nginx` file in this directory with relevant environment variables
-  a. TODO: add description of required variables
-2. `chmod +x init-letsencrypt.sh` + `./init-letsencrypt.sh` to setup https certificate
-2. `docker compose up -d --build` to start nginx, postgres, web/api servers, and the background indexing job
+1. Set up a `.env` file in this directory with relevant environment variables.
+   - Use the `env.template` as a reference.
+2. SKIP this step if running locally. If you are running this for production and need https do the following:
+   - Set up a `.env.nginx` file in this directory based on `env.nginx.template`.
+   - `chmod +x init-letsencrypt.sh` + `./init-letsencrypt.sh` to set up https certificate.
+3. Run one of the docker compose commands below depending on your environment:
+   - For Local:
+     - `docker compose -f docker-compose.dev.yml -p danswer-stack up -d --build`
+     - This will start Web/API servers, Postgres (backend DB), Qdrant (vector DB), and the background indexing job.
+   - For Prod:
+     - `docker compose -f docker-compose.prod.yml -p danswer-stack up -d --build`
+     - This will additionally run certbot and start Nginx.
+4. To shut down the deployment run:
+   - For Local:
+     - `docker compose -f docker-compose.dev.yml -p danswer-stack down`
+   - For Prod:
+     - `docker compose -f docker-compose.prod.yml -p danswer-stack down`

deployment/data/nginx/app.conf.template

@@ -7,11 +7,11 @@ upstream app_server {
 
     # for a TCP configuration
     # TODO: use gunicorn to manage multiple processes
-    server api:8080 fail_timeout=0;
+    server api_server:8080 fail_timeout=0;
 }
 
 upstream web_server {
-    server web:3000 fail_timeout=0;
+    server web_server:3000 fail_timeout=0;
 }
 
 server {

deployment/docker-compose.dev.yml

@@ -0,0 +1,64 @@
+version: '3'
+services:
+  api_server:
+    build:
+      context: ../backend
+      dockerfile: Dockerfile
+    depends_on:
+      - relational_db
+      - vector_db
+    restart: always
+    ports:
+      - "8080:8080"
+    env_file:
+      - .env
+    environment:
+      - POSTGRES_HOST=relational_db
+      - QDRANT_HOST=vector_db
+    volumes:
+      - local_dynamic_storage:/home/storage
+  background:
+    build:
+      context: ../backend
+      dockerfile: Dockerfile.background
+    depends_on:
+      - relational_db
+      - vector_db
+    restart: always
+    env_file:
+      - .env
+    environment:
+      - POSTGRES_HOST=relational_db
+      - QDRANT_HOST=vector_db
+    volumes:
+      - local_dynamic_storage:/home/storage
+  web_server:
+    build:
+      context: ../web
+      dockerfile: Dockerfile
+    depends_on:
+      - api_server
+    restart: always
+    environment:
+      - INTERNAL_AUTH_URL=http://api_server:8080
+    ports:
+      - "3000:3000"
+  relational_db:
+    image: postgres:15.2-alpine
+    restart: always
+    # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
+    env_file:
+      - .env
+    ports:
+      - "5432:5432"
+    volumes:
+      - db_volume:/var/lib/postgresql/data
+  vector_db:
+    image: qdrant/qdrant:v1.1.3
+    restart: always
+    volumes:
+      - qdrant_volume:/qdrant/storage
+volumes:
+  local_dynamic_storage:
+  db_volume:
+  qdrant_volume:

deployment/docker-compose.prod.yml

@@ -1,20 +1,18 @@
 version: '3'
 services:
-  api:
+  api_server:
     build:
       context: ../backend
       dockerfile: Dockerfile
     depends_on:
-      - db
-      - qdrant
-    # uncomment for local testing
-    # ports:
-    #   - "8080:8080"
+      - relational_db
+      - vector_db
+    restart: always
     env_file:
       - .env
     environment:
-      - POSTGRES_HOST=db
-      - QDRANT_HOST=qdrant
+      - POSTGRES_HOST=relational_db
+      - QDRANT_HOST=vector_db
     volumes:
       - local_dynamic_storage:/home/storage
   background:
@@ -22,24 +20,26 @@ services:
       context: ../backend
       dockerfile: Dockerfile.background
     depends_on:
-      - db
-      - qdrant
+      - relational_db
+      - vector_db
+    restart: always
     env_file:
       - .env
     environment:
-      - POSTGRES_HOST=db
-      - QDRANT_HOST=qdrant
+      - POSTGRES_HOST=relational_db
+      - QDRANT_HOST=vector_db
     volumes:
       - local_dynamic_storage:/home/storage
-  web:
+  web_server:
     build:
       context: ../web
       dockerfile: Dockerfile
     depends_on:
-      - api
+      - api_server
+    restart: always
     environment:
-      - INTERNAL_AUTH_URL=http://api:8080
-  db:
+      - INTERNAL_AUTH_URL=http://api_server:8080
+  relational_db:
     image: postgres:15.2-alpine
     restart: always
     # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
@@ -47,14 +47,16 @@ services:
       - .env
     ports:
       - "5432:5432"
-    volumes: 
+    volumes:
       - db_volume:/var/lib/postgresql/data
-  qdrant:
+  vector_db:
     image: qdrant/qdrant:v1.1.3
+    restart: always
     volumes:
       - qdrant_volume:/qdrant/storage
   nginx:
     image: nginx:1.23.4-alpine
+    restart: always
     ports:
       - "80:80"
       - "443:443"
@@ -70,6 +72,7 @@ services:
   # follows https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
   certbot:
     image: certbot/certbot
+    restart: always
     volumes:
       - ./data/certbot/conf:/etc/letsencrypt
       - ./data/certbot/www:/var/www/certbot

deployment/env.dev.template

@@ -0,0 +1,21 @@
+# Fill in the values and copy the contents of this file to .env in the deployment directory
+# Some valid default values are provided where applicable, delete the variables which you don't set values for
+
+
+# Insert your OpenAI API key here, currently the only Generative AI endpoint for QA that we support is OpenAI
+OPENAI_API_KEY=
+
+# Choose between "openai-chat-completion" and "openai-completion"
+INTERNAL_MODEL_VERSION=openai-chat-completion
+
+# Use a valid model for the choice above, consult https://platform.openai.com/docs/models/model-endpoint-compatibility
+OPENAPI_MODEL_VERSION=gpt-3.5-turbo
+
+
+# Can leave these as defaults
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+
+
+# Auth not necessary for local
+DISABLE_AUTH=True

deployment/env.nginx.template

@@ -0,0 +1,3 @@
+# DOMAIN is necessary for https setup, EMAIL is optional
+DOMAIN=
+EMAIL=

deployment/env.prod.template

@@ -0,0 +1,52 @@
+# Fill in the values and copy the contents of this file to .env in the deployment directory
+# Some valid default values are provided where applicable, delete the variables which you don't set values for
+
+
+# THE SECTION BELOW INCLUDE MUST HAVE CONFIGS
+# Insert your OpenAI API key here, currently the only Generative AI endpoint for QA that we support is OpenAI
+OPENAI_API_KEY=
+# Choose between "openai-chat-completion" and "openai-completion"
+INTERNAL_MODEL_VERSION=openai-chat-completion
+# Use a valid model for the choice above, consult https://platform.openai.com/docs/models/model-endpoint-compatibility
+OPENAPI_MODEL_VERSION=gpt-4
+
+# Could be something like danswer.companyname.com. Requires additional setup if not localhost
+WEB_DOMAIN=http://localhost:3000
+
+
+# CONNECTOR CONFIGS (set for the ones you are using, delete the others)
+GITHUB_ACCESS_TOKEN=
+
+GOOGLE_DRIVE_CREDENTIAL_JSON=
+GOOGLE_DRIVE_TOKENS_JSON=
+
+
+# BACKEND DB can leave these as defaults
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+
+
+# AUTH CONFIGS
+DISABLE_AUTH=False
+# Feel free remove everything after if DISABLE_AUTH=True
+
+# Currently frontend page doesn't have basic auth, use OAuth if user auth is enabled.
+ENABLE_OAUTH=True
+# The two settings below are only required if ENABLE_OAUTH is true
+GOOGLE_OAUTH_CLIENT_ID=
+GOOGLE_OAUTH_CLIENT_SECRET=
+
+# Used to generate values for security verification, use a random string
+SECRET=
+
+# How long before user needs to reauthenticate, default to 1 day. (cookie expiration time)
+SESSION_EXPIRE_TIME_SECONDS=86400
+
+# Only relevant if using basic auth
+REQUIRE_EMAIL_VERIFICATION=True
+# The five settings below are only required if REQUIRE_EMAIL_VERIFICATION is True
+VALID_EMAIL_DOMAIN=
+SMTP_SERVER=
+SMTP_PORT=
+SMTP_USER=
+SMTP_PASS=

deployment/init-letsencrypt.sh

@@ -6,7 +6,7 @@ set -o allexport
 source .env.nginx
 set +o allexport
 
-if ! [ -x "$(command -v docker compose)" ]; then
+if ! docker compose --version >/dev/null 2>&1; then
   echo 'Error: docker compose is not installed.' >&2
   exit 1
 fi