From d35d7ee83307eac64f28ab2053ae5150e414c2f5 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 6 Jul 2024 13:17:05 -0700
Subject: [PATCH] Evaluation Pipeline Touchup (#1783)

---
 backend/.gitignore                                            | 1 -
 backend/tests/regression/answer_quality/cli_utils.py          | 4 ++--
 backend/tests/regression/answer_quality/relari.py             | 4 ++++
 ...arch_test_config.yaml => search_test_config.yaml.template} | 3 +++
 deployment/docker_compose/docker-compose.search-testing.yml   | 2 ++
 5 files changed, 11 insertions(+), 3 deletions(-)
 rename backend/tests/regression/answer_quality/{search_test_config.yaml => search_test_config.yaml.template} (88%)

diff --git a/backend/.gitignore b/backend/.gitignore
index f42f056d8..b1c4f4db7 100644
--- a/backend/.gitignore
+++ b/backend/.gitignore
@@ -9,4 +9,3 @@ api_keys.py
 vespa-app.zip
 dynamic_config_storage/
 celerybeat-schedule*
-search_test_config.yaml
diff --git a/backend/tests/regression/answer_quality/cli_utils.py b/backend/tests/regression/answer_quality/cli_utils.py
index 3d15ce3bb..c66cfc1df 100644
--- a/backend/tests/regression/answer_quality/cli_utils.py
+++ b/backend/tests/regression/answer_quality/cli_utils.py
@@ -68,6 +68,7 @@ def set_env_variables(
     if use_cloud_gpu:
         env_vars["MODEL_SERVER_HOST"] = remote_server_ip
         env_vars["MODEL_SERVER_PORT"] = remote_server_port
+        env_vars["INDEXING_MODEL_SERVER_HOST"] = remote_server_ip
 
     for env_var_name, env_var in env_vars.items():
         os.environ[env_var_name] = env_var
@@ -78,10 +79,9 @@ def start_docker_compose(
     run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool
 ) -> None:
     print("Starting Docker Compose...")
-    os.chdir(os.path.expanduser("~/danswer/deployment/docker_compose"))
+    os.chdir("../deployment/docker_compose")
     command = f"docker compose -f docker-compose.search-testing.yml -p danswer-stack{run_suffix} up -d"
     command += " --build"
-    command += " --pull always"
     command += " --force-recreate"
     if not launch_web_ui:
         command += " --scale web_server=0"
diff --git a/backend/tests/regression/answer_quality/relari.py b/backend/tests/regression/answer_quality/relari.py
index 0894a0eb1..21e9d3838 100644
--- a/backend/tests/regression/answer_quality/relari.py
+++ b/backend/tests/regression/answer_quality/relari.py
@@ -105,4 +105,8 @@ def main() -> None:
 
 
 if __name__ == "__main__":
+    """
+    To run a different set of questions, update the questions_file in search_test_config.yaml
+    If there is more than one instance of Danswer running, specify the suffix in search_test_config.yaml
+    """
     main()
diff --git a/backend/tests/regression/answer_quality/search_test_config.yaml b/backend/tests/regression/answer_quality/search_test_config.yaml.template
similarity index 88%
rename from backend/tests/regression/answer_quality/search_test_config.yaml
rename to backend/tests/regression/answer_quality/search_test_config.yaml.template
index 4711b0d54..2e8e7ca27 100644
--- a/backend/tests/regression/answer_quality/search_test_config.yaml
+++ b/backend/tests/regression/answer_quality/search_test_config.yaml.template
@@ -1,3 +1,6 @@
+# Copy this to search_test_config.yaml and fill in the values to run the eval pipeline
+# Don't forget to also update the .env_eval file with the correct values
+
 # Directory where test results will be saved
 output_folder: "~/danswer_test_results"
 
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index e540ab82c..186c47ebc 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -22,6 +22,7 @@ services:
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=True
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -49,6 +50,7 @@ services:
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=True
     extra_hosts: