Evaluation Pipeline Touchup (#1783)

backend/.gitignore

@@ -9,4 +9,3 @@ api_keys.py
 vespa-app.zip
 dynamic_config_storage/
 celerybeat-schedule*
-search_test_config.yaml

backend/tests/regression/answer_quality/cli_utils.py

@@ -68,6 +68,7 @@ def set_env_variables(
     if use_cloud_gpu:
         env_vars["MODEL_SERVER_HOST"] = remote_server_ip
         env_vars["MODEL_SERVER_PORT"] = remote_server_port
+        env_vars["INDEXING_MODEL_SERVER_HOST"] = remote_server_ip
 
     for env_var_name, env_var in env_vars.items():
         os.environ[env_var_name] = env_var
@@ -78,10 +79,9 @@ def start_docker_compose(
     run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool
 ) -> None:
     print("Starting Docker Compose...")
-    os.chdir(os.path.expanduser("~/danswer/deployment/docker_compose"))
+    os.chdir("../deployment/docker_compose")
     command = f"docker compose -f docker-compose.search-testing.yml -p danswer-stack{run_suffix} up -d"
     command += " --build"
-    command += " --pull always"
     command += " --force-recreate"
     if not launch_web_ui:
         command += " --scale web_server=0"

backend/tests/regression/answer_quality/relari.py

@@ -105,4 +105,8 @@ def main() -> None:
 
 
 if __name__ == "__main__":
+    """
+    To run a different set of questions, update the questions_file in search_test_config.yaml
+    If there is more than one instance of Danswer running, specify the suffix in search_test_config.yaml
+    """
     main()

backend/tests/regression/answer_quality/search_test_config.yaml.template

@@ -1,3 +1,6 @@
+# Copy this to search_test_config.yaml and fill in the values to run the eval pipeline
+# Don't forget to also update the .env_eval file with the correct values
+
 # Directory where test results will be saved
 output_folder: "~/danswer_test_results"
 

deployment/docker_compose/docker-compose.search-testing.yml

@@ -22,6 +22,7 @@ services:
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=True
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -49,6 +50,7 @@ services:
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=True
     extra_hosts: