Search Eval (#1873)

backend/tests/regression/answer_quality/cli_utils.py

@@ -144,7 +144,7 @@ def _is_port_in_use(port: int) -> bool:
 
 
 def start_docker_compose(
-    run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool
+    run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool, only_state: bool = False
 ) -> None:
     print("Starting Docker Compose...")
     os.chdir(os.path.dirname(__file__))
@@ -152,18 +152,22 @@ def start_docker_compose(
     command = f"docker compose -f docker-compose.search-testing.yml -p danswer-stack{run_suffix} up -d"
     command += " --build"
     command += " --force-recreate"
-    if use_cloud_gpu:
-        command += " --scale indexing_model_server=0"
-        command += " --scale inference_model_server=0"
-    if launch_web_ui:
-        web_ui_port = 3000
-        while _is_port_in_use(web_ui_port):
-            web_ui_port += 1
-        print(f"UI will be launched at http://localhost:{web_ui_port}")
-        os.environ["NGINX_PORT"] = str(web_ui_port)
+
+    if only_state:
+        command += " index relational_db"
     else:
-        command += " --scale web_server=0"
-        command += " --scale nginx=0"
+        if use_cloud_gpu:
+            command += " --scale indexing_model_server=0"
+            command += " --scale inference_model_server=0"
+        if launch_web_ui:
+            web_ui_port = 3000
+            while _is_port_in_use(web_ui_port):
+                web_ui_port += 1
+            print(f"UI will be launched at http://localhost:{web_ui_port}")
+            os.environ["NGINX_PORT"] = str(web_ui_port)
+        else:
+            command += " --scale web_server=0"
+            command += " --scale nginx=0"
 
     print("Docker Command:\n", command)
 

backend/tests/regression/answer_quality/run_eval_pipeline.py

@@ -39,9 +39,11 @@ def main() -> None:
     if config.commit_sha:
         switch_to_commit(config.commit_sha)
 
-    start_docker_compose(run_suffix, config.launch_web_ui, config.use_cloud_gpu)
+    start_docker_compose(
+        run_suffix, config.launch_web_ui, config.use_cloud_gpu, config.only_state
+    )
 
-    if not config.existing_test_suffix:
+    if not config.existing_test_suffix and not config.only_state:
         upload_test_files(config.zipped_documents_file, run_suffix)
 
         run_qa_test_and_save_results(run_suffix)

backend/tests/regression/answer_quality/search_test_config.yaml.template

@@ -19,6 +19,9 @@ clean_up_docker_containers: true
 # Whether to launch a web UI for the test
 launch_web_ui: false
 
+# Whether to only run Vespa and Postgres
+only_state: false
+
 # Whether to use a cloud GPU for processing
 use_cloud_gpu: false