Search Eval (#1873)

backend/tests/regression/answer_quality/cli_utils.py

@@ -144,7 +144,7 @@ def _is_port_in_use(port: int) -> bool:
 
 
 def start_docker_compose(
-    run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool
+    run_suffix: str, launch_web_ui: bool, use_cloud_gpu: bool, only_state: bool = False
 ) -> None:
     print("Starting Docker Compose...")
     os.chdir(os.path.dirname(__file__))
@@ -152,6 +152,10 @@ def start_docker_compose(
     command = f"docker compose -f docker-compose.search-testing.yml -p danswer-stack{run_suffix} up -d"
     command += " --build"
     command += " --force-recreate"
+
+    if only_state:
+        command += " index relational_db"
+    else:
         if use_cloud_gpu:
             command += " --scale indexing_model_server=0"
             command += " --scale inference_model_server=0"

backend/tests/regression/answer_quality/run_eval_pipeline.py

@@ -39,9 +39,11 @@ def main() -> None:
     if config.commit_sha:
         switch_to_commit(config.commit_sha)
 
-    start_docker_compose(run_suffix, config.launch_web_ui, config.use_cloud_gpu)
+    start_docker_compose(
+        run_suffix, config.launch_web_ui, config.use_cloud_gpu, config.only_state
+    )
 
-    if not config.existing_test_suffix:
+    if not config.existing_test_suffix and not config.only_state:
         upload_test_files(config.zipped_documents_file, run_suffix)
 
         run_qa_test_and_save_results(run_suffix)

backend/tests/regression/answer_quality/search_test_config.yaml.template

@@ -19,6 +19,9 @@ clean_up_docker_containers: true
 # Whether to launch a web UI for the test
 launch_web_ui: false
 
+# Whether to only run Vespa and Postgres
+only_state: false
+
 # Whether to use a cloud GPU for processing
 use_cloud_gpu: false