diff --git a/backend/tests/regression/answer_quality/README.md b/backend/tests/regression/answer_quality/README.md
index d7f5b9ae0f1..f74f8ee4360 100644
--- a/backend/tests/regression/answer_quality/README.md
+++ b/backend/tests/regression/answer_quality/README.md
@@ -70,7 +70,7 @@ Edit `search_test_config.yaml` to set:
 - model_server_port
     - This is the port of the remote model server
     - Only need to set this if use_cloud_gpu is true
-- existing_test_suffix
+- existing_test_suffix (THIS IS NOT A SUFFIX ANYMORE, TODO UPDATE THE DOCS HERE)
     - Use this if you would like to relaunch a previous test instance
     - Input the suffix of the test you'd like to re-launch 
     - (E.g. to use the data from folder "test-1234-5678" put "-1234-5678")
diff --git a/backend/tests/regression/answer_quality/cli_utils.py b/backend/tests/regression/answer_quality/cli_utils.py
index c5338b5a53a..c40a8b7e598 100644
--- a/backend/tests/regression/answer_quality/cli_utils.py
+++ b/backend/tests/regression/answer_quality/cli_utils.py
@@ -97,7 +97,7 @@ def get_docker_container_env_vars(suffix: str) -> dict:
 
 def manage_data_directories(suffix: str, base_path: str, use_cloud_gpu: bool) -> None:
     # Use the user's home directory as the base path
-    target_path = os.path.join(os.path.expanduser(base_path), f"test{suffix}")
+    target_path = os.path.join(os.path.expanduser(base_path), suffix)
     directories = {
         "DANSWER_POSTGRES_DATA_DIR": os.path.join(target_path, "postgres/"),
         "DANSWER_VESPA_DATA_DIR": os.path.join(target_path, "vespa/"),
diff --git a/backend/tests/regression/answer_quality/run_qa.py b/backend/tests/regression/answer_quality/run_qa.py
index b8a4d559ad8..f4982058393 100644
--- a/backend/tests/regression/answer_quality/run_qa.py
+++ b/backend/tests/regression/answer_quality/run_qa.py
@@ -46,19 +46,16 @@ def _get_test_output_folder(config: dict) -> str:
     base_output_folder = os.path.expanduser(config["output_folder"])
     if config["run_suffix"]:
         base_output_folder = os.path.join(
-            base_output_folder, ("test" + config["run_suffix"]), "evaluations_output"
+            base_output_folder, config["run_suffix"], "evaluations_output"
         )
     else:
         base_output_folder = os.path.join(base_output_folder, "no_defined_suffix")
 
     counter = 1
-    run_suffix = config["run_suffix"][1:]
-    output_folder_path = os.path.join(base_output_folder, f"{run_suffix}_run_1")
+    output_folder_path = os.path.join(base_output_folder, "run_1")
     while os.path.exists(output_folder_path):
         output_folder_path = os.path.join(
-            output_folder_path.replace(
-                f"{run_suffix}_run_{counter-1}", f"{run_suffix}_run_{counter}"
-            ),
+            output_folder_path.replace(f"run_{counter-1}", f"run_{counter}"),
         )
         counter += 1
 
@@ -163,6 +160,9 @@ def _process_and_write_query_results(config: dict) -> None:
         if not result.get("answer"):
             invalid_answer_count += 1
 
+        if not result.get("context_data_list"):
+            raise RuntimeError("Search failed, this is a critical failure!")
+
     _update_metadata_file(test_output_folder, invalid_answer_count)
 
     if invalid_answer_count: