improve web testing (#3162)

* shared admin level test dependency

* change to on - push (recommended by chromatic)

* change playwright reporter to list, name test jobs

* use test tags ... much cleaner

* test vs prod

* try copying templates

* run with localhost?

* revert to dev

* new tests and a bit of refactoring

* add additional checks so that page snapshots reflect loaded state

* more admin tests

* User Management tests

* remaining admin pages

* test search and chat

* await fix and exclude UI that changes with dates.

.github/workflows/pr-chromatic-tests.yml

@@ -3,12 +3,7 @@ concurrency:
   group: Run-Chromatic-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
   cancel-in-progress: true
 
-on:
-  merge_group:
-  pull_request:
-    branches:
-      - main
-      - 'release/**'
+on: push
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -16,6 +11,8 @@ env:
 
 jobs:
   playwright-tests:
+    name: Playwright Tests
+
     # See https://runs-on.com/runners/linux/
     runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
     steps:
@@ -108,7 +105,7 @@ jobs:
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
-      - name: Start Docker containers 
+      - name: Start Docker containers
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -193,7 +190,8 @@ jobs:
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v
 
   chromatic-tests:
-    name: Run Chromatic
+    name: Chromatic Tests
+    
     needs: playwright-tests
     runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
     steps:

deployment/docker_compose/README.md

@@ -5,7 +5,7 @@
 For general information, please read the instructions in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md).
 
 ## Deploy in a system without GPU support
-This part is elaborated precisely in  in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md) in section *Docker Compose*. If you have any questions, please feel free to open an issue or get in touch in slack for support.
+This part is elaborated precisely in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md) in section *Docker Compose*. If you have any questions, please feel free to open an issue or get in touch in slack for support.
 
 ## Deploy in a system with GPU support
 Running Model servers with GPU support while indexing and querying can result in significant improvements in performance. This is highly recommended if you have access to resources. Currently, Danswer offloads embedding model and tokenizers to the GPU VRAM and the size needed depends on chosen embedding model. For example, the embedding model `nomic-ai/nomic-embed-text-v1` takes up about 1GB of VRAM. That means running this model for inference and embedding pipeline would require roughly 2GB of VRAM.

web/.gitignore

@@ -34,3 +34,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+/admin_auth.json
+/build-archive.log
+

web/playwright.config.ts

@@ -1,8 +1,8 @@
-import { defineConfig } from "@playwright/test";
+import { defineConfig, devices } from "@playwright/test";
 
 export default defineConfig({
-  // Other Playwright config options
   testDir: "./tests/e2e", // Folder for test files
+  reporter: "list",
   // Configure paths for screenshots
   // expect: {
   //   toMatchSnapshot: {
@@ -11,4 +11,30 @@ export default defineConfig({
   // },
   // reporter: [["html", { outputFolder: "test-results/output/report" }]], // HTML report location
   // outputDir: "test-results/output/screenshots", // Set output folder for test artifacts
+  projects: [
+    {
+      // dependency for admin workflows
+      name: "admin_setup",
+      testMatch: /.*\admin_auth.setup\.ts/,
+    },
+    {
+      // tests admin workflows
+      name: "chromium-admin",
+      grep: /@admin/,
+      use: {
+        ...devices["Desktop Chrome"],
+        // Use prepared auth state.
+        storageState: "admin_auth.json",
+      },
+      dependencies: ["admin_setup"],
+    },
+    {
+      // tests logged out / guest workflows
+      name: "chromium-guest",
+      grep: /@guest/,
+      use: {
+        ...devices["Desktop Chrome"],
+      },
+    },
+  ],
 });

web/src/app/admin/api-key/page.tsx

@@ -29,9 +29,7 @@ import { deleteApiKey, regenerateApiKey } from "./lib";
 import { DanswerApiKeyForm } from "./DanswerApiKeyForm";
 import { APIKey } from "./types";
 
-const API_KEY_TEXT = `
-API Keys allow you to access Danswer APIs programmatically. Click the button below to generate a new API Key.
-`;
+const API_KEY_TEXT = `API Keys allow you to access Danswer APIs programmatically. Click the button below to generate a new API Key.`;
 
 function NewApiKeyModal({
   apiKey,

web/tests/e2e/admin_add_connector.spec.ts

@@ -0,0 +1,14 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Connectors - Add Connector",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/add-connector");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Add Connector");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(/^Storage/);
+  }
+);

web/tests/e2e/admin_api_key.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - API Keys",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/api-key");
+    await expect(page.locator("h1.text-3xl")).toHaveText("API Keys");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^API Keys allow you to access Danswer APIs programmatically/
+    );
+    await expect(
+      page.getByRole("button", { name: "Create API Key" })
+    ).toHaveCount(1);
+  }
+);

web/tests/e2e/admin_assistants.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Assistants",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/assistants");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Assistants");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Assistants are a way to build/
+    );
+  }
+);

web/tests/e2e/admin_auth.setup.ts

@@ -0,0 +1,24 @@
+// dependency for all admin user tests
+
+import { test as setup, expect } from "@playwright/test";
+import { TEST_CREDENTIALS } from "./constants";
+
+setup("authenticate", async ({ page }) => {
+  const { email, password } = TEST_CREDENTIALS;
+
+  await page.goto("http://localhost:3000/search");
+
+  await page.waitForURL("http://localhost:3000/auth/login?next=%2Fsearch");
+
+  await expect(page).toHaveTitle("Danswer");
+
+  await page.fill("#email", email);
+  await page.fill("#password", password);
+
+  // Click the login button
+  await page.click('button[type="submit"]');
+
+  await page.waitForURL("http://localhost:3000/search");
+
+  await page.context().storageState({ path: "admin_auth.json" });
+});

web/tests/e2e/admin_bots.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Slack Bots",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/bots");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Slack Bots");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Setup Slack bots that connect to Danswer./
+    );
+  }
+);

web/tests/e2e/admin_configuration_document_processing.spec.ts

@@ -0,0 +1,18 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - Document Processing",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto(
+      "http://localhost:3000/admin/configuration/document-processing"
+    );
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Processing");
+    await expect(page.locator("h3.text-2xl")).toHaveText(
+      "Process with Unstructured API"
+    );
+  }
+);

web/tests/e2e/admin_configuration_llm.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - LLM",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/configuration/llm");
+    await expect(page.locator("h1.text-3xl")).toHaveText("LLM Setup");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Enabled LLM Providers"
+    );
+  }
+);

web/tests/e2e/admin_configuration_search.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - Search Settings",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/configuration/search");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Search Settings");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Embedding Model"
+    );
+  }
+);

web/tests/e2e/admin_documents_explorer.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Feedback",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/explorer");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Explorer");
+    await expect(page.locator("div.flex.text-emphasis.mt-3")).toHaveText(
+      "Search for a document above to modify its boost or hide it from searches."
+    );
+  }
+);

web/tests/e2e/admin_documents_feedback.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Feedback",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/feedback");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Feedback");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Most Liked Documents"
+    );
+    await expect(page.locator("h1.text-lg").nth(1)).toHaveText(
+      "Most Disliked Documents"
+    );
+  }
+);

web/tests/e2e/admin_documents_sets.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Document Sets",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/sets");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Sets");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^Document Sets allow you to group logically connected documents into a single bundle./
+    );
+  }
+);

web/tests/e2e/admin_groups.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - Groups",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/groups");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Manage User Groups");
+    await expect(
+      page.getByRole("button", { name: "Create New User Group" })
+    ).toHaveCount(1);
+  }
+);

web/tests/e2e/admin_indexing_status.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Connectors - Existing Connectors",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/indexing/status");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Existing Connectors");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^It looks like you don't have any connectors setup yet./
+    );
+  }
+);

web/tests/e2e/admin_performance_custom_analytics.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Performance - Custom Analytics",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/performance/custom-analytics");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Custom Analytics");
+    await expect(page.locator("div.font-medium").nth(0)).toHaveText(
+      "Custom Analytics is not enabled."
+    );
+  }
+);

web/tests/e2e/admin_performance_query_history.spec.ts

@@ -0,0 +1,14 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Performance - Query History",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/performance/query-history");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Query History");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText("Feedback Type");
+  }
+);

web/tests/e2e/admin_performance_usage.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test.describe("Admin Performance Usage", () => {
+  // Ignores the diff for elements targeted by the specified list of selectors
+  test.use({ ignoreSelectors: ["button", "svg"] });
+
+  test(
+    "Admin - Performance - Usage Statistics",
+    {
+      tag: "@admin",
+    },
+    async ({ page }, testInfo) => {
+      await page.goto("http://localhost:3000/admin/performance/usage");
+      await expect(page.locator("h1.text-3xl")).toHaveText("Usage Statistics");
+      await expect(page.locator("h1.text-lg").nth(0)).toHaveText("Usage");
+      await expect(page.locator("h1.text-lg").nth(1)).toHaveText("Feedback");
+    }
+  );
+});

web/tests/e2e/admin_prompt_library.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Prompt Library",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/prompt-library");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Prompt Library");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^Create prompts that can be accessed/
+    );
+  }
+);

web/tests/e2e/admin_settings.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Settings - Workspace Settings",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/settings");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Workspace Settings");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Manage general Danswer settings applicable to all users in the workspace./
+    );
+    await expect(
+      page.getByRole("button", { name: "Set Retention Limit" })
+    ).toHaveCount(1);
+  }
+);

web/tests/e2e/admin_standard_answer.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Standard Answers",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/standard-answer");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Standard Answers");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Manage the standard answers for pre-defined questions./
+    );
+  }
+);

web/tests/e2e/admin_token_rate_limits.spec.ts

@@ -0,0 +1,22 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - Token Rate Limits",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/token-rate-limits");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Token Rate Limits");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Token rate limits enable you control how many tokens can be spent in a given time period./
+    );
+    await expect(
+      page.getByRole("button", { name: "Create a Token Rate Limit" })
+    ).toHaveCount(1);
+    await expect(page.locator("h1.text-lg")).toHaveText(
+      "Global Token Rate Limits"
+    );
+  }
+);

web/tests/e2e/admin_tools.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Tools",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/tools");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Tools");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      "Tools allow assistants to retrieve information or take actions."
+    );
+  }
+);

web/tests/e2e/admin_users.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - Groups",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/users");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Manage Users");
+    await expect(page.locator("div.font-bold").nth(0)).toHaveText(
+      "Invited Users"
+    );
+    await expect(page.locator("div.font-bold").nth(1)).toHaveText(
+      "Current Users"
+    );
+  }
+);

web/tests/e2e/admin_whitelabeling.spec.ts

@@ -0,0 +1,18 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Performance - Whitelabeling",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/whitelabeling");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Whitelabeling");
+    await expect(page.locator("div.block").nth(0)).toHaveText(
+      "Application Name"
+    );
+    await expect(page.locator("div.block").nth(1)).toHaveText("Custom Logo");
+    await expect(page.getByRole("button", { name: "Update" })).toHaveCount(1);
+  }
+);

web/tests/e2e/chat.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Chat",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/chat");
+    await expect(page.locator("div.text-2xl").nth(0)).toHaveText("General");
+    await expect(page.getByRole("button", { name: "Search S" })).toHaveClass(
+      /text-text-application-untoggled/
+    );
+    await expect(page.getByRole("button", { name: "Chat D" })).toHaveClass(
+      /text-text-application-toggled/
+    );
+  }
+);

web/tests/e2e/constants.js

@@ -0,0 +1,5 @@
+// constants.js
+export const TEST_CREDENTIALS = {
+  email: "admin_user@test.com",
+  password: "test",
+};

web/tests/e2e/home.spec.ts

@@ -1,27 +1,31 @@
 // ➕ Add this line
 import { test, expect, takeSnapshot } from "@chromatic-com/playwright";
+import { TEST_CREDENTIALS } from "./constants";
 
 // Then use as normal 👇
-test("Homepage", async ({ page }, testInfo) => {
-  // Test redirect to login, and redirect to search after login
+test(
+  "Homepage",
+  {
+    tag: "@guest",
+  },
+  async ({ page }, testInfo) => {
+    // Test redirect to login, and redirect to search after login
+    const { email, password } = TEST_CREDENTIALS;
 
-  // move these into a constants file or test fixture soon
-  let email = "admin_user@test.com";
-  let password = "test";
+    await page.goto("http://localhost:3000/search");
 
-  await page.goto("http://localhost:3000/search");
+    await page.waitForURL("http://localhost:3000/auth/login?next=%2Fsearch");
 
-  await page.waitForURL("http://localhost:3000/auth/login?next=%2Fsearch");
+    await expect(page).toHaveTitle("Danswer");
 
-  await expect(page).toHaveTitle("Danswer");
+    await takeSnapshot(page, "Before login", testInfo);
 
-  await takeSnapshot(page, "Before login", testInfo);
+    await page.fill("#email", email);
+    await page.fill("#password", password);
 
-  await page.fill("#email", email);
-  await page.fill("#password", password);
+    // Click the login button
+    await page.click('button[type="submit"]');
 
-  // Click the login button
-  await page.click('button[type="submit"]');
-
-  await page.waitForURL("http://localhost:3000/search");
-});
+    await page.waitForURL("http://localhost:3000/search");
+  }
+);

web/tests/e2e/search.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Search",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/search");
+    await expect(page.locator("div.text-3xl")).toHaveText("Unlock Knowledge");
+    await expect(page.getByRole("button", { name: "Search S" })).toHaveClass(
+      /text-text-application-toggled/
+    );
+    await expect(page.getByRole("button", { name: "Chat D" })).toHaveClass(
+      /text-text-application-untoggled/
+    );
+  }
+);