Update e2e frontend tests (#3843)

* fix input prompts

* assistant ordering validation

* k

* Revert "fix input prompts"

This reverts commit a4b577bdd7.

* fix alembic

* foreign key updates

* Revert "foreign key updates"

This reverts commit fe17795a037f831790d69229e1067ccb5aab5bd9.

* improve e2e tests

* fix admin

web/tests/e2e/admin_auth.setup.ts

@@ -1,24 +1,9 @@
 // dependency for all admin user tests
+import { test as setup } from "@playwright/test";
 
-import { test as setup, expect } from "@playwright/test";
-import { TEST_CREDENTIALS } from "./constants";
-
-setup("authenticate", async ({ page }) => {
-  const { email, password } = TEST_CREDENTIALS;
-
+setup("authenticate as admin", async ({ browser }) => {
+  const context = await browser.newContext({ storageState: "admin_auth.json" });
+  const page = await context.newPage();
   await page.goto("http://localhost:3000/chat");
-
-  await page.waitForURL("http://localhost:3000/auth/login?next=%2Fchat");
-
-  await expect(page).toHaveTitle("Onyx");
-
-  await page.fill("#email", email);
-  await page.fill("#password", password);
-
-  // Click the login button
-  await page.click('button[type="submit"]');
-
   await page.waitForURL("http://localhost:3000/chat");
-
-  await page.context().storageState({ path: "admin_auth.json" });
 });

web/tests/e2e/admin_oauth_redirect_uri.spec.ts

@@ -1,65 +1,43 @@
-import { test, expect } from "@chromatic-com/playwright";
+import { test, expect } from "@playwright/test";
 
-test(
-  "Admin - OAuth Redirect - Missing Code",
-  {
-    tag: "@admin",
-  },
-  async ({ page }, testInfo) => {
-    await page.goto(
-      "http://localhost:3000/admin/connectors/slack/oauth/callback?state=xyz"
-    );
+test.use({ storageState: "admin_auth.json" });
 
-    await expect(page.locator("p.text-text-500")).toHaveText(
-      "Missing authorization code."
-    );
-  }
-);
+test("Admin - OAuth Redirect - Missing Code", async ({ page }) => {
+  await page.goto(
+    "http://localhost:3000/admin/connectors/slack/oauth/callback?state=xyz"
+  );
 
-test(
-  "Admin - OAuth Redirect - Missing State",
-  {
-    tag: "@admin",
-  },
-  async ({ page }, testInfo) => {
-    await page.goto(
-      "http://localhost:3000/admin/connectors/slack/oauth/callback?code=123"
-    );
+  await expect(page.locator("p.text-text-500")).toHaveText(
+    "Missing authorization code."
+  );
+});
 
-    await expect(page.locator("p.text-text-500")).toHaveText(
-      "Missing state parameter."
-    );
-  }
-);
+test("Admin - OAuth Redirect - Missing State", async ({ page }) => {
+  await page.goto(
+    "http://localhost:3000/admin/connectors/slack/oauth/callback?code=123"
+  );
 
-test(
-  "Admin - OAuth Redirect - Invalid Connector",
-  {
-    tag: "@admin",
-  },
-  async ({ page }, testInfo) => {
-    await page.goto(
-      "http://localhost:3000/admin/connectors/invalid-connector/oauth/callback?code=123&state=xyz"
-    );
+  await expect(page.locator("p.text-text-500")).toHaveText(
+    "Missing state parameter."
+  );
+});
 
-    await expect(page.locator("p.text-text-500")).toHaveText(
-      "invalid_connector is not a valid source type."
-    );
-  }
-);
+test("Admin - OAuth Redirect - Invalid Connector", async ({ page }) => {
+  await page.goto(
+    "http://localhost:3000/admin/connectors/invalid-connector/oauth/callback?code=123&state=xyz"
+  );
 
-test(
-  "Admin - OAuth Redirect - No Session",
-  {
-    tag: "@admin",
-  },
-  async ({ page }, testInfo) => {
-    await page.goto(
-      "http://localhost:3000/admin/connectors/slack/oauth/callback?code=123&state=xyz"
-    );
+  await expect(page.locator("p.text-text-500")).toHaveText(
+    "invalid_connector is not a valid source type."
+  );
+});
 
-    await expect(page.locator("p.text-text-500")).toHaveText(
-      "An error occurred during the OAuth process. Please try again."
-    );
-  }
-);
+test("Admin - OAuth Redirect - No Session", async ({ page }) => {
+  await page.goto(
+    "http://localhost:3000/admin/connectors/slack/oauth/callback?code=123&state=xyz"
+  );
+
+  await expect(page.locator("p.text-text-500")).toHaveText(
+    "An error occurred during the OAuth process. Please try again."
+  );
+});

web/tests/e2e/admin_pages.spec.ts

@@ -2,6 +2,8 @@ import { test, expect } from "@playwright/test";
 import chromaticSnpashots from "./chromaticSnpashots.json";
 import type { Page } from "@playwright/test";
 
+test.use({ storageState: "admin_auth.json" });
+
 async function verifyAdminPageNavigation(
   page: Page,
   path: string,
@@ -13,7 +15,10 @@ async function verifyAdminPageNavigation(
   }
 ) {
   await page.goto(`http://localhost:3000/admin/${path}`);
-  await expect(page.locator("h1.text-3xl")).toHaveText(pageTitle);
+
+  await expect(page.locator("h1.text-3xl")).toHaveText(pageTitle, {
+    timeout: 2000,
+  });
 
   if (options?.paragraphText) {
     await expect(page.locator("p.text-sm").nth(0)).toHaveText(
@@ -35,18 +40,12 @@ async function verifyAdminPageNavigation(
 }
 
 for (const chromaticSnapshot of chromaticSnpashots) {
-  test(
-    `Admin - ${chromaticSnapshot.name}`,
-    {
-      tag: "@admin",
-    },
-    async ({ page }) => {
-      await verifyAdminPageNavigation(
-        page,
-        chromaticSnapshot.path,
-        chromaticSnapshot.pageTitle,
-        chromaticSnapshot.options
-      );
-    }
-  );
+  test(`Admin - ${chromaticSnapshot.name}`, async ({ page }) => {
+    await verifyAdminPageNavigation(
+      page,
+      chromaticSnapshot.path,
+      chromaticSnapshot.pageTitle,
+      chromaticSnapshot.options
+    );
+  });
 }

web/tests/e2e/assisant_ordering.spec.ts

@@ -0,0 +1,54 @@
+import { test, expect } from "@playwright/test";
+
+// Use pre-signed in "admin" storage state
+test.use({
+  storageState: "admin_auth.json",
+});
+
+test("Chat workflow", async ({ page }) => {
+  // Initial setup
+  await page.goto("http://localhost:3000/chat", { timeout: 3000 });
+
+  // Interact with Art assistant
+  await page.locator("button").filter({ hasText: "Art" }).click();
+  await page.getByPlaceholder("Message Art assistant...").fill("Hi");
+  await page.keyboard.press("Enter");
+  await page.waitForTimeout(3000);
+
+  // Start a new chat
+  await page.getByRole("link", { name: "Start New Chat" }).click();
+  await page.waitForNavigation({ waitUntil: "networkidle" });
+
+  // Check for expected text
+  await expect(page.getByText("Assistant for generating")).toBeVisible();
+
+  // Interact with General assistant
+  await page.locator("button").filter({ hasText: "General" }).click();
+
+  // Check URL after clicking General assistant
+  await expect(page).toHaveURL("http://localhost:3000/chat?assistantId=-1", {
+    timeout: 5000,
+  });
+
+  // Create a new assistant
+  await page.getByRole("button", { name: "Explore Assistants" }).click();
+  await page.getByRole("button", { name: "Create" }).click();
+  await page.getByTestId("name").click();
+  await page.getByTestId("name").fill("Test Assistant");
+  await page.getByTestId("description").click();
+  await page.getByTestId("description").fill("Test Assistant Description");
+  await page.getByTestId("system_prompt").click();
+  await page.getByTestId("system_prompt").fill("Test Assistant Instructions");
+  await page.getByRole("button", { name: "Create" }).click();
+
+  // Verify new assistant creation
+  await expect(page.getByText("Test Assistant Description")).toBeVisible({
+    timeout: 5000,
+  });
+
+  // Start another new chat
+  await page.getByRole("link", { name: "Start New Chat" }).click();
+  await expect(page.getByText("Assistant with access to")).toBeVisible({
+    timeout: 5000,
+  });
+});

web/tests/e2e/constants.js

@@ -1,4 +1,9 @@
-export const TEST_CREDENTIALS = {
+export const TEST_USER_CREDENTIALS = {
+  email: "user1@test.com",
+  password: "User1Password123!",
+};
+
+export const TEST_ADMIN_CREDENTIALS = {
   email: "admin_user@test.com",
   password: "TestPassword123!",
 };

web/tests/e2e/global-setup.ts

@@ -0,0 +1,22 @@
+import { chromium, FullConfig } from "@playwright/test";
+import { loginAs } from "./utils/auth";
+
+async function globalSetup(config: FullConfig) {
+  const browser = await chromium.launch();
+
+  const adminContext = await browser.newContext();
+  const adminPage = await adminContext.newPage();
+  await loginAs(adminPage, "admin");
+  await adminContext.storageState({ path: "admin_auth.json" });
+  await adminContext.close();
+
+  const userContext = await browser.newContext();
+  const userPage = await userContext.newPage();
+  await loginAs(userPage, "user");
+  await userContext.storageState({ path: "user_auth.json" });
+  await userContext.close();
+
+  await browser.close();
+}
+
+export default globalSetup;

web/tests/e2e/home.spec.ts

@@ -1,35 +0,0 @@
-// ➕ Add this line
-import { test, expect, takeSnapshot } from "@chromatic-com/playwright";
-import { TEST_CREDENTIALS } from "./constants";
-
-// Then use as normal 👇
-test(
-  "Homepage",
-  {
-    tag: "@guest",
-  },
-  async ({ page }, testInfo) => {
-    // Test redirect to login, and redirect to search after login
-    const { email, password } = TEST_CREDENTIALS;
-
-    await page.goto("http://localhost:3000/chat");
-
-    await page.waitForURL("http://localhost:3000/auth/login?next=%2Fchat");
-
-    await expect(page).toHaveTitle("Onyx");
-
-    await takeSnapshot(page, "Before login", testInfo);
-
-    await page.fill("#email", email);
-    await page.fill("#password", password);
-
-    // Click the login button
-    await page.click('button[type="submit"]');
-
-    await page.waitForURL("http://localhost:3000/chat");
-
-    await page.getByPlaceholder("Send a message or try using @ or /");
-
-    await expect(page.locator("body")).not.toContainText("Initializing Onyx");
-  }
-);

web/tests/e2e/utils/auth.ts

@@ -0,0 +1,37 @@
+import { Page } from "@playwright/test";
+import { TEST_ADMIN_CREDENTIALS, TEST_USER_CREDENTIALS } from "../constants";
+
+// Basic function which logs in a user (either admin or regular user) to the application
+// It handles both successful login attempts and potential timeouts, with a retry mechanism
+export async function loginAs(page: Page, userType: "admin" | "user") {
+  const { email, password } =
+    userType === "admin" ? TEST_ADMIN_CREDENTIALS : TEST_USER_CREDENTIALS;
+  await page.goto("http://localhost:3000/auth/login", { timeout: 1000 });
+
+  await page.fill("#email", email);
+  await page.fill("#password", password);
+
+  // Click the login button
+  await page.click('button[type="submit"]');
+
+  try {
+    await page.waitForURL("http://localhost:3000/chat", { timeout: 4000 });
+  } catch (error) {
+    console.log(`Timeout occurred. Current URL: ${page.url()}`);
+
+    // If redirect to /chat doesn't happen, go to /auth/login
+    await page.goto("http://localhost:3000/auth/signup", { timeout: 1000 });
+
+    await page.fill("#email", email);
+    await page.fill("#password", password);
+
+    // Click the login button
+    await page.click('button[type="submit"]');
+
+    try {
+      await page.waitForURL("http://localhost:3000/chat", { timeout: 4000 });
+    } catch (error) {
+      console.log(`Timeout occurred again. Current URL: ${page.url()}`);
+    }
+  }
+}