Frontend testing (#4500)

* add o3 + o4 mini

* k

* see which ones fail

* attempt

* k

* k

* llm ordering passing

* all tests passing

* quick bump

* Revert "add o3 + o4 mini"

This reverts commit 4cfa1984ec85ea1a10b0943ef2ee3bdff597be78.

* k

* k

web/admin2_auth.json

@@ -2,14 +2,14 @@
   "cookies": [
     {
       "name": "fastapiusersauth",
-      "value": "9HrehHtJj1-5UXudkc96qNBS1Aq5yFDFNCPlLR7PW7k",
+      "value": "h3qhacpHbE4_09HcOLlVW4lSee48m1UbjYTUiKYwNiw",
       "domain": "localhost",
       "path": "/",
-      "expires": 1740532793.140733,
+      "expires": 1745624493.119168,
       "httpOnly": true,
       "secure": false,
       "sameSite": "Lax"
     }
   ],
   "origins": []
-}
+}

web/playwright.config.ts

@@ -2,7 +2,7 @@ import { defineConfig, devices } from "@playwright/test";
 
 export default defineConfig({
   globalSetup: require.resolve("./tests/e2e/global-setup"),
-  timeout: 60000, // 60 seconds timeout
+  timeout: 100000, // 100 seconds timeout
   reporter: [
     ["list"],
     // Warning: uncommenting the html reporter may cause the chromatic-archives
@@ -23,7 +23,7 @@ export default defineConfig({
         viewport: { width: 1280, height: 720 },
         storageState: "admin_auth.json",
       },
-      testIgnore: ["**/codeUtils.test.ts", "**/chat/**/*.spec.ts"],
+      testIgnore: ["**/codeUtils.test.ts"],
     },
   ],
 });

web/tests/e2e/chat/current_assistant.spec.ts

@@ -3,7 +3,6 @@ import { dragElementAbove, dragElementBelow } from "../utils/dragUtils";
 import { loginAsRandomUser } from "../utils/auth";
 
 test("Assistant Drag and Drop", async ({ page }) => {
-  test.fail();
   await page.context().clearCookies();
   await loginAsRandomUser(page);
 

web/tests/e2e/chat/live_assistant.spec.ts

@@ -41,7 +41,7 @@ test("Chat workflow", async ({ page }) => {
 
   // Test creation of a new assistant
   await page.getByRole("button", { name: "Explore Assistants" }).click();
-  await page.getByRole("button", { name: "Create" }).click();
+  await page.getByRole("button", { name: "Create", exact: true }).click();
   await page.getByTestId("name").click();
   await page.getByTestId("name").fill("Test Assistant");
   await page.getByTestId("description").click();

web/tests/e2e/chat/llm_ordering.spec.ts

@@ -9,26 +9,26 @@ import {
 } from "../utils/chatActions";
 
 test("LLM Ordering and Model Switching", async ({ page }) => {
-  test.fail();
-
   // Setup: Clear cookies and log in as a random user
   await page.context().clearCookies();
   await loginAsRandomUser(page);
 
   // Navigate to the chat page and verify URL
   await page.goto("http://localhost:3000/chat");
-  await page.waitForSelector("#onyx-chat-input-textarea");
+  await page.waitForSelector("#onyx-chat-input-textarea", { timeout: 10000 });
   await expect(page.url()).toBe("http://localhost:3000/chat");
 
-  // Configure user settings: Set default model to GPT 4 Turbo
+  // Configure user settings: Set default model to o3 Mini
   await page.locator("#onyx-user-dropdown").click();
   await page.getByText("User Settings").click();
-  await page.getByRole("combobox").click();
+  await page.getByRole("combobox").nth(1).click();
-  await page.getByLabel("GPT 4 Turbo", { exact: true }).click();
+  await page.getByLabel("o3 Mini", { exact: true }).click();
   await page.getByLabel("Close modal").click();
-  await verifyCurrentModel(page, "GPT 4 Turbo");
+  await page.waitForTimeout(2000);
-
+  await verifyCurrentModel(page, "o3 Mini");
   // Test Art Assistant: Should use its own model (GPT 4o)
+  await page.reload();
+  await page.waitForSelector("#onyx-chat-input-textarea", { timeout: 10000 });
   await navigateToAssistantInHistorySidebar(
     page,
     "[-3]",
@@ -37,44 +37,40 @@ test("LLM Ordering and Model Switching", async ({ page }) => {
   await sendMessage(page, "Sample message");
   await verifyCurrentModel(page, "GPT 4o");
 
-  // Verify model persistence for Art Assistant
-  await sendMessage(page, "Sample message");
-
   // Test new chat: Should use Art Assistant's model initially
   await startNewChat(page);
   await expect(page.getByText("Assistant for generating")).toBeVisible();
   await verifyCurrentModel(page, "GPT 4o");
 
-  // Test another new chat: Should use user's default model (GPT 4 Turbo)
+  // Test another new chat: Should use user's default model (o3 Mini)
   await startNewChat(page);
-  await verifyCurrentModel(page, "GPT 4 Turbo");
+  await verifyCurrentModel(page, "o3 Mini");
 
   // Test model switching within a chat
-  await switchModel(page, "O1 Mini");
+  await switchModel(page, "o1 Mini");
   await sendMessage(page, "Sample message");
-  await verifyCurrentModel(page, "O1 Mini");
+  await verifyCurrentModel(page, "o1 Mini");
 
   // Create a custom assistant with a specific model
   await page.getByRole("button", { name: "Explore Assistants" }).click();
-  await page.getByRole("button", { name: "Create" }).click();
+  await page.getByRole("button", { name: "Create", exact: true }).click();
   await page.waitForTimeout(2000);
   await page.getByTestId("name").fill("Sample Name");
   await page.getByTestId("description").fill("Sample Description");
   await page.getByTestId("system_prompt").fill("Sample Instructions");
-  await page.getByRole("combobox").click();
   await page
-    .getByLabel("GPT 4 Turbo (Preview)")
+    .locator('button[role="combobox"] > span:has-text("User Default")')
-    .getByText("GPT 4 Turbo (Preview)")
     .click();
+  await page.getByLabel("o3 Mini").getByText("o3 Mini").click();
   await page.getByRole("button", { name: "Create" }).click();
 
   // Verify custom assistant uses its specified model
   await page.locator("#onyx-chat-input-textarea").fill("");
-  await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
+  await verifyCurrentModel(page, "o3 Mini");
 
   // Ensure model persistence for custom assistant
   await sendMessage(page, "Sample message");
-  await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
+  await verifyCurrentModel(page, "o3 Mini");
 
   // Switch back to Art Assistant and verify its model
   await navigateToAssistantInHistorySidebar(

web/tests/e2e/global-setup.ts

@@ -21,30 +21,6 @@ async function globalSetup(config: FullConfig) {
   await loginAs(admin2Page, "admin2");
   await admin2Context.storageState({ path: "admin2_auth.json" });
   await admin2Context.close();
-
-  const adminContext2 = await browser.newContext({
-    storageState: "admin_auth.json",
-  });
-  const adminPage2 = await adminContext2.newPage();
-  await inviteAdmin2AsAdmin1(adminPage2);
-  await adminContext2.close();
-
-  // Test admin2 access after invitation
-  const admin2TestContext = await browser.newContext({
-    storageState: "admin2_auth.json",
-  });
-  const admin2TestPage = await admin2TestContext.newPage();
-  await admin2TestPage.goto("http://localhost:3000/admin/indexing/status");
-
-  // Ensure we stay on the admin page
-  if (admin2TestPage.url() !== "http://localhost:3000/admin/indexing/status") {
-    throw new Error(
-      `Admin2 was not able to access the admin page after invitation. Actual URL: ${admin2TestPage.url()}`
-    );
-  }
-
-  await admin2TestContext.close();
-
   await browser.close();
 }
 

web/tests/e2e/utils/auth.ts

@@ -18,7 +18,7 @@ export async function loginAs(
         ? TEST_ADMIN2_CREDENTIALS
         : TEST_USER_CREDENTIALS;
 
-  await page.goto("http://localhost:3000/auth/login", { timeout: 1000 });
+  await page.goto("http://localhost:3000/auth/login");
 
   await page.fill("#email", email);
   await page.fill("#password", password);
@@ -32,7 +32,7 @@ export async function loginAs(
     console.log(`Timeout occurred. Current URL: ${page.url()}`);
 
     // If redirect to /chat doesn't happen, go to /auth/login
-    await page.goto("http://localhost:3000/auth/signup", { timeout: 1000 });
+    await page.goto("http://localhost:3000/auth/signup");
 
     await page.fill("#email", email);
     await page.fill("#password", password);
@@ -76,7 +76,14 @@ export async function loginAsRandomUser(page: Page) {
   // Click the signup button
   await page.click('button[type="submit"]');
   try {
-    await page.waitForURL("http://localhost:3000/chat");
+    // Wait for 2 seconds to ensure the signup process completes
+    await page.waitForTimeout(3000);
+    // Refresh the page to ensure everything is loaded properly
+    // await page.reload();
+
+    await page.waitForURL("http://localhost:3000/chat?new_team=true");
+    // Wait for the page to be fully loaded after refresh
+    await page.waitForLoadState("networkidle");
   } catch (error) {
     console.log(`Timeout occurred. Current URL: ${page.url()}`);
     throw new Error("Failed to sign up and redirect to chat page");

web/tests/e2e/utils/chatActions.ts

@@ -36,13 +36,11 @@ export async function verifyCurrentModel(page: Page, modelName: string) {
 // Start of Selection
 export async function switchModel(page: Page, modelName: string) {
   await page.getByTestId("llm-popover-trigger").click();
-  await page
+  await page.getByRole("button", { name: new RegExp(modelName, "i") }).click();
-    .getByRole("button", { name: `Logo ${modelName}`, exact: true })
-    .click();
   await page.waitForTimeout(1000);
 }
 
 export async function startNewChat(page: Page) {
-  await page.getByRole("link", { name: "Start New Chat" }).click();
+  await page.getByRole("link", { name: "New Chat" }).click();
   await expect(page.locator('div[data-testid="chat-intro"]')).toBeVisible();
 }