E2e assistant tests (#3869)

* adding llm override logic

* update

* general cleanup

* fix various tests

* rm

* update

* update

* better comments

* k

* k

* update to pass tests

* clarify content

* improve timeout

web/tests/e2e/assisant_ordering.spec.ts

@@ -1,54 +0,0 @@
-import { test, expect } from "@playwright/test";
-
-// Use pre-signed in "admin" storage state
-test.use({
-  storageState: "admin_auth.json",
-});
-
-test("Chat workflow", async ({ page }) => {
-  // Initial setup
-  await page.goto("http://localhost:3000/chat", { timeout: 3000 });
-
-  // Interact with Art assistant
-  await page.locator("button").filter({ hasText: "Art" }).click();
-  await page.getByPlaceholder("Message Art assistant...").fill("Hi");
-  await page.keyboard.press("Enter");
-  await page.waitForTimeout(3000);
-
-  // Start a new chat
-  await page.getByRole("link", { name: "Start New Chat" }).click();
-  await page.waitForNavigation({ waitUntil: "networkidle" });
-
-  // Check for expected text
-  await expect(page.getByText("Assistant for generating")).toBeVisible();
-
-  // Interact with General assistant
-  await page.locator("button").filter({ hasText: "General" }).click();
-
-  // Check URL after clicking General assistant
-  await expect(page).toHaveURL("http://localhost:3000/chat?assistantId=-1", {
-    timeout: 5000,
-  });
-
-  // Create a new assistant
-  await page.getByRole("button", { name: "Explore Assistants" }).click();
-  await page.getByRole("button", { name: "Create" }).click();
-  await page.getByTestId("name").click();
-  await page.getByTestId("name").fill("Test Assistant");
-  await page.getByTestId("description").click();
-  await page.getByTestId("description").fill("Test Assistant Description");
-  await page.getByTestId("system_prompt").click();
-  await page.getByTestId("system_prompt").fill("Test Assistant Instructions");
-  await page.getByRole("button", { name: "Create" }).click();
-
-  // Verify new assistant creation
-  await expect(page.getByText("Test Assistant Description")).toBeVisible({
-    timeout: 5000,
-  });
-
-  // Start another new chat
-  await page.getByRole("link", { name: "Start New Chat" }).click();
-  await expect(page.getByText("Assistant with access to")).toBeVisible({
-    timeout: 5000,
-  });
-});

web/tests/e2e/chat/current_assistant.spec.ts

@@ -0,0 +1,54 @@
+import { test, expect } from "@playwright/test";
+import { dragElementAbove, dragElementBelow } from "../utils/dragUtils";
+import { loginAsRandomUser } from "../utils/auth";
+
+test("Assistant Drag and Drop", async ({ page }) => {
+  await page.context().clearCookies();
+  await loginAsRandomUser(page);
+
+  // Navigate to the chat page
+  await page.goto("http://localhost:3000/chat");
+
+  // Helper function to get the current order of assistants
+  const getAssistantOrder = async () => {
+    const assistants = await page.$$('[data-testid^="assistant-["]');
+    return Promise.all(
+      assistants.map(async (assistant) => {
+        const nameElement = await assistant.$("p");
+        return nameElement ? nameElement.textContent() : "";
+      })
+    );
+  };
+
+  // Get the initial order
+  const initialOrder = await getAssistantOrder();
+
+  // Drag second assistant above first
+  const secondAssistant = page.locator('[data-testid^="assistant-["]').nth(1);
+  const firstAssistant = page.locator('[data-testid^="assistant-["]').nth(0);
+
+  await dragElementAbove(secondAssistant, firstAssistant, page);
+
+  // Check new order
+  const orderAfterDragUp = await getAssistantOrder();
+  expect(orderAfterDragUp[0]).toBe(initialOrder[1]);
+  expect(orderAfterDragUp[1]).toBe(initialOrder[0]);
+
+  // Drag last assistant to second position
+  const assistants = page.locator('[data-testid^="assistant-["]');
+  const lastIndex = (await assistants.count()) - 1;
+  const lastAssistant = assistants.nth(lastIndex);
+  const secondPosition = assistants.nth(1);
+
+  await page.waitForTimeout(3000);
+  await dragElementBelow(lastAssistant, secondPosition, page);
+
+  // Check new order
+  const orderAfterDragDown = await getAssistantOrder();
+  expect(orderAfterDragDown[1]).toBe(initialOrder[lastIndex]);
+
+  // Refresh and verify order
+  await page.reload();
+  const orderAfterRefresh = await getAssistantOrder();
+  expect(orderAfterRefresh).toEqual(orderAfterDragDown);
+});

web/tests/e2e/chat/live_assistant.spec.ts

@@ -0,0 +1,70 @@
+import { test, expect } from "@playwright/test";
+import { loginAsRandomUser } from "../utils/auth";
+import {
+  navigateToAssistantInHistorySidebar,
+  sendMessage,
+  startNewChat,
+  switchModel,
+} from "../utils/chatActions";
+
+test("Chat workflow", async ({ page }) => {
+  // Clear cookies and log in as a random user
+  await page.context().clearCookies();
+  await loginAsRandomUser(page);
+
+  // Navigate to the chat page
+  await page.goto("http://localhost:3000/chat");
+
+  // Test interaction with the Art assistant
+  await navigateToAssistantInHistorySidebar(
+    page,
+    "[-3]",
+    "Assistant for generating"
+  );
+  await sendMessage(page, "Hi");
+
+  // Start a new chat session
+  await startNewChat(page);
+
+  // Verify the presence of the expected text
+  await expect(page.getByText("Assistant for generating")).toBeVisible();
+
+  // Test interaction with the General assistant
+  await navigateToAssistantInHistorySidebar(
+    page,
+    "[-1]",
+    "Assistant with no search"
+  );
+
+  // Verify the URL after selecting the General assistant
+  await expect(page).toHaveURL("http://localhost:3000/chat?assistantId=-1");
+
+  // Test creation of a new assistant
+  await page.getByRole("button", { name: "Explore Assistants" }).click();
+  await page.getByRole("button", { name: "Create" }).click();
+  await page.getByTestId("name").click();
+  await page.getByTestId("name").fill("Test Assistant");
+  await page.getByTestId("description").click();
+  await page.getByTestId("description").fill("Test Assistant Description");
+  await page.getByTestId("system_prompt").click();
+  await page.getByTestId("system_prompt").fill("Test Assistant Instructions");
+  await page.getByRole("button", { name: "Create" }).click();
+
+  // Verify the successful creation of the new assistant
+  await expect(page.getByText("Test Assistant Description")).toBeVisible({
+    timeout: 5000,
+  });
+
+  // Start another new chat session
+  await startNewChat(page);
+
+  // Verify the presence of the default assistant text
+  try {
+    await expect(page.getByText("Assistant with access to")).toBeVisible({
+      timeout: 5000,
+    });
+  } catch (error) {
+    console.error("Live Assistant final page content:");
+    console.error(await page.content());
+  }
+});

web/tests/e2e/chat/llm_ordering.spec.ts

@@ -0,0 +1,84 @@
+import { test, expect } from "@playwright/test";
+import { loginAsRandomUser } from "../utils/auth";
+import {
+  navigateToAssistantInHistorySidebar,
+  sendMessage,
+  verifyCurrentModel,
+  switchModel,
+  startNewChat,
+} from "../utils/chatActions";
+
+test("LLM Ordering and Model Switching", async ({ page }) => {
+  // Setup: Clear cookies and log in as a random user
+  await page.context().clearCookies();
+  await loginAsRandomUser(page);
+
+  // Navigate to the chat page and verify URL
+  await page.goto("http://localhost:3000/chat");
+  await page.waitForSelector("#onyx-chat-input-textarea");
+  await expect(page.url()).toBe("http://localhost:3000/chat");
+
+  // Configure user settings: Set default model to GPT 4 Turbo
+  await page.locator("#onyx-user-dropdown").click();
+  await page.getByText("User Settings").click();
+  await page.getByRole("combobox").click();
+  await page.getByLabel("GPT 4 Turbo", { exact: true }).click();
+  await page.getByLabel("Close modal").click();
+  await verifyCurrentModel(page, "GPT 4 Turbo");
+
+  // Test Art Assistant: Should use its own model (GPT 4o)
+  await navigateToAssistantInHistorySidebar(
+    page,
+    "[-3]",
+    "Assistant for generating"
+  );
+  await sendMessage(page, "Sample message");
+  await verifyCurrentModel(page, "GPT 4o");
+
+  // Verify model persistence for Art Assistant
+  await sendMessage(page, "Sample message");
+
+  // Test new chat: Should use Art Assistant's model initially
+  await startNewChat(page);
+  await expect(page.getByText("Assistant for generating")).toBeVisible();
+  await verifyCurrentModel(page, "GPT 4o");
+
+  // Test another new chat: Should use user's default model (GPT 4 Turbo)
+  await startNewChat(page);
+  await verifyCurrentModel(page, "GPT 4 Turbo");
+
+  // Test model switching within a chat
+  await switchModel(page, "O1 Mini");
+  await sendMessage(page, "Sample message");
+  await verifyCurrentModel(page, "O1 Mini");
+
+  // Create a custom assistant with a specific model
+  await page.getByRole("button", { name: "Explore Assistants" }).click();
+  await page.getByRole("button", { name: "Create" }).click();
+  await page.waitForTimeout(2000);
+  await page.getByTestId("name").fill("Sample Name");
+  await page.getByTestId("description").fill("Sample Description");
+  await page.getByTestId("system_prompt").fill("Sample Instructions");
+  await page.getByRole("combobox").click();
+  await page
+    .getByLabel("GPT 4 Turbo (Preview)")
+    .getByText("GPT 4 Turbo (Preview)")
+    .click();
+  await page.getByRole("button", { name: "Create" }).click();
+
+  // Verify custom assistant uses its specified model
+  await page.locator("#onyx-chat-input-textarea").fill("");
+  await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
+
+  // Ensure model persistence for custom assistant
+  await sendMessage(page, "Sample message");
+  await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
+
+  // Switch back to Art Assistant and verify its model
+  await navigateToAssistantInHistorySidebar(
+    page,
+    "[-3]",
+    "Assistant for generating"
+  );
+  await verifyCurrentModel(page, "GPT 4o");
+});

web/tests/e2e/utils/auth.ts

@@ -35,3 +35,40 @@ export async function loginAs(page: Page, userType: "admin" | "user") {
     }
   }
 }
+// Function to generate a random email and password
+const generateRandomCredentials = () => {
+  const randomString = Math.random().toString(36).substring(2, 10);
+  const specialChars = "!@#$%^&*()_+{}[]|:;<>,.?~";
+  const randomSpecialChar =
+    specialChars[Math.floor(Math.random() * specialChars.length)];
+  const randomUpperCase = String.fromCharCode(
+    65 + Math.floor(Math.random() * 26)
+  );
+  const randomNumber = Math.floor(Math.random() * 10);
+
+  return {
+    email: `test_${randomString}@example.com`,
+    password: `P@ssw0rd_${randomUpperCase}${randomSpecialChar}${randomNumber}${randomString}`,
+  };
+};
+
+// Function to sign up a new random user
+export async function loginAsRandomUser(page: Page) {
+  const { email, password } = generateRandomCredentials();
+
+  await page.goto("http://localhost:3000/auth/signup");
+
+  await page.fill("#email", email);
+  await page.fill("#password", password);
+
+  // Click the signup button
+  await page.click('button[type="submit"]');
+  try {
+    await page.waitForURL("http://localhost:3000/chat");
+  } catch (error) {
+    console.log(`Timeout occurred. Current URL: ${page.url()}`);
+    throw new Error("Failed to sign up and redirect to chat page");
+  }
+
+  return { email, password };
+}

web/tests/e2e/utils/chatActions.ts

@@ -0,0 +1,48 @@
+import { Page } from "@playwright/test";
+import { expect } from "@playwright/test";
+
+export async function navigateToAssistantInHistorySidebar(
+  page: Page,
+  testId: string,
+  description: string
+) {
+  await page.getByTestId(`assistant-${testId}`).click();
+  try {
+    await expect(page.getByText(description)).toBeVisible();
+  } catch (error) {
+    console.error("Error in navigateToAssistantInHistorySidebar:", error);
+    const pageText = await page.textContent("body");
+    console.log("Page text:", pageText);
+    throw error;
+  }
+}
+
+export async function sendMessage(page: Page, message: string) {
+  await page.locator("#onyx-chat-input-textarea").click();
+  await page.locator("#onyx-chat-input-textarea").fill(message);
+  await page.locator("#onyx-chat-input-send-button").click();
+  await page.waitForSelector("#onyx-ai-message");
+  await page.waitForTimeout(2000);
+}
+
+export async function verifyCurrentModel(page: Page, modelName: string) {
+  await page.waitForTimeout(1000);
+  const chatInput = page.locator("#onyx-chat-input");
+  const text = await chatInput.textContent();
+  expect(text).toContain(modelName);
+  await page.waitForTimeout(1000);
+}
+
+// Start of Selection
+export async function switchModel(page: Page, modelName: string) {
+  await page.getByTestId("llm-popover-trigger").click();
+  await page
+    .getByRole("button", { name: `Logo ${modelName}`, exact: true })
+    .click();
+  await page.waitForTimeout(1000);
+}
+
+export async function startNewChat(page: Page) {
+  await page.getByRole("link", { name: "Start New Chat" }).click();
+  await expect(page.locator('div[data-testid="chat-intro"]')).toBeVisible();
+}

web/tests/e2e/utils/dragUtils.ts

@@ -0,0 +1,74 @@
+import { Locator, Page } from "@playwright/test";
+
+/**
+ * Drag "source" above (higher Y) "target" by using mouse events.
+ * Positions the cursor on the lower half of source, then moves to the top half of the target.
+ */
+export async function dragElementAbove(
+  sourceLocator: Locator,
+  targetLocator: Locator,
+  page: Page
+) {
+  // Get bounding boxes
+  const sourceBB = await sourceLocator.boundingBox();
+  const targetBB = await targetLocator.boundingBox();
+  if (!sourceBB || !targetBB) {
+    throw new Error("Source/target bounding boxes not found.");
+  }
+
+  // Move over source, press mouse down
+  await page.mouse.move(
+    sourceBB.x + sourceBB.width / 2,
+    sourceBB.y + sourceBB.height * 0.75 // Move to 3/4 down the source element
+  );
+  await page.mouse.down();
+
+  // Move to a point slightly above the target's center
+  await page.mouse.move(
+    targetBB.x + targetBB.width / 2,
+    targetBB.y + targetBB.height * 0.1, // Move to 1/10 down the target element
+    { steps: 20 } // Increase steps for smoother drag
+  );
+  await page.mouse.up();
+
+  // Increase wait time for DnD transitions
+  await page.waitForTimeout(200);
+}
+
+/**
+ * Drag "source" below (higher Y → lower Y) "target" using mouse events.
+ */
+export async function dragElementBelow(
+  sourceLocator: Locator,
+  targetLocator: Locator,
+  page: Page
+) {
+  // Get bounding boxes
+  const sourceBB = await targetLocator.boundingBox();
+  const targetBB = await sourceLocator.boundingBox();
+  if (!sourceBB || !targetBB) {
+    throw new Error("Source/target bounding boxes not found.");
+  }
+
+  // Move over source, press mouse down
+  await page.mouse.move(
+    sourceBB.x + sourceBB.width / 2,
+    sourceBB.y + sourceBB.height * 0.25 // Move to 1/4 down the source element
+  );
+  await page.mouse.down();
+
+  // Move to a point well below the target's bottom edge
+  await page.mouse.move(
+    targetBB.x + targetBB.width / 2,
+    targetBB.y + targetBB.height + 50, // Move 50 pixels below the target element
+    { steps: 50 } // Keep the same number of steps for smooth drag
+  );
+
+  // Hold for a moment to ensure the drag is registered
+  await page.waitForTimeout(500);
+
+  await page.mouse.up();
+
+  // Wait for DnD transitions and potential animations
+  await page.waitForTimeout(1000);
+}