E2e assistant tests (#3869)

* adding llm override logic

* update

* general cleanup

* fix various tests

* rm

* update

* update

* better comments

* k

* k

* update to pass tests

* clarify content

* improve timeout
This commit is contained in:
pablonyx
2025-02-01 12:05:53 -08:00
committed by GitHub
parent a82cac5361
commit 3c34ddcc4f
23 changed files with 405 additions and 76 deletions

View File

@@ -0,0 +1,54 @@
import { test, expect } from "@playwright/test";
import { dragElementAbove, dragElementBelow } from "../utils/dragUtils";
import { loginAsRandomUser } from "../utils/auth";
test("Assistant Drag and Drop", async ({ page }) => {
await page.context().clearCookies();
await loginAsRandomUser(page);
// Navigate to the chat page
await page.goto("http://localhost:3000/chat");
// Helper function to get the current order of assistants
const getAssistantOrder = async () => {
const assistants = await page.$$('[data-testid^="assistant-["]');
return Promise.all(
assistants.map(async (assistant) => {
const nameElement = await assistant.$("p");
return nameElement ? nameElement.textContent() : "";
})
);
};
// Get the initial order
const initialOrder = await getAssistantOrder();
// Drag second assistant above first
const secondAssistant = page.locator('[data-testid^="assistant-["]').nth(1);
const firstAssistant = page.locator('[data-testid^="assistant-["]').nth(0);
await dragElementAbove(secondAssistant, firstAssistant, page);
// Check new order
const orderAfterDragUp = await getAssistantOrder();
expect(orderAfterDragUp[0]).toBe(initialOrder[1]);
expect(orderAfterDragUp[1]).toBe(initialOrder[0]);
// Drag last assistant to second position
const assistants = page.locator('[data-testid^="assistant-["]');
const lastIndex = (await assistants.count()) - 1;
const lastAssistant = assistants.nth(lastIndex);
const secondPosition = assistants.nth(1);
await page.waitForTimeout(3000);
await dragElementBelow(lastAssistant, secondPosition, page);
// Check new order
const orderAfterDragDown = await getAssistantOrder();
expect(orderAfterDragDown[1]).toBe(initialOrder[lastIndex]);
// Refresh and verify order
await page.reload();
const orderAfterRefresh = await getAssistantOrder();
expect(orderAfterRefresh).toEqual(orderAfterDragDown);
});

View File

@@ -0,0 +1,70 @@
import { test, expect } from "@playwright/test";
import { loginAsRandomUser } from "../utils/auth";
import {
navigateToAssistantInHistorySidebar,
sendMessage,
startNewChat,
switchModel,
} from "../utils/chatActions";
test("Chat workflow", async ({ page }) => {
// Clear cookies and log in as a random user
await page.context().clearCookies();
await loginAsRandomUser(page);
// Navigate to the chat page
await page.goto("http://localhost:3000/chat");
// Test interaction with the Art assistant
await navigateToAssistantInHistorySidebar(
page,
"[-3]",
"Assistant for generating"
);
await sendMessage(page, "Hi");
// Start a new chat session
await startNewChat(page);
// Verify the presence of the expected text
await expect(page.getByText("Assistant for generating")).toBeVisible();
// Test interaction with the General assistant
await navigateToAssistantInHistorySidebar(
page,
"[-1]",
"Assistant with no search"
);
// Verify the URL after selecting the General assistant
await expect(page).toHaveURL("http://localhost:3000/chat?assistantId=-1");
// Test creation of a new assistant
await page.getByRole("button", { name: "Explore Assistants" }).click();
await page.getByRole("button", { name: "Create" }).click();
await page.getByTestId("name").click();
await page.getByTestId("name").fill("Test Assistant");
await page.getByTestId("description").click();
await page.getByTestId("description").fill("Test Assistant Description");
await page.getByTestId("system_prompt").click();
await page.getByTestId("system_prompt").fill("Test Assistant Instructions");
await page.getByRole("button", { name: "Create" }).click();
// Verify the successful creation of the new assistant
await expect(page.getByText("Test Assistant Description")).toBeVisible({
timeout: 5000,
});
// Start another new chat session
await startNewChat(page);
// Verify the presence of the default assistant text
try {
await expect(page.getByText("Assistant with access to")).toBeVisible({
timeout: 5000,
});
} catch (error) {
console.error("Live Assistant final page content:");
console.error(await page.content());
}
});

View File

@@ -0,0 +1,84 @@
import { test, expect } from "@playwright/test";
import { loginAsRandomUser } from "../utils/auth";
import {
navigateToAssistantInHistorySidebar,
sendMessage,
verifyCurrentModel,
switchModel,
startNewChat,
} from "../utils/chatActions";
test("LLM Ordering and Model Switching", async ({ page }) => {
// Setup: Clear cookies and log in as a random user
await page.context().clearCookies();
await loginAsRandomUser(page);
// Navigate to the chat page and verify URL
await page.goto("http://localhost:3000/chat");
await page.waitForSelector("#onyx-chat-input-textarea");
await expect(page.url()).toBe("http://localhost:3000/chat");
// Configure user settings: Set default model to GPT 4 Turbo
await page.locator("#onyx-user-dropdown").click();
await page.getByText("User Settings").click();
await page.getByRole("combobox").click();
await page.getByLabel("GPT 4 Turbo", { exact: true }).click();
await page.getByLabel("Close modal").click();
await verifyCurrentModel(page, "GPT 4 Turbo");
// Test Art Assistant: Should use its own model (GPT 4o)
await navigateToAssistantInHistorySidebar(
page,
"[-3]",
"Assistant for generating"
);
await sendMessage(page, "Sample message");
await verifyCurrentModel(page, "GPT 4o");
// Verify model persistence for Art Assistant
await sendMessage(page, "Sample message");
// Test new chat: Should use Art Assistant's model initially
await startNewChat(page);
await expect(page.getByText("Assistant for generating")).toBeVisible();
await verifyCurrentModel(page, "GPT 4o");
// Test another new chat: Should use user's default model (GPT 4 Turbo)
await startNewChat(page);
await verifyCurrentModel(page, "GPT 4 Turbo");
// Test model switching within a chat
await switchModel(page, "O1 Mini");
await sendMessage(page, "Sample message");
await verifyCurrentModel(page, "O1 Mini");
// Create a custom assistant with a specific model
await page.getByRole("button", { name: "Explore Assistants" }).click();
await page.getByRole("button", { name: "Create" }).click();
await page.waitForTimeout(2000);
await page.getByTestId("name").fill("Sample Name");
await page.getByTestId("description").fill("Sample Description");
await page.getByTestId("system_prompt").fill("Sample Instructions");
await page.getByRole("combobox").click();
await page
.getByLabel("GPT 4 Turbo (Preview)")
.getByText("GPT 4 Turbo (Preview)")
.click();
await page.getByRole("button", { name: "Create" }).click();
// Verify custom assistant uses its specified model
await page.locator("#onyx-chat-input-textarea").fill("");
await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
// Ensure model persistence for custom assistant
await sendMessage(page, "Sample message");
await verifyCurrentModel(page, "GPT 4 Turbo (Preview)");
// Switch back to Art Assistant and verify its model
await navigateToAssistantInHistorySidebar(
page,
"[-3]",
"Assistant for generating"
);
await verifyCurrentModel(page, "GPT 4o");
});