diff --git a/web/.gitignore b/web/.gitignore index c72b98d44..f87b13aa7 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -35,6 +35,8 @@ yarn-error.log* *.tsbuildinfo next-env.d.ts +# playwright testing temp files /admin_auth.json +/user_auth.json /build-archive.log - +/test-results diff --git a/web/README.md b/web/README.md index 38af9cd58..91d8fd067 100644 --- a/web/README.md +++ b/web/README.md @@ -21,3 +21,42 @@ Open [http://localhost:3000](http://localhost:3000) with your browser to see the _Note:_ if you are having problems accessing the ^, try setting the `WEB_DOMAIN` env variable to `http://127.0.0.1:3000` and accessing it there. + +## Testing +This testing process will reset your application into a clean state. +Don't run these tests if you don't want to do this! + +Bring up the entire application. + + +1. Reset the instance + +```cd backend +export PYTEST_IGNORE_SKIP=true +pytest -s tests/integration/tests/playwright/test_playwright.py +``` + +2. Run playwright + +``` +cd web +npx playwright test +``` + +3. Inspect results + +By default, playwright.config.ts is configured to output the results to: + +``` +web/test-results +``` + +4. Upload results to Chromatic (Optional) + +This step would normally not be run by third party developers, but first party devs +may use this for local troubleshooting and testing. + +``` +cd web +npx chromatic --playwright --project-token={your token here} +``` \ No newline at end of file diff --git a/web/package-lock.json b/web/package-lock.json index 3188f851e..e3a2b7f59 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -84,11 +84,11 @@ "yup": "^1.4.0" }, "devDependencies": { - "@chromatic-com/playwright": "^0.10.0", + "@chromatic-com/playwright": "^0.10.2", "@tailwindcss/typography": "^0.5.10", "@types/chrome": "^0.0.287", "@types/jest": "^29.5.14", - "chromatic": "^11.18.1", + "chromatic": "^11.25.2", "eslint": "^8.48.0", "eslint-config-next": "^14.1.0", "jest": "^29.7.0", @@ -757,9 +757,9 @@ "license": "MIT" }, "node_modules/@chromatic-com/playwright": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@chromatic-com/playwright/-/playwright-0.10.0.tgz", - "integrity": "sha512-QjKnOfuIcq9Y97QwA3MMVzOceXn1ikelUeC8gy60d2PbsQ2NNxH2n/PrAJ8Sllr225mXD1ts9xBH+Hq3+Blo5A==", + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@chromatic-com/playwright/-/playwright-0.10.2.tgz", + "integrity": "sha512-SfP4I0rWPeSNW5VtV7eiuNSsZYK9IdVPTBT1SnUFJd3lACS1YJJd5s8pTisJvgh5Q8u9VNGWXfeuV3ddGJyRtw==", "dev": true, "license": "MIT", "dependencies": { @@ -8577,9 +8577,9 @@ } }, "node_modules/chromatic": { - "version": "11.18.1", - "resolved": "https://registry.npmjs.org/chromatic/-/chromatic-11.18.1.tgz", - "integrity": "sha512-hkNT9vA6K9+PnE/khhZYBnRCOm8NonaQDs7RZ8YHFo7/lh1b/x/uFMkTjWjaj/mkM6QOR/evu5VcZMtcaauSlw==", + "version": "11.25.2", + "resolved": "https://registry.npmjs.org/chromatic/-/chromatic-11.25.2.tgz", + "integrity": "sha512-/9eQWn6BU1iFsop86t8Au21IksTRxwXAl7if8YHD05L2AbuMjClLWZo5cZojqrJHGKDhTqfrC2X2xE4uSm0iKw==", "dev": true, "license": "MIT", "bin": { diff --git a/web/package.json b/web/package.json index f2db4b905..07ddf07d2 100644 --- a/web/package.json +++ b/web/package.json @@ -87,11 +87,11 @@ "yup": "^1.4.0" }, "devDependencies": { - "@chromatic-com/playwright": "^0.10.0", + "@chromatic-com/playwright": "^0.10.2", "@tailwindcss/typography": "^0.5.10", "@types/chrome": "^0.0.287", "@types/jest": "^29.5.14", - "chromatic": "^11.18.1", + "chromatic": "^11.25.2", "eslint": "^8.48.0", "eslint-config-next": "^14.1.0", "jest": "^29.7.0", diff --git a/web/playwright.config.ts b/web/playwright.config.ts index 9570c46d1..fa9e69483 100644 --- a/web/playwright.config.ts +++ b/web/playwright.config.ts @@ -2,7 +2,19 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ globalSetup: require.resolve("./tests/e2e/global-setup"), - timeout: 30000, // 30 seconds timeout + timeout: 60000, // 60 seconds timeout + reporter: [ + ["list"], + // Warning: uncommenting the html reporter may cause the chromatic-archives + // directory to be deleted after the test run, which will break CI. + // [ + // 'html', + // { + // outputFolder: 'test-results', // or whatever directory you want + // open: 'never', // can be 'always' | 'on-failure' | 'never' + // }, + // ], + ], projects: [ { name: "admin", diff --git a/web/tests/e2e/admin_auth.setup.ts b/web/tests/e2e/admin_auth.setup.ts index 593ee7d1f..7edd59cb8 100644 --- a/web/tests/e2e/admin_auth.setup.ts +++ b/web/tests/e2e/admin_auth.setup.ts @@ -1,5 +1,5 @@ // dependency for all admin user tests -import { test as setup } from "@playwright/test"; +import { test as setup } from "@chromatic-com/playwright"; setup("authenticate as admin", async ({ browser }) => { const context = await browser.newContext({ storageState: "admin_auth.json" }); diff --git a/web/tests/e2e/admin_oauth_redirect_uri.spec.ts b/web/tests/e2e/admin_oauth_redirect_uri.spec.ts index fdec61ee3..f543574e3 100644 --- a/web/tests/e2e/admin_oauth_redirect_uri.spec.ts +++ b/web/tests/e2e/admin_oauth_redirect_uri.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "@chromatic-com/playwright"; test.use({ storageState: "admin_auth.json" }); diff --git a/web/tests/e2e/admin_pages.spec.ts b/web/tests/e2e/admin_pages.spec.ts index 8c6198070..09c6a15d4 100644 --- a/web/tests/e2e/admin_pages.spec.ts +++ b/web/tests/e2e/admin_pages.spec.ts @@ -1,5 +1,5 @@ -import { test, expect } from "@playwright/test"; -import chromaticSnpashots from "./chromaticSnpashots.json"; +import { test, expect } from "@chromatic-com/playwright"; +import chromaticSnapshots from "./chromaticSnapshots.json"; import type { Page } from "@playwright/test"; test.use({ storageState: "admin_auth.json" }); @@ -39,7 +39,7 @@ async function verifyAdminPageNavigation( } } -for (const chromaticSnapshot of chromaticSnpashots) { +for (const chromaticSnapshot of chromaticSnapshots) { test(`Admin - ${chromaticSnapshot.name}`, async ({ page }) => { await verifyAdminPageNavigation( page, diff --git a/web/tests/e2e/chat/current_assistant.spec.ts b/web/tests/e2e/chat/current_assistant.spec.ts index f6d65f98b..556fde890 100644 --- a/web/tests/e2e/chat/current_assistant.spec.ts +++ b/web/tests/e2e/chat/current_assistant.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "@chromatic-com/playwright"; import { dragElementAbove, dragElementBelow } from "../utils/dragUtils"; import { loginAsRandomUser } from "../utils/auth"; diff --git a/web/tests/e2e/chat/live_assistant.spec.ts b/web/tests/e2e/chat/live_assistant.spec.ts index c54fb9b26..b4ce652dc 100644 --- a/web/tests/e2e/chat/live_assistant.spec.ts +++ b/web/tests/e2e/chat/live_assistant.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "@chromatic-com/playwright"; import { loginAsRandomUser } from "../utils/auth"; import { navigateToAssistantInHistorySidebar, diff --git a/web/tests/e2e/chat/llm_ordering.spec.ts b/web/tests/e2e/chat/llm_ordering.spec.ts index 3b9a363be..a96022dac 100644 --- a/web/tests/e2e/chat/llm_ordering.spec.ts +++ b/web/tests/e2e/chat/llm_ordering.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "@chromatic-com/playwright"; import { loginAsRandomUser } from "../utils/auth"; import { navigateToAssistantInHistorySidebar, diff --git a/web/tests/e2e/chromaticSnpashots.json b/web/tests/e2e/chromaticSnapshots.json similarity index 100% rename from web/tests/e2e/chromaticSnpashots.json rename to web/tests/e2e/chromaticSnapshots.json diff --git a/web/tests/e2e/utils/chatActions.ts b/web/tests/e2e/utils/chatActions.ts index aa35fbee7..f53eb7595 100644 --- a/web/tests/e2e/utils/chatActions.ts +++ b/web/tests/e2e/utils/chatActions.ts @@ -1,5 +1,5 @@ import { Page } from "@playwright/test"; -import { expect } from "@playwright/test"; +import { expect } from "@chromatic-com/playwright"; export async function navigateToAssistantInHistorySidebar( page: Page,