mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] e2e: add a combinatorial-flows spec with strict post-conditions, page+console error capture, and a self-check that proves must() actually fails
This commit is contained in:
Eric
@@ -0,0 +1,329 @@
|
||||
import { test, expect, ElectronApplication, Page, Locator } from '@playwright/test';
|
||||
import { launchApp, waitForMainWindow } from '../helpers/launch';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
// Strict combinatorial pass that exercises the app the way a real user does:
|
||||
// every click has a positive post-condition (route/state/element change), every
|
||||
// page error and console error is captured, every renderer crash is asserted
|
||||
// against a backend-log budget, and the toggle/theme matrices flip both ways.
|
||||
// Silent skips are NOT allowed: a missing target fails the step. This is the
|
||||
// gate that's supposed to actually catch regressions; deep-coverage.spec.ts is
|
||||
// the cheaper mount-only smoke that runs alongside it.
|
||||
|
||||
function backendLogPath(): string {
|
||||
if (process.platform === 'win32') return path.join(process.env.APPDATA || '', 'OpenSwarm', 'data', 'backend.log');
|
||||
if (process.platform === 'darwin') return path.join(os.homedir(), 'Library', 'Application Support', 'OpenSwarm', 'data', 'backend.log');
|
||||
return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'OpenSwarm', 'data', 'backend.log');
|
||||
}
|
||||
function rendererCrashes(): number {
|
||||
try { return (fs.readFileSync(backendLogPath(), 'utf8').match(/renderer process gone/g) || []).length; }
|
||||
catch { return 0; }
|
||||
}
|
||||
|
||||
// Console noise we don't fail on; everything else is treated as a real bug.
|
||||
const CONSOLE_WHITELIST: RegExp[] = [
|
||||
/DevTools listening/i,
|
||||
/Autofill\.enable/i,
|
||||
/Autofill\.setAddresses/i,
|
||||
/electron-store/i,
|
||||
/\[HMR\]/i,
|
||||
/downloadable font/i,
|
||||
/chrome-extension/i,
|
||||
];
|
||||
|
||||
type ErrEvent = { kind: 'pageerror' | 'console'; text: string };
|
||||
|
||||
test.describe.configure({ mode: 'serial' });
|
||||
test.describe('combinatorial user flows', () => {
|
||||
let app: ElectronApplication;
|
||||
let page: Page;
|
||||
let baselineCrashes = 0;
|
||||
let errors: ErrEvent[] = [];
|
||||
|
||||
// Strict click: locator MUST resolve to >=1 visible element. No silent skips.
|
||||
const must = async (loc: Locator, label: string) => {
|
||||
const count = await loc.count();
|
||||
expect(count, `expected at least one match for: ${label}`).toBeGreaterThan(0);
|
||||
await expect(loc.first(), `${label}: not visible`).toBeVisible({ timeout: 15_000 });
|
||||
return loc.first();
|
||||
};
|
||||
const clickMust = async (loc: Locator, label: string) => {
|
||||
const el = await must(loc, label);
|
||||
await el.click({ timeout: 8_000 });
|
||||
return el;
|
||||
};
|
||||
const errorsSince = (mark: number) => errors.slice(mark).filter((e) => !CONSOLE_WHITELIST.some((rx) => rx.test(e.text)));
|
||||
const assertNoNew = (mark: number, label: string) => {
|
||||
const now = rendererCrashes();
|
||||
expect(now, `renderer crashed during: ${label}`).toBe(baselineCrashes);
|
||||
const fresh = errorsSince(mark);
|
||||
expect(fresh.map((e) => `${e.kind}: ${e.text}`).join('\n'), `unexpected errors during: ${label}`).toBe('');
|
||||
};
|
||||
|
||||
test.beforeAll(async () => {
|
||||
app = await launchApp();
|
||||
page = await waitForMainWindow(app);
|
||||
page.on('pageerror', (e) => errors.push({ kind: 'pageerror', text: String(e?.message ?? e) }));
|
||||
page.on('console', (m) => { if (m.type() === 'error') errors.push({ kind: 'console', text: m.text() }); });
|
||||
baselineCrashes = rendererCrashes();
|
||||
});
|
||||
test.afterAll(async () => { await app?.close().catch(() => {}); });
|
||||
|
||||
// The "test the test" sanity check: prove our must() helper fails loudly when
|
||||
// a target is missing. If this ever passes silently, every later assertion is
|
||||
// also unreliable, so the whole suite is invalid and we want to know early.
|
||||
test('self-check: must() actually fails on a missing target', async () => {
|
||||
let threw = false;
|
||||
try { await must(page.locator('#__definitely_not_in_dom__'), 'self-check sentinel'); }
|
||||
catch { threw = true; }
|
||||
expect(threw, 'must() did NOT fail on missing element; the strict-click guarantee is broken').toBe(true);
|
||||
});
|
||||
|
||||
test('home: react root mounted, no banner-only fallback', async () => {
|
||||
const mark = errors.length;
|
||||
const root = page.locator('#root');
|
||||
await expect(root).toBeVisible();
|
||||
const childCount = await root.evaluate((el) => el.childElementCount);
|
||||
expect(childCount, 'react root rendered no children').toBeGreaterThan(0);
|
||||
assertNoNew(mark, 'home render');
|
||||
});
|
||||
|
||||
test('sidebar: every primary nav item navigates to its surface', async () => {
|
||||
const mark = errors.length;
|
||||
|
||||
// Make sure sidebar is expanded so the labels are clickable.
|
||||
const sidebarToggle = page.locator('[data-onboarding="sidebar-toggle"]');
|
||||
if ((await sidebarToggle.getAttribute('aria-expanded')) === 'false') await sidebarToggle.click();
|
||||
await expect(sidebarToggle).toHaveAttribute('aria-expanded', 'true');
|
||||
|
||||
// Customization expands inline; clicking should reveal Skills/Actions/Modes.
|
||||
const customization = page.locator('[data-onboarding="sidebar-customization"]');
|
||||
await clickMust(customization, 'sidebar customization');
|
||||
await expect(customization).toHaveAttribute('aria-expanded', 'true', { timeout: 5_000 });
|
||||
for (const label of ['Skills', 'Actions', 'Modes']) {
|
||||
const item = page.getByText(label, { exact: true });
|
||||
await clickMust(item, `customization > ${label}`);
|
||||
await expect(page.locator('#root')).toBeVisible();
|
||||
const url = page.url();
|
||||
expect(url, `URL did not change to customization route for ${label}`).toMatch(/customization|skills|actions|modes/i);
|
||||
assertNoNew(mark, `nav ${label}`);
|
||||
}
|
||||
|
||||
// Apps section.
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-apps"]'), 'sidebar apps');
|
||||
await expect.poll(() => page.url(), { timeout: 5_000 }).toMatch(/apps/);
|
||||
assertNoNew(mark, 'nav Apps');
|
||||
|
||||
// Dashboards section.
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-dashboards"]'), 'sidebar dashboards');
|
||||
await expect.poll(() => page.url(), { timeout: 5_000 }).toMatch(/dashboard|^[^?#]*\/?$/);
|
||||
assertNoNew(mark, 'nav Dashboards');
|
||||
});
|
||||
|
||||
test('settings modal: opens, every tab activates, closes', async ({}, info) => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-settings-button"]'), 'sidebar Settings');
|
||||
// Modal title is unique to the open settings dialog.
|
||||
await expect(page.getByText('Settings', { exact: true }).first()).toBeVisible();
|
||||
|
||||
for (const tab of ['General', 'Models', 'Usage', 'Commands']) {
|
||||
const tabLoc = page.getByRole('tab', { name: tab });
|
||||
await clickMust(tabLoc, `settings tab ${tab}`);
|
||||
await expect(tabLoc.first()).toHaveAttribute('aria-selected', 'true');
|
||||
await page.screenshot({ path: info.outputPath(`settings-${tab.toLowerCase()}.png`) });
|
||||
assertNoNew(mark, `settings tab ${tab}`);
|
||||
}
|
||||
|
||||
// Close via the dedicated close button (which has a stable data-onboarding hook).
|
||||
await clickMust(page.locator('[data-onboarding="settings-close-button"]'), 'settings close');
|
||||
await expect(page.getByRole('tab', { name: 'General' })).toHaveCount(0, { timeout: 5_000 });
|
||||
assertNoNew(mark, 'settings close');
|
||||
});
|
||||
|
||||
test('settings: theme toggle actually flips and persists', async () => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-settings-button"]'), 'open settings');
|
||||
// General is the default tab; assert + force to be safe.
|
||||
await clickMust(page.getByRole('tab', { name: 'General' }), 'tab General');
|
||||
|
||||
const readMode = () => page.evaluate(() => localStorage.getItem('self-swarm-theme-mode'));
|
||||
const before = await readMode();
|
||||
const target = before === 'dark' ? 'Light' : 'Dark';
|
||||
await clickMust(page.getByRole('button', { name: target }), `theme button ${target}`);
|
||||
await expect.poll(readMode, { timeout: 5_000 }).not.toBe(before);
|
||||
const flipped = await readMode();
|
||||
expect(flipped, 'theme localStorage did not flip').not.toBe(before);
|
||||
|
||||
// Computed background must visibly change.
|
||||
const bg = await page.evaluate(() => getComputedStyle(document.body).backgroundColor);
|
||||
expect(bg, 'body background did not pick up the new theme tokens').not.toBe('');
|
||||
|
||||
// Revert so later tests start from the same state.
|
||||
const back = before === 'dark' ? 'Dark' : 'Light';
|
||||
await clickMust(page.getByRole('button', { name: back }), `revert theme ${back}`);
|
||||
await expect.poll(readMode, { timeout: 5_000 }).toBe(before);
|
||||
|
||||
await clickMust(page.locator('[data-onboarding="settings-close-button"]'), 'close settings');
|
||||
assertNoNew(mark, 'theme flip + revert');
|
||||
});
|
||||
|
||||
test('settings: every Switch on General flips, reverts, and the renderer survives', async () => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-settings-button"]'), 'open settings');
|
||||
await clickMust(page.getByRole('tab', { name: 'General' }), 'tab General');
|
||||
|
||||
// MUI Switch renders an inner <input type=checkbox>. Limit to inputs that
|
||||
// are interactable so we don't pick up off-screen ones from other tabs.
|
||||
const switches = page.locator('.MuiSwitch-root input[type="checkbox"]');
|
||||
const n = await switches.count();
|
||||
expect(n, 'no Switch components found on General tab; selectors drifted').toBeGreaterThan(0);
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const sw = switches.nth(i);
|
||||
const before = await sw.isChecked();
|
||||
// MUI hides the input; click the parent label/root to toggle the way a user would.
|
||||
await sw.locator('xpath=ancestor::*[contains(@class,"MuiSwitch-root")][1]').click({ timeout: 4_000 });
|
||||
await expect.poll(() => sw.isChecked(), { timeout: 5_000 }).toBe(!before);
|
||||
// Revert so the test is hermetic for the next switch.
|
||||
await sw.locator('xpath=ancestor::*[contains(@class,"MuiSwitch-root")][1]').click({ timeout: 4_000 });
|
||||
await expect.poll(() => sw.isChecked(), { timeout: 5_000 }).toBe(before);
|
||||
assertNoNew(mark, `switch #${i} flip+revert`);
|
||||
}
|
||||
|
||||
await clickMust(page.locator('[data-onboarding="settings-close-button"]'), 'close settings');
|
||||
assertNoNew(mark, 'all-switches matrix');
|
||||
});
|
||||
|
||||
test('onboarding: See all todos opens the roadmap, Escape closes it', async () => {
|
||||
const mark = errors.length;
|
||||
const roadmapTrigger = page.getByText('See all todos', { exact: true });
|
||||
// Roadmap is gated on the panel being visible; if it isn't, that itself is
|
||||
// a state we explicitly need to know about, so fail the assertion.
|
||||
await clickMust(roadmapTrigger, 'See all todos');
|
||||
// Roadmap modal has a unique aria-label="Close roadmap" close button.
|
||||
await expect(page.locator('[aria-label="Close roadmap"]')).toBeVisible({ timeout: 8_000 });
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(page.locator('[aria-label="Close roadmap"]')).toHaveCount(0, { timeout: 5_000 });
|
||||
assertNoNew(mark, 'roadmap open + escape');
|
||||
});
|
||||
|
||||
test('dashboard toolbar: New Agent opens compose with contentEditable that accepts typing', async ({}, info) => {
|
||||
const mark = errors.length;
|
||||
// Make sure we're on a dashboard (the toolbar lives there).
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-dashboards"]'), 'sidebar dashboards');
|
||||
await clickMust(page.locator('[data-onboarding="new-agent-button"]'), 'toolbar New Agent');
|
||||
|
||||
const editor = page.locator('[data-onboarding="chat-input"]');
|
||||
await expect(editor.first(), 'EditorSurface contentEditable did not mount').toBeVisible({ timeout: 10_000 });
|
||||
await editor.first().click();
|
||||
await page.keyboard.type('hello agent', { delay: 15 });
|
||||
await expect.poll(async () => (await editor.first().innerText()).trim(), { timeout: 5_000 }).toContain('hello agent');
|
||||
await page.screenshot({ path: info.outputPath('new-agent-typed.png') });
|
||||
// Don't actually send; clear and dismiss so we don't hit a real provider.
|
||||
await page.keyboard.press('Escape').catch(() => {});
|
||||
assertNoNew(mark, 'New Agent compose + type');
|
||||
});
|
||||
|
||||
test('dashboard toolbar: Browser card mounts (webview path, not grey iframe)', async ({}, info) => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="browser-button"]'), 'toolbar Browser');
|
||||
// Wait for at least one <webview> to attach. A grey iframe = no webview = fail.
|
||||
await page.waitForFunction(() => document.querySelectorAll('webview').length > 0, undefined, { timeout: 15_000 });
|
||||
const webviews = await page.locator('webview').count();
|
||||
expect(webviews, 'no <webview> attached after Browser click; render path collapsed to iframe').toBeGreaterThan(0);
|
||||
await page.screenshot({ path: info.outputPath('browser-card.png') });
|
||||
assertNoNew(mark, 'Browser card mount (webview)');
|
||||
});
|
||||
|
||||
test('dashboard toolbar: Add note + Add App + History each mount their surfaces', async () => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.getByRole('button', { name: 'Add note' }), 'toolbar Add note');
|
||||
assertNoNew(mark, 'Add note mount');
|
||||
|
||||
await clickMust(page.getByRole('button', { name: 'Add App' }), 'toolbar Add App');
|
||||
// Picker is a dialog; closing via Escape is enough.
|
||||
await page.keyboard.press('Escape').catch(() => {});
|
||||
assertNoNew(mark, 'Add App picker');
|
||||
|
||||
await clickMust(page.getByRole('button', { name: 'History' }), 'toolbar History');
|
||||
await page.keyboard.press('Escape').catch(() => {});
|
||||
assertNoNew(mark, 'History panel');
|
||||
});
|
||||
|
||||
test('modes: edit screen mounts RichPromptEditor and accepts typing (TSF crash class)', async () => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-customization"]'), 'open customization');
|
||||
await clickMust(page.getByText('Modes', { exact: true }), 'go to Modes');
|
||||
// Modes list might be empty on a clean profile; we still want to enter the
|
||||
// editor by either an existing row or the create flow. Fail if neither path exists.
|
||||
const editIcons = page.locator('[aria-label="Edit"], [aria-label*="edit mode" i]');
|
||||
const newBtn = page.getByRole('button', { name: /new mode|create mode|add mode/i });
|
||||
if (await editIcons.count()) {
|
||||
await editIcons.first().click({ timeout: 5_000 });
|
||||
} else if (await newBtn.count()) {
|
||||
await newBtn.first().click({ timeout: 5_000 });
|
||||
} else {
|
||||
// Neither edit nor create button means a UI regression on Modes; fail explicitly.
|
||||
expect.fail('Modes screen exposes no edit-or-create entry point; rich editor is unreachable');
|
||||
}
|
||||
// RichPromptEditor uses a contentEditable; verify one is mounted somewhere on the route.
|
||||
await page.waitForFunction(
|
||||
() => Array.from(document.querySelectorAll('[contenteditable="true"]')).length > 0,
|
||||
undefined,
|
||||
{ timeout: 15_000 },
|
||||
);
|
||||
await page.keyboard.type('test prompt', { delay: 10 });
|
||||
assertNoNew(mark, 'Modes RichPromptEditor mount + type');
|
||||
await page.keyboard.press('Escape').catch(() => {});
|
||||
});
|
||||
|
||||
test('theme x toggle matrix: dark + first switch flipped, light + first switch flipped, all reverted', async () => {
|
||||
const mark = errors.length;
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-settings-button"]'), 'open settings (matrix)');
|
||||
await clickMust(page.getByRole('tab', { name: 'General' }), 'tab General');
|
||||
const readMode = () => page.evaluate(() => localStorage.getItem('self-swarm-theme-mode'));
|
||||
const initialMode = await readMode();
|
||||
const switches = page.locator('.MuiSwitch-root input[type="checkbox"]');
|
||||
expect(await switches.count()).toBeGreaterThan(0);
|
||||
const sw = switches.first();
|
||||
const switchRoot = sw.locator('xpath=ancestor::*[contains(@class,"MuiSwitch-root")][1]');
|
||||
const initialSwitch = await sw.isChecked();
|
||||
|
||||
for (const targetMode of ['dark', 'light'] as const) {
|
||||
const btn = page.getByRole('button', { name: targetMode === 'dark' ? 'Dark' : 'Light' });
|
||||
await clickMust(btn, `set theme ${targetMode}`);
|
||||
await expect.poll(readMode, { timeout: 5_000 }).toBe(targetMode);
|
||||
await switchRoot.click({ timeout: 4_000 });
|
||||
await expect.poll(() => sw.isChecked(), { timeout: 5_000 }).toBe(!initialSwitch);
|
||||
await switchRoot.click({ timeout: 4_000 });
|
||||
await expect.poll(() => sw.isChecked(), { timeout: 5_000 }).toBe(initialSwitch);
|
||||
assertNoNew(mark, `theme=${targetMode} x switch[0] flip+revert`);
|
||||
}
|
||||
|
||||
if (initialMode) {
|
||||
await clickMust(page.getByRole('button', { name: initialMode === 'dark' ? 'Dark' : 'Light' }), 'restore theme');
|
||||
await expect.poll(readMode, { timeout: 5_000 }).toBe(initialMode);
|
||||
}
|
||||
await clickMust(page.locator('[data-onboarding="settings-close-button"]'), 'close settings (matrix)');
|
||||
assertNoNew(mark, 'theme x toggle matrix');
|
||||
});
|
||||
|
||||
test('resilience: open + close Settings 3x without state corruption', async () => {
|
||||
const mark = errors.length;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await clickMust(page.locator('[data-onboarding="sidebar-settings-button"]'), `open settings round ${i}`);
|
||||
await expect(page.getByRole('tab', { name: 'General' })).toBeVisible({ timeout: 5_000 });
|
||||
await clickMust(page.locator('[data-onboarding="settings-close-button"]'), `close settings round ${i}`);
|
||||
await expect(page.getByRole('tab', { name: 'General' })).toHaveCount(0, { timeout: 5_000 });
|
||||
assertNoNew(mark, `settings open/close round ${i}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('zero unexpected errors and zero new renderer crashes across whole walkthrough', () => {
|
||||
expect(rendererCrashes(), 'renderer crashed somewhere; see earlier annotations').toBe(baselineCrashes);
|
||||
const dirty = errors.filter((e) => !CONSOLE_WHITELIST.some((rx) => rx.test(e.text)));
|
||||
expect(dirty.map((e) => `${e.kind}: ${e.text}`).join('\n'), 'unexpected page/console errors during walkthrough').toBe('');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user