From 4a0accc85796986fc9164a2d43e114bab5ef6937 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 2 Jul 2026 18:53:58 +0530 Subject: [PATCH 01/91] feat(core,backend): emit console/stdout/stderr events into trace.zip and round-trip them in the reader --- packages/backend/src/trace-reader-types.ts | 17 +++ packages/backend/src/trace-reader-utils.ts | 58 +++++++- packages/backend/src/trace-reader.ts | 26 ++-- packages/backend/tests/trace-reader.test.ts | 38 +++++- packages/core/src/index.ts | 2 + packages/core/src/trace-console.ts | 93 +++++++++++++ packages/core/src/trace-exporter.ts | 141 +++++--------------- packages/core/src/trace-snapshots.ts | 102 ++++++++++++++ packages/core/tests/trace-console.test.ts | 111 +++++++++++++++ 9 files changed, 465 insertions(+), 123 deletions(-) create mode 100644 packages/core/src/trace-console.ts create mode 100644 packages/core/src/trace-snapshots.ts create mode 100644 packages/core/tests/trace-console.test.ts diff --git a/packages/backend/src/trace-reader-types.ts b/packages/backend/src/trace-reader-types.ts index 63bc3139..3a9d5e77 100644 --- a/packages/backend/src/trace-reader-types.ts +++ b/packages/backend/src/trace-reader-types.ts @@ -24,6 +24,22 @@ export interface ScreencastFrameEvent { timestamp: number } +export interface ConsoleEvent { + type: 'console' + time: number + messageType: string + text: string + args?: { preview: string; value: unknown }[] +} + +export interface StdioEvent { + type: 'stdout' | 'stderr' + timestamp: number + text?: string + /** Extension field restoring the test-vs-terminal origin; absent in foreign zips. */ + source?: 'test' | 'terminal' +} + export interface ContextOptionsEvent { type: 'context-options' wallTime: number @@ -54,4 +70,5 @@ export interface CategorizedEvents { befores: Map afters: Map frameEvents: ScreencastFrameEvent[] + consoleEvents: (ConsoleEvent | StdioEvent)[] } diff --git a/packages/backend/src/trace-reader-utils.ts b/packages/backend/src/trace-reader-utils.ts index 16a0972e..b7162c12 100644 --- a/packages/backend/src/trace-reader-utils.ts +++ b/packages/backend/src/trace-reader-utils.ts @@ -3,13 +3,20 @@ import { TraceType, + type ConsoleLog, + type LogLevel, type Metadata, type NetworkRequest, type TracePlayerFrame, type Viewport } from '@wdio/devtools-shared' -import type { ContextOptionsEvent, HarSnapshot } from './trace-reader-types.js' +import type { + ConsoleEvent, + ContextOptionsEvent, + HarSnapshot, + StdioEvent +} from './trace-reader-types.js' export function parseNdjson(text: string): Record[] { return text @@ -44,9 +51,9 @@ export function paramsToArgs( return indexKeys.map((key) => params[key]) } -// Playwright/vibium-style action label, built from class.method + the most -// meaningful param (value for fill/type, url for navigate, nothing for click) — -// matching what `playwright show-trace` renders for the same trace. +// Trace-viewer action label, built from class.method + the most meaningful +// param (value for fill/type, url for navigate, nothing for click) — +// matching what standard trace viewers render for the same trace. export function actionLabel( cls: string, method: string, @@ -109,7 +116,7 @@ export function harToNetworkRequest( ): NetworkRequest { const started = Date.parse(snapshot.startedDateTime) const startTime = Number.isFinite(started) ? started : 0 - // A foreign trace.zip (the reader accepts any Vibium/Playwright zip) can carry + // A foreign trace.zip (the reader accepts any standard-format zip) can carry // a pending or failed request with no response or content; default those. const response = snapshot.response const content = response?.content @@ -137,6 +144,47 @@ export function harToNetworkRequest( } } +const LOG_LEVELS: ReadonlySet = new Set([ + 'trace', + 'debug', + 'log', + 'info', + 'warn', + 'error' +]) + +// Reverse of the writer's level mapping ('warn' → 'warning'); a foreign +// trace zip can carry levels outside our union, which default to 'log'. +function fromTraceLevel(messageType: string): LogLevel { + if (messageType === 'warning') { + return 'warn' + } + return LOG_LEVELS.has(messageType) ? (messageType as LogLevel) : 'log' +} + +export function buildConsoleLogs( + consoleEvents: (ConsoleEvent | StdioEvent)[], + wallTime: number +): ConsoleLog[] { + return consoleEvents.map((event) => { + if (event.type === 'console') { + return { + type: fromTraceLevel(event.messageType), + args: event.args?.map((arg) => arg.value) ?? [event.text], + timestamp: wallTime + event.time, + source: 'browser' as const + } + } + return { + type: event.type === 'stderr' ? ('error' as const) : ('log' as const), + args: [event.text ?? ''], + timestamp: wallTime + event.timestamp, + // Our zips carry the origin; foreign stdio events default to terminal. + source: event.source ?? ('terminal' as const) + } + }) +} + export function nearestFrame( frames: TracePlayerFrame[], timestamp: number diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index d8643f39..6146998f 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -1,10 +1,11 @@ // Reads a trace.zip produced by core/trace-exporter.ts back into a player // payload. The writer is the inverse: this reconstructs commands from // before/after events, the frame filmstrip from screencast-frame events + -// resources/*.jpeg, and network requests from the HAR resource-snapshot -// entries. Fields the zip never carried (console logs, mutations, sources, -// suites) come back empty. Constants, event types, and pure helpers live in the -// sibling trace-reader-{constants,types,utils}.ts files. +// resources/*.jpeg, console logs from console/stdout/stderr events, and +// network requests from the HAR resource-snapshot entries. Fields the zip +// never carried (mutations, sources, suites) come back empty. Constants, +// event types, and pure helpers live in the sibling +// trace-reader-{constants,types,utils}.ts files. import fs from 'node:fs/promises' import { unzipSync, strFromU8 } from 'fflate' @@ -20,12 +21,15 @@ import type { AfterEvent, BeforeEvent, CategorizedEvents, + ConsoleEvent, ContextOptionsEvent, HarSnapshot, - ScreencastFrameEvent + ScreencastFrameEvent, + StdioEvent } from './trace-reader-types.js' import { actionLabel, + buildConsoleLogs, buildMetadata, harToNetworkRequest, nearestFrame, @@ -39,6 +43,7 @@ function categorizeEvents( const befores = new Map() const afters = new Map() const frameEvents: ScreencastFrameEvent[] = [] + const consoleEvents: (ConsoleEvent | StdioEvent)[] = [] let ctx: ContextOptionsEvent | undefined for (const event of events) { switch (event.type) { @@ -58,9 +63,14 @@ function categorizeEvents( case 'screencast-frame': frameEvents.push(event as unknown as ScreencastFrameEvent) break + case 'console': + case 'stdout': + case 'stderr': + consoleEvents.push(event as unknown as ConsoleEvent | StdioEvent) + break } } - return { ctx, befores, afters, frameEvents } + return { ctx, befores, afters, frameEvents, consoleEvents } } function buildFrames( @@ -100,7 +110,7 @@ function buildCommands( command: REVERSE_ACTION_MAP[`${before.class}.${before.method}`] ?? before.method, args: paramsToArgs(before.params), - // Show the Playwright/vibium label (`Element.fill("x")`); the command + // Show the trace-style label (`Element.fill("x")`); the command // name above still drives the UI's category colour and icon. title: actionLabel(before.class, before.method, before.params), startTime: wallTime + before.startTime, @@ -143,7 +153,7 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { const trace: TraceLog = { mutations: [], logs: [], - consoleLogs: [], + consoleLogs: buildConsoleLogs(categorized.consoleEvents, wallTime), networkRequests, metadata: buildMetadata(categorized.ctx), commands, diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index f2aa880e..8df8f9ff 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -73,7 +73,18 @@ function fixtureZip(): Uint8Array { width: 1024, height: 768, timestamp: 260 - } + }, + { + type: 'console', + time: 120, + pageId: 'page@abcd1234', + messageType: 'warning', + text: 'low disk', + args: [{ preview: 'low disk', value: 'low disk' }], + location: { url: '', lineNumber: 0, columnNumber: 0 } + }, + { type: 'stdout', timestamp: 130, text: 'spec started', source: 'test' }, + { type: 'stderr', timestamp: 140, text: 'worker warning' } ] const networkEntry = { type: 'resource-snapshot', @@ -159,9 +170,32 @@ describe('parseTraceZip', () => { (trace.metadata.capabilities as { browserName: string }).browserName ).toBe('chrome') expect(trace.metadata.sessionId).toBe('abcd1234') - expect(trace.consoleLogs).toEqual([]) expect(trace.mutations).toEqual([]) expect(trace.suites).toEqual([]) expect(trace.sources).toEqual({}) }) + + it('reconstructs console logs from console and stdio events', () => { + const { trace } = parseTraceZip(fixtureZip()) + expect(trace.consoleLogs).toEqual([ + { + type: 'warn', + args: ['low disk'], + timestamp: WALL_TIME + 120, + source: 'browser' + }, + { + type: 'log', + args: ['spec started'], + timestamp: WALL_TIME + 130, + source: 'test' + }, + { + type: 'error', + args: ['worker warning'], + timestamp: WALL_TIME + 140, + source: 'terminal' + } + ]) + }) }) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 21fcdf08..36b8bcd7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -8,8 +8,10 @@ export * from './assert-patcher.js' export * from './element-snapshot.js' export * from './element-scripts.js' export * from './element-types.js' +export * from './trace-console.js' export * from './trace-exporter.js' export * from './trace-har.js' +export * from './trace-snapshots.js' export * from './trace-zip-writer.js' export * from './bidi.js' export * from './console.js' diff --git a/packages/core/src/trace-console.ts b/packages/core/src/trace-console.ts new file mode 100644 index 00000000..f9db30fd --- /dev/null +++ b/packages/core/src/trace-console.ts @@ -0,0 +1,93 @@ +// Maps captured ConsoleLog entries into trace-event vocabulary: browser +// console entries become `console` events; test/terminal output becomes +// `stdout`/`stderr` events (which carry no location semantics — matching +// what we capture for Node-side lines). + +import type { ConsoleLog, LogSource } from '@wdio/devtools-shared' + +export interface ConsoleEvent { + type: 'console' + time: number + pageId?: string + messageType: string + text: string + args?: { preview: string; value: unknown }[] + location: { url: string; lineNumber: number; columnNumber: number } +} + +export interface StdioEvent { + type: 'stdout' | 'stderr' + timestamp: number + text?: string + /** Extension field: preserves the test-vs-terminal origin the standard + * stdio vocabulary can't express. Foreign viewers ignore it. */ + source?: Extract +} + +// Keeps a pathological run (console.log in a tight loop) from producing a +// trace.trace the viewer can't open. +const MAX_CONSOLE_EVENTS = 10_000 + +/** The trace format's console vocabulary uses 'warning'; 'trace' has no + * equivalent — 'debug' is the nearest severity. */ +function toTraceLevel(level: ConsoleLog['type']): string { + if (level === 'warn') { + return 'warning' + } + if (level === 'trace') { + return 'debug' + } + return level +} + +function previewArg(arg: unknown): string { + if (typeof arg === 'string') { + return arg + } + try { + return JSON.stringify(arg) ?? String(arg) + } catch { + return String(arg) + } +} + +export function buildConsoleEvents( + logs: ConsoleLog[], + pageId: string, + wallTime: number +): (ConsoleEvent | StdioEvent)[] { + const capped = logs.slice(0, MAX_CONSOLE_EVENTS) + const events: (ConsoleEvent | StdioEvent)[] = capped.map((log) => { + const time = Math.max(0, log.timestamp - wallTime) + const text = log.args.map(previewArg).join(' ') + // Untagged entries predate source tagging; they came from the page. + if (log.source === 'browser' || log.source === undefined) { + return { + type: 'console', + time, + pageId, + messageType: toTraceLevel(log.type), + text, + args: log.args.map((arg) => ({ preview: previewArg(arg), value: arg })), + // Location isn't captured at the console patch site; the event + // shape requires the field, so it ships zeroed. + location: { url: '', lineNumber: 0, columnNumber: 0 } + } satisfies ConsoleEvent + } + return { + type: log.type === 'error' || log.type === 'warn' ? 'stderr' : 'stdout', + timestamp: time, + text, + source: log.source + } satisfies StdioEvent + }) + if (logs.length > MAX_CONSOLE_EVENTS) { + const last = capped[capped.length - 1] + events.push({ + type: 'stderr', + timestamp: last ? Math.max(0, last.timestamp - wallTime) : 0, + text: `[devtools] console truncated: dropped ${logs.length - MAX_CONSOLE_EVENTS} entries` + }) + } + return events +} diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index 24488442..8fd7d575 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -20,6 +20,16 @@ import { FILL_METHODS, type TraceAction } from './action-mapping.js' +import { + buildConsoleEvents, + type ConsoleEvent, + type StdioEvent +} from './trace-console.js' +import { + buildFilmstripEvents, + buildSnapshotResources, + type ScreencastFrameEvent +} from './trace-snapshots.js' import { networkRequestToHar } from './trace-har.js' import { buildTraceZip, type TraceZipResource } from './trace-zip-writer.js' @@ -52,7 +62,7 @@ interface BeforeEvent { pageId: string params: Record title: string - /** Playwright-compatible API name (e.g. 'page.goBack', 'element.click'). */ + /** Trace-viewer API name (e.g. 'page.goBack', 'element.click'). */ apiName: string /** CallId of the Tracing.tracingGroup that wraps this action (if any). */ parentId?: string @@ -67,22 +77,13 @@ interface AfterEvent { parentId?: string } -interface ScreencastFrameEvent { - type: 'screencast-frame' - pageId: string - sha1: string - elements?: string - snapshot?: string - width: number - height: number - timestamp: number -} - type TraceEvent = | ContextOptionsEvent | BeforeEvent | AfterEvent | ScreencastFrameEvent + | ConsoleEvent + | StdioEvent function shortId(sessionId?: string): string { return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) @@ -294,63 +295,6 @@ function buildNetworkNdjson( return Buffer.from(lines.join('\n'), 'utf8') } -function buildSnapshotResources( - snapshots: ActionSnapshot[], - pageId: string -): TraceZipResource[] { - const out: TraceZipResource[] = [] - for (const snap of snapshots) { - const base = `${pageId}-${snap.timestamp}` - if (snap.screenshot) { - out.push({ - resourceName: `${base}.jpeg`, - data: Buffer.from(snap.screenshot, 'base64') - }) - } - if (snap.elements && snap.elements.length) { - out.push({ - resourceName: `${base}-elements.json`, - data: Buffer.from(JSON.stringify(snap.elements), 'utf8') - }) - } - if (snap.snapshotText) { - out.push({ - resourceName: `${base}-snapshot.txt`, - data: Buffer.from(snap.snapshotText, 'utf8') - }) - } - } - return out -} - -function buildScreencastFrames( - snapshots: ActionSnapshot[], - pageId: string, - wallTime: number, - viewport: { width: number; height: number } -): ScreencastFrameEvent[] { - return snapshots - .filter((s) => s.screenshot) - .map((s) => { - const base = `${pageId}-${s.timestamp}` - const frame: ScreencastFrameEvent = { - type: 'screencast-frame', - pageId, - sha1: `${base}.jpeg`, - width: viewport.width, - height: viewport.height, - timestamp: Math.max(0, s.timestamp - wallTime) - } - if (s.elements && s.elements.length) { - frame.elements = `${base}-elements.json` - } - if (s.snapshotText) { - frame.snapshot = `${base}-snapshot.txt` - } - return frame - }) -} - /** * Build a trace.zip buffer from the captured TraceLog. * Filters commands through ACTION_MAP and renames to trace vocabulary; @@ -368,13 +312,19 @@ function eventTime(e: TraceEvent): number { return e.endTime case 'screencast-frame': return e.timestamp + case 'console': + return e.time + case 'stdout': + case 'stderr': + return e.timestamp } } /** At the same timestamp T: an action's `after` ends first, then the - * snapshot captured at the action boundary, then the next action's `before`. - * Matches the viewer's expectation that the screencast frame shows the - * state between the previous action's completion and the next one's start. */ + * snapshot captured at the action boundary, then console output observed + * at the boundary, then the next action's `before`. Matches the viewer's + * expectation that the screencast frame shows the state between the + * previous action's completion and the next one's start. */ function eventOrder(e: TraceEvent): number { switch (e.type) { case 'context-options': @@ -383,8 +333,12 @@ function eventOrder(e: TraceEvent): number { return 1 case 'screencast-frame': return 2 - case 'before': + case 'console': + case 'stdout': + case 'stderr': return 3 + case 'before': + return 4 } } @@ -464,41 +418,12 @@ function buildTraceBundle( const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } const snapshots = trace.actionSnapshots ?? [] const ctxOptions = buildContextOptions(trace, contextId, wallTime) - const events: TraceEvent[] = [ctxOptions] - - // Emit initial screencast-frame (timestamp=0) using the first snapshot's - // resources so trace viewers show the page state before any interaction. - const firstSnap = snapshots.find((s) => s.screenshot) - if (firstSnap) { - const base = `${pageId}-${firstSnap.timestamp}` - const initFrame: ScreencastFrameEvent = { - type: 'screencast-frame', - pageId, - sha1: `${base}.jpeg`, - width: viewport.width, - height: viewport.height, - timestamp: 0 - } - if (firstSnap.elements && firstSnap.elements.length) { - initFrame.elements = `${base}-elements.json` - } - if (firstSnap.snapshotText) { - initFrame.snapshot = `${base}-snapshot.txt` - } - events.push(initFrame) - } - - events.push( - // Skip the first snapshot in buildScreencastFrames — it was already emitted - // as the initial t=0 frame above. - ...buildScreencastFrames( - firstSnap ? snapshots.filter((s) => s !== firstSnap) : snapshots, - pageId, - wallTime, - viewport - ), - ...buildActionEvents(trace.commands, pageId, wallTime, opts.testMetadata) - ) + const events: TraceEvent[] = [ + ctxOptions, + ...buildFilmstripEvents(snapshots, pageId, wallTime, viewport), + ...buildActionEvents(trace.commands, pageId, wallTime, opts.testMetadata), + ...buildConsoleEvents(trace.consoleLogs, pageId, wallTime) + ] events.sort(compareEvents) const ctxBName = ctxOptions.title return { diff --git a/packages/core/src/trace-snapshots.ts b/packages/core/src/trace-snapshots.ts new file mode 100644 index 00000000..0f139bdb --- /dev/null +++ b/packages/core/src/trace-snapshots.ts @@ -0,0 +1,102 @@ +// Per-action snapshot resources and the screencast-frame filmstrip derived +// from them. Split from trace-exporter.ts; the exporter composes these into +// the trace event stream. + +import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { TraceZipResource } from './trace-zip-writer.js' + +export interface ScreencastFrameEvent { + type: 'screencast-frame' + pageId: string + sha1: string + elements?: string + snapshot?: string + width: number + height: number + timestamp: number +} + +export function buildSnapshotResources( + snapshots: ActionSnapshot[], + pageId: string +): TraceZipResource[] { + const out: TraceZipResource[] = [] + for (const snap of snapshots) { + const base = `${pageId}-${snap.timestamp}` + if (snap.screenshot) { + out.push({ + resourceName: `${base}.jpeg`, + data: Buffer.from(snap.screenshot, 'base64') + }) + } + if (snap.elements && snap.elements.length) { + out.push({ + resourceName: `${base}-elements.json`, + data: Buffer.from(JSON.stringify(snap.elements), 'utf8') + }) + } + if (snap.snapshotText) { + out.push({ + resourceName: `${base}-snapshot.txt`, + data: Buffer.from(snap.snapshotText, 'utf8') + }) + } + } + return out +} + +function frameForSnapshot( + snap: ActionSnapshot, + pageId: string, + timestamp: number, + viewport: { width: number; height: number } +): ScreencastFrameEvent { + const base = `${pageId}-${snap.timestamp}` + const frame: ScreencastFrameEvent = { + type: 'screencast-frame', + pageId, + sha1: `${base}.jpeg`, + width: viewport.width, + height: viewport.height, + timestamp + } + if (snap.elements && snap.elements.length) { + frame.elements = `${base}-elements.json` + } + if (snap.snapshotText) { + frame.snapshot = `${base}-snapshot.txt` + } + return frame +} + +/** + * Full filmstrip for the trace: the first snapshot is re-anchored to t=0 so + * viewers show the page state before any interaction; the rest keep their + * wall-time offsets. + */ +export function buildFilmstripEvents( + snapshots: ActionSnapshot[], + pageId: string, + wallTime: number, + viewport: { width: number; height: number } +): ScreencastFrameEvent[] { + const firstSnap = snapshots.find((s) => s.screenshot) + const events: ScreencastFrameEvent[] = [] + if (firstSnap) { + events.push(frameForSnapshot(firstSnap, pageId, 0, viewport)) + } + for (const snap of snapshots) { + if (snap === firstSnap || !snap.screenshot) { + continue + } + events.push( + frameForSnapshot( + snap, + pageId, + Math.max(0, snap.timestamp - wallTime), + viewport + ) + ) + } + return events +} diff --git a/packages/core/tests/trace-console.test.ts b/packages/core/tests/trace-console.test.ts new file mode 100644 index 00000000..5bb6ee59 --- /dev/null +++ b/packages/core/tests/trace-console.test.ts @@ -0,0 +1,111 @@ +import { describe, it, expect } from 'vitest' +import { buildConsoleEvents } from '@wdio/devtools-core' +import type { ConsoleLog } from '@wdio/devtools-shared' + +const WALL_TIME = 1000 +const PAGE_ID = 'page@abc123' + +function log(overrides: Partial = {}): ConsoleLog { + return { + type: 'log', + args: ['hello'], + timestamp: WALL_TIME + 50, + source: 'browser', + ...overrides + } +} + +describe('buildConsoleEvents', () => { + it('returns empty array for no logs', () => { + expect(buildConsoleEvents([], PAGE_ID, WALL_TIME)).toEqual([]) + }) + + it('maps browser logs to console events with monotonic offsets', () => { + const events = buildConsoleEvents([log()], PAGE_ID, WALL_TIME) + expect(events).toEqual([ + { + type: 'console', + time: 50, + pageId: PAGE_ID, + messageType: 'log', + text: 'hello', + args: [{ preview: 'hello', value: 'hello' }], + location: { url: '', lineNumber: 0, columnNumber: 0 } + } + ]) + }) + + it('treats untagged logs as browser console', () => { + const events = buildConsoleEvents( + [log({ source: undefined })], + PAGE_ID, + WALL_TIME + ) + expect(events[0]!.type).toBe('console') + }) + + it("maps 'warn' to 'warning' and 'trace' to 'debug'", () => { + const events = buildConsoleEvents( + [log({ type: 'warn' }), log({ type: 'trace' })], + PAGE_ID, + WALL_TIME + ) + expect( + events.map((e) => (e.type === 'console' ? e.messageType : '')) + ).toEqual(['warning', 'debug']) + }) + + it('previews non-string args as JSON and joins into text', () => { + const events = buildConsoleEvents( + [log({ args: ['count', { a: 1 }, 2] })], + PAGE_ID, + WALL_TIME + ) + const event = events[0]! + expect(event.type).toBe('console') + if (event.type === 'console') { + expect(event.text).toBe('count {"a":1} 2') + expect(event.args?.[1]).toEqual({ preview: '{"a":1}', value: { a: 1 } }) + } + }) + + it('routes test/terminal logs to stdout/stderr by level with source kept', () => { + const events = buildConsoleEvents( + [ + log({ source: 'test', type: 'log', args: ['out'] }), + log({ source: 'test', type: 'error', args: ['bad'] }), + log({ source: 'terminal', type: 'warn', args: ['careful'] }) + ], + PAGE_ID, + WALL_TIME + ) + expect(events).toEqual([ + { type: 'stdout', timestamp: 50, text: 'out', source: 'test' }, + { type: 'stderr', timestamp: 50, text: 'bad', source: 'test' }, + { type: 'stderr', timestamp: 50, text: 'careful', source: 'terminal' } + ]) + }) + + it('floors offsets at zero for logs before wallTime', () => { + const events = buildConsoleEvents( + [log({ timestamp: WALL_TIME - 500 })], + PAGE_ID, + WALL_TIME + ) + const event = events[0]! + expect(event.type === 'console' ? event.time : -1).toBe(0) + }) + + it('caps output and appends a truncation marker', () => { + const logs = Array.from({ length: 10_001 }, (_, i) => + log({ timestamp: WALL_TIME + i }) + ) + const events = buildConsoleEvents(logs, PAGE_ID, WALL_TIME) + expect(events).toHaveLength(10_001) + const last = events[events.length - 1]! + expect(last.type).toBe('stderr') + if (last.type === 'stderr') { + expect(last.text).toContain('dropped 1 entries') + } + }) +}) From fcded20e0d4393375e8549c8d200b0bacce2d8e5 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 2 Jul 2026 19:56:46 +0530 Subject: [PATCH 02/91] =?UTF-8?q?feat(app):=20trace-player=20layout=20pari?= =?UTF-8?q?ty=20=E2=80=94=20full=20workbench=20with=20top-docked=20timelin?= =?UTF-8?q?e=20strip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/app/src/app.ts | 7 +- .../browser/trace-player-controls.ts | 145 ++++++++ .../browser/trace-timeline-constants.ts | 26 +- .../browser/trace-timeline-styles.ts | 45 +-- .../src/components/browser/trace-timeline.ts | 313 +++++------------- packages/app/src/components/workbench.ts | 135 ++++++-- packages/app/src/controller/constants.ts | 9 + 7 files changed, 357 insertions(+), 323 deletions(-) create mode 100644 packages/app/src/components/browser/trace-player-controls.ts diff --git a/packages/app/src/app.ts b/packages/app/src/app.ts index d9a89e5d..d8076a4b 100644 --- a/packages/app/src/app.ts +++ b/packages/app/src/app.ts @@ -232,9 +232,10 @@ export class WebdriverIODevtoolsApplication extends Element { class="flex h-[calc(100%-40px)] w-full relative" > ${ - // Only render the test-suite sidebar (and its resize slider) when the - // trace came from a testrunner — the player (standalone) has no tree, - // so the slider would otherwise show a stray dragger on hover. + // Only render the test-suite sidebar (and its resize slider) for a + // live testrunner session. The player has no run/rerun affordances, + // so the tree is dead weight even for testrunner-captured zips. + !this.dataManager.playerMode && this.dataManager.traceType === TraceType.Testrunner ? html` { + this.playerState = (event as CustomEvent).detail + } + + #emit(name: string, detail?: unknown): void { + window.dispatchEvent(new CustomEvent(name, { detail })) + } + + #button( + title: string, + icon: TemplateResult, + onClick: () => void, + extra = '' + ): TemplateResult { + return html`` + } + + #renderSpeedSelect(speed: number): TemplateResult { + return html` + + ` + } + + render() { + const { currentMs, duration, playing, speed } = this.playerState + return html` +
+ ${formatTimecode(currentMs)} + / + ${formatTimecode(duration)} + + ${this.#button( + 'Restart', + html``, + () => this.#emit(PLAYER_RESTART_EVENT) + )} + ${this.#button( + 'Previous action', + html``, + () => this.#emit(KBD.step, { dir: -1 }) + )} + ${this.#button( + playing ? 'Pause' : 'Play', + playing + ? html`` + : html``, + () => this.#emit(KBD.togglePlay), + 'text-chartsBlue' + )} + ${this.#button( + 'Next action', + html``, + () => this.#emit(KBD.step, { dir: 1 }) + )} + ${this.#renderSpeedSelect(speed)} +
+ ` + } +} + +declare global { + interface HTMLElementTagNameMap { + [COMPONENT]: TracePlayerControls + } +} diff --git a/packages/app/src/components/browser/trace-timeline-constants.ts b/packages/app/src/components/browser/trace-timeline-constants.ts index d88f1c61..1e6a1dad 100644 --- a/packages/app/src/components/browser/trace-timeline-constants.ts +++ b/packages/app/src/components/browser/trace-timeline-constants.ts @@ -1,19 +1,15 @@ -import type { ActionCategory } from '../workbench/actionItems/category.js' - /** Playback speed multipliers offered in the timeline controls. */ export const SPEEDS = [0.5, 1, 2, 3, 5] -/** Width of the track-label gutter (px) — lanes start after it. */ -export const GUTTER = 80 - -/** Right breathing room (px) so end-of-timeline markers don't hug the edge. */ -export const INSET = 14 - -/** Tailwind background class per action category, for the timeline chips. */ -export const CATEGORY_BG: Record = { - navigation: 'bg-chartsBlue', - input: 'bg-chartsPurple', - assertion: 'bg-chartsGreen', - query: 'bg-chartsYellow', - other: 'bg-gray-500' +/** Window events linking the controls bar and the timeline strip — same + * decoupling pattern as the KBD events, so either side can be re-homed. */ +export const PLAYER_STATE_EVENT = 'trace-player:state' +export const PLAYER_RESTART_EVENT = 'trace-player:restart' +export const PLAYER_SPEED_EVENT = 'trace-player:speed' + +export interface PlayerState { + currentMs: number + duration: number + playing: boolean + speed: number } diff --git a/packages/app/src/components/browser/trace-timeline-styles.ts b/packages/app/src/components/browser/trace-timeline-styles.ts index f2e276ad..d65cbe24 100644 --- a/packages/app/src/components/browser/trace-timeline-styles.ts +++ b/packages/app/src/components/browser/trace-timeline-styles.ts @@ -1,7 +1,6 @@ import { css } from 'lit' -/** Styles for the trace-player timeline: host layout, hidden scrollbars, and - * the network-detail drawer. Detail-block styles come from networkStyles. */ +/** Styles for the trace-player timeline strip: host layout + hidden scrollbars. */ export const timelineStyles = css` :host { position: relative; @@ -19,46 +18,4 @@ export const timelineStyles = css` .no-scrollbar::-webkit-scrollbar { display: none; } - .net-drawer { - position: absolute; - left: 0; - right: 0; - bottom: 0; - max-height: 62%; - display: flex; - flex-direction: column; - background: var(--vscode-sideBar-background); - border-top: 1px solid var(--accent, #ff7a3c); - box-shadow: 0 -16px 40px -24px #000; - z-index: 30; - } - .net-drawer-head { - display: flex; - align-items: center; - gap: 8px; - padding: 6px 10px; - border-bottom: 1px solid var(--vscode-panel-border); - font-size: 12px; - } - .net-drawer-head .url { - font-family: monospace; - font-size: 11.5px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - opacity: 0.85; - } - .net-drawer-head .close { - margin-left: auto; - cursor: pointer; - padding: 2px 6px; - border-radius: 4px; - } - .net-drawer-head .close:hover { - background: var(--vscode-toolbar-hoverBackground); - } - .net-drawer-body { - overflow: auto; - padding: 4px 0; - } ` diff --git a/packages/app/src/components/browser/trace-timeline.ts b/packages/app/src/components/browser/trace-timeline.ts index c2a62d52..2abc3753 100644 --- a/packages/app/src/components/browser/trace-timeline.ts +++ b/packages/app/src/components/browser/trace-timeline.ts @@ -1,41 +1,30 @@ import { Element } from '@core/element' -import { html, nothing, type TemplateResult } from 'lit' +import { html, type TemplateResult } from 'lit' import { customElement, state, query } from 'lit/decorators.js' import { consume } from '@lit/context' import type { CommandLog, TracePlayerFrame } from '@wdio/devtools-shared' -import { - commandContext, - framesContext, - networkRequestContext -} from '../../controller/context.js' -import { commandCategory } from '../workbench/actionItems/category.js' +import { commandContext, framesContext } from '../../controller/context.js' import { activeTimestampAt } from '../workbench/active-entry.js' -import { networkStyles } from '../workbench/network/styles.js' -import { renderNetworkRequestDetail } from '../workbench/network/request-detail.js' import { KBD } from '../../controller/keyboard.js' import { - CATEGORY_BG, - GUTTER, - INSET, - SPEEDS + PLAYER_RESTART_EVENT, + PLAYER_SPEED_EVENT, + PLAYER_STATE_EVENT, + SPEEDS, + type PlayerState } from './trace-timeline-constants.js' import { formatTimecode, imageMime } from './trace-timeline-utils.js' import { timelineStyles } from './trace-timeline-styles.js' -import '~icons/mdi/play.js' -import '~icons/mdi/pause.js' -import '~icons/mdi/skip-previous.js' -import '~icons/mdi/skip-next.js' -import '~icons/mdi/restart.js' - const COMPONENT = 'wdio-devtools-trace-timeline' /** - * Trace-player timeline (replaces the workbench dock in `pnpm show-trace` - * mode). Owns the playback clock, the screenshot filmstrip, the per-track - * timeline (actions / network / console), and the playhead. Advancing the - * clock dispatches `show-command` so the reused browser pane swaps screenshots. + * Trace-player timeline strip, docked above the workbench in `pnpm show-trace` + * mode. Owns the playback clock, the screenshot filmstrip, and the playhead; + * the controls bar (trace-player-controls) and keyboard drive it via window + * events, and it broadcasts its state back the same way. Advancing the clock + * dispatches `show-command` so the reused browser pane and actions list follow. */ @customElement(COMPONENT) export class TraceTimeline extends Element { @@ -47,10 +36,6 @@ export class TraceTimeline extends Element { @state() frames: TracePlayerFrame[] = [] - @consume({ context: networkRequestContext, subscribe: true }) - @state() - networkRequests: NetworkRequest[] = [] - /** Playback position in ms relative to the recording start. */ @state() currentMs = 0 @state() playing = false @@ -61,14 +46,11 @@ export class TraceTimeline extends Element { #activeTimestamp?: number #started = false - @query('[data-lanes]') lanesEl?: HTMLElement + @query('[data-scrub]') scrubEl?: HTMLElement #dragging = false - /** Network request whose detail drawer is open, or undefined. */ - @state() selectedRequest?: NetworkRequest - - static styles = [...Element.styles, networkStyles, timelineStyles] + static styles = [...Element.styles, timelineStyles] connectedCallback(): void { super.connectedCallback() @@ -76,6 +58,8 @@ export class TraceTimeline extends Element { window.addEventListener(KBD.step, this.#onKbdStep) window.addEventListener(KBD.jump, this.#onKbdJump) window.addEventListener(KBD.speed, this.#onKbdSpeed) + window.addEventListener(PLAYER_RESTART_EVENT, this.#onRestartEvent) + window.addEventListener(PLAYER_SPEED_EVENT, this.#onSpeedEvent) } disconnectedCallback(): void { @@ -87,6 +71,13 @@ export class TraceTimeline extends Element { window.removeEventListener(KBD.step, this.#onKbdStep) window.removeEventListener(KBD.jump, this.#onKbdJump) window.removeEventListener(KBD.speed, this.#onKbdSpeed) + window.removeEventListener(PLAYER_RESTART_EVENT, this.#onRestartEvent) + window.removeEventListener(PLAYER_SPEED_EVENT, this.#onSpeedEvent) + } + + #onRestartEvent = (): void => this.#restart() + #onSpeedEvent = (event: Event): void => { + this.speed = (event as CustomEvent<{ value: number }>).detail.value } #onKbdTogglePlay = (): void => this.#togglePlay() @@ -152,6 +143,17 @@ export class TraceTimeline extends Element { if (!this.#started && this.commands.length) { this.#syncActiveCommand() } + // Mirror playback state to the controls bar on the tab-header line. + window.dispatchEvent( + new CustomEvent(PLAYER_STATE_EVENT, { + detail: { + currentMs: this.currentMs, + duration: this.#duration, + playing: this.playing, + speed: this.speed + } + }) + ) } #stopRaf(): void { @@ -243,23 +245,14 @@ export class TraceTimeline extends Element { } } - #onSpeedChange(event: Event): void { - this.speed = Number((event.target as HTMLSelectElement).value) - } - - // ─── scrubbing (free-flow playhead drag) ─────────────────────────────────── + // ─── scrubbing (drag anywhere on the strip) ─────────────────────────────── #fractionFromClientX(clientX: number): number { - const rect = this.lanesEl?.getBoundingClientRect() - if (!rect) { + const rect = this.scrubEl?.getBoundingClientRect() + if (!rect || rect.width <= 0) { return 0 } - const laneStart = rect.left + GUTTER - const laneWidth = rect.width - GUTTER - INSET - if (laneWidth <= 0) { - return 0 - } - return Math.min(1, Math.max(0, (clientX - laneStart) / laneWidth)) + return Math.min(1, Math.max(0, (clientX - rect.left) / rect.width)) } #onPointerDown = (event: PointerEvent): void => { @@ -290,72 +283,6 @@ export class TraceTimeline extends Element { // ─── render ─────────────────────────────────────────────────────────────── - #ctrlButton( - title: string, - icon: TemplateResult, - onClick: () => void, - extra = '' - ): TemplateResult { - return html`` - } - - #renderControls(): TemplateResult { - return html` -
- ${this.#ctrlButton( - 'Restart', - html``, - () => this.#restart() - )} - ${this.#ctrlButton( - 'Previous action', - html``, - () => this.#step(-1) - )} - ${this.#ctrlButton( - this.playing ? 'Pause' : 'Play', - this.playing - ? html`` - : html``, - () => this.#togglePlay(), - 'text-chartsBlue' - )} - ${this.#ctrlButton( - 'Next action', - html``, - () => this.#step(1) - )} - ${formatTimecode(this.currentMs)} - / - ${formatTimecode(this.#duration)} - -
- ` - } - /** Timestamp of the frame nearest the playhead — drives filmstrip highlight. */ get #activeFrameTimestamp(): number | undefined { const clock = this.#start + this.currentMs @@ -371,16 +298,10 @@ export class TraceTimeline extends Element { return best } - // CSS left for a marker inside a track body (which starts after the gutter), - // leaving INSET of right margin so end-of-timeline markers don't hug the edge. - #laneLeft(fraction: number): string { - return `calc(${fraction} * (100% - ${INSET}px))` - } - #renderFilmstrip(): TemplateResult { if (!this.frames.length) { return html`
No frames captured
` @@ -388,150 +309,64 @@ export class TraceTimeline extends Element { const activeFrame = this.#activeFrameTimestamp return html`
-
-
- ${this.frames.map( - (frame) => - html`` - )} -
-
- ` - } - - #renderTrack( - label: string, - body: TemplateResult | typeof nothing - ): TemplateResult { - return html` -
-
- ${label} -
-
${body}
+ ${this.frames.map( + (frame) => + html`` + )}
` } - #renderActionsTrack(): TemplateResult { - const body = html`${this.#sortedCommands.map((command) => { - const ts = command.timestamp ?? 0 - const fraction = this.#fraction(ts) - const active = ts === this.#activeTimestamp - const color = CATEGORY_BG[commandCategory(command.command)] - // Track chips stay compact with the short command name; the full - // Playwright label is the hover tooltip (and the left Actions list). - return html`` - })}` - return this.#renderTrack('Actions', body) - } - - #renderNetworkTrack(): TemplateResult { - if (!this.networkRequests.length) { - return this.#renderTrack('Network', nothing) - } - const body = html`${this.networkRequests.map((request) => { - const leftFr = this.#fraction(request.startTime) - const rawFr = Math.max(0.004, (request.time ?? 0) / this.#duration) - const widthFr = Math.min(rawFr, 1 - leftFr) - const selected = this.selectedRequest?.id === request.id - // stopPropagation so a click selects the request rather than scrubbing the - // playhead (the lanes container owns the pointerdown drag handler). - return html`
` - })}` - return this.#renderTrack('Network', body) - } - - #renderNetworkDrawer(): TemplateResult | typeof nothing { - const req = this.selectedRequest - if (!req) { - return nothing - } + // Slim full-width ruler under the controls: action ticks + drag-to-scrub. + #renderRuler(): TemplateResult { return html` -
-
- ${req.method} ${req.url} - ✕ -
-
${renderNetworkRequestDetail(req)}
+
+ ${this.#sortedCommands.map((command) => { + const ts = command.timestamp ?? 0 + return html`
` + })}
` } #renderPlayhead(): TemplateResult { const fraction = Math.min(1, Math.max(0, this.currentMs / this.#duration)) - // Anchored at the gutter and inset on the right so it tracks the same lane - // coordinates as the action/network markers. return html`
` } render() { return html` - ${this.#renderControls()} ${this.#renderFilmstrip()}
- ${this.#renderActionsTrack()} ${this.#renderNetworkTrack()} - ${this.#renderTrack('Console', nothing)} ${this.#renderPlayhead()} + ${this.#renderRuler()} ${this.#renderFilmstrip()} + ${this.#renderPlayhead()}
- ${this.#renderNetworkDrawer()} ` } } diff --git a/packages/app/src/components/workbench.ts b/packages/app/src/components/workbench.ts index 5b87cd9d..0d56d7d9 100644 --- a/packages/app/src/components/workbench.ts +++ b/packages/app/src/components/workbench.ts @@ -26,23 +26,36 @@ import './workbench/network.js' import './workbench/compare.js' import './browser/snapshot.js' import './browser/trace-timeline.js' +import './browser/trace-player-controls.js' import { + HEADER_HEIGHT, MIN_WORKBENCH_HEIGHT, MIN_METATAB_WIDTH, ACTIONS_DEFAULT_WIDTH, BROWSER_HEIGHT_RATIO, - RERENDER_TIMEOUT + PLAYER_CONTROLS_HEIGHT, + PLAYER_DOCK_DEFAULT_HEIGHT, + PLAYER_DOCK_MIN_HEIGHT, + RERENDER_TIMEOUT, + TRACE_TIMELINE_MIN_HEIGHT, + TRACE_TIMELINE_DEFAULT_HEIGHT } from '../controller/constants.js' const COMPONENT = 'wdio-devtools-workbench' + +/** Pixel value from a DragController position string (`flex-basis: 123px`). */ +function basisPx(position: string): number | undefined { + const value = parseFloat(position.split(':')[1] ?? '') + return Number.isFinite(value) ? value : undefined +} @customElement(COMPONENT) export class DevtoolsWorkbench extends Element { #toolbarCollapsed = localStorage.getItem('toolbar') === 'true' #workbenchSidebarCollapsed = localStorage.getItem('workbenchSidebar') === 'true' - // Trace-player mode (`pnpm show-trace`): hide the Metadata tab and swap the - // workbench tabs for the timeline player. + // Trace-player mode (`pnpm show-trace`): the full workbench renders as in + // live mode, plus the playback timeline docked above the browser pane. @property({ type: Boolean }) playerMode = false @@ -99,6 +112,33 @@ export class DevtoolsWorkbench extends Element { direction: Direction.horizontal }) + #dragTimeline = new DragController(this, { + localStorageKey: 'traceTimelineHeight', + minPosition: TRACE_TIMELINE_MIN_HEIGHT, + maxPosition: window.innerHeight * 0.4, + initialPosition: TRACE_TIMELINE_DEFAULT_HEIGHT, + getContainerEl: () => this.#getVerticalWindow(), + direction: Direction.vertical + }) + + // Player-mode browser-pane height. Separate controller + storage key so + // resizing the player never disturbs the live-mode split. + #dragVerticalPlayer = new DragController(this, { + localStorageKey: 'playerBrowserHeight', + minPosition: MIN_WORKBENCH_HEIGHT, + maxPosition: window.innerHeight * 0.8, + initialPosition: Math.max( + MIN_WORKBENCH_HEIGHT, + window.innerHeight - + HEADER_HEIGHT - + PLAYER_CONTROLS_HEIGHT - + TRACE_TIMELINE_DEFAULT_HEIGHT - + PLAYER_DOCK_DEFAULT_HEIGHT + ), + getContainerEl: () => this.#getVerticalWindow(), + direction: Direction.vertical + }) + async #getHorizontalWindow() { await this.updateComplete return this.horizontalResizerWindow as Element @@ -144,13 +184,43 @@ export class DevtoolsWorkbench extends Element { if (this.#toolbarCollapsed) { return '' } - const m = this.#dragVertical.getPosition().match(/(\d+(?:\.\d+)?)px/) - const raw = m ? parseFloat(m[1]) : window.innerHeight * BROWSER_HEIGHT_RATIO + if (this.playerMode) { + // Player proportions: the snapshot dominates — pane gets everything the + // timeline, controls bar, and a compact dock don't need, clamped in CSS + // so the dock never drops below its minimum inside the viewport. + const fallback = + window.innerHeight - + HEADER_HEIGHT - + PLAYER_CONTROLS_HEIGHT - + this.#timelinePaneHeight() - + PLAYER_DOCK_DEFAULT_HEIGHT + const raw = basisPx(this.#dragVerticalPlayer.getPosition()) ?? fallback + const paneHeight = Math.max(MIN_WORKBENCH_HEIGHT, raw) + const maxHeight = `calc(100vh - ${ + HEADER_HEIGHT + PLAYER_CONTROLS_HEIGHT + PLAYER_DOCK_MIN_HEIGHT + }px - ${this.#timelinePaneHeight()}px)` + return `flex:0 0 ${paneHeight}px; height:${paneHeight}px; max-height:${maxHeight}; min-height:0;` + } + const raw = + basisPx(this.#dragVertical.getPosition()) ?? + window.innerHeight * BROWSER_HEIGHT_RATIO const capped = Math.min(raw, window.innerHeight * 0.7) const paneHeight = Math.max(MIN_WORKBENCH_HEIGHT, capped) return `flex:0 0 ${paneHeight}px; height:${paneHeight}px; max-height:70vh; min-height:0;` } + #timelinePaneHeight(): number { + const raw = + basisPx(this.#dragTimeline.getPosition()) ?? TRACE_TIMELINE_DEFAULT_HEIGHT + const capped = Math.min(raw, window.innerHeight * 0.4) + return Math.max(TRACE_TIMELINE_MIN_HEIGHT, capped) + } + + #computeTimelinePaneStyle(): string { + const paneHeight = this.#timelinePaneHeight() + return `flex:0 0 ${paneHeight}px; height:${paneHeight}px; max-height:40vh; min-height:0;` + } + #computeSidebarStyle(): string { if (this.#workbenchSidebarCollapsed) { return 'width:0; flex:0 0 0; overflow:hidden;' @@ -173,11 +243,9 @@ export class DevtoolsWorkbench extends Element { - ${this.playerMode - ? nothing - : html` - - `} + + +
diff --git a/packages/app/src/components/browser/trace-timeline-constants.ts b/packages/app/src/components/browser/trace-timeline-constants.ts index 1e6a1dad..9f057994 100644 --- a/packages/app/src/components/browser/trace-timeline-constants.ts +++ b/packages/app/src/components/browser/trace-timeline-constants.ts @@ -1,8 +1,16 @@ /** Playback speed multipliers offered in the timeline controls. */ export const SPEEDS = [0.5, 1, 2, 3, 5] -/** Window events linking the controls bar and the timeline strip — same - * decoupling pattern as the KBD events, so either side can be re-homed. */ +/** Candidate ruler intervals (ms); tickStep picks the smallest fitting one. */ +export const TICK_STEPS = [ + 100, 250, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 30_000, 60_000, 120_000, + 300_000, 600_000 +] + +/** Ruler divisions to aim for — keeps labels readable at any duration. */ +export const TICK_TARGET_DIVISIONS = 14 + +/** Window events linking the controls bar and the timeline strip (KBD-style). */ export const PLAYER_STATE_EVENT = 'trace-player:state' export const PLAYER_RESTART_EVENT = 'trace-player:restart' export const PLAYER_SPEED_EVENT = 'trace-player:speed' diff --git a/packages/app/src/components/browser/trace-timeline-styles.ts b/packages/app/src/components/browser/trace-timeline-styles.ts index d65cbe24..f1482adc 100644 --- a/packages/app/src/components/browser/trace-timeline-styles.ts +++ b/packages/app/src/components/browser/trace-timeline-styles.ts @@ -1,6 +1,6 @@ import { css } from 'lit' -/** Styles for the trace-player timeline strip: host layout + hidden scrollbars. */ +/** Host layout for the trace-player timeline strip. */ export const timelineStyles = css` :host { position: relative; @@ -11,11 +11,4 @@ export const timelineStyles = css` background-color: var(--vscode-editor-background); color: var(--vscode-foreground); } - .no-scrollbar { - scrollbar-width: none; - -ms-overflow-style: none; - } - .no-scrollbar::-webkit-scrollbar { - display: none; - } ` diff --git a/packages/app/src/components/browser/trace-timeline-utils.ts b/packages/app/src/components/browser/trace-timeline-utils.ts index 1083ee65..ef99a363 100644 --- a/packages/app/src/components/browser/trace-timeline-utils.ts +++ b/packages/app/src/components/browser/trace-timeline-utils.ts @@ -1,9 +1,37 @@ +import { + TICK_STEPS, + TICK_TARGET_DIVISIONS +} from './trace-timeline-constants.js' + /** Detect image mime from a base64 string's magic bytes — trace screenshots * may be PNG (polling capture) or JPEG (CDP), and the zip names both `.jpeg`. */ export function imageMime(base64: string): string { return base64.startsWith('/9j/') ? 'image/jpeg' : 'image/png' } +export function tickStep( + durationMs: number, + targetTicks = TICK_TARGET_DIVISIONS +): number { + const raw = durationMs / targetTicks + return ( + TICK_STEPS.find((step) => step >= raw) ?? TICK_STEPS[TICK_STEPS.length - 1] + ) +} + +/** Ruler tick label: `500ms`, `3.5s`, `1:15`. */ +export function formatTickLabel(ms: number): string { + if (ms < 1_000) { + return `${ms}ms` + } + if (ms < 60_000) { + return `${(ms / 1_000).toFixed(1)}s` + } + const minutes = Math.floor(ms / 60_000) + const seconds = Math.round((ms % 60_000) / 1_000) + return `${minutes}:${String(seconds).padStart(2, '0')}` +} + /** `m:ss.cc` timecode (e.g. 32_270ms → `0:32.27`). */ export function formatTimecode(ms: number): string { const safe = Number.isFinite(ms) && ms > 0 ? ms : 0 diff --git a/packages/app/src/components/browser/trace-timeline.ts b/packages/app/src/components/browser/trace-timeline.ts index 2abc3753..21f7ffce 100644 --- a/packages/app/src/components/browser/trace-timeline.ts +++ b/packages/app/src/components/browser/trace-timeline.ts @@ -14,18 +14,17 @@ import { SPEEDS, type PlayerState } from './trace-timeline-constants.js' -import { formatTimecode, imageMime } from './trace-timeline-utils.js' +import { + formatTickLabel, + formatTimecode, + imageMime, + tickStep +} from './trace-timeline-utils.js' import { timelineStyles } from './trace-timeline-styles.js' const COMPONENT = 'wdio-devtools-trace-timeline' -/** - * Trace-player timeline strip, docked above the workbench in `pnpm show-trace` - * mode. Owns the playback clock, the screenshot filmstrip, and the playhead; - * the controls bar (trace-player-controls) and keyboard drive it via window - * events, and it broadcasts its state back the same way. Advancing the clock - * dispatches `show-command` so the reused browser pane and actions list follow. - */ +/** Player timeline strip: owns the playback clock, filmstrip, and playhead; wired to the controls bar and keyboard via window events, and drives the workbench via `show-command`. */ @customElement(COMPONENT) export class TraceTimeline extends Element { @consume({ context: commandContext, subscribe: true }) @@ -298,7 +297,45 @@ export class TraceTimeline extends Element { return best } - #renderFilmstrip(): TemplateResult { + get #ticks(): number[] { + const step = tickStep(this.#duration) + const out: number[] = [] + for (let t = step; t < this.#duration; t += step) { + out.push(t) + } + return out + } + + // Faint vertical gridlines at each ruler tick, spanning the whole strip. + #renderGridlines(): TemplateResult { + return html`${this.#ticks.map( + (tick) => + html`
` + )}` + } + + // Ruler labels stay inside the strip via the bounded translateX trick. + #renderRulerLabels(): TemplateResult { + return html` +
+ ${this.#ticks.map((tick) => { + const fraction = tick / this.#duration + return html`${formatTickLabel(tick)}` + })} +
+ ` + } + + // Thumbnails sit at their wall-clock position along the axis. + #renderThumbTrack(): TemplateResult { if (!this.frames.length) { return html`
- ${this.frames.map( - (frame) => - html`` - )} +
+ ${this.frames.map((frame) => { + const fraction = this.#fraction(frame.timestamp) + const active = frame.timestamp === activeFrame + return html`` + })}
` } - // Slim full-width ruler under the controls: action ticks + drag-to-scrub. - #renderRuler(): TemplateResult { + // Bottom scrub bar: full-width line, action tick marks, draggable knob. + #renderScrubBar(): TemplateResult { + const fraction = Math.min(1, Math.max(0, this.currentMs / this.#duration)) return html` -
+
+
${this.#sortedCommands.map((command) => { - const ts = command.timestamp ?? 0 + const tickFraction = this.#fraction(command.timestamp ?? 0) return html`
` })} +
` } - #renderPlayhead(): TemplateResult { - const fraction = Math.min(1, Math.max(0, this.currentMs / this.#duration)) - return html`
` - } - render() { return html`
- ${this.#renderRuler()} ${this.#renderFilmstrip()} - ${this.#renderPlayhead()} + ${this.#renderGridlines()} ${this.#renderRulerLabels()} + ${this.#renderThumbTrack()} ${this.#renderScrubBar()}
` } diff --git a/packages/app/src/components/workbench.ts b/packages/app/src/components/workbench.ts index 0d56d7d9..0bb9f243 100644 --- a/packages/app/src/components/workbench.ts +++ b/packages/app/src/components/workbench.ts @@ -6,10 +6,11 @@ import { consume } from '@lit/context' import { DragController, Direction } from '../utils/DragController.js' import { consoleLogContext, + metadataContext, networkRequestContext, baselineContext } from '../controller/context.js' -import type { PreservedAttempt } from '@wdio/devtools-shared' +import type { Metadata, PreservedAttempt } from '@wdio/devtools-shared' import '~icons/mdi/arrow-collapse-down.js' import '~icons/mdi/arrow-collapse-up.js' @@ -28,6 +29,7 @@ import './browser/snapshot.js' import './browser/trace-timeline.js' import './browser/trace-player-controls.js' import { + BROWSER_BACKDROP_GRADIENT, HEADER_HEIGHT, MIN_WORKBENCH_HEIGHT, MIN_METATAB_WIDTH, @@ -36,6 +38,7 @@ import { PLAYER_CONTROLS_HEIGHT, PLAYER_DOCK_DEFAULT_HEIGHT, PLAYER_DOCK_MIN_HEIGHT, + PLAYER_SNAPSHOT_WIDTH_RATIO, RERENDER_TIMEOUT, TRACE_TIMELINE_MIN_HEIGHT, TRACE_TIMELINE_DEFAULT_HEIGHT @@ -54,8 +57,7 @@ export class DevtoolsWorkbench extends Element { #workbenchSidebarCollapsed = localStorage.getItem('workbenchSidebar') === 'true' - // Trace-player mode (`pnpm show-trace`): the full workbench renders as in - // live mode, plus the playback timeline docked above the browser pane. + // Trace-player mode: full workbench plus the timeline strip and controls bar. @property({ type: Boolean }) playerMode = false @@ -71,6 +73,10 @@ export class DevtoolsWorkbench extends Element { @state() baselines: Map | undefined = undefined + @consume({ context: metadataContext, subscribe: true }) + @state() + metadata: Metadata | undefined = undefined + static styles = [ ...Element.styles, css` @@ -115,18 +121,18 @@ export class DevtoolsWorkbench extends Element { #dragTimeline = new DragController(this, { localStorageKey: 'traceTimelineHeight', minPosition: TRACE_TIMELINE_MIN_HEIGHT, - maxPosition: window.innerHeight * 0.4, + maxPosition: () => window.innerHeight * 0.4, initialPosition: TRACE_TIMELINE_DEFAULT_HEIGHT, getContainerEl: () => this.#getVerticalWindow(), direction: Direction.vertical }) - // Player-mode browser-pane height. Separate controller + storage key so - // resizing the player never disturbs the live-mode split. + // Player-mode pane height; own storage key so it never disturbs the live split. + // The live max bound keeps the handle (and pane) inside the current budget. #dragVerticalPlayer = new DragController(this, { - localStorageKey: 'playerBrowserHeight', + localStorageKey: 'playerPaneHeight', minPosition: MIN_WORKBENCH_HEIGHT, - maxPosition: window.innerHeight * 0.8, + maxPosition: () => this.#playerPaneBudget(), initialPosition: Math.max( MIN_WORKBENCH_HEIGHT, window.innerHeight - @@ -139,6 +145,27 @@ export class DevtoolsWorkbench extends Element { direction: Direction.vertical }) + // Player snapshot keeps the recorded viewport's shape, slightly narrowed. + #playerAspectRatio(): string { + const viewport = this.metadata?.viewport + const width = Math.round( + (viewport?.width || 1280) * PLAYER_SNAPSHOT_WIDTH_RATIO + ) + return `${width} / ${viewport?.height || 800}` + } + + // Space left for the snapshot pane once the fixed rows and dock minimum eat theirs. + #playerPaneBudget(): number { + return Math.max( + MIN_WORKBENCH_HEIGHT, + window.innerHeight - + HEADER_HEIGHT - + PLAYER_CONTROLS_HEIGHT - + PLAYER_DOCK_MIN_HEIGHT - + this.#timelinePaneHeight() + ) + } + async #getHorizontalWindow() { await this.updateComplete return this.horizontalResizerWindow as Element @@ -185,21 +212,12 @@ export class DevtoolsWorkbench extends Element { return '' } if (this.playerMode) { - // Player proportions: the snapshot dominates — pane gets everything the - // timeline, controls bar, and a compact dock don't need, clamped in CSS - // so the dock never drops below its minimum inside the viewport. - const fallback = - window.innerHeight - - HEADER_HEIGHT - - PLAYER_CONTROLS_HEIGHT - - this.#timelinePaneHeight() - - PLAYER_DOCK_DEFAULT_HEIGHT - const raw = basisPx(this.#dragVerticalPlayer.getPosition()) ?? fallback - const paneHeight = Math.max(MIN_WORKBENCH_HEIGHT, raw) + // Snapshot pane dominates; the CSS clamp keeps the dock minimum in view. + // Literal getPosition() basis lets adjustPosition sync slider ↔ clamped height. const maxHeight = `calc(100vh - ${ HEADER_HEIGHT + PLAYER_CONTROLS_HEIGHT + PLAYER_DOCK_MIN_HEIGHT }px - ${this.#timelinePaneHeight()}px)` - return `flex:0 0 ${paneHeight}px; height:${paneHeight}px; max-height:${maxHeight}; min-height:0;` + return `flex-grow:0; flex-shrink:0; ${this.#dragVerticalPlayer.getPosition()}; max-height:${maxHeight}; min-height:${MIN_WORKBENCH_HEIGHT}px;` } const raw = basisPx(this.#dragVertical.getPosition()) ?? @@ -344,6 +362,31 @@ export class DevtoolsWorkbench extends Element { ` } + #renderBrowserPane() { + // Player: the boxed host goes transparent and the pane carries the shared + // backdrop, so the aspect box blends instead of showing a gradient seam. + const playerPaneExtra = this.playerMode + ? ` background:${BROWSER_BACKDROP_GRADIENT};` + : '' + return html` +
+ ${this.playerMode + ? html`
+ +
` + : html``} +
+ ` + } + // Full-width playback strip above the workbench row — player mode only. #renderTimelineStrip() { if (!this.playerMode) { @@ -388,12 +431,7 @@ export class DevtoolsWorkbench extends Element {
-
- -
+ ${this.#renderBrowserPane()} ${!this.#toolbarCollapsed ? (this.playerMode ? this.#dragVerticalPlayer diff --git a/packages/app/src/controller/constants.ts b/packages/app/src/controller/constants.ts index 105e74ad..6992ebf4 100644 --- a/packages/app/src/controller/constants.ts +++ b/packages/app/src/controller/constants.ts @@ -14,6 +14,11 @@ export const PLAYER_DOCK_MIN_HEIGHT = 140 export const PLAYER_DOCK_DEFAULT_HEIGHT = 220 /** Controls bar on the tab-header line in player mode (matches h-10 headers). */ export const PLAYER_CONTROLS_HEIGHT = 40 +/** Width factor on the player snapshot's aspect box — trims width, keeps height. */ +export const PLAYER_SNAPSHOT_WIDTH_RATIO = 0.9 +/** Backdrop behind the browser chrome — shared with the snapshot component styles. */ +export const BROWSER_BACKDROP_GRADIENT = + 'radial-gradient(120% 120% at 50% 0%, var(--vscode-editorWidget-background), var(--vscode-editor-background))' /** Fixed app-header height (see header.ts / app.ts `h-[calc(100%-40px)]`). */ export const HEADER_HEIGHT = 40 export const LOG_ICONS: Record = { diff --git a/packages/app/src/controller/keyboard.ts b/packages/app/src/controller/keyboard.ts index 87559253..f94ce52c 100644 --- a/packages/app/src/controller/keyboard.ts +++ b/packages/app/src/controller/keyboard.ts @@ -32,7 +32,7 @@ function isTyping(event: KeyboardEvent): boolean { ) } -function emit(name: string, detail?: unknown): void { +export function emit(name: string, detail?: unknown): void { window.dispatchEvent(new CustomEvent(name, { detail })) } diff --git a/packages/app/src/utils/DragController.ts b/packages/app/src/utils/DragController.ts index 593d452d..d675570f 100644 --- a/packages/app/src/utils/DragController.ts +++ b/packages/app/src/utils/DragController.ts @@ -14,15 +14,22 @@ export enum Direction { type DragControllerHost = HTMLElement & ReactiveControllerHost type AsyncGetElFn = () => Element | Promise +/** Bounds accept getters so panes with a layout-dependent budget clamp live. */ +type Bound = number | (() => number) + interface DragControllerOptions { initialPosition: number direction: Direction localStorageKey?: string - minPosition?: number - maxPosition?: number + minPosition?: Bound + maxPosition?: Bound getContainerEl: AsyncGetElFn } +function resolveBound(bound: Bound | undefined): number | undefined { + return typeof bound === 'function' ? bound() : bound +} + type State = 'dragging' | 'idle' const defaultOptions = { @@ -107,16 +114,18 @@ export class DragController implements ReactiveController { } #setPosition(x: number, y: number) { + const min = resolveBound(this.#options.minPosition) ?? 0 + const max = resolveBound(this.#options.maxPosition) if (this.#options.direction === Direction.horizontal) { - let nx = Math.max(x, this.#options.minPosition || 0) - if (this.#options.maxPosition !== undefined) { - nx = Math.min(nx, this.#options.maxPosition) + let nx = Math.max(x, min) + if (max !== undefined) { + nx = Math.min(nx, max) } this.#x = nx } else { - let ny = Math.max(y, this.#options.minPosition || 0) - if (this.#options.maxPosition !== undefined) { - ny = Math.min(ny, this.#options.maxPosition) + let ny = Math.max(y, min) + if (max !== undefined) { + ny = Math.min(ny, max) } this.#y = ny } diff --git a/packages/app/tests/trace-timeline-utils.test.ts b/packages/app/tests/trace-timeline-utils.test.ts new file mode 100644 index 00000000..e3197c7b --- /dev/null +++ b/packages/app/tests/trace-timeline-utils.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from 'vitest' +import { + formatTickLabel, + tickStep +} from '../src/components/browser/trace-timeline-utils.js' + +describe('tickStep', () => { + it('picks the smallest step yielding at most the target tick count', () => { + expect(tickStep(10_000)).toBe(1_000) + expect(tickStep(78_460)).toBe(10_000) + expect(tickStep(1_200)).toBe(100) + }) + + it('caps at the largest step for very long traces', () => { + expect(tickStep(3 * 60 * 60 * 1000)).toBe(600_000) + }) +}) + +describe('formatTickLabel', () => { + it('formats sub-second ticks as milliseconds', () => { + expect(formatTickLabel(500)).toBe('500ms') + }) + + it('formats seconds with one decimal', () => { + expect(formatTickLabel(1_000)).toBe('1.0s') + expect(formatTickLabel(3_500)).toBe('3.5s') + }) + + it('formats minutes as m:ss', () => { + expect(formatTickLabel(75_000)).toBe('1:15') + expect(formatTickLabel(60_000)).toBe('1:00') + }) +}) From fef6978d76136375945a01dc0fc7f49f34101ae9 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 3 Jul 2026 17:35:17 +0530 Subject: [PATCH 04/91] refactor(shared,backend,core): derive LogLevel from runtime LOG_LEVELS; tighten comments --- packages/backend/src/trace-reader-constants.ts | 5 ++++- packages/backend/src/trace-reader-utils.ts | 15 +++------------ packages/core/src/trace-console.ts | 12 ++++-------- packages/core/src/trace-snapshots.ts | 6 +----- packages/shared/src/types.ts | 10 +++++++++- 5 files changed, 21 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/trace-reader-constants.ts b/packages/backend/src/trace-reader-constants.ts index 6e738464..0572b51f 100644 --- a/packages/backend/src/trace-reader-constants.ts +++ b/packages/backend/src/trace-reader-constants.ts @@ -1,4 +1,7 @@ -import { ACTION_MAP } from '@wdio/devtools-shared' +import { ACTION_MAP, LOG_LEVELS } from '@wdio/devtools-shared' + +/** Runtime lookup for narrowing foreign trace levels to the shared union. */ +export const LOG_LEVEL_SET: ReadonlySet = new Set(LOG_LEVELS) // Inverse of ACTION_MAP, derived so it can never drift from the forward map. // The forward map is many-to-one (url/navigateTo/get all → Page.navigate); the diff --git a/packages/backend/src/trace-reader-utils.ts b/packages/backend/src/trace-reader-utils.ts index b7162c12..e1f8e3dd 100644 --- a/packages/backend/src/trace-reader-utils.ts +++ b/packages/backend/src/trace-reader-utils.ts @@ -11,6 +11,7 @@ import { type Viewport } from '@wdio/devtools-shared' +import { LOG_LEVEL_SET } from './trace-reader-constants.js' import type { ConsoleEvent, ContextOptionsEvent, @@ -144,22 +145,12 @@ export function harToNetworkRequest( } } -const LOG_LEVELS: ReadonlySet = new Set([ - 'trace', - 'debug', - 'log', - 'info', - 'warn', - 'error' -]) - -// Reverse of the writer's level mapping ('warn' → 'warning'); a foreign -// trace zip can carry levels outside our union, which default to 'log'. +// Reverse level mapping; foreign levels outside our union default to 'log'. function fromTraceLevel(messageType: string): LogLevel { if (messageType === 'warning') { return 'warn' } - return LOG_LEVELS.has(messageType) ? (messageType as LogLevel) : 'log' + return LOG_LEVEL_SET.has(messageType) ? (messageType as LogLevel) : 'log' } export function buildConsoleLogs( diff --git a/packages/core/src/trace-console.ts b/packages/core/src/trace-console.ts index f9db30fd..78deb7aa 100644 --- a/packages/core/src/trace-console.ts +++ b/packages/core/src/trace-console.ts @@ -19,17 +19,14 @@ export interface StdioEvent { type: 'stdout' | 'stderr' timestamp: number text?: string - /** Extension field: preserves the test-vs-terminal origin the standard - * stdio vocabulary can't express. Foreign viewers ignore it. */ + /** Extension field: test-vs-terminal origin; foreign viewers ignore it. */ source?: Extract } -// Keeps a pathological run (console.log in a tight loop) from producing a -// trace.trace the viewer can't open. +// Caps pathological runs (console.log in a loop) so the trace stays openable. const MAX_CONSOLE_EVENTS = 10_000 -/** The trace format's console vocabulary uses 'warning'; 'trace' has no - * equivalent — 'debug' is the nearest severity. */ +/** Trace vocabulary uses 'warning'; 'trace' maps to the nearest severity, 'debug'. */ function toTraceLevel(level: ConsoleLog['type']): string { if (level === 'warn') { return 'warning' @@ -69,8 +66,7 @@ export function buildConsoleEvents( messageType: toTraceLevel(log.type), text, args: log.args.map((arg) => ({ preview: previewArg(arg), value: arg })), - // Location isn't captured at the console patch site; the event - // shape requires the field, so it ships zeroed. + // Location isn't captured at the patch site; the required field ships zeroed. location: { url: '', lineNumber: 0, columnNumber: 0 } } satisfies ConsoleEvent } diff --git a/packages/core/src/trace-snapshots.ts b/packages/core/src/trace-snapshots.ts index 0f139bdb..9e98704d 100644 --- a/packages/core/src/trace-snapshots.ts +++ b/packages/core/src/trace-snapshots.ts @@ -69,11 +69,7 @@ function frameForSnapshot( return frame } -/** - * Full filmstrip for the trace: the first snapshot is re-anchored to t=0 so - * viewers show the page state before any interaction; the rest keep their - * wall-time offsets. - */ +/** Filmstrip events; the first snapshot is re-anchored to t=0 for pre-interaction state. */ export function buildFilmstripEvents( snapshots: ActionSnapshot[], pageId: string, diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 23ad3c7f..2f209aed 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -4,7 +4,15 @@ // these shapes. The backend stores and forwards them. The app consumes them. // See ARCHITECTURE.md §2 and CLAUDE.md §2.1. -export type LogLevel = 'trace' | 'debug' | 'log' | 'info' | 'warn' | 'error' +export const LOG_LEVELS = [ + 'trace', + 'debug', + 'log', + 'info', + 'warn', + 'error' +] as const +export type LogLevel = (typeof LOG_LEVELS)[number] /** Where a captured ConsoleLog entry originated. */ export type LogSource = 'browser' | 'test' | 'terminal' From dfa885650a3c8d72f488a19e7877caf7a1efc9fe Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 3 Jul 2026 17:35:44 +0530 Subject: [PATCH 05/91] fix(backend): skip tracing-group markers when reconstructing player commands --- packages/backend/src/trace-reader.ts | 5 +++++ packages/backend/tests/trace-reader.test.ts | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index 6146998f..3c6b84ea 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -103,6 +103,11 @@ function buildCommands( const commands: CommandLog[] = [] let maxOffset = 0 for (const [callId, before] of events.befores) { + // Group markers are structure, not actions — as command rows their end + // timestamp ties with the last action and steals the active highlight. + if (before.class === 'Tracing') { + continue + } const after = events.afters.get(callId) const endOffset = after?.endTime ?? before.startTime maxOffset = Math.max(maxOffset, endOffset) diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index 8df8f9ff..7d1f059e 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -36,6 +36,14 @@ function fixtureZip(): Uint8Array { params: { selector: '#name', value: 'vishnu' } }, { type: 'after', callId: 'call@2', endTime: 160 }, + { + type: 'before', + callId: 'call@0', + startTime: 0, + class: 'Tracing', + method: 'tracingGroup', + params: { name: 'my test' } + }, { type: 'before', callId: 'call@3', @@ -50,6 +58,7 @@ function fixtureZip(): Uint8Array { endTime: 260, error: { message: 'boom' } }, + { type: 'after', callId: 'call@0', endTime: 260 }, { type: 'screencast-frame', pageId: 'page@abcd1234', @@ -175,6 +184,12 @@ describe('parseTraceZip', () => { expect(trace.sources).toEqual({}) }) + it('skips tracing group markers so the last command stays the last action', () => { + const { trace } = parseTraceZip(fixtureZip()) + expect(trace.commands.some((c) => c.command === 'tracingGroup')).toBe(false) + expect(trace.commands[trace.commands.length - 1].command).toBe('click') + }) + it('reconstructs console logs from console and stdio events', () => { const { trace } = parseTraceZip(fixtureZip()) expect(trace.consoleLogs).toEqual([ From d6d07b4b464788041ba4e12ba356c2a2d56e395d Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 3 Jul 2026 19:23:03 +0530 Subject: [PATCH 06/91] feat(shared,backend,app): nested action tree with failure rollup in the trace player --- examples/wdio/wdio.conf.ts | 2 +- .../src/components/workbench/action-tree.ts | 72 +++ .../workbench/actionItems/command.ts | 9 +- .../workbench/actionItems/duration.ts | 16 +- .../components/workbench/actionItems/group.ts | 69 +++ .../components/workbench/actionItems/item.ts | 17 + .../app/src/components/workbench/actions.ts | 106 ++++- .../src/components/workbench/call-source.ts | 45 ++ .../app/src/components/workbench/source.ts | 66 ++- .../src/components/workbench/source/styles.ts | 13 + packages/app/src/controller/DataManager.ts | 92 ++-- packages/app/src/controller/context.ts | 7 + packages/app/tests/action-tree.test.ts | 95 ++++ packages/app/tests/call-source.test.ts | 93 +++- packages/app/tests/duration.test.ts | 9 + .../backend/src/trace-reader-constants.ts | 12 + packages/backend/src/trace-reader-groups.ts | 165 +++++++ packages/backend/src/trace-reader-types.ts | 21 +- packages/backend/src/trace-reader-utils.ts | 99 +++- packages/backend/src/trace-reader.ts | 258 ++++++++--- packages/backend/tests/trace-reader.test.ts | 435 +++++++++++++++++- packages/core/src/index.ts | 4 + packages/core/src/sha1.ts | 6 + packages/core/src/trace-action-events.ts | 195 ++++++++ packages/core/src/trace-exporter.ts | 186 ++------ packages/core/src/trace-frame-snapshots.ts | 154 +++++++ packages/core/src/trace-sources.ts | 65 +++ packages/core/tests/trace-exporter.test.ts | 181 +++++++- .../core/tests/trace-frame-snapshots.test.ts | 180 ++++++++ packages/core/tests/trace-sources.test.ts | 71 +++ packages/shared/src/trace-player.ts | 22 + 31 files changed, 2458 insertions(+), 307 deletions(-) create mode 100644 packages/app/src/components/workbench/action-tree.ts create mode 100644 packages/app/src/components/workbench/actionItems/group.ts create mode 100644 packages/app/tests/action-tree.test.ts create mode 100644 packages/backend/src/trace-reader-groups.ts create mode 100644 packages/core/src/sha1.ts create mode 100644 packages/core/src/trace-action-events.ts create mode 100644 packages/core/src/trace-frame-snapshots.ts create mode 100644 packages/core/src/trace-sources.ts create mode 100644 packages/core/tests/trace-frame-snapshots.test.ts create mode 100644 packages/core/tests/trace-sources.test.ts diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/wdio.conf.ts index 0f7b3470..b6305819 100644 --- a/examples/wdio/wdio.conf.ts +++ b/examples/wdio/wdio.conf.ts @@ -131,7 +131,7 @@ export const config: Options.Testrunner = { [ 'devtools', { - mode: 'live' as const, + mode: 'trace' as const, screencast: { enabled: true, pollIntervalMs: 200 } } ] diff --git a/packages/app/src/components/workbench/action-tree.ts b/packages/app/src/components/workbench/action-tree.ts new file mode 100644 index 00000000..723f83af --- /dev/null +++ b/packages/app/src/components/workbench/action-tree.ts @@ -0,0 +1,72 @@ +// Pure helpers behind the player's collapsible action tree: flattening the +// group tree into render rows and deciding which groups start expanded. + +import type { + TraceActionChild, + TraceActionGroupNode +} from '@wdio/devtools-shared' + +export interface GroupRow { + kind: 'group' + group: TraceActionGroupNode + depth: number + expanded: boolean +} + +export interface CommandRow { + kind: 'command' + commandIndex: number + depth: number +} + +export type ActionTreeRow = GroupRow | CommandRow + +/** Command indices anywhere under a group, nested groups included. */ +export function collectCommandIndices(group: TraceActionGroupNode): number[] { + const indices: number[] = [] + for (const child of group.children) { + if ('group' in child) { + indices.push(...collectCommandIndices(child.group)) + } else { + indices.push(child.commandIndex) + } + } + return indices +} + +/** Groups open by default when failed or when holding the active command. */ +export function defaultExpanded( + group: TraceActionGroupNode, + activeCommandIndex?: number +): boolean { + if (group.failed) { + return true + } + return ( + activeCommandIndex !== undefined && + collectCommandIndices(group).includes(activeCommandIndex) + ) +} + +/** Flatten the tree into render rows, descending only into expanded groups. */ +export function flattenActionTree( + children: TraceActionChild[], + isExpanded: (group: TraceActionGroupNode) => boolean, + depth = 0 +): ActionTreeRow[] { + const rows: ActionTreeRow[] = [] + for (const child of children) { + if ('group' in child) { + const expanded = isExpanded(child.group) + rows.push({ kind: 'group', group: child.group, depth, expanded }) + if (expanded) { + rows.push( + ...flattenActionTree(child.group.children, isExpanded, depth + 1) + ) + } + } else { + rows.push({ kind: 'command', commandIndex: child.commandIndex, depth }) + } + } + return rows +} diff --git a/packages/app/src/components/workbench/actionItems/command.ts b/packages/app/src/components/workbench/actionItems/command.ts index a587369e..1a596f56 100644 --- a/packages/app/src/components/workbench/actionItems/command.ts +++ b/packages/app/src/components/workbench/actionItems/command.ts @@ -33,6 +33,10 @@ export class CommandItem extends ActionItem { @property({ type: Object, attribute: true }) entry?: CommandLog + willUpdate(): void { + this.failed = Boolean(this.entry?.error) + } + #highlightLine() { const event = new CustomEvent('show-command', { detail: { @@ -85,7 +89,10 @@ export class CommandItem extends ActionItem { @click="${() => this.#highlightLine()}" > ${this.iconChip(this.#renderIcon(entry.command))} - ${entry.title ?? entry.command} ${this.renderTime()} diff --git a/packages/app/src/components/workbench/actionItems/duration.ts b/packages/app/src/components/workbench/actionItems/duration.ts index db4b7fbc..5d2df61f 100644 --- a/packages/app/src/components/workbench/actionItems/duration.ts +++ b/packages/app/src/components/workbench/actionItems/duration.ts @@ -3,17 +3,19 @@ export type DurationHeat = 'fast' | 'mid' | 'slow' const ONE_SECOND = 1000 const ONE_MINUTE = ONE_SECOND * 60 -/** Human-readable duration: `ms` under a second, `s` under a minute, `m s` above. */ +/** Human-readable duration: `ms` under a second, `s` under a minute, `m s` + * above. Rounds first — reconstructed traces carry fractional-ms clocks. */ export function formatDuration(ms: number): string { - if (ms > ONE_MINUTE) { - const minutes = Math.floor(ms / ONE_MINUTE) - const seconds = Math.floor((ms - minutes * ONE_MINUTE) / ONE_SECOND) + const rounded = Math.round(ms) + if (rounded > ONE_MINUTE) { + const minutes = Math.floor(rounded / ONE_MINUTE) + const seconds = Math.floor((rounded - minutes * ONE_MINUTE) / ONE_SECOND) return `${minutes}m ${seconds}s` } - if (ms > ONE_SECOND) { - return `${(ms / ONE_SECOND).toFixed(2)}s` + if (rounded > ONE_SECOND) { + return `${(rounded / ONE_SECOND).toFixed(2)}s` } - return `${ms}ms` + return `${rounded}ms` } /** Bucket a step duration so slow steps stand out: fast < 500ms ≤ mid < 2s ≤ slow. */ diff --git a/packages/app/src/components/workbench/actionItems/group.ts b/packages/app/src/components/workbench/actionItems/group.ts new file mode 100644 index 00000000..10780076 --- /dev/null +++ b/packages/app/src/components/workbench/actionItems/group.ts @@ -0,0 +1,69 @@ +import { html } from 'lit' +import { customElement, property } from 'lit/decorators.js' + +import type { TraceActionGroupNode } from '@wdio/devtools-shared' + +import { ActionItem } from './item.js' +import '~icons/mdi/chevron-right.js' + +const SOURCE_COMPONENT = 'wdio-devtools-group-item' + +/** Collapsible step/group row of the trace player's action tree. */ +@customElement(SOURCE_COMPONENT) +export class GroupItem extends ActionItem { + @property({ type: Object }) + group?: TraceActionGroupNode + + /** Whether the group's children are currently rendered below it. */ + @property({ type: Boolean, reflect: true }) + expanded = false + + willUpdate(): void { + this.failed = Boolean(this.group?.failed) + this.duration = this.group + ? this.group.endTime - this.group.startTime + : undefined + } + + #toggle() { + this.dispatchEvent( + new CustomEvent('group-toggle', { + detail: { callId: this.group?.callId, expanded: this.expanded }, + bubbles: true, + composed: true + }) + ) + } + + render() { + if (!this.group) { + return + } + return html` + + ` + } +} + +declare global { + interface HTMLElementTagNameMap { + [SOURCE_COMPONENT]: GroupItem + } +} diff --git a/packages/app/src/components/workbench/actionItems/item.ts b/packages/app/src/components/workbench/actionItems/item.ts index 931a2299..945a79db 100644 --- a/packages/app/src/components/workbench/actionItems/item.ts +++ b/packages/app/src/components/workbench/actionItems/item.ts @@ -29,6 +29,10 @@ export class ActionItem extends Element { @property({ type: Boolean, reflect: true }) active = false + /** Whether this row's action errored — drives the red row treatment. */ + @property({ type: Boolean, reflect: true }) + failed = false + static styles = [ ...Element.styles, css` @@ -67,6 +71,19 @@ export class ActionItem extends Element { :host([active]) .ic { border-color: var(--accent); } + :host([failed]) button { + background: color-mix( + in srgb, + var(--vscode-charts-red) 8%, + transparent + ); + box-shadow: inset 2px 0 0 var(--vscode-charts-red); + } + :host([failed][active]) button { + box-shadow: + inset 2px 0 0 var(--vscode-charts-red), + inset 0 0 0 1px var(--vscode-panel-border); + } ` ] diff --git a/packages/app/src/components/workbench/actions.ts b/packages/app/src/components/workbench/actions.ts index 86f4e856..842f4f56 100644 --- a/packages/app/src/components/workbench/actions.ts +++ b/packages/app/src/components/workbench/actions.ts @@ -1,21 +1,38 @@ import { Element } from '@core/element' -import { html, css } from 'lit' +import { html, css, nothing } from 'lit' import { customElement, state } from 'lit/decorators.js' import { consume } from '@lit/context' -import type { CommandLog } from '@wdio/devtools-shared' -import { mutationContext, commandContext } from '../../controller/context.js' +import type { + CommandLog, + TraceActionChild, + TraceActionGroupNode +} from '@wdio/devtools-shared' +import { + mutationContext, + commandContext, + actionGroupsContext +} from '../../controller/context.js' import '../placeholder.js' import './actionItems/command.js' +import './actionItems/group.js' import './actionItems/mutation.js' import { stepDurations } from './actionItems/duration.js' import { activeTimestampAt } from './active-entry.js' +import { + defaultExpanded, + flattenActionTree, + type ActionTreeRow +} from './action-tree.js' type TimelineEntry = TraceMutation | CommandLog const SOURCE_COMPONENT = 'wdio-devtools-actions' +/** Horizontal shift per tree depth level in the player's action tree. */ +const TREE_INDENT_PX = 14 + @customElement(SOURCE_COMPONENT) export class DevtoolsActions extends Element { static styles = [ @@ -47,6 +64,11 @@ export class DevtoolsActions extends Element { background: var(--vscode-panel-border); pointer-events: none; } + + /* Tree mode indents rows, so the straight rail no longer lines up. */ + .timeline.tree::before { + display: none; + } ` ] @@ -56,12 +78,32 @@ export class DevtoolsActions extends Element { @consume({ context: commandContext, subscribe: true }) commands: CommandLog[] = [] + @consume({ context: actionGroupsContext, subscribe: true }) + groups?: TraceActionChild[] + // The selected timeline row, tracked by object reference — timestamps aren't // unique (commands logged in the same millisecond would all match), so // reference identity is what highlights exactly one row. @state() private activeEntry?: TimelineEntry + // User chevron toggles, by group callId; unset groups follow the default + // (failed or containing the active command → open). + @state() + private expandOverrides: ReadonlyMap = new Map() + + #onGroupToggle = (event: Event) => { + const { callId, expanded } = ( + event as CustomEvent<{ callId?: string; expanded: boolean }> + ).detail + if (!callId) { + return + } + const next = new Map(this.expandOverrides) + next.set(callId, !expanded) + this.expandOverrides = next + } + #onShowCommand = (event: Event) => { const command = (event as CustomEvent<{ command?: CommandLog }>).detail ?.command @@ -146,7 +188,65 @@ export class DevtoolsActions extends Element { } } + // Player tree mode: group rows expand/collapse; leaf rows are the same + // command items as the flat list, indented under their group. + #renderTree(rootChildren: TraceActionChild[]) { + const commands = this.commands || [] + const activeIndex = + this.activeEntry && 'command' in this.activeEntry + ? commands.indexOf(this.activeEntry) + : -1 + const isExpanded = (group: TraceActionGroupNode) => + this.expandOverrides.get(group.callId) ?? + defaultExpanded(group, activeIndex >= 0 ? activeIndex : undefined) + const rows = flattenActionTree(rootChildren, isExpanded) + const baseline = commands[0]?.timestamp ?? 0 + const gaps = stepDurations(commands.map((command) => command.timestamp)) + return html`
+ ${rows.map((row) => this.#renderTreeRow(row, commands, baseline, gaps))} +
` + } + + #renderTreeRow( + row: ActionTreeRow, + commands: CommandLog[], + baseline: number, + gaps: Array + ) { + const indent = `padding-left: ${row.depth * TREE_INDENT_PX}px` + if (row.kind === 'group') { + return html` + + ` + } + const entry = commands[row.commandIndex] + if (!entry) { + return nothing + } + // Reconstructed zips carry the real invocation span; gap is the fallback. + const duration = + entry.startTime !== undefined + ? entry.timestamp - entry.startTime + : gaps[row.commandIndex] + return html` + + ` + } + render() { + if (this.groups?.length) { + return this.#renderTree(this.groups) + } const entries = this.#sortedEntries() if (!entries.length) { diff --git a/packages/app/src/components/workbench/call-source.ts b/packages/app/src/components/workbench/call-source.ts index 9349b184..3fa2d5ca 100644 --- a/packages/app/src/components/workbench/call-source.ts +++ b/packages/app/src/components/workbench/call-source.ts @@ -23,6 +23,51 @@ export function parseCallSource( } } +/** Path with any trailing `:line` / `:line:column` suffix stripped — some + * recorded traces glue the line onto the file, so lookups compare clean paths. */ +export function normalizeSourcePath(path: string): string { + const match = path.match(/:\d+:\d+$/) || path.match(/:\d+$/) + return match && match.index ? path.slice(0, match.index) : path +} + +/** Key in `sources` holding `file`'s content — exact match first, then a + * normalized-path match so suffixed keys and clean queries still pair up. */ +export function resolveSourceFile( + sources: Record, + file: string +): string | undefined { + if (file in sources) { + return file + } + const target = normalizeSourcePath(file) + return Object.keys(sources).find((key) => normalizeSourcePath(key) === target) +} + +/** Normalized display list: every captured source file plus any file referenced + * by a command call source, deduplicated by clean path. */ +export function listSourceFiles( + sources: Record, + callSources: (string | undefined)[] +): string[] { + const files: string[] = [] + const seen = new Set() + const add = (path: string) => { + const normalized = normalizeSourcePath(path) + if (!seen.has(normalized)) { + seen.add(normalized) + files.push(normalized) + } + } + Object.keys(sources).forEach(add) + for (const callSource of callSources) { + const parsed = callSource ? parseCallSource(callSource) : null + if (parsed) { + add(parsed.file) + } + } + return files +} + /** Last path segment, handling both POSIX (`/`) and Windows (`\`) separators. */ export function fileBasename(path: string): string { const segments = path.split(/[/\\]/) diff --git a/packages/app/src/components/workbench/source.ts b/packages/app/src/components/workbench/source.ts index 81a12d0b..fcdb6fe5 100644 --- a/packages/app/src/components/workbench/source.ts +++ b/packages/app/src/components/workbench/source.ts @@ -13,7 +13,14 @@ import type { CommandLog } from '@wdio/devtools-shared' import { sourceContext, commandContext } from '../../controller/context.js' import { commandCategory, type ActionCategory } from './actionItems/category.js' -import { parseCallSource, fileBasename, pathSegments } from './call-source.js' +import { + parseCallSource, + fileBasename, + pathSegments, + normalizeSourcePath, + resolveSourceFile, + listSourceFiles +} from './call-source.js' import { sourceStyles } from './source/styles.js' import '../placeholder.js' @@ -95,12 +102,26 @@ export class DevtoolsSource extends Element { return document.body.classList.contains('dark') } + /** Captured files plus files referenced by command call sources (clean paths). */ + get #fileList(): string[] { + return listSourceFiles( + this.sources || {}, + (this.commands || []).map((c) => c.callSource) + ) + } + /** File to show: an explicit selection/call-site, else the first available. */ get #effectiveFile(): string | undefined { - if (this.activeFile && this.sources?.[this.activeFile]) { + const files = this.#fileList + if (this.activeFile && files.includes(this.activeFile)) { return this.activeFile } - return Object.keys(this.sources || {})[0] + return files[0] + } + + #contentFor(file: string): string | undefined { + const key = resolveSourceFile(this.sources || {}, file) + return key !== undefined ? this.sources[key] : undefined } connectedCallback(): void { @@ -145,8 +166,7 @@ export class DevtoolsSource extends Element { super.disconnectedCallback() window.removeEventListener('app-source-highlight', this.#onHighlight) window.removeEventListener('app-source-track', this.#onTrack) - this.#editorView?.destroy() - this.#editorView = undefined + this.#unmountEditor() this.#tabObserver?.disconnect() this.#tabObserver = undefined this.#themeObserver?.disconnect() @@ -158,6 +178,10 @@ export class DevtoolsSource extends Element { if (!target) { return } + if (this.#contentFor(target) === undefined) { + this.#unmountEditor() + return + } this.#mountEditor(target) this.#refreshCallSite() } @@ -167,10 +191,11 @@ export class DevtoolsSource extends Element { if (!parsed) { return } - this.activeFile = parsed.file - this.callSiteFile = parsed.file + const file = normalizeSourcePath(parsed.file) + this.activeFile = file + this.callSiteFile = file this.callSiteLine = parsed.line - const cmd = this.#commandAt(parsed.file, parsed.line) + const cmd = this.#commandAt(file, parsed.line) this.callSiteCommand = cmd?.command this.callSiteCategory = cmd ? commandCategory(cmd.command) : 'other' if (activateTab) { @@ -184,7 +209,11 @@ export class DevtoolsSource extends Element { return false } const parsed = parseCallSource(c.callSource) - return parsed?.file === file && parsed.line === line + return ( + !!parsed && + normalizeSourcePath(parsed.file) === file && + parsed.line === line + ) }) } @@ -192,9 +221,15 @@ export class DevtoolsSource extends Element { this.activeFile = file } + #unmountEditor() { + this.#editorView?.destroy() + this.#editorView = undefined + this.#mountedFile = undefined + } + #mountEditor(filePath: string) { - const source = this.sources?.[filePath] - if (!source) { + const source = this.#contentFor(filePath) + if (source === undefined) { return } const container = @@ -256,7 +291,7 @@ export class DevtoolsSource extends Element { } #renderFileTabs(active: string) { - return Object.keys(this.sources || {}).map( + return this.#fileList.map( (file) => html` ` } @@ -197,6 +214,9 @@ export class DevtoolsTab extends Element { @property({ type: Number }) badge?: number + @property({ type: String }) + badgeTone?: BadgeTone + static styles = [ ...Element.styles, css` diff --git a/packages/app/src/components/workbench.ts b/packages/app/src/components/workbench.ts index 0bb9f243..a97b20c9 100644 --- a/packages/app/src/components/workbench.ts +++ b/packages/app/src/components/workbench.ts @@ -8,9 +8,17 @@ import { consoleLogContext, metadataContext, networkRequestContext, - baselineContext + baselineContext, + commandContext, + suiteContext } from '../controller/context.js' -import type { Metadata, PreservedAttempt } from '@wdio/devtools-shared' +import type { + CommandLog, + Metadata, + PreservedAttempt +} from '@wdio/devtools-shared' +import type { SuiteStatsFragment } from '../controller/types.js' +import { collectErrors } from './workbench/errors/collect.js' import '~icons/mdi/arrow-collapse-down.js' import '~icons/mdi/arrow-collapse-up.js' @@ -24,6 +32,7 @@ import './workbench/logs.js' import './workbench/console.js' import './workbench/metadata.js' import './workbench/network.js' +import './workbench/errors.js' import './workbench/compare.js' import './browser/snapshot.js' import './browser/trace-timeline.js' @@ -73,6 +82,14 @@ export class DevtoolsWorkbench extends Element { @state() baselines: Map | undefined = undefined + @consume({ context: commandContext, subscribe: true }) + @state() + commands: CommandLog[] | undefined = undefined + + @consume({ context: suiteContext, subscribe: true }) + @state() + suites: Record[] | undefined = undefined + @consume({ context: metadataContext, subscribe: true }) @state() metadata: Metadata | undefined = undefined @@ -317,6 +334,44 @@ export class DevtoolsWorkbench extends Element { ` } + #errorCount(): number { + return collectErrors(this.commands, this.suites).length + } + + // Dock tab list — extracted so #renderWorkbenchTabs stays under the size cap. + #renderDockTabItems() { + return html` + + + + + + + + + + + + + + + + ${this.#renderCompareTabIfAvailable()} + ` + } + #renderWorkbenchTabs() { return html` - - - - - - - - - - - - - ${this.#renderCompareTabIfAvailable()} + ${this.#renderDockTabItems()}
+ ` + } + + render() { + const errors = collectErrors(this.commands, this.suites) + if (!errors.length) { + return html` +
+
✓
+
No errors
+
+ ` + } + return html`${errors.map((error) => this.#renderEntry(error))}` + } +} + +declare global { + interface HTMLElementTagNameMap { + [COMPONENT]: DevtoolsErrors + } +} diff --git a/packages/app/src/components/workbench/errors/collect.ts b/packages/app/src/components/workbench/errors/collect.ts new file mode 100644 index 00000000..ca7429e1 --- /dev/null +++ b/packages/app/src/components/workbench/errors/collect.ts @@ -0,0 +1,232 @@ +/** + * Pure error-collection for the workbench Errors tab. Merges failed commands + * (from `commandContext`) with failed tests (from `suiteContext`) into a single + * ordered, de-duplicated list. Kept framework-free and side-effect-free so the + * tab component only has to render what this returns. + */ + +import type { CommandLog } from '@wdio/devtools-shared' +import type { + SuiteStatsFragment, + TestStatsFragment +} from '../../../controller/types.js' +import { stripAnsi } from '../console-filter.js' + +/** One row in the Errors tab. */ +export interface CollectedError { + /** Failing action/step or test title — the row heading. */ + title: string + /** Error message shown message-first, monospace. */ + message: string + /** Optional stack, rendered under the message when present. */ + stack?: string + /** `file:line:col` source anchor for the "open source" link. */ + callSource?: string + /** The failing command, when the error came from one — lets the tab dispatch + * `show-command` to select and scroll to that action. */ + command?: CommandLog + /** Command timestamp; drives ordering and the `show-command` elapsed time. */ + timestamp?: number + /** Assertion expected value, rendered as a labelled row when present. */ + expected?: string + /** Assertion received/actual value, rendered as a labelled row when present. */ + actual?: string +} + +const ASSERTION_COMMAND_RE = /^(expect|assert|verify)\./ + +/** Display string for an expected/actual value that may already be serialized. */ +function displayValue(value: unknown): string | undefined { + if (value === undefined || value === null) { + return undefined + } + if (typeof value === 'string') { + return value + } + try { + return JSON.stringify(value) + } catch { + return String(value) + } +} + +/** Assertion commands carry `[actual, expected]` in args (see the exporter's + * Assert params + synthesizeExpectFailure). */ +function assertionValues(command: CommandLog): { + actual?: string + expected?: string +} { + if (!ASSERTION_COMMAND_RE.test(command.command) || command.args.length < 2) { + return {} + } + return { + actual: displayValue(command.args[0]), + expected: displayValue(command.args[1]) + } +} + +interface ReadableError { + message?: string + name?: string + stack?: string + expected?: unknown + actual?: unknown +} + +/** Split trailing `at …` stack-frame lines off the message body. */ +function splitStack(clean: string): { body: string; stack?: string } { + const lines = clean.split('\n') + const idx = lines.findIndex((line) => /^\s*at\s/.test(line)) + if (idx === -1) { + return { body: clean.trimEnd() } + } + return { + body: lines.slice(0, idx).join('\n').trimEnd(), + stack: lines.slice(idx).join('\n').trim() + } +} + +/** Trim each line and drop blanks — assertion libraries indent continuation + * lines, which would otherwise show as ragged whitespace. */ +function dedent(text: string): string { + return text + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .join('\n') +} + +/** Pull `Expected:` / `Received:` values out of a matcher body and return the + * remaining headline. The labels may be indented (expect-webdriverio pads + * them), and `Received:` can span several lines up to the end of the body. */ +function extractDiff(body: string): { + headline: string + expected?: string + actual?: string +} { + const expected = body.match(/^[ \t]*Expected:[ \t]*(.*)$/m)?.[1]?.trim() + const receivedAt = body.search(/^[ \t]*Received:/m) + let actual: string | undefined + let headline = body + if (receivedAt !== -1) { + actual = dedent( + body.slice(receivedAt).replace(/^[ \t]*Received:[ \t]*/, '') + ) + headline = body.slice(0, receivedAt) + } + if (expected !== undefined) { + headline = headline.replace(/^[ \t]*Expected:[ \t]*.*$/m, '') + } + return { headline: dedent(headline), expected, actual } +} + +/** Clean, structured view of any error-ish value: ANSI stripped, stack split + * off the message, and assertion Expected/Received pulled into fields. */ +function readError(error: unknown): + | { + message: string + stack?: string + expected?: string + actual?: string + } + | undefined { + if (!error || typeof error !== 'object') { + return undefined + } + const e = error as ReadableError + const raw = e.message?.trim() || e.name?.trim() || '' + if (!raw && !e.stack) { + return undefined + } + const { body, stack } = splitStack(stripAnsi(raw)) + const diff = extractDiff(body) + return { + message: diff.headline || 'Error', + stack: e.stack ? stripAnsi(e.stack) : stack, + expected: diff.expected ?? displayValue(e.expected), + actual: diff.actual ?? displayValue(e.actual) + } +} + +/** Failed leaf tests across every suite map, deduped by uid (last wins, matching + * the sidebar's root-suite dedup so we read the freshest fragment). */ +function collectFailedTests( + suites: Record[] | undefined +): TestStatsFragment[] { + const byUid = new Map() + const visit = (suite: SuiteStatsFragment) => { + for (const test of suite.tests ?? []) { + if (test.state === 'failed') { + byUid.set(test.uid, test) + } + } + for (const child of suite.suites ?? []) { + visit(child) + } + } + for (const map of suites ?? []) { + for (const suite of Object.values(map)) { + visit(suite) + } + } + return [...byUid.values()] +} + +function commandErrors(commands: CommandLog[] | undefined): CollectedError[] { + return (commands ?? []) + .flatMap((command) => { + const read = readError(command.error) + if (!read) { + return [] + } + const values = assertionValues(command) + return [ + { + title: command.title ?? command.command, + message: read.message, + stack: read.stack, + callSource: command.callSource, + command, + timestamp: command.timestamp, + expected: values.expected ?? read.expected, + actual: values.actual ?? read.actual + } + ] + }) + .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)) +} + +/** + * Build the Errors-tab list from the live/player contexts. + * + * Command failures come first (time-ordered) because they carry the clickable + * action; a failed test that only echoes a command's message is dropped so the + * same failure isn't listed twice (e.g. a Cucumber `Then` fails as both the + * assertion command and the scenario). + */ +export function collectErrors( + commands: CommandLog[] | undefined, + suites: Record[] | undefined +): CollectedError[] { + const fromCommands = commandErrors(commands) + const seenMessages = new Set(fromCommands.map((e) => e.message)) + + const fromTests = collectFailedTests(suites).flatMap((test) => { + const read = readError(test.error ?? test.errors?.[0]) + if (!read || seenMessages.has(read.message)) { + return [] + } + return [ + { + title: test.fullTitle || test.title || test.uid, + message: read.message, + stack: read.stack, + callSource: test.callSource, + expected: read.expected, + actual: read.actual + } + ] + }) + + return [...fromCommands, ...fromTests] +} diff --git a/packages/app/src/components/workbench/source.ts b/packages/app/src/components/workbench/source.ts index fcdb6fe5..5b336bda 100644 --- a/packages/app/src/components/workbench/source.ts +++ b/packages/app/src/components/workbench/source.ts @@ -110,13 +110,11 @@ export class DevtoolsSource extends Element { ) } - /** File to show: an explicit selection/call-site, else the first available. */ + /** File to show: an explicit selection/call-site wins even when it wasn't + * captured as a source (the toolbar then shows a not-captured state instead + * of silently falling back to a different file), else the first available. */ get #effectiveFile(): string | undefined { - const files = this.#fileList - if (this.activeFile && files.includes(this.activeFile)) { - return this.activeFile - } - return files[0] + return this.activeFile ?? this.#fileList[0] } #contentFor(file: string): string | undefined { From 26c98bd50cfe1409d8d24a0946c3aef85db32ab3 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 10:57:11 +0530 Subject: [PATCH 15/91] fix(service,nightwatch): distinct rerun-stable uids for Cucumber steps and outline rows --- .../src/helpers/cucumberScenarioBuilder.ts | 12 +++- packages/service/src/reporter.ts | 45 ++++++++++-- packages/service/tests/reporter.test.ts | 72 +++++++++++++++++++ 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts index 9ea426e9..54ab1da3 100644 --- a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts +++ b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts @@ -57,7 +57,9 @@ function buildScenarioStepTest( ? `${featureAbsPath}:${stepLines[i]}` : undefined return { - uid: deterministicUid(featureUri, `step:${scenarioName}:${step.text}`), + // Scope by the scenario uid (which carries the scenario line) so identical + // step text in sibling scenarios and outline example rows stays distinct. + uid: deterministicUid(featureUri, `step:${scenarioUid}:${step.text}`), cid: DEFAULTS.CID, title: stepLabel, fullTitle: `${scenarioName} ${stepLabel}`, @@ -87,8 +89,12 @@ export function buildCucumberScenarioSuite( parentFeatureSuiteUid } = input // deterministicUid (no counter) so the SAME scenario gets the SAME uid - // across retries — that's what makes retry-coalescing work upstream. - const scenarioUid = deterministicUid(featureUri, `scenario:${scenarioName}`) + // across retries — that's what makes retry-coalescing work upstream. The + // scenario line disambiguates outline example rows that share a name. + const scenarioUid = deterministicUid( + featureUri, + `scenario:${scenarioName}:${scenarioLine}` + ) const scenarioSuite: SuiteStats = { uid: scenarioUid, cid: DEFAULTS.CID, diff --git a/packages/service/src/reporter.ts b/packages/service/src/reporter.ts index ad792131..801a86ad 100644 --- a/packages/service/src/reporter.ts +++ b/packages/service/src/reporter.ts @@ -26,8 +26,13 @@ function isScenario(item: SuiteStats | TestStats): boolean { // Generate stable UID for a WDIO suite/test stats object. Handles WDIO's // Cucumber-specific shapes (scenarios with featureFile/featureLine, or with // numeric uid + example-row fallback), then delegates the Mocha/Jasmine path -// to core's generateStableUid. -function generateStableUid(item: SuiteStats | TestStats): string { +// to core's generateStableUid. `parentScope` (the owning scenario's stable +// uid) disambiguates Cucumber steps so identical step text in sibling +// scenarios yields distinct, rerun-stable uids. +function generateStableUid( + item: SuiteStats | TestStats, + parentScope?: string +): string { // For Cucumber scenarios, prefer the feature file URI:line as the stable // discriminator. The Cucumber pickle carries the actual line of the example // row, which is stable across reruns regardless of how many examples run. @@ -60,11 +65,25 @@ function generateStableUid(item: SuiteStats | TestStats): string { ) } + // Cucumber step: scope by the owning scenario's stable uid via + // deterministicUid (no run-order counter), so two scenarios sharing step + // text — and scenario-outline example rows — get distinct, rerun-stable uids. + const stepFile = 'file' in item ? (item.file ?? '') : '' + if (parentScope) { + return deterministicUid( + stepFile, + parentScope, + String(item.fullTitle || item.title) + ) + } + // For Mocha/Jasmine tests and suites, use only stable identifiers // that don't change between full and partial runs // DO NOT use cid or parent as they can vary based on run context - const file = 'file' in item ? (item.file ?? '') : '' - return generateStableUidByFileName(file, String(item.fullTitle || item.title)) + return generateStableUidByFileName( + stepFile, + String(item.fullTitle || item.title) + ) } /** @@ -154,6 +173,9 @@ export class TestReporter extends WebdriverIOReporter { #loadSource: (location: string) => void #currentSpecFile?: string #suitePath: string[] = [] + /** Stable uid of the Cucumber scenario currently open, used to scope its + * step uids. Undefined outside a scenario (Mocha/Jasmine). */ + #currentScenarioUid?: string constructor( options: Reporters.Options, @@ -205,6 +227,11 @@ export class TestReporter extends WebdriverIOReporter { // Generate stable UID for consistent identification across reruns suiteStats.uid = generateStableUid(suiteStats) + // Track the open Cucumber scenario so its steps scope their uids to it. + if (isScenario(suiteStats)) { + this.#currentScenarioUid = suiteStats.uid + } + this.#currentSpecFile = suiteStats.file setCurrentSpecFile(suiteStats.file) @@ -252,8 +279,10 @@ export class TestReporter extends WebdriverIOReporter { this.#loadSource(testStats.file) } - // Generate stable UID after enriching metadata for consistent test identification - testStats.uid = generateStableUid(testStats) + // Generate stable UID after enriching metadata for consistent test + // identification. Cucumber steps are scoped by their scenario's uid so + // identical step text across scenarios stays distinct. + testStats.uid = generateStableUid(testStats, this.#currentScenarioUid) this.#sendUpstream() } @@ -290,6 +319,10 @@ export class TestReporter extends WebdriverIOReporter { onSuiteEnd(suiteStats: SuiteStats): void { super.onSuiteEnd(suiteStats) + // Stop scoping steps once the owning scenario closes. + if (isScenario(suiteStats) && suiteStats.uid === this.#currentScenarioUid) { + this.#currentScenarioUid = undefined + } // Pop the suite we pushed on start if ( suiteStats.title && diff --git a/packages/service/tests/reporter.test.ts b/packages/service/tests/reporter.test.ts index 03461046..bdaccdca 100644 --- a/packages/service/tests/reporter.test.ts +++ b/packages/service/tests/reporter.test.ts @@ -324,4 +324,76 @@ describe('TestReporter - Rerun & Stable UID', () => { expect(() => reporter.report).not.toThrow() }) }) + + describe('Cucumber step uid scoping (no cross-scenario collision)', () => { + const FEATURE = '/proj/features/login.feature' + const STEP = + 'I should see a flash message saying You logged into a secure area!' + + const scenario = (title: string, line: number): SuiteStats => + ({ + uid: `raw-${title}`, + title, + fullTitle: `Login ${title}`, + file: FEATURE, + type: 'scenario', + argument: { uri: FEATURE, line } + }) as unknown as SuiteStats + + const step = (line: number): TestStats => + ({ + uid: 'raw-step', + title: STEP, + fullTitle: STEP, + file: FEATURE, + type: 'test', + argument: { uri: FEATURE, line } + }) as unknown as TestStats + + // Drive a scenario's step through the reporter and return the assigned uid. + const runStep = ( + r: TestReporter, + scen: SuiteStats, + stepStats: TestStats + ): string => { + r.onSuiteStart(scen) + r.onTestStart(stepStats) + r.onTestEnd(stepStats) + r.onSuiteEnd(scen) + return stepStats.uid + } + + it('gives identical step text in sibling scenarios distinct uids', () => { + const scenA = scenario('Scenario A', 5) + const scenB = scenario('Scenario B', 20) + const uidA = runStep(reporter, scenA, step(8)) + const uidB = runStep(reporter, scenB, step(23)) + expect(uidA).not.toBe(uidB) + }) + + it('keeps a step uid stable when its scenario is rerun alone', () => { + // Full run: scenario A then B. + const uidBFull = (() => { + runStep(reporter, scenario('Scenario A', 5), step(8)) + return runStep(reporter, scenario('Scenario B', 20), step(23)) + })() + + // Rerun scenario B on its own (fresh reporter resets the counter). A + // run-order-counter uid would shift to A's slot here; scoping by the + // scenario keeps it stable. + const reporter2 = new TestReporter( + { logFile: '/tmp/test.log' }, + sendUpstream as any + ) + const uidBAlone = runStep(reporter2, scenario('Scenario B', 20), step(23)) + + expect(uidBAlone).toBe(uidBFull) + }) + + it('distinguishes scenario-outline example rows (same title, different line)', () => { + const row1 = runStep(reporter, scenario('greet ', 10), step(11)) + const row2 = runStep(reporter, scenario('greet ', 14), step(15)) + expect(row1).not.toBe(row2) + }) + }) }) From 3cc47e3c59ee087b0e12d6216bdbef40275fa1ce Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 10:58:24 +0530 Subject: [PATCH 16/91] fix(core,service): collapse same-timestamp snapshots so blank frames don't clobber results --- packages/core/src/trace-snapshots.ts | 34 ++++- packages/service/src/snapshot-dedupe.ts | 20 +++ .../service/tests/dedupe-snapshots.test.ts | 124 +++++++++++++++++- 3 files changed, 174 insertions(+), 4 deletions(-) diff --git a/packages/core/src/trace-snapshots.ts b/packages/core/src/trace-snapshots.ts index 9e98704d..ab7ed3f5 100644 --- a/packages/core/src/trace-snapshots.ts +++ b/packages/core/src/trace-snapshots.ts @@ -16,10 +16,39 @@ export interface ScreencastFrameEvent { timestamp: number } +// Two snapshots at the same timestamp (a real post-action capture and a blank +// end-of-scenario one) map to the same `${pageId}-${ts}` resource name, so a +// last-wins write lets a blank frame clobber the real one — and the real +// screenshot and real elements can land on different captures. Collapse to one +// per timestamp: largest screenshot, richest metadata, merged across duplicates. +function collapseByTimestamp(snapshots: ActionSnapshot[]): ActionSnapshot[] { + const byTs = new Map() + const order: number[] = [] + for (const snap of snapshots) { + const existing = byTs.get(snap.timestamp) + if (!existing) { + byTs.set(snap.timestamp, { ...snap }) + order.push(snap.timestamp) + continue + } + if ((snap.screenshot?.length ?? 0) > (existing.screenshot?.length ?? 0)) { + existing.screenshot = snap.screenshot + } + if (!existing.elements?.length && snap.elements?.length) { + existing.elements = snap.elements + } + if (!existing.snapshotText && snap.snapshotText) { + existing.snapshotText = snap.snapshotText + } + } + return order.map((ts) => byTs.get(ts)!) +} + export function buildSnapshotResources( - snapshots: ActionSnapshot[], + rawSnapshots: ActionSnapshot[], pageId: string ): TraceZipResource[] { + const snapshots = collapseByTimestamp(rawSnapshots) const out: TraceZipResource[] = [] for (const snap of snapshots) { const base = `${pageId}-${snap.timestamp}` @@ -71,11 +100,12 @@ function frameForSnapshot( /** Filmstrip events; the first snapshot is re-anchored to t=0 for pre-interaction state. */ export function buildFilmstripEvents( - snapshots: ActionSnapshot[], + rawSnapshots: ActionSnapshot[], pageId: string, wallTime: number, viewport: { width: number; height: number } ): ScreencastFrameEvent[] { + const snapshots = collapseByTimestamp(rawSnapshots) const firstSnap = snapshots.find((s) => s.screenshot) const events: ScreencastFrameEvent[] = [] if (firstSnap) { diff --git a/packages/service/src/snapshot-dedupe.ts b/packages/service/src/snapshot-dedupe.ts index d7e14a00..5e51cccd 100644 --- a/packages/service/src/snapshot-dedupe.ts +++ b/packages/service/src/snapshot-dedupe.ts @@ -19,3 +19,23 @@ export function dedupeSnapshotsByTimestamp( } return [...best.values()].sort((a, b) => a.timestamp - b.timestamp) } + +/** Insert a snapshot, or — when one already shares its timestamp — keep only + * the richer screenshot, replacing in place to preserve any index ranges into + * the list. Applies the dedupe heuristic at capture time so a blank + * end-of-scenario frame can't clobber the last action's real result on export + * paths that don't run dedupeSnapshotsByTimestamp (e.g. per-spec traces). */ +export function upsertRichestSnapshot( + snapshots: ActionSnapshot[], + snap: ActionSnapshot +): void { + const idx = snapshots.findIndex((s) => s.timestamp === snap.timestamp) + if (idx === -1) { + snapshots.push(snap) + return + } + const existing = snapshots[idx]! + if ((snap.screenshot?.length ?? 0) > (existing.screenshot?.length ?? 0)) { + snapshots[idx] = snap + } +} diff --git a/packages/service/tests/dedupe-snapshots.test.ts b/packages/service/tests/dedupe-snapshots.test.ts index c33b9b38..d1ac50ee 100644 --- a/packages/service/tests/dedupe-snapshots.test.ts +++ b/packages/service/tests/dedupe-snapshots.test.ts @@ -1,6 +1,13 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' import { describe, it, expect } from 'vitest' -import type { ActionSnapshot } from '@wdio/devtools-shared' -import { dedupeSnapshotsByTimestamp } from '../src/snapshot-dedupe.js' +import { writeTraceZip, type TraceCapturer } from '@wdio/devtools-core' +import { TraceType, type ActionSnapshot } from '@wdio/devtools-shared' +import { + dedupeSnapshotsByTimestamp, + upsertRichestSnapshot +} from '../src/snapshot-dedupe.js' function snap(timestamp: number, screenshot: string): ActionSnapshot { return { timestamp, command: 'click', screenshot } @@ -34,3 +41,116 @@ describe('dedupeSnapshotsByTimestamp', () => { expect(dedupeSnapshotsByTimestamp([])).toEqual([]) }) }) + +describe('upsertRichestSnapshot', () => { + const blank = 'AA' + const content = 'A'.repeat(100) + + it("does not let a blank __final__ clobber the last action's real frame", () => { + // The last action's post-capture (real) and the end-of-scenario __final__ + // (blank) share the last action's timestamp; the real frame must survive. + const list: ActionSnapshot[] = [snap(50, content), snap(100, content)] + const final: ActionSnapshot = { + timestamp: 100, + command: '__final__', + screenshot: blank + } + upsertRichestSnapshot(list, final) + expect(list).toHaveLength(2) + expect(list[1].screenshot).toBe(content) + expect(list[1].command).toBe('click') + }) + + it('replaces in place when the final capture is richer', () => { + // Mirror the opposite case: the action was screenshotted mid-navigation + // (blank) and the settled __final__ is the real frame. + const list: ActionSnapshot[] = [snap(50, content), snap(100, blank)] + const final: ActionSnapshot = { + timestamp: 100, + command: '__final__', + screenshot: content + } + upsertRichestSnapshot(list, final) + expect(list).toHaveLength(2) + expect(list[1].screenshot).toBe(content) + expect(list[1].command).toBe('__final__') + }) + + it('preserves array length/indices on a timestamp collision', () => { + // Spec-range slicing indexes into this array, so a collision must never + // change its length — only replace in place or skip. + const list: ActionSnapshot[] = [snap(100, content)] + upsertRichestSnapshot(list, snap(100, blank)) + expect(list).toHaveLength(1) + }) + + it('appends when the timestamp is new', () => { + const list: ActionSnapshot[] = [snap(100, content)] + upsertRichestSnapshot(list, snap(200, blank)) + expect(list.map((s) => s.timestamp)).toEqual([100, 200]) + }) +}) + +describe('final-frame regression (capture → export)', () => { + it("exports the last action's real result, not the blank __final__ frame", async () => { + // Base64 payloads chosen so byte-length ranks blank < real < result and + // each round-trips cleanly through the resource writer's base64 decode. + const blank = 'AA' + const real = 'R'.repeat(200) + const result = 'B'.repeat(400) + + // The service builds pre/post captures per action; the last action's + // post-capture (result) collides on timestamp with the trailing __final__. + const snapshots: ActionSnapshot[] = [ + { timestamp: 1200, command: 'url', screenshot: real }, + { timestamp: 1200, command: 'click', screenshot: real }, + { timestamp: 1400, command: 'click', screenshot: result } + ] + upsertRichestSnapshot(snapshots, { + timestamp: 1400, + command: '__final__', + screenshot: blank + }) + const prepared = dedupeSnapshotsByTimestamp(snapshots) + + const capturer: TraceCapturer = { + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { command: 'url', args: [], timestamp: 1200, startTime: 1150 }, + { command: 'click', args: [], timestamp: 1400, startTime: 1350 } + ], + sources: new Map(), + metadata: { + type: TraceType.Testrunner, + viewport: { + width: 800, + height: 600, + offsetLeft: 0, + offsetTop: 0, + scale: 1 + } + }, + startWallTime: 1000 + } + + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'final-frame-')) + const dir = await writeTraceZip(capturer, { + outputDir, + sessionId: 'sess1234', + format: 'ndjson-directory', + actionSnapshots: prepared + }) + + const frame = await fs.readFile( + path.join(dir, 'resources', 'page@sess1234-1400.jpeg') + ) + // The last action's frame is the real result, never the blank capture. + expect(frame.toString('base64')).toBe(result) + expect(frame.length).not.toBe(Buffer.from(blank, 'base64').length) + + await fs.rm(outputDir, { recursive: true, force: true }) + }) +}) From 5ea851f36b6421cde8bc0106866184c1fe469726 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 11:00:47 +0530 Subject: [PATCH 17/91] chore: Added more actions to the action mapping to appear in the timeline --- packages/app/tests/errors-collect.test.ts | 337 ++++++++++++++++++++++ packages/shared/src/trace-actions.ts | 37 ++- 2 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 packages/app/tests/errors-collect.test.ts diff --git a/packages/app/tests/errors-collect.test.ts b/packages/app/tests/errors-collect.test.ts new file mode 100644 index 00000000..c92e3680 --- /dev/null +++ b/packages/app/tests/errors-collect.test.ts @@ -0,0 +1,337 @@ +import { describe, it, expect } from 'vitest' +import type { CommandLog } from '@wdio/devtools-shared' + +import { collectErrors } from '../src/components/workbench/errors/collect.js' +import type { + SuiteStatsFragment, + TestStatsFragment +} from '../src/controller/types.js' + +function command(overrides: Partial): CommandLog { + return { + command: 'click', + args: [], + timestamp: 0, + ...overrides + } +} + +function suiteMap( + ...suites: SuiteStatsFragment[] +): Record[] { + return [Object.fromEntries(suites.map((s) => [s.uid, s]))] +} + +function failedTest(overrides: Partial): TestStatsFragment { + return { + uid: 't1', + state: 'failed', + ...overrides + } +} + +describe('collectErrors', () => { + it('returns an empty list when nothing failed', () => { + expect(collectErrors([], [])).toEqual([]) + expect(collectErrors(undefined, undefined)).toEqual([]) + expect( + collectErrors( + [command({ command: 'click' })], + suiteMap({ + uid: 's1', + state: 'passed', + tests: [failedTest({ state: 'passed' })] + }) + ) + ).toEqual([]) + }) + + it('collects failed commands with title, message, stack and source', () => { + const errors = collectErrors( + [ + command({ + command: 'expect', + title: 'expect(el).toHaveText', + callSource: 'file:///spec.ts:12:3', + error: { + name: 'Error', + message: 'expected foo, got bar', + stack: 'at spec.ts:12' + }, + timestamp: 100 + }) + ], + [] + ) + expect(errors).toHaveLength(1) + expect(errors[0]).toMatchObject({ + title: 'expect(el).toHaveText', + message: 'expected foo, got bar', + stack: 'at spec.ts:12', + callSource: 'file:///spec.ts:12:3', + timestamp: 100 + }) + expect(errors[0].command?.command).toBe('expect') + }) + + it('falls back to the command name when there is no title', () => { + const [error] = collectErrors( + [ + command({ + command: 'navigateTo', + error: { name: 'Error', message: 'boom' } + }) + ], + [] + ) + expect(error.title).toBe('navigateTo') + }) + + it('orders command errors by timestamp', () => { + const errors = collectErrors( + [ + command({ + command: 'second', + error: { name: 'Error', message: 'b' }, + timestamp: 200 + }), + command({ + command: 'first', + error: { name: 'Error', message: 'a' }, + timestamp: 100 + }) + ], + [] + ) + expect(errors.map((e) => e.message)).toEqual(['a', 'b']) + }) + + it('collects failed tests from nested suites', () => { + const errors = collectErrors( + [], + suiteMap({ + uid: 'feature', + state: 'failed', + suites: [ + { + uid: 'scenario', + state: 'failed', + tests: [ + failedTest({ + uid: 'step-1', + title: 'Then it should pass', + fullTitle: 'Scenario > Then it should pass', + callSource: 'steps.ts:8:1', + error: { name: 'AssertionError', message: 'nope' } + }) + ] + } + ] + }) + ) + expect(errors).toHaveLength(1) + expect(errors[0]).toMatchObject({ + title: 'Scenario > Then it should pass', + message: 'nope', + callSource: 'steps.ts:8:1' + }) + expect(errors[0].command).toBeUndefined() + }) + + it('reads the first entry of errors[] when error is absent', () => { + const [error] = collectErrors( + [], + suiteMap({ + uid: 's', + state: 'failed', + tests: [ + failedTest({ + uid: 'a', + errors: [{ name: 'Error', message: 'from errors array' } as Error] + }) + ] + }) + ) + expect(error.message).toBe('from errors array') + }) + + it('ignores failed tests that carry no error payload', () => { + const errors = collectErrors( + [], + suiteMap({ + uid: 's', + state: 'failed', + tests: [failedTest({ uid: 'a' })] + }) + ) + expect(errors).toEqual([]) + }) + + it('dedupes a test failure that only echoes a command failure', () => { + const shared = 'expect(locator).toHaveText failed' + const errors = collectErrors( + [ + command({ + command: 'expect', + error: { name: 'Error', message: shared }, + timestamp: 5 + }) + ], + suiteMap({ + uid: 's', + state: 'failed', + tests: [ + failedTest({ + uid: 'a', + title: 'the scenario', + error: { name: 'Error', message: shared } + }) + ] + }) + ) + expect(errors).toHaveLength(1) + expect(errors[0].command?.command).toBe('expect') + }) + + it('keeps a distinct test failure alongside command failures', () => { + const errors = collectErrors( + [ + command({ + command: 'click', + error: { name: 'Error', message: 'click failed' }, + timestamp: 1 + }) + ], + suiteMap({ + uid: 's', + state: 'failed', + tests: [ + failedTest({ + uid: 'a', + title: 'setup', + error: { name: 'Error', message: 'hook failed' } + }) + ] + }) + ) + expect(errors.map((e) => e.message)).toEqual([ + 'click failed', + 'hook failed' + ]) + }) + + it('dedupes failed tests by uid across repeated suite maps', () => { + const test = failedTest({ + uid: 'dup', + title: 'flaky', + error: { name: 'Error', message: 'x' } + }) + const errors = collectErrors( + [], + [ + { s: { uid: 's', state: 'failed', tests: [test] } }, + { s: { uid: 's', state: 'failed', tests: [test] } } + ] + ) + expect(errors).toHaveLength(1) + }) + + it('strips ANSI, splits the stack, and pulls Expected/Received from a cucumber message', () => { + const raw = + 'Expect $(`#flash`) to have text\n\n' + + 'Expected: StringContaining "secure area"\n' + + 'Received: "invalid"\n' + + ' at World. (/specs/steps.ts:31:20)\n' + + ' at process.processTicksAndRejections (node:internal:104:5)' + const [error] = collectErrors( + [ + command({ + command: 'expect.assertion', + error: { name: 'Error', message: raw } + }) + ], + [] + ) + expect(error.message).toBe('Expect $(`#flash`) to have text') + expect(error.expected).toBe('StringContaining "secure area"') + expect(error.actual).toBe('"invalid"') + expect(error.stack).toContain('at World.') + expect(error.message).not.toContain('') + expect(error.message).not.toContain('at World') + }) + + it('extracts indented Expected/Received and dedents the value', () => { + const raw = + 'Expect $(`#flash`) to have text\n\n' + + ' Expected: StringContaining "secure area"\n' + + ' Received: "invalid!\n×"\n' + + ' at World. (/specs/steps.ts:31:20)' + const [error] = collectErrors( + [command({ command: 'getText', error: { name: 'Error', message: raw } })], + [] + ) + expect(error.message).toBe('Expect $(`#flash`) to have text') + expect(error.expected).toBe('StringContaining "secure area"') + expect(error.actual).toBe('"invalid!\n×"') + expect(error.stack).toContain('at World.') + }) + + it('extracts expected/received from an assertion command', () => { + const [error] = collectErrors( + [ + command({ + command: 'expect.toHaveText', + args: ['Your username is invalid!', 'You logged into a secure area!'], + error: { name: 'Error', message: 'text mismatch' } + }) + ], + [] + ) + expect(error.actual).toBe('Your username is invalid!') + expect(error.expected).toBe('You logged into a secure area!') + }) + + it('extracts expected/received from a failed-test matcher error', () => { + const [error] = collectErrors( + [], + suiteMap({ + uid: 's', + state: 'failed', + tests: [ + failedTest({ + uid: 'a', + title: 'the scenario', + error: { + name: 'Error', + message: 'mismatch', + expected: 42, + actual: 7 + } as unknown as Error + }) + ] + }) + ) + expect(error.expected).toBe('42') + expect(error.actual).toBe('7') + }) + + it('places command errors before test errors', () => { + const errors = collectErrors( + [ + command({ + command: 'click', + error: { name: 'Error', message: 'cmd' }, + timestamp: 999 + }) + ], + suiteMap({ + uid: 's', + state: 'failed', + tests: [ + failedTest({ uid: 'a', error: { name: 'Error', message: 'test' } }) + ] + }) + ) + expect(errors.map((e) => e.message)).toEqual(['cmd', 'test']) + }) +}) diff --git a/packages/shared/src/trace-actions.ts b/packages/shared/src/trace-actions.ts index bae73200..c3bd7462 100644 --- a/packages/shared/src/trace-actions.ts +++ b/packages/shared/src/trace-actions.ts @@ -33,6 +33,8 @@ export const TRACKED_ASSERT_METHODS = [ // assert. (node:assert), verify. (nightwatch soft variants), and // expect. (synthesized failing-matcher entries) all render as Assert. +// Only FAILING expect-webdriverio matchers reach the command log today (via the +// reporter); recording passing matchers needs a per-adapter capture change. const ASSERT_COMMAND_RE = /^(?:assert|verify|expect)\.(\w+)$/ export function mapAssertCommand(command: string): TraceAction | null { @@ -68,5 +70,38 @@ export const ACTION_MAP: Record = { execute: { class: 'Page', method: 'evaluate' }, executeAsync: { class: 'Page', method: 'evaluate' }, switchToFrame: { class: 'Frame', method: 'goto' }, - touchAction: { class: 'Element', method: 'tap' } + touchAction: { class: 'Element', method: 'tap' }, + // WDIO element reads — surfaced so query steps appear in the timeline the way + // locator queries do in standard trace viewers. Adapters already capture + // these; only the export allow-list kept them out. + getText: { class: 'Element', method: 'getText' }, + getValue: { class: 'Element', method: 'getValue' }, + getAttribute: { class: 'Element', method: 'getAttribute' }, + getProperty: { class: 'Element', method: 'getProperty' }, + getCSSProperty: { class: 'Element', method: 'getCSSProperty' }, + getTagName: { class: 'Element', method: 'getTagName' }, + getLocation: { class: 'Element', method: 'getLocation' }, + getSize: { class: 'Element', method: 'getSize' }, + isDisplayed: { class: 'Element', method: 'isDisplayed' }, + isExisting: { class: 'Element', method: 'isExisting' }, + isEnabled: { class: 'Element', method: 'isEnabled' }, + isSelected: { class: 'Element', method: 'isSelected' }, + isClickable: { class: 'Element', method: 'isClickable' }, + isFocused: { class: 'Element', method: 'isFocused' }, + // Explicit user-facing waits (not the internal polling loops behind them). + waitForDisplayed: { class: 'Element', method: 'waitForDisplayed' }, + waitForExist: { class: 'Element', method: 'waitForExist' }, + waitForEnabled: { class: 'Element', method: 'waitForEnabled' }, + waitForClickable: { class: 'Element', method: 'waitForClickable' }, + waitUntil: { class: 'Browser', method: 'waitForFunction' }, + // WDIO page/browser reads + getTitle: { class: 'Page', method: 'getTitle' }, + getUrl: { class: 'Page', method: 'getUrl' }, + getPageSource: { class: 'Page', method: 'getPageSource' }, + // Selenium read aliases — normalized onto the WDIO names above so both runners + // read identically. getText/getAttribute/getTagName/isDisplayed/isEnabled/ + // isSelected share the command name across runners and need no alias. + getCssValue: { class: 'Element', method: 'getCSSProperty' }, + getRect: { class: 'Element', method: 'getRect' }, + getCurrentUrl: { class: 'Page', method: 'getUrl' } } From 1ff07ccbf264c24810c728a89dfc1b426b314d85 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 11:56:54 +0530 Subject: [PATCH 18/91] chore(examples): split wdio example into cucumber/ and mocha/ with shared page objects --- .../wdio/cucumber/features/login-fail.feature | 9 +++ .../{ => cucumber}/features/login.feature | 2 +- .../features/step-definitions/steps.ts | 4 +- examples/wdio/{ => cucumber}/wdio.conf.ts | 0 .../wdio/{ => cucumber}/wdio.mobile.conf.ts | 0 examples/wdio/cucumber/wdio.retention.conf.ts | 68 +++++++++++++++++++ .../wdio/{ => cucumber}/wdio.trace.conf.ts | 0 examples/wdio/mocha/specs/login-fail.e2e.ts | 17 +++++ examples/wdio/mocha/specs/login.e2e.ts | 26 +++++++ examples/wdio/mocha/wdio.conf.ts | 53 +++++++++++++++ examples/wdio/package.json | 9 ++- .../{features => }/pageobjects/login.page.ts | 0 .../wdio/{features => }/pageobjects/page.ts | 0 .../{features => }/pageobjects/secure.page.ts | 0 examples/wdio/tsconfig.json | 8 ++- 15 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 examples/wdio/cucumber/features/login-fail.feature rename examples/wdio/{ => cucumber}/features/login.feature (84%) rename examples/wdio/{ => cucumber}/features/step-definitions/steps.ts (89%) rename examples/wdio/{ => cucumber}/wdio.conf.ts (100%) rename examples/wdio/{ => cucumber}/wdio.mobile.conf.ts (100%) create mode 100644 examples/wdio/cucumber/wdio.retention.conf.ts rename examples/wdio/{ => cucumber}/wdio.trace.conf.ts (100%) create mode 100644 examples/wdio/mocha/specs/login-fail.e2e.ts create mode 100644 examples/wdio/mocha/specs/login.e2e.ts create mode 100644 examples/wdio/mocha/wdio.conf.ts rename examples/wdio/{features => }/pageobjects/login.page.ts (100%) rename examples/wdio/{features => }/pageobjects/page.ts (100%) rename examples/wdio/{features => }/pageobjects/secure.page.ts (100%) diff --git a/examples/wdio/cucumber/features/login-fail.feature b/examples/wdio/cucumber/features/login-fail.feature new file mode 100644 index 00000000..c8d325b8 --- /dev/null +++ b/examples/wdio/cucumber/features/login-fail.feature @@ -0,0 +1,9 @@ +Feature: Retention check — deliberately failing login + + # Logs in with bad credentials but asserts the SUCCESS message, so the Then + # step fails. Used to verify retain-on-failure keeps this spec's trace while + # dropping the passing login.feature. Reuses the existing step definitions. + Scenario: Failing assertion exercises retain-on-failure + Given I am on the login page + When I login with foobar and barfoo + Then I should see a flash message saying You logged into a secure area! diff --git a/examples/wdio/features/login.feature b/examples/wdio/cucumber/features/login.feature similarity index 84% rename from examples/wdio/features/login.feature rename to examples/wdio/cucumber/features/login.feature index e6dd0f87..8b9bcf9a 100644 --- a/examples/wdio/features/login.feature +++ b/examples/wdio/cucumber/features/login.feature @@ -8,5 +8,5 @@ Feature: The Internet Guinea Pig Website Examples: | username | password | message | - | tomsmith | SuperSecretPassword! | You logged into a secure area! | + | tomsmith | SuperSecretPassword1! | You logged into a secure area! | | foobar | barfoo | Your username is invalid! | diff --git a/examples/wdio/features/step-definitions/steps.ts b/examples/wdio/cucumber/features/step-definitions/steps.ts similarity index 89% rename from examples/wdio/features/step-definitions/steps.ts rename to examples/wdio/cucumber/features/step-definitions/steps.ts index 32522982..3367b631 100644 --- a/examples/wdio/features/step-definitions/steps.ts +++ b/examples/wdio/cucumber/features/step-definitions/steps.ts @@ -1,8 +1,8 @@ import { Given, When, Then, After } from '@wdio/cucumber-framework' import { browser, expect } from '@wdio/globals' -import LoginPage from '../pageobjects/login.page.js' -import SecurePage from '../pageobjects/secure.page.js' +import LoginPage from '../../../pageobjects/login.page.js' +import SecurePage from '../../../pageobjects/secure.page.js' const pages = { login: LoginPage diff --git a/examples/wdio/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts similarity index 100% rename from examples/wdio/wdio.conf.ts rename to examples/wdio/cucumber/wdio.conf.ts diff --git a/examples/wdio/wdio.mobile.conf.ts b/examples/wdio/cucumber/wdio.mobile.conf.ts similarity index 100% rename from examples/wdio/wdio.mobile.conf.ts rename to examples/wdio/cucumber/wdio.mobile.conf.ts diff --git a/examples/wdio/cucumber/wdio.retention.conf.ts b/examples/wdio/cucumber/wdio.retention.conf.ts new file mode 100644 index 00000000..a71b97bb --- /dev/null +++ b/examples/wdio/cucumber/wdio.retention.conf.ts @@ -0,0 +1,68 @@ +import path from 'node:path' + +// Disposable harness for verifying tracePolicy end-to-end. Runs one passing +// spec (login.feature) and one failing spec (login-fail.feature) at spec +// granularity so retain-on-failure can be seen dropping the passing spec's +// trace while keeping the failing one. Change `tracePolicy` below to try each. + +const __dirname = path.resolve(path.dirname(new URL(import.meta.url).pathname)) + +export const config: WebdriverIO.Config = { + runner: 'local', + tsConfigPath: './tsconfig.json', + + specs: ['./features/login.feature', './features/login-fail.feature'], + + maxInstances: 1, + + capabilities: [ + { + browserName: 'chrome', + browserVersion: '149.0.7827.201', // specify chromium browser version for testing + 'goog:chromeOptions': { + args: [ + '--headless', + '--disable-gpu', + '--remote-allow-origins=*', + '--window-size=1600,1200' + ] + } + } + ], + + logLevel: 'warn', + baseUrl: 'http://localhost', + waitforTimeout: 10000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + + services: [ + [ + 'devtools', + { + mode: 'trace' as const, + traceGranularity: 'spec' as const + // tracePolicy: 'retain-on-failure' as const + } + ] + ], + + framework: 'cucumber', + reporters: ['spec'], + + cucumberOpts: { + require: [ + path.resolve(__dirname, 'features', 'step-definitions', 'steps.ts') + ], + backtrace: false, + requireModule: [], + dryRun: false, + failFast: false, + snippets: true, + source: true, + strict: false, + tagExpression: '', + timeout: 60000, + ignoreUndefinedDefinitions: false + } +} diff --git a/examples/wdio/wdio.trace.conf.ts b/examples/wdio/cucumber/wdio.trace.conf.ts similarity index 100% rename from examples/wdio/wdio.trace.conf.ts rename to examples/wdio/cucumber/wdio.trace.conf.ts diff --git a/examples/wdio/mocha/specs/login-fail.e2e.ts b/examples/wdio/mocha/specs/login-fail.e2e.ts new file mode 100644 index 00000000..b939a28b --- /dev/null +++ b/examples/wdio/mocha/specs/login-fail.e2e.ts @@ -0,0 +1,17 @@ +import { expect } from '@wdio/globals' + +import LoginPage from '../../pageobjects/login.page.js' +import SecurePage from '../../pageobjects/secure.page.js' + +// Deliberately failing — mirrors login-fail.feature, so the Errors tab and +// tracePolicy: 'retain-on-failure' have something to exercise. +describe('Login (failing)', () => { + it('asserts the wrong flash message so the run fails', async () => { + console.log('[TEST] submitting invalid credentials') + await LoginPage.open() + await LoginPage.login('foobar', 'barfoo') + await expect(SecurePage.flashAlert).toHaveText( + expect.stringContaining('You logged into a secure area!') + ) + }) +}) diff --git a/examples/wdio/mocha/specs/login.e2e.ts b/examples/wdio/mocha/specs/login.e2e.ts new file mode 100644 index 00000000..0282a744 --- /dev/null +++ b/examples/wdio/mocha/specs/login.e2e.ts @@ -0,0 +1,26 @@ +import { expect } from '@wdio/globals' + +import LoginPage from '../../pageobjects/login.page.js' +import SecurePage from '../../pageobjects/secure.page.js' + +describe('Login', () => { + it('logs into the secure area with valid credentials', async () => { + console.log('[TEST] logging in with valid credentials') + await LoginPage.open() + await LoginPage.login('tomsmith', 'SuperSecretPassword!') + await expect(SecurePage.flashAlert).toBeExisting() + await expect(SecurePage.flashAlert).toHaveText( + expect.stringContaining('You logged into a secure area!') + ) + console.log('[TEST] secure area reached') + }) + + it('shows an error message for an invalid username', async () => { + console.log('[TEST] logging in with an invalid username') + await LoginPage.open() + await LoginPage.login('foobar', 'barfoo') + await expect(SecurePage.flashAlert).toHaveText( + expect.stringContaining('Your username is invalid!') + ) + }) +}) diff --git a/examples/wdio/mocha/wdio.conf.ts b/examples/wdio/mocha/wdio.conf.ts new file mode 100644 index 00000000..474602e3 --- /dev/null +++ b/examples/wdio/mocha/wdio.conf.ts @@ -0,0 +1,53 @@ +import type { Options } from '@wdio/types' + +// Mocha counterpart to wdio.conf.ts (which runs the Cucumber example). Same +// capabilities and devtools service; only the framework + spec layout differ. +export const config: Options.Testrunner = { + runner: 'local', + autoCompileOpts: { + autoCompile: true, + tsNodeOpts: { + project: './tsconfig.json', + transpileOnly: true + } + }, + specs: ['./specs/**/*.e2e.ts'], + exclude: [], + maxInstances: 10, + capabilities: [ + { + browserName: 'chrome', + 'goog:chromeOptions': { + args: [ + '--headless', + '--disable-gpu', + '--remote-allow-origins=*', + '--window-size=1600,900' + ] + } + } + ], + logLevel: 'debug', + bail: 0, + baseUrl: 'http://localhost', + waitforTimeout: 10000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + services: [ + [ + 'devtools', + { + mode: 'trace' as const, + traceGranularity: 'spec' as const + // tracePolicy: 'retain-on-failure' as const + // screencast: { enabled: true, pollIntervalMs: 200 } + } + ] + ], + framework: 'mocha', + reporters: ['spec'], + mochaOpts: { + ui: 'bdd', + timeout: 60000 + } +} diff --git a/examples/wdio/package.json b/examples/wdio/package.json index c783a577..4bbb0957 100644 --- a/examples/wdio/package.json +++ b/examples/wdio/package.json @@ -6,6 +6,7 @@ "devDependencies": { "@wdio/cli": "9.28.0", "@wdio/cucumber-framework": "9.28.0", + "@wdio/mocha-framework": "9.28.0", "@wdio/devtools-service": "workspace:*", "@wdio/globals": "9.28.0", "@wdio/local-runner": "9.28.0", @@ -18,8 +19,10 @@ "typescript": "^6.0.3" }, "scripts": { - "wdio": "wdio run ./wdio.conf.ts", - "mobile": "wdio run ./wdio.mobile.conf.ts", - "trace": "wdio run ./wdio.trace.conf.ts" + "cucumber": "wdio run ./cucumber/wdio.conf.ts", + "mocha": "wdio run ./mocha/wdio.conf.ts", + "mobile": "wdio run ./cucumber/wdio.mobile.conf.ts", + "trace": "wdio run ./cucumber/wdio.trace.conf.ts", + "retention": "wdio run ./cucumber/wdio.retention.conf.ts" } } diff --git a/examples/wdio/features/pageobjects/login.page.ts b/examples/wdio/pageobjects/login.page.ts similarity index 100% rename from examples/wdio/features/pageobjects/login.page.ts rename to examples/wdio/pageobjects/login.page.ts diff --git a/examples/wdio/features/pageobjects/page.ts b/examples/wdio/pageobjects/page.ts similarity index 100% rename from examples/wdio/features/pageobjects/page.ts rename to examples/wdio/pageobjects/page.ts diff --git a/examples/wdio/features/pageobjects/secure.page.ts b/examples/wdio/pageobjects/secure.page.ts similarity index 100% rename from examples/wdio/features/pageobjects/secure.page.ts rename to examples/wdio/pageobjects/secure.page.ts diff --git a/examples/wdio/tsconfig.json b/examples/wdio/tsconfig.json index 8acd81f7..8b8a6786 100644 --- a/examples/wdio/tsconfig.json +++ b/examples/wdio/tsconfig.json @@ -11,7 +11,8 @@ "node", "@wdio/globals/types", "expect-webdriverio", - "@wdio/cucumber-framework" + "@wdio/cucumber-framework", + "@wdio/mocha-framework" ], "skipLibCheck": true, "noEmit": true, @@ -24,7 +25,8 @@ "noFallthroughCasesInSwitch": true }, "include": [ - "test", - "wdio.conf.ts" + "cucumber", + "mocha", + "pageobjects" ] } From 353422431e0bc48a2c948afa0dbec98d7f61603a Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 12:09:20 +0530 Subject: [PATCH 19/91] chore: Update path in readme and update pnpm-lock.yaml --- ARCHITECTURE.md | 2 +- README.md | 2 +- package.json | 3 +- pnpm-lock.yaml | 183 +++++++++++++++--------------------------------- 4 files changed, 60 insertions(+), 130 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6d4fecc0..1324f454 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -144,7 +144,7 @@ The execution environment is the browser, not Node, so this package cannot impor Per-framework demo projects used for manual verification. -- `examples/wdio/` — WebdriverIO with Mocha (default). Run via `pnpm demo:wdio`. +- `examples/wdio/` — WebdriverIO, split into `cucumber/` and `mocha/` (shared page objects in `pageobjects/`). Run via `pnpm demo:wdio` (Cucumber) or `pnpm demo:wdio:mocha`. - `examples/nightwatch/` — Nightwatch (both vanilla and Cucumber). Run via `pnpm demo:nightwatch`. - `examples/selenium/` — Selenium with subdirs for `mocha-test/`, `jest-test/`, `cucumber-test/`, `jasmine-test/`, `vitest-test/`. `pnpm demo:selenium` runs mocha; `pnpm --filter @wdio/selenium-devtools example:` runs the others. diff --git a/README.md b/README.md index 96e7adf8..fa87cc71 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ services: [[DevToolsHookService, { Adapters detect mobile sessions via `platformName: 'android' | 'ios'` (case-insensitive) and adjust the per-action snapshot to extract elements from the mobile XML tree instead of the DOM. The trace's `context-options` records `title: 'android' — ` / `'ios' — ` so the viewer labels frames correctly. -A reference WDIO config is at [examples/wdio/wdio.mobile.conf.ts](examples/wdio/wdio.mobile.conf.ts). Prereqs to run it end-to-end with a local emulator: +A reference WDIO config is at [examples/wdio/cucumber/wdio.mobile.conf.ts](examples/wdio/cucumber/wdio.mobile.conf.ts). Prereqs to run it end-to-end with a local emulator: 1. **Java JDK** — `brew install --cask temurin` 2. **Android SDK** — `brew install --cask android-commandlinetools` then `yes | sdkmanager --licenses && sdkmanager "platform-tools" "emulator" "system-images;android-34;google_apis_playstore;arm64-v8a"`. The brew cask installs sdkmanager under `/opt/homebrew/share/android-commandlinetools/`, and sdkmanager downloads other SDK pieces alongside it — set `ANDROID_HOME` to that path (not `~/Library/Android/sdk/`). diff --git a/package.json b/package.json index 49292cfd..f12d3491 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "build": "pnpm -r build", - "demo:wdio": "wdio run ./examples/wdio/wdio.conf.ts", + "demo:wdio": "wdio run ./examples/wdio/cucumber/wdio.conf.ts", + "demo:wdio:mocha": "wdio run ./examples/wdio/mocha/wdio.conf.ts", "demo:nightwatch": "pnpm --filter @wdio/nightwatch-devtools example", "demo:selenium": "pnpm --filter @wdio/selenium-devtools example", "dev": "pnpm --parallel dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b338e0b8..a2354ac3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -141,6 +141,9 @@ importers: '@wdio/local-runner': specifier: 9.28.0 version: 9.28.0(@wdio/globals@9.28.0)(webdriverio@9.28.0(puppeteer-core@21.11.0)) + '@wdio/mocha-framework': + specifier: 9.28.0 + version: 9.28.0 '@wdio/spec-reporter': specifier: 9.28.0 version: 9.28.0 @@ -365,7 +368,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.0.16 - version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) packages/nightwatch-devtools: dependencies: @@ -1859,49 +1862,42 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@napi-rs/nice-linux-arm64-musl@1.1.1': resolution: {integrity: sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [musl] '@napi-rs/nice-linux-ppc64-gnu@1.1.1': resolution: {integrity: sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==} engines: {node: '>= 10'} cpu: [ppc64] os: [linux] - libc: [glibc] '@napi-rs/nice-linux-riscv64-gnu@1.1.1': resolution: {integrity: sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] - libc: [glibc] '@napi-rs/nice-linux-s390x-gnu@1.1.1': resolution: {integrity: sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==} engines: {node: '>= 10'} cpu: [s390x] os: [linux] - libc: [glibc] '@napi-rs/nice-linux-x64-gnu@1.1.1': resolution: {integrity: sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@napi-rs/nice-linux-x64-musl@1.1.1': resolution: {integrity: sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [musl] '@napi-rs/nice-openharmony-arm64@1.1.1': resolution: {integrity: sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==} @@ -2027,42 +2023,36 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.0.3': resolution: {integrity: sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.0.3': resolution: {integrity: sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.0.3': resolution: {integrity: sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.0.3': resolution: {integrity: sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.0.3': resolution: {integrity: sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.0.3': resolution: {integrity: sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==} @@ -2133,79 +2123,66 @@ packages: resolution: {integrity: sha512-T1dMEQhXA/jkJ/jyMIw9IovK8bSUq7A8kLIlvZTb/6YIVsp2zLavr4F3oyllHWo7eIVJRyE5n3tUjQJEbE1IuQ==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.62.0': resolution: {integrity: sha512-2as0LgT7qQpyceQq6VUJYnumUMUrgGQCWIiDIN9DE0/tglsk6o66uCB4f3djRawAltvfCNLyZZrsqbPA6inCsA==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.62.0': resolution: {integrity: sha512-bVURMg+6eNN9C/yc0aVjooZcwTTtYF4YW3xta5pP0//r3o1V8gXEHXWCndj47w/HhwsFroZrFhR+6uQP5T0n0g==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.62.0': resolution: {integrity: sha512-Ful8pM/2yYI83PViWdFdpZhdI8HJ5qsXANe5atypbHDf+KIBBDsZsbyy8hbXnULVvW9NsTh5DHwbcBftyLTfiw==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.62.0': resolution: {integrity: sha512-9Gp/DgrkzfUBmNPVTyPTvay+4xEP7M/clXpj3efXBcm6uTIVIgDg4rqUpqKXvLEuFRVuEpSAOkhgNeecvaZ4Cg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.62.0': resolution: {integrity: sha512-m9tsJz54LUXkSYM8+8PG81B9IKK5r+2T0clMq4QrS16xFosufU7firBDAZEsDheDs7wTlP7h3++S7lMsU955HA==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.62.0': resolution: {integrity: sha512-3UvJ5PNVU16aJf6M3tFI24pWzAl2/ynfbyRN3ICyQajK1lSkrnVYNnLz3v04J32qKa0FczJc22zeToc0lr2A3w==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.62.0': resolution: {integrity: sha512-vRWUAbYLGHBZS6Q8Msb2sfnf1fvJf+47t8l/TwOerM2qArzy+IeNMTHrYLHXh95h8MoatPHI5hhSZNs+mGXKPg==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.62.0': resolution: {integrity: sha512-c00T5SYENHAt86cfW47URaP3Us5vLC/4QO7GYud1G5VNRffCwwCuBspwqYrriuJB+5m0WFzClCn9wed0FBjKvg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.62.0': resolution: {integrity: sha512-krrCDilhXOwFkSkO3Wm9I/f9H0L92XHHwy2fwxjukxIbh0dem8gZqOW5Y8BsHrpJv5qwlRBV+Wl4ZFyRWhUpwg==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.62.0': resolution: {integrity: sha512-7pfYFSTc4/rUC/FtAI0Qp6QthDBCIi6/AuP1xYqFk5vanI6KnL5dWKP60OM/05LOsbwTmIcvr6eXC4CJuJ75IA==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.62.0': resolution: {integrity: sha512-7SDIalKeIpG0Ifogbbdn58HmSotYMlf23K3dCJEmiVd9Fg36Vmni82iPQec27N3wY4Bvbxftkxz6vSx9OcouTg==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.62.0': resolution: {integrity: sha512-eRZevouTH2i1HeAVLqJuLnt256krQkGY0TN6WsTmsIhuzbh457HuWDMakKwmi0Cjadux983CoSr8Lim2QhUIFw==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.62.0': resolution: {integrity: sha512-3oVS7FLGa4U1qcvao9ylGxrjXZyUQqR8UwxEcnUEyPX53O/C/mKDZegNXTdHCP+h3e6ta/f1EN38Yif1mmZHYg==} @@ -2327,28 +2304,24 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.3.1': resolution: {integrity: sha512-Bwv9KwOvE0VKa86xPFif9b9c3Y1NxOV1P0gLti/IYaWEsQYZXDlxfGEtA8mdDZ7SG3wyNXAWYT5SIn3giL57oA==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.3.1': resolution: {integrity: sha512-Ymi8O8T15HYQdOUWUtTI6ldN0neHP85FC+Qz32xTcZ7iJXtem/x8ITev0o1e9e5rkqj4lONZfTRLvkmin1+tKg==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.3.1': resolution: {integrity: sha512-M+P/91qJ6uILLw4k2G93GMDRAXj61SMvFQYt39AqvUqYgExXpLL5aepfns7sj4HiAQeolirQF9E0lzRvdf4zPQ==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.3.1': resolution: {integrity: sha512-zsM8uOeqvVGHsAXsJxsT28ttosFahLJKCLOTUBqRAtKnVgGSRitds9T432QiT8b77Yga7JIBkulIRRlJPtYhRA==} @@ -2454,6 +2427,9 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + '@types/mocha@10.0.10': + resolution: {integrity: sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q==} + '@types/node@25.9.3': resolution: {integrity: sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==} @@ -2606,61 +2582,51 @@ packages: resolution: {integrity: sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==} cpu: [arm64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.12.2': resolution: {integrity: sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==} cpu: [arm64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-loong64-gnu@1.12.2': resolution: {integrity: sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==} cpu: [loong64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-loong64-musl@1.12.2': resolution: {integrity: sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==} cpu: [loong64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.12.2': resolution: {integrity: sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==} cpu: [ppc64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.12.2': resolution: {integrity: sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==} cpu: [riscv64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.12.2': resolution: {integrity: sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==} cpu: [riscv64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.12.2': resolution: {integrity: sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==} cpu: [s390x] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.12.2': resolution: {integrity: sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==} cpu: [x64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.12.2': resolution: {integrity: sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==} cpu: [x64] os: [linux] - libc: [musl] '@unrs/resolver-binding-openharmony-arm64@1.12.2': resolution: {integrity: sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==} @@ -2785,6 +2751,10 @@ packages: resolution: {integrity: sha512-HdzDrRs+ywAqbXGKqe1i/bLtCv47plz4TvsHFH3j729OooT5VH38ctFn5aLXgECmiAKDkmH/A6kOq2Zh5DIxww==} engines: {node: '>=18.20.0'} + '@wdio/mocha-framework@9.28.0': + resolution: {integrity: sha512-UdkgigdyxJu0Rpx2ZRSNuB8u8B5nmLBeRjn5JHYTXBp8Ubi+wgFR1+EdLg8k2z+CtuYe3bNHBw7vPpBYuQMLZw==} + engines: {node: '>=18.20.0'} + '@wdio/protocols@8.40.3': resolution: {integrity: sha512-wK7+eyrB3TAei8RwbdkcyoNk2dPu+mduMBOdPJjp8jf/mavd15nIUXLID1zA+w5m1Qt1DsT1NbvaeO9+aJQ33A==} @@ -5262,28 +5232,24 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -7644,7 +7610,7 @@ snapshots: '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -7803,7 +7769,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/template': 7.29.7 '@babel/types': 7.29.7 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -8484,7 +8450,7 @@ snapshots: '@eslint/config-array@0.23.5': dependencies: '@eslint/object-schema': 3.0.5 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) minimatch: 10.2.5 transitivePeerDependencies: - supports-color @@ -9168,7 +9134,7 @@ snapshots: '@puppeteer/browsers@2.13.2': dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.5.0 @@ -9514,6 +9480,8 @@ snapshots: '@types/json5@0.0.29': {} + '@types/mocha@10.0.10': {} + '@types/node@25.9.3': dependencies: undici-types: 7.24.6 @@ -9584,7 +9552,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) eslint: 10.5.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: @@ -9594,7 +9562,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -9613,7 +9581,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) eslint: 10.5.0(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 @@ -9628,7 +9596,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) minimatch: 10.2.5 semver: 7.8.4 tinyglobby: 0.2.17 @@ -9767,14 +9735,6 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': - dependencies: - '@vitest/spy': 4.1.9 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) - '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 @@ -9958,6 +9918,20 @@ snapshots: safe-regex2: 5.1.1 strip-ansi: 7.2.0 + '@wdio/mocha-framework@9.28.0': + dependencies: + '@types/mocha': 10.0.10 + '@types/node': 25.9.3 + '@wdio/logger': 9.18.0 + '@wdio/types': 9.28.0 + '@wdio/utils': 9.28.0 + mocha: 10.8.2 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + '@wdio/protocols@8.40.3': {} '@wdio/protocols@9.28.0': {} @@ -10084,7 +10058,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -11515,7 +11489,7 @@ snapshots: '@types/estree': 1.0.9 ajv: 6.15.0 cross-spawn: 7.0.6 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) escape-string-regexp: 4.0.0 eslint-scope: 9.1.2 eslint-visitor-keys: 5.0.1 @@ -11632,7 +11606,7 @@ snapshots: extract-zip@2.0.1: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -11963,7 +11937,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 6.0.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -11971,7 +11945,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 8.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12167,35 +12141,35 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color http-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12492,7 +12466,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -13578,7 +13552,7 @@ snapshots: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-uri: 6.0.5 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 @@ -13590,7 +13564,7 @@ snapshots: pac-proxy-agent@9.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-uri: 8.0.0 http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 @@ -13880,7 +13854,7 @@ snapshots: proxy-agent@6.5.0: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 @@ -13893,7 +13867,7 @@ snapshots: proxy-agent@8.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 lru-cache: 7.18.3 @@ -14413,7 +14387,7 @@ snapshots: socks-proxy-agent@10.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14421,7 +14395,7 @@ snapshots: socks-proxy-agent@8.0.5: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14630,7 +14604,7 @@ snapshots: cosmiconfig: 9.0.2(typescript@6.0.3) css-functions-list: 3.3.3 css-tree: 3.2.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) fast-glob: 3.3.3 fastest-levenshtein: 1.0.16 file-entry-cache: 11.1.3 @@ -14893,7 +14867,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) esbuild: 0.27.7 fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 @@ -15026,7 +15000,7 @@ snapshots: '@rollup/pluginutils': 5.4.0(rollup@4.62.0) '@volar/typescript': 2.4.28 compare-versions: 6.1.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) kolorist: 1.8.0 local-pkg: 1.2.1 magic-string: 0.30.21 @@ -15169,21 +15143,6 @@ snapshots: optionalDependencies: rollup: 4.62.0 - vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): - dependencies: - lightningcss: 1.32.0 - picomatch: 4.0.4 - postcss: 8.5.15 - rolldown: 1.0.3 - tinyglobby: 0.2.17 - optionalDependencies: - '@types/node': 25.9.3 - esbuild: 0.27.7 - fsevents: 2.3.3 - jiti: 2.7.0 - tsx: 4.22.4 - yaml: 2.9.0 - vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 @@ -15199,36 +15158,6 @@ snapshots: tsx: 4.22.4 yaml: 2.9.0 - vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): - dependencies: - '@vitest/expect': 4.1.9 - '@vitest/mocker': 4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) - '@vitest/pretty-format': 4.1.9 - '@vitest/runner': 4.1.9 - '@vitest/snapshot': 4.1.9 - '@vitest/spy': 4.1.9 - '@vitest/utils': 4.1.9 - es-module-lexer: 2.1.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.3 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.1.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 - tinyglobby: 0.2.17 - tinyrainbow: 3.1.0 - vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 25.9.3 - '@vitest/coverage-v8': 4.1.9(@vitest/browser@4.1.9)(vitest@4.1.9) - happy-dom: 20.10.4 - jsdom: 24.1.3 - transitivePeerDependencies: - - msw - vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 @@ -15271,7 +15200,7 @@ snapshots: dependencies: chalk: 4.1.2 commander: 9.5.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color From 174f9b55381cf9e661f3c7f0e81e7058ceb92c3c Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 12:09:49 +0530 Subject: [PATCH 20/91] fix(core): drop dependency-internal assertions from the trace --- packages/core/src/assert-patcher.ts | 5 ++ packages/core/tests/assert-patcher.test.ts | 62 +++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index ab30c1f7..8917a1ef 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -57,6 +57,11 @@ function makeAssertEmitters( onCommand: (cmd: CapturedAssert) => void ): { passed: () => void; failed: (err: unknown) => void } { const callInfo = getCallSourceFromStack() + // No user-code frame means the assert came from a dependency or framework + // internal, not the user's test — drop it so it never reaches the trace. + if (callInfo.filePath === undefined) { + return { passed: () => {}, failed: () => {} } + } const startedAt = Date.now() const sanitizedArgs = args.map(safeSerializeAssertArg) const emit = (result: 'passed' | undefined, error: Error | undefined) => diff --git a/packages/core/tests/assert-patcher.test.ts b/packages/core/tests/assert-patcher.test.ts index 641c2e59..86774967 100644 --- a/packages/core/tests/assert-patcher.test.ts +++ b/packages/core/tests/assert-patcher.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, vi } from 'vitest' import assert from 'node:assert' import { ASSERT_PATCHED_SYMBOL, @@ -8,6 +8,19 @@ import { safeSerializeAssertArg, type CapturedAssert } from '../src/assert-patcher.js' +import { getCallSourceFromStack } from '../src/stack.js' + +// Stub the call-source resolver so tests choose whether an assert looks like +// it originated in user code (a frame) or a dependency/internal (no frame). +vi.mock('../src/stack.js', () => ({ + getCallSourceFromStack: vi.fn() +})) + +const USER_FRAME = { + filePath: '/specs/login.ts', + callSource: '/specs/login.ts:12:3' +} +const INTERNAL_FRAME = { filePath: undefined, callSource: 'unknown:0' } // Snapshot real methods once so each test starts from a fresh assert. const ASSERT_MUT = assert as unknown as Record @@ -21,6 +34,8 @@ beforeEach(() => { for (const m of TRACKED_ASSERT_METHODS) { ASSERT_MUT[m] = originals[m] } + // Default: every assert looks like it came from user code so captures fire. + vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) }) describe('safeSerializeAssertArg', () => { @@ -68,6 +83,51 @@ describe('patchNodeAssert', () => { expect(results).toEqual(['passed', undefined]) // first resolved, second rejected }) + // Only assertions that originate in user test code should reach the trace. + // Dependency/framework-internal asserts have no user-code frame on the + // stack (getCallSourceFromStack yields 'unknown:0') and must be dropped. + describe('user-origin filtering', () => { + it('drops a passing assert that has no user-code frame', () => { + vi.mocked(getCallSourceFromStack).mockReturnValue(INTERNAL_FRAME) + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + assert.equal(1, 1) + expect(captured).toHaveLength(0) + }) + + it('drops a failing internal assert but still re-throws', () => { + vi.mocked(getCallSourceFromStack).mockReturnValue(INTERNAL_FRAME) + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + expect(() => assert.equal(1, 2)).toThrow() + expect(captured).toHaveLength(0) + }) + + it('drops an internal async assert (rejects/doesNotReject)', async () => { + vi.mocked(getCallSourceFromStack).mockReturnValue(INTERNAL_FRAME) + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + await assert.doesNotReject(async () => 1) + await expect(assert.rejects(async () => 1)).rejects.toThrow() + expect(captured).toHaveLength(0) + }) + + it('keeps a user-origin assert and carries its callSource through', () => { + vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + assert.equal(1, 1) + expect(() => assert.equal(1, 2)).toThrow() + expect(captured).toHaveLength(2) + expect(captured[0]).toMatchObject({ + result: 'passed', + callSource: USER_FRAME.callSource + }) + expect(captured[1].result).toBeUndefined() + expect(captured[1].error).toBeInstanceOf(Error) + }) + }) + it('is idempotent — second patch is a no-op and sets the guard symbol', () => { const a: CapturedAssert[] = [] patchNodeAssert((c) => a.push(c)) From cc4f595048083b3284cd6ffebbccb26a2f633639 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 12:40:08 +0530 Subject: [PATCH 21/91] fix(service): give Cucumber scenario-outline rows distinct trace groups --- packages/service/src/action-snapshot.ts | 28 +++- packages/service/src/index.ts | 126 +++++++----------- packages/service/src/test-metadata.ts | 13 ++ packages/service/tests/trace-metadata.test.ts | 21 ++- 4 files changed, 108 insertions(+), 80 deletions(-) diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts index 96cf60b4..12710114 100644 --- a/packages/service/src/action-snapshot.ts +++ b/packages/service/src/action-snapshot.ts @@ -8,9 +8,13 @@ // library. No external input reaches new Function() — the lint flag here is // a false positive given the closed input set. -import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' +import { + captureActionSnapshot as coreCapture, + mapCommandToAction +} from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' import { isNativeMobile, mobilePlatform } from './mobile.js' +import { INTERNAL_COMMANDS } from './constants.js' function reviveScript(src: string): () => unknown { // `src` from core/element-scripts is already a self-invoking IIFE @@ -55,6 +59,28 @@ export async function waitForActionResult( await browser.pause(250).catch(() => undefined) } +/** Post-action capture: settle the resulting page, screenshot it, and push the + * snapshot stamped at the latest logged action. No-op for internal/non-mapped + * commands. Skipped by the caller outside trace mode. */ +export async function captureActionResult( + browser: WebdriverIO.Browser, + command: string, + actionSnapshots: ActionSnapshot[], + stampTimestamp: () => number +): Promise { + if (!mapCommandToAction(command) || INTERNAL_COMMANDS.includes(command)) { + return + } + if (!isNativeMobile(browser)) { + await waitForActionResult(browser) + } + const snap = await captureActionSnapshot(browser, command) + if (snap) { + snap.timestamp = stampTimestamp() + actionSnapshots.push(snap) + } +} + export function captureActionSnapshot( browser: WebdriverIO.Browser, command: string diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index f3388600..c2af6580 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -1,7 +1,6 @@ /// import logger from '@wdio/logger' import { - deterministicUid, errorMessage, finalizeScreencast, finalizeTraceExport, @@ -14,11 +13,15 @@ import { type TraceExportContext } from '@wdio/devtools-core' import { captureExpectFailure, wireAssertCapture } from './assert-capture.js' -import { stampTestState, testMetadataUid } from './test-metadata.js' +import { + cucumberScenarioUid, + stampTestState, + testMetadataUid +} from './test-metadata.js' import { resolveCallSourceFromFrame } from './call-source.js' import { - captureActionSnapshot, - waitForActionResult + captureActionResult, + captureActionSnapshot } from './action-snapshot.js' import { dedupeSnapshotsByTimestamp, @@ -128,6 +131,21 @@ export default class DevToolsHookService implements Services.ServiceInstance { ) } + /** Record a spec boundary (spec granularity) and flush the previous spec. */ + #recordBoundary(specFile: string | undefined): void { + if (!specFile) { + return + } + const prevRange = recordSpecBoundary( + this.#boundaryContext, + specFile, + this.#options.traceGranularity + ) + if (prevRange) { + this.#fireAndForgetFlush(prevRange) + } + } + /** Assemble the framework-agnostic trace-export context from this service's * state. Output dir ignores the spec range — WDIO writes next to config. */ #traceContext(browser: WebdriverIO.Browser): TraceExportContext { @@ -286,33 +304,24 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** - * Hook for Cucumber framework. - * Detects feature-file boundaries for per-spec tracing and tags commands - * with a stable testUid so tracingGroup spans render in the trace output. - * WDIO passes the Cucumber World as the first argument. - */ - beforeScenario(world?: { pickle?: { uri?: string; name?: string } }) { + /** Cucumber hook: records feature-file boundaries and tags commands with a stable testUid. */ + beforeScenario(world?: { + pickle?: { uri?: string; name?: string; astNodeIds?: readonly string[] } + }) { this.resetStack() const featureFile = world?.pickle?.uri const scenarioName = world?.pickle?.name - // ── Per-spec boundary detection (Cucumber) ── - if (featureFile) { - const prevRange = recordSpecBoundary( - this.#boundaryContext, - featureFile, - this.#options.traceGranularity - ) - if (prevRange) { - this.#fireAndForgetFlush(prevRange) - } - } + this.#recordBoundary(featureFile) // ── Test identity for command tagging ── if (featureFile && scenarioName) { - const uid = deterministicUid(featureFile, scenarioName) + const uid = cucumberScenarioUid( + featureFile, + scenarioName, + world?.pickle?.astNodeIds + ) this.#currentTestUid = uid this.#testMetadata.set(uid, { title: scenarioName, @@ -321,28 +330,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** - * Hook for Mocha/Jasmine frameworks. - * It does the exact same thing as beforeScenario. - */ + /** Mocha/Jasmine hook: the beforeScenario equivalent for file-based specs. */ beforeTest(test?: { file?: string; title?: string; fullTitle?: string }) { this.resetStack() - const newSpec = test?.file - - // ── Per-spec boundary detection ── - // Only tracked when traceGranularity is 'spec'. Records array index - // ranges so #flushSpecTrace can slice the accumulated data per spec. - if (newSpec) { - const prevRange = recordSpecBoundary( - this.#boundaryContext, - newSpec, - this.#options.traceGranularity - ) - if (prevRange) { - this.#fireAndForgetFlush(prevRange) - } - } + this.#recordBoundary(test?.file) // Track test identity for command tagging. Generate a stable UID // from file + title so commands can be partitioned across reruns. @@ -381,12 +373,15 @@ export default class DevToolsHookService implements Services.ServiceInstance { } async afterScenario( - world?: { pickle?: { uri?: string; name?: string } }, + world?: { + pickle?: { uri?: string; name?: string; astNodeIds?: readonly string[] } + }, result?: { error?: unknown; passed?: boolean; skipped?: boolean } ) { - const { uri, name } = world?.pickle ?? {} + const { uri, name, astNodeIds } = world?.pickle ?? {} if (uri && name) { - stampTestState(this.#testMetadata, deterministicUid(uri, name), result) + const uid = cucumberScenarioUid(uri, name, astNodeIds) + stampTestState(this.#testMetadata, uid, result) } await this.#finalizePerScenario() } @@ -434,38 +429,6 @@ export default class DevToolsHookService implements Services.ServiceInstance { return Date.now() } - // Post-action capture: the state RESULTING from the action just completed. - // The pre-action capture in beforeCommand only records the state before the - // NEXT mapped action — when intervening commands (assertions, reloadSession) - // change the page first, an action's result (e.g. the page a click navigated - // to) is never captured. - // - // readyState alone is unreliable: right after a click the OLD document still - // reports 'complete', so a naive wait snapshots a blank mid-navigation frame. - // Instead, beforeCommand tags the document; if the tag is gone the action - // navigated, so we wait for the NEW document to finish loading AND render - // content before screenshotting its destination. Stamped at this command's - // own end (the latest logged action). - async #captureActionResult(command: string): Promise { - if ( - this.#options.mode !== 'trace' || - !this.#browser || - !mapCommandToAction(command) || - INTERNAL_COMMANDS.includes(command) - ) { - return - } - const browser = this.#browser - if (!isNativeMobile(browser)) { - await waitForActionResult(browser) - } - const snap = await captureActionSnapshot(browser, command) - if (snap) { - snap.timestamp = this.#lastActionTimestamp() - this.#actionSnapshots.push(snap) - } - } - private resetStack() { this.#commandStack = [] this.#sessionCapturer.resetLastSelector() @@ -574,7 +537,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { frame.startTimestamp, this.#currentTestUid ) - await this.#captureActionResult(command) + if (this.#options.mode === 'trace') { + await captureActionResult( + this.#browser, + command, + this.#actionSnapshots, + () => this.#lastActionTimestamp() + ) + } return captured } } diff --git a/packages/service/src/test-metadata.ts b/packages/service/src/test-metadata.ts index bf802762..339e1dc1 100644 --- a/packages/service/src/test-metadata.ts +++ b/packages/service/src/test-metadata.ts @@ -13,6 +13,19 @@ export function testMetadataUid( return file ? deterministicUid(file, title) : title } +/** Scenario key that separates scenario-outline example rows: they share a + * name, so the pickle's astNodeIds (distinct per row, stable across reruns) + * are folded in. beforeScenario and afterScenario derive it identically. */ +export function cucumberScenarioUid( + uri: string, + name: string, + astNodeIds?: readonly string[] +): string { + return astNodeIds?.length + ? deterministicUid(uri, name, astNodeIds.join(':')) + : deterministicUid(uri, name) +} + /** Canonical test state from a WDIO afterTest/afterScenario result. */ export function resultToState(result: { passed?: boolean diff --git a/packages/service/tests/trace-metadata.test.ts b/packages/service/tests/trace-metadata.test.ts index 044067c2..fe88dc1a 100644 --- a/packages/service/tests/trace-metadata.test.ts +++ b/packages/service/tests/trace-metadata.test.ts @@ -1,7 +1,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import type * as DevtoolsCore from '@wdio/devtools-core' import { deterministicUid } from '@wdio/devtools-core' -import { resultToState, testMetadataUid } from '../src/test-metadata.js' +import { + cucumberScenarioUid, + resultToState, + testMetadataUid +} from '../src/test-metadata.js' // Captures the ctx handed to finalizeTraceExport so the test can inspect the // state stamped onto testMetadata and the policy that flowed in. @@ -36,6 +40,7 @@ vi.mock('../src/session.js', () => ({ // Keep the after* hooks from touching a real browser/CDP. vi.mock('../src/action-snapshot.js', () => ({ captureActionSnapshot: vi.fn().mockResolvedValue(null), + captureActionResult: vi.fn().mockResolvedValue(undefined), waitForActionResult: vi.fn().mockResolvedValue(undefined) })) @@ -66,6 +71,20 @@ describe('test-metadata helpers', () => { ) expect(testMetadataUid(undefined, 'renders')).toBe('renders') }) + + it('cucumberScenarioUid separates outline rows sharing a name by astNodeIds', () => { + const uri = '/proj/features/login.feature' + const row1 = cucumberScenarioUid(uri, 'log in', ['node-1']) + const row2 = cucumberScenarioUid(uri, 'log in', ['node-2']) + // Distinct example rows → distinct uids (so they render as separate groups). + expect(row1).not.toBe(row2) + // A rerun of the same row → same uid (retry-coalescing stays intact). + expect(cucumberScenarioUid(uri, 'log in', ['node-1'])).toBe(row1) + // No astNodeIds → plain name-based uid. + expect(cucumberScenarioUid(uri, 'log in')).toBe( + deterministicUid(uri, 'log in') + ) + }) }) describe('DevtoolsService - afterTest state stamping', () => { From d25d39fbb0374c5f781286cc7e7fd107b56fe50e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Wed, 8 Jul 2026 14:04:09 +0530 Subject: [PATCH 22/91] fix(service): show waits as actions and suppress their internal polling --- examples/wdio/cucumber/wdio.conf.ts | 6 +++-- examples/wdio/mocha/wdio.conf.ts | 6 +++-- packages/service/src/constants.ts | 2 -- packages/service/tests/index.test.ts | 33 ++++++++++++++++++---------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/examples/wdio/cucumber/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts index b6305819..68f7ee8b 100644 --- a/examples/wdio/cucumber/wdio.conf.ts +++ b/examples/wdio/cucumber/wdio.conf.ts @@ -54,7 +54,9 @@ export const config: Options.Testrunner = { // and 30 processes will get spawned. The property handles how many capabilities // from the same test should run tests. // - maxInstances: 10, + // Live mode drives a single-session dashboard; >1 worker streams two feature + // files into it at once and neither renders cleanly. One instance = readable demo. + maxInstances: 1, // // If you have trouble getting all important capabilities together, check out the // Sauce Labs platform configurator - a great tool to configure your capabilities: @@ -131,7 +133,7 @@ export const config: Options.Testrunner = { [ 'devtools', { - mode: 'trace' as const, + mode: 'live' as const, screencast: { enabled: true, pollIntervalMs: 200 } } ] diff --git a/examples/wdio/mocha/wdio.conf.ts b/examples/wdio/mocha/wdio.conf.ts index 474602e3..7ccee43f 100644 --- a/examples/wdio/mocha/wdio.conf.ts +++ b/examples/wdio/mocha/wdio.conf.ts @@ -13,7 +13,9 @@ export const config: Options.Testrunner = { }, specs: ['./specs/**/*.e2e.ts'], exclude: [], - maxInstances: 10, + // Live mode drives a single-session dashboard; >1 worker streams two sessions + // into it at once and neither renders cleanly. One instance = readable demo. + maxInstances: 1, capabilities: [ { browserName: 'chrome', @@ -37,7 +39,7 @@ export const config: Options.Testrunner = { [ 'devtools', { - mode: 'trace' as const, + mode: 'live' as const, traceGranularity: 'spec' as const // tracePolicy: 'retain-on-failure' as const // screencast: { enabled: true, pollIntervalMs: 200 } diff --git a/packages/service/src/constants.ts b/packages/service/src/constants.ts index 46e38878..a30f6e3a 100644 --- a/packages/service/src/constants.ts +++ b/packages/service/src/constants.ts @@ -43,7 +43,6 @@ export const INTERNAL_COMMANDS = [ 'emit', 'browsingContextLocateNodes', 'browsingContextNavigate', - 'waitUntil', 'getTitle', 'getUrl', 'getWindowSize', @@ -51,7 +50,6 @@ export const INTERNAL_COMMANDS = [ 'deleteSession', 'findElementFromShadowRoot', 'findElementsFromShadowRoot', - 'waitForExist', 'browsingContextGetTree', 'scriptCallFunction', 'getElement', diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index 0ce30b79..5ce1e1f5 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -106,13 +106,7 @@ describe('DevtoolsService - Internal Command Filtering', () => { describe('beforeCommand', () => { it('should not add internal commands to command stack', () => { - const internalCommands = [ - 'getTitle', - 'waitUntil', - 'getUrl', - 'execute', - 'findElement' - ] + const internalCommands = ['getTitle', 'getUrl', 'execute', 'findElement'] internalCommands.forEach((cmd) => service.beforeCommand(cmd as any, [])) expect(true).toBe(true) }) @@ -138,19 +132,34 @@ describe('DevtoolsService - Internal Command Filtering', () => { executeCommand('url', ['https://example.com']) executeCommand('getTitle', [], 'Page Title') // internal executeCommand('click', ['.button']) - executeCommand('waitUntil', [expect.any(Function)], true) // internal + executeCommand('waitUntil', [expect.any(Function)], true) // a user wait executeCommand('getText', ['.result'], 'Success') - // Only user commands (url, click, getText) should be captured - expect(mockSessionCapturerInstance.afterCommand).toHaveBeenCalledTimes(3) + // getTitle is internal; the rest — including the wait — are user actions. + expect(mockSessionCapturerInstance.afterCommand).toHaveBeenCalledTimes(4) const capturedCommands = mockSessionCapturerInstance.afterCommand.mock.calls.map( (call) => call[1] ) - expect(capturedCommands).toEqual(['url', 'click', 'getText']) + expect(capturedCommands).toEqual(['url', 'click', 'waitUntil', 'getText']) expect(capturedCommands).not.toContain('getTitle') - expect(capturedCommands).not.toContain('waitUntil') + }) + + it('captures a wait once and suppresses the commands it polls', () => { + // waitUntil opens; its predicate polls isDisplayed while the wait is + // still on the stack, so those polls are top-level-suppressed. + service.beforeCommand('waitUntil' as any, [expect.any(Function)]) + service.beforeCommand('isDisplayed' as any, []) + service.afterCommand('isDisplayed' as any, [], false) + service.beforeCommand('isDisplayed' as any, []) + service.afterCommand('isDisplayed' as any, [], true) + service.afterCommand('waitUntil' as any, [expect.any(Function)], true) + + const captured = mockSessionCapturerInstance.afterCommand.mock.calls.map( + (call) => call[1] + ) + expect(captured).toEqual(['waitUntil']) }) // Service-fired commands (preload injection, Puppeteer handle for CDP, From 6aeb37b2223403630d44c39f1ba28bc27cba71a7 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:47:46 +0530 Subject: [PATCH 23/91] feat(core,adapters): capture per-test retry attempts for retry-aware policies --- packages/core/src/attempt-tracker.ts | 39 +++++ packages/core/src/index.ts | 1 + packages/core/src/trace-finalizer.ts | 11 +- packages/core/tests/attempt-tracker.test.ts | 46 ++++++ .../src/cucumber-lifecycle.ts | 26 ++-- .../src/helpers/cucumberScenarioBuilder.ts | 13 +- packages/nightwatch-devtools/src/index.ts | 53 ++++--- .../src/plugin-internals.ts | 6 + packages/nightwatch-devtools/src/reporter.ts | 22 ++- .../nightwatch-devtools/src/session-init.ts | 14 +- .../nightwatch-devtools/src/test-lifecycle.ts | 3 + .../nightwatch-devtools/src/trace-context.ts | 60 ++++++++ .../tests/attemptTracking.test.ts | 134 ++++++++++++++++++ .../src/helpers/testManager.ts | 21 ++- packages/selenium-devtools/src/index.ts | 54 +++---- .../src/runnerHooks/mocha.ts | 3 +- .../src/session-lifecycle.ts | 5 +- .../selenium-devtools/src/test-management.ts | 59 ++++++-- packages/selenium-devtools/src/types.ts | 51 ++----- packages/service/src/index.ts | 53 +++++-- packages/service/src/reporter.ts | 4 + packages/service/src/test-metadata.ts | 32 ++++- packages/service/tests/trace-metadata.test.ts | 51 +++++++ 23 files changed, 606 insertions(+), 155 deletions(-) create mode 100644 packages/core/src/attempt-tracker.ts create mode 100644 packages/core/tests/attempt-tracker.test.ts create mode 100644 packages/nightwatch-devtools/src/trace-context.ts create mode 100644 packages/nightwatch-devtools/tests/attemptTracking.test.ts diff --git a/packages/core/src/attempt-tracker.ts b/packages/core/src/attempt-tracker.ts new file mode 100644 index 00000000..e4915584 --- /dev/null +++ b/packages/core/src/attempt-tracker.ts @@ -0,0 +1,39 @@ +/** + * Framework-agnostic per-test attempt counting. Every supported runner + * re-enters its per-test start hook when a test is retried, so recording the + * same uid again yields an incremented attempt number (0-based: the first run + * is attempt 0, the first retry is attempt 1). This is the primary, + * runner-independent retry signal feeding `TestOutcome.attempt` for the + * retry-aware trace policies (see trace-retention.ts). + */ +export class TestAttemptTracker { + #attempts = new Map() + #sawRetry = false + + /** Record a starting test; returns its attempt number (0 first, +1 per rerun). */ + recordStart(uid: string): number { + const prior = this.#attempts.get(uid) + const attempt = prior === undefined ? 0 : prior + 1 + this.#attempts.set(uid, attempt) + if (attempt > 0) { + this.#sawRetry = true + } + return attempt + } + + /** Latest attempt recorded for `uid`, or undefined if it never started. */ + attemptFor(uid: string): number | undefined { + return this.#attempts.get(uid) + } + + /** True once any test has started more than once (a retry occurred). */ + get sawRetry(): boolean { + return this.#sawRetry + } + + /** Clear all state — used at session boundaries and reuse-mode reconnects. */ + reset(): void { + this.#attempts.clear() + this.#sawRetry = false + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index db5b315d..3c3325e2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ export * from './action-mapping.js' export * from './action-snapshot.js' +export * from './attempt-tracker.js' export * from './with-timeout.js' export * from './assert-patcher.js' export * from './element-snapshot.js' diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index f0de3023..1750d3a4 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -40,6 +40,9 @@ export interface TraceExportContext { mode?: DevToolsMode /** undefined → treated as `on` (always retain) by the retention evaluator. */ policy?: TraceRetentionPolicy + /** True when the adapter fed real per-test attempt numbers (B4); retry-aware + * policies degrade to retain-on-failure when this is false. */ + attemptInfoAvailable?: boolean granularity?: TraceGranularity format?: TraceFormat capturer: TraceCapturer @@ -75,9 +78,9 @@ function toOutcomes(metadata: TestMetadataMap): TestOutcome[] { } /** - * Evaluate the retention policy for one trace slice. B4 supplies real attempt - * info; today `attemptInfoAvailable` is false and the default `on` policy - * always retains — but the decision is wired now so B3 only flips the option. + * Evaluate the retention policy for one trace slice. Adapters that feed real + * per-test attempt numbers also set `attemptInfoAvailable`; when they don't, + * retry-aware policies degrade to retain-on-failure (see trace-retention.ts). */ function shouldRetain( ctx: TraceExportContext, @@ -85,7 +88,7 @@ function shouldRetain( ): boolean { return shouldRetainTrace(ctx.policy, { outcomes: toOutcomes(metadata), - attemptInfoAvailable: false + attemptInfoAvailable: ctx.attemptInfoAvailable ?? false }).retain } diff --git a/packages/core/tests/attempt-tracker.test.ts b/packages/core/tests/attempt-tracker.test.ts new file mode 100644 index 00000000..5d9a44b4 --- /dev/null +++ b/packages/core/tests/attempt-tracker.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest' +import { TestAttemptTracker } from '../src/attempt-tracker.js' + +describe('TestAttemptTracker', () => { + it('numbers the first start 0 and each rerun +1', () => { + const t = new TestAttemptTracker() + expect(t.recordStart('a')).toBe(0) + expect(t.recordStart('a')).toBe(1) + expect(t.recordStart('a')).toBe(2) + }) + + it('tracks attempts per uid independently', () => { + const t = new TestAttemptTracker() + expect(t.recordStart('a')).toBe(0) + expect(t.recordStart('b')).toBe(0) + expect(t.recordStart('a')).toBe(1) + expect(t.attemptFor('a')).toBe(1) + expect(t.attemptFor('b')).toBe(0) + }) + + it('attemptFor is undefined for an unseen uid', () => { + const t = new TestAttemptTracker() + expect(t.attemptFor('missing')).toBeUndefined() + }) + + it('sawRetry flips only after a second start of some test', () => { + const t = new TestAttemptTracker() + expect(t.sawRetry).toBe(false) + t.recordStart('a') + t.recordStart('b') + expect(t.sawRetry).toBe(false) + t.recordStart('a') + expect(t.sawRetry).toBe(true) + }) + + it('reset clears attempts and the retry flag', () => { + const t = new TestAttemptTracker() + t.recordStart('a') + t.recordStart('a') + expect(t.sawRetry).toBe(true) + t.reset() + expect(t.sawRetry).toBe(false) + expect(t.attemptFor('a')).toBeUndefined() + expect(t.recordStart('a')).toBe(0) + }) +}) diff --git a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts index afb8a832..201e2ca0 100644 --- a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts +++ b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts @@ -13,7 +13,11 @@ import logger from '@wdio/logger' import { errorMessage } from '@wdio/devtools-core' -import { WS_SCOPE } from '@wdio/devtools-shared' +import { + WS_SCOPE, + type CucumberPickle, + type CucumberPickleStep +} from '@wdio/devtools-shared' import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' @@ -32,20 +36,8 @@ import { parseCucumberScenario } from './helpers/utils.js' const log = logger('@wdio/nightwatch-devtools:cucumber') -/** Minimal shapes for the Cucumber objects we touch. Cucumber's own types - * vary across major versions; we pin only fields we read. */ -export interface CucumberPickleStep { - text?: string - astNodeIds?: string[] - location?: { line?: number } -} -export interface CucumberPickle { - uri?: string - name?: string - location?: { line?: number } - astNodeIds?: string[] - steps?: CucumberPickleStep[] -} +/** Cucumber's result shape varies across major versions; we pin only the + * status field we read. Pickle shapes come from shared. */ export interface CucumberResult { status?: string } @@ -66,6 +58,7 @@ export interface CucumberLifecycleCtx { setCurrentStep(s: unknown): void getCurrentStep(): unknown setCurrentTest(t: unknown): void + recordAttempt(uid: string): number } type MutStep = { @@ -178,7 +171,8 @@ export async function initCucumberScenario( stepLines, stepKeywords, scenarioLine, - parentFeatureSuiteUid: featureSuite.uid + parentFeatureSuiteUid: featureSuite.uid, + recordAttempt: (uid) => ctx.recordAttempt(uid) }) attachScenarioToFeature(ctx, featureSuite, scenarioSuite) ctx.setCurrentScenarioSuite(scenarioSuite) diff --git a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts index 54ab1da3..b60d5ad4 100644 --- a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts +++ b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts @@ -21,6 +21,9 @@ export interface CucumberScenarioBuildInput { scenarioLine: number /** Parent feature-suite uid — scenarios nest under this. */ parentFeatureSuiteUid: string + /** Records the scenario start under its (retry-stable) uid and returns the + * 0-based attempt number stamped on every step. Omitted → attempt 0. */ + recordAttempt?: (scenarioUid: string) => number } /** @@ -36,6 +39,7 @@ export interface CucumberScenarioBuildInput { function buildScenarioStepTest( input: CucumberScenarioBuildInput, scenarioUid: string, + attempt: number, i: number ): SuiteStats['tests'][number] { const { @@ -69,7 +73,9 @@ function buildScenarioStepTest( end: null, type: 'test' as const, file: featureUri, - retries: 0, + // Scenario-level attempt from the tracker (0 first run, +1 per retry); + // flows to TestOutcome.attempt via collectSuiteTestMetadata. + retries: attempt, _duration: 0, hooks: [], callSource @@ -95,6 +101,7 @@ export function buildCucumberScenarioSuite( featureUri, `scenario:${scenarioName}:${scenarioLine}` ) + const attempt = input.recordAttempt?.(scenarioUid) ?? 0 const scenarioSuite: SuiteStats = { uid: scenarioUid, cid: DEFAULTS.CID, @@ -116,7 +123,9 @@ export function buildCucumberScenarioSuite( : undefined } for (let i = 0; i < steps.length; i++) { - scenarioSuite.tests.push(buildScenarioStepTest(input, scenarioUid, i)) + scenarioSuite.tests.push( + buildScenarioStepTest(input, scenarioUid, attempt, i) + ) } return scenarioSuite } diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 41a6c404..e94877bc 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -7,20 +7,25 @@ import { fileURLToPath } from 'node:url' import { - collectSuiteTestMetadata, errorMessage, finalizeTraceExport, flushRangeTrace, recordSpecBoundary, - resolveAdapterOutputDir, + TestAttemptTracker, tracePolicyModeWarning, type SpecRange, type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' +import { buildTraceContext } from './trace-context.js' import { wireAssertCapture } from './helpers/assertCapture.js' import { stop as stopBackend } from '@wdio/devtools-backend' -import { REUSE_ENV, SCREENCAST_DEFAULTS } from '@wdio/devtools-shared' +import { + REUSE_ENV, + SCREENCAST_DEFAULTS, + type CucumberPickle, + type CucumberPickleStep +} from '@wdio/devtools-shared' import logger from '@wdio/logger' import { handleReuseMode, @@ -53,8 +58,6 @@ import { cucumberAfter as cucumberLifecycleAfter, cucumberBeforeStep as cucumberLifecycleBeforeStep, cucumberAfterStep as cucumberLifecycleAfterStep, - type CucumberPickle, - type CucumberPickleStep, type CucumberResult } from './cucumber-lifecycle.js' import { @@ -116,6 +119,10 @@ class NightwatchDevToolsPlugin { #bidiEnabled = false #bidiAttachAttempted = false + // Nightwatch `--retries` and cross-worker reruns may reset this in-process + // tracker; only retries that re-enter this process's start hook are counted. + #attemptTracker = new TestAttemptTracker() + constructor(options: DevToolsOptions = {}) { const mode = options.mode ?? 'live' const ignore = mode === 'trace' && options.screencast?.enabled === true @@ -286,7 +293,10 @@ class NightwatchDevToolsPlugin { clearExecutionData: () => { self.testReporter.clearExecutionData() self.suiteManager.clearExecutionData() + self.#attemptTracker.reset() }, + recordAttempt: (uid) => self.#attemptTracker.recordStart(uid), + attemptFor: (uid) => self.#attemptTracker.attemptFor(uid), buildMetadataOptions: () => self.#buildMetadataOptions(), ensureSessionInitialized: (b) => self.#ensureSessionInitialized(b), wrapBrowserOnce: (b) => self.#wrapBrowserOnce(b), @@ -544,24 +554,21 @@ class NightwatchDevToolsPlugin { /** Assemble the framework-agnostic trace-export context from plugin state. * Output dir ignores the spec range — nightwatch writes next to config. */ #traceContext(sessionId: string): TraceExportContext { - return { - mode: this.options.mode, - policy: this.options.tracePolicy, - granularity: this.options.traceGranularity, - format: this.options.traceFormat, - capturer: this.sessionCapturer, - actionSnapshots: this.sessionCapturer.actionSnapshots, - sessionId, - testMetadata: collectSuiteTestMetadata( - this.suiteManager.getAllSuites().values() - ), - ranges: this.#specRanges, - flushed: this.#flushedSpecs, - resolveOutputDir: () => - resolveAdapterOutputDir({ configPath: this.#configPath }), - awaitPending: this.sessionCapturer.snapshotCaptures, - log: (level, msg) => log[level](msg) - } + return buildTraceContext( + { + mode: this.options.mode, + policy: this.options.tracePolicy, + granularity: this.options.traceGranularity, + format: this.options.traceFormat, + capturer: this.sessionCapturer, + suites: this.suiteManager.getAllSuites().values(), + ranges: this.#specRanges, + flushed: this.#flushedSpecs, + configPath: this.#configPath, + log: (level, msg) => log[level](msg) + }, + sessionId + ) } async #writeTraceIfNeeded(): Promise { diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index afce58b4..1d4d45a2 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -73,4 +73,10 @@ export interface PluginInternals { testIcon(state: TestStats['state']): string setCucumberRunner(v: boolean): void getRerunLabel(): string | undefined + + /** Records a test/scenario start under its retry-stable uid; returns the + * 0-based attempt number (0 first run, +1 per rerun). */ + recordAttempt(uid: string): number + /** Latest attempt recorded for `uid`, or undefined if it never started. */ + attemptFor(uid: string): number | undefined } diff --git a/packages/nightwatch-devtools/src/reporter.ts b/packages/nightwatch-devtools/src/reporter.ts index a6e8282e..63626d27 100644 --- a/packages/nightwatch-devtools/src/reporter.ts +++ b/packages/nightwatch-devtools/src/reporter.ts @@ -1,5 +1,5 @@ import logger from '@wdio/logger' -import { TestReporterBase } from '@wdio/devtools-core' +import { TestReporterBase, type ReporterUpstream } from '@wdio/devtools-core' import { REUSE_ENV } from '@wdio/devtools-shared' import { DEFAULTS } from './constants.js' import { extractTestMetadata, generateStableUid } from './helpers/utils.js' @@ -11,6 +11,15 @@ export class TestReporter extends TestReporterBase { #currentSpecFile?: string #testNamesCache = new Map() #currentSuite?: SuiteStats + #attemptFor: (uid: string) => number | undefined + + constructor( + report: ReporterUpstream, + attemptFor: (uid: string) => number | undefined + ) { + super(report) + this.#attemptFor = attemptFor + } onSuiteStart(suiteStats: SuiteStats) { this.#currentSpecFile = suiteStats.file @@ -92,11 +101,12 @@ export class TestReporter extends TestReporterBase { cachedNames.forEach((testName) => { if (!processedTestNames.has(testName)) { + const uid = generateStableUid({ + file: suiteStats.file, + fullTitle: `${suiteStats.title} ${testName}` + } as TestStats) const skippedTest: TestStats = { - uid: generateStableUid({ - file: suiteStats.file, - fullTitle: `${suiteStats.title} ${testName}` - } as TestStats), + uid, cid: DEFAULTS.CID, title: testName, fullTitle: `${suiteStats.title} ${testName}`, @@ -106,7 +116,7 @@ export class TestReporter extends TestReporterBase { end: new Date(), type: 'test', file: suiteStats.file, - retries: DEFAULTS.RETRIES, + retries: this.#attemptFor(uid) ?? DEFAULTS.RETRIES, _duration: DEFAULTS.DURATION, hooks: [] } diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 71a1c21b..2bccfc82 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -57,6 +57,7 @@ export interface SessionInitCtx { getCurrentTest(): unknown getCurrentScenarioSuite(): SuiteStats | null buildMetadataOptions(): unknown + attemptFor(uid: string): number | undefined } async function handleSessionChange( @@ -84,11 +85,14 @@ function initReporterChain(ctx: SessionInitCtx): void { // These must NOT be recreated on session change — doing so generates a // new feature suite with a fresh start timestamp, which DataManager sees // as a new run and wipes all accumulated commands. - ctx.testReporter = new TestReporter((suitesData) => { - if (ctx.sessionCapturer) { - ctx.sessionCapturer.sendUpstream('suites', suitesData) - } - }) + ctx.testReporter = new TestReporter( + (suitesData) => { + if (ctx.sessionCapturer) { + ctx.sessionCapturer.sendUpstream('suites', suitesData) + } + }, + (uid) => ctx.attemptFor(uid) + ) ctx.testManager = new TestManager(ctx.testReporter) ctx.suiteManager = new SuiteManager(ctx.testReporter) ctx.browserProxy = new BrowserProxy( diff --git a/packages/nightwatch-devtools/src/test-lifecycle.ts b/packages/nightwatch-devtools/src/test-lifecycle.ts index 4bbd52e6..5caf0537 100644 --- a/packages/nightwatch-devtools/src/test-lifecycle.ts +++ b/packages/nightwatch-devtools/src/test-lifecycle.ts @@ -41,6 +41,7 @@ export interface TestLifecycleCtx { incrementCount(state: TestStats['state']): void testIcon(state: TestStats['state']): string setCurrentTest(t: unknown): void + recordAttempt(uid: string): number } interface SuiteMetadata { @@ -124,6 +125,8 @@ export async function startNextTest( } const test = ctx.testManager.findTestInSuite(currentSuite, currentTestName) if (test) { + // Nightwatch has no per-test retry index; the tracker is the retry signal. + test.retries = ctx.recordAttempt(test.uid) test.state = TEST_STATE.RUNNING as TestStats['state'] test.start = new Date() test.end = null diff --git a/packages/nightwatch-devtools/src/trace-context.ts b/packages/nightwatch-devtools/src/trace-context.ts new file mode 100644 index 00000000..1e451ac7 --- /dev/null +++ b/packages/nightwatch-devtools/src/trace-context.ts @@ -0,0 +1,60 @@ +/** + * Assembles the framework-agnostic trace-export context from plugin state. + * + * Extracted from `NightwatchDevToolsPlugin.#traceContext` so the assembly is + * unit-testable and to keep `index.ts` under the file-size cap. Both the + * per-spec boundary flush and the final trace write share this one builder. + */ + +import { + collectSuiteTestMetadata, + resolveAdapterOutputDir, + type SpecRange, + type TraceExportContext +} from '@wdio/devtools-core' +import type { SessionCapturer } from './session.js' +import type { + DevToolsMode, + SuiteStats, + TraceFormat, + TraceGranularity, + TraceRetentionPolicy +} from './types.js' + +export interface TraceContextInput { + mode: DevToolsMode + policy: TraceRetentionPolicy + granularity: TraceGranularity + format: TraceFormat + capturer: SessionCapturer + suites: Iterable + ranges: SpecRange[] + flushed: Set + configPath: string | undefined + log: (level: 'info' | 'warn', msg: string) => void +} + +export function buildTraceContext( + input: TraceContextInput, + sessionId: string +): TraceExportContext { + return { + mode: input.mode, + policy: input.policy, + granularity: input.granularity, + format: input.format, + capturer: input.capturer, + actionSnapshots: input.capturer.actionSnapshots, + sessionId, + testMetadata: collectSuiteTestMetadata(input.suites), + ranges: input.ranges, + flushed: input.flushed, + resolveOutputDir: () => + resolveAdapterOutputDir({ configPath: input.configPath }), + awaitPending: input.capturer.snapshotCaptures, + // Nightwatch feeds real per-test attempt numbers via TestAttemptTracker + // (B4), so retry-aware policies use per-test attempts, not the fallback. + attemptInfoAvailable: true, + log: input.log + } +} diff --git a/packages/nightwatch-devtools/tests/attemptTracking.test.ts b/packages/nightwatch-devtools/tests/attemptTracking.test.ts new file mode 100644 index 00000000..c08313c4 --- /dev/null +++ b/packages/nightwatch-devtools/tests/attemptTracking.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, vi } from 'vitest' +import { + TestAttemptTracker, + collectSuiteTestMetadata +} from '@wdio/devtools-core' +import { buildCucumberScenarioSuite } from '../src/helpers/cucumberScenarioBuilder.js' +import { buildTraceContext } from '../src/trace-context.js' +import { startNextTest, type TestLifecycleCtx } from '../src/test-lifecycle.js' +import { TEST_STATE } from '../src/constants.js' +import type { SessionCapturer } from '../src/session.js' +import type { SuiteStats, TestStats } from '../src/types.js' + +function scenarioInput(recordAttempt: (uid: string) => number) { + return { + featureUri: 'login.feature', + scenarioName: 'Valid login', + featureName: 'Login', + stepDefFiles: [], + steps: [{ text: 'I open the page' }, { text: 'I log in' }], + stepLines: [0, 0], + stepKeywords: ['Given', 'When'], + scenarioLine: 3, + parentFeatureSuiteUid: 'feature-uid', + recordAttempt + } +} + +function makeTest(uid: string, title: string): TestStats { + return { + uid, + cid: '0-0', + title, + fullTitle: title, + parent: 'suite-uid', + state: TEST_STATE.PENDING, + start: new Date(), + end: null, + type: 'test', + file: '/spec.ts', + retries: 0, + _duration: 0, + hooks: [] + } +} + +describe('cucumber scenario retry attempt', () => { + it('stamps the tracker attempt on every step and flows it to metadata.attempt', () => { + const tracker = new TestAttemptTracker() + const record = (uid: string) => tracker.recordStart(uid) + + const first = buildCucumberScenarioSuite(scenarioInput(record)) + expect((first.tests as TestStats[]).map((t) => t.retries)).toEqual([0, 0]) + + // Same scenario name + line → same (retry-stable) uid → attempt increments. + const retried = buildCucumberScenarioSuite(scenarioInput(record)) + expect((retried.tests as TestStats[]).map((t) => t.retries)).toEqual([1, 1]) + expect(tracker.sawRetry).toBe(true) + + const metadata = collectSuiteTestMetadata([retried]) + for (const step of retried.tests as TestStats[]) { + expect(metadata.get(step.uid)?.attempt).toBe(1) + } + }) + + it('defaults step attempt to 0 when no tracker is wired', () => { + const suite = buildCucumberScenarioSuite({ + ...scenarioInput(() => 0), + recordAttempt: undefined + }) + expect((suite.tests as TestStats[]).every((t) => t.retries === 0)).toBe( + true + ) + }) +}) + +describe('regular test retry attempt via startNextTest', () => { + it('records the attempt under the test uid and increments on rerun', async () => { + const tracker = new TestAttemptTracker() + const test = makeTest('test-uid', 'my test') + const suite = { + uid: 'suite-uid', + tests: [test] + } as unknown as SuiteStats + const ctx = { + suiteManager: { markSuiteAsRunning: vi.fn() }, + testManager: { findTestInSuite: () => test }, + testReporter: { onTestStart: vi.fn() }, + setCurrentTest: vi.fn(), + recordAttempt: (uid: string) => tracker.recordStart(uid) + } as unknown as TestLifecycleCtx + + await startNextTest(ctx, suite, 'my test', new Set()) + expect(test.retries).toBe(0) + + await startNextTest(ctx, suite, 'my test', new Set(['my test'])) + expect(test.retries).toBe(1) + }) +}) + +describe('buildTraceContext', () => { + it('sets attemptInfoAvailable and carries retries through to attempt', () => { + const capturer = { + actionSnapshots: [], + snapshotCaptures: [] + } as unknown as SessionCapturer + const test = makeTest('t1', 'retried test') + test.retries = 2 + const suite = { + uid: 'suite-uid', + tests: [test], + suites: [] + } as unknown as SuiteStats + + const ctx = buildTraceContext( + { + mode: 'trace', + policy: 'on', + granularity: 'session', + format: 'zip', + capturer, + suites: [suite], + ranges: [], + flushed: new Set(), + configPath: undefined, + log: () => {} + }, + 'session-1' + ) + + expect(ctx.attemptInfoAvailable).toBe(true) + expect(ctx.sessionId).toBe('session-1') + expect(ctx.testMetadata.get('t1')?.attempt).toBe(2) + }) +}) diff --git a/packages/selenium-devtools/src/helpers/testManager.ts b/packages/selenium-devtools/src/helpers/testManager.ts index fc3bfebf..70fcb14b 100644 --- a/packages/selenium-devtools/src/helpers/testManager.ts +++ b/packages/selenium-devtools/src/helpers/testManager.ts @@ -1,5 +1,9 @@ import logger from '@wdio/logger' -import { createTestStats, stampTestEnd } from '@wdio/devtools-core' +import { + createTestStats, + stampTestEnd, + TestAttemptTracker +} from '@wdio/devtools-core' import { DEFAULTS, TEST_STATE } from '../constants.js' import type { SuiteStats, TestStats } from '../types.js' import type { TestReporter } from '../reporter.js' @@ -19,6 +23,9 @@ export class TestManager { #currentTest: TestStats | null = null #lastMarkedTest: TestStats | null = null #mode: 'session' | 'marked' = 'session' + // Per-test attempt counter. Persists for the whole in-process session so + // same-process retries (Mocha/Jest/etc re-entering the start hook) accumulate. + #attemptTracker = new TestAttemptTracker() /** Set true the first time the user calls startMarkedTest. Once true we * never auto-create the synthetic session test — orphan commands attach * to the most-recently-marked test instead. */ @@ -90,7 +97,7 @@ export class TestManager { */ startMarkedTest( name: string, - opts: { file?: string; callSource?: string } = {} + opts: { file?: string; callSource?: string; attempt?: number } = {} ): TestStats { if (!this.#userTookOver) { this.#userTookOver = true @@ -115,14 +122,22 @@ export class TestManager { const file = opts.file || this.suite.file // Scope by parent so two suites with the same test/step name don't // collide on signatureCounter disambiguation across rerun processes. + const signature = `${this.suite.uid}::${name}` const test = createTestStats({ - uid: generateStableUid(file, `${this.suite.uid}::${name}`), + uid: generateStableUid(file, signature), cid: DEFAULTS.CID, title: name, file, parent: this.suite.uid, callSource: opts.callSource }) + // deterministicUid is retry-stable (no counter), so re-entering with the + // same logical test increments the heuristic. Mocha supplies an + // authoritative per-test retry index via opts.attempt; prefer it. + const heuristicAttempt = this.#attemptTracker.recordStart( + deterministicUid(file, signature) + ) + test.retries = opts.attempt ?? heuristicAttempt log.info( `Started marked test "${name}" (callSource: ${opts.callSource || 'n/a'})` ) diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 030e935f..2fcdb885 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -30,8 +30,10 @@ import { startScenario as tmStartScenario, endScenario as tmEndScenario, flushPendingTestActions as tmFlushPendingTestActions, + buildStartTestMeta, type StartTestMeta, - type StartScenarioMeta + type StartScenarioMeta, + type PendingTestAction } from './test-management.js' import type { PluginInternals } from './plugin-internals.js' import { @@ -104,16 +106,7 @@ class SeleniumDevToolsPlugin { #uiReadyPromise?: Promise // First it() body fires before onDriverCreated's async setup completes — // buffer startTest/endTest until testManager exists. - #pendingTestActions: Array< - | { - kind: 'start' - name: string - meta: { file?: string; callSource?: string } - suiteName?: string - suiteCallSource?: string - } - | { kind: 'end'; state: TestStats['state'] } - > = [] + #pendingTestActions: PendingTestAction[] = [] // Cucumber Before fires before the driver-build Before — stash and replay. #pendingScenario: { name: string @@ -596,27 +589,24 @@ patchNodeAssert((cmd) => { // Runner globals are published after `--require`, so retry briefly. function registerHooks() { return tryRegisterRunnerHooks({ - onTestStart: (name, file, callSource, suiteName, suiteCallSource) => { - const meta: { - file?: string - callSource?: string - suiteName?: string - suiteCallSource?: string - } = {} - if (file) { - meta.file = file - } - if (callSource) { - meta.callSource = callSource - } - if (suiteName) { - meta.suiteName = suiteName - } - if (suiteCallSource) { - meta.suiteCallSource = suiteCallSource - } - plugin.startTest(name, meta) - }, + onTestStart: ( + name, + file, + callSource, + suiteName, + suiteCallSource, + attempt + ) => + plugin.startTest( + name, + buildStartTestMeta( + file, + callSource, + suiteName, + suiteCallSource, + attempt + ) + ), onTestEnd: (state) => { plugin.endTest(state === 'pending' ? 'skipped' : state) }, diff --git a/packages/selenium-devtools/src/runnerHooks/mocha.ts b/packages/selenium-devtools/src/runnerHooks/mocha.ts index 5f2ad0f5..e7b8a616 100644 --- a/packages/selenium-devtools/src/runnerHooks/mocha.ts +++ b/packages/selenium-devtools/src/runnerHooks/mocha.ts @@ -87,7 +87,8 @@ function makeMochaBeforeEach( parentTitle, parentTitle ? resolveCallSource(test.file, parentTitle, 'suite') - : undefined + : undefined, + typeof test._currentRetry === 'number' ? test._currentRetry : undefined ) } } diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 294d0921..b12699dd 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -290,7 +290,7 @@ export async function onSessionEnd(ctx: SessionLifecycleCtx): Promise { * file and the session trace next to the run's first test file. Test metadata * is recomputed from the suite tree so a boundary flush sees the current tree. */ -function buildTraceExportContext( +export function buildTraceExportContext( ctx: SessionLifecycleCtx, capturer: SessionCapturer, sessionId: string, @@ -306,6 +306,9 @@ function buildTraceExportContext( actionSnapshots: ctx.actionSnapshots, sessionId, testMetadata: collectSuiteTestMetadata(root ? [root] : []), + // TestStats.retries carries the per-test attempt (Mocha authoritative, + // other runners heuristic), so retry-aware policies can trust it. + attemptInfoAvailable: true, ranges: ctx.specRanges, flushed: ctx.flushedSpecs, resolveOutputDir: (range) => diff --git a/packages/selenium-devtools/src/test-management.ts b/packages/selenium-devtools/src/test-management.ts index 842c0654..e62d17de 100644 --- a/packages/selenium-devtools/src/test-management.ts +++ b/packages/selenium-devtools/src/test-management.ts @@ -24,7 +24,7 @@ export type PendingTestAction = | { kind: 'start' name: string - meta: { file?: string; callSource?: string } + meta: { file?: string; callSource?: string; attempt?: number } suiteName?: string suiteCallSource?: string } @@ -54,6 +54,8 @@ export interface StartTestMeta { callSource?: string suiteName?: string suiteCallSource?: string + /** Authoritative per-test retry index when the runner exposes one (Mocha). */ + attempt?: number } export interface StartScenarioMeta { @@ -63,6 +65,53 @@ export interface StartScenarioMeta { featureCallSource?: string } +/** Assemble a StartTestMeta from a runner hook's positional callback args. */ +export function buildStartTestMeta( + file?: string, + callSource?: string, + suiteName?: string, + suiteCallSource?: string, + attempt?: number +): StartTestMeta { + const meta: StartTestMeta = {} + if (file) { + meta.file = file + } + if (callSource) { + meta.callSource = callSource + } + if (suiteName) { + meta.suiteName = suiteName + } + if (suiteCallSource) { + meta.suiteCallSource = suiteCallSource + } + if (typeof attempt === 'number') { + meta.attempt = attempt + } + return meta +} + +type ResolvedTestMeta = { file?: string; callSource?: string; attempt?: number } + +function resolveStartMeta( + meta: StartTestMeta, + file: string | undefined, + callSource: string | undefined +): ResolvedTestMeta { + const resolved: ResolvedTestMeta = {} + if (file) { + resolved.file = file + } + if (callSource && callSource !== 'unknown:0') { + resolved.callSource = callSource + } + if (typeof meta.attempt === 'number') { + resolved.attempt = meta.attempt + } + return resolved +} + export function startTest( ctx: TestManagementCtx, name: string, @@ -74,13 +123,7 @@ export function startTest( const stackInfo = getCallSourceFromStack() const file = meta.file || stackInfo.filePath const callSource = meta.callSource || stackInfo.callSource - const resolvedMeta: { file?: string; callSource?: string } = {} - if (file) { - resolvedMeta.file = file - } - if (callSource && callSource !== 'unknown:0') { - resolvedMeta.callSource = callSource - } + const resolvedMeta = resolveStartMeta(meta, file, callSource) if (!ctx.suiteManager || !ctx.testReporter) { ctx.pendingTestActions.push({ kind: 'start', diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index e801799d..af764d69 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -20,31 +20,13 @@ export { type TraceRetentionPolicy } from '@wdio/devtools-shared' -export interface DevToolsOptions { - port?: number - hostname?: string +export interface DevToolsOptions extends BaseDevToolsOptions { /** Open a Chrome window pointing at the UI. Default true. */ openUi?: boolean - /** `live` (default) launches the DevTools UI; `trace` skips it. Overrides `openUi`. */ - mode?: DevToolsMode - /** Trace output layout — `zip` (default) writes a single archive, - * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ - traceFormat?: TraceFormat - /** Trace output granularity — `session` (default) writes one trace per - * worker session; `spec` writes one trace per spec file. Only applies in - * trace mode. */ - traceGranularity?: TraceGranularity - /** Trace retention policy — gates which traces are kept (e.g. - * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ - tracePolicy?: TraceRetentionPolicy /** Capture screenshots after each command. Default true. */ captureScreenshots?: boolean - /** Capture node:assert assertions as first-class commands. Default true. */ - captureAssertions?: boolean /** Command template for per-test rerun. {{testName}} is substituted. */ rerunCommand?: string - /** Per-session screencast recording. Disabled by default. */ - screencast?: ScreencastOptions /** * Force the *test* browser into headless mode by injecting --headless=new * into Chrome capabilities. The dashboard window (auto-opened by openUi) @@ -55,13 +37,7 @@ export interface DevToolsOptions { // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { - DevToolsMode, - ScreencastOptions, - TraceFormat, - TraceGranularity, - TraceRetentionPolicy -} from '@wdio/devtools-shared' +import type { BaseDevToolsOptions, TestStatus } from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** @@ -133,24 +109,16 @@ export interface ElementOriginals { getTagName?: (element: unknown) => Promise } -// ─── bidi ─────────────────────────────────────────────────────────────────── - -import type { ConsoleLog, NetworkRequest } from '@wdio/devtools-shared' - -export interface BidiHandlerSinks { - pushConsoleLog: (entry: ConsoleLog) => void - pushNetworkRequest: (entry: NetworkRequest) => void - replaceNetworkRequest: (id: string, entry: NetworkRequest) => void -} - // ─── runnerHooks ──────────────────────────────────────────────────────────── export interface MochaTestCtx { title?: string file?: string - state?: 'passed' | 'failed' | 'pending' | 'running' | 'skipped' + state?: TestStatus duration?: number parent?: { title?: string } + /** Mocha's authoritative retry index — 0 on the first run, +1 per retry. */ + _currentRetry?: number } export interface RunnerHookCallbacks { @@ -159,9 +127,12 @@ export interface RunnerHookCallbacks { file?: string, callSource?: string, suiteName?: string, - suiteCallSource?: string + suiteCallSource?: string, + // Authoritative per-test retry index when the runner exposes one (Mocha's + // `_currentRetry`); undefined leaves the heuristic tracker to decide. + attempt?: number ) => void - onTestEnd: (state: 'passed' | 'failed' | 'skipped' | 'pending') => void + onTestEnd: (state: Exclude) => void // Cucumber-only: scenario boundary creates a sub-suite under the feature // rootSuite; subsequent onTestStart/onTestEnd attach as Gherkin steps inside. onScenarioStart?: ( @@ -171,7 +142,7 @@ export interface RunnerHookCallbacks { featureName?: string, featureCallSource?: string ) => void - onScenarioEnd?: (state: 'passed' | 'failed' | 'skipped' | 'pending') => void + onScenarioEnd?: (state: Exclude) => void // Fires from the runner's after-all hook so the dashboard suite header // updates without waiting for process exit. onTestRunComplete?: (summary: { diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index c2af6580..cbe6c196 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -8,15 +8,23 @@ import { mapCommandToAction, recordSpecBoundary, resolveAdapterOutputDir, + TestAttemptTracker, tracePolicyModeWarning, type SpecRange, type TraceExportContext } from '@wdio/devtools-core' -import { captureExpectFailure, wireAssertCapture } from './assert-capture.js' +import { + captureExpectFailure, + expectAssertionToCommandLog, + wireAssertCapture, + type ExpectAssertion +} from './assert-capture.js' import { cucumberScenarioUid, + resolveTestAttempt, stampTestState, - testMetadataUid + testMetadataUid, + type TestOutcomeResult } from './test-metadata.js' import { resolveCallSourceFromFrame } from './call-source.js' import { @@ -100,6 +108,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Map of testUid → metadata for trace group events and per-spec partitioning. */ #testMetadata: TestMetadataMap = new Map() + /** Per-test attempt counter. specFileRetries spawns a fresh worker (hence a + * fresh instance) per retry, so this only reflects same-process retries + * (Mocha this.retries(n)); cross-worker attempts rely on the WDIO result. */ + #attemptTracker = new TestAttemptTracker() + /** Index ranges into the session capturer's flat arrays, one per spec file. */ #specRanges: SpecRange[] = [] @@ -159,6 +172,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { sessionId: browser.sessionId, capabilities: browser.capabilities, testMetadata: this.#testMetadata, + attemptInfoAvailable: true, ranges: this.#specRanges, flushed: this.#flushedSpecs, resolveOutputDir: () => this.#outputDir, @@ -323,6 +337,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { world?.pickle?.astNodeIds ) this.#currentTestUid = uid + this.#attemptTracker.recordStart(uid) this.#testMetadata.set(uid, { title: scenarioName, specFile: featureFile @@ -344,6 +359,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (testTitle) { const uid = testMetadataUid(test?.file, testTitle) this.#currentTestUid = uid + this.#attemptTracker.recordStart(uid) this.#testMetadata.set(uid, { title: testTitle, specFile: test?.file ?? '' @@ -372,16 +388,23 @@ export default class DevToolsHookService implements Services.ServiceInstance { ) } + /** Stamp final state + the resolved 0-based attempt onto the test's metadata + * entry, taking the max of the tracker count and WDIO's retry field. */ + #stampOutcome(uid: string, result?: TestOutcomeResult): void { + const fallback = this.#attemptTracker.attemptFor(uid) ?? 0 + const attempt = resolveTestAttempt(result, fallback) + stampTestState(this.#testMetadata, uid, result, attempt) + } + async afterScenario( world?: { pickle?: { uri?: string; name?: string; astNodeIds?: readonly string[] } }, - result?: { error?: unknown; passed?: boolean; skipped?: boolean } + result?: TestOutcomeResult ) { const { uri, name, astNodeIds } = world?.pickle ?? {} if (uri && name) { - const uid = cucumberScenarioUid(uri, name, astNodeIds) - stampTestState(this.#testMetadata, uid, result) + this.#stampOutcome(cucumberScenarioUid(uri, name, astNodeIds), result) } await this.#finalizePerScenario() } @@ -389,20 +412,28 @@ export default class DevToolsHookService implements Services.ServiceInstance { async afterTest( test?: { file?: string; title?: string; fullTitle?: string }, _context?: unknown, - result?: { error?: unknown; passed?: boolean; skipped?: boolean } + result?: TestOutcomeResult ) { this.#captureExpectFailure(result?.error) const testTitle = test?.fullTitle || test?.title if (testTitle) { - stampTestState( - this.#testMetadata, - testMetadataUid(test?.file, testTitle), - result - ) + this.#stampOutcome(testMetadataUid(test?.file, testTitle), result) } await this.#finalizePerScenario() } + /** expect-webdriverio fires this per matcher (pass or fail) via WDIO's + * assertion hook — capture it as an `expect.` action so passing + * assertions show in the trace, not just failures. */ + afterAssertion(params: ExpectAssertion): void { + if (this.#options.captureAssertions === false) { + return + } + this.#sessionCapturer.captureAssertCommand( + expectAssertionToCommandLog(params, this.#currentTestUid) + ) + } + async #finalizePerScenario() { if (this.#options.mode !== 'trace' || !this.#browser) { return diff --git a/packages/service/src/reporter.ts b/packages/service/src/reporter.ts index 801a86ad..ee22f23b 100644 --- a/packages/service/src/reporter.ts +++ b/packages/service/src/reporter.ts @@ -314,6 +314,10 @@ export class TestReporter extends WebdriverIOReporter { matcherResult: rawErr.matcherResult } as Error } + // WDIO stamps the 0-based attempt on each test:start, so the final attempt's + // stat already carries the retry count; the field is optional upstream, so + // pin it to a number for the shared TestStats.retries contract. + testStats.retries = testStats.retries ?? 0 this.#sendUpstream() } diff --git a/packages/service/src/test-metadata.ts b/packages/service/src/test-metadata.ts index 339e1dc1..dc2babad 100644 --- a/packages/service/src/test-metadata.ts +++ b/packages/service/src/test-metadata.ts @@ -4,6 +4,15 @@ import { deterministicUid } from '@wdio/devtools-core' import type { TestMetadataMap, TestStatus } from '@wdio/devtools-shared' +/** The subset of a WDIO afterTest/afterScenario hook result this adapter reads: + * pass/skip state, the error, and WDIO's authoritative 0-based retry count. */ +export interface TestOutcomeResult { + error?: unknown + passed?: boolean + skipped?: boolean + retries?: { attempts?: number } +} + /** Stable per-test key. beforeTest and afterTest derive it identically so keys * match. File-less runners (some non-WDIO frameworks) key on the title alone. */ export function testMetadataUid( @@ -37,15 +46,32 @@ export function resultToState(result: { return result.passed ? 'passed' : 'failed' } -/** Stamp the final state onto the metadata entry beforeTest/beforeScenario - * created, so retention can gate its trace. No-op when there's no entry. */ +/** Stamp the final state (and, when known, the 0-based attempt) onto the + * metadata entry beforeTest/beforeScenario created, so retention can gate its + * trace per attempt. No-op when there's no entry. */ export function stampTestState( metadata: TestMetadataMap, uid: string, - result?: { passed?: boolean; skipped?: boolean } + result?: Pick, + attempt?: number ): void { const entry = result && metadata.get(uid) if (entry) { entry.state = resultToState(result) + if (attempt !== undefined) { + entry.attempt = attempt + } } } + +/** Resolve the 0-based attempt for a test. WDIO's mocha framework reports + * `retries.attempts` as 0 even on a retry, so it can't override the in-process + * tracker — take the max so a present-but-zero runner field never clobbers a + * real retry count, while a genuine runner value still wins when it's higher. */ +export function resolveTestAttempt( + result: Pick | undefined, + fallback: number +): number { + const attempts = result?.retries?.attempts + return Math.max(typeof attempts === 'number' ? attempts : 0, fallback) +} diff --git a/packages/service/tests/trace-metadata.test.ts b/packages/service/tests/trace-metadata.test.ts index fe88dc1a..9b2ee653 100644 --- a/packages/service/tests/trace-metadata.test.ts +++ b/packages/service/tests/trace-metadata.test.ts @@ -161,6 +161,57 @@ describe('DevtoolsService - afterTest state stamping', () => { ).toBe('failed') }) + it('flags attemptInfoAvailable so retry-aware policies use per-test attempt', async () => { + const ctx = (await runTest('trace opts', { passed: true })) as { + attemptInfoAvailable?: boolean + } + expect(ctx.attemptInfoAvailable).toBe(true) + }) + + it('uses the tracker attempt even when WDIO reports retries.attempts:0', async () => { + const service = new DevToolsHookService({ + mode: 'trace', + tracePolicy: 'retain-on-failure-and-retries' + }) + await service.before({} as never, [], mockBrowser) + const title = 'flaky login' + // Two starts before a single end simulate a same-process (Mocha) retry. + // WDIO's mocha framework reports retries.attempts:0 even on the retry, so + // the runner field must NOT clobber the tracker's real count of 1. + service.beforeTest({ file, fullTitle: title }) + service.beforeTest({ file, fullTitle: title }) + await service.afterTest( + { file, fullTitle: title }, + {}, + { passed: true, retries: { attempts: 0 } } + ) + await service.after() + const ctx = finalizeTraceExport.mock.calls.at(-1)?.[0] as { + testMetadata: Map + } + expect(ctx.testMetadata.get(deterministicUid(file, title))?.attempt).toBe(1) + }) + + it('uses the runner attempts count when it exceeds the tracker', async () => { + const service = new DevToolsHookService({ + mode: 'trace', + tracePolicy: 'retain-on-failure-and-retries' + }) + await service.before({} as never, [], mockBrowser) + const title = 'checkout retries' + service.beforeTest({ file, fullTitle: title }) + await service.afterTest( + { file, fullTitle: title }, + {}, + { passed: true, retries: { attempts: 3 } } + ) + await service.after() + const ctx = finalizeTraceExport.mock.calls.at(-1)?.[0] as { + testMetadata: Map + } + expect(ctx.testMetadata.get(deterministicUid(file, title))?.attempt).toBe(3) + }) + it('stamps a Cucumber scenario state in afterScenario', async () => { const service = new DevToolsHookService({ mode: 'trace', From 379122ead2c781b5bc3b61b5df7e330741209b5b Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:48:48 +0530 Subject: [PATCH 24/91] feat(service,core): render expect-webdriverio matchers as trace actions --- packages/core/src/assert-patcher.ts | 40 ++++++++ packages/core/tests/assert-patcher.test.ts | 40 ++++++++ .../tests/attempt-capture.test.ts | 95 +++++++++++++++++++ packages/service/src/assert-capture.ts | 54 ++++++++++- 4 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 packages/selenium-devtools/tests/attempt-capture.test.ts diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index 8917a1ef..71ebd9fe 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -2,6 +2,7 @@ import { createRequire } from 'node:module' import { TRACKED_ASSERT_METHODS, type CommandLog } from '@wdio/devtools-shared' import { getCallSourceFromStack } from './stack.js' import { toError } from './error.js' +import { stripAnsi } from './console.js' export { TRACKED_ASSERT_METHODS } @@ -152,6 +153,45 @@ export function capturedAssertToCommandLog( return entry } +/** + * Params any adapter's matcher tap produces. `prefix` selects the command + * namespace (`expect` / `assert` / `verify`), all of which map to an `Assert` + * action via the shared action map. Adapters do the thin framework-specific + * extraction (matcher name, args, pass flag, message); this conversion is the + * single shared path — the generic counterpart to `capturedAssertToCommandLog` + * for libraries that aren't node:assert (expect-webdriverio, chai, Nightwatch). + */ +export interface MatcherAssertion { + prefix?: string + method: string + args?: unknown[] + passed: boolean + message?: string | (() => string) + callSource?: string +} + +export function matcherAssertionToCommandLog( + input: MatcherAssertion, + testUid?: string +): CommandLog { + const command = `${input.prefix ?? 'expect'}.${input.method}` + const message = + typeof input.message === 'function' ? input.message() : input.message + return capturedAssertToCommandLog( + { + command, + args: (input.args ?? []).map(safeSerializeAssertArg), + result: input.passed ? 'passed' : undefined, + error: input.passed + ? undefined + : new Error(stripAnsi(message ?? `${command} failed`)), + callSource: input.callSource, + timestamp: Date.now() + }, + testUid + ) +} + /** * Patch `node:assert` so each tracked method emits a `CapturedAssert` to the * supplied hook. Idempotent across calls (guarded by `ASSERT_PATCHED_SYMBOL`). diff --git a/packages/core/tests/assert-patcher.test.ts b/packages/core/tests/assert-patcher.test.ts index 86774967..38bea721 100644 --- a/packages/core/tests/assert-patcher.test.ts +++ b/packages/core/tests/assert-patcher.test.ts @@ -4,6 +4,7 @@ import { ASSERT_PATCHED_SYMBOL, TRACKED_ASSERT_METHODS, capturedAssertToCommandLog, + matcherAssertionToCommandLog, patchNodeAssert, safeSerializeAssertArg, type CapturedAssert @@ -245,3 +246,42 @@ describe('capturedAssertToCommandLog', () => { expect(entry.testUid).toBeUndefined() }) }) + +describe('matcherAssertionToCommandLog', () => { + it('builds a passing expect. command (default prefix)', () => { + const entry = matcherAssertionToCommandLog( + { method: 'toHaveTitle', args: ['The Internet'], passed: true }, + 'uid-1' + ) + expect(entry).toMatchObject({ + command: 'expect.toHaveTitle', + args: ['The Internet'], + result: 'passed', + testUid: 'uid-1' + }) + expect(entry.error).toBeUndefined() + }) + + it('carries the ANSI-stripped message as the error when failed', () => { + const entry = matcherAssertionToCommandLog({ + method: 'toHaveText', + args: ['foo'], + passed: false, + message: () => 'expected foo' + }) + expect(entry.result).toBeUndefined() + expect(entry.error).toMatchObject({ message: 'expected foo' }) + }) + + it('honors a non-default prefix and sanitizes args', () => { + const entry = matcherAssertionToCommandLog({ + prefix: 'verify', + method: 'equal', + args: [/re/, () => 1], + passed: true + }) + expect(entry.command).toBe('verify.equal') + // RegExp → string, function → '[Function]' (via safeSerializeAssertArg). + expect(entry.args).toEqual(['/re/', '[Function]']) + }) +}) diff --git a/packages/selenium-devtools/tests/attempt-capture.test.ts b/packages/selenium-devtools/tests/attempt-capture.test.ts new file mode 100644 index 00000000..255280f6 --- /dev/null +++ b/packages/selenium-devtools/tests/attempt-capture.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, vi } from 'vitest' +import { collectSuiteTestMetadata } from '@wdio/devtools-core' + +import { resetSignatureCounters } from '../src/helpers/utils.js' +import { TestManager } from '../src/helpers/testManager.js' +import { SuiteManager } from '../src/helpers/suiteManager.js' +import { TestReporter } from '../src/reporter.js' +import { SessionCapturer } from '../src/session.js' +import { + buildTraceExportContext, + type SessionLifecycleCtx +} from '../src/session-lifecycle.js' + +function makeManager() { + resetSignatureCounters() + const reporter = new TestReporter(vi.fn()) + const suiteManager = new SuiteManager(reporter) + const rootSuite = suiteManager.getOrCreateRootSuite('/spec.ts', 'Suite') + const mgr = new TestManager(rootSuite, reporter, suiteManager) + return { mgr, suiteManager, rootSuite } +} + +describe('retry/attempt capture', () => { + it('re-entering the start hook for the same test increments retries (heuristic)', () => { + const { mgr } = makeManager() + + const first = mgr.startMarkedTest('flaky') + expect(first.retries).toBe(0) + + mgr.endCurrent('failed') + const retry = mgr.startMarkedTest('flaky') + expect(retry.retries).toBe(1) + + mgr.endCurrent('failed') + const retry2 = mgr.startMarkedTest('flaky') + expect(retry2.retries).toBe(2) + }) + + it('prefers the authoritative Mocha attempt over the heuristic', () => { + const { mgr } = makeManager() + + // First start: heuristic would be 0, authoritative says 2 → 2 wins. + const t = mgr.startMarkedTest('login', { attempt: 2 }) + expect(t.retries).toBe(2) + + mgr.endCurrent('failed') + // Re-entry: heuristic would be 1, authoritative says 5 → 5 wins. + const retry = mgr.startMarkedTest('login', { attempt: 5 }) + expect(retry.retries).toBe(5) + }) + + it('flows retries into metadata attempt and the trace ctx flags attempt info', () => { + const { mgr, suiteManager, rootSuite } = makeManager() + + mgr.startMarkedTest('flaky') + mgr.endCurrent('failed') + const retry = mgr.startMarkedTest('flaky') + mgr.endCurrent('passed') + expect(retry.retries).toBe(1) + + const metadata = collectSuiteTestMetadata([rootSuite]) + expect(metadata.get(retry.uid)?.attempt).toBe(1) + + const capturer = new SessionCapturer() + try { + // Minimal structural ctx: buildTraceExportContext only reads options, + // suiteManager and the trace accumulators, so we cast a partial to the + // full lifecycle interface rather than standing up the whole plugin. + const ctx = { + options: { + mode: 'trace', + tracePolicy: 'retain-on-failure-and-retries', + traceGranularity: 'session', + traceFormat: 'zip' + }, + suiteManager, + actionSnapshots: [], + specRanges: [], + flushedSpecs: new Set(), + snapshotCaptures: [] + } as unknown as SessionLifecycleCtx + + const traceCtx = buildTraceExportContext( + ctx, + capturer, + 'sess-1', + '/spec.ts' + ) + expect(traceCtx.attemptInfoAvailable).toBe(true) + expect(traceCtx.testMetadata.get(retry.uid)?.attempt).toBe(1) + } finally { + capturer.cleanup() + } + }) +}) diff --git a/packages/service/src/assert-capture.ts b/packages/service/src/assert-capture.ts index 43c75131..2c277cf8 100644 --- a/packages/service/src/assert-capture.ts +++ b/packages/service/src/assert-capture.ts @@ -4,10 +4,11 @@ import logger from '@wdio/logger' import { capturedAssertToCommandLog, + matcherAssertionToCommandLog, patchNodeAssert, stripAnsi } from '@wdio/devtools-core' -import type { SerializedError } from '@wdio/devtools-shared' +import type { CommandLog, SerializedError } from '@wdio/devtools-shared' import type { SessionCapturer } from './session.js' const log = logger('@wdio/devtools-service:assert-capture') @@ -34,8 +35,10 @@ export function wireAssertCapture( * Normalize a framework failure into a SerializedError. Cucumber hands a plain * message string (@wdio/cucumber-framework getResultObject → world.result.message); * Mocha/Jasmine hand an Error object. Returns undefined when there's nothing to - * show, or for a node:assert AssertionError (the patcher already made that its - * own command). ANSI is stripped so every consumer gets a clean message. + * show, or when the failure was already captured as its own command: a + * node:assert AssertionError (via the patcher) or an expect-webdriverio matcher + * error (via afterAssertion — it carries `matcherResult`). Skipping those keeps + * `failLastAction` from double-marking a passing command. ANSI is stripped. */ export function toCommandError(error: unknown): SerializedError | undefined { if (!error) { @@ -48,8 +51,13 @@ export function toCommandError(error: unknown): SerializedError | undefined { if (typeof error !== 'object') { return undefined } - const err = error as { name?: string; message?: string; stack?: string } - if (err.name === 'AssertionError') { + const err = error as { + name?: string + message?: string + stack?: string + matcherResult?: unknown + } + if (err.name === 'AssertionError' || err.matcherResult !== undefined) { return undefined } return { @@ -79,3 +87,39 @@ export function captureExpectFailure( capturer.failLastAction(testUid, commandError) } } + +/** The subset of expect-webdriverio's afterAssertion hook params we read to + * turn a matcher call into a trace command. The matcher passes `{ pass }` at + * runtime, but @wdio/types declares `{ result }`, so we accept and read both. */ +export interface ExpectAssertion { + matcherName: string + expectedValue?: unknown + result: { pass?: boolean; result?: boolean; message?: () => string } +} + +/** + * Adapt expect-webdriverio's afterAssertion params to the shared matcher + * converter. Framework-specific extraction only (matcher name, expectedValue → + * args, the runtime `pass` vs typed `result` flag); the actual CommandLog + * shaping lives once in core's `matcherAssertionToCommandLog`. + */ +export function expectAssertionToCommandLog( + params: ExpectAssertion, + testUid: string | undefined +): CommandLog { + const { matcherName, expectedValue, result } = params + return matcherAssertionToCommandLog( + { + method: matcherName, + args: + expectedValue === undefined + ? [] + : Array.isArray(expectedValue) + ? expectedValue + : [expectedValue], + passed: result.pass ?? result.result ?? false, + message: result.message + }, + testUid + ) +} From bca2d4392f80c013cd648167ba5fcf9acec8c222 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:49:14 +0530 Subject: [PATCH 25/91] test(backend): remove skip-forever foreign-zip tests --- .../backend/tests/trace-reader-bodies.test.ts | 27 +----------- packages/backend/tests/trace-reader.test.ts | 43 +------------------ 2 files changed, 2 insertions(+), 68 deletions(-) diff --git a/packages/backend/tests/trace-reader-bodies.test.ts b/packages/backend/tests/trace-reader-bodies.test.ts index 947daf38..62f2cec3 100644 --- a/packages/backend/tests/trace-reader-bodies.test.ts +++ b/packages/backend/tests/trace-reader-bodies.test.ts @@ -1,8 +1,7 @@ import { createHash } from 'node:crypto' -import { existsSync } from 'node:fs' import { describe, it, expect } from 'vitest' import { zipSync, strToU8 } from 'fflate' -import { parseTraceZip, readTraceZip } from '../src/trace-reader.js' +import { parseTraceZip } from '../src/trace-reader.js' const sha1 = (data: string): string => createHash('sha1').update(data).digest('hex') @@ -142,27 +141,3 @@ describe('trace-reader response bodies', () => { expect(bodyByUrl('https://example.com/missing')).toBeUndefined() }) }) - -// Machine-local foreign zip; exercises the extension-suffixed _sha1 convention -// against a real archive when present. -const REAL_FOREIGN_ZIP = - '/Users/vishnu.p@browserstack.com/Documents/Test projects/Test-playwright/test-results/trace-features-trace-feature-showcase-chromium-retry2/trace.zip' - -describe('foreign zip response bodies', () => { - it.skipIf(!existsSync(REAL_FOREIGN_ZIP))( - 'restores bodies referenced via extension-suffixed _sha1 entries', - async () => { - const data = await readTraceZip(REAL_FOREIGN_ZIP) - const requests = data.trace.networkRequests - const apiRequest = requests.find((r) => - r.url.startsWith('https://api.github.com/') - ) - expect(apiRequest?.responseBody).toContain('"full_name"') - const css = requests.find((r) => r.url.endsWith('base.css')) - expect(css?.responseBody?.length).toBeGreaterThan(0) - // Entries without a _sha1 ref stay body-less. - const script = requests.find((r) => r.url.endsWith('.js')) - expect(script?.responseBody).toBeUndefined() - } - ) -}) diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index be736c90..25f9a2f9 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -1,12 +1,11 @@ import { createHash } from 'node:crypto' -import { existsSync } from 'node:fs' import { describe, it, expect } from 'vitest' import { zipSync, strToU8 } from 'fflate' import type { TraceActionChild, TraceActionGroupNode } from '@wdio/devtools-shared' -import { parseTraceZip, readTraceZip } from '../src/trace-reader.js' +import { parseTraceZip } from '../src/trace-reader.js' import { buildSources, stackToCallSource } from '../src/trace-reader-utils.js' import type { BeforeEvent } from '../src/trace-reader-types.js' @@ -598,43 +597,3 @@ describe('glued callSource recovery from older zips', () => { expect(sources).toEqual({ [clean]: 'glued source' }) }) }) - -// Manual verification against a real foreign zip; skipped where absent. -const FOREIGN_ZIP_PATH = - process.env.FOREIGN_TRACE_ZIP ?? - '/Users/vishnu.p@browserstack.com/Documents/Test projects/Test-playwright/test-results/trace-features-trace-feature-showcase-chromium/trace.zip' - -describe('readTraceZip against a real foreign zip', () => { - it.skipIf(!existsSync(FOREIGN_ZIP_PATH))( - 'renders actions, filmstrip, network and a sane duration', - async () => { - const { trace, frames, startTime, duration, groups } = - await readTraceZip(FOREIGN_ZIP_PATH) - expect(trace.commands.length).toBeGreaterThan(10) - expect(frames.length).toBeGreaterThan(10) - expect(frames.every((f) => f.screenshot.length > 0)).toBe(true) - expect(startTime).toBeGreaterThan(1_700_000_000_000) - expect(duration).toBeGreaterThan(5_000) - expect(duration).toBeLessThan(60_000) - for (const frame of frames) { - expect(frame.timestamp).toBeGreaterThanOrEqual(startTime) - expect(frame.timestamp).toBeLessThanOrEqual(startTime + duration) - } - expect(trace.networkRequests.length).toBeGreaterThan(0) - expect(trace.consoleLogs.length).toBeGreaterThanOrEqual(3) - expect(Object.keys(trace.sources).length).toBeGreaterThan(0) - expect(trace.commands.some((c) => c.error)).toBe(true) - // The action tree covers every command exactly once and flags failures. - const tree = allGroups(groups ?? []) - expect(tree.length).toBeGreaterThan(5) - expect([...commandIndices(groups ?? [])].sort((a, b) => a - b)).toEqual( - trace.commands.map((_, index) => index) - ) - const failed = tree.filter((group) => group.failed) - expect(failed.length).toBeGreaterThan(0) - expect(failed.some((g) => g.title.startsWith('Soft assertion'))).toBe( - true - ) - } - ) -}) From 3b9f9ac5c6e5e4ecea117f76aeab7370b8774e4a Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:50:27 +0530 Subject: [PATCH 26/91] chore(examples): add WDIO retry harness for retry-aware trace policies --- examples/wdio/mocha/retry/flaky.e2e.ts | 19 +++++++++ examples/wdio/mocha/wdio.retry.conf.ts | 58 ++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 78 insertions(+) create mode 100644 examples/wdio/mocha/retry/flaky.e2e.ts create mode 100644 examples/wdio/mocha/wdio.retry.conf.ts diff --git a/examples/wdio/mocha/retry/flaky.e2e.ts b/examples/wdio/mocha/retry/flaky.e2e.ts new file mode 100644 index 00000000..0b87bc83 --- /dev/null +++ b/examples/wdio/mocha/retry/flaky.e2e.ts @@ -0,0 +1,19 @@ +import { browser, expect } from '@wdio/globals' + +// Deterministically flaky: throws on the first attempt and passes on the retry. +// The module-level counter survives mocha's in-process retry, so attempt 0 fails +// and attempt 1 succeeds. The test ends PASSED — so retain-on-failure would drop +// its trace, but on-first-retry keeps it, which is exactly what B4 verifies: +// the retry attempt is captured and is distinct from a failure. +let attempts = 0 + +describe('Flaky (passes on retry)', () => { + it('fails the first attempt, then passes', async () => { + await browser.url('https://the-internet.herokuapp.com/login') + attempts += 1 + if (attempts === 1) { + throw new Error('intentional first-attempt failure — should retry') + } + await expect(browser).toHaveTitle('The Internet') + }) +}) diff --git a/examples/wdio/mocha/wdio.retry.conf.ts b/examples/wdio/mocha/wdio.retry.conf.ts new file mode 100644 index 00000000..90b87858 --- /dev/null +++ b/examples/wdio/mocha/wdio.retry.conf.ts @@ -0,0 +1,58 @@ +import type { Options } from '@wdio/types' + +// Disposable harness for verifying B4 (retry-aware trace policies). Runs the +// deterministically-flaky spec (fails once, passes on retry) alongside a clean +// passing spec at spec granularity. With tracePolicy 'on-first-retry' only the +// flaky spec's trace is retained — proving the retry attempt is captured and is +// distinct from failure (the flaky test ends PASSED, so retain-on-failure would +// drop it). Flip tracePolicy below to exercise the other retry-aware policies. +export const config: Options.Testrunner = { + runner: 'local', + autoCompileOpts: { + autoCompile: true, + tsNodeOpts: { + project: './tsconfig.json', + transpileOnly: true + } + }, + specs: ['./retry/flaky.e2e.ts', './specs/login.e2e.ts'], + exclude: [], + maxInstances: 1, + capabilities: [ + { + browserName: 'chrome', + 'goog:chromeOptions': { + args: [ + '--headless', + '--disable-gpu', + '--remote-allow-origins=*', + '--window-size=1600,900' + ] + } + } + ], + logLevel: 'warn', + bail: 0, + baseUrl: 'http://localhost', + waitforTimeout: 10000, + connectionRetryTimeout: 120000, + connectionRetryCount: 3, + services: [ + [ + 'devtools', + { + mode: 'trace' as const, + traceGranularity: 'spec' as const, + tracePolicy: 'on-first-retry' as const + } + ] + ], + framework: 'mocha', + reporters: ['spec'], + mochaOpts: { + ui: 'bdd', + timeout: 60000, + // 1 retry = 2 total attempts; the flaky spec fails attempt 0, passes attempt 1. + retries: 1 + } +} diff --git a/package.json b/package.json index f12d3491..9c120190 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "build": "pnpm -r build", "demo:wdio": "wdio run ./examples/wdio/cucumber/wdio.conf.ts", "demo:wdio:mocha": "wdio run ./examples/wdio/mocha/wdio.conf.ts", + "demo:wdio:retry": "wdio run ./examples/wdio/mocha/wdio.retry.conf.ts", "demo:nightwatch": "pnpm --filter @wdio/nightwatch-devtools example", "demo:selenium": "pnpm --filter @wdio/selenium-devtools example", "dev": "pnpm --parallel dev", From ca6b78675acccd0efef9eb6b76f6c1c01e50801a Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:51:49 +0530 Subject: [PATCH 27/91] docs: correct node:assert note; record retain-on-first-failure limitation --- .gitignore | 2 ++ CLAUDE.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e95e0003..47c58732 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ coverage/ # pnpm state, cache, logs, and debug files /packages/**/*.mjs + +.claude diff --git a/CLAUDE.md b/CLAUDE.md index 0416c6ae..f06ea24d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -233,8 +233,9 @@ Documented divergences from the conventions above. They exist today as debt to b ### Architecture - `replaceCommand` has two semantics — Selenium mutates in place (preserves `_id`/`id` for chained calls); Nightwatch splices and reissues. Both call the same `core/suite-helpers` factories; the storage strategy stays adapter-specific because runner integrations differ. Could be unified by parameterizing the policy if the divergence ever causes a real problem. -- `patchNodeAssert` is wired only in `selenium-devtools`. The shared helper lives in `core/assert-patcher`; Service and Nightwatch can opt in via a one-line call when ready. Not auto-enabled — both communities lean on chai/expect. +- `patchNodeAssert` (via `core/assert-patcher`) is now wired in all three adapters, default-on behind each adapter's `captureAssertions` option (opt out with `captureAssertions: false`). Framework matcher libraries differ: Service additionally taps expect-webdriverio's `afterAssertion` hook so passing+failing matchers render as `expect.*` actions; Selenium (chai/jest) and Nightwatch (`browser.assert`/`verify`) don't yet surface passing framework-matcher assertions as trace actions. - BiDi is auto-attached in Service and Selenium; Nightwatch is opt-in via `bidi: true` and requires `webSocketUrl: true` in capabilities. +- Retry-aware trace policies aren't uniform across adapters. Selenium emits one metadata entry per attempt (counter-based `generateStableUid` uid + retry-stable tracker key), so all six policies — including `retain-on-first-failure` (needs the first attempt's outcome) — evaluate correctly. Service and Nightwatch key metadata on the retry-stable uid and overwrite/mutate per attempt, collapsing to the *final* attempt's outcome; the retry-count policies (`retain-on-failure`, `on-first-retry`, `on-all-retries`, `retain-on-failure-and-retries`) are still correct, but `retain-on-first-failure` can't see a failed-then-passed first attempt and under-retains. Proper fix rides with B5 (per-test granularity preserves per-attempt slices); until then all three set `attemptInfoAvailable: true`. ### File-size (raw line counts; soft cap is 500 logic lines) From 78a963523925ea47ec0fc346f6351ebe7be8c37a Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 13:59:41 +0530 Subject: [PATCH 28/91] refactor(shared,adapters): consolidate adapter types into shared --- packages/nightwatch-devtools/src/types.ts | 41 ++-------------- .../src/runnerHooks/cucumber.ts | 18 +++---- packages/service/src/types.ts | 41 ++-------------- packages/shared/src/types.ts | 47 +++++++++++++++++++ 4 files changed, 59 insertions(+), 88 deletions(-) diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index e8ef90f6..ab776db5 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -23,13 +23,7 @@ export { type TraceLog } from '@wdio/devtools-shared' -import type { - DevToolsMode, - ScreencastOptions, - TraceFormat, - TraceGranularity, - TraceRetentionPolicy -} from '@wdio/devtools-shared' +import type { BaseDevToolsOptions } from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -87,16 +81,7 @@ export interface StepLocation { line: number } -export interface DevToolsOptions { - port?: number - hostname?: string - /** - * Screencast recording options. When enabled, a continuous video of the - * browser session is recorded and saved as a .webm file at the end of the - * test run. Polling mode only on Nightwatch (no CDP push); works on every - * browser Nightwatch supports. - */ - screencast?: ScreencastOptions +export interface DevToolsOptions extends BaseDevToolsOptions { /** * Enable WebDriver BiDi capture (browser console + JS exceptions + network * via `selenium-webdriver/bidi`). Requires `webSocketUrl: true` in your @@ -105,21 +90,6 @@ export interface DevToolsOptions { * entries. Defaults to `false` — opt-in. */ bidi?: boolean - /** Capture node:assert assertions as first-class commands (Nightwatch's - * built-in asserts already flow as commands). Default true. */ - captureAssertions?: boolean - /** `live` (default) launches the DevTools UI; `trace` skips it. */ - mode?: DevToolsMode - /** Trace output layout — `zip` (default) writes a single archive, - * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ - traceFormat?: TraceFormat - /** Trace output granularity — `session` (default) writes one trace per - * worker session; `spec` writes one trace per spec file. Only applies in - * trace mode. */ - traceGranularity?: TraceGranularity - /** Trace retention policy — gates which traces are kept (e.g. - * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ - tracePolicy?: TraceRetentionPolicy } export interface NightwatchBrowser { @@ -144,12 +114,7 @@ export interface NightwatchBrowser { webdriver?: { host?: string } [key: string]: unknown } - currentTest?: { - name?: string - module?: string - group?: string - [key: string]: unknown - } + currentTest?: NightwatchCurrentTest results?: unknown queue?: unknown } diff --git a/packages/selenium-devtools/src/runnerHooks/cucumber.ts b/packages/selenium-devtools/src/runnerHooks/cucumber.ts index cffc6c76..86f8d3c2 100644 --- a/packages/selenium-devtools/src/runnerHooks/cucumber.ts +++ b/packages/selenium-devtools/src/runnerHooks/cucumber.ts @@ -1,6 +1,11 @@ import { createRequire } from 'node:module' import logger from '@wdio/logger' import { errorMessage } from '@wdio/devtools-core' +import type { + CucumberPickle, + CucumberPickleStep, + TestStatus +} from '@wdio/devtools-shared' import type { RunnerHookCallbacks } from '../types.js' const log = logger('@wdio/selenium-devtools:runnerHooks:cucumber') @@ -178,17 +183,6 @@ interface GherkinFeature { location?: { line?: number } children?: GherkinFeatureChild[] } -interface CucumberPickleStep { - text?: string - astNodeIds?: string[] - location?: { line?: number } -} -interface CucumberPickle { - name?: string - uri?: string - location?: { line?: number } - astNodeIds?: string[] -} interface CucumberTestCase { gherkinDocument?: { feature?: GherkinFeature } pickle?: CucumberPickle @@ -227,7 +221,7 @@ function populateGherkinIndex( } } -type ScenarioState = 'passed' | 'failed' | 'pending' +type ScenarioState = Extract function mapCucumberStatus(status: string): ScenarioState | 'skipped' { const s = status.toUpperCase() diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index 6aa1a33e..d21dcff3 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,15 +23,10 @@ export { type Viewport } from '@wdio/devtools-shared' +import type { BaseDevToolsOptions } from '@wdio/devtools-shared' + // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. -import type { - DevToolsMode, - ScreencastOptions, - TraceFormat, - TraceGranularity, - TraceRetentionPolicy -} from '@wdio/devtools-shared' export type { DevToolsMode, ScreencastFrame, @@ -45,16 +40,7 @@ export interface ExtendedCapabilities extends WebdriverIO.Capabilities { 'wdio:devtoolsOptions'?: ServiceOptions } -export interface ServiceOptions { - /** - * port to launch the application on (default: random) - */ - port?: number - /** - * hostname to launch the application on - * @default localhost - */ - hostname?: string +export interface ServiceOptions extends BaseDevToolsOptions { /** * capabilities used to launch the devtools application * @default @@ -67,27 +53,6 @@ export interface ServiceOptions { * } */ devtoolsCapabilities?: WebdriverIO.Capabilities - /** - * Screencast recording options. When enabled, a continuous video of the - * browser session is recorded and saved as a .webm file. Chrome/Chromium - * uses CDP push mode; all other browsers fall back to screenshot polling. - */ - screencast?: ScreencastOptions - /** Capture node:assert assertions (and failing `expect` matchers) as - * first-class commands. Default true. */ - captureAssertions?: boolean - /** `live` (default) launches the DevTools UI; `trace` skips it. */ - mode?: DevToolsMode - /** Trace output layout — `zip` (default) writes a single archive, - * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ - traceFormat?: TraceFormat - /** Trace output granularity — `session` (default) writes one trace per - * worker session; `spec` writes one trace per spec file. Only applies in - * trace mode. */ - traceGranularity?: TraceGranularity - /** Trace retention policy — gates which traces are kept (e.g. - * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ - tracePolicy?: TraceRetentionPolicy } declare namespace WebdriverIO { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index c2902e19..f90db146 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -283,6 +283,53 @@ export const SCREENCAST_DEFAULTS: Required = { pollIntervalMs: 200 } +/** + * Options every framework adapter accepts. Each adapter's own options interface + * extends this and adds only its framework-specific fields (e.g. WDIO's + * devtoolsCapabilities, Selenium's openUi, Nightwatch's bidi). + */ +export interface BaseDevToolsOptions { + /** Port to launch the application on (default: random). */ + port?: number + /** Hostname to launch the application on. @default localhost */ + hostname?: string + /** Screencast recording options. When enabled, a continuous video of the + * browser session is recorded and saved as a .webm file. */ + screencast?: ScreencastOptions + /** Capture node:assert assertions (and framework `expect` matchers where + * supported) as first-class commands. Default true. */ + captureAssertions?: boolean + /** `live` (default) launches the DevTools UI; `trace` skips it. */ + mode?: DevToolsMode + /** Trace output layout — `zip` (default) writes a single archive, + * `ndjson-directory` unpacks into `trace-/`. Only applies in trace mode. */ + traceFormat?: TraceFormat + /** Trace output granularity — `session` (default) writes one trace per + * worker session; `spec` writes one per spec file. Only applies in trace mode. */ + traceGranularity?: TraceGranularity + /** Trace retention policy — gates which traces are kept (e.g. + * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ + tracePolicy?: TraceRetentionPolicy +} + +/** Minimal Cucumber pickle-step shape — only the fields the adapters read. + * Cucumber's own types vary across versions, so we pin just these. */ +export interface CucumberPickleStep { + text?: string + astNodeIds?: string[] + location?: { line?: number } +} + +/** Minimal Cucumber pickle shape — only the fields the adapters read. `steps` + * is present only where the adapter walks step boundaries (Nightwatch). */ +export interface CucumberPickle { + name?: string + uri?: string + location?: { line?: number } + astNodeIds?: string[] + steps?: CucumberPickleStep[] +} + export interface Metadata { type: TraceType url?: string From 7d6578a9d60797228ea1af226c6a4fb2939b3c79 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 14:00:03 +0530 Subject: [PATCH 29/91] test(service): cover expect-webdriverio matcher capture --- packages/service/tests/assert-capture.test.ts | 88 +++++++++++++++++-- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/packages/service/tests/assert-capture.test.ts b/packages/service/tests/assert-capture.test.ts index 55ed1ee8..d1486257 100644 --- a/packages/service/tests/assert-capture.test.ts +++ b/packages/service/tests/assert-capture.test.ts @@ -6,19 +6,20 @@ import { } from '@wdio/devtools-core' import { captureExpectFailure, + expectAssertionToCommandLog, toCommandError, wireAssertCapture } from '../src/assert-capture.js' import type { SessionCapturer } from '../src/session.js' describe('toCommandError', () => { - it('normalizes a matcher Error object (ANSI stripped)', () => { - const error = Object.assign(new Error('expected 1 to be 2'), { - matcherResult: { matcherName: 'toBe', actual: 1, expected: 2 } - }) + it('normalizes a plain Error object (ANSI stripped)', () => { + // Matcher errors are skipped now (afterAssertion owns them); a plain + // thrown Error still routes through failLastAction, so test that path. + const error = new Error('something broke') expect(toCommandError(error)).toMatchObject({ name: 'Error', - message: 'expected 1 to be 2' + message: 'something broke' }) }) @@ -30,11 +31,17 @@ describe('toCommandError', () => { }) }) - it('returns undefined for node:assert AssertionError, empties and non-errors', () => { + it('returns undefined for self-captured errors (AssertionError / matcher), empties and non-errors', () => { const assertionError = Object.assign(new Error('a !== b'), { name: 'AssertionError' }) expect(toCommandError(assertionError)).toBeUndefined() + // expect-webdriverio matcher errors carry matcherResult and are already + // captured by afterAssertion — must not double-mark via failLastAction. + const matcherError = Object.assign(new Error('expected a to be b'), { + matcherResult: { pass: false } + }) + expect(toCommandError(matcherError)).toBeUndefined() expect(toCommandError(' ')).toBeUndefined() expect(toCommandError(undefined)).toBeUndefined() expect(toCommandError(42)).toBeUndefined() @@ -65,6 +72,17 @@ describe('captureExpectFailure', () => { captureExpectFailure(capturer, 'test-1', undefined, true) expect(capturer.failLastAction).not.toHaveBeenCalled() }) + + it('does not mark an action for a self-captured matcher error', () => { + const capturer = fakeCapturer() + // A failing expect-webdriverio matcher (carries matcherResult) is already + // its own command via afterAssertion — failLastAction must stay off it. + const matcherError = Object.assign(new Error('expected'), { + matcherResult: { pass: false } + }) + captureExpectFailure(capturer, 'test-1', matcherError, true) + expect(capturer.failLastAction).not.toHaveBeenCalled() + }) }) describe('wireAssertCapture', () => { @@ -121,3 +139,61 @@ describe('wireAssertCapture', () => { expect(spy).not.toHaveBeenCalled() }) }) + +describe('expectAssertionToCommandLog', () => { + it('captures a passing matcher as an expect. command', () => { + const entry = expectAssertionToCommandLog( + { + matcherName: 'toHaveTitle', + expectedValue: 'The Internet', + result: { pass: true, message: () => 'ok' } + }, + 'uid-1' + ) + expect(entry).toMatchObject({ + command: 'expect.toHaveTitle', + args: ['The Internet'], + result: 'passed', + testUid: 'uid-1' + }) + expect(entry.error).toBeUndefined() + }) + + it('captures a failing matcher with its ANSI-stripped message as the error', () => { + const entry = expectAssertionToCommandLog( + { + matcherName: 'toHaveText', + expectedValue: 'foo', + result: { pass: false, message: () => 'expected foo' } + }, + undefined + ) + expect(entry.result).toBeUndefined() + expect(entry.error).toMatchObject({ message: 'expected foo' }) + }) + + it('spreads an array expectedValue and reads the typed `result` flag', () => { + // @wdio/types declares the pass flag on `result`, not `pass` — read both. + const entry = expectAssertionToCommandLog( + { + matcherName: 'toHaveAttribute', + expectedValue: ['href', '/x'], + result: { result: true } + }, + undefined + ) + expect(entry).toMatchObject({ + command: 'expect.toHaveAttribute', + args: ['href', '/x'], + result: 'passed' + }) + }) + + it('treats a matcher with no expectedValue as a no-arg assertion', () => { + const entry = expectAssertionToCommandLog( + { matcherName: 'toBeClickable', result: { pass: true } }, + undefined + ) + expect(entry).toMatchObject({ command: 'expect.toBeClickable', args: [] }) + }) +}) From 2c117d55c8271bbe423f2341da94a8af92b42763 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Thu, 9 Jul 2026 16:32:07 +0530 Subject: [PATCH 30/91] feat(core): per-test trace granularity foundation --- packages/core/src/spec-trace-helpers.ts | 188 ++++++++++++++---- packages/core/src/trace-finalizer.ts | 62 ++++-- .../core/tests/spec-trace-helpers.test.ts | 129 ++++++++++++ packages/core/tests/trace-finalizer.test.ts | 104 ++++++++++ 4 files changed, 424 insertions(+), 59 deletions(-) diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index e49f2bd6..dd091efb 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -18,9 +18,15 @@ import { deterministicUid } from './uid.js' // ─── SpecRange ──────────────────────────────────────────────────────────────── -/** Index ranges into a SessionCapturer's flat arrays for a single spec file. */ +/** Index ranges into a SessionCapturer's flat arrays for one trace slice + * (a spec file, or a single test under `test` granularity). */ export interface SpecRange { specFile: string + /** Dedupe/identity key: spec path for spec slices; testUid for test slices, + * or `${testUid}-retry${n}` so each retried attempt is its own slice. */ + key: string + /** Present only for test-granularity slices; the base (non-retry) testUid. */ + testUid?: string commandStartIdx: number consoleStartIdx: number networkStartIdx: number @@ -63,6 +69,25 @@ export function buildSpecSessionId( return `${base}-${hash}-${sessionId.slice(0, 8)}` } +// ─── Test slice session ID ────────────────────────────────────────────────── + +/** + * Build a collision-safe test-level session ID from the slice's spec file, its + * identity `key` (testUid, or `${testUid}-retry${n}` for retries), and the + * parent session ID. Hashing the key keeps retries and sibling tests in the + * same spec from colliding on filename, while the spec basename keeps the name + * human-readable. + */ +export function buildTestSliceSessionId( + specFile: string, + key: string, + sessionId: string +): string { + const base = sanitizeSpecName(specFile) + const hash = deterministicUid(key).split('-').pop()!.slice(0, 8) + return `${base}-${hash}-${sessionId.slice(0, 8)}` +} + // ─── TraceCapturer slice ───────────────────────────────────────────────────── /** @@ -127,11 +152,29 @@ export function filterTestMetadataBySpec( return filtered } +/** + * Filter a full `testUid → metadata` map down to a single test's entry. The + * per-test analog of {@link filterTestMetadataBySpec}: a test slice's metadata + * is just that one test's entry, attached as its tracingGroup name. + */ +export function filterTestMetadataByUid( + allMetadata: TestMetadataMap, + testUid: string +): TestMetadataMap { + const filtered: TestMetadataMap = new Map() + const entry = allMetadata.get(testUid) + if (entry) { + filtered.set(testUid, entry) + } + return filtered +} + // ─── Spec boundary recording ────────────────────────────────────────────────── /** - * Minimal context needed by `recordSpecBoundary` to detect spec-file - * transitions and capture array index ranges. + * Minimal context needed by `recordSliceBoundary` to detect spec-file / test + * transitions and capture array index ranges. `flushedSpecs` holds already- + * flushed slice keys (spec paths or test keys), shared with the finalizer. */ export interface SpecBoundaryContext { specRanges: SpecRange[] @@ -146,32 +189,28 @@ export interface SpecBoundaryContext { actionSnapshots: ArrayLike } -/** - * Record a spec-file boundary. When `traceGranularity` is `'spec'` and the - * spec file has changed, this pushes a new `SpecRange` and returns the - * previous range so the caller can flush its trace artifact. - * - * Returns `null` when no flush is needed (same spec, or granularity isn't - * `'spec'`, or no capturer). - */ -export function recordSpecBoundary( +/** Push a new slice range and return the previous (unflushed) range to flush. + * `suppressSameKey` skips recording when the incoming key matches the last + * range's — used for spec granularity so consecutive tests in one file share + * a slice; test granularity records every attempt (retries included). */ +function pushSliceRange( ctx: SpecBoundaryContext, specFile: string, - traceGranularity: TraceGranularity | undefined + key: string, + testUid: string | undefined, + suppressSameKey: boolean ): SpecRange | null { - if (traceGranularity !== 'spec') { - return null - } const lastRange = ctx.specRanges[ctx.specRanges.length - 1] - if (lastRange && lastRange.specFile === specFile) { + if (suppressSameKey && lastRange && lastRange.key === key) { return null } - const prevRange = - lastRange && !ctx.flushedSpecs.has(lastRange.specFile) ? lastRange : null + lastRange && !ctx.flushedSpecs.has(lastRange.key) ? lastRange : null ctx.specRanges.push({ specFile, + key, + testUid, commandStartIdx: ctx.capturer.commandsLog.length, consoleStartIdx: ctx.capturer.consoleLogs.length, networkStartIdx: ctx.capturer.networkRequests.length, @@ -183,6 +222,48 @@ export function recordSpecBoundary( return prevRange } +/** + * Record a trace-slice boundary. For `spec` granularity, a new slice starts + * when the spec file changes (existing behavior). For `test` granularity, a + * new slice starts on every recorded test — including retries: a repeated + * `testUid` is keyed `${testUid}-retry${n}` so each attempt is its own slice. + * Returns the previous, not-yet-flushed range so the caller can flush it, or + * `null` when nothing needs flushing (same spec, missing testUid, or a + * non-sliced granularity). + */ +export function recordSliceBoundary( + ctx: SpecBoundaryContext, + granularity: TraceGranularity | undefined, + specFile: string, + testUid?: string +): SpecRange | null { + if (granularity === 'spec') { + return pushSliceRange(ctx, specFile, specFile, undefined, true) + } + if (granularity === 'test' && testUid !== undefined) { + const priorAttempts = ctx.specRanges.filter( + (r) => r.testUid === testUid + ).length + const key = + priorAttempts === 0 ? testUid : `${testUid}-retry${priorAttempts}` + return pushSliceRange(ctx, specFile, key, testUid, false) + } + return null +} + +/** + * Record a spec-file boundary. Thin back-compat wrapper over + * {@link recordSliceBoundary}; behavior is unchanged for `spec` granularity + * and returns `null` for every other granularity. + */ +export function recordSpecBoundary( + ctx: SpecBoundaryContext, + specFile: string, + traceGranularity: TraceGranularity | undefined +): SpecRange | null { + return recordSliceBoundary(ctx, traceGranularity, specFile) +} + // ─── Spec trace I/O ──────────────────────────────────────────────────────────── /** @@ -205,41 +286,66 @@ export interface WriteSpecTraceInput { capabilities?: unknown } -/** - * Write a standalone trace artifact (zip or ndjson-directory) for a single - * spec file. This is the shared I/O path — all three adapters delegate to it - * from their own `flushSpecTrace` wrappers. - */ -export async function writeSpecTrace( - input: WriteSpecTraceInput +/** Slice the parent capturer/snapshots for one range and write the artifact + * under `sliceSessionId` with the pre-filtered `testMetadata`. Shared by the + * spec and test write paths so both slice identically. */ +async function writeSliceTrace( + input: WriteSpecTraceInput, + sliceSessionId: string, + testMetadata: TestMetadataMap ): Promise { - const specCapturer = buildSpecCapturer( + const sliceCapturer = buildSpecCapturer( input.capturer, input.range, input.nextRange ) - const specSnapshots = input.actionSnapshots.slice( + const sliceSnapshots = input.actionSnapshots.slice( input.range.snapshotCount, input.nextRange?.snapshotCount ?? input.actionSnapshots.length ) - const specSessionId = buildSpecSessionId( - input.range.specFile, - input.sessionId - ) - - const testMetadata = filterTestMetadataBySpec( - input.testMetadata, - input.range.specFile - ) - - return writeTraceZip(specCapturer, { + return writeTraceZip(sliceCapturer, { outputDir: input.outputDir, - sessionId: specSessionId, + sessionId: sliceSessionId, capabilities: input.capabilities, - actionSnapshots: specSnapshots.length > 0 ? specSnapshots : undefined, + actionSnapshots: sliceSnapshots.length > 0 ? sliceSnapshots : undefined, format: input.format, testMetadata }) } + +/** + * Write a standalone trace artifact (zip or ndjson-directory) for a single + * spec file. This is the shared I/O path — all three adapters delegate to it + * from their own `flushSpecTrace` wrappers. + */ +export async function writeSpecTrace( + input: WriteSpecTraceInput +): Promise { + return writeSliceTrace( + input, + buildSpecSessionId(input.range.specFile, input.sessionId), + filterTestMetadataBySpec(input.testMetadata, input.range.specFile) + ) +} + +/** + * Write a standalone trace artifact for a single test slice. Reuses + * {@link WriteSpecTraceInput}; the slice identity comes from `range.key` + * (retry-aware) and its metadata from the base `range.testUid`. + */ +export async function writeTestSliceTrace( + input: WriteSpecTraceInput +): Promise { + const testUid = input.range.testUid ?? input.range.key + return writeSliceTrace( + input, + buildTestSliceSessionId( + input.range.specFile, + input.range.key, + input.sessionId + ), + filterTestMetadataByUid(input.testMetadata, testUid) + ) +} diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index 1750d3a4..840d9a51 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -17,7 +17,9 @@ import type { import { errorMessage } from './error.js' import { filterTestMetadataBySpec, + filterTestMetadataByUid, writeSpecTrace, + writeTestSliceTrace, type SpecRange } from './spec-trace-helpers.js' import { shouldRetainTrace, type TestOutcome } from './trace-retention.js' @@ -69,6 +71,22 @@ const SPEC_WITHOUT_BOUNDARIES_WARNING = '(the runner may not expose per-test hooks). Falling back to ' + 'session-level trace.' +const TEST_WITHOUT_BOUNDARIES_WARNING = + 'traceGranularity is "test" but no test boundaries were detected ' + + '(the runner may not expose per-test hooks). Falling back to ' + + 'session-level trace.' + +/** Above this many slices, warn to pair granularity with a retention policy. */ +const SLICE_COUNT_WARN_THRESHOLD = 200 + +function sliceCountWarning(count: number): string { + return ( + `traceGranularity produced ${count} trace slices. Consider pairing it ` + + 'with a retention policy (e.g. tracePolicy: "retain-on-failure") to ' + + 'avoid writing hundreds of trace archives.' + ) +} + /** Project a metadata slice onto the retention evaluator's outcome shape. */ function toOutcomes(metadata: TestMetadataMap): TestOutcome[] { return Array.from(metadata.values(), (m) => ({ @@ -93,30 +111,31 @@ function shouldRetain( } /** - * Policy-aware single-range flush: dedupes via `ctx.flushed`, applies the - * retention decision, and delegates the byte-level slicing/naming to - * `writeSpecTrace`. Returns the artifact, or undefined when the range was - * already flushed. + * Policy-aware single-range flush: dedupes via `ctx.flushed` on the slice + * `key`, applies the retention decision, and delegates the byte-level + * slicing/naming to `writeSpecTrace` (spec slices) or `writeTestSliceTrace` + * (test slices, distinguished by `range.testUid`). Returns the artifact, or + * undefined when the range was already flushed. */ export async function flushRangeTrace( ctx: TraceExportContext, range: SpecRange, nextRange?: SpecRange ): Promise { - if (ctx.flushed.has(range.specFile)) { + if (ctx.flushed.has(range.key)) { return undefined } - ctx.flushed.add(range.specFile) + ctx.flushed.add(range.key) - const sliceMetadata = filterTestMetadataBySpec( - ctx.testMetadata, - range.specFile - ) + const isTestSlice = range.testUid !== undefined + const sliceMetadata = isTestSlice + ? filterTestMetadataByUid(ctx.testMetadata, range.testUid!) + : filterTestMetadataBySpec(ctx.testMetadata, range.specFile) const artifact: TraceArtifact = { kind: 'trace', path: '', - scope: 'spec', - key: range.specFile, + scope: isTestSlice ? 'test' : 'spec', + key: range.key, testUids: Array.from(sliceMetadata.keys()), retained: shouldRetain(ctx, sliceMetadata) } @@ -125,7 +144,8 @@ export async function flushRangeTrace( return artifact } - artifact.path = await writeSpecTrace({ + const writeSlice = isTestSlice ? writeTestSliceTrace : writeSpecTrace + artifact.path = await writeSlice({ range, nextRange, capturer: ctx.capturer, @@ -138,7 +158,7 @@ export async function flushRangeTrace( }) ctx.log?.( 'info', - `Trace for spec "${range.specFile}" saved to ${artifact.path}` + `Trace for ${isTestSlice ? 'test' : 'spec'} "${range.key}" saved to ${artifact.path}` ) ctx.onArtifact?.(artifact) return artifact @@ -202,9 +222,9 @@ async function flushAllRanges( /** * Entry point for the after/end-of-run hook. No-op outside trace mode. Awaits - * any pending snapshot captures, then fans out to per-spec or session writes. - * `spec` granularity with no recorded boundaries warns and falls back to a - * single session-level trace. + * any pending snapshot captures, then fans out to per-spec, per-test, or + * session writes. `spec`/`test` granularity with no recorded boundaries warns + * and falls back to a single session-level trace. */ export async function finalizeTraceExport( ctx: TraceExportContext @@ -215,11 +235,17 @@ export async function finalizeTraceExport( if (ctx.awaitPending?.length) { await Promise.allSettled(ctx.awaitPending) } - if (ctx.granularity === 'spec' && ctx.ranges.length > 0) { + const sliced = ctx.granularity === 'spec' || ctx.granularity === 'test' + if (sliced && ctx.ranges.length > 0) { + if (ctx.ranges.length > SLICE_COUNT_WARN_THRESHOLD) { + ctx.log?.('warn', sliceCountWarning(ctx.ranges.length)) + } return flushAllRanges(ctx) } if (ctx.granularity === 'spec') { ctx.log?.('warn', SPEC_WITHOUT_BOUNDARIES_WARNING) + } else if (ctx.granularity === 'test') { + ctx.log?.('warn', TEST_WITHOUT_BOUNDARIES_WARNING) } const artifact = await safely(ctx, () => writeSessionTrace(ctx)) return artifact ? [artifact] : [] diff --git a/packages/core/tests/spec-trace-helpers.test.ts b/packages/core/tests/spec-trace-helpers.test.ts index f41efc5c..4be8ad6e 100644 --- a/packages/core/tests/spec-trace-helpers.test.ts +++ b/packages/core/tests/spec-trace-helpers.test.ts @@ -2,8 +2,13 @@ import { describe, it, expect } from 'vitest' import { buildSpecCapturer, buildSpecSessionId, + buildTestSliceSessionId, filterTestMetadataBySpec, + filterTestMetadataByUid, + recordSliceBoundary, + recordSpecBoundary, sanitizeSpecName, + type SpecBoundaryContext, type SpecRange, type TraceCapturer } from '@wdio/devtools-core' @@ -30,6 +35,7 @@ function capturer(): TraceCapturer { const range = (over: Partial = {}): SpecRange => ({ specFile: '/a.js', + key: over.key ?? over.specFile ?? '/a.js', commandStartIdx: 0, consoleStartIdx: 0, networkStartIdx: 0, @@ -39,6 +45,24 @@ const range = (over: Partial = {}): SpecRange => ({ ...over }) +function boundaryCtx( + over: Partial = {} +): SpecBoundaryContext { + return { + specRanges: [], + flushedSpecs: new Set(), + capturer: { + commandsLog: [], + consoleLogs: [], + networkRequests: [], + mutations: [], + traceLogs: [] + }, + actionSnapshots: [], + ...over + } +} + describe('buildSpecCapturer', () => { it('slices from the range start to the end when no nextRange is given', () => { const sliced = buildSpecCapturer(capturer(), range({ commandStartIdx: 2 })) @@ -91,3 +115,108 @@ describe('spec name / session id', () => { expect(a.startsWith('login-')).toBe(true) }) }) + +describe('filterTestMetadataByUid', () => { + it('keeps only the entry for the given uid', () => { + const all: TestMetadataMap = new Map([ + ['u1', { title: 'A', specFile: '/a.js' }], + ['u2', { title: 'B', specFile: '/b.js' }] + ]) + expect([...filterTestMetadataByUid(all, 'u1').keys()]).toEqual(['u1']) + expect(filterTestMetadataByUid(all, 'missing').size).toBe(0) + }) +}) + +describe('buildTestSliceSessionId', () => { + it('derives distinct, stable ids per key and stays readable', () => { + const a = buildTestSliceSessionId('/dir/login.js', 'u1', 'session-xyz') + const b = buildTestSliceSessionId('/dir/login.js', 'u2', 'session-xyz') + const retry = buildTestSliceSessionId( + '/dir/login.js', + 'u1-retry1', + 'session-xyz' + ) + expect(a).not.toBe(b) + expect(a).not.toBe(retry) + expect(a).toBe( + buildTestSliceSessionId('/dir/login.js', 'u1', 'session-xyz') + ) + expect(a.startsWith('login-')).toBe(true) + }) +}) + +describe('recordSliceBoundary (test granularity)', () => { + it('opens a new slice per test uid and returns the previous range', () => { + const ctx = boundaryCtx({ + capturer: { + commandsLog: [1, 2], + consoleLogs: [1], + networkRequests: [], + mutations: [1, 2, 3], + traceLogs: [] + }, + actionSnapshots: [1, 1] + }) + expect(recordSliceBoundary(ctx, 'test', '/a.js', 'u1')).toBeNull() + expect(ctx.specRanges).toHaveLength(1) + expect(ctx.specRanges[0]!.key).toBe('u1') + expect(ctx.specRanges[0]!.testUid).toBe('u1') + expect(ctx.specRanges[0]!.commandStartIdx).toBe(2) + expect(ctx.specRanges[0]!.snapshotCount).toBe(2) + + const prev = recordSliceBoundary(ctx, 'test', '/a.js', 'u2') + expect(prev?.key).toBe('u1') + expect(ctx.specRanges).toHaveLength(2) + expect(ctx.specRanges[1]!.key).toBe('u2') + }) + + it('keys each retry of the same uid as its own slice', () => { + const ctx = boundaryCtx() + recordSliceBoundary(ctx, 'test', '/a.js', 'u1') + recordSliceBoundary(ctx, 'test', '/a.js', 'u1') + recordSliceBoundary(ctx, 'test', '/a.js', 'u1') + expect(ctx.specRanges.map((r) => r.key)).toEqual([ + 'u1', + 'u1-retry1', + 'u1-retry2' + ]) + expect(ctx.specRanges.every((r) => r.testUid === 'u1')).toBe(true) + }) + + it('returns null when no testUid is provided', () => { + const ctx = boundaryCtx() + expect(recordSliceBoundary(ctx, 'test', '/a.js')).toBeNull() + expect(ctx.specRanges).toHaveLength(0) + }) + + it('does not return a previous range already in the flushed set', () => { + const ctx = boundaryCtx() + recordSliceBoundary(ctx, 'test', '/a.js', 'u1') + ctx.flushedSpecs.add('u1') + expect(recordSliceBoundary(ctx, 'test', '/a.js', 'u2')).toBeNull() + }) +}) + +describe('recordSliceBoundary / recordSpecBoundary (spec granularity)', () => { + it('keeps the spec-file behavior: one slice per spec, key equals specFile', () => { + const ctx = boundaryCtx() + expect(recordSliceBoundary(ctx, 'spec', '/a.js')).toBeNull() + expect(recordSliceBoundary(ctx, 'spec', '/a.js')).toBeNull() + expect(ctx.specRanges).toHaveLength(1) + expect(ctx.specRanges[0]!.key).toBe('/a.js') + expect(ctx.specRanges[0]!.testUid).toBeUndefined() + + const prev = recordSliceBoundary(ctx, 'spec', '/b.js') + expect(prev?.specFile).toBe('/a.js') + expect(ctx.specRanges).toHaveLength(2) + }) + + it('recordSpecBoundary returns null for session and test granularities', () => { + expect(recordSpecBoundary(boundaryCtx(), '/a.js', 'session')).toBeNull() + expect(recordSpecBoundary(boundaryCtx(), '/a.js', 'test')).toBeNull() + const ctx = boundaryCtx() + recordSpecBoundary(ctx, '/a.js', 'spec') + expect(ctx.specRanges).toHaveLength(1) + expect(ctx.specRanges[0]!.key).toBe('/a.js') + }) +}) diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index 73a9ae36..04362b01 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -4,6 +4,7 @@ import path from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { buildSpecSessionId, + buildTestSliceSessionId, finalizeTraceExport, flushRangeTrace, type SpecRange, @@ -51,6 +52,7 @@ function meta(entries: Array<[string, TestMetadataEntry]>): TestMetadataMap { function range(specFile: string, startIdx: number): SpecRange { return { specFile, + key: specFile, commandStartIdx: startIdx, consoleStartIdx: 0, networkStartIdx: 0, @@ -60,6 +62,15 @@ function range(specFile: string, startIdx: number): SpecRange { } } +function testRange( + specFile: string, + testUid: string, + startIdx: number, + key = testUid +): SpecRange { + return { ...range(specFile, startIdx), key, testUid } +} + describe('finalizeTraceExport', () => { let outputDir: string let artifacts: TraceArtifact[] @@ -158,6 +169,99 @@ describe('finalizeTraceExport', () => { ).toBe(true) }) + it('fans out one trace per recorded test range', async () => { + const result = await finalizeTraceExport( + baseCtx({ + granularity: 'test', + ranges: [testRange('/a.js', 'a1', 0), testRange('/a.js', 'a2', 2)], + testMetadata: meta([ + ['a1', { title: 'A1', specFile: '/a.js' }], + ['a2', { title: 'A2', specFile: '/a.js' }] + ]) + }) + ) + expect(result).toHaveLength(2) + expect(result.map((a) => a.scope)).toEqual(['test', 'test']) + expect(result.map((a) => a.key)).toEqual(['a1', 'a2']) + // Each test slice carries only its own test's metadata. + expect(result[0]!.testUids).toEqual(['a1']) + expect(result[1]!.testUids).toEqual(['a2']) + const nameA1 = buildTestSliceSessionId('/a.js', 'a1', 'abcd1234') + const nameA2 = buildTestSliceSessionId('/a.js', 'a2', 'abcd1234') + expect(nameA1).not.toBe(nameA2) + expect(await exists(path.join(outputDir, `trace-${nameA1}.zip`))).toBe(true) + expect(await exists(path.join(outputDir, `trace-${nameA2}.zip`))).toBe(true) + }) + + it('treats a retry-keyed range as its own test slice', async () => { + const result = await finalizeTraceExport( + baseCtx({ + granularity: 'test', + ranges: [ + testRange('/a.js', 'a1', 0), + testRange('/a.js', 'a1', 2, 'a1-retry1') + ], + testMetadata: meta([['a1', { title: 'A1', specFile: '/a.js' }]]) + }) + ) + expect(result).toHaveLength(2) + expect(result.map((a) => a.key)).toEqual(['a1', 'a1-retry1']) + const first = buildTestSliceSessionId('/a.js', 'a1', 'abcd1234') + const retry = buildTestSliceSessionId('/a.js', 'a1-retry1', 'abcd1234') + expect(first).not.toBe(retry) + expect(await exists(path.join(outputDir, `trace-${first}.zip`))).toBe(true) + expect(await exists(path.join(outputDir, `trace-${retry}.zip`))).toBe(true) + }) + + it('warns and falls back to a session trace when test has no boundaries', async () => { + const result = await finalizeTraceExport( + baseCtx({ granularity: 'test', ranges: [] }) + ) + expect(result).toHaveLength(1) + expect(result[0]!.scope).toBe('session') + expect(await exists(path.join(outputDir, 'trace-abcd1234.zip'))).toBe(true) + expect( + logs.some( + ([level, msg]) => + level === 'warn' && msg.includes('no test boundaries were detected') + ) + ).toBe(true) + }) + + it('warns to pair with a retention policy above the slice-count threshold', async () => { + const ranges = Array.from({ length: 201 }, (_, i) => + testRange('/a.js', `u${i}`, i) + ) + const result = await finalizeTraceExport( + baseCtx({ + granularity: 'test', + // retain-on-failure + all-passing declines every write, so the guard + // is exercised without emitting 201 archives. + policy: 'retain-on-failure', + ranges, + testMetadata: meta( + ranges.map((r) => [ + r.testUid!, + { + title: r.testUid!, + specFile: '/a.js', + state: 'passed', + attempt: 0 + } + ]) + ) + }) + ) + expect(result).toHaveLength(201) + expect(result.every((a) => !a.retained)).toBe(true) + expect(await fs.readdir(outputDir)).toEqual([]) + expect( + logs.some( + ([level, msg]) => level === 'warn' && msg.includes('retention policy') + ) + ).toBe(true) + }) + it('does not rewrite a range already in the flushed set', async () => { const flushed = new Set(['/a.js']) const result = await finalizeTraceExport( From 647e396bcbd021dd6ff893d4b9640ce10e7be823 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 10 Jul 2026 23:23:01 +0530 Subject: [PATCH 31/91] feat(trace): per-test granularity, output folders, and test-results/ parent --- .gitignore | 3 + packages/core/src/finalize-screencast.ts | 3 + packages/core/src/output-dir.ts | 13 +- packages/core/src/spec-trace-helpers.ts | 70 ++++- packages/core/src/trace-exporter.ts | 8 +- packages/core/src/trace-finalizer.ts | 41 ++- packages/core/tests/output-dir.test.ts | 25 +- .../core/tests/spec-trace-helpers.test.ts | 119 ++++++++- packages/core/tests/trace-finalizer.test.ts | 24 +- .../src/cucumber-lifecycle.ts | 27 +- .../src/plugin-internals.ts | 13 +- .../nightwatch-devtools/src/test-lifecycle.ts | 11 +- .../nightwatch-devtools/src/trace-slices.ts | 102 ++++++++ .../tests/traceSlices.test.ts | 155 +++++++++++ packages/selenium-devtools/src/index.ts | 23 +- .../selenium-devtools/src/plugin-internals.ts | 2 + .../src/session-lifecycle.ts | 116 +++++++-- .../tests/attempt-capture.test.ts | 1 + .../tests/trace-granularity.test.ts | 191 ++++++++++++++ packages/service/src/trace-slices.ts | 49 ++++ .../service/tests/trace-granularity.test.ts | 240 ++++++++++++++++++ 21 files changed, 1167 insertions(+), 69 deletions(-) create mode 100644 packages/nightwatch-devtools/src/trace-slices.ts create mode 100644 packages/nightwatch-devtools/tests/traceSlices.test.ts create mode 100644 packages/selenium-devtools/tests/trace-granularity.test.ts create mode 100644 packages/service/src/trace-slices.ts create mode 100644 packages/service/tests/trace-granularity.test.ts diff --git a/.gitignore b/.gitignore index 47c58732..e8a7a934 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ packages/nightwatch-devtools/nightwatch-video-*.webm trace-*.zip examples/**/trace-*/ +# test results +examples/**/test-results/ + # vitest --coverage output coverage/ diff --git a/packages/core/src/finalize-screencast.ts b/packages/core/src/finalize-screencast.ts index e8c6b00c..4e1b0b9c 100644 --- a/packages/core/src/finalize-screencast.ts +++ b/packages/core/src/finalize-screencast.ts @@ -62,6 +62,9 @@ export async function finalizeScreencast({ const candidate = outputDir || process.cwd() let videoPath = path.join(candidate, fileName) try { + // Create the (test-results) dir if absent, then confirm it's writable; + // fall back to tmpdir on any failure so a bad path never aborts the run. + fs.mkdirSync(candidate, { recursive: true }) fs.accessSync(candidate, fs.constants.W_OK) } catch { videoPath = path.join(os.tmpdir(), fileName) diff --git a/packages/core/src/output-dir.ts b/packages/core/src/output-dir.ts index fa659ace..463c5960 100644 --- a/packages/core/src/output-dir.ts +++ b/packages/core/src/output-dir.ts @@ -25,6 +25,9 @@ export interface ResolveAdapterOutputDirInput { const NODE_MODULES_SEGMENT = `${path.sep}node_modules${path.sep}` +/** All run output is grouped under this subfolder. */ +const OUTPUT_SUBDIR = 'test-results' + function isWritable(dir: string): boolean { try { fs.accessSync(dir, fs.constants.W_OK) @@ -36,9 +39,11 @@ function isWritable(dir: string): boolean { /** * Resolve the directory where an adapter should write output files - * (screencast .webm, trace JSON, etc.). + * (screencast .webm, trace JSON, etc.). Every artifact is grouped under a + * single `test-results/` subfolder so a run's output is + * self-contained regardless of where the base directory resolves to. * - * Priority: + * The base directory is resolved by priority: * 1. `userConfiguredDir` — explicit opt-in, honored as-is. * 2. `dirname(testFilePath)` — same folder as the spec that just ran. * 3. `dirname(configPath)` — fallback to the framework config dir. @@ -56,6 +61,10 @@ function isWritable(dir: string): boolean { export function resolveAdapterOutputDir( input: ResolveAdapterOutputDirInput = {} ): string { + return path.join(resolveBaseDir(input), OUTPUT_SUBDIR) +} + +function resolveBaseDir(input: ResolveAdapterOutputDirInput): string { const fallback = input.fallbackDir ?? process.cwd() // userConfiguredDir bypasses the node_modules and writability filters // because the user opted into it explicitly — surprising overrides are diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index dd091efb..2eb69298 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -6,6 +6,7 @@ * is the single source of truth — adapters import from here. */ +import path from 'node:path' import type { ActionSnapshot, TestMetadataMap, @@ -88,6 +89,44 @@ export function buildTestSliceSessionId( return `${base}-${hash}-${sessionId.slice(0, 8)}` } +// ─── Test slice output folder ──────────────────────────────────────────────── + +/** Max slug length so a title doesn't blow past filesystem path limits. */ +const MAX_SLUG_LENGTH = 60 + +/** Lowercase, collapse runs of non-alphanumerics to `-`, trim edge dashes, + * and cap length (trimming a dash the cut may leave behind). */ +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, MAX_SLUG_LENGTH) + .replace(/-+$/g, '') +} + +/** + * Build the per-test output folder name: `--<browser>[-retryN]`. + * The spec basename is sanitized via {@link sanitizeSpecName}; title and browser + * are slugified. An empty title falls back to a short hash of `key`, and a + * `${uid}-retry${n}` key appends a `-retry<N>` suffix so retries don't collide. + */ +export function buildTestSliceFolder( + specFile: string, + testTitle: string | undefined, + browser: string | undefined, + key: string +): string { + const specBase = sanitizeSpecName(path.basename(specFile)) + const titleSlug = + slugify(testTitle ?? '') || + deterministicUid(key).split('-').pop()!.slice(0, 8) + const browserSlug = slugify(browser ?? '') || 'browser' + const retryMatch = key.match(/-retry(\d+)$/) + const retrySuffix = retryMatch ? `-retry${retryMatch[1]}` : '' + return `${specBase}-${titleSlug}-${browserSlug}${retrySuffix}` +} + // ─── TraceCapturer slice ───────────────────────────────────────────────────── /** @@ -288,11 +327,13 @@ export interface WriteSpecTraceInput { /** Slice the parent capturer/snapshots for one range and write the artifact * under `sliceSessionId` with the pre-filtered `testMetadata`. Shared by the - * spec and test write paths so both slice identically. */ + * spec and test write paths so both slice identically. `overrides` lets the + * test path redirect into a named folder with a fixed `trace` file stem. */ async function writeSliceTrace( input: WriteSpecTraceInput, sliceSessionId: string, - testMetadata: TestMetadataMap + testMetadata: TestMetadataMap, + overrides: { outputDir?: string; fileStem?: string } = {} ): Promise<string> { const sliceCapturer = buildSpecCapturer( input.capturer, @@ -306,8 +347,9 @@ async function writeSliceTrace( ) return writeTraceZip(sliceCapturer, { - outputDir: input.outputDir, + outputDir: overrides.outputDir ?? input.outputDir, sessionId: sliceSessionId, + fileStem: overrides.fileStem, capabilities: input.capabilities, actionSnapshots: sliceSnapshots.length > 0 ? sliceSnapshots : undefined, format: input.format, @@ -331,14 +373,27 @@ export async function writeSpecTrace( } /** - * Write a standalone trace artifact for a single test slice. Reuses - * {@link WriteSpecTraceInput}; the slice identity comes from `range.key` - * (retry-aware) and its metadata from the base `range.testUid`. + * Write a standalone trace artifact for a single test slice into its own + * folder: `<outputDir>/<specBasename>-<titleSlug>-<browserSlug>[-retryN]/trace.zip`. + * Reuses {@link WriteSpecTraceInput}; the folder is the slice's external + * identity (title/browser/retry), while {@link buildTestSliceSessionId} names + * the sessionId embedded inside the archive. */ export async function writeTestSliceTrace( input: WriteSpecTraceInput ): Promise<string> { const testUid = input.range.testUid ?? input.range.key + const title = input.testMetadata.get(testUid)?.title + // capabilities is framework-typed unknown; read only browserName here. + const browserName = ( + input.capabilities as { browserName?: string } | undefined + )?.browserName + const folder = buildTestSliceFolder( + input.range.specFile, + title, + browserName, + input.range.key + ) return writeSliceTrace( input, buildTestSliceSessionId( @@ -346,6 +401,7 @@ export async function writeTestSliceTrace( input.range.key, input.sessionId ), - filterTestMetadataByUid(input.testMetadata, testUid) + filterTestMetadataByUid(input.testMetadata, testUid), + { outputDir: path.join(input.outputDir, folder), fileStem: 'trace' } ) } diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index bc223e14..40079b9a 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -466,6 +466,9 @@ export interface WriteTraceZipOptions { format?: TraceFormat /** Test metadata keyed by testUid for Tracing.tracingGroup events. */ testMetadata?: TestMetadataMap + /** Base name for the artifact (zip file stem / directory name). Defaults to + * `trace-<sessionId>`; per-test slices pass `'trace'` inside a named folder. */ + fileStem?: string } /** @@ -501,14 +504,15 @@ export async function writeTraceZip( wallTimeOverride: capturer.startWallTime, testMetadata: opts.testMetadata } + const stem = opts.fileStem ?? `trace-${opts.sessionId}` if (opts.format === 'ndjson-directory') { - const dir = path.join(opts.outputDir, `trace-${opts.sessionId}`) + const dir = path.join(opts.outputDir, stem) await fs.mkdir(dir, { recursive: true }) await exportTraceDirectory(traceLog, dir, exportOpts) return dir } const zip = await exportTraceZip(traceLog, exportOpts) - const zipPath = path.join(opts.outputDir, `trace-${opts.sessionId}.zip`) + const zipPath = path.join(opts.outputDir, `${stem}.zip`) await fs.writeFile(zipPath, zip) return zipPath } diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index 840d9a51..415b6289 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -164,6 +164,36 @@ export async function flushRangeTrace( return artifact } +/** + * Flush one slice via {@link flushRangeTrace}, logging the shared spec/test + * error string on failure so a failed boundary flush can't abort the next test. + * All three adapters wrapped this identically; the label + identity are derived + * from `range.testUid` (test slice → `test "<key>"`, else `spec "<specFile>"`), + * so each call site keeps its exact message. Errors are logged and swallowed + * (resolves `undefined`), so callers `await` it when the write must land before + * a retry overwrites metadata, or fire-and-forget (`void`, or tracked in an + * in-flight list) otherwise. Callers keep their own find-current-range strategy + * and pass the resolved range in. + */ +export async function flushRangeLogged( + ctx: TraceExportContext, + range: SpecRange +): Promise<TraceArtifact | undefined> { + try { + return await flushRangeTrace(ctx, range) + } catch (err) { + const label = + range.testUid !== undefined + ? `test "${range.key}"` + : `spec "${range.specFile}"` + ctx.log?.( + 'warn', + `Failed to flush trace for ${label}: ${errorMessage(err)}` + ) + return undefined + } +} + async function writeSessionTrace( ctx: TraceExportContext ): Promise<TraceArtifact | undefined> { @@ -211,8 +241,15 @@ async function flushAllRanges( ctx: TraceExportContext ): Promise<TraceArtifact[]> { const artifacts: TraceArtifact[] = [] - for (const range of ctx.ranges) { - const artifact = await safely(ctx, () => flushRangeTrace(ctx, range)) + // Bound each slice by the next range's start indices; the final range (no + // nextRange) runs to the end of the arrays. Without this, every slice would + // run to the end and each test slice would swallow all later tests. + for (let i = 0; i < ctx.ranges.length; i++) { + const range = ctx.ranges[i]! + const nextRange = ctx.ranges[i + 1] + const artifact = await safely(ctx, () => + flushRangeTrace(ctx, range, nextRange) + ) if (artifact) { artifacts.push(artifact) } diff --git a/packages/core/tests/output-dir.test.ts b/packages/core/tests/output-dir.test.ts index 12d80315..f44ba3ef 100644 --- a/packages/core/tests/output-dir.test.ts +++ b/packages/core/tests/output-dir.test.ts @@ -4,6 +4,9 @@ import path from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { resolveAdapterOutputDir } from '../src/output-dir.js' +/** Every resolved dir is grouped under this subfolder (Playwright-style). */ +const grouped = (base: string) => path.join(base, 'test-results') + describe('resolveAdapterOutputDir', () => { let tmpDir: string @@ -15,10 +18,16 @@ describe('resolveAdapterOutputDir', () => { fs.rmSync(tmpDir, { recursive: true, force: true }) }) + it('groups the resolved dir under a test-results/ subfolder', () => { + expect(resolveAdapterOutputDir({ fallbackDir: tmpDir })).toBe( + path.join(tmpDir, 'test-results') + ) + }) + it('returns userConfiguredDir verbatim when set, even if non-existent', () => { expect( resolveAdapterOutputDir({ userConfiguredDir: '/whatever/path' }) - ).toBe('/whatever/path') + ).toBe(grouped('/whatever/path')) }) it('prefers testFilePath dir over configPath dir over fallback', () => { @@ -35,14 +44,14 @@ describe('resolveAdapterOutputDir', () => { configPath, fallbackDir: tmpDir }) - ).toBe(path.dirname(testFile)) + ).toBe(grouped(path.dirname(testFile))) }) it('falls back to configPath dir when testFilePath is missing', () => { const configPath = path.join(tmpDir, 'wdio.conf.ts') fs.writeFileSync(configPath, '') expect(resolveAdapterOutputDir({ configPath, fallbackDir: tmpDir })).toBe( - tmpDir + grouped(tmpDir) ) }) @@ -59,11 +68,11 @@ describe('resolveAdapterOutputDir', () => { testFilePath: testFile, configPath }) - ).toBe(tmpDir) + ).toBe(grouped(tmpDir)) }) it('falls back to process.cwd() when no inputs are given', () => { - expect(resolveAdapterOutputDir()).toBe(process.cwd()) + expect(resolveAdapterOutputDir()).toBe(grouped(process.cwd())) }) it('falls back to fallbackDir when given and none of the candidates are writable', () => { @@ -72,12 +81,12 @@ describe('resolveAdapterOutputDir', () => { testFilePath: '/definitely/missing/a.test.ts', fallbackDir: tmpDir }) - ).toBe(tmpDir) + ).toBe(grouped(tmpDir)) }) it('userConfiguredDir bypasses node_modules skip (explicit opt-in)', () => { const nm = '/some/node_modules/pkg/dir' - expect(resolveAdapterOutputDir({ userConfiguredDir: nm })).toBe(nm) + expect(resolveAdapterOutputDir({ userConfiguredDir: nm })).toBe(grouped(nm)) }) it('returns fallback (cwd) when all candidate dirs are missing', () => { @@ -86,6 +95,6 @@ describe('resolveAdapterOutputDir', () => { testFilePath: '/missing/x.test.ts', configPath: '/missing/wdio.conf.ts' }) - ).toBe(process.cwd()) + ).toBe(grouped(process.cwd())) }) }) diff --git a/packages/core/tests/spec-trace-helpers.test.ts b/packages/core/tests/spec-trace-helpers.test.ts index 4be8ad6e..95257afe 100644 --- a/packages/core/tests/spec-trace-helpers.test.ts +++ b/packages/core/tests/spec-trace-helpers.test.ts @@ -1,16 +1,23 @@ -import { describe, it, expect } from 'vitest' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, it, expect } from 'vitest' import { buildSpecCapturer, buildSpecSessionId, + buildTestSliceFolder, buildTestSliceSessionId, filterTestMetadataBySpec, filterTestMetadataByUid, recordSliceBoundary, recordSpecBoundary, sanitizeSpecName, + writeSpecTrace, + writeTestSliceTrace, type SpecBoundaryContext, type SpecRange, - type TraceCapturer + type TraceCapturer, + type WriteSpecTraceInput } from '@wdio/devtools-core' import { TraceType, type TestMetadataMap } from '@wdio/devtools-shared' @@ -197,6 +204,114 @@ describe('recordSliceBoundary (test granularity)', () => { }) }) +describe('buildTestSliceFolder', () => { + it('combines sanitized spec, title slug, and browser slug', () => { + expect( + buildTestSliceFolder( + '/tests/login.e2e.js', + 'shows an error message for an invalid username', + 'chrome', + 'u1' + ) + ).toBe('login_e2e-shows-an-error-message-for-an-invalid-username-chrome') + }) + + it('appends a -retry<N> suffix when the key is a retry key', () => { + expect( + buildTestSliceFolder('/a.js', 'My Test', 'chrome', 'u1-retry2') + ).toBe('a-my-test-chrome-retry2') + }) + + it('defaults the browser slug to "browser" when the browser is absent', () => { + expect(buildTestSliceFolder('/a.js', 'My Test', undefined, 'u1')).toBe( + 'a-my-test-browser' + ) + }) + + it('falls back to a stable short hash of the key when the title is empty', () => { + const folder = buildTestSliceFolder('/a.js', '', 'chrome', 'u1') + expect(folder).toMatch(/^a-[a-z0-9]+-chrome$/) + expect(buildTestSliceFolder('/a.js', undefined, 'chrome', 'u1')).toBe( + folder + ) + }) + + it('lowercases, collapses non-alphanumerics, and caps the slug length', () => { + const folder = buildTestSliceFolder( + '/a.js', + `${'A'.repeat(80)} !!! End`, + 'Chrome', + 'u1' + ) + const titleSlug = folder.slice('a-'.length, folder.lastIndexOf('-chrome')) + expect(titleSlug.length).toBeLessThanOrEqual(60) + expect(titleSlug).toMatch(/^[a-z0-9-]+$/) + expect(titleSlug.endsWith('-')).toBe(false) + }) +}) + +describe('writeTestSliceTrace / writeSpecTrace output layout', () => { + let outputDir: string + beforeEach(async () => { + outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slice-layout-')) + }) + afterEach(async () => { + await fs.rm(outputDir, { recursive: true, force: true }) + }) + + const writableCapturer = (): TraceCapturer => ({ + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { command: 'url', args: ['https://example.test'], timestamp: 1000 } + ], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + }) + + const input = ( + over: Partial<WriteSpecTraceInput> = {} + ): WriteSpecTraceInput => ({ + range: range({ specFile: '/tests/login.js', key: 'u1', testUid: 'u1' }), + capturer: writableCapturer(), + actionSnapshots: [], + sessionId: 'sess1234', + outputDir, + testMetadata: new Map([ + ['u1', { title: 'My Test', specFile: '/tests/login.js' }] + ]), + capabilities: { browserName: 'firefox' }, + ...over + }) + + it('writes a test slice into <folder>/trace.zip', async () => { + const written = await writeTestSliceTrace(input()) + const folder = buildTestSliceFolder( + '/tests/login.js', + 'My Test', + 'firefox', + 'u1' + ) + expect(folder).toBe('login-my-test-firefox') + expect(written).toBe(path.join(outputDir, folder, 'trace.zip')) + await expect(fs.access(written)).resolves.toBeUndefined() + }) + + it('keeps the spec write flat as trace-<id>.zip (unchanged layout)', async () => { + const written = await writeSpecTrace( + input({ + range: range({ specFile: '/tests/login.js', key: '/tests/login.js' }) + }) + ) + const name = buildSpecSessionId('/tests/login.js', 'sess1234') + expect(written).toBe(path.join(outputDir, `trace-${name}.zip`)) + await expect(fs.access(written)).resolves.toBeUndefined() + }) +}) + describe('recordSliceBoundary / recordSpecBoundary (spec granularity)', () => { it('keeps the spec-file behavior: one slice per spec, key equals specFile', () => { const ctx = boundaryCtx() diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index 04362b01..6d2b5013 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -4,7 +4,7 @@ import path from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { buildSpecSessionId, - buildTestSliceSessionId, + buildTestSliceFolder, finalizeTraceExport, flushRangeTrace, type SpecRange, @@ -186,11 +186,13 @@ describe('finalizeTraceExport', () => { // Each test slice carries only its own test's metadata. expect(result[0]!.testUids).toEqual(['a1']) expect(result[1]!.testUids).toEqual(['a2']) - const nameA1 = buildTestSliceSessionId('/a.js', 'a1', 'abcd1234') - const nameA2 = buildTestSliceSessionId('/a.js', 'a2', 'abcd1234') - expect(nameA1).not.toBe(nameA2) - expect(await exists(path.join(outputDir, `trace-${nameA1}.zip`))).toBe(true) - expect(await exists(path.join(outputDir, `trace-${nameA2}.zip`))).toBe(true) + // Each slice lands in its own <spec>--<title>-<browser>/trace.zip folder. + const folderA1 = buildTestSliceFolder('/a.js', 'A1', undefined, 'a1') + const folderA2 = buildTestSliceFolder('/a.js', 'A2', undefined, 'a2') + expect(folderA1).not.toBe(folderA2) + expect(result[0]!.path).toBe(path.join(outputDir, folderA1, 'trace.zip')) + expect(await exists(path.join(outputDir, folderA1, 'trace.zip'))).toBe(true) + expect(await exists(path.join(outputDir, folderA2, 'trace.zip'))).toBe(true) }) it('treats a retry-keyed range as its own test slice', async () => { @@ -206,11 +208,13 @@ describe('finalizeTraceExport', () => { ) expect(result).toHaveLength(2) expect(result.map((a) => a.key)).toEqual(['a1', 'a1-retry1']) - const first = buildTestSliceSessionId('/a.js', 'a1', 'abcd1234') - const retry = buildTestSliceSessionId('/a.js', 'a1-retry1', 'abcd1234') + // The retry attempt gets a distinct folder via the -retry<N> suffix. + const first = buildTestSliceFolder('/a.js', 'A1', undefined, 'a1') + const retry = buildTestSliceFolder('/a.js', 'A1', undefined, 'a1-retry1') expect(first).not.toBe(retry) - expect(await exists(path.join(outputDir, `trace-${first}.zip`))).toBe(true) - expect(await exists(path.join(outputDir, `trace-${retry}.zip`))).toBe(true) + expect(retry.endsWith('-retry1')).toBe(true) + expect(await exists(path.join(outputDir, first, 'trace.zip'))).toBe(true) + expect(await exists(path.join(outputDir, retry, 'trace.zip'))).toBe(true) }) it('warns and falls back to a session trace when test has no boundaries', async () => { diff --git a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts index 201e2ca0..44197c0e 100644 --- a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts +++ b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts @@ -19,7 +19,6 @@ import { type CucumberPickleStep } from '@wdio/devtools-shared' -import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { TestManager } from './helpers/testManager.js' import type { SuiteManager } from './helpers/suiteManager.js' @@ -33,6 +32,11 @@ import { import { buildCucumberScenarioSuite } from './helpers/cucumberScenarioBuilder.js' import { scanFeatureFile } from './helpers/featureFileScan.js' import { parseCucumberScenario } from './helpers/utils.js' +import { + recordTestSliceBoundary, + flushTestSlice, + type TestSliceCtx +} from './trace-slices.js' const log = logger('@wdio/nightwatch-devtools:cucumber') @@ -42,8 +46,7 @@ export interface CucumberResult { status?: string } -export interface CucumberLifecycleCtx { - readonly sessionCapturer: SessionCapturer +export interface CucumberLifecycleCtx extends TestSliceCtx { readonly testReporter: TestReporter readonly testManager: TestManager readonly suiteManager: SuiteManager @@ -132,6 +135,15 @@ function normalizeSteps( return (pickleSteps ?? []).map((s) => ({ text: s.text ?? '' })) } +function captureFeatureSources( + ctx: CucumberLifecycleCtx, + paths: string[] +): void { + for (const p of paths) { + ctx.sessionCapturer.captureSource(p).catch(() => {}) + } +} + export async function initCucumberScenario( ctx: CucumberLifecycleCtx, browser: NightwatchBrowser, @@ -148,9 +160,7 @@ export async function initCucumberScenario( stepDefFiles, capturedPaths } = scanFeatureFile(featureUri) - for (const p of capturedPaths) { - ctx.sessionCapturer.captureSource(p).catch(() => {}) - } + captureFeatureSources(ctx, capturedPaths) const { featureSuite, scenarioLine, stepLines, stepKeywords } = createFeatureSuite( ctx, @@ -175,6 +185,8 @@ export async function initCucumberScenario( recordAttempt: (uid) => ctx.recordAttempt(uid) }) attachScenarioToFeature(ctx, featureSuite, scenarioSuite) + // The scenario is the `test` unit; its steps are the leaf metadata entries. + recordTestSliceBoundary(ctx, featureUri, scenarioSuite.uid) ctx.setCurrentScenarioSuite(scenarioSuite) ctx.setCurrentStep(null) ctx.setCurrentTest(null) @@ -222,6 +234,9 @@ export async function finalizeCucumberScenario( ctx.setCurrentTest(null) } await ctx.sessionCapturer.captureTrace(browser) + // Flush before the next attempt's attachScenarioToFeature overwrites this + // scenario's suite (and thus its outcome) in the tree. + flushTestSlice(ctx) } catch (err) { log.error(`Failed to finalize Cucumber scenario: ${errorMessage(err)}`) } diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 1d4d45a2..1753d09c 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -7,6 +7,7 @@ * four) while still letting each lifecycle module narrow its dependencies. */ +import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { ScreencastRecorder } from './screencast.js' @@ -18,7 +19,8 @@ import type { NightwatchBrowser, ScreencastOptions, SuiteStats, - TestStats + TestStats, + TraceGranularity } from './types.js' export interface PluginInternals { @@ -29,6 +31,7 @@ export interface PluginInternals { readonly mode: DevToolsMode readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean + readonly captureAssertions: boolean // Runtime instances (mutable — bringup/session-change replaces them) sessionCapturer: SessionCapturer @@ -79,4 +82,12 @@ export interface PluginInternals { recordAttempt(uid: string): number /** Latest attempt recorded for `uid`, or undefined if it never started. */ attemptFor(uid: string): number | undefined + + // Per-test trace slicing (`test` granularity). Boundary state is shared with + // the finalizer; flushTraceRange writes one slice via the plugin's context. + readonly traceMode: boolean + readonly traceGranularity: TraceGranularity + readonly specRanges: SpecRange[] + readonly flushedSpecs: Set<string> + flushTraceRange(range: SpecRange): Promise<TraceArtifact | undefined> } diff --git a/packages/nightwatch-devtools/src/test-lifecycle.ts b/packages/nightwatch-devtools/src/test-lifecycle.ts index 5caf0537..7192c12a 100644 --- a/packages/nightwatch-devtools/src/test-lifecycle.ts +++ b/packages/nightwatch-devtools/src/test-lifecycle.ts @@ -9,7 +9,6 @@ */ import logger from '@wdio/logger' -import type { SessionCapturer } from './session.js' import type { TestReporter } from './reporter.js' import type { TestManager } from './helpers/testManager.js' import type { SuiteManager } from './helpers/suiteManager.js' @@ -26,11 +25,11 @@ import { DEFAULTS, TIMING, TEST_STATE } from './constants.js' import { resolveSpecFilePath } from './helpers/specFileResolver.js' import { closePreviousTest } from './helpers/closePreviousTest.js' import { extractTestMetadata, determineTestState } from './helpers/utils.js' +import { recordTestSliceBoundary, type TestSliceCtx } from './trace-slices.js' const log = logger('@wdio/nightwatch-devtools:test-lifecycle') -export interface TestLifecycleCtx { - readonly sessionCapturer: SessionCapturer +export interface TestLifecycleCtx extends TestSliceCtx { readonly testReporter: TestReporter readonly testManager: TestManager readonly suiteManager: SuiteManager @@ -118,7 +117,8 @@ export async function startNextTest( ctx: TestLifecycleCtx, currentSuite: SuiteStats, currentTestName: string, - processedTests: Set<string> + processedTests: Set<string>, + specFile: string | null ): Promise<void> { if (processedTests.size === 0) { ctx.suiteManager.markSuiteAsRunning(currentSuite) @@ -127,6 +127,9 @@ export async function startNextTest( if (test) { // Nightwatch has no per-test retry index; the tracker is the retry signal. test.retries = ctx.recordAttempt(test.uid) + if (specFile) { + recordTestSliceBoundary(ctx, specFile, test.uid) + } test.state = TEST_STATE.RUNNING as TestStats['state'] test.start = new Date() test.end = null diff --git a/packages/nightwatch-devtools/src/trace-slices.ts b/packages/nightwatch-devtools/src/trace-slices.ts new file mode 100644 index 00000000..bdd0cf0d --- /dev/null +++ b/packages/nightwatch-devtools/src/trace-slices.ts @@ -0,0 +1,102 @@ +/** + * Trace-slice boundary recording for `spec` and `test` granularity. + * + * `spec` slices are recorded/flushed at each spec-file transition (unchanged + * from the original inline `beforeEach` logic). For `test`, Nightwatch builds + * test outcomes into the suite tree in place — a retry overwrites the previous + * attempt's state (regular tests reuse the same test object; cucumber replaces + * the scenario suite). So each attempt's slice is flushed at its own + * test/scenario END, before the next attempt can overwrite the outcome the + * flush reads. Regular tests drive this from test-lifecycle, cucumber scenarios + * from cucumber-lifecycle; both share this module. + */ + +import { + recordSliceBoundary, + recordSpecBoundary, + type SpecBoundaryContext, + type SpecRange, + type TraceArtifact +} from '@wdio/devtools-core' +import type { SessionCapturer } from './session.js' +import type { TraceGranularity } from './types.js' + +export interface TestSliceCtx { + readonly sessionCapturer: SessionCapturer + readonly traceMode: boolean + readonly traceGranularity: TraceGranularity + readonly specRanges: SpecRange[] + readonly flushedSpecs: Set<string> + flushTraceRange(range: SpecRange): Promise<TraceArtifact | undefined> +} + +function sliceActive(ctx: TestSliceCtx): boolean { + return ctx.traceMode && ctx.traceGranularity === 'test' +} + +function boundaryContext(ctx: TestSliceCtx): SpecBoundaryContext { + return { + specRanges: ctx.specRanges, + flushedSpecs: ctx.flushedSpecs, + capturer: ctx.sessionCapturer, + actionSnapshots: ctx.sessionCapturer.actionSnapshots + } +} + +function flushPrevious(ctx: TestSliceCtx, prevRange: SpecRange | null): void { + if (!prevRange) { + return + } + // flushTraceRange (→ core flushRangeLogged) logs+swallows on failure. + void ctx.flushTraceRange(prevRange) +} + +/** + * Record a spec-file boundary at test/scenario start and eagerly flush the + * previous spec's slice. No-op for `session`/`test` granularity (core returns + * null). Preserves the original `beforeEach` behavior verbatim. + */ +export function recordSpecSliceBoundary( + ctx: TestSliceCtx, + specFile: string +): void { + const prevRange = recordSpecBoundary( + boundaryContext(ctx), + specFile, + ctx.traceGranularity + ) + flushPrevious(ctx, prevRange) +} + +/** + * Record a per-test slice boundary at test/scenario start. No-op outside trace + * mode + `test` granularity. Retries push a distinct range (core keys a repeat + * `${testUid}-retry${n}`), so each attempt becomes its own artifact. + */ +export function recordTestSliceBoundary( + ctx: TestSliceCtx, + specFile: string, + testUid: string +): void { + if (!sliceActive(ctx)) { + return + } + recordSliceBoundary(boundaryContext(ctx), 'test', specFile, testUid) +} + +/** + * Flush the current test's slice at test/scenario end — before a retry can + * overwrite its outcome in the suite tree. Fire-and-forget; the finalize pass + * is the safety net for any range this misses. No-op outside trace + `test`. + */ +export function flushTestSlice(ctx: TestSliceCtx): void { + if (!sliceActive(ctx)) { + return + } + const range = ctx.specRanges[ctx.specRanges.length - 1] + if (!range) { + return + } + // flushTraceRange (→ core flushRangeLogged) logs+swallows on failure. + void ctx.flushTraceRange(range) +} diff --git a/packages/nightwatch-devtools/tests/traceSlices.test.ts b/packages/nightwatch-devtools/tests/traceSlices.test.ts new file mode 100644 index 00000000..b42e92d5 --- /dev/null +++ b/packages/nightwatch-devtools/tests/traceSlices.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, vi } from 'vitest' +import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' +import { + recordTestSliceBoundary, + recordSpecSliceBoundary, + flushTestSlice, + type TestSliceCtx +} from '../src/trace-slices.js' +import type { SessionCapturer } from '../src/session.js' +import type { TraceGranularity } from '../src/types.js' + +function makeCtx(traceMode: boolean, granularity: TraceGranularity) { + const capturer = { + commandsLog: [], + consoleLogs: [], + networkRequests: [], + mutations: [], + traceLogs: [], + actionSnapshots: [] + } as unknown as SessionCapturer + const artifacts: TraceArtifact[] = [] + const flushTraceRange = vi.fn( + async (range: SpecRange): Promise<TraceArtifact> => { + const artifact: TraceArtifact = { + kind: 'trace', + path: `/out/${range.key}.zip`, + scope: range.testUid ? 'test' : 'spec', + key: range.key, + testUids: range.testUid ? [range.testUid] : [], + retained: true + } + artifacts.push(artifact) + return artifact + } + ) + const ctx: TestSliceCtx = { + sessionCapturer: capturer, + traceMode, + traceGranularity: granularity, + specRanges: [], + flushedSpecs: new Set<string>(), + flushTraceRange + } + return { ctx, artifacts, flushTraceRange } +} + +describe('test granularity — recordTestSliceBoundary', () => { + it('records a per-test slice keyed on the test uid', () => { + const { ctx } = makeCtx(true, 'test') + recordTestSliceBoundary(ctx, '/login.spec.js', 'uid-1') + expect(ctx.specRanges).toHaveLength(1) + expect(ctx.specRanges[0]).toMatchObject({ + key: 'uid-1', + testUid: 'uid-1', + specFile: '/login.spec.js' + }) + }) + + it('keys a retry of the same test as a distinct slice', () => { + const { ctx } = makeCtx(true, 'test') + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') // retry + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-2') // sibling test + expect(ctx.specRanges.map((r) => r.key)).toEqual([ + 'uid-1', + 'uid-1-retry1', + 'uid-2' + ]) + // Every slice stays a test slice (base uid preserved for metadata filter). + expect(ctx.specRanges.map((r) => r.testUid)).toEqual([ + 'uid-1', + 'uid-1', + 'uid-2' + ]) + }) +}) + +describe('test granularity — flushTestSlice', () => { + it('emits one artifact per test slice, flushing the current test at its end', () => { + const { ctx, artifacts, flushTraceRange } = makeCtx(true, 'test') + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + flushTestSlice(ctx) + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-2') + flushTestSlice(ctx) + + expect(flushTraceRange).toHaveBeenCalledTimes(2) + expect(artifacts.map((a) => a.key)).toEqual(['uid-1', 'uid-2']) + expect(artifacts.every((a) => a.scope === 'test')).toBe(true) + }) + + it('flushes each attempt separately so retries become distinct artifacts', () => { + const { ctx, artifacts } = makeCtx(true, 'test') + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + flushTestSlice(ctx) // attempt 0 flushed before the retry can overwrite it + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + flushTestSlice(ctx) // attempt 1 + + expect(artifacts.map((a) => a.key)).toEqual(['uid-1', 'uid-1-retry1']) + }) + + it('does nothing when there is no recorded slice', () => { + const { ctx, flushTraceRange } = makeCtx(true, 'test') + flushTestSlice(ctx) + expect(flushTraceRange).not.toHaveBeenCalled() + }) +}) + +describe('test-slice helpers are inert outside trace + test granularity', () => { + it('no-ops for spec and session granularity', () => { + for (const granularity of ['spec', 'session'] as TraceGranularity[]) { + const { ctx, flushTraceRange } = makeCtx(true, granularity) + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + flushTestSlice(ctx) + expect(ctx.specRanges).toHaveLength(0) + expect(flushTraceRange).not.toHaveBeenCalled() + } + }) + + it('no-ops in live mode even at test granularity', () => { + const { ctx, flushTraceRange } = makeCtx(false, 'test') + recordTestSliceBoundary(ctx, '/a.spec.js', 'uid-1') + flushTestSlice(ctx) + expect(ctx.specRanges).toHaveLength(0) + expect(flushTraceRange).not.toHaveBeenCalled() + }) +}) + +describe('spec granularity is unchanged by the test-slice work', () => { + it('records one slice per spec file and flushes the previous on transition', () => { + const { ctx, artifacts, flushTraceRange } = makeCtx(true, 'spec') + recordSpecSliceBoundary(ctx, '/a.spec.js') + expect(ctx.specRanges.map((r) => r.key)).toEqual(['/a.spec.js']) + expect(flushTraceRange).not.toHaveBeenCalled() // no previous spec yet + + recordSpecSliceBoundary(ctx, '/a.spec.js') // same spec → shared slice + expect(ctx.specRanges).toHaveLength(1) + + recordSpecSliceBoundary(ctx, '/b.spec.js') // new spec → flush previous + expect(ctx.specRanges.map((r) => r.key)).toEqual([ + '/a.spec.js', + '/b.spec.js' + ]) + expect(artifacts.map((a) => a.key)).toEqual(['/a.spec.js']) + expect(artifacts[0].scope).toBe('spec') + }) + + it('records nothing for session or test granularity', () => { + for (const granularity of ['session', 'test'] as TraceGranularity[]) { + const { ctx, flushTraceRange } = makeCtx(true, granularity) + recordSpecSliceBoundary(ctx, '/a.spec.js') + expect(ctx.specRanges).toHaveLength(0) + expect(flushTraceRange).not.toHaveBeenCalled() + } + }) +}) diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 2fcdb885..7289cd81 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -21,7 +21,8 @@ import { onDriverEnd as sessionOnDriverEnd, onSessionEnd as sessionOnSessionEnd, setPluginRef, - recordSpecBoundary + recordTraceBoundary, + flushCurrentTestTrace } from './session-lifecycle.js' import type { SpecRange } from '@wdio/devtools-core' import { @@ -122,6 +123,9 @@ class SeleniumDevToolsPlugin { /** Set of spec files already flushed to disk. */ #flushedSpecs = new Set<string>() + /** In-flight per-test eager flushes (test granularity), awaited at finalize. */ + #traceFlushes: Promise<unknown>[] = [] + constructor(options: DevToolsOptions = {}) { this.#options = { port: options.port ?? 3000, @@ -427,6 +431,9 @@ class SeleniumDevToolsPlugin { get flushedSpecs() { return self.#flushedSpecs }, + get traceFlushes() { + return self.#traceFlushes + }, setFinalized: (v) => { self.#finalized = v }, @@ -451,20 +458,17 @@ class SeleniumDevToolsPlugin { /** Public API: start a marked test. */ startTest(name: string, meta: StartTestMeta = {}) { tmStartTest(this.#getInternals(), name, meta) - if (meta.file) { - recordSpecBoundary(this.#getInternals(), meta.file) - } + recordTraceBoundary(this.#getInternals(), meta.file) } endTest(state: TestStats['state'] = 'passed') { tmEndTest(this.#getInternals(), state) + flushCurrentTestTrace(this.#getInternals()) } startScenario(name: string, meta: StartScenarioMeta = {}) { tmStartScenario(this.#getInternals(), name, meta) - if (meta.file) { - recordSpecBoundary(this.#getInternals(), meta.file) - } + recordTraceBoundary(this.#getInternals(), meta.file) } endScenario(state: TestStats['state'] = 'passed') { @@ -473,6 +477,11 @@ class SeleniumDevToolsPlugin { #flushPendingTestActions() { tmFlushPendingTestActions(this.#getInternals()) + // The first test's startTest fires before the driver/capturer exists, so + // its per-test boundary is recorded here once capture is live. + if (this.#options.traceGranularity === 'test') { + recordTraceBoundary(this.#getInternals(), this.#testFilePath) + } } async onDriverCreated(driver: SeleniumDriverLike) { diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index c769c4ad..a9672e50 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -67,6 +67,8 @@ export interface PluginInternals { // Per-spec trace tracking (populated at spec file boundaries). readonly specRanges: SpecRange[] readonly flushedSpecs: Set<string> + // In-flight per-test eager flushes (test granularity), awaited at finalize. + readonly traceFlushes: Promise<unknown>[] // Plugin-side delegates setFinalized(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index b12699dd..856d05ee 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -14,9 +14,11 @@ import { errorMessage, finalizeScreencast, finalizeTraceExport, - flushRangeTrace, + flushRangeLogged, + recordSliceBoundary as coreRecordSliceBoundary, recordSpecBoundary as coreRecordSpecBoundary, resolveAdapterOutputDir, + type SpecBoundaryContext, type SpecRange, type TraceExportContext } from '@wdio/devtools-core' @@ -79,6 +81,9 @@ export interface SessionLifecycleCtx { // Per-spec trace tracking (populated at spec file boundaries). readonly specRanges: SpecRange[] readonly flushedSpecs: Set<string> + // In-flight per-test eager flushes (test granularity); awaited at finalize + // so the last test's write completes before the process exits. + readonly traceFlushes: Promise<unknown>[] setFinalized(v: boolean): void ensureBackendStarted(): Promise<void> @@ -315,30 +320,55 @@ export function buildTraceExportContext( resolveAdapterOutputDir({ testFilePath: range ? range.specFile : testFilePath }), - awaitPending: ctx.snapshotCaptures, + awaitPending: [...ctx.snapshotCaptures, ...ctx.traceFlushes], log: (level, msg) => log[level](msg) } } +/** Narrow view of the lifecycle ctx that the core boundary recorder needs. */ +function boundaryContext( + ctx: SessionLifecycleCtx, + capturer: SessionCapturer +): SpecBoundaryContext { + return { + specRanges: ctx.specRanges, + flushedSpecs: ctx.flushedSpecs, + capturer, + actionSnapshots: ctx.actionSnapshots + } +} + /** - * Record a spec-file boundary in the session lifecycle context. Called from - * the plugin's startTest / startScenario when `traceGranularity` is `'spec'`. - * Flushes the previous spec's trace if it hasn't been written yet. + * Record a trace-slice boundary for the active granularity, called from the + * plugin's startTest / startScenario. `spec` keys by spec file (flushing the + * previous spec lazily at the next boundary); `test` keys by the just-started + * marked test's uid so each test — and each retry, which selenium gives a + * distinct uid — becomes its own slice. No-op for `session`. */ -export function recordSpecBoundary( +export function recordTraceBoundary( ctx: SessionLifecycleCtx, - specFile: string + specFile: string | undefined ): void { + if (ctx.options.traceGranularity === 'test') { + recordTestBoundary(ctx, specFile) + return + } + if (specFile) { + recordSpecBoundary(ctx, specFile) + } +} + +/** + * Record a spec-file boundary and lazily flush the previous spec's trace if it + * hasn't been written yet. `coreRecordSpecBoundary` no-ops for non-spec + * granularities, so this only records under `spec`. + */ +function recordSpecBoundary(ctx: SessionLifecycleCtx, specFile: string): void { if (!ctx.sessionCapturer) { return } const prevRange = coreRecordSpecBoundary( - { - specRanges: ctx.specRanges, - flushedSpecs: ctx.flushedSpecs, - capturer: ctx.sessionCapturer, - actionSnapshots: ctx.actionSnapshots - }, + boundaryContext(ctx, ctx.sessionCapturer), specFile, ctx.options.traceGranularity ) @@ -346,7 +376,7 @@ export function recordSpecBoundary( if (!prevRange || !sessionId) { return } - void flushRangeTrace( + void flushRangeLogged( buildTraceExportContext( ctx, ctx.sessionCapturer, @@ -354,13 +384,63 @@ export function recordSpecBoundary( ctx.testFilePath ), prevRange - ).catch((err) => - log.warn( - `Failed to flush trace for spec "${prevRange.specFile}": ${errorMessage(err)}` - ) ) } +/** + * Record a per-test boundary keyed by the currently-active marked test's uid. + * The first test's startTest fires before the driver exists (no capturer yet), + * so flushPendingTestActions re-invokes this once capture is live; the + * same-uid guard keeps that replay from minting a spurious retry slice. + */ +function recordTestBoundary( + ctx: SessionLifecycleCtx, + specFile: string | undefined +): void { + const testUid = ctx.testManager?.getCurrentTest()?.uid + const file = specFile ?? ctx.testFilePath + if (!ctx.sessionCapturer || !testUid || !file) { + return + } + const lastRange = ctx.specRanges[ctx.specRanges.length - 1] + if (lastRange?.testUid === testUid) { + return + } + coreRecordSliceBoundary( + boundaryContext(ctx, ctx.sessionCapturer), + 'test', + file, + testUid + ) +} + +/** + * Eager-flush the just-ended test's trace slice (test granularity), after + * endTest has finalized its state so collectSuiteTestMetadata sees the final + * outcome. flushRangeTrace dedupes by key, so finalizeTraceExport won't + * re-write it; the promise is tracked so finalize awaits the last write. + */ +export function flushCurrentTestTrace(ctx: SessionLifecycleCtx): void { + if (ctx.options.traceGranularity !== 'test' || !ctx.sessionCapturer) { + return + } + const sessionId = ctx.sessionCapturer.metadata?.sessionId + const currentRange = ctx.specRanges[ctx.specRanges.length - 1] + if (!sessionId || currentRange?.testUid === undefined) { + return + } + const flush = flushRangeLogged( + buildTraceExportContext( + ctx, + ctx.sessionCapturer, + sessionId, + ctx.testFilePath + ), + currentRange + ) + ctx.traceFlushes.push(flush) +} + async function writeTraceIfNeeded( ctx: SessionLifecycleCtx, capturer: SessionCapturer | undefined, diff --git a/packages/selenium-devtools/tests/attempt-capture.test.ts b/packages/selenium-devtools/tests/attempt-capture.test.ts index 255280f6..2f2840ff 100644 --- a/packages/selenium-devtools/tests/attempt-capture.test.ts +++ b/packages/selenium-devtools/tests/attempt-capture.test.ts @@ -77,6 +77,7 @@ describe('retry/attempt capture', () => { actionSnapshots: [], specRanges: [], flushedSpecs: new Set<string>(), + traceFlushes: [], snapshotCaptures: [] } as unknown as SessionLifecycleCtx diff --git a/packages/selenium-devtools/tests/trace-granularity.test.ts b/packages/selenium-devtools/tests/trace-granularity.test.ts new file mode 100644 index 00000000..8f133a24 --- /dev/null +++ b/packages/selenium-devtools/tests/trace-granularity.test.ts @@ -0,0 +1,191 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { resetSignatureCounters } from '../src/helpers/utils.js' +import { TestManager } from '../src/helpers/testManager.js' +import { SuiteManager } from '../src/helpers/suiteManager.js' +import { TestReporter } from '../src/reporter.js' +import { SessionCapturer } from '../src/session.js' +import { + flushCurrentTestTrace, + recordTraceBoundary, + type SessionLifecycleCtx +} from '../src/session-lifecycle.js' +import type { TraceGranularity } from '../src/types.js' + +const SESSION_ID = 'sess-abcd1234ef' + +const capturers: SessionCapturer[] = [] +const tmpDirs: string[] = [] + +afterEach(() => { + while (capturers.length) { + capturers.pop()!.cleanup() + } + while (tmpDirs.length) { + fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }) + } +}) + +function makeTmpSpec(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sel-trace-gran-')) + tmpDirs.push(dir) + return path.join(dir, 'login.spec.ts') +} + +function makeCtx( + granularity: TraceGranularity, + specFile: string, + withSessionId = false +) { + resetSignatureCounters() + const reporter = new TestReporter(vi.fn()) + const suiteManager = new SuiteManager(reporter) + const rootSuite = suiteManager.getOrCreateRootSuite(specFile, 'Suite') + const testManager = new TestManager(rootSuite, reporter, suiteManager) + const capturer = new SessionCapturer() + capturers.push(capturer) + if (withSessionId) { + // Only sessionId matters to the flush path; the rest of Metadata is + // filled with writeTraceZip's defaults, so a partial cast is safe here. + capturer.metadata = { sessionId: SESSION_ID } as SessionCapturer['metadata'] + } + + // Minimal structural ctx: the boundary/flush helpers read only the trace + // accumulators, options, capturer, test/suite managers and testFilePath, so + // we cast a partial to the full lifecycle interface. + const ctx = { + options: { + mode: 'trace', + tracePolicy: 'on', + traceGranularity: granularity, + traceFormat: 'zip' + }, + sessionCapturer: capturer, + testManager, + suiteManager, + testFilePath: specFile, + actionSnapshots: [], + snapshotCaptures: [], + specRanges: [], + flushedSpecs: new Set<string>(), + traceFlushes: [] + } as unknown as SessionLifecycleCtx + + return { ctx, capturer, testManager } +} + +describe('trace granularity: test', () => { + it('records one per-test slice keyed by the started test uid', () => { + const spec = makeTmpSpec() + const { ctx, testManager } = makeCtx('test', spec) + + const test = testManager.startMarkedTest('logs in') + recordTraceBoundary(ctx, spec) + + expect(ctx.specRanges).toHaveLength(1) + expect(ctx.specRanges[0].testUid).toBe(test.uid) + expect(ctx.specRanges[0].key).toBe(test.uid) + expect(ctx.specRanges[0].specFile).toBe(spec) + }) + + it('re-recording the same active test is idempotent (no spurious slice)', () => { + const spec = makeTmpSpec() + const { ctx, testManager } = makeCtx('test', spec) + + testManager.startMarkedTest('logs in') + recordTraceBoundary(ctx, spec) + // Mirrors the buffered-first-test replay: startTest recorded nothing (no + // capturer yet), flushPendingTestActions re-invokes for the same test. + recordTraceBoundary(ctx, spec) + + expect(ctx.specRanges).toHaveLength(1) + }) + + it('a retry gets its own distinct slice (selenium gives each attempt a uid)', () => { + const spec = makeTmpSpec() + const { ctx, testManager } = makeCtx('test', spec) + + const first = testManager.startMarkedTest('flaky') + recordTraceBoundary(ctx, spec) + testManager.endCurrent('failed') + + const retry = testManager.startMarkedTest('flaky') + recordTraceBoundary(ctx, spec) + testManager.endCurrent('passed') + + expect(first.uid).not.toBe(retry.uid) + expect(ctx.specRanges).toHaveLength(2) + expect(ctx.specRanges.map((r) => r.key)).toEqual([first.uid, retry.uid]) + }) + + it('eager-flushes each test to its own artifact at test end', async () => { + const spec = makeTmpSpec() + const outDir = path.dirname(spec) + const { ctx, testManager } = makeCtx('test', spec, true) + + testManager.startMarkedTest('logs in') + recordTraceBoundary(ctx, spec) + testManager.endCurrent('passed') + flushCurrentTestTrace(ctx) + + testManager.startMarkedTest('logs out') + recordTraceBoundary(ctx, spec) + testManager.endCurrent('passed') + flushCurrentTestTrace(ctx) + + expect(ctx.traceFlushes).toHaveLength(2) + // Both slice keys are recorded as flushed synchronously, so end-of-run + // finalizeTraceExport dedupes and won't re-write them. + expect(ctx.flushedSpecs.size).toBe(2) + await Promise.all(ctx.traceFlushes) + + // Per-test slices land under test-results/<spec--title-browser>/trace.zip, + // so recurse to find them rather than scanning the flat dir. + const zips = fs + .readdirSync(outDir, { recursive: true }) + .map(String) + .filter((f) => f.endsWith('.zip')) + expect(zips).toHaveLength(2) + expect(zips.every((f) => path.basename(f) === 'trace.zip')).toBe(true) + expect(zips.every((f) => f.includes('test-results'))).toBe(true) + }) + + it('flushCurrentTestTrace is a no-op with no recorded range', () => { + const spec = makeTmpSpec() + const { ctx } = makeCtx('test', spec, true) + flushCurrentTestTrace(ctx) + expect(ctx.traceFlushes).toHaveLength(0) + expect(ctx.flushedSpecs.size).toBe(0) + }) +}) + +describe('trace granularity: spec/session unchanged', () => { + it('spec granularity still records one slice per spec file, keyed by file', () => { + const specA = makeTmpSpec() + const specB = makeTmpSpec() + const { ctx } = makeCtx('spec', specA) + + recordTraceBoundary(ctx, specA) + recordTraceBoundary(ctx, specA) // same file → shares the slice + recordTraceBoundary(ctx, specB) + + expect(ctx.specRanges.map((r) => r.key)).toEqual([specA, specB]) + expect(ctx.specRanges.every((r) => r.testUid === undefined)).toBe(true) + }) + + it('session granularity records no slices and never eager-flushes', () => { + const spec = makeTmpSpec() + const { ctx, testManager } = makeCtx('session', spec, true) + + testManager.startMarkedTest('logs in') + recordTraceBoundary(ctx, spec) + testManager.endCurrent('passed') + flushCurrentTestTrace(ctx) + + expect(ctx.specRanges).toHaveLength(0) + expect(ctx.traceFlushes).toHaveLength(0) + }) +}) diff --git a/packages/service/src/trace-slices.ts b/packages/service/src/trace-slices.ts new file mode 100644 index 00000000..fcc36962 --- /dev/null +++ b/packages/service/src/trace-slices.ts @@ -0,0 +1,49 @@ +// Trace-slice flushing for the WDIO adapter: previous-slice flush at boundary +// changes plus the eager per-test flush. Kept out of index.ts so the slice +// selection and flush I/O are unit-testable and the god-file stays lean. + +import { + flushRangeLogged, + type SpecRange, + type TraceExportContext +} from '@wdio/devtools-core' + +/** The range for the test that just ended is the most recent slice recorded + * under this base testUid — retries push a new range under the same testUid, + * so reverse-scanning finds the attempt whose afterTest is now firing. */ +export function findCurrentTestRange( + ranges: readonly SpecRange[], + testUid: string +): SpecRange | undefined { + for (let i = ranges.length - 1; i >= 0; i--) { + if (ranges[i]!.testUid === testUid) { + return ranges[i] + } + } + return undefined +} + +/** Fire-and-forget flush of the previous unflushed slice at a boundary change + * (spec granularity, or a test slice whose eager flush was missed). Errors are + * logged, never thrown, so a failed flush can't abort the next test. */ +export function flushPrevSlice( + ctx: TraceExportContext, + range: SpecRange +): void { + void flushRangeLogged(ctx, range) +} + +/** Awaited flush of the just-ended test's slice (test granularity), so this + * attempt's just-stamped metadata is written before a retry's beforeTest + * overwrites the entry. No-op when the test recorded no range. */ +export async function flushTestSlice( + ctx: TraceExportContext, + ranges: readonly SpecRange[], + testUid: string +): Promise<void> { + const range = findCurrentTestRange(ranges, testUid) + if (!range) { + return + } + await flushRangeLogged(ctx, range) +} diff --git a/packages/service/tests/trace-granularity.test.ts b/packages/service/tests/trace-granularity.test.ts new file mode 100644 index 00000000..c43b6b66 --- /dev/null +++ b/packages/service/tests/trace-granularity.test.ts @@ -0,0 +1,240 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type * as DevtoolsCore from '@wdio/devtools-core' +import { deterministicUid, type SpecRange } from '@wdio/devtools-core' +import { findCurrentTestRange } from '../src/trace-slices.js' + +// Records the key/state observed at each per-slice flush and replays the real +// dedupe (flushed.add) so recordSliceBoundary's prev-slice logic behaves as in +// production — otherwise a boundary change would re-flush an already-flushed +// slice. Capturing state at call time is what proves the eager flush sees this +// attempt's outcome before a retry's beforeTest overwrites the entry. +const flushedSlices: Array<{ key: string; testUid?: string; state?: string }> = + [] + +type FlushCtx = { + flushed: Set<string> + testMetadata: Map<string, { state?: string }> +} + +const flushRangeTrace = vi.fn( + (ctx: FlushCtx, range: { key: string; testUid?: string }) => { + ctx.flushed.add(range.key) + flushedSlices.push({ + key: range.key, + testUid: range.testUid, + state: range.testUid + ? ctx.testMetadata.get(range.testUid)?.state + : undefined + }) + return Promise.resolve(undefined) + } +) + +const finalizeTraceExport = vi.fn().mockResolvedValue([]) + +vi.mock('stack-trace', () => ({ parse: () => [] })) + +const mockSessionCapturerInstance = { + afterCommand: vi.fn(), + sendUpstream: vi.fn(), + injectScript: vi.fn().mockResolvedValue(undefined), + captureAssertCommand: vi.fn(), + failLastAction: vi.fn(), + resetLastSelector: vi.fn(), + resetRetryTracker: vi.fn(), + cleanup: vi.fn(), + commandsLog: [] as unknown[], + sources: new Map(), + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + metadata: { url: 'http://test.com', viewport: {} } +} + +vi.mock('../src/session.js', () => ({ + SessionCapturer: vi.fn(function (this: unknown) { + return mockSessionCapturerInstance + }) +})) + +vi.mock('../src/action-snapshot.js', () => ({ + captureActionSnapshot: vi.fn().mockResolvedValue(null), + captureActionResult: vi.fn().mockResolvedValue(undefined), + waitForActionResult: vi.fn().mockResolvedValue(undefined) +})) + +vi.mock('@wdio/devtools-core', async (importOriginal) => { + const actual = await importOriginal<typeof DevtoolsCore>() + return { + ...actual, + finalizeTraceExport: (ctx: unknown) => finalizeTraceExport(ctx), + // The adapter now flushes via core's flushRangeLogged wrapper; route it to + // the same spy so call-count/argument assertions still observe each flush. + flushRangeLogged: (ctx: unknown, range: unknown) => + flushRangeTrace( + ctx as FlushCtx, + range as { key: string; testUid?: string } + ), + flushRangeTrace: (ctx: unknown, range: unknown) => + flushRangeTrace( + ctx as FlushCtx, + range as { key: string; testUid?: string } + ) + } +}) + +// Imported after the mocks are declared so the mocked core module is used. +const { default: DevToolsHookService } = await import('../src/index.js') + +describe('findCurrentTestRange', () => { + const mk = (key: string, testUid?: string): SpecRange => ({ + specFile: 'f', + key, + testUid, + commandStartIdx: 0, + consoleStartIdx: 0, + networkStartIdx: 0, + mutationStartIdx: 0, + traceLogStartIdx: 0, + snapshotCount: 0 + }) + + it('returns the most recent range recorded under the base testUid', () => { + const ranges = [mk('a', 'a'), mk('b', 'b'), mk('b-retry1', 'b')] + // A retry pushes a new range under the same testUid; the latest one wins. + expect(findCurrentTestRange(ranges, 'b')?.key).toBe('b-retry1') + expect(findCurrentTestRange(ranges, 'a')?.key).toBe('a') + }) + + it('returns undefined when no range matches (spec/session slices)', () => { + expect( + findCurrentTestRange([mk('spec.ts', undefined)], 'x') + ).toBeUndefined() + expect(findCurrentTestRange([], 'x')).toBeUndefined() + }) +}) + +describe('DevtoolsService - trace granularity slicing', () => { + const file = '/proj/specs/login.spec.ts' + const mockBrowser = { + isBidi: true, + sessionId: 'sess-1', + scriptAddPreloadScript: vi.fn().mockResolvedValue(undefined), + takeScreenshot: vi.fn().mockResolvedValue('shot'), + execute: vi + .fn() + .mockResolvedValue({ width: 1, height: 1, offsetLeft: 0, offsetTop: 0 }), + on: vi.fn(), + emit: vi.fn(), + options: { rootDir: '/proj' }, + capabilities: { browserName: 'chrome' } + } as never + + beforeEach(() => { + vi.clearAllMocks() + finalizeTraceExport.mockResolvedValue([]) + flushedSlices.length = 0 + }) + + async function newService(granularity: 'session' | 'spec' | 'test') { + const service = new DevToolsHookService({ + mode: 'trace', + tracePolicy: 'retain-on-failure', + traceGranularity: granularity + }) + await service.before({} as never, [], mockBrowser) + return service + } + + it('test granularity: records a per-test slice at beforeTest and eager-flushes it at afterTest', async () => { + const service = await newService('test') + const title = 'login works' + const uid = deterministicUid(file, title) + + service.beforeTest({ file, fullTitle: title }) + expect(flushRangeTrace).not.toHaveBeenCalled() + + await service.afterTest({ file, fullTitle: title }, {}, { passed: true }) + + expect(flushRangeTrace).toHaveBeenCalledTimes(1) + // The flushed range is the one recorded at beforeTest (found via testUid), + // proving both the start-boundary and the eager end-flush. + expect(flushRangeTrace.mock.calls[0]![1]).toMatchObject({ + key: uid, + testUid: uid + }) + expect(flushedSlices).toEqual([{ key: uid, testUid: uid, state: 'passed' }]) + }) + + it('test granularity: a retried test produces a distinct retry slice, each with its own attempt outcome', async () => { + const service = await newService('test') + const title = 'flaky login' + const uid = deterministicUid(file, title) + + // Attempt 1 fails, then a same-process retry (Mocha) passes. + service.beforeTest({ file, fullTitle: title }) + await service.afterTest( + { file, fullTitle: title }, + {}, + { passed: false, error: new Error('boom') } + ) + service.beforeTest({ file, fullTitle: title }) + await service.afterTest({ file, fullTitle: title }, {}, { passed: true }) + + expect(flushRangeTrace).toHaveBeenCalledTimes(2) + // Each attempt is its own slice, and each slice was written with that + // attempt's just-stamped state — the failed first attempt survives the + // retry's beforeTest overwrite (the retain-on-first-failure fix). + expect(flushedSlices).toEqual([ + { key: uid, testUid: uid, state: 'failed' }, + { key: `${uid}-retry1`, testUid: uid, state: 'passed' } + ]) + }) + + it('test granularity: Cucumber scenario eager-flushes its slice at afterScenario', async () => { + const service = await newService('test') + const uri = '/proj/features/login.feature' + const name = 'log in' + const uid = deterministicUid(uri, name) + + service.beforeScenario({ pickle: { uri, name } }) + await service.afterScenario({ pickle: { uri, name } }, { passed: true }) + + expect(flushRangeTrace).toHaveBeenCalledTimes(1) + expect(flushRangeTrace.mock.calls[0]![1]).toMatchObject({ + key: uid, + testUid: uid + }) + }) + + it('spec granularity: no eager flush at afterTest; flushes only when the spec file changes', async () => { + const service = await newService('spec') + const other = '/proj/specs/cart.spec.ts' + + service.beforeTest({ file, fullTitle: 't1' }) + await service.afterTest({ file, fullTitle: 't1' }, {}, { passed: true }) + // Two tests in the same spec: neither the end-flush nor a boundary fires. + service.beforeTest({ file, fullTitle: 't2' }) + await service.afterTest({ file, fullTitle: 't2' }, {}, { passed: true }) + expect(flushRangeTrace).not.toHaveBeenCalled() + + // A new spec file flushes the previous spec's slice (fire-and-forget). + service.beforeTest({ file: other, fullTitle: 't3' }) + expect(flushRangeTrace).toHaveBeenCalledTimes(1) + const range = flushRangeTrace.mock.calls[0]![1] as SpecRange + expect(range.key).toBe(file) + expect(range.testUid).toBeUndefined() + }) + + it('session granularity: records no slices and never flushes per test', async () => { + const service = await newService('session') + + service.beforeTest({ file, fullTitle: 't1' }) + await service.afterTest({ file, fullTitle: 't1' }, {}, { passed: true }) + service.beforeTest({ file, fullTitle: 't2' }) + await service.afterTest({ file, fullTitle: 't2' }, {}, { passed: false }) + + expect(flushRangeTrace).not.toHaveBeenCalled() + }) +}) From 9db3873c4bfccd9c532b12aa35bf293c18422b36 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:23:26 +0530 Subject: [PATCH 32/91] feat(service): capture expect-webdriverio matchers as assertion actions --- packages/core/src/assert-patcher.ts | 10 +- packages/service/src/action-snapshot.ts | 18 ++ packages/service/src/assert-capture.ts | 45 +++- packages/service/src/call-source.ts | 12 +- packages/service/src/index.ts | 218 ++++++++++++++---- .../service/tests/action-snapshot.test.ts | 29 +++ packages/service/tests/assert-capture.test.ts | 56 +++++ packages/service/tests/assertion-rows.test.ts | 157 +++++++++++++ packages/service/tests/index.test.ts | 64 +++++ 9 files changed, 557 insertions(+), 52 deletions(-) create mode 100644 packages/service/tests/action-snapshot.test.ts create mode 100644 packages/service/tests/assertion-rows.test.ts diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index 71ebd9fe..c6b91bf2 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -168,6 +168,10 @@ export interface MatcherAssertion { passed: boolean message?: string | (() => string) callSource?: string + /** Explicit display label for the action row. Falls back to `command` when + * absent — set it when the framework carries a richer human message than + * `prefix.method` (e.g. Nightwatch's "Testing if the page title contains …"). */ + title?: string } export function matcherAssertionToCommandLog( @@ -177,7 +181,7 @@ export function matcherAssertionToCommandLog( const command = `${input.prefix ?? 'expect'}.${input.method}` const message = typeof input.message === 'function' ? input.message() : input.message - return capturedAssertToCommandLog( + const entry = capturedAssertToCommandLog( { command, args: (input.args ?? []).map(safeSerializeAssertArg), @@ -190,6 +194,10 @@ export function matcherAssertionToCommandLog( }, testUid ) + if (input.title) { + entry.title = input.title + } + return entry } /** diff --git a/packages/service/src/action-snapshot.ts b/packages/service/src/action-snapshot.ts index 12710114..f8dc7fe9 100644 --- a/packages/service/src/action-snapshot.ts +++ b/packages/service/src/action-snapshot.ts @@ -81,6 +81,24 @@ export async function captureActionResult( } } +/** Capture a DOM snapshot for a synthesized action row (e.g. an `expect.*` + * assertion) and push it stamped at the row's OWN timestamp — the trace + * player's Snapshot tab claims it by timestamp the same way it claims a + * regular command's post-action snapshot (see FrameSnapshotIndex.claimAfter). + * Mirrors the tail of `captureActionResult` for a command with no page-settle. */ +export async function pushActionSnapshotAt( + browser: WebdriverIO.Browser, + command: string, + timestamp: number, + actionSnapshots: ActionSnapshot[] +): Promise<void> { + const snap = await captureActionSnapshot(browser, command) + if (snap) { + snap.timestamp = timestamp + actionSnapshots.push(snap) + } +} + export function captureActionSnapshot( browser: WebdriverIO.Browser, command: string diff --git a/packages/service/src/assert-capture.ts b/packages/service/src/assert-capture.ts index 2c277cf8..49725ec8 100644 --- a/packages/service/src/assert-capture.ts +++ b/packages/service/src/assert-capture.ts @@ -9,10 +9,44 @@ import { stripAnsi } from '@wdio/devtools-core' import type { CommandLog, SerializedError } from '@wdio/devtools-shared' +import { parse } from 'stack-trace' +import { + resolveCallSourceFromFrame, + resolveFilePathFromFrame +} from './call-source.js' +import { isUserSpecFile } from './utils.js' import type { SessionCapturer } from './session.js' const log = logger('@wdio/devtools-service:assert-capture') +/** + * Capture the user's `expect()` call site from the SYNCHRONOUS stack at matcher + * entry (call from `beforeAssertion`). The matcher then runs async, so this is + * the only point a user frame is still on the stack — reading it in + * `afterAssertion` resolves to the service bundle instead. Mirrors + * `beforeCommand`'s resolver (`parse(new Error()).reverse()` → first user-spec + * frame → `resolveCallSourceFromFrame`) so assertion rows share regular + * commands' Source-tab behaviour. Also loads that file's source via + * `captureSource` so the tab renders. Returns `undefined` when no user frame is + * present (row falls back to no callSource, exactly as before this fix). + */ +export function resolveAssertionCallSource( + captureSource: (filePath: string) => void +): string | undefined { + Error.stackTraceLimit = 20 + const frame = parse(new Error('')) + .reverse() + .find((f) => isUserSpecFile(f.getFileName())) + if (!frame) { + return undefined + } + const filePath = resolveFilePathFromFrame(frame) + if (filePath) { + captureSource(filePath) + } + return resolveCallSourceFromFrame(frame) +} + /** * Patch node:assert so every tracked assertion lands in the session capturer * as a command. Getters are read at capture time — the capturer instance is @@ -101,11 +135,15 @@ export interface ExpectAssertion { * Adapt expect-webdriverio's afterAssertion params to the shared matcher * converter. Framework-specific extraction only (matcher name, expectedValue → * args, the runtime `pass` vs typed `result` flag); the actual CommandLog - * shaping lives once in core's `matcherAssertionToCommandLog`. + * shaping lives once in core's `matcherAssertionToCommandLog`. `callSource` is + * the user's `expect()` call site captured in `beforeAssertion` (the matcher + * runs async, so afterAssertion's own stack no longer holds a user frame) — it + * makes the row's Source tab point at the spec, not the service bundle. */ export function expectAssertionToCommandLog( params: ExpectAssertion, - testUid: string | undefined + testUid: string | undefined, + callSource?: string ): CommandLog { const { matcherName, expectedValue, result } = params return matcherAssertionToCommandLog( @@ -118,7 +156,8 @@ export function expectAssertionToCommandLog( ? expectedValue : [expectedValue], passed: result.pass ?? result.result ?? false, - message: result.message + message: result.message, + callSource }, testUid ) diff --git a/packages/service/src/call-source.ts b/packages/service/src/call-source.ts index 59325905..53a6fc80 100644 --- a/packages/service/src/call-source.ts +++ b/packages/service/src/call-source.ts @@ -2,8 +2,8 @@ import type { parse } from 'stack-trace' type StackFrame = ReturnType<typeof parse>[number] -/** `<file>:<line>:<column>` from a parsed stack frame; strips file:// and query. */ -export function resolveCallSourceFromFrame( +/** Absolute file path from a parsed stack frame; strips file:// and query. */ +export function resolveFilePathFromFrame( frame: StackFrame ): string | undefined { const rawFile = frame.getFileName() ?? undefined @@ -18,6 +18,14 @@ export function resolveCallSourceFromFrame( if (absPath?.includes('?')) { absPath = absPath.split('?')[0] } + return absPath +} + +/** `<file>:<line>:<column>` from a parsed stack frame; strips file:// and query. */ +export function resolveCallSourceFromFrame( + frame: StackFrame +): string | undefined { + const absPath = resolveFilePathFromFrame(frame) if (absPath === undefined) { return undefined } diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index cbe6c196..5a266694 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -4,9 +4,8 @@ import { errorMessage, finalizeScreencast, finalizeTraceExport, - flushRangeTrace, mapCommandToAction, - recordSpecBoundary, + recordSliceBoundary, resolveAdapterOutputDir, TestAttemptTracker, tracePolicyModeWarning, @@ -16,6 +15,7 @@ import { import { captureExpectFailure, expectAssertionToCommandLog, + resolveAssertionCallSource, wireAssertCapture, type ExpectAssertion } from './assert-capture.js' @@ -27,9 +27,11 @@ import { type TestOutcomeResult } from './test-metadata.js' import { resolveCallSourceFromFrame } from './call-source.js' +import { flushPrevSlice, flushTestSlice } from './trace-slices.js' import { captureActionResult, - captureActionSnapshot + captureActionSnapshot, + pushActionSnapshotAt } from './action-snapshot.js' import { dedupeSnapshotsByTimestamp, @@ -102,6 +104,24 @@ export default class DevToolsHookService implements Services.ServiceInstance { */ #commandStack: CommandFrame[] = [] + /** Depth of the current expect-webdriverio matcher evaluation. While > 0, the + * matcher's internal WebDriver commands (getText/isExisting polling) are + * suppressed so only the `expect.<matcher>` assertion row is captured — + * matching Playwright and the Nightwatch native-assert behaviour. */ + #assertionDepth = 0 + + /** Matcher start timestamps (stack, paired with #assertionDepth) so a captured + * assertion spans [matcher start → end] — its poll duration — instead of + * collapsing to a zero-width point at completion, which left the screencast + * tracking the preceding command during the poll. */ + #assertionStartTimes: number[] = [] + + /** User `expect()` call sites (stack, paired with #assertionStartTimes), + * captured in beforeAssertion while a user frame is still synchronous. The + * matcher runs async, so afterAssertion's own stack resolves to the service + * bundle — this parallel stack lets each row keep its spec-file callSource. */ + #assertionCallSources: (string | undefined)[] = [] + /** Current test UID, set in beforeTest(), used by afterCommand() to tag commands. */ #currentTestUid?: string @@ -119,7 +139,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Set of spec files already flushed to disk. */ #flushedSpecs = new Set<string>() - /** Build the boundary context for recordSpecBoundary — the same shape is + /** Build the boundary context for recordSliceBoundary — the same shape is * needed in both beforeTest and beforeScenario. */ get #boundaryContext() { return { @@ -130,33 +150,41 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** Fire-and-forget flush of a previous spec's trace. The error log is - * inline so the spec-file reference stays precise. */ - #fireAndForgetFlush(prevRange: SpecRange): void { - if (!this.#browser) { + /** Record a trace-slice boundary. `spec` slices per file; `test` per test + * (retries keyed per attempt by core); `session` records nothing. The + * previous-slice flush fires for `spec`; `test` slices eager-flush at their + * own test end (see #eagerFlushTestSlice) so this is only a missed-slice net. */ + #recordBoundary(specFile: string | undefined, testUid?: string): void { + if (!specFile) { return } - void flushRangeTrace(this.#traceContext(this.#browser), prevRange).catch( - (err) => - log.warn( - `Failed to flush trace for spec "${prevRange.specFile}": ${errorMessage(err)}` - ) + const prevRange = recordSliceBoundary( + this.#boundaryContext, + this.#options.traceGranularity, + specFile, + testUid ) + if (prevRange && this.#browser) { + flushPrevSlice(this.#traceContext(this.#browser), prevRange) + } } - /** Record a spec boundary (spec granularity) and flush the previous spec. */ - #recordBoundary(specFile: string | undefined): void { - if (!specFile) { + /** Eager per-test flush at test end (test granularity only), run after the + * outcome is stamped so this attempt's metadata is written before a retry + * overwrites it; the end-of-run finalizer then dedupes it via the key set. */ + async #eagerFlushTestSlice(testUid: string): Promise<void> { + if ( + this.#options.traceGranularity !== 'test' || + this.#options.mode !== 'trace' || + !this.#browser + ) { return } - const prevRange = recordSpecBoundary( - this.#boundaryContext, - specFile, - this.#options.traceGranularity + await flushTestSlice( + this.#traceContext(this.#browser), + this.#specRanges, + testUid ) - if (prevRange) { - this.#fireAndForgetFlush(prevRange) - } } /** Assemble the framework-agnostic trace-export context from this service's @@ -326,16 +354,21 @@ export default class DevToolsHookService implements Services.ServiceInstance { const featureFile = world?.pickle?.uri const scenarioName = world?.pickle?.name + // Derived before recording the boundary so `test` granularity keys the + // slice on the same uid the metadata map uses. + const uid = + featureFile && scenarioName + ? cucumberScenarioUid( + featureFile, + scenarioName, + world?.pickle?.astNodeIds + ) + : undefined - this.#recordBoundary(featureFile) + this.#recordBoundary(featureFile, uid) // ── Test identity for command tagging ── - if (featureFile && scenarioName) { - const uid = cucumberScenarioUid( - featureFile, - scenarioName, - world?.pickle?.astNodeIds - ) + if (uid && scenarioName && featureFile) { this.#currentTestUid = uid this.#attemptTracker.recordStart(uid) this.#testMetadata.set(uid, { @@ -349,15 +382,17 @@ export default class DevToolsHookService implements Services.ServiceInstance { beforeTest(test?: { file?: string; title?: string; fullTitle?: string }) { this.resetStack() - this.#recordBoundary(test?.file) - // Track test identity for command tagging. Generate a stable UID // from file + title so commands can be partitioned across reruns. // WDIO's Test type always provides `fullTitle`; `title` is a - // fallback for non-WDIO frameworks. + // fallback for non-WDIO frameworks. Derived before the boundary so + // `test` granularity keys the slice on the metadata-map uid. const testTitle = test?.fullTitle || test?.title - if (testTitle) { - const uid = testMetadataUid(test?.file, testTitle) + const uid = testTitle ? testMetadataUid(test?.file, testTitle) : undefined + + this.#recordBoundary(test?.file, uid) + + if (uid && testTitle) { this.#currentTestUid = uid this.#attemptTracker.recordStart(uid) this.#testMetadata.set(uid, { @@ -403,10 +438,16 @@ export default class DevToolsHookService implements Services.ServiceInstance { result?: TestOutcomeResult ) { const { uri, name, astNodeIds } = world?.pickle ?? {} - if (uri && name) { - this.#stampOutcome(cucumberScenarioUid(uri, name, astNodeIds), result) + const uid = + uri && name ? cucumberScenarioUid(uri, name, astNodeIds) : undefined + if (uid) { + this.#stampOutcome(uid, result) } await this.#finalizePerScenario() + // Flush now so this slice includes the final snapshot and stamped outcome. + if (uid) { + await this.#eagerFlushTestSlice(uid) + } } async afterTest( @@ -416,22 +457,81 @@ export default class DevToolsHookService implements Services.ServiceInstance { ) { this.#captureExpectFailure(result?.error) const testTitle = test?.fullTitle || test?.title - if (testTitle) { - this.#stampOutcome(testMetadataUid(test?.file, testTitle), result) + const uid = testTitle ? testMetadataUid(test?.file, testTitle) : undefined + if (uid) { + this.#stampOutcome(uid, result) } await this.#finalizePerScenario() + // Flush now so this slice includes the final snapshot and stamped outcome. + if (uid) { + await this.#eagerFlushTestSlice(uid) + } } - /** expect-webdriverio fires this per matcher (pass or fail) via WDIO's - * assertion hook — capture it as an `expect.<matcher>` action so passing - * assertions show in the trace, not just failures. */ - afterAssertion(params: ExpectAssertion): void { + /** expect-webdriverio fires this before each matcher evaluates. The matcher's + * internal polling commands run inside the before→after window and must not + * surface as their own rows — only the `expect.<matcher>` assertion should. */ + beforeAssertion(): void { + // Matchers don't nest, so any residual depth here is a prior matcher whose + // afterAssertion was skipped by a hard throw. Reset before pushing so the + // suppression window can't stay stuck open across assertions. + if (this.#assertionDepth > 0) { + this.#assertionDepth = 0 + this.#assertionStartTimes = [] + this.#assertionCallSources = [] + } + this.#assertionDepth++ + this.#assertionStartTimes.push(Date.now()) + // Capture the user's expect() call site now — see #assertionCallSources. + this.#assertionCallSources.push( + resolveAssertionCallSource( + (file) => void this.#sessionCapturer.captureSource(file) + ) + ) + } + + async afterAssertion(params: ExpectAssertion): Promise<void> { + // Decrement first (even when capture is off) so the window stays balanced. + const startTime = this.#assertionStartTimes.pop() + const callSource = this.#assertionCallSources.pop() + if (this.#assertionDepth > 0) { + this.#assertionDepth-- + } if (this.#options.captureAssertions === false) { return } - this.#sessionCapturer.captureAssertCommand( - expectAssertionToCommandLog(params, this.#currentTestUid) + const entry = expectAssertionToCommandLog( + params, + this.#currentTestUid, + callSource ) + // Span the matcher's poll window so the row's duration is real and the + // screencast tracks the assertion (not the preceding command) during it. + if (startTime !== undefined) { + entry.startTime = startTime + } + // The suppressed matcher-internal command carried the DOM screenshot; + // capture one here so the assertion row's Snapshot panel isn't blank. + if (this.#browser && !isNativeMobile(this.#browser)) { + try { + entry.screenshot = await this.#browser.takeScreenshot() + } catch (err) { + // best-effort: a missing screenshot must not fail the assertion hook + log.debug(`assertion screenshot skipped: ${errorMessage(err)}`) + } + } + // Trace mode: push a DOM action-snapshot stamped at this row's timestamp so + // the trace player's Snapshot tab renders it, exactly like a regular + // command's post-action snapshot (captureActionResult). + if (this.#options.mode === 'trace' && this.#browser) { + await pushActionSnapshotAt( + this.#browser, + entry.command, + entry.timestamp, + this.#actionSnapshots + ) + } + this.#sessionCapturer.captureAssertCommand(entry) } async #finalizePerScenario() { @@ -462,6 +562,9 @@ export default class DevToolsHookService implements Services.ServiceInstance { private resetStack() { this.#commandStack = [] + this.#assertionDepth = 0 + this.#assertionStartTimes = [] + this.#assertionCallSources = [] this.#sessionCapturer.resetLastSelector() this.#sessionCapturer.resetRetryTracker() } @@ -503,7 +606,30 @@ export default class DevToolsHookService implements Services.ServiceInstance { Error.stackTraceLimit = 20 const stack = parse(new Error('')).reverse() const source = stack.find((frame) => isUserSpecFile(frame.getFileName())) - if (source && this.#commandStack.length === 0) { + // #assertionDepth > 0 → we believe we're inside an expect matcher; its + // internal commands trace back to the user's expect() line but must not + // become their own rows (only the expect.<matcher> assertion should). + // expect-webdriverio doesn't wrap afterAssertion in try/finally, though, so + // a matcher whose internal command hard-throws skips it and leaves the depth + // stuck ≥1 — which would then suppress every later user command. When a + // top-level user command arrives with the depth stuck but no live + // expect-webdriverio frame on the stack, the matcher already ended: self-heal + // the leftover depth + parallel stacks so this command captures normally. + if (source && this.#commandStack.length === 0 && this.#assertionDepth > 0) { + const inMatcher = stack.some((frame) => + frame.getFileName()?.includes('expect-webdriverio') + ) + if (!inMatcher) { + this.#assertionDepth = 0 + this.#assertionStartTimes = [] + this.#assertionCallSources = [] + } + } + if ( + source && + this.#commandStack.length === 0 && + this.#assertionDepth === 0 + ) { this.#pushTopLevelCommandFrame( command, resolveCallSourceFromFrame(source) diff --git a/packages/service/tests/action-snapshot.test.ts b/packages/service/tests/action-snapshot.test.ts new file mode 100644 index 00000000..f81be33c --- /dev/null +++ b/packages/service/tests/action-snapshot.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from 'vitest' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import { pushActionSnapshotAt } from '../src/action-snapshot.js' + +const mockBrowser = () => + ({ + execute: vi.fn().mockResolvedValue([]), + takeScreenshot: vi.fn().mockResolvedValue('SHOT'), + getUrl: vi.fn().mockResolvedValue('http://example.com/'), + getTitle: vi.fn().mockResolvedValue('Example') + }) as unknown as WebdriverIO.Browser + +describe('pushActionSnapshotAt', () => { + it('captures a DOM snapshot and stamps it at the row timestamp', async () => { + const snapshots: ActionSnapshot[] = [] + await pushActionSnapshotAt( + mockBrowser(), + 'expect.toExist', + 12345, + snapshots + ) + expect(snapshots).toHaveLength(1) + // Stamped at the row's own timestamp — not the capture time — so the trace + // player's FrameSnapshotIndex.claimAfter(cmd.timestamp) matches it. + expect(snapshots[0]!.timestamp).toBe(12345) + expect(snapshots[0]!.command).toBe('expect.toExist') + expect(snapshots[0]!.screenshot).toBe('SHOT') + }) +}) diff --git a/packages/service/tests/assert-capture.test.ts b/packages/service/tests/assert-capture.test.ts index d1486257..534945e3 100644 --- a/packages/service/tests/assert-capture.test.ts +++ b/packages/service/tests/assert-capture.test.ts @@ -7,11 +7,30 @@ import { import { captureExpectFailure, expectAssertionToCommandLog, + resolveAssertionCallSource, toCommandError, wireAssertCapture } from '../src/assert-capture.js' import type { SessionCapturer } from '../src/session.js' +const stackFrames = vi.hoisted(() => ({ + value: [] as Array<{ + getFileName: () => string | null + getLineNumber: () => number | null + getColumnNumber: () => number | null + }> +})) + +// Only resolveAssertionCallSource reads 'stack-trace'; node:assert capture uses +// core's stacktrace-parser path, so mocking here doesn't affect those tests. +vi.mock('stack-trace', () => ({ parse: () => stackFrames.value })) + +const frame = (file: string | null, line = 1, column = 1) => ({ + getFileName: () => file, + getLineNumber: () => line, + getColumnNumber: () => column +}) + describe('toCommandError', () => { it('normalizes a plain Error object (ANSI stripped)', () => { // Matcher errors are skipped now (afterAssertion owns them); a plain @@ -196,4 +215,41 @@ describe('expectAssertionToCommandLog', () => { ) expect(entry).toMatchObject({ command: 'expect.toBeClickable', args: [] }) }) + + it('forwards the captured callSource onto the assertion row', () => { + const entry = expectAssertionToCommandLog( + { matcherName: 'toExist', result: { pass: true } }, + 'uid-1', + '/proj/specs/login.e2e.ts:30:7' + ) + expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') + }) +}) + +describe('resolveAssertionCallSource', () => { + it('returns the outermost user-spec frame and loads its source', () => { + // innermost → outermost: service bundle, expect-webdriverio matcher, the + // user spec, then node internals. The user spec must win, not the bundle. + stackFrames.value = [ + frame('/repo/packages/service/dist/index.js', 4045, 12), + frame('/repo/node_modules/expect-webdriverio/lib/matchers.js', 9, 3), + frame('/proj/specs/login.e2e.ts', 30, 7), + frame('node:internal/process/task_queues', 95, 5) + ] + const captured: string[] = [] + const callSource = resolveAssertionCallSource((f) => captured.push(f)) + expect(callSource).toBe('/proj/specs/login.e2e.ts:30:7') + expect(callSource).not.toContain('/dist/') + expect(captured).toEqual(['/proj/specs/login.e2e.ts']) + }) + + it('returns undefined and loads nothing when only dependency frames exist', () => { + stackFrames.value = [ + frame('/repo/node_modules/expect-webdriverio/lib/matchers.js', 9, 3), + frame('node:internal/process/task_queues', 95, 5) + ] + const captured: string[] = [] + expect(resolveAssertionCallSource((f) => captured.push(f))).toBeUndefined() + expect(captured).toEqual([]) + }) }) diff --git a/packages/service/tests/assertion-rows.test.ts b/packages/service/tests/assertion-rows.test.ts new file mode 100644 index 00000000..5a5b09ac --- /dev/null +++ b/packages/service/tests/assertion-rows.test.ts @@ -0,0 +1,157 @@ +import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest' +import assert from 'node:assert' +import { + ASSERT_PATCHED_SYMBOL, + TRACKED_ASSERT_METHODS +} from '@wdio/devtools-core' + +// Controlled synchronous stack for the beforeAssertion call-source walk. +const stackFrames = vi.hoisted(() => ({ + value: [] as Array<{ + getFileName: () => string | null + getLineNumber: () => number | null + getColumnNumber: () => number | null + }> +})) +vi.mock('stack-trace', () => ({ parse: () => stackFrames.value })) + +const capturer = vi.hoisted(() => ({ + captureAssertCommand: vi.fn(), + captureSource: vi.fn().mockResolvedValue(undefined), + injectScript: vi.fn().mockResolvedValue(undefined), + sendUpstream: vi.fn(), + cleanup: vi.fn(), + resetLastSelector: vi.fn(), + resetRetryTracker: vi.fn(), + commandsLog: [] as unknown[], + sources: new Map<string, string>() +})) +vi.mock('../src/session.js', () => ({ + SessionCapturer: vi.fn(function (this: unknown) { + return capturer + }) +})) + +const pushActionSnapshotAt = vi.hoisted(() => + vi.fn().mockResolvedValue(undefined) +) +vi.mock('../src/action-snapshot.js', () => ({ + pushActionSnapshotAt, + captureActionSnapshot: vi.fn().mockResolvedValue(null), + captureActionResult: vi.fn().mockResolvedValue(undefined) +})) + +import DevToolsHookService from '../src/index.js' +import type { ExpectAssertion } from '../src/assert-capture.js' + +const userFrame = { + getFileName: () => '/proj/specs/login.e2e.ts', + getLineNumber: () => 30, + getColumnNumber: () => 7 +} + +const mockBrowser = { + isBidi: true, + sessionId: 's1', + options: {}, + capabilities: {}, + takeScreenshot: vi.fn().mockResolvedValue('SHOT'), + execute: vi + .fn() + .mockResolvedValue({ width: 1, height: 1, offsetLeft: 0, offsetTop: 0 }), + on: vi.fn(), + emit: vi.fn() +} as unknown as WebdriverIO.Browser + +// before() wires node:assert capture; restore the real methods afterwards. +const ASSERT_MUT = assert as unknown as Record<string | symbol, unknown> +const originals: Record<string, unknown> = {} +for (const method of TRACKED_ASSERT_METHODS) { + originals[method] = ASSERT_MUT[method] +} +afterAll(() => { + delete ASSERT_MUT[ASSERT_PATCHED_SYMBOL] + for (const method of TRACKED_ASSERT_METHODS) { + ASSERT_MUT[method] = originals[method] + } +}) + +const capturedEntry = () => capturer.captureAssertCommand.mock.calls[0]![0] + +describe('DevtoolsService — expect.* assertion rows', () => { + beforeEach(() => { + vi.clearAllMocks() + stackFrames.value = [userFrame] + }) + + it('trace mode: user-spec callSource, spec source loaded, DOM snapshot pushed', async () => { + const service = new DevToolsHookService({ mode: 'trace' }) + await service.before({} as never, [], mockBrowser) + + service.beforeAssertion() + const params: ExpectAssertion = { + matcherName: 'toExist', + result: { pass: true, message: () => 'ok' } + } + await service.afterAssertion(params) + + const entry = capturedEntry() + expect(entry.command).toBe('expect.toExist') + // The regression: callSource must point at the user's spec, not the + // service bundle (…/service/dist/index.js). + expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') + expect(entry.callSource).not.toContain('/dist/') + // Source of that file is loaded so the Source tab can render it. + expect(capturer.captureSource).toHaveBeenCalledWith( + '/proj/specs/login.e2e.ts' + ) + // Live-mode screenshot kept for the CommandLog. + expect(entry.screenshot).toBe('SHOT') + // Trace-player Snapshot tab: a DOM snapshot stamped at the row timestamp. + expect(pushActionSnapshotAt).toHaveBeenCalledWith( + mockBrowser, + 'expect.toExist', + entry.timestamp, + expect.any(Array) + ) + // WDIO reconciles rows by timestamp (like every regular WDIO command), + // so assertion rows carry no public id — parity with regular commands. + expect(entry.id).toBeUndefined() + }) + + it('live mode: keeps the screenshot but pushes no DOM snapshot', async () => { + const service = new DevToolsHookService({ mode: 'live' }) + await service.before({} as never, [], mockBrowser) + + service.beforeAssertion() + await service.afterAssertion({ + matcherName: 'toHaveText', + expectedValue: 'Hi', + result: { pass: false, message: () => 'nope' } + }) + + const entry = capturedEntry() + expect(entry.command).toBe('expect.toHaveText') + expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') + expect(entry.error).toMatchObject({ message: 'nope' }) + expect(entry.screenshot).toBe('SHOT') + expect(pushActionSnapshotAt).not.toHaveBeenCalled() + }) + + it('captureAssertions: false suppresses the row but keeps the window balanced', async () => { + const service = new DevToolsHookService({ + mode: 'trace', + captureAssertions: false + }) + await service.before({} as never, [], mockBrowser) + + service.beforeAssertion() + await service.afterAssertion({ + matcherName: 'toExist', + result: { pass: true } + }) + + expect(capturer.captureAssertCommand).not.toHaveBeenCalled() + expect(pushActionSnapshotAt).not.toHaveBeenCalled() + }) +}) diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index 5ce1e1f5..1809fa6c 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -15,6 +15,8 @@ const mockSessionCapturerInstance = { afterCommand: vi.fn(), sendUpstream: vi.fn(), injectScript: vi.fn().mockResolvedValue(undefined), + captureSource: vi.fn(), + captureAssertCommand: vi.fn(), cleanup: vi.fn(), commandsLog: [], sources: new Map(), @@ -280,3 +282,65 @@ describe('DevtoolsService - Screencast Integration', () => { expect(mockScreencastRecorder.start).toHaveBeenCalledWith(mockBrowser) }) }) + +describe('DevtoolsService - assertion suppression self-heal', () => { + let service: DevToolsHookService + const mockBrowser = { + isBidi: true, + sessionId: 'heal-session', + scriptAddPreloadScript: vi.fn().mockResolvedValue(undefined), + takeScreenshot: vi.fn().mockResolvedValue('screenshot'), + execute: vi.fn().mockResolvedValue({ + width: 1200, + height: 800, + offsetLeft: 0, + offsetTop: 0 + }), + on: vi.fn(), + emit: vi.fn() + } as any + + const capturedCommands = () => + mockSessionCapturerInstance.afterCommand.mock.calls.map((call) => call[1]) + + beforeEach(async () => { + vi.clearAllMocks() + service = new DevToolsHookService() + await service.before({} as any, [], mockBrowser) + vi.clearAllMocks() + }) + + // expect-webdriverio doesn't wrap afterAssertion in try/finally, so a matcher + // whose internal command hard-throws runs beforeAssertion but skips + // afterAssertion, leaving #assertionDepth stuck ≥ 1. Without the self-heal + // that stuck depth suppresses every later user command in the test. + it('captures a top-level command after a matcher throw left the assertion depth stuck', async () => { + service.beforeAssertion() + service.beforeAssertion() // depth now 2, no matching afterAssertion + + // The mocked stack is a user-spec frame with no expect-webdriverio frame, + // so the command is recognised as top-level and the leftover depth resets. + service.beforeCommand('click' as any, ['.button']) + await service.afterCommand('click' as any, ['.button'], undefined) + + expect(capturedCommands()).toEqual(['click']) + }) + + it('a fresh beforeAssertion resets a stuck depth so the window stays balanced', async () => { + service.beforeAssertion() + service.beforeAssertion() // leftover depth from a prior throw + + // A new matcher starts (resetting the stuck depth) and completes normally. + service.beforeAssertion() + await service.afterAssertion({ + matcherName: 'toBeDisplayed', + result: { pass: true } + } as any) + + // Depth is balanced again, so the next top-level command is captured. + service.beforeCommand('setValue' as any, ['.input', 'hi']) + await service.afterCommand('setValue' as any, ['.input', 'hi'], undefined) + + expect(capturedCommands()).toEqual(['setValue']) + }) +}) From d552ab2f8d0b246872c87cfc08abcd174448bb01 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:23:51 +0530 Subject: [PATCH 33/91] feat(nightwatch): capture native browser.assert/verify as assertion actions --- .../src/helpers/browserProxy.ts | 99 ++++- .../src/helpers/nativeAssertions.ts | 299 +++++++++++++++ packages/nightwatch-devtools/src/index.ts | 80 +++-- .../nightwatch-devtools/src/session-init.ts | 4 +- packages/nightwatch-devtools/src/session.ts | 35 ++ packages/nightwatch-devtools/src/types.ts | 16 +- .../tests/attemptTracking.test.ts | 4 +- .../tests/browserProxy.test.ts | 168 +++++++++ .../tests/nativeAssertions.test.ts | 340 ++++++++++++++++++ 9 files changed, 1011 insertions(+), 34 deletions(-) create mode 100644 packages/nightwatch-devtools/src/helpers/nativeAssertions.ts create mode 100644 packages/nightwatch-devtools/tests/browserProxy.test.ts create mode 100644 packages/nightwatch-devtools/tests/nativeAssertions.test.ts diff --git a/packages/nightwatch-devtools/src/helpers/browserProxy.ts b/packages/nightwatch-devtools/src/helpers/browserProxy.ts index 019ebfb2..dff01a1f 100644 --- a/packages/nightwatch-devtools/src/helpers/browserProxy.ts +++ b/packages/nightwatch-devtools/src/helpers/browserProxy.ts @@ -10,11 +10,16 @@ import { } from '../constants.js' import { getCallSourceFromStack } from './utils.js' import { serializeCommandResult } from './serializeCommandResult.js' +import { + latestResolvedScreenshot, + pendingAssertionCommand +} from './nativeAssertions.js' import { RetryTracker, toError } from '@wdio/devtools-core' import type { SessionCapturer } from '../session.js' import type { TestManager } from './testManager.js' import type { CommandLog, + NativeAssertCall, NightwatchBrowser, NightwatchCurrentTest, CommandStackFrame @@ -42,10 +47,20 @@ export class BrowserProxy { */ private retryTracker = new RetryTracker() + /** + * Per-test buffer of explicit `browser.assert.*` / `browser.verify.*` calls, + * recorded synchronously at call time by {@link wrapAssertionNamespaces}. + * Nightwatch exposes no per-assertion hook and its test-end results carry no + * source location for passing assertions, so call-time capture is the only + * way to get real args + callSource. Drained (and cleared) each `afterEach`. + */ + private nativeAssertCalls: NativeAssertCall[] = [] + constructor( private sessionCapturer: SessionCapturer, private testManager: TestManager, - private getCurrentTest: () => { uid?: string } | null + private getCurrentTest: () => { uid?: string } | null, + private captureAssertions = true ) {} /** @@ -60,6 +75,78 @@ export class BrowserProxy { this.commandStack = [] this.lastCommandSig = null this.retryTracker.reset() + this.nativeAssertCalls = [] + } + + /** Hand off this test's recorded native-assertion calls and clear the + * buffer so the next test starts fresh. */ + drainNativeAssertCalls(): NativeAssertCall[] { + const calls = this.nativeAssertCalls + this.nativeAssertCalls = [] + return calls + } + + /** + * Replace `browser.assert` / `browser.verify` (both are Nightwatch Proxies + * whose `get` returns a fresh function per access) with a recording Proxy: + * on each method call it captures `{prefix, method, args, callSource}` from a + * user-code frame, streams a neutral pending row immediately (live), then + * delegates to the ORIGINAL namespace method so Nightwatch's queue, chaining, + * and abortOnFailure semantics (assert aborts, verify does not) are + * byte-for-byte unchanged. Called once per browser. + */ + private wrapAssertionNamespaces(browser: NightwatchBrowser): void { + // captureAssertions:false → leave assert/verify original so no neutral + // pending rows stream (the afterEach finalize path is gated to match). + if (!this.captureAssertions) { + return + } + // Cast once: the assert/verify namespaces aren't on the public type; each + // is a dynamic method bag reached by property name. + const b = browser as unknown as Record<string, unknown> + ;(['assert', 'verify'] as const).forEach((prefix) => { + const original = b[prefix] + if (!original || typeof original !== 'object') { + return + } + b[prefix] = new Proxy(original as object, { + get: (target, name, receiver) => { + const orig = Reflect.get(target, name, receiver) + // `not` (negation Proxy) and non-method props pass through untouched. + if (typeof orig !== 'function' || typeof name !== 'string') { + return orig + } + return (...args: unknown[]) => { + const callInfo = getCallSourceFromStack() + if (callInfo.filePath !== undefined) { + this.emitPendingAssertion({ + prefix, + method: name, + args, + callSource: callInfo.callSource, + timestamp: Date.now() + }) + } + return (orig as (...a: unknown[]) => unknown)(...args) + } + } + }) + }) + } + + /** Stream a neutral pending row for an explicit assert/verify call the moment + * it's invoked, and buffer the call (with its emitted row) for `afterEach` + * to finalize pass/fail in place. */ + private emitPendingAssertion(call: NativeAssertCall): void { + const testUid = this.getCurrentTest()?.uid + const entry = pendingAssertionCommand( + call, + testUid, + latestResolvedScreenshot(this.sessionCapturer) + ) + this.sessionCapturer.captureAssertCommand(entry) + call.entry = entry + this.nativeAssertCalls.push(call) } getCurrentTestFullPath(): string | null { @@ -189,6 +276,7 @@ export class BrowserProxy { wrappedMethods.push(methodName) }) + this.wrapAssertionNamespaces(browser) this.proxiedBrowsers.add(browser as object) log.info(`✓ Wrapped ${wrappedMethods.length} browser methods`) } @@ -307,6 +395,7 @@ export class BrowserProxy { logArgs: unknown[], cmdSig: string, callSource: string | undefined, + hasUserSource: boolean, commandTimestamp: number, testUid: string | undefined, userCallback: Function | null @@ -318,7 +407,12 @@ export class BrowserProxy { methodName ) const effectiveUid = this.getCurrentTest()?.uid ?? testUid - if (effectiveUid) { + // Only surface commands that originate from a user-code frame. Commands + // Nightwatch issues from inside its own queue (e.g. the getTitle a + // `browser.assert.titleContains` runs) execute in a detached tick with no + // user frame on the stack, so they'd otherwise leak as top-level actions + // with an "unknown" source. Mirrors the service's user-spec-source guard. + if (effectiveUid && hasUserSource) { if (this.retryTracker.isRetry(cmdSig)) { this.handleRetryReplacement( browser, @@ -402,6 +496,7 @@ export class BrowserProxy { logArgs, cmdSig, callSource, + callInfo.filePath !== undefined, commandTimestamp, this.getCurrentTest()?.uid, userCallback diff --git a/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts new file mode 100644 index 00000000..16dd3bd2 --- /dev/null +++ b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts @@ -0,0 +1,299 @@ +// Native-assertion capture: turns explicit `browser.assert.*` / +// `browser.verify.*` calls into concise trace/UI action rows with real args, +// a clickable source location, and pass/fail colour — streamed LIVE. +// +// Nightwatch exposes no per-assertion hook, and its test-end +// `currentTest.results` carries no source location for PASSING assertions +// (results.assertions[i].stackTrace is '' on pass) and only stringified args. +// So each explicit call is intercepted at CALL TIME +// (BrowserProxy.wrapAssertionNamespaces): a neutral "pending" row is emitted +// immediately (concise title, real args, callSource) so rows stream in one by +// one like normal commands. At TEST END the pass/fail truth + verbose failure +// message are read from results.assertions and the already-emitted row is +// UPDATED IN PLACE (same stable id) — never re-created, so no duplicates. +// Iterating the recorded calls (not results.assertions) also excludes +// Nightwatch's implicit command-generated assertions (e.g. +// waitForElementVisible's "element was visible" entry). + +import logger from '@wdio/logger' +import { + matcherAssertionToCommandLog, + safeSerializeAssertArg, + stripAnsi +} from '@wdio/devtools-core' +import type { SessionCapturer } from '../session.js' +import type { + CommandLog, + NativeAssertCall, + NightwatchBrowser, + NightwatchCurrentTest +} from '../types.js' + +const log = logger('@wdio/nightwatch-devtools:nativeAssertions') + +/** + * One entry Nightwatch pushes to `results.assertions` (from + * `NightwatchAssertion.getAssertResult`, lib/assertion/assertion.js). Carries + * only a human message — no method/args. `failure === false` is the reliable + * pass signal; any truthy `failure` (message string, or `true`) means failed. + */ +interface NwAssertionEntry { + message?: string + fullMsg?: string + failure?: string | boolean +} + +/** One entry Nightwatch pushes to `results.commands` (lib/reporter/index.js + * logCommandResult). For an assert/verify `name` is the namespaced method and + * `startTime`/`endTime` are the real queue-execution window in ms + * (treenode.js). Read only to position the finalized row on the timeline. */ +interface NwCommandEntry { + name?: string + startTime?: number + endTime?: number +} + +const ASSERT_CMD_RE = /^(assert|verify)\.\w+$/ + +/** Real per-assertion execution windows, in call order, aligned to `count` + * recorded calls. Nightwatch enqueues assertions synchronously (all at once) + * but runs them later one at a time, so the enqueue timestamp clusters the + * rows; this recovers each row's true execution time so the trace timeline + * spreads them out. Returns `null` per slot when the executed-command count + * doesn't line up (e.g. retries) — the enqueue timestamp is kept then. */ +function assertCommandTimings( + commands: NwCommandEntry[], + count: number +): Array<{ startTime: number; endTime: number } | null> { + const executed = commands.filter( + (c) => typeof c?.name === 'string' && ASSERT_CMD_RE.test(c.name) + ) + if (executed.length !== count) { + return new Array(count).fill(null) + } + return executed.map((c) => + typeof c.startTime === 'number' && typeof c.endTime === 'number' + ? { startTime: c.startTime, endTime: c.endTime } + : null + ) +} + +/** Nightwatch embeds the assertion arguments in the message text + * (AssertionInstance.initialize → Logger.formatMessage), so a passing entry + * for `titleContains('Example')` reads "Testing if the page title contains + * 'Example'". Match a call to its result entry when every string/number arg + * appears in the message. */ +function messageMatchesArgs(entry: NwAssertionEntry, args: unknown[]): boolean { + const text = String(entry.fullMsg ?? entry.message ?? '') + const literals = args.filter( + (a): a is string | number => typeof a === 'string' || typeof a === 'number' + ) + return literals.length > 0 && literals.every((a) => text.includes(String(a))) +} + +interface Outcome { + passed: boolean + message: string +} + +/** + * Pair each recorded call with its `results.assertions` entry for pass/fail + + * verbose message. Both lists are in call/execution order and each explicit + * call produces exactly one assertion entry, so: match by args-in-message + * first (most specific, skips interleaved implicit entries), then fall back to + * the next unconsumed entry positionally. A call with no matching entry left + * (never happens for a real assertion) is dropped (`null`). + */ +function correlate( + calls: NativeAssertCall[], + assertions: NwAssertionEntry[] +): Array<Outcome | null> { + const consumed = new Array(assertions.length).fill(false) + const toOutcome = (idx: number): Outcome => { + consumed[idx] = true + const entry = assertions[idx] + return { + passed: !entry.failure, + message: stripAnsi(String(entry.fullMsg ?? entry.message ?? '')).trim() + } + } + const matched = calls.map((call) => { + const idx = assertions.findIndex( + (entry, i) => !consumed[i] && messageMatchesArgs(entry, call.args) + ) + return idx === -1 ? null : toOutcome(idx) + }) + return matched.map((outcome) => { + if (outcome) { + return outcome + } + const idx = consumed.findIndex((used) => !used) + return idx === -1 ? null : toOutcome(idx) + }) +} + +/** Last already-resolved screenshot in the command log — the DOM the assertion + * evaluated against (title/most asserts don't mutate it). Synchronous, so it's + * usable from the call-time wrapper. */ +export function latestResolvedScreenshot( + capturer: SessionCapturer +): string | null { + for (let i = capturer.commandsLog.length - 1; i >= 0; i--) { + const shot = capturer.commandsLog[i]?.screenshot + if (shot) { + return shot + } + } + return null +} + +/** Reuse the nearest preceding command's screenshot; if the fire-and-forget + * capture hasn't resolved yet (race), fall back to a fresh end-of-test one. */ +async function resolveAssertionScreenshot( + capturer: SessionCapturer, + browser: NightwatchBrowser +): Promise<string | null> { + return ( + latestResolvedScreenshot(capturer) ?? + (await capturer.takeScreenshotViaHttp(browser)) + ) +} + +/** `assert.titleContains('SOFT_FAIL_ME')` — the concise row label. Strings are + * quoted, objects elided; never the verbose failure message. */ +function conciseTitle(call: NativeAssertCall): string { + const preview = call.args + .map((a) => + typeof a === 'string' + ? `'${a}'` + : a !== null && typeof a === 'object' + ? '…' + : String(a) + ) + .join(', ') + return `${call.prefix}.${call.method}(${preview})` +} + +/** + * Build the neutral "pending" row emitted the moment an assert/verify is + * called — everything known at call time (concise title, real args, callSource, + * screenshot) but NO result/error yet, so it renders neutral (not red/green) + * and streams in like a normal in-flight command. `startTime`/`timestamp` are + * the call time; the row is finalized in place later by + * {@link captureNativeAssertions}. + */ +export function pendingAssertionCommand( + call: NativeAssertCall, + testUid: string | undefined, + screenshot: string | null +): CommandLog { + const entry: CommandLog = { + command: `${call.prefix}.${call.method}`, + args: call.args.map(safeSerializeAssertArg), + title: conciseTitle(call), + timestamp: call.timestamp, + startTime: call.timestamp + } + if (call.callSource) { + entry.callSource = call.callSource + } + if (testUid) { + entry.testUid = testUid + } + if (screenshot) { + entry.screenshot = screenshot + } + return entry +} + +/** Update one streamed pending row in place: apply pass/fail + verbose error, + * a screenshot, and its real execution window, then re-broadcast by stable id. */ +function finalizeAssertionRow( + capturer: SessionCapturer, + call: NativeAssertCall, + outcome: Outcome, + timing: { startTime: number; endTime: number } | null, + screenshot: string | null, + testUid: string | undefined +): void { + const entry = call.entry! + const finalized = matcherAssertionToCommandLog( + { + prefix: call.prefix, + method: call.method, + args: call.args, + passed: outcome.passed, + message: outcome.message || `${call.prefix}.${call.method} failed`, + callSource: call.callSource, + title: entry.title + }, + testUid + ) + entry.result = finalized.result + entry.error = finalized.error + if (!entry.screenshot && screenshot) { + entry.screenshot = screenshot + } + // Reposition the row on its REAL execution window (Nightwatch enqueues all + // asserts at once, so the emit/enqueue timestamp clustered them). The row is + // matched for replacement by its stable id, so the old enqueue timestamp is + // what the UI still keys on until this swap lands. + const oldTimestamp = entry.timestamp + if (timing) { + entry.startTime = timing.startTime + entry.timestamp = + timing.endTime > timing.startTime ? timing.endTime : timing.startTime + } + capturer.sendReplaceCommand(oldTimestamp, entry) + log.info(`[assert] ${entry.title} → ${outcome.passed ? 'pass' : 'fail'}`) +} + +/** + * Finalize the streamed pending rows at test-end: correlate the recorded calls + * with `results.assertions`, then UPDATE each row's `entry` in place with + * pass/fail (`result`) + the verbose failure message (`error`, failures only) + * + a screenshot + its real execution window, and re-broadcast via + * `sendReplaceCommand` keyed on the row's stable id. No new rows are created + * (no duplicates); a call with no matching result is left pending (defensive). + */ +export async function captureNativeAssertions( + capturer: SessionCapturer, + browser: NightwatchBrowser, + currentTest: NightwatchCurrentTest | undefined, + testUid: string | undefined, + calls: NativeAssertCall[] +): Promise<void> { + if (calls.length === 0) { + return + } + // Boundary cast: `results` is Nightwatch's loosely-typed per-test bag; we read + // only the assertions + commands arrays whose shapes are documented above. + const results = currentTest?.results as + | { assertions?: NwAssertionEntry[]; commands?: NwCommandEntry[] } + | undefined + const assertions = Array.isArray(results?.assertions) + ? results.assertions + : [] + const outcomes = correlate(calls, assertions) + const timings = assertCommandTimings( + Array.isArray(results?.commands) ? results.commands : [], + calls.length + ) + // One shared screenshot for all rows — same DOM, ran consecutively. + const screenshot = await resolveAssertionScreenshot(capturer, browser) + + calls.forEach((call, index) => { + const outcome = outcomes[index] + // Leave an unmatched (or never-emitted) row in its last state. + if (call.entry && outcome) { + finalizeAssertionRow( + capturer, + call, + outcome, + timings[index], + screenshot, + testUid + ) + } + }) +} diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index e94877bc..db5cefc0 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -9,8 +9,7 @@ import { fileURLToPath } from 'node:url' import { errorMessage, finalizeTraceExport, - flushRangeTrace, - recordSpecBoundary, + flushRangeLogged, TestAttemptTracker, tracePolicyModeWarning, type SpecRange, @@ -72,6 +71,8 @@ import { ensureSessionInitialized, finalizeCurrentScreencast } from './session-init.js' +import { captureNativeAssertions } from './helpers/nativeAssertions.js' +import { flushTestSlice, recordSpecSliceBoundary } from './trace-slices.js' import { getTestIcon, incrementCounters, @@ -179,6 +180,9 @@ class NightwatchDevToolsPlugin { get bidiEnabled() { return self.#bidiEnabled }, + get captureAssertions() { + return self.options.captureAssertions + }, get sessionCapturer() { return self.sessionCapturer }, @@ -305,11 +309,29 @@ class NightwatchDevToolsPlugin { setCucumberRunner: (v) => { self.#isCucumberRunner = v }, - getRerunLabel: () => self.#getRerunLabel() + getRerunLabel: () => self.#getRerunLabel(), + get traceMode() { + return self.options.mode === 'trace' + }, + get traceGranularity() { + return self.options.traceGranularity + }, + get specRanges() { + return self.#specRanges + }, + get flushedSpecs() { + return self.#flushedSpecs + }, + flushTraceRange: (range) => self.#flushSpecTrace(range) } return this.#internals } + /** Boundary cast: currentTest is Nightwatch's loose bag; only uid is read. */ + #currentTestUid(): string | undefined { + return (this.#currentTest as { uid?: string } | null)?.uid + } + #handleReuseMode(): void { handleReuseMode(this.#getInternals()) } @@ -330,8 +352,7 @@ class NightwatchDevToolsPlugin { if (this.options.captureAssertions) { wireAssertCapture( () => this.sessionCapturer, - // Boundary cast: currentTest is Nightwatch's loose bag; only uid is read. - () => (this.#currentTest as { uid?: string } | null)?.uid + () => this.#currentTestUid() ) } } @@ -401,13 +422,15 @@ class NightwatchDevToolsPlugin { async #startNextTest( currentSuite: SuiteStats, currentTestName: string, - processedTests: Set<string> + processedTests: Set<string>, + specFile: string | null ): Promise<void> { await startNextTest( this.#getInternals(), currentSuite, currentTestName, - processedTests + processedTests, + specFile ) } @@ -453,25 +476,8 @@ class NightwatchDevToolsPlugin { this.sessionCapturer.captureSource(fullPath).catch(() => {}) } - // ── Per-spec boundary detection ── if (fullPath) { - const prevRange = recordSpecBoundary( - { - specRanges: this.#specRanges, - flushedSpecs: this.#flushedSpecs, - capturer: this.sessionCapturer, - actionSnapshots: this.sessionCapturer.actionSnapshots - }, - fullPath, - this.options.traceGranularity - ) - if (prevRange) { - void this.#flushSpecTrace(prevRange).catch((err) => - log.warn( - `Failed to flush trace for spec "${prevRange.specFile}": ${errorMessage(err)}` - ) - ) - } + recordSpecSliceBoundary(this.#getInternals(), fullPath) } await this.#closePreviousRunningTest(currentSuite, testFile, currentTest) @@ -483,7 +489,12 @@ class NightwatchDevToolsPlugin { processedTests ) if (currentTestName) { - await this.#startNextTest(currentSuite, currentTestName, processedTests) + await this.#startNextTest( + currentSuite, + currentTestName, + processedTests, + fullPath + ) } this.#wrapBrowserOnce(browser) } @@ -497,7 +508,18 @@ class NightwatchDevToolsPlugin { if (browser && this.sessionCapturer) { try { await this.#closeOutTestcases(browser) + if (this.options.captureAssertions) { + await captureNativeAssertions( + this.sessionCapturer, + browser, + browser.currentTest as NightwatchCurrentTest | undefined, + this.#currentTestUid(), + this.browserProxy.drainNativeAssertCalls() + ) + } await this.sessionCapturer.captureTrace(browser) + // Flush this test's slice before the next test overwrites its outcome. + flushTestSlice(this.#getInternals()) } catch (err) { log.error(`Failed to capture trace: ${errorMessage(err)}`) } @@ -542,13 +564,15 @@ class NightwatchDevToolsPlugin { logRunSummary(this.#getInternals()) } - /** Thin wrapper so boundary flushes and the final flush share one path. */ + /** Thin wrapper so boundary flushes and the final flush share one path. + * flushRangeLogged logs+swallows a failed flush (shared spec/test string) so + * the fire-and-forget boundary callers don't each re-implement the guard. */ #flushSpecTrace(range: SpecRange): Promise<TraceArtifact | undefined> { const sessionId = this.sessionCapturer.metadata?.sessionId if (!sessionId) { return Promise.resolve(undefined) } - return flushRangeTrace(this.#traceContext(sessionId), range) + return flushRangeLogged(this.#traceContext(sessionId), range) } /** Assemble the framework-agnostic trace-export context from plugin state. diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 2bccfc82..e9d1004a 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -39,6 +39,7 @@ export interface SessionInitCtx { readonly screencastOptions: ScreencastOptions readonly bidiEnabled: boolean readonly mode: DevToolsMode + readonly captureAssertions: boolean sessionCapturer: SessionCapturer testReporter: TestReporter @@ -98,7 +99,8 @@ function initReporterChain(ctx: SessionInitCtx): void { ctx.browserProxy = new BrowserProxy( ctx.sessionCapturer, ctx.testManager, - () => ctx.getCurrentTest() ?? ctx.getCurrentScenarioSuite() + () => ctx.getCurrentTest() ?? ctx.getCurrentScenarioSuite(), + ctx.captureAssertions ) } diff --git a/packages/nightwatch-devtools/src/session.ts b/packages/nightwatch-devtools/src/session.ts index ae830af7..ed9c5457 100644 --- a/packages/nightwatch-devtools/src/session.ts +++ b/packages/nightwatch-devtools/src/session.ts @@ -213,6 +213,41 @@ export class SessionCapturer extends SessionCapturerBase { ) } + /** + * Ingest a pre-built assertion entry (native `browser.assert`/`verify` + * synthesized from Nightwatch's results, or a node:assert capture) into the + * same command stream driver commands use. Unlike {@link captureCommand} this + * preserves the entry's `title` (the human assertion message) and never + * dedups — each call is a distinct action row. Assigns a fresh `_id` and a + * matching stable public `id` (so a later `sendReplaceCommand` can update + * this exact row in place — native asserts stream a pending row at call time, + * then finalize their pass/fail), pushes, and broadcasts. + */ + captureAssertCommand(entry: CommandLog): void { + const withId = entry as CommandLog & { _id?: number } + withId._id = this.commandCounter++ + withId.id = withId._id + this.commandsLog.push(withId) + if ( + this.traceMode === 'trace' && + !entry.error && + this.#browser && + mapCommandToAction(entry.command) + ) { + const browser = this.#browser + this.snapshotCaptures.push( + captureActionSnapshot(browser, entry.command, () => + this.takeScreenshotViaHttp(browser) + ).then((snap) => { + if (snap) { + this.actionSnapshots.push(snap) + } + }) + ) + } + this.sendCommand(withId) + } + /** * Replace an already-captured command entry (used for retried commands so * only the final execution result is shown in the UI). diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index ab776db5..203dbbdb 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -23,7 +23,7 @@ export { type TraceLog } from '@wdio/devtools-shared' -import type { BaseDevToolsOptions } from '@wdio/devtools-shared' +import type { BaseDevToolsOptions, CommandLog } from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -31,6 +31,20 @@ export interface CommandStackFrame { signature: string } +/** One explicit `browser.assert.*` / `browser.verify.*` call, recorded at call + * time by BrowserProxy. A neutral "pending" row (`entry`) is streamed live at + * call time; `captureNativeAssertions` finalizes its pass/fail at test-end. */ +export interface NativeAssertCall { + prefix: 'assert' | 'verify' + method: string + args: unknown[] + callSource?: string + /** Wall-clock at call time; also the streamed row's timestamp/startTime. */ + timestamp: number + /** The pending row emitted at call time, updated in place when finalized. */ + entry?: CommandLog +} + export interface NightwatchTestCase { passed: number failed: number diff --git a/packages/nightwatch-devtools/tests/attemptTracking.test.ts b/packages/nightwatch-devtools/tests/attemptTracking.test.ts index c08313c4..30df38e2 100644 --- a/packages/nightwatch-devtools/tests/attemptTracking.test.ts +++ b/packages/nightwatch-devtools/tests/attemptTracking.test.ts @@ -89,10 +89,10 @@ describe('regular test retry attempt via startNextTest', () => { recordAttempt: (uid: string) => tracker.recordStart(uid) } as unknown as TestLifecycleCtx - await startNextTest(ctx, suite, 'my test', new Set()) + await startNextTest(ctx, suite, 'my test', new Set(), null) expect(test.retries).toBe(0) - await startNextTest(ctx, suite, 'my test', new Set(['my test'])) + await startNextTest(ctx, suite, 'my test', new Set(['my test']), null) expect(test.retries).toBe(1) }) }) diff --git a/packages/nightwatch-devtools/tests/browserProxy.test.ts b/packages/nightwatch-devtools/tests/browserProxy.test.ts new file mode 100644 index 00000000..d69ffcf3 --- /dev/null +++ b/packages/nightwatch-devtools/tests/browserProxy.test.ts @@ -0,0 +1,168 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type { CommandLog, NightwatchBrowser } from '../src/types.js' + +// browserProxy resolves the caller's source via this helper; stub it so each +// test can decide whether the command looks user-issued or framework-internal. +// vi.hoisted so the stub exists when the hoisted vi.mock factory runs. +const { getCallSourceFromStack } = vi.hoisted(() => ({ + getCallSourceFromStack: vi.fn() +})) +vi.mock('../src/helpers/utils.js', () => ({ getCallSourceFromStack })) + +import { BrowserProxy } from '../src/helpers/browserProxy.js' +import type { SessionCapturer } from '../src/session.js' +import type { TestManager } from '../src/helpers/testManager.js' + +function makeCapturer() { + const commandsLog: (CommandLog & { _id?: number })[] = [] + let counter = 0 + const captureCommand = vi.fn( + async ( + command: string, + args: unknown[], + result: unknown, + error: Error | undefined, + testUid?: string, + callSource?: string, + timestamp?: number + ) => { + commandsLog.push({ + _id: counter++, + command, + args, + result, + error, + testUid, + callSource, + timestamp + }) + return true + } + ) + const capturer = { + commandsLog, + captureCommand, + replaceCommand: vi.fn(), + sendCommand: vi.fn(), + sendReplaceCommand: vi.fn(), + takeScreenshotViaHttp: vi.fn(async () => null), + captureTrace: vi.fn(async () => {}) + } as unknown as SessionCapturer + return { capturer, commandsLog, captureCommand } +} + +function makeTestManager() { + return { + detectTestBoundary: vi.fn(() => ''), + startTestIfPending: vi.fn() + } as unknown as TestManager +} + +/** A browser whose single command echoes its result into the capture callback + * Nightwatch appends as the last argument. */ +function makeBrowser() { + return { + titleContains: (_arg: unknown, cb: (result: unknown) => void) => { + cb('Example Domain') + return undefined + } + } as unknown as NightwatchBrowser +} + +describe('BrowserProxy internal-command suppression', () => { + beforeEach(() => { + getCallSourceFromStack.mockReset() + }) + + it('captures a command issued from a user-code frame', () => { + const { capturer, commandsLog, captureCommand } = makeCapturer() + getCallSourceFromStack.mockReturnValue({ + filePath: '/tests/spec.js', + callSource: '/tests/spec.js:5' + }) + const proxy = new BrowserProxy(capturer, makeTestManager(), () => ({ + uid: 'test-1' + })) + const browser = makeBrowser() + proxy.wrapBrowserCommands(browser) + ;( + browser as unknown as Record<string, (a: unknown) => unknown> + ).titleContains('Example') + + expect(captureCommand).toHaveBeenCalledTimes(1) + expect(commandsLog).toHaveLength(1) + expect(commandsLog[0].command).toBe('titleContains') + expect(commandsLog[0].callSource).toBe('/tests/spec.js:5') + }) + + it('suppresses a framework-internal command with no user-code frame', () => { + const { capturer, commandsLog, captureCommand } = makeCapturer() + // Mirrors the getTitle a `browser.assert.titleContains` issues from inside + // Nightwatch's queue: no user frame, so getCallSourceFromStack returns none. + getCallSourceFromStack.mockReturnValue({ + filePath: undefined, + callSource: 'unknown:0' + }) + const proxy = new BrowserProxy(capturer, makeTestManager(), () => ({ + uid: 'test-1' + })) + const browser = makeBrowser() + proxy.wrapBrowserCommands(browser) + ;( + browser as unknown as Record<string, (a: unknown) => unknown> + ).titleContains('Example') + + expect(captureCommand).not.toHaveBeenCalled() + expect(commandsLog).toHaveLength(0) + }) +}) + +describe('BrowserProxy captureAssertions gating', () => { + beforeEach(() => { + getCallSourceFromStack.mockReset() + }) + + /** A browser exposing `assert`/`verify` namespace objects (as Nightwatch + * does), so the test can check whether wrapAssertionNamespaces replaced them + * with a recording Proxy. */ + function makeAssertBrowser() { + return { + assert: { titleContains: vi.fn() }, + verify: { titleContains: vi.fn() } + } as unknown as NightwatchBrowser & { + assert: object + verify: object + } + } + + it('leaves assert/verify namespaces original when captureAssertions is false', () => { + const { capturer } = makeCapturer() + const browser = makeAssertBrowser() + const originalAssert = browser.assert + const originalVerify = browser.verify + const proxy = new BrowserProxy( + capturer, + makeTestManager(), + () => ({ uid: 't1' }), + false + ) + proxy.wrapBrowserCommands(browser) + // No wrapping → the namespaces are untouched, so no pending rows can stream. + expect(browser.assert).toBe(originalAssert) + expect(browser.verify).toBe(originalVerify) + }) + + it('wraps assert/verify namespaces by default (captureAssertions true)', () => { + const { capturer } = makeCapturer() + const browser = makeAssertBrowser() + const originalAssert = browser.assert + const originalVerify = browser.verify + const proxy = new BrowserProxy(capturer, makeTestManager(), () => ({ + uid: 't1' + })) + proxy.wrapBrowserCommands(browser) + // Replaced with a recording Proxy → no longer the original object. + expect(browser.assert).not.toBe(originalAssert) + expect(browser.verify).not.toBe(originalVerify) + }) +}) diff --git a/packages/nightwatch-devtools/tests/nativeAssertions.test.ts b/packages/nightwatch-devtools/tests/nativeAssertions.test.ts new file mode 100644 index 00000000..d904dbde --- /dev/null +++ b/packages/nightwatch-devtools/tests/nativeAssertions.test.ts @@ -0,0 +1,340 @@ +import { describe, it, expect, vi } from 'vitest' +import { + captureNativeAssertions, + latestResolvedScreenshot, + pendingAssertionCommand +} from '../src/helpers/nativeAssertions.js' +import type { SessionCapturer } from '../src/session.js' +import type { + CommandLog, + NativeAssertCall, + NightwatchBrowser, + NightwatchCurrentTest +} from '../src/types.js' + +/** Fake capturer mimicking the live stream: `captureAssertCommand` assigns a + * stable public `id` and appends to `commandsLog` (like the real one); + * `sendReplaceCommand` records in-place updates; `takeScreenshotViaHttp` is + * the fresh-fallback. */ +function makeFakeCapturer( + commandsLog: CommandLog[] = [], + freshScreenshot: string | null = 'FRESH_SHOT' +) { + const sent: CommandLog[] = [] + const replaced: Array<{ oldTimestamp: number; command: CommandLog }> = [] + let counter = 100 + const captureAssertCommand = vi.fn( + (entry: CommandLog & { _id?: number; id?: number }) => { + entry._id = counter++ + entry.id = entry._id + commandsLog.push(entry) + sent.push(entry) + } + ) + const sendReplaceCommand = vi.fn( + (oldTimestamp: number, command: CommandLog) => { + replaced.push({ oldTimestamp, command }) + } + ) + const takeScreenshotViaHttp = vi.fn(() => Promise.resolve(freshScreenshot)) + const capturer = { + commandsLog, + captureAssertCommand, + sendReplaceCommand, + takeScreenshotViaHttp + } as unknown as SessionCapturer + return { + capturer, + commandsLog, + sent, + replaced, + captureAssertCommand, + sendReplaceCommand, + takeScreenshotViaHttp + } +} + +const browser = {} as unknown as NightwatchBrowser + +/** Mimic BrowserProxy.emitPendingAssertion: stream a neutral pending row at + * call time and attach it to the call for later finalization. */ +function emitPending( + capturer: SessionCapturer, + calls: NativeAssertCall[], + testUid: string | undefined +) { + for (const call of calls) { + const entry = pendingAssertionCommand( + call, + testUid, + latestResolvedScreenshot(capturer) + ) + capturer.captureAssertCommand(entry) + call.entry = entry + } +} + +/** Nightwatch's `getAssertResult` entries: pass → `failure: false`; fail → + * `failure` is a message string. The message embeds the assertion args. */ +function passing(message: string) { + return { message, fullMsg: message, failure: false as const } +} +function failing(message: string) { + return { message, fullMsg: message, failure: `${message} — failed` } +} + +function currentTestWith( + assertions: unknown[], + commands: unknown[] = [] +): NightwatchCurrentTest { + return { + name: 'renders asserts', + module: 'assert-check', + results: { assertions, commands } + } as unknown as NightwatchCurrentTest +} + +let clock = 1000 +function call( + prefix: 'assert' | 'verify', + method: string, + args: unknown[], + callSource = 'spec.js:8' +): NativeAssertCall { + return { prefix, method, args, callSource, timestamp: clock++ } +} + +describe('captureNativeAssertions (live pending → finalize)', () => { + it('streams neutral pending rows at call time, then finalizes pass/fail in place with a stable id and no duplicates', async () => { + // Preceding real commands: the last with a resolved screenshot is the DOM + // the assertions evaluated against — reused for the pending rows. + const precedingCommands: CommandLog[] = [ + { command: 'url', args: [], timestamp: 1, screenshot: 'URL_SHOT' }, + { + command: 'waitForElementVisible', + args: ['body'], + timestamp: 2, + screenshot: 'BODY_SHOT' + } + ] + const { + capturer, + sent, + replaced, + commandsLog, + captureAssertCommand, + sendReplaceCommand, + takeScreenshotViaHttp + } = makeFakeCapturer(precedingCommands) + const calls = [ + call('verify', 'titleContains', ['Example'], 'spec.js:11'), + call('verify', 'titleContains', ['SOFT_FAIL_ME'], 'spec.js:12'), + call('assert', 'titleContains', ['Example'], 'spec.js:15'), + call('assert', 'titleContains', ['HARD_FAIL_ME'], 'spec.js:16') + ] + + // 1) CALL TIME: each call streams one neutral pending row. + emitPending(capturer, calls, 'test-uid') + expect(captureAssertCommand).toHaveBeenCalledTimes(4) + expect(sent).toHaveLength(4) + for (const row of sent) { + expect(row.result).toBeUndefined() // neutral — not green + expect(row.error).toBeUndefined() // neutral — not red + expect(row.testUid).toBe('test-uid') + expect(row.screenshot).toBe('BODY_SHOT') // reused preceding DOM + expect(typeof (row as { id?: number }).id).toBe('number') + } + expect(sent[0].title).toBe("verify.titleContains('Example')") + expect(sent[0].command).toBe('verify.titleContains') + expect(sent[0].args).toEqual(['Example']) + expect(sent[0].callSource).toBe('spec.js:11') + expect(sent[0].timestamp).toBe(sent[0].startTime) + const pendingIds = sent.map((r) => (r as { id?: number }).id) + + // 2) TEST END: results.assertions includes the implicit waitForElementVisible + // assertion (first entry) which must NOT create/finalize a row. + const assertions = [ + passing('Element <body> was visible after 16 milliseconds'), + passing("Testing if the page title contains 'Example'"), + failing("Testing if the page title contains 'SOFT_FAIL_ME'"), + passing("Testing if the page title contains 'Example'"), + failing("Testing if the page title contains 'HARD_FAIL_ME'") + ] + await captureNativeAssertions( + capturer, + browser, + currentTestWith(assertions), + 'test-uid', + calls + ) + + // Each row updated exactly once, in place — no new rows, no duplicates. + expect(captureAssertCommand).toHaveBeenCalledTimes(4) + expect(sendReplaceCommand).toHaveBeenCalledTimes(4) + expect(commandsLog).toHaveLength(2 + 4) + expect(takeScreenshotViaHttp).not.toHaveBeenCalled() + + // Same row objects, same stable ids (updated, not recreated). + const finalized = calls.map((c) => c.entry!) + expect(finalized.map((r) => (r as { id?: number }).id)).toEqual(pendingIds) + + // Pass/fail applied; failures carry the verbose message as error. + expect(finalized[0].result).toBe('passed') + expect(finalized[0].error).toBeUndefined() + expect(finalized[1].result).toBeUndefined() + expect((finalized[1].error as { message: string }).message).toContain( + 'SOFT_FAIL_ME' + ) + expect(finalized[2].result).toBe('passed') // duplicate 'Example' → 2nd entry + expect(finalized[3].error).toBeDefined() + + // Labels/args/callSource preserved from the pending row. + expect(finalized[3].title).toBe("assert.titleContains('HARD_FAIL_ME')") + expect(finalized[3].callSource).toBe('spec.js:16') + + // With no results.commands timing, rows keep their enqueue timestamp, so + // the replace is keyed on that same timestamp. + replaced.forEach(({ oldTimestamp, command }) => { + expect(oldTimestamp).toBe(command.timestamp) + }) + }) + + it('repositions each row on its real execution window from results.commands (not enqueue time)', async () => { + const { capturer, sent, replaced } = makeFakeCapturer() + const calls = [ + call('verify', 'titleContains', ['Example']), + call('assert', 'titleContains', ['SOFT_FAIL_ME']) + ] + emitPending(capturer, calls, 'uid') + // Both enqueued ~together (clock++), clustered. + const enqueue = sent.map((r) => r.timestamp) + + // Nightwatch ran them ~29ms apart; results.commands carries the real window. + const assertions = [ + passing("Testing if the page title contains 'Example'"), + failing("Testing if the page title contains 'SOFT_FAIL_ME'") + ] + const commands = [ + { name: 'url', startTime: 5000, endTime: 5100 }, + { name: 'verify.titleContains', startTime: 6000, endTime: 6029 }, + { name: 'assert.titleContains', startTime: 6100, endTime: 6132 } + ] + await captureNativeAssertions( + capturer, + browser, + currentTestWith(assertions, commands), + 'uid', + calls + ) + + const finalized = calls.map((c) => c.entry!) + // startTime/timestamp now reflect the real execution window, spread apart. + expect(finalized[0].startTime).toBe(6000) + expect(finalized[0].timestamp).toBe(6029) + expect(finalized[1].startTime).toBe(6100) + expect(finalized[1].timestamp).toBe(6132) + expect(finalized[1].timestamp - finalized[0].timestamp).toBeGreaterThan(50) + // The replace is keyed on the ORIGINAL enqueue timestamp (stable id also + // matches), while the row now carries its real execution timestamp. + expect(replaced[0].oldTimestamp).toBe(enqueue[0]) + expect(replaced[0].command.timestamp).toBe(6029) + }) + + it('finalizes with a fresh screenshot when no preceding one has resolved yet', async () => { + // Race: preceding capture is fire-and-forget and unresolved → pending rows + // stream without a screenshot; finalize takes a fresh end-of-test one. + const pending: CommandLog[] = [ + { command: 'waitForElementVisible', args: ['body'], timestamp: 2 } + ] + const { capturer, sent, takeScreenshotViaHttp } = makeFakeCapturer( + pending, + 'FRESH_SHOT' + ) + const calls = [call('verify', 'titleContains', ['Example'])] + emitPending(capturer, calls, 'uid') + expect(sent[0].screenshot).toBeUndefined() + + await captureNativeAssertions( + capturer, + browser, + currentTestWith([ + passing("Testing if the page title contains 'Example'") + ]), + 'uid', + calls + ) + expect(takeScreenshotViaHttp).toHaveBeenCalledTimes(1) + expect(calls[0].entry!.screenshot).toBe('FRESH_SHOT') + expect(calls[0].entry!.result).toBe('passed') + }) + + it('preserves real multi-arg assertions (no faked args)', async () => { + const { capturer, sent, replaced } = makeFakeCapturer() + const calls = [call('assert', 'containsText', ['#btn', 'Save'])] + emitPending(capturer, calls, 'uid') + expect(sent[0].args).toEqual(['#btn', 'Save']) + expect(sent[0].title).toBe("assert.containsText('#btn', 'Save')") + + await captureNativeAssertions( + capturer, + browser, + currentTestWith([ + failing("Testing if element <#btn> contains text 'Save'") + ]), + 'uid', + calls + ) + expect(replaced).toHaveLength(1) + expect(calls[0].entry!.error).toBeDefined() + }) + + it('falls back to positional correlation when args are not literals', async () => { + const { capturer, sent } = makeFakeCapturer() + // Element-object arg won't substring-match; positional order still pairs + // the single call with the single explicit assertion outcome. + const calls = [call('assert', 'elementPresent', [{ selector: 'body' }])] + emitPending(capturer, calls, 'uid') + expect(sent[0].title).toBe('assert.elementPresent(…)') + + await captureNativeAssertions( + capturer, + browser, + currentTestWith([failing('Testing if element <body> is present')]), + 'uid', + calls + ) + expect(calls[0].entry!.error).toBeDefined() + }) + + it('leaves an uncorrelated pending row in its neutral state (defensive)', async () => { + const { capturer, sent, sendReplaceCommand } = makeFakeCapturer() + const calls = [call('assert', 'ok', [true])] + emitPending(capturer, calls, 'uid') + // No matching results entry — row must not be dropped or mis-coloured. + await captureNativeAssertions( + capturer, + browser, + currentTestWith([]), + 'uid', + calls + ) + expect(sent).toHaveLength(1) + expect(sendReplaceCommand).not.toHaveBeenCalled() + expect(calls[0].entry!.result).toBeUndefined() + expect(calls[0].entry!.error).toBeUndefined() + }) + + it('is a no-op when there are no recorded calls (implicit assertions ignored)', async () => { + const { capturer, sendReplaceCommand, takeScreenshotViaHttp } = + makeFakeCapturer() + await captureNativeAssertions( + capturer, + browser, + currentTestWith([passing('Element <body> was visible')]), + 'uid', + [] + ) + expect(sendReplaceCommand).not.toHaveBeenCalled() + expect(takeScreenshotViaHttp).not.toHaveBeenCalled() + }) +}) From 6e0377ff71e303d6035224dffb6ead1cf3cab8bb Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:24:34 +0530 Subject: [PATCH 34/91] fix(core): rate-limit repeated terminal lines and fix failLastAction --- examples/wdio/cucumber/wdio.conf.ts | 2 +- examples/wdio/cucumber/wdio.mobile.conf.ts | 2 +- examples/wdio/mocha/wdio.conf.ts | 2 +- packages/core/src/index.ts | 1 + packages/core/src/session-capturer.ts | 26 +++++-- packages/core/src/terminal-throttle.ts | 70 ++++++++++++++++++ .../core/tests/session-capturer-base.test.ts | 46 ++++++++++++ packages/core/tests/terminal-throttle.test.ts | 74 +++++++++++++++++++ 8 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 packages/core/src/terminal-throttle.ts create mode 100644 packages/core/tests/terminal-throttle.test.ts diff --git a/examples/wdio/cucumber/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts index 68f7ee8b..4c535acc 100644 --- a/examples/wdio/cucumber/wdio.conf.ts +++ b/examples/wdio/cucumber/wdio.conf.ts @@ -89,7 +89,7 @@ export const config: Options.Testrunner = { // Define all options that are relevant for the WebdriverIO instance here // // Level of logging verbosity: trace | debug | info | warn | error | silent - logLevel: 'debug', + logLevel: 'warn', // // Set specific log levels per logger // loggers: diff --git a/examples/wdio/cucumber/wdio.mobile.conf.ts b/examples/wdio/cucumber/wdio.mobile.conf.ts index 7f693ab4..13618ffe 100644 --- a/examples/wdio/cucumber/wdio.mobile.conf.ts +++ b/examples/wdio/cucumber/wdio.mobile.conf.ts @@ -44,7 +44,7 @@ export const config: Options.Testrunner = { // eslint-disable-next-line @typescript-eslint/no-explicit-any ] as any, - logLevel: 'info', + logLevel: 'warn', bail: 0, baseUrl: 'http://localhost', waitforTimeout: 15000, diff --git a/examples/wdio/mocha/wdio.conf.ts b/examples/wdio/mocha/wdio.conf.ts index 7ccee43f..966226f9 100644 --- a/examples/wdio/mocha/wdio.conf.ts +++ b/examples/wdio/mocha/wdio.conf.ts @@ -29,7 +29,7 @@ export const config: Options.Testrunner = { } } ], - logLevel: 'debug', + logLevel: 'warn', bail: 0, baseUrl: 'http://localhost', waitforTimeout: 10000, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 3c3325e2..89c6c301 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -25,6 +25,7 @@ export * from './console.js' export * from './uid.js' export * from './net.js' export * from './stack.js' +export * from './terminal-throttle.js' export * from './error.js' export * from './finalize-screencast.js' export * from './output-dir.js' diff --git a/packages/core/src/session-capturer.ts b/packages/core/src/session-capturer.ts index 82cfe0f6..f5b90f4e 100644 --- a/packages/core/src/session-capturer.ts +++ b/packages/core/src/session-capturer.ts @@ -21,6 +21,7 @@ import { isInternalStreamLine, stripAnsi } from './console.js' +import { TerminalLineThrottle } from './terminal-throttle.js' /** * Foundation class for adapter SessionCapturers. Owns the cross-framework @@ -62,6 +63,11 @@ export abstract class SessionCapturerBase { // stdout, OR when stream forwarding wants to log via console. #isCapturingConsole = false #isCapturingStream = false + // Collapses high-frequency identical terminal lines (e.g. WDIO's per-command + // COMMAND/RESULT logger frames reprinted every ~100ms during an expect poll) + // so they don't flood the console lane. Terminal-source only — user + // console.* is source='test' and never throttled. + #terminalThrottle = new TerminalLineThrottle() // Command bookkeeping — used by adapters that emit commands themselves // (nightwatch, selenium). The WDIO service adapter doesn't call sendCommand @@ -209,20 +215,27 @@ export abstract class SessionCapturerBase { } /** - * Mark the most recent user action of a test as failed. Assertion and step - * failures aren't their own command, so the action that ran (e.g. the query - * an expectation read) carries the error; broadcasts the swap so live mode - * highlights it too. Returns false when no eligible action is found. + * Mark the most recent user action of a test as failed — for framework + * failures that aren't captured as their own command. Broadcasts the swap so + * live mode highlights it too. Returns false when no eligible action is found, + * OR when that most-recent action already carries an error: the failure is + * then already represented (e.g. an expect matcher captured as its own row via + * afterAssertion), so it must not bleed onto an earlier *passing* action. */ failLastAction(testUid: string | undefined, error: SerializedError): boolean { for (let i = this.commandsLog.length - 1; i >= 0; i--) { const command = this.commandsLog[i] - if (command.error || !mapCommandToAction(command.command)) { + if (!mapCommandToAction(command.command)) { continue } if (testUid && command.testUid !== testUid) { continue } + // First (most-recent) action for this test. If it's already failed, the + // failure is captured — stop rather than marking an earlier passing one. + if (command.error) { + return false + } command.error = error this.sendReplaceCommand(command.timestamp, command) return true @@ -432,7 +445,8 @@ export abstract class SessionCapturerBase { if ( !clean || this.isInternalStreamLine(clean) || - SPINNER_RE.test(clean) + SPINNER_RE.test(clean) || + !this.#terminalThrottle.shouldEmit(clean) ) { continue } diff --git a/packages/core/src/terminal-throttle.ts b/packages/core/src/terminal-throttle.ts new file mode 100644 index 00000000..017cbf00 --- /dev/null +++ b/packages/core/src/terminal-throttle.ts @@ -0,0 +1,70 @@ +/** + * Rate-limits high-frequency identical terminal (stdout/stderr) lines so a + * polling framework that reprints the same line every ~100ms doesn't flood the + * trace's console lane. The motivating case: WDIO's logger writes a + * `COMMAND`/`RESULT` frame for every WebDriver command, so an `expect` that + * polls for its full 10s timeout emits hundreds of near-identical lines. + * + * Only terminal-source capture goes through this — user `console.*` is + * captured as `source: 'test'` and never throttled, so real user output is + * untouched. Distinct lines always pass immediately; a repeat is emitted at + * most once per window. + */ + +/** + * A repeated identical terminal line is emitted at most once per this window + * (ms). Sized so a 100ms poll collapses ~10:1 while a human-paced reprint of + * the same line (>1s apart) still shows every time. + */ +export const TERMINAL_REPEAT_WINDOW_MS = 1000 + +/** + * Leading ISO-8601 timestamp emitted by most structured loggers (`@wdio/logger`'s + * `%t %l %n:` template, pino, winston, …). It's the only volatile part of + * otherwise-identical successive log frames, so it's stripped from the throttle + * key — never from the emitted text. + */ +const LEADING_ISO_TIMESTAMP_RE = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d\d\d)?Z {1,4}/ + +/** Key a terminal line for repeat-detection: drop a leading ISO timestamp so + * successive log frames of the same message collapse. */ +export function terminalRepeatKey(line: string): string { + return line.replace(LEADING_ISO_TIMESTAMP_RE, '') +} + +export class TerminalLineThrottle { + #lastEmitted = new Map<string, number>() + readonly #windowMs: number + + constructor(windowMs: number = TERMINAL_REPEAT_WINDOW_MS) { + this.#windowMs = windowMs + } + + /** + * True if the line should be emitted, false if it's a within-window repeat of + * a line already emitted. The window is anchored to the last *emit*, not the + * last occurrence, so a sustained stream of one line still emits once per + * window instead of going silent after the first. Expired keys are pruned on + * each emit, so state stays bounded by the number of distinct lines seen + * within the window. + */ + shouldEmit(line: string, now: number = Date.now()): boolean { + const key = terminalRepeatKey(line) + const last = this.#lastEmitted.get(key) + if (last !== undefined && now - last < this.#windowMs) { + return false + } + this.#lastEmitted.set(key, now) + this.#prune(now) + return true + } + + #prune(now: number): void { + for (const [key, ts] of this.#lastEmitted) { + if (now - ts >= this.#windowMs) { + this.#lastEmitted.delete(key) + } + } + } +} diff --git a/packages/core/tests/session-capturer-base.test.ts b/packages/core/tests/session-capturer-base.test.ts index d89455ae..92ee7722 100644 --- a/packages/core/tests/session-capturer-base.test.ts +++ b/packages/core/tests/session-capturer-base.test.ts @@ -152,3 +152,49 @@ describe('sendCommand', () => { expect(cap.upstream.filter((u) => u.scope === 'commands')).toHaveLength(1) }) }) + +describe('failLastAction', () => { + const boom = { name: 'Error', message: 'boom' } + + it('marks the most-recent action of the test when it has no error', () => { + cap.commandsLog.push({ + command: 'expect.toBeExisting', + args: [], + timestamp: 1, + testUid: 't1' + }) + expect(cap.failLastAction('t1', boom)).toBe(true) + expect(cap.commandsLog[0]!.error).toEqual(boom) + }) + + it('does not bleed onto an earlier passing action when the latest one already failed', () => { + // Regression: a failing expect matcher is captured as its own row via + // afterAssertion, so failLastAction must stop at it — NOT stamp the error + // onto the preceding passing assertion (the toBeExisting-shown-red bug). + const matcherErr = { name: 'Error', message: 'to have text' } + cap.commandsLog.push( + { command: 'expect.toBeExisting', args: [], timestamp: 1, testUid: 't1' }, + { + command: 'expect.toHaveText', + args: [], + timestamp: 2, + testUid: 't1', + error: matcherErr + } + ) + expect(cap.failLastAction('t1', boom)).toBe(false) + expect(cap.commandsLog[0]!.error).toBeUndefined() + expect(cap.commandsLog[1]!.error).toEqual(matcherErr) + }) + + it('ignores actions belonging to a different test', () => { + cap.commandsLog.push({ + command: 'expect.toBeExisting', + args: [], + timestamp: 1, + testUid: 'other' + }) + expect(cap.failLastAction('t1', boom)).toBe(false) + expect(cap.commandsLog[0]!.error).toBeUndefined() + }) +}) diff --git a/packages/core/tests/terminal-throttle.test.ts b/packages/core/tests/terminal-throttle.test.ts new file mode 100644 index 00000000..4e26f11b --- /dev/null +++ b/packages/core/tests/terminal-throttle.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from 'vitest' +import { + TERMINAL_REPEAT_WINDOW_MS, + TerminalLineThrottle, + terminalRepeatKey +} from '../src/terminal-throttle.js' + +describe('terminalRepeatKey', () => { + it('strips a leading ISO-8601 timestamp so log frames of the same message match', () => { + const a = '2026-07-09T22:12:38.497Z INFO webdriver: COMMAND getText("#x")' + const b = '2026-07-09T22:12:38.597Z INFO webdriver: COMMAND getText("#x")' + expect(terminalRepeatKey(a)).toBe(terminalRepeatKey(b)) + expect(terminalRepeatKey(a)).toBe('INFO webdriver: COMMAND getText("#x")') + }) + + it('leaves lines without a leading timestamp untouched', () => { + expect(terminalRepeatKey('[TEST] hello')).toBe('[TEST] hello') + }) +}) + +describe('TerminalLineThrottle', () => { + it('emits distinct lines immediately', () => { + const t = new TerminalLineThrottle() + expect(t.shouldEmit('one', 0)).toBe(true) + expect(t.shouldEmit('two', 0)).toBe(true) + expect(t.shouldEmit('three', 0)).toBe(true) + }) + + it('suppresses a within-window repeat but re-emits once the window passes', () => { + const t = new TerminalLineThrottle(1000) + expect(t.shouldEmit('poll', 0)).toBe(true) // first + expect(t.shouldEmit('poll', 100)).toBe(false) // repeat within window + expect(t.shouldEmit('poll', 900)).toBe(false) + expect(t.shouldEmit('poll', 1000)).toBe(true) // window elapsed → re-emit + expect(t.shouldEmit('poll', 1100)).toBe(false) + }) + + it('anchors the window to the last emit, not the last occurrence (a sustained stream still emits ~1/window)', () => { + const t = new TerminalLineThrottle(1000) + let emitted = 0 + // 100ms poll for 10s = 101 occurrences of the same line. + for (let ms = 0; ms <= 10000; ms += 100) { + if (t.shouldEmit('COMMAND getText', ms)) { + emitted++ + } + } + // ~1 per second rather than ~100 total — flood collapsed ~10:1. + expect(emitted).toBe(11) + }) + + it('collapses successive WDIO logger frames of the same command despite changing timestamps', () => { + const t = new TerminalLineThrottle(1000) + const line = (ms: number) => + `2026-07-09T22:12:${String(30 + Math.floor(ms / 1000)).padStart(2, '0')}.${String(ms % 1000).padStart(3, '0')}Z INFO webdriver: COMMAND getText("#x")` + expect(t.shouldEmit(line(0), 0)).toBe(true) + expect(t.shouldEmit(line(100), 100)).toBe(false) + expect(t.shouldEmit(line(200), 200)).toBe(false) + }) + + it('treats different messages independently', () => { + const t = new TerminalLineThrottle(1000) + expect(t.shouldEmit('COMMAND getText', 0)).toBe(true) + expect(t.shouldEmit('RESULT ""', 10)).toBe(true) // interleaved, distinct + expect(t.shouldEmit('COMMAND getText', 100)).toBe(false) // repeat + expect(t.shouldEmit('RESULT ""', 110)).toBe(false) // repeat + }) + + it('defaults to the exported window constant', () => { + const t = new TerminalLineThrottle() + expect(t.shouldEmit('x', 0)).toBe(true) + expect(t.shouldEmit('x', TERMINAL_REPEAT_WINDOW_MS - 1)).toBe(false) + expect(t.shouldEmit('x', TERMINAL_REPEAT_WINDOW_MS)).toBe(true) + }) +}) From 65412406c56bc4ad64461b66507531670ddf4c44 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:24:50 +0530 Subject: [PATCH 35/91] fix(app): real per-action duration and span-based active highlight --- .../src/components/browser/trace-timeline.ts | 20 ++-- .../workbench/actionItems/duration.ts | 20 ++++ .../app/src/components/workbench/actions.ts | 18 +--- .../src/components/workbench/active-entry.ts | 59 ++++++++--- packages/app/tests/active-entry.test.ts | 97 +++++++++++++++---- packages/app/tests/duration.test.ts | 33 +++++++ 6 files changed, 191 insertions(+), 56 deletions(-) diff --git a/packages/app/src/components/browser/trace-timeline.ts b/packages/app/src/components/browser/trace-timeline.ts index 21f7ffce..07cb81a1 100644 --- a/packages/app/src/components/browser/trace-timeline.ts +++ b/packages/app/src/components/browser/trace-timeline.ts @@ -5,7 +5,7 @@ import { consume } from '@lit/context' import type { CommandLog, TracePlayerFrame } from '@wdio/devtools-shared' import { commandContext, framesContext } from '../../controller/context.js' -import { activeTimestampAt } from '../workbench/active-entry.js' +import { activeSpanAt } from '../workbench/active-entry.js' import { KBD } from '../../controller/keyboard.js' import { PLAYER_RESTART_EVENT, @@ -42,7 +42,7 @@ export class TraceTimeline extends Element { #rafId?: number #rafLast = 0 - #activeTimestamp?: number + #activeCommand?: CommandLog #started = false @query('[data-scrub]') scrubEl?: HTMLElement @@ -229,19 +229,15 @@ export class TraceTimeline extends Element { return } const clock = this.#start + this.currentMs - const timestamps = sorted.map((c) => c.timestamp ?? 0) - const activeTs = activeTimestampAt(timestamps, clock) ?? timestamps[0] - if (this.#started && activeTs === this.#activeTimestamp) { + const command = activeSpanAt(sorted, clock) ?? sorted[0] + if (this.#started && command === this.#activeCommand) { return } this.#started = true - this.#activeTimestamp = activeTs - const command = sorted.find((c) => (c.timestamp ?? 0) === activeTs) - if (command) { - window.dispatchEvent( - new CustomEvent('show-command', { detail: { command } }) - ) - } + this.#activeCommand = command + window.dispatchEvent( + new CustomEvent('show-command', { detail: { command } }) + ) } // ─── scrubbing (drag anywhere on the strip) ─────────────────────────────── diff --git a/packages/app/src/components/workbench/actionItems/duration.ts b/packages/app/src/components/workbench/actionItems/duration.ts index 5d2df61f..f0dd5d63 100644 --- a/packages/app/src/components/workbench/actionItems/duration.ts +++ b/packages/app/src/components/workbench/actionItems/duration.ts @@ -1,3 +1,5 @@ +import type { CommandLog, TraceMutation } from '@wdio/devtools-shared' + export type DurationHeat = 'fast' | 'mid' | 'slow' const ONE_SECOND = 1000 @@ -29,6 +31,24 @@ export function durationHeat(ms: number): DurationHeat { return 'fast' } +/** + * True per-action duration: a command's own execution span (`timestamp − + * startTime`) when it has one, else the inter-action `gapFallback`. Prefer the + * span — the gap over-counts idle time before an action, so e.g. an assertion + * whose internal polling commands are suppressed would otherwise report the + * navigation gap that preceded it rather than its own runtime. Used by both the + * flat (live) and grouped (trace-player) action views so they agree. + */ +export function entryDuration( + entry: CommandLog | TraceMutation, + gapFallback: number | undefined +): number | undefined { + if ('command' in entry && entry.startTime !== undefined) { + return entry.timestamp - entry.startTime + } + return gapFallback +} + /** * Per-step duration for each timeline entry: the gap to the next entry. The * final entry has no next, so it falls back to the gap from the previous entry diff --git a/packages/app/src/components/workbench/actions.ts b/packages/app/src/components/workbench/actions.ts index 842f4f56..5e630c8e 100644 --- a/packages/app/src/components/workbench/actions.ts +++ b/packages/app/src/components/workbench/actions.ts @@ -18,8 +18,8 @@ import '../placeholder.js' import './actionItems/command.js' import './actionItems/group.js' import './actionItems/mutation.js' -import { stepDurations } from './actionItems/duration.js' -import { activeTimestampAt } from './active-entry.js' +import { entryDuration, stepDurations } from './actionItems/duration.js' +import { activeSpanAt } from './active-entry.js' import { defaultExpanded, flattenActionTree, @@ -128,12 +128,7 @@ export class DevtoolsActions extends Element { // every timeupdate tick. #onScreencastProgress = (event: Event) => { const { time } = (event as CustomEvent<{ time: number }>).detail - const entries = this.#sortedEntries() - const timestamp = activeTimestampAt( - entries.map((entry) => entry.timestamp), - time - ) - const active = entries.find((entry) => entry.timestamp === timestamp) + const active = activeSpanAt(this.#sortedEntries(), time) if (active === this.activeEntry) { return } @@ -228,10 +223,7 @@ export class DevtoolsActions extends Element { return nothing } // Reconstructed zips carry the real invocation span; gap is the fallback. - const duration = - entry.startTime !== undefined - ? entry.timestamp - entry.startTime - : gaps[row.commandIndex] + const duration = entryDuration(entry, gaps[row.commandIndex]) return html` <wdio-devtools-command-item style=${indent} @@ -257,7 +249,7 @@ export class DevtoolsActions extends Element { const rows = entries.map((entry, index) => { const elapsedTime = entry.timestamp - baselineTimestamp - const duration = durations[index] + const duration = entryDuration(entry, durations[index]) const active = entry === this.activeEntry if ('command' in entry) { diff --git a/packages/app/src/components/workbench/active-entry.ts b/packages/app/src/components/workbench/active-entry.ts index 3cdd2ed2..edbe3eb9 100644 --- a/packages/app/src/components/workbench/active-entry.ts +++ b/packages/app/src/components/workbench/active-entry.ts @@ -1,20 +1,51 @@ +/** Timeline item occupying a span from `startTime` (inclusive) to `timestamp` + * (inclusive). Items without a `startTime` occupy the single point `timestamp`. */ +export interface TimeSpanned { + timestamp: number + startTime?: number +} + /** - * Given action timestamps sorted ascending and a playback time (wall-clock ms), - * return the timestamp of the action that is "current" — the latest one at or - * before `time`. Returns undefined when `time` precedes the first action, so - * nothing is highlighted before the first command runs. + * Given timeline items and a playback time (wall-clock ms), return the item + * that is "current" at `time`. An item's span is `[startTime ?? timestamp, + * timestamp]` and the item is current when `time` falls inside it — so a + * long-running command (e.g. a polling `expect.*` matcher) stays selected for + * its whole duration, not just at completion. When several spans contain `time` + * (nested/overlapping commands), the one that started most recently wins, with + * ties broken toward the tighter span — the most specific action in progress. + * When none contain `time` (a point-like command, or a gap between actions), + * fall back to the latest item that has already ended at or before `time`. + * Returns undefined when `time` precedes every item, so nothing is highlighted + * before the first action runs. */ -export function activeTimestampAt( - sortedTimestamps: number[], +export function activeSpanAt<T extends TimeSpanned>( + items: readonly T[], time: number -): number | undefined { - let active: number | undefined - for (const ts of sortedTimestamps) { - if (ts <= time) { - active = ts - } else { - break +): T | undefined { + let containing: T | undefined + let containingStart = -Infinity + let containingEnd = Infinity + let ended: T | undefined + let endedAt = -Infinity + + for (const item of items) { + const end = item.timestamp + const start = item.startTime ?? end + if ( + start <= time && + time <= end && + (start > containingStart || + (start === containingStart && end < containingEnd)) + ) { + containing = item + containingStart = start + containingEnd = end + } + if (end <= time && end >= endedAt) { + ended = item + endedAt = end } } - return active + + return containing ?? ended } diff --git a/packages/app/tests/active-entry.test.ts b/packages/app/tests/active-entry.test.ts index 9dcf416f..c894f456 100644 --- a/packages/app/tests/active-entry.test.ts +++ b/packages/app/tests/active-entry.test.ts @@ -1,28 +1,91 @@ import { describe, it, expect } from 'vitest' -import { activeTimestampAt } from '../src/components/workbench/active-entry.js' +import { + activeSpanAt, + type TimeSpanned +} from '../src/components/workbench/active-entry.js' -describe('activeTimestampAt', () => { - const stamps = [100, 200, 300, 400] +describe('activeSpanAt', () => { + // Point-like commands (no startTime) — the pre-span behaviour must be intact. + describe('point-like items (no startTime)', () => { + const points: TimeSpanned[] = [ + { timestamp: 100 }, + { timestamp: 200 }, + { timestamp: 300 }, + { timestamp: 400 } + ] - it('returns undefined before the first action', () => { - expect(activeTimestampAt(stamps, 50)).toBeUndefined() - }) + it('returns undefined before the first item', () => { + expect(activeSpanAt(points, 50)).toBeUndefined() + }) - it('returns the action exactly at the playback time', () => { - expect(activeTimestampAt(stamps, 200)).toBe(200) - }) + it('returns the item exactly at the playback time', () => { + expect(activeSpanAt(points, 200)).toBe(points[1]) + }) - it('returns the latest action at or before the playback time', () => { - expect(activeTimestampAt(stamps, 250)).toBe(200) - expect(activeTimestampAt(stamps, 399)).toBe(300) - }) + it('returns the latest item at or before the playback time', () => { + expect(activeSpanAt(points, 250)).toBe(points[1]) + expect(activeSpanAt(points, 399)).toBe(points[2]) + }) - it('clamps to the last action once playback passes it', () => { - expect(activeTimestampAt(stamps, 999)).toBe(400) + it('clamps to the last item once playback passes it', () => { + expect(activeSpanAt(points, 999)).toBe(points[3]) + }) + + it('returns undefined for an empty timeline', () => { + expect(activeSpanAt([], 100)).toBeUndefined() + }) }) - it('returns undefined for an empty timeline', () => { - expect(activeTimestampAt([], 100)).toBeUndefined() + describe('spanned items (startTime → timestamp)', () => { + it('selects a long-running command while the clock is inside its span', () => { + const before = { timestamp: 1000 } + const poll = { startTime: 1000, timestamp: 11000 } + const items: TimeSpanned[] = [before, poll] + // Mid-poll: the span contains the clock, so the poll stays selected even + // though the preceding command ended earlier. + expect(activeSpanAt(items, 5000)).toBe(poll) + expect(activeSpanAt(items, 1500)).toBe(poll) + // At the span end the poll is still selected. + expect(activeSpanAt(items, 11000)).toBe(poll) + }) + + it('picks the most recently started span when several contain the clock', () => { + const outer = { startTime: 100, timestamp: 10000 } + const inner = { startTime: 4000, timestamp: 6000 } + const items: TimeSpanned[] = [outer, inner] + expect(activeSpanAt(items, 5000)).toBe(inner) + // Outside the inner span but still inside the outer one → outer. + expect(activeSpanAt(items, 2000)).toBe(outer) + expect(activeSpanAt(items, 8000)).toBe(outer) + }) + + it('breaks a start-time tie toward the tighter span', () => { + const wide = { startTime: 100, timestamp: 9000 } + const tight = { startTime: 100, timestamp: 3000 } + const items: TimeSpanned[] = [wide, tight] + expect(activeSpanAt(items, 2000)).toBe(tight) + }) + + it('falls back to the last-ended item when no span contains the clock', () => { + const first = { startTime: 100, timestamp: 2000 } + const second = { startTime: 5000, timestamp: 7000 } + const items: TimeSpanned[] = [first, second] + // Clock sits in the gap between the two spans → the last one that ended. + expect(activeSpanAt(items, 3500)).toBe(first) + }) + + it('returns undefined when the clock precedes the first span', () => { + const items: TimeSpanned[] = [{ startTime: 1000, timestamp: 5000 }] + expect(activeSpanAt(items, 500)).toBeUndefined() + }) + + it('prefers a containing span over an already-ended earlier item', () => { + const ended = { timestamp: 1000 } + const running = { startTime: 900, timestamp: 8000 } + const items: TimeSpanned[] = [ended, running] + // Clock is past the point command but inside the running span → running. + expect(activeSpanAt(items, 2000)).toBe(running) + }) }) }) diff --git a/packages/app/tests/duration.test.ts b/packages/app/tests/duration.test.ts index 0bfc90ba..92ab1c87 100644 --- a/packages/app/tests/duration.test.ts +++ b/packages/app/tests/duration.test.ts @@ -3,8 +3,10 @@ import { describe, it, expect } from 'vitest' import { formatDuration, durationHeat, + entryDuration, stepDurations } from '../src/components/workbench/actionItems/duration.js' +import type { CommandLog, TraceMutation } from '@wdio/devtools-shared' describe('formatDuration', () => { it('shows milliseconds under a second', () => { @@ -73,3 +75,34 @@ describe('stepDurations', () => { expect(stepDurations([0, 0, 0, 500])).toEqual([0, 0, 500, 500]) }) }) + +describe('entryDuration', () => { + const cmd = (over: Partial<CommandLog> = {}): CommandLog => ({ + command: 'expect.toExist', + args: [], + timestamp: 10603, + ...over + }) + + it('uses the command execution span (timestamp − startTime) when present', () => { + // Regression: an assertion whose internal polling is suppressed sits after a + // long navigation gap; the row must show its own 603ms runtime, not the gap. + expect(entryDuration(cmd({ startTime: 10000 }), 10650)).toBe(603) + }) + + it('falls back to the inter-action gap when the command has no startTime', () => { + expect(entryDuration(cmd(), 420)).toBe(420) + }) + + it('falls back to the gap for a mutation entry (no startTime field)', () => { + const mutation = { + type: 'childList', + timestamp: 5 + } as unknown as TraceMutation + expect(entryDuration(mutation, 42)).toBe(42) + }) + + it('returns undefined when neither a span nor a gap is available', () => { + expect(entryDuration(cmd(), undefined)).toBeUndefined() + }) +}) From 8f5cb9fa9a97255874ecf28ade44d7623c60ad42 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:25:14 +0530 Subject: [PATCH 36/91] fix(app): sidebar icon alignment and resizable-pane drag --- packages/app/src/components/sidebar/test-suite.ts | 11 +++++++---- packages/app/src/utils/DragController.ts | 11 +++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/app/src/components/sidebar/test-suite.ts b/packages/app/src/components/sidebar/test-suite.ts index cf0d43f3..9cc7e0ee 100644 --- a/packages/app/src/components/sidebar/test-suite.ts +++ b/packages/app/src/components/sidebar/test-suite.ts @@ -287,10 +287,11 @@ export class ExplorerTestEntry extends CollapseableEntry { return this.state === TestState.RUNNING } get testStateIcon() { - // Fixed-height box (= the label's line-height) centred so the icon aligns - // with the first line of the title whether it wraps or not — no margin hacks. + // Vertically centred in the row so the status icon lines up with the + // row's action buttons (run/stop), which are also centre-aligned. const box = (inner: unknown) => - html`<span class="w-4 h-[18px] shrink-0 flex items-center justify-center" + html`<span + class="w-4 h-[18px] shrink-0 self-center grid place-items-center" >${inner}</span >` if (this.isRunning) { @@ -407,7 +408,9 @@ export class ExplorerTestEntry extends CollapseableEntry { class="row flex w-full items-start text-sm group/sidebar rounded-md my-0.5 px-1 py-1 cursor-pointer hover:bg-toolbarHoverBackground" > <button - class="flex-none pointer px-2 h-8 ${hasNoChildren ? 'hidden' : ''}" + class="flex-none pointer px-2 h-[18px] flex items-center justify-center ${hasNoChildren + ? 'hidden' + : ''}" @click="${() => this.#toggleEntry()}" > <icon-mdi-menu-down diff --git a/packages/app/src/utils/DragController.ts b/packages/app/src/utils/DragController.ts index d675570f..e74d8d6e 100644 --- a/packages/app/src/utils/DragController.ts +++ b/packages/app/src/utils/DragController.ts @@ -256,6 +256,17 @@ export class DragController implements ReactiveController { // if slider was removed (collapsed) and re-added, re-init pointer tracker if (this.#draggableEl !== draggableEl) { this.#draggableEl = draggableEl as HTMLElement + // The container often doesn't exist at construction (the sidebar renders + // only after a connection), so it must be resolved here too — otherwise + // the tracker attaches but #handleWindowMove bails on a null container + // and the drag silently no-ops. + if (!this.#containerEl) { + const containerEl = await this.#options.getContainerEl() + if (containerEl) { + this.#containerEl = containerEl as HTMLElement + window.onresize = () => this.#adjustPosition() + } + } this.#init() } } From 0b9609b4b39ddaf9666a3efedebd214248a8e489 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:26:04 +0530 Subject: [PATCH 37/91] chore(examples): assertion + retry harnesses, chromedriver bump --- .../nightwatch/tests/assert-capture-check.js | 18 +++++++ examples/nightwatch/tests/login.js | 47 ------------------- examples/nightwatch/tests/sample.js | 21 --------- .../features/step-definitions/steps.ts | 8 ++-- examples/wdio/mocha/wdio.retry.conf.ts | 4 +- packages/nightwatch-devtools/package.json | 2 +- pnpm-lock.yaml | 30 +++++++++--- 7 files changed, 49 insertions(+), 81 deletions(-) create mode 100644 examples/nightwatch/tests/assert-capture-check.js delete mode 100644 examples/nightwatch/tests/login.js delete mode 100644 examples/nightwatch/tests/sample.js diff --git a/examples/nightwatch/tests/assert-capture-check.js b/examples/nightwatch/tests/assert-capture-check.js new file mode 100644 index 00000000..6f8511a3 --- /dev/null +++ b/examples/nightwatch/tests/assert-capture-check.js @@ -0,0 +1,18 @@ +// Verification harness for native-assert trace capture (browser.assert / verify). +// Run `pnpm demo:nightwatch` and inspect the dashboard Actions: the PASSING +// asserts must render green and the FAILING ones RED. If a failing assert shows +// green, the classic-chained capture is mis-reporting failures (the known risk +// in nativeAssertCapture — the wrapper sees the enqueue, not the queued result). +describe('Native assert capture check', function () { + it('renders passing and failing native asserts', async function (browser) { + await browser.url('https://example.com').waitForElementVisible('body', 5000) + + // Soft verify.* first — never aborts the test, so all four always run. + browser.verify.titleContains('Example') // PASS → expect green + browser.verify.titleContains('SOFT_FAIL_ME') // FAIL → expect RED + + // Hard assert.* — the classic-chained/queued path under test. + browser.assert.titleContains('Example') // PASS → expect green + browser.assert.titleContains('HARD_FAIL_ME') // FAIL → expect RED + }) +}) diff --git a/examples/nightwatch/tests/login.js b/examples/nightwatch/tests/login.js deleted file mode 100644 index d8964d8d..00000000 --- a/examples/nightwatch/tests/login.js +++ /dev/null @@ -1,47 +0,0 @@ -describe('The Internet Guinea Pig Website', function () { - afterEach(async function (browser) { - await browser.end() - }) - - it('should log into the secure area with valid credentials', async function (browser) { - console.log('[TEST] Navigating to login page') - await browser - .url('https://the-internet.herokuapp.com/login') - .waitForElementVisible('body') - - console.log('[TEST] Attempting login with username: tomsmith') - await browser - .setValue('#username', 'tomsmith') - .setValue('#password', 'SuperSecretPassword!') - .click('button[type="submit"]') - - console.log( - '[TEST] Verifying flash message: You logged into a secure area!' - ) - await browser - .waitForElementVisible('#flash', 10000) - .assert.textContains('#flash', 'You logged into a secure area!') - - console.log('[TEST] Flash message verified successfully') - }) - - it('should show error with invalid credentials', async function (browser) { - console.log('[TEST] Navigating to login page') - await browser - .url('https://the-internet.herokuapp.com/login') - .waitForElementVisible('body') - - console.log('[TEST] Attempting login with username: foobar') - await browser - .setValue('#username', 'foobar') - .setValue('#password', 'barfoo') - .click('button[type="submit"]') - - console.log('[TEST] Verifying flash message: Your username is invalid!') - await browser - .waitForElementVisible('#flash', 10000) - .assert.textContains('#flash', 'Your username is invalid!') - - console.log('[TEST] Flash message verified successfully') - }) -}) diff --git a/examples/nightwatch/tests/sample.js b/examples/nightwatch/tests/sample.js deleted file mode 100644 index ff2e4b2a..00000000 --- a/examples/nightwatch/tests/sample.js +++ /dev/null @@ -1,21 +0,0 @@ -describe('Sample Nightwatch Test with DevTools', function () { - it('should navigate to example.com and check title', async function (browser) { - await browser - .url('https://example.com') - .waitForElementVisible('body', 5000) - .assert.titleContains('Example') - .assert.visible('h1') - - const result = await browser.getText('h1') - browser.assert.ok(result.includes('Example'), 'H1 contains "Example"') - }) - - it('should perform basic interactions', async function (browser) { - await browser - .url('https://www.google.com') - .waitForElementVisible('body', 5000) - .assert.visible('textarea[name="q"]') - .setValue('textarea[name="q"]', 'WebdriverIO DevTools') - .pause(1000) - }) -}) diff --git a/examples/wdio/cucumber/features/step-definitions/steps.ts b/examples/wdio/cucumber/features/step-definitions/steps.ts index 3367b631..5d02133e 100644 --- a/examples/wdio/cucumber/features/step-definitions/steps.ts +++ b/examples/wdio/cucumber/features/step-definitions/steps.ts @@ -26,9 +26,9 @@ When(/^I login with (\w+) and (.+)$/, async (username, password) => { Then(/^I should see a flash message saying (.*)$/, async (message) => { console.log(`[TEST] Verifying flash message: ${message}`) - const el = await SecurePage.flashAlert - await expect(el).toBeExisting() - await expect(el).toHaveText(expect.stringContaining(message)) + await expect(SecurePage.flashAlert).toBeExisting() + await expect(SecurePage.flashAlert).toHaveText( + expect.stringContaining(message) + ) console.log('[TEST] Flash message verified successfully') - // await browser.pause(15000) }) diff --git a/examples/wdio/mocha/wdio.retry.conf.ts b/examples/wdio/mocha/wdio.retry.conf.ts index 90b87858..d9c873bb 100644 --- a/examples/wdio/mocha/wdio.retry.conf.ts +++ b/examples/wdio/mocha/wdio.retry.conf.ts @@ -42,8 +42,8 @@ export const config: Options.Testrunner = { 'devtools', { mode: 'trace' as const, - traceGranularity: 'spec' as const, - tracePolicy: 'on-first-retry' as const + traceGranularity: 'test' as const, + tracePolicy: 'retain-on-failure' as const } ] ], diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index f0bd5800..526a6ffd 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -62,7 +62,7 @@ "@types/ws": "^8.18.1", "@wdio/devtools-core": "workspace:^", "@wdio/devtools-shared": "workspace:^", - "chromedriver": "^148.0.4", + "chromedriver": "^150.0.0", "nightwatch": "^3.16.0", "tsup": "^8.5.1", "typescript": "^6.0.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2354ac3..528cbb11 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -109,7 +109,7 @@ importers: version: link:../../packages/nightwatch-devtools nightwatch: specifier: ^3.16.0 - version: 3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@148.0.4) + version: 3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@150.0.1) examples/selenium: dependencies: @@ -422,11 +422,11 @@ importers: specifier: workspace:^ version: link:../shared chromedriver: - specifier: ^148.0.4 - version: 148.0.4 + specifier: ^150.0.0 + version: 150.0.1 nightwatch: specifier: ^3.16.0 - version: 3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@148.0.4) + version: 3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@150.0.1) tsup: specifier: ^8.5.1 version: 8.5.1(@microsoft/api-extractor@7.58.9(@types/node@25.9.3))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@6.0.3)(yaml@2.9.0) @@ -3338,6 +3338,11 @@ packages: engines: {node: '>=22'} hasBin: true + chromedriver@150.0.1: + resolution: {integrity: sha512-vpYskeQFyZGPNMEwc/5fuvfnxzfjljCO1EvwuF0dZGQ8KE0repYQar7eThTlT1IjznwVrirYRgXrRsJts7DRIQ==} + engines: {node: '>=22'} + hasBin: true + chromium-bidi@0.5.8: resolution: {integrity: sha512-blqh+1cEQbHBKmok3rVJkBlBxt9beKBgOsxbFgs7UJcoVbbeZ+K7+6liAsjgpc8l1Xd55cQUy14fXZdGSb4zIw==} peerDependencies: @@ -10656,6 +10661,19 @@ snapshots: - debug - supports-color + chromedriver@150.0.1: + dependencies: + '@testim/chrome-version': 1.1.4 + adm-zip: 0.5.17 + axios: 1.16.1 + compare-versions: 6.1.1 + proxy-agent: 8.0.1 + proxy-from-env: 2.1.0 + tcp-port-used: 1.0.2 + transitivePeerDependencies: + - debug + - supports-color + chromium-bidi@0.5.8(devtools-protocol@0.0.1232444): dependencies: devtools-protocol: 0.0.1232444 @@ -13280,7 +13298,7 @@ snapshots: dependencies: axe-core: 4.12.0 - nightwatch@3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@148.0.4): + nightwatch@3.16.0(@cucumber/cucumber@13.0.0)(chromedriver@150.0.1): dependencies: '@nightwatch/chai': 5.0.3 '@nightwatch/html-reporter-template': 0.3.0 @@ -13318,7 +13336,7 @@ snapshots: uuid: 8.3.2 optionalDependencies: '@cucumber/cucumber': 13.0.0 - chromedriver: 148.0.4 + chromedriver: 150.0.1 transitivePeerDependencies: - bufferutil - canvas From 4a03f831aa0576b59d52fe7ceb302123221c49f7 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 10 Jul 2026 23:26:25 +0530 Subject: [PATCH 38/91] docs: test-results/ layout, traceGranularity:'test', tracePolicy, and known-debt --- CLAUDE.md | 8 +++++++- README.md | 10 ++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f06ea24d..14fff1a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,10 +236,16 @@ Documented divergences from the conventions above. They exist today as debt to b - `patchNodeAssert` (via `core/assert-patcher`) is now wired in all three adapters, default-on behind each adapter's `captureAssertions` option (opt out with `captureAssertions: false`). Framework matcher libraries differ: Service additionally taps expect-webdriverio's `afterAssertion` hook so passing+failing matchers render as `expect.*` actions; Selenium (chai/jest) and Nightwatch (`browser.assert`/`verify`) don't yet surface passing framework-matcher assertions as trace actions. - BiDi is auto-attached in Service and Selenium; Nightwatch is opt-in via `bidi: true` and requires `webSocketUrl: true` in capabilities. - Retry-aware trace policies aren't uniform across adapters. Selenium emits one metadata entry per attempt (counter-based `generateStableUid` uid + retry-stable tracker key), so all six policies — including `retain-on-first-failure` (needs the first attempt's outcome) — evaluate correctly. Service and Nightwatch key metadata on the retry-stable uid and overwrite/mutate per attempt, collapsing to the *final* attempt's outcome; the retry-count policies (`retain-on-failure`, `on-first-retry`, `on-all-retries`, `retain-on-failure-and-retries`) are still correct, but `retain-on-first-failure` can't see a failed-then-passed first attempt and under-retains. Proper fix rides with B5 (per-test granularity preserves per-attempt slices); until then all three set `attemptInfoAvailable: true`. +- Nightwatch negated native asserts (`browser.assert.not.*` / `verify.not.*`) produce no trace row: the `not` namespace is a nested Proxy that `wrapAssertionNamespaces` passes through untouched, and the internal command it issues is dropped by the `hasUserSource` gate in `browserProxy`. Positive `assert.*`/`verify.*` still record. +- Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. +- Nightwatch native asserts populate all at once in live mode and finalize pass/fail in one test-end batch: Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin (`client.queue.tree` / `client.reporter` aren't on `browser`), so call-time capture streams neutral pending rows and `afterEach` reconciles outcomes. Trace-timeline positions are corrected from `results.commands`. +- Service assertion-suppression self-heal is command-triggered: a swallowed soft-assert (`expect.soft` / user `try/catch`) whose matcher hard-throws leaves `#assertionDepth` stuck until the next top-level user command's stack shows no `expect-webdriverio` frame (heal) or the next `beforeAssertion` (reset). A stuck depth with no following command or assertion before test end persists until the next test's `resetStack()` — benign (no later command in that test to suppress). ### File-size (raw line counts; soft cap is 500 logic lines) -None of the entries below trigger the `max-lines` lint rule after `skipBlankLines`/`skipComments`. They're documented because their raw line count is over 500, and the next substantive change to any of them should still look for an extraction opportunity. +Most entries below don't trigger the `max-lines` lint rule after `skipBlankLines`/`skipComments`; they're documented because their raw line count is over 500, and the next substantive change to any of them should still look for an extraction opportunity. The service plugin is the exception — it's now over the *logic*-line cap. + +- `packages/service/src/index.ts` — over the 500-**logic**-line soft cap (the WDIO plugin god-file: session/screencast/command/assertion/trace-slice wiring in one class). Split candidate; the trace-slice, assertion-capture, and screencast concerns are the natural extraction seams. - `packages/nightwatch-devtools/src/index.ts` (~536 raw). Cucumber/test/run-lifecycle, session-init, event-hub modules already extracted; remainder is the `PluginInternals` accessor bag plus per-method delegators plus the factory. Accept-as-is. - `packages/selenium-devtools/src/index.ts` (~560 raw). Session/test-lifecycle extracted; remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Same situation as nightwatch. diff --git a/README.md b/README.md index fa87cc71..09f93c2e 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ When BiDi is active in Selenium or Nightwatch, the per-command Chrome performanc ### 📦 Trace mode (trace.zip) -Headless capture path — no DevTools UI window opens. At session end the adapter writes a trace artifact next to the user's output directory, suitable for offline replay, AI-agent diffing, or any consumer that prefers a portable artifact over a live UI. +Headless capture path — no DevTools UI window opens. At session end the adapter writes trace artifacts into a `test-results/` folder (created next to the resolved spec/config directory), suitable for offline replay, AI-agent diffing, or any consumer that prefers a portable artifact over a live UI. | Adapter | How to enable | |---|---| @@ -79,7 +79,7 @@ Headless capture path — no DevTools UI window opens. At session end the adapte | **Nightwatch** | `globals: nightwatchDevtools({ mode: 'trace' })` | The trace artifact contains: -- `trace.trace` — NDJSON `context-options` + `before`/`after` action events. When test hooks are available (Mocha's `it()` / Cucumber's `Scenario()`), each test becomes a [`Tracing.tracingGroup`](https://github.com/VibiumDev/vibium/blob/main/docs/explanation/recording-format.md#action-groups-user-defined) span — an open/close `before`/`after` pair with `method: "tracingGroup"` and `params.name` set to the test title. Child actions inside the group carry `parentId` pointing back to the group's `callId`, so timeline viewers render tests as labelled spans wrapping their commands. +- `trace.trace` — NDJSON `context-options` + `before`/`after` action events. When test hooks are available (Mocha's `it()` / Cucumber's `Scenario()`), each test becomes a `Tracing.tracingGroup` span — an open/close `before`/`after` pair with `method: "tracingGroup"` and `params.name` set to the test title. Child actions inside the group carry `parentId` pointing back to the group's `callId`, so timeline viewers render tests as labelled spans wrapping their commands. - `trace.network` — HAR-style network entries derived from the existing capture - `resources/page@<id>-<ts>.jpeg` — screenshot per user-facing action - `resources/elements-page@<id>-<ts>.json` — flat interactable element list extracted by the page-injected scripts in `@wdio/devtools-core/element-scripts` @@ -103,7 +103,7 @@ npx show-trace path/to/trace.zip # in a project that installs an adapter The `show-trace` bin is exposed by each adapter (`@wdio/devtools-service`, `@wdio/nightwatch-devtools`, `@wdio/selenium-devtools`), so `pnpm show-trace <zip>` / `npx show-trace <zip>` work in any project that installs one — no extra dependency. -**Other viewers.** Because the format is unchanged, the same `.zip` also drops into [player.vibium.dev](https://player.vibium.dev) or `npx playwright show-trace <path>`. The format follows the [Vibium recording format](https://github.com/VibiumDev/vibium/blob/main/docs/explanation/recording-format.md) spec — a Playwright-compatible NDJSON schema that the ecosystem already renders. This is the same format [`@wdio/mcp`](https://webdriver.io/docs/mcp) uses for AI-driven session recording. +**Other viewers.** The trace uses a portable NDJSON schema, so the same `.zip` also opens in other compatible trace viewers that read the format. This is the same format [`@wdio/mcp`](https://webdriver.io/docs/mcp) uses for AI-driven session recording. #### Options @@ -111,7 +111,9 @@ The `show-trace` bin is exposed by each adapter (`@wdio/devtools-service`, `@wdi |--------|--------|---------|-------------| | `mode` | `'live'` \| `'trace'` | `'live'` | `'live'` launches the DevTools UI; `'trace'` writes an offline artifact. | | `traceFormat` | `'zip'` \| `'ndjson-directory'` | `'zip'` | Output layout. `'zip'` writes a single archive; `'ndjson-directory'` unpacks into `trace-<id>/`. | -| `traceGranularity` | `'session'` \| `'spec'` | `'session'` | `'session'` writes one trace per worker; `'spec'` writes one trace per spec file — smaller artifacts, easier to navigate. | +| `traceGranularity` | `'session'` \| `'spec'` \| `'test'` | `'session'` | `'session'` writes one trace per worker; `'spec'` one per spec file; `'test'` one per test into its own `<spec>-<title>-<browser>[-retryN]/trace.zip` folder — the smallest, most navigable artifacts, and the best pairing for a retention policy. | +| `tracePolicy` | `'on'` \| `'retain-on-failure'` \| `'retain-on-first-failure'` \| `'on-first-retry'` \| `'on-all-retries'` \| `'retain-on-failure-and-retries'` | `'on'` | Which traces to keep. `'on'` keeps every trace; the rest keep only failing/retried tests — pairs well with `traceGranularity: 'test'`. | +| `captureAssertions` | `boolean` | `true` | Capture assertions as action rows: `node:assert` (all adapters), WebdriverIO `expect(...)` matchers, and Nightwatch `browser.assert`/`verify`. Set `false` to opt out. | WDIO config example: From 96f7807f2446a87fbabdd0c116a1314da4c258ff Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:39:51 +0530 Subject: [PATCH 39/91] fix(core): bound the pending-snapshot settle in trace finalize --- packages/core/src/trace-finalizer.ts | 55 ++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index 415b6289..f0459590 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -79,6 +79,57 @@ const TEST_WITHOUT_BOUNDARIES_WARNING = /** Above this many slices, warn to pair granularity with a retention policy. */ const SLICE_COUNT_WARN_THRESHOLD = 200 +/** Cap on how long finalize waits for still-in-flight snapshot captures. A + * fire-and-forget capture against a tearing-down session can hang forever + * (never resolves nor rejects); blocking the export on it would deadlock the + * whole run (finalize never returns → the runner never tears the session down + * → the capture stays stuck). Past this bound we write whatever snapshots + * resolved in time — the same "flush from what exists" rule slices use. */ +const PENDING_SETTLE_TIMEOUT_MS = 5000 + +/** + * `Promise.allSettled` bounded by a timeout. Resolves when every pending + * capture settles OR the cap elapses, whichever comes first — a stuck capture + * can never hang the export. Returns whether it timed out so the caller can + * warn. The timer is unref'd so it can't itself keep the process alive. + */ +async function settlePending( + pending: Promise<unknown>[], + timeoutMs: number +): Promise<boolean> { + let timer: ReturnType<typeof setTimeout> | undefined + const timedOut = new Promise<true>((resolve) => { + timer = setTimeout(() => resolve(true), timeoutMs) + timer.unref?.() + }) + const settled = Promise.allSettled(pending).then(() => false) + const result = await Promise.race([settled, timedOut]) + if (timer) { + clearTimeout(timer) + } + return result +} + +/** Settle in-flight snapshot captures under the timeout cap, warning if the + * bound elapses. Never throws; never hangs. */ +async function awaitPendingCaptures(ctx: TraceExportContext): Promise<void> { + if (!ctx.awaitPending?.length) { + return + } + const timedOut = await settlePending( + ctx.awaitPending, + PENDING_SETTLE_TIMEOUT_MS + ) + if (timedOut) { + ctx.log?.( + 'warn', + `One or more of ${ctx.awaitPending.length} snapshot capture(s) did not ` + + `settle within ${PENDING_SETTLE_TIMEOUT_MS}ms; writing trace with the ` + + 'snapshots captured so far.' + ) + } +} + function sliceCountWarning(count: number): string { return ( `traceGranularity produced ${count} trace slices. Consider pairing it ` + @@ -269,9 +320,7 @@ export async function finalizeTraceExport( if (ctx.mode !== 'trace') { return [] } - if (ctx.awaitPending?.length) { - await Promise.allSettled(ctx.awaitPending) - } + await awaitPendingCaptures(ctx) const sliced = ctx.granularity === 'spec' || ctx.granularity === 'test' if (sliced && ctx.ranges.length > 0) { if (ctx.ranges.length > SLICE_COUNT_WARN_THRESHOLD) { From 3a9493f93eecf822097d003f136f637b741727ab Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:40:05 +0530 Subject: [PATCH 40/91] fix(core): order trace actions chronologically, not by insertion --- packages/core/src/trace-action-events.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/core/src/trace-action-events.ts b/packages/core/src/trace-action-events.ts index 7cd44eab..29eeb0b2 100644 --- a/packages/core/src/trace-action-events.ts +++ b/packages/core/src/trace-action-events.ts @@ -297,7 +297,18 @@ export function buildActionEvents( snapshotIndex?: FrameSnapshotIndex ): ActionEvent[] { const stream: ActionStream = { events: [], prevEndMs: 0, callCounter: 0 } - for (const cmd of commands) { + // Process in chronological order, not insertion order. Deferred rows (e.g. + // Nightwatch native asserts, finalized in one batch at test-end) are appended + // late but carry their real call-time `startTime`; since pushActionPair floors + // each start at the running prevEndMs, out-of-order input would clamp those + // late rows to the end of the timeline (clustering them after the last real + // command). A stable sort by start time restores true positions and keeps + // equal-time rows in insertion order (each command owns its own before/after + // pair, so pairing is unaffected). + const ordered = [...commands].sort( + (a, b) => (a.startTime ?? a.timestamp) - (b.startTime ?? b.timestamp) + ) + for (const cmd of ordered) { const action = mapCommandToAction(cmd.command) if (!action) { continue From 9fdbd6a4b8d3ba86492cc001ec2f4155c33ed6e8 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:40:25 +0530 Subject: [PATCH 41/91] fix(core): capture strict-mode node:assert and label asserts from args --- packages/core/src/action-mapping.ts | 12 +++--- packages/core/src/assert-patcher.ts | 44 +++++++++++++++----- packages/core/tests/assert-patcher.test.ts | 22 +++++++++- packages/core/tests/trace-assertions.test.ts | 19 ++++++++- 4 files changed, 79 insertions(+), 18 deletions(-) diff --git a/packages/core/src/action-mapping.ts b/packages/core/src/action-mapping.ts index 368ca8e3..eb31c2b2 100644 --- a/packages/core/src/action-mapping.ts +++ b/packages/core/src/action-mapping.ts @@ -46,8 +46,12 @@ function formatAssertValue(value: unknown): string { : text } -// Prefer normalized actual/expected params (nightwatch collapses them into -// the result); fall back to the first two positional args (node:assert order). +// The label mirrors the CALL the user wrote — its positional args +// (`assert.equal(a, b)`, `titleContains('x')`). The normalized actual/expected +// params drive the result diff, NOT the label: a single-arg assert like +// titleContains passes only the expected, so labelling it with the derived +// actual would misrepresent the call. Fall back to actual/expected only when +// no args survived (reader round-trips that dropped them). function formatAssertTitle( action: TraceAction, args: unknown[], @@ -55,9 +59,7 @@ function formatAssertTitle( command?: string ): string { const values = - params && ('actual' in params || 'expected' in params) - ? [params.actual, params.expected] - : args.slice(0, 2) + args.length > 0 ? args.slice(0, 2) : [params?.actual, params?.expected] const label = values .filter((value) => value !== undefined) .map(formatAssertValue) diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index c6b91bf2..c60f21ab 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -200,6 +200,29 @@ export function matcherAssertionToCommandLog( return entry } +/** Wrap every tracked method on one assert namespace object in place; returns + * how many were patched. Used for both `assert.*` and `assert.strict.*`. */ +function patchAssertNamespace( + nsObj: Record<string | symbol, unknown>, + onCommand: (cmd: CapturedAssert) => void +): number { + let count = 0 + for (const methodName of TRACKED_ASSERT_METHODS) { + const original = nsObj[methodName] + if (typeof original !== 'function') { + continue + } + nsObj[methodName] = makePatchedAssertMethod( + methodName, + nsObj, + original as (...a: unknown[]) => unknown, + onCommand + ) + count++ + } + return count +} + /** * Patch `node:assert` so each tracked method emits a `CapturedAssert` to the * supplied hook. Idempotent across calls (guarded by `ASSERT_PATCHED_SYMBOL`). @@ -239,19 +262,20 @@ export function patchNodeAssert( } assertObj[ASSERT_PATCHED_SYMBOL] = true - for (const methodName of TRACKED_ASSERT_METHODS) { - const original = assertObj[methodName] - if (typeof original !== 'function') { - continue - } - assertObj[methodName] = makePatchedAssertMethod( - methodName, - assertObj, - original as (...a: unknown[]) => unknown, + let patched = patchAssertNamespace(assertObj, onCommand) + // `import { strict as assert }` and `assert.strict.*` reference a separate + // object with its own method identities; patch it too so strict-mode + // assertions are captured the same as the default ones. `assert.strict` is a + // callable (strict-mode `assert()`) with the methods hung off it, so it's a + // function, not a plain object. + const strict = assertObj.strict + if (strict && (typeof strict === 'object' || typeof strict === 'function')) { + patched += patchAssertNamespace( + strict as Record<string | symbol, unknown>, onCommand ) } - log('info', `Patched ${TRACKED_ASSERT_METHODS.length} node:assert method(s)`) + log('info', `Patched ${patched} node:assert method(s)`) return true } diff --git a/packages/core/tests/assert-patcher.test.ts b/packages/core/tests/assert-patcher.test.ts index 38bea721..5eda95da 100644 --- a/packages/core/tests/assert-patcher.test.ts +++ b/packages/core/tests/assert-patcher.test.ts @@ -23,17 +23,24 @@ const USER_FRAME = { } const INTERNAL_FRAME = { filePath: undefined, callSource: 'unknown:0' } -// Snapshot real methods once so each test starts from a fresh assert. +// Snapshot real methods once so each test starts from a fresh assert. Both the +// default namespace and `assert.strict` are patched, so restore both. const ASSERT_MUT = assert as unknown as Record<string | symbol, unknown> +const STRICT_MUT = ( + assert as unknown as { strict: Record<string | symbol, unknown> } +).strict const originals: Record<string, unknown> = {} +const strictOriginals: Record<string, unknown> = {} for (const m of TRACKED_ASSERT_METHODS) { originals[m] = ASSERT_MUT[m] + strictOriginals[m] = STRICT_MUT[m] } beforeEach(() => { delete ASSERT_MUT[ASSERT_PATCHED_SYMBOL] for (const m of TRACKED_ASSERT_METHODS) { ASSERT_MUT[m] = originals[m] + STRICT_MUT[m] = strictOriginals[m] } // Default: every assert looks like it came from user code so captures fire. vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) @@ -84,6 +91,19 @@ describe('patchNodeAssert', () => { expect(results).toEqual(['passed', undefined]) // first resolved, second rejected }) + // `import { strict as assert }` (and `assert.strict.*`) reference a separate + // object; without patching it, strict-mode assertions were never captured. + it('captures assertions on the strict namespace too', () => { + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + assert.strict.equal(1, 1) + expect(() => assert.strict.equal(1, 2)).toThrow() + expect(captured.map((c) => [c.command, c.result])).toEqual([ + ['assert.equal', 'passed'], + ['assert.equal', undefined] + ]) + }) + // Only assertions that originate in user test code should reach the trace. // Dependency/framework-internal asserts have no user-code frame on the // stack (getCallSourceFromStack yields 'unknown:0') and must be dropped. diff --git a/packages/core/tests/trace-assertions.test.ts b/packages/core/tests/trace-assertions.test.ts index 0d3d4ccd..d28b8d55 100644 --- a/packages/core/tests/trace-assertions.test.ts +++ b/packages/core/tests/trace-assertions.test.ts @@ -67,7 +67,10 @@ describe('formatActionTitle for asserts', () => { ).toBe('assert.strictEqual("a", "b")') }) - it('prefers normalized actual/expected params over positional args', () => { + it('labels from the call args, not the derived actual/expected', () => { + // textContains('#el', 'foo') passes the selector + expected as args; the + // real "actual" ('bar') lives in params for the result diff and must NOT + // leak into the concise label, which mirrors the call the user wrote. expect( formatActionTitle( action, @@ -75,6 +78,17 @@ describe('formatActionTitle for asserts', () => { { actual: 'bar', expected: 'foo' }, 'verify.textContains' ) + ).toBe('verify.textContains("#el", "foo")') + }) + + it('falls back to actual/expected params only when no args survive', () => { + expect( + formatActionTitle( + action, + [], + { actual: 'bar', expected: 'foo' }, + 'verify.textContains' + ) ).toBe('verify.textContains("bar", "foo")') }) @@ -169,7 +183,8 @@ describe('buildActionEvents with assert commands', () => { expected: 'foo', message: 'nope' }) - expect(before.title).toBe('verify.textContains("bar", "foo")') + // Label mirrors the call args; actual/expected stay in params for the diff. + expect(before.title).toBe('verify.textContains("#el", "foo")') // No Error instance on the entry — the failure comes from the collapsed result. expect(afterOf(events, before.callId)?.error).toEqual({ message: 'nope' }) }) From a8f8cf4fc2ff9799d2b19688d4cec31f22552f79 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:40:42 +0530 Subject: [PATCH 42/91] fix(nightwatch): emit native asserts at test-end with real outcomes --- .../src/helpers/browserProxy.ts | 10 +- .../src/helpers/nativeAssertions.ts | 162 ++++++++++++++---- packages/nightwatch-devtools/src/index.ts | 2 + .../nightwatch-devtools/src/trace-context.ts | 6 +- .../tests/nativeAssertions.test.ts | 123 ++++++++----- 5 files changed, 219 insertions(+), 84 deletions(-) diff --git a/packages/nightwatch-devtools/src/helpers/browserProxy.ts b/packages/nightwatch-devtools/src/helpers/browserProxy.ts index dff01a1f..21225506 100644 --- a/packages/nightwatch-devtools/src/helpers/browserProxy.ts +++ b/packages/nightwatch-devtools/src/helpers/browserProxy.ts @@ -134,9 +134,12 @@ export class BrowserProxy { }) } - /** Stream a neutral pending row for an explicit assert/verify call the moment - * it's invoked, and buffer the call (with its emitted row) for `afterEach` - * to finalize pass/fail in place. */ + /** Buffer an explicit assert/verify call at invocation time (prebuilding its + * row with args/callSource/screenshot), but do NOT stream it yet. Nightwatch + * exposes no per-assertion result hook, so streaming here would show every + * assert as a neutral (green) row while the test is still Running — before + * its pass/fail is known. `captureNativeAssertions` emits the rows at test-end + * with real outcomes + execution timing instead. */ private emitPendingAssertion(call: NativeAssertCall): void { const testUid = this.getCurrentTest()?.uid const entry = pendingAssertionCommand( @@ -144,7 +147,6 @@ export class BrowserProxy { testUid, latestResolvedScreenshot(this.sessionCapturer) ) - this.sessionCapturer.captureAssertCommand(entry) call.entry = entry this.nativeAssertCalls.push(call) } diff --git a/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts index 16dd3bd2..1134047d 100644 --- a/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts +++ b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts @@ -1,19 +1,22 @@ // Native-assertion capture: turns explicit `browser.assert.*` / // `browser.verify.*` calls into concise trace/UI action rows with real args, -// a clickable source location, and pass/fail colour — streamed LIVE. +// a clickable source location, and pass/fail colour. // -// Nightwatch exposes no per-assertion hook, and its test-end -// `currentTest.results` carries no source location for PASSING assertions -// (results.assertions[i].stackTrace is '' on pass) and only stringified args. -// So each explicit call is intercepted at CALL TIME -// (BrowserProxy.wrapAssertionNamespaces): a neutral "pending" row is emitted -// immediately (concise title, real args, callSource) so rows stream in one by -// one like normal commands. At TEST END the pass/fail truth + verbose failure -// message are read from results.assertions and the already-emitted row is -// UPDATED IN PLACE (same stable id) — never re-created, so no duplicates. -// Iterating the recorded calls (not results.assertions) also excludes +// Nightwatch exposes no per-assertion hook. Each explicit call is intercepted +// at CALL TIME (BrowserProxy.wrapAssertionNamespaces) and BUFFERED — concise +// title, real args, callSource, the per-test testUid, and the preceding +// screenshot — but NOT streamed, because streaming a neutral row mid-run would +// render every assert green before its outcome is known. At test-end +// `captureNativeAssertions` reads the pass/fail truth + verbose failure message +// from the results bag and emits each row once, positioned on its real +// execution window (the exporter re-sorts by the buffered call timestamp). +// Correlating the recorded calls (not the raw results) also excludes // Nightwatch's implicit command-generated assertions (e.g. // waitForElementVisible's "element was visible" entry). +// +// The plugin `afterEach` fires once per describe-suite (not per `it`), so the +// finalize reads assertions from `results.testcases` (all tests) — see +// gatherResultAssertions. import logger from '@wdio/logger' import { @@ -96,6 +99,39 @@ interface Outcome { message: string } +/** + * All assertion entries for the suite, in declaration order. Nightwatch's + * plugin `afterEach` fires once per describe-suite (not per `it`), so the flat + * `results.assertions` reflects only the last testcase; the full per-test + * breakdown lives in `results.testcases[title].assertions`. Flattening every + * testcase's entries lets each buffered call correlate to its own test's + * outcome (without this, only the last test's asserts get a pass/fail and the + * rest render neutral). Falls back to the flat list for single-test modules or + * older Nightwatch that doesn't populate `testcases`. + */ +function gatherResultAssertions( + results: + | { + assertions?: NwAssertionEntry[] + testcases?: Record<string, { assertions?: unknown[] }> + } + | undefined +): NwAssertionEntry[] { + const testcases = results?.testcases + if (testcases && typeof testcases === 'object') { + const all: NwAssertionEntry[] = [] + for (const tc of Object.values(testcases)) { + if (Array.isArray(tc?.assertions)) { + all.push(...(tc.assertions as NwAssertionEntry[])) + } + } + if (all.length > 0) { + return all + } + } + return Array.isArray(results?.assertions) ? results.assertions : [] +} + /** * Pair each recorded call with its `results.assertions` entry for pass/fail + * verbose message. Both lists are in call/execution order and each explicit @@ -206,8 +242,55 @@ export function pendingAssertionCommand( return entry } -/** Update one streamed pending row in place: apply pass/fail + verbose error, - * a screenshot, and its real execution window, then re-broadcast by stable id. */ +/** Collapsed pass/fail result core's `collapsedAssertResult` reads. */ +interface CollapsedAssertResult { + passed: boolean + expected?: unknown + actual?: unknown + message?: string +} + +/** Nightwatch failure messages end with `… but got: "<actual>"`. Pull out the + * real observed value: Nightwatch passes only the EXPECTED as an arg, so the + * actual lives in the message (and only on failure). Undefined when absent. + * Uses indexOf + slice (no backtracking-prone regex) on the message tail. */ +function parseActualFromMessage(message: string): string | undefined { + const marker = 'but got:' + const idx = message.lastIndexOf(marker) + if (idx === -1) { + return undefined + } + let rest = message.slice(idx + marker.length).trim() + rest = rest.replace(/ \(\d+ms\)$/, '').trim() // drop trailing "(123ms)" + rest = rest.replace(/^["']/, '').replace(/["']$/, '').trim() // strip quotes + return rest || undefined +} + +/** Build the collapsed `{passed, expected, actual?, message}` result core's + * `buildAssertParams` prefers over the positional `[actual, expected]` arg + * convention — which is wrong for Nightwatch, whose asserts pass only the + * expected value (`titleContains('x')`), never the actual. */ +function collapsedAssertResult( + call: NativeAssertCall, + outcome: Outcome +): CollapsedAssertResult { + const result: CollapsedAssertResult = { + passed: outcome.passed, + expected: call.args.length <= 1 ? call.args[0] : call.args, + message: outcome.message + } + const actual = outcome.passed + ? undefined + : parseActualFromMessage(outcome.message) + if (actual !== undefined) { + result.actual = actual + } + return result +} + +/** Emit one assertion row at test-end: apply pass/fail + verbose error, a + * screenshot, and its real execution window, then send it. Rows are NOT + * streamed at call time, so this is the single emit (no neutral pending row). */ function finalizeAssertionRow( capturer: SessionCapturer, call: NativeAssertCall, @@ -229,32 +312,34 @@ function finalizeAssertionRow( }, testUid ) - entry.result = finalized.result + // Collapsed object result (not the plain 'passed' string) so the trace's + // action params show a true actual-vs-expected diff instead of mislabelling + // Nightwatch's single expected-arg as the actual. + entry.result = collapsedAssertResult(call, outcome) entry.error = finalized.error if (!entry.screenshot && screenshot) { entry.screenshot = screenshot } - // Reposition the row on its REAL execution window (Nightwatch enqueues all - // asserts at once, so the emit/enqueue timestamp clustered them). The row is - // matched for replacement by its stable id, so the old enqueue timestamp is - // what the UI still keys on until this swap lands. - const oldTimestamp = entry.timestamp + // Position the row on its REAL execution window (Nightwatch enqueues all + // asserts at once, so the call-time timestamp clustered them). The row was + // never streamed, so send it now — a single emit with the final outcome. if (timing) { entry.startTime = timing.startTime entry.timestamp = timing.endTime > timing.startTime ? timing.endTime : timing.startTime } - capturer.sendReplaceCommand(oldTimestamp, entry) + capturer.captureAssertCommand(entry) log.info(`[assert] ${entry.title} → ${outcome.passed ? 'pass' : 'fail'}`) } /** - * Finalize the streamed pending rows at test-end: correlate the recorded calls - * with `results.assertions`, then UPDATE each row's `entry` in place with - * pass/fail (`result`) + the verbose failure message (`error`, failures only) - * + a screenshot + its real execution window, and re-broadcast via - * `sendReplaceCommand` keyed on the row's stable id. No new rows are created - * (no duplicates); a call with no matching result is left pending (defensive). + * Emit the native assertion rows at test-end: correlate the recorded calls with + * `results.assertions`, then send each row with pass/fail (`result`) + the + * verbose failure message (`error`, failures only) + a screenshot + its real + * execution window. Rows are buffered (not streamed) at call time — Nightwatch + * has no per-assertion result hook, so streaming them during the run would show + * every assert as a neutral (green) row before its outcome is known. A call with + * no matching result is skipped (defensive). */ export async function captureNativeAssertions( capturer: SessionCapturer, @@ -269,11 +354,13 @@ export async function captureNativeAssertions( // Boundary cast: `results` is Nightwatch's loosely-typed per-test bag; we read // only the assertions + commands arrays whose shapes are documented above. const results = currentTest?.results as - | { assertions?: NwAssertionEntry[]; commands?: NwCommandEntry[] } + | { + assertions?: NwAssertionEntry[] + commands?: NwCommandEntry[] + testcases?: Record<string, { assertions?: unknown[] }> + } | undefined - const assertions = Array.isArray(results?.assertions) - ? results.assertions - : [] + const assertions = gatherResultAssertions(results) const outcomes = correlate(calls, assertions) const timings = assertCommandTimings( Array.isArray(results?.commands) ? results.commands : [], @@ -283,9 +370,11 @@ export async function captureNativeAssertions( const screenshot = await resolveAssertionScreenshot(capturer, browser) calls.forEach((call, index) => { + if (!call.entry) { + return + } const outcome = outcomes[index] - // Leave an unmatched (or never-emitted) row in its last state. - if (call.entry && outcome) { + if (outcome) { finalizeAssertionRow( capturer, call, @@ -294,6 +383,15 @@ export async function captureNativeAssertions( screenshot, testUid ) + return + } + // No execution outcome correlated to this call (defensive): emit the + // buffered row in its neutral state so the assertion still appears — never + // dropped, never mis-coloured as passed/failed. Rows are only buffered at + // call time, so without this an uncorrelated assert would vanish. + if (!call.entry.screenshot && screenshot) { + call.entry.screenshot = screenshot } + capturer.captureAssertCommand(call.entry) }) } diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index db5cefc0..53b1a3dd 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -589,6 +589,8 @@ class NightwatchDevToolsPlugin { ranges: this.#specRanges, flushed: this.#flushedSpecs, configPath: this.#configPath, + testFilePath: + this.browserProxy?.getCurrentTestFullPath?.() ?? undefined, log: (level, msg) => log[level](msg) }, sessionId diff --git a/packages/nightwatch-devtools/src/trace-context.ts b/packages/nightwatch-devtools/src/trace-context.ts index 1e451ac7..359737b9 100644 --- a/packages/nightwatch-devtools/src/trace-context.ts +++ b/packages/nightwatch-devtools/src/trace-context.ts @@ -31,6 +31,7 @@ export interface TraceContextInput { ranges: SpecRange[] flushed: Set<string> configPath: string | undefined + testFilePath: string | undefined log: (level: 'info' | 'warn', msg: string) => void } @@ -50,7 +51,10 @@ export function buildTraceContext( ranges: input.ranges, flushed: input.flushed, resolveOutputDir: () => - resolveAdapterOutputDir({ configPath: input.configPath }), + resolveAdapterOutputDir({ + testFilePath: input.testFilePath, + configPath: input.configPath + }), awaitPending: input.capturer.snapshotCaptures, // Nightwatch feeds real per-test attempt numbers via TestAttemptTracker // (B4), so retry-aware policies use per-test attempts, not the fallback. diff --git a/packages/nightwatch-devtools/tests/nativeAssertions.test.ts b/packages/nightwatch-devtools/tests/nativeAssertions.test.ts index d904dbde..352875f2 100644 --- a/packages/nightwatch-devtools/tests/nativeAssertions.test.ts +++ b/packages/nightwatch-devtools/tests/nativeAssertions.test.ts @@ -56,21 +56,20 @@ function makeFakeCapturer( const browser = {} as unknown as NightwatchBrowser -/** Mimic BrowserProxy.emitPendingAssertion: stream a neutral pending row at - * call time and attach it to the call for later finalization. */ +/** Mimic BrowserProxy.emitPendingAssertion: build the row and buffer it on the + * call, but do NOT stream it — rows are emitted at test-end by + * captureNativeAssertions once their pass/fail is known. */ function emitPending( capturer: SessionCapturer, calls: NativeAssertCall[], testUid: string | undefined ) { for (const call of calls) { - const entry = pendingAssertionCommand( + call.entry = pendingAssertionCommand( call, testUid, latestResolvedScreenshot(capturer) ) - capturer.captureAssertCommand(entry) - call.entry = entry } } @@ -104,8 +103,8 @@ function call( return { prefix, method, args, callSource, timestamp: clock++ } } -describe('captureNativeAssertions (live pending → finalize)', () => { - it('streams neutral pending rows at call time, then finalizes pass/fail in place with a stable id and no duplicates', async () => { +describe('captureNativeAssertions (buffer → finalize at test-end)', () => { + it('buffers rows at call time (nothing streamed), then emits pass/fail once at test-end with no duplicates', async () => { // Preceding real commands: the last with a resolved screenshot is the DOM // the assertions evaluated against — reused for the pending rows. const precedingCommands: CommandLog[] = [ @@ -120,7 +119,6 @@ describe('captureNativeAssertions (live pending → finalize)', () => { const { capturer, sent, - replaced, commandsLog, captureAssertCommand, sendReplaceCommand, @@ -133,23 +131,23 @@ describe('captureNativeAssertions (live pending → finalize)', () => { call('assert', 'titleContains', ['HARD_FAIL_ME'], 'spec.js:16') ] - // 1) CALL TIME: each call streams one neutral pending row. + // 1) CALL TIME: rows are buffered, NOT streamed — nothing is sent, so no + // assert shows as a neutral (green) row before its outcome is known. emitPending(capturer, calls, 'test-uid') - expect(captureAssertCommand).toHaveBeenCalledTimes(4) - expect(sent).toHaveLength(4) - for (const row of sent) { + expect(captureAssertCommand).not.toHaveBeenCalled() + expect(sent).toHaveLength(0) + const buffered = calls.map((c) => c.entry!) + for (const row of buffered) { expect(row.result).toBeUndefined() // neutral — not green expect(row.error).toBeUndefined() // neutral — not red expect(row.testUid).toBe('test-uid') expect(row.screenshot).toBe('BODY_SHOT') // reused preceding DOM - expect(typeof (row as { id?: number }).id).toBe('number') } - expect(sent[0].title).toBe("verify.titleContains('Example')") - expect(sent[0].command).toBe('verify.titleContains') - expect(sent[0].args).toEqual(['Example']) - expect(sent[0].callSource).toBe('spec.js:11') - expect(sent[0].timestamp).toBe(sent[0].startTime) - const pendingIds = sent.map((r) => (r as { id?: number }).id) + expect(buffered[0].title).toBe("verify.titleContains('Example')") + expect(buffered[0].command).toBe('verify.titleContains') + expect(buffered[0].args).toEqual(['Example']) + expect(buffered[0].callSource).toBe('spec.js:11') + expect(buffered[0].timestamp).toBe(buffered[0].startTime) // 2) TEST END: results.assertions includes the implicit waitForElementVisible // assertion (first entry) which must NOT create/finalize a row. @@ -168,46 +166,45 @@ describe('captureNativeAssertions (live pending → finalize)', () => { calls ) - // Each row updated exactly once, in place — no new rows, no duplicates. + // Rows emitted exactly once at test-end — no call-time stream, no replace. expect(captureAssertCommand).toHaveBeenCalledTimes(4) - expect(sendReplaceCommand).toHaveBeenCalledTimes(4) + expect(sendReplaceCommand).not.toHaveBeenCalled() + expect(sent).toHaveLength(4) expect(commandsLog).toHaveLength(2 + 4) expect(takeScreenshotViaHttp).not.toHaveBeenCalled() - // Same row objects, same stable ids (updated, not recreated). const finalized = calls.map((c) => c.entry!) - expect(finalized.map((r) => (r as { id?: number }).id)).toEqual(pendingIds) - // Pass/fail applied; failures carry the verbose message as error. - expect(finalized[0].result).toBe('passed') + // Pass/fail applied as a collapsed { passed, expected, actual? } result; + // failures also carry the verbose message as error. + expect(finalized[0].result).toMatchObject({ passed: true }) expect(finalized[0].error).toBeUndefined() - expect(finalized[1].result).toBeUndefined() + expect(finalized[1].result).toMatchObject({ passed: false }) expect((finalized[1].error as { message: string }).message).toContain( 'SOFT_FAIL_ME' ) - expect(finalized[2].result).toBe('passed') // duplicate 'Example' → 2nd entry + expect(finalized[2].result).toMatchObject({ passed: true }) // 2nd 'Example' expect(finalized[3].error).toBeDefined() // Labels/args/callSource preserved from the pending row. expect(finalized[3].title).toBe("assert.titleContains('HARD_FAIL_ME')") expect(finalized[3].callSource).toBe('spec.js:16') - // With no results.commands timing, rows keep their enqueue timestamp, so - // the replace is keyed on that same timestamp. - replaced.forEach(({ oldTimestamp, command }) => { - expect(oldTimestamp).toBe(command.timestamp) + // With no results.commands timing, rows keep their call-time timestamp. + finalized.forEach((row) => { + expect(row.timestamp).toBe(row.startTime) }) }) it('repositions each row on its real execution window from results.commands (not enqueue time)', async () => { - const { capturer, sent, replaced } = makeFakeCapturer() + const { capturer, sent } = makeFakeCapturer() const calls = [ call('verify', 'titleContains', ['Example']), call('assert', 'titleContains', ['SOFT_FAIL_ME']) ] emitPending(capturer, calls, 'uid') - // Both enqueued ~together (clock++), clustered. - const enqueue = sent.map((r) => r.timestamp) + // Buffered ~together (clock++), clustered at call time — nothing sent yet. + expect(sent).toHaveLength(0) // Nightwatch ran them ~29ms apart; results.commands carries the real window. const assertions = [ @@ -234,10 +231,10 @@ describe('captureNativeAssertions (live pending → finalize)', () => { expect(finalized[1].startTime).toBe(6100) expect(finalized[1].timestamp).toBe(6132) expect(finalized[1].timestamp - finalized[0].timestamp).toBeGreaterThan(50) - // The replace is keyed on the ORIGINAL enqueue timestamp (stable id also - // matches), while the row now carries its real execution timestamp. - expect(replaced[0].oldTimestamp).toBe(enqueue[0]) - expect(replaced[0].command.timestamp).toBe(6029) + // Rows are sent (not replaced) at test-end, each carrying its real window. + expect(sent).toHaveLength(2) + expect(sent[0].timestamp).toBe(6029) + expect(sent[1].timestamp).toBe(6132) }) it('finalizes with a fresh screenshot when no preceding one has resolved yet', async () => { @@ -246,13 +243,13 @@ describe('captureNativeAssertions (live pending → finalize)', () => { const pending: CommandLog[] = [ { command: 'waitForElementVisible', args: ['body'], timestamp: 2 } ] - const { capturer, sent, takeScreenshotViaHttp } = makeFakeCapturer( + const { capturer, takeScreenshotViaHttp } = makeFakeCapturer( pending, 'FRESH_SHOT' ) const calls = [call('verify', 'titleContains', ['Example'])] emitPending(capturer, calls, 'uid') - expect(sent[0].screenshot).toBeUndefined() + expect(calls[0].entry!.screenshot).toBeUndefined() await captureNativeAssertions( capturer, @@ -265,15 +262,15 @@ describe('captureNativeAssertions (live pending → finalize)', () => { ) expect(takeScreenshotViaHttp).toHaveBeenCalledTimes(1) expect(calls[0].entry!.screenshot).toBe('FRESH_SHOT') - expect(calls[0].entry!.result).toBe('passed') + expect(calls[0].entry!.result).toMatchObject({ passed: true }) }) it('preserves real multi-arg assertions (no faked args)', async () => { - const { capturer, sent, replaced } = makeFakeCapturer() + const { capturer, sent } = makeFakeCapturer() const calls = [call('assert', 'containsText', ['#btn', 'Save'])] emitPending(capturer, calls, 'uid') - expect(sent[0].args).toEqual(['#btn', 'Save']) - expect(sent[0].title).toBe("assert.containsText('#btn', 'Save')") + expect(calls[0].entry!.args).toEqual(['#btn', 'Save']) + expect(calls[0].entry!.title).toBe("assert.containsText('#btn', 'Save')") await captureNativeAssertions( capturer, @@ -284,17 +281,17 @@ describe('captureNativeAssertions (live pending → finalize)', () => { 'uid', calls ) - expect(replaced).toHaveLength(1) + expect(sent).toHaveLength(1) expect(calls[0].entry!.error).toBeDefined() }) it('falls back to positional correlation when args are not literals', async () => { - const { capturer, sent } = makeFakeCapturer() + const { capturer } = makeFakeCapturer() // Element-object arg won't substring-match; positional order still pairs // the single call with the single explicit assertion outcome. const calls = [call('assert', 'elementPresent', [{ selector: 'body' }])] emitPending(capturer, calls, 'uid') - expect(sent[0].title).toBe('assert.elementPresent(…)') + expect(calls[0].entry!.title).toBe('assert.elementPresent(…)') await captureNativeAssertions( capturer, @@ -324,6 +321,38 @@ describe('captureNativeAssertions (live pending → finalize)', () => { expect(calls[0].entry!.error).toBeUndefined() }) + it('surfaces a real actual-vs-expected from the failure message (not the expected-only arg)', async () => { + const { capturer, sent } = makeFakeCapturer() + const calls = [call('assert', 'titleContains', ['This Is Not The Title'])] + emitPending(capturer, calls, 'uid') + + await captureNativeAssertions( + capturer, + browser, + currentTestWith([ + failing( + "Testing if the page title contains 'This Is Not The Title' in " + + '5000ms - expected "contains \'This Is Not The Title\'" but got: ' + + '"Example Domain"' + ) + ]), + 'uid', + calls + ) + + // Nightwatch passes only the EXPECTED as an arg; the real actual lives in + // the "but got: …" clause of the message. The collapsed result must carry + // the true diff, not mislabel the expected arg as the actual. + const result = sent[0].result as { + passed: boolean + expected: unknown + actual: unknown + } + expect(result.passed).toBe(false) + expect(result.expected).toBe('This Is Not The Title') + expect(result.actual).toBe('Example Domain') + }) + it('is a no-op when there are no recorded calls (implicit assertions ignored)', async () => { const { capturer, sendReplaceCommand, takeScreenshotViaHttp } = makeFakeCapturer() From aa7b852fc5db2d8f861796247b290c1a54443d65 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:40:57 +0530 Subject: [PATCH 43/91] fix(service): keep expect matchers to a single row --- packages/service/src/index.ts | 58 +++++++++++++++++++--------- packages/service/tests/index.test.ts | 52 +++++++++++++++++++------ 2 files changed, 79 insertions(+), 31 deletions(-) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 5a266694..064be3a4 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -110,6 +110,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { * matching Playwright and the Nightwatch native-assert behaviour. */ #assertionDepth = 0 + /** True once the current matcher has issued an internal command whose stack + * shows an expect-webdriverio frame. Gates the stuck-depth self-heal: the + * matcher's FIRST internal command (an element re-lookup) traces to the + * user's expect() line but its sync stack has no matcher frame yet, which + * would otherwise be misread as "matcher ended" and reset the depth mid-run + * — capturing every later matcher command as a duplicate row. */ + #matcherStarted = false + /** Matcher start timestamps (stack, paired with #assertionDepth) so a captured * assertion spans [matcher start → end] — its poll duration — instead of * collapsing to a zero-width point at completion, which left the screencast @@ -481,6 +489,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#assertionCallSources = [] } this.#assertionDepth++ + this.#matcherStarted = false this.#assertionStartTimes.push(Date.now()) // Capture the user's expect() call site now — see #assertionCallSources. this.#assertionCallSources.push( @@ -497,6 +506,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (this.#assertionDepth > 0) { this.#assertionDepth-- } + this.#matcherStarted = false if (this.#options.captureAssertions === false) { return } @@ -563,6 +573,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { private resetStack() { this.#commandStack = [] this.#assertionDepth = 0 + this.#matcherStarted = false this.#assertionStartTimes = [] this.#assertionCallSources = [] this.#sessionCapturer.resetLastSelector() @@ -586,6 +597,30 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } + /** Maintain the expect-matcher suppression window for an incoming command. + * `inMatcher` marks the matcher as started (so its trailing internal + * commands stay suppressed); otherwise self-heal a depth left stuck by a + * matcher that hard-threw (skipping afterAssertion) — but only once the + * matcher has actually started, so its own leading element re-lookup (no + * matcher frame on the sync stack yet) can't reset the depth mid-run. */ + #trackAssertionWindow(inMatcher: boolean, hasSource: boolean): void { + if (inMatcher) { + this.#matcherStarted = true + return + } + if ( + hasSource && + this.#commandStack.length === 0 && + this.#assertionDepth > 0 && + this.#matcherStarted + ) { + this.#assertionDepth = 0 + this.#assertionStartTimes = [] + this.#assertionCallSources = [] + this.#matcherStarted = false + } + } + async beforeCommand(command: string, args: string[]) { if (!this.#browser) { return @@ -606,25 +641,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { Error.stackTraceLimit = 20 const stack = parse(new Error('')).reverse() const source = stack.find((frame) => isUserSpecFile(frame.getFileName())) - // #assertionDepth > 0 → we believe we're inside an expect matcher; its - // internal commands trace back to the user's expect() line but must not - // become their own rows (only the expect.<matcher> assertion should). - // expect-webdriverio doesn't wrap afterAssertion in try/finally, though, so - // a matcher whose internal command hard-throws skips it and leaves the depth - // stuck ≥1 — which would then suppress every later user command. When a - // top-level user command arrives with the depth stuck but no live - // expect-webdriverio frame on the stack, the matcher already ended: self-heal - // the leftover depth + parallel stacks so this command captures normally. - if (source && this.#commandStack.length === 0 && this.#assertionDepth > 0) { - const inMatcher = stack.some((frame) => - frame.getFileName()?.includes('expect-webdriverio') - ) - if (!inMatcher) { - this.#assertionDepth = 0 - this.#assertionStartTimes = [] - this.#assertionCallSources = [] - } - } + const inMatcher = + this.#assertionDepth > 0 && + stack.some((frame) => frame.getFileName()?.includes('expect-webdriverio')) + this.#trackAssertionWindow(inMatcher, Boolean(source)) if ( source && this.#commandStack.length === 0 && diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index 1809fa6c..a4f4a229 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -2,14 +2,26 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import type * as DevtoolsCore from '@wdio/devtools-core' import DevToolsHookService from '../src/index.js' -const fakeFrame = { - getFileName: () => '/test/specs/fake.spec.ts', - getLineNumber: () => 1, - getColumnNumber: () => 1 -} +// Controllable stack: `frames` defaults to a single user-spec frame (commands +// read as top-level). A test can splice in `matcherFrame` to simulate a command +// issued from inside an expect-webdriverio matcher. +const stackMock = vi.hoisted(() => { + const userFrame = { + getFileName: () => '/test/specs/fake.spec.ts', + getLineNumber: () => 1, + getColumnNumber: () => 1 + } + const matcherFrame = { + getFileName: () => + '/node_modules/expect-webdriverio/lib/matchers/toHaveText.js', + getLineNumber: () => 1, + getColumnNumber: () => 1 + } + return { frames: [userFrame], userFrame, matcherFrame } +}) // Create mock instance that will be returned by SessionCapturer constructor vi.mock('stack-trace', () => ({ - parse: () => [fakeFrame] + parse: () => stackMock.frames })) const mockSessionCapturerInstance = { afterCommand: vi.fn(), @@ -305,6 +317,7 @@ describe('DevtoolsService - assertion suppression self-heal', () => { beforeEach(async () => { vi.clearAllMocks() + stackMock.frames = [stackMock.userFrame] service = new DevToolsHookService() await service.before({} as any, [], mockBrowser) vi.clearAllMocks() @@ -314,18 +327,33 @@ describe('DevtoolsService - assertion suppression self-heal', () => { // whose internal command hard-throws runs beforeAssertion but skips // afterAssertion, leaving #assertionDepth stuck ≥ 1. Without the self-heal // that stuck depth suppresses every later user command in the test. - it('captures a top-level command after a matcher throw left the assertion depth stuck', async () => { - service.beforeAssertion() - service.beforeAssertion() // depth now 2, no matching afterAssertion - - // The mocked stack is a user-spec frame with no expect-webdriverio frame, - // so the command is recognised as top-level and the leftover depth resets. + it('captures a top-level command after a matcher that started then threw left the assertion depth stuck', async () => { + service.beforeAssertion() // depth 1 + // The matcher issues an internal command (stack shows an expect-webdriverio + // frame) → marks the matcher started, then hard-throws (afterAssertion + // skipped), leaving the depth stuck. + stackMock.frames = [stackMock.userFrame, stackMock.matcherFrame] + service.beforeCommand('getText' as any, ['#el']) + // A later top-level user command with no matcher frame self-heals the depth. + stackMock.frames = [stackMock.userFrame] service.beforeCommand('click' as any, ['.button']) await service.afterCommand('click' as any, ['.button'], undefined) expect(capturedCommands()).toEqual(['click']) }) + it('keeps suppressing the matcher’s own leading command (no premature self-heal)', async () => { + service.beforeAssertion() // depth 1 + // The matcher's FIRST internal command is an element re-lookup whose sync + // stack has no expect-webdriverio frame yet. It must NOT reset the depth — + // otherwise the subsequent matcher commands would surface as duplicate rows. + stackMock.frames = [stackMock.userFrame] + service.beforeCommand('$' as any, ['#el']) + await service.afterCommand('$' as any, ['#el'], undefined) + + expect(capturedCommands()).toEqual([]) + }) + it('a fresh beforeAssertion resets a stuck depth so the window stays balanced', async () => { service.beforeAssertion() service.beforeAssertion() // leftover depth from a prior throw From eef4d8f2ad65ffb236c5da63edee70fa0c33f4bc Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:41:16 +0530 Subject: [PATCH 44/91] fix(app): render assertions consistently across frameworks --- .../app/src/components/browser/snapshot.ts | 22 ++++++- .../workbench/actionItems/category.ts | 9 ++- .../workbench/actionItems/command.ts | 25 +++++++- .../components/workbench/compare/markers.ts | 8 +++ .../workbench/compare/renderDetailBlock.ts | 60 +++++++++++++++---- .../app/src/components/workbench/errors.ts | 19 ++++-- .../components/workbench/errors/collect.ts | 22 ++++++- 7 files changed, 138 insertions(+), 27 deletions(-) diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts index ce9ae709..9fe5f7b4 100644 --- a/packages/app/src/components/browser/snapshot.ts +++ b/packages/app/src/components/browser/snapshot.ts @@ -249,7 +249,7 @@ export class DevtoolsBrowser extends Element { } async #renderCommandScreenshot(command?: CommandLog) { - this.#screenshotData = command?.screenshot ?? null + this.#screenshotData = this.#screenshotForCommand(command) // Follow the selected command's page in the address bar — commands carry no // URL, so resolve it from the navigation active at the command's time. if (command) { @@ -466,6 +466,26 @@ export class DevtoolsBrowser extends Element { this.requestUpdate() } + /** Screenshot for the selected command. Assertions (and other snapshot-less + * commands) carry none, so fall back to the nearest PRECEDING command's frame + * — the page state the assertion observed — instead of a blank preview. */ + #screenshotForCommand(command?: CommandLog): string | null { + if (!command) { + return null + } + if (command.screenshot) { + return command.screenshot + } + const cmds = this.commands ?? [] + const idx = cmds.indexOf(command) + for (let i = (idx === -1 ? cmds.length : idx) - 1; i >= 0; i--) { + if (cmds[i].screenshot) { + return cmds[i].screenshot! + } + } + return null + } + /** Latest screenshot from any command — auto-updates the preview as tests run. */ get #latestAutoScreenshot(): string | null { if (!this.commands?.length) { diff --git a/packages/app/src/components/workbench/actionItems/category.ts b/packages/app/src/components/workbench/actionItems/category.ts index e40861a8..28b25db9 100644 --- a/packages/app/src/components/workbench/actionItems/category.ts +++ b/packages/app/src/components/workbench/actionItems/category.ts @@ -78,6 +78,11 @@ const QUERY = new Set([ 'findElements' ]) +/** assert.* (node:assert), expect.* (expect-webdriverio), verify.* (Nightwatch) + * — assertion rows across every framework, so they get the check icon + green + * like the WDIO state matchers above instead of the generic fallback. */ +const ASSERTION_PREFIX_RE = /^(?:assert|expect|verify)\./ + /** Group a command by intent so the timeline can colour it consistently. */ export function commandCategory(command: string): ActionCategory { if (NAVIGATION.has(command)) { @@ -86,7 +91,7 @@ export function commandCategory(command: string): ActionCategory { if (INPUT.has(command)) { return 'input' } - if (ASSERTION.has(command)) { + if (ASSERTION.has(command) || ASSERTION_PREFIX_RE.test(command)) { return 'assertion' } if (QUERY.has(command)) { @@ -133,7 +138,7 @@ export function commandIcon(command: string): ActionIcon { if (INPUT.has(command)) { return 'click' } - if (ASSERTION.has(command)) { + if (ASSERTION.has(command) || ASSERTION_PREFIX_RE.test(command)) { return 'assert' } if (QUERY.has(command)) { diff --git a/packages/app/src/components/workbench/actionItems/command.ts b/packages/app/src/components/workbench/actionItems/command.ts index 1a596f56..02631b0d 100644 --- a/packages/app/src/components/workbench/actionItems/command.ts +++ b/packages/app/src/components/workbench/actionItems/command.ts @@ -15,11 +15,20 @@ import '~icons/mdi/target.js' import '~icons/mdi/keyboard-outline.js' import '~icons/mdi/cursor-default-click-outline.js' import '~icons/mdi/check-circle-outline.js' +import '~icons/mdi/close-circle-outline.js' import '~icons/mdi/text.js' import '~icons/mdi/code-tags.js' const SOURCE_COMPONENT = 'wdio-devtools-command-item' +function capitalizeAssertLabel(label: string): string { + return label.replace( + /^(assert|expect|verify)\./, + (_m, prefix: string) => + prefix.charAt(0).toUpperCase() + prefix.slice(1) + '.' + ) +} + const CATEGORY_COLOR: Record<ActionCategory, string> = { navigation: 'text-chartsBlue', input: 'text-chartsPurple', @@ -48,8 +57,18 @@ export class CommandItem extends ActionItem { } #renderIcon(command: string): TemplateResult { - const cls = `${ICON_CLASS} ${CATEGORY_COLOR[commandCategory(command)]}` - switch (commandIcon(command)) { + // Failed commands render red (matching the label); a failed assertion also + // swaps its green ✓-circle for a red ✗-circle so it never reads as passed. + const cls = `${ICON_CLASS} ${ + this.failed ? 'text-chartsRed' : CATEGORY_COLOR[commandCategory(command)] + }` + const icon = commandIcon(command) + if (this.failed && icon === 'assert') { + return html`<icon-mdi-close-circle-outline + class="${cls}" + ></icon-mdi-close-circle-outline>` + } + switch (icon) { case 'navigate': return html`<icon-mdi-arrow-top-right class="${cls}" @@ -93,7 +112,7 @@ export class CommandItem extends ActionItem { class="text-[12.5px] flex-wrap text-left break-all ${this.failed ? 'text-chartsRed' : ''}" - >${entry.title ?? entry.command}</code + >${capitalizeAssertLabel(entry.title ?? entry.command)}</code > ${this.renderTime()} </button> diff --git a/packages/app/src/components/workbench/compare/markers.ts b/packages/app/src/components/workbench/compare/markers.ts index ccec430b..18945b4f 100644 --- a/packages/app/src/components/workbench/compare/markers.ts +++ b/packages/app/src/components/workbench/compare/markers.ts @@ -75,6 +75,14 @@ function renderStatusMarker( >✗ in failed step</span >` } + // A command that itself errored is a failure even when its step state didn't + // resolve — e.g. the rerun/latest side, whose live step state isn't tracked + // the way the baseline's PreservedStep is. Without this it shows a green ✓. + if (cmd.error?.message) { + return html`<span class="marker error" title="Failed: ${cmd.error.message}" + >✗ failed</span + >` + } if (step?.state === 'passed') { return html`<span class="marker ok" diff --git a/packages/app/src/components/workbench/compare/renderDetailBlock.ts b/packages/app/src/components/workbench/compare/renderDetailBlock.ts index 1c4c03ed..81dfcd80 100644 --- a/packages/app/src/components/workbench/compare/renderDetailBlock.ts +++ b/packages/app/src/components/workbench/compare/renderDetailBlock.ts @@ -10,9 +10,35 @@ import type { PreservedAttempt, PreservedStep } from '@wdio/devtools-shared' -import { safeJson } from './compareUtils.js' +import { cleanErrorMessage, safeJson } from './compareUtils.js' import { computeDetailBlockData } from './stepResolution.js' +/** Assertion commands carry `[actual, expected]` in args (mirrors the Errors + * tab). Surfacing those instead of the raw node:assert message keeps the + * detail free of the "+ actual - expected" diff noise + ANSI. */ +const ASSERTION_CMD_RE = /^(?:assert|expect|verify)\./ + +function assertionArgs( + cmd: CommandLog +): { actual: unknown; expected: unknown } | undefined { + if (!ASSERTION_CMD_RE.test(cmd.command)) { + return undefined + } + // Prefer a collapsed { actual, expected } result (Nightwatch native asserts) + // over positional args, which mislabel a single expected-only arg as actual. + const result = cmd.result + if (result && typeof result === 'object') { + const r = result as { actual?: unknown; expected?: unknown } + if ('actual' in r || 'expected' in r) { + return { actual: r.actual, expected: r.expected } + } + } + if ((cmd.args?.length ?? 0) < 2) { + return undefined + } + return { actual: cmd.args![0], expected: cmd.args![1] } +} + /** Hooks the detail-block renderers need to reach component state. */ export interface DetailBlockCtx { baseline: PreservedAttempt | undefined @@ -102,22 +128,32 @@ export function renderDetailBlock( ctx.findStepFor(cmd, side), allCmdsThisSide ) + const assertVals = assertionArgs(cmd) return html` <div class="detail-block"> <h4>${label} · ${cmd.command}</h4> ${renderDetailStepBanner(data.step, data.stepText)} <pre>args: ${data.argsStr}</pre> - ${cmd.error - ? html`<pre style="color:var(--vscode-charts-red,#f48771);"> -error: ${cmd.error.message || String(cmd.error)}</pre - >` - : html`<pre>result: ${data.resultStr}</pre>`} - ${renderExpectedActualAssertion( - data.expected, - data.actual, - data.assertionMessage, - data.fallbackExpected - )} + ${assertVals + ? renderExpectedActualAssertion( + assertVals.expected, + assertVals.actual, + undefined, + undefined + ) + : html` + ${cmd.error + ? html`<pre style="color:var(--vscode-charts-red,#f48771);"> +error: ${cleanErrorMessage(cmd.error.message || String(cmd.error))}</pre + >` + : html`<pre>result: ${data.resultStr}</pre>`} + ${renderExpectedActualAssertion( + data.expected, + data.actual, + data.assertionMessage, + data.fallbackExpected + )} + `} ${cmd.screenshot ? html`<img src="${cmd.screenshot.startsWith('data:') diff --git a/packages/app/src/components/workbench/errors.ts b/packages/app/src/components/workbench/errors.ts index f4470005..ef497850 100644 --- a/packages/app/src/components/workbench/errors.ts +++ b/packages/app/src/components/workbench/errors.ts @@ -16,6 +16,11 @@ function shortSource(callSource: string): string { return callSource.split(/[\\/]/).slice(-3).join('/') } +/** Show assertion values readably: quote strings, stringify everything else. */ +function fmtValue(value: unknown): string { + return typeof value === 'string' ? `'${value}'` : String(value) +} + @customElement(COMPONENT) export class DevtoolsErrors extends Element { @consume({ context: commandContext, subscribe: true }) @@ -155,13 +160,13 @@ export class DevtoolsErrors extends Element { return nothing } return html`<div class="error-diff"> + ${error.actual !== undefined + ? html`<span class="label">Actual</span + ><span class="received">${fmtValue(error.actual)}</span>` + : nothing} ${error.expected !== undefined ? html`<span class="label">Expected</span - ><span class="expected">${error.expected}</span>` - : nothing} - ${error.actual !== undefined - ? html`<span class="label">Received</span - ><span class="received">${error.actual}</span>` + ><span class="expected">${fmtValue(error.expected)}</span>` : nothing} </div>` } @@ -178,7 +183,9 @@ export class DevtoolsErrors extends Element { @${shortSource(error.callSource)} </button>` : nothing} - <div class="error-title">${error.message}</div> + ${error.expected === undefined && error.actual === undefined + ? html`<div class="error-title">${error.message}</div>` + : nothing} ${this.#renderDiff(error)} ${error.stack ? html`<details class="error-stack"> diff --git a/packages/app/src/components/workbench/errors/collect.ts b/packages/app/src/components/workbench/errors/collect.ts index ca7429e1..f28d8065 100644 --- a/packages/app/src/components/workbench/errors/collect.ts +++ b/packages/app/src/components/workbench/errors/collect.ts @@ -50,13 +50,29 @@ function displayValue(value: unknown): string | undefined { } } -/** Assertion commands carry `[actual, expected]` in args (see the exporter's - * Assert params + synthesizeExpectFailure). */ +/** Actual/expected for an assertion row. Prefer a collapsed `{actual, expected}` + * result (Nightwatch native asserts, and any framework that surfaces a + * structured result) over the positional `[actual, expected]` arg convention — + * the latter is wrong for asserts that pass only an expected value + * (`titleContains('x')`), which would mislabel the expected as the actual. */ function assertionValues(command: CommandLog): { actual?: string expected?: string } { - if (!ASSERTION_COMMAND_RE.test(command.command) || command.args.length < 2) { + if (!ASSERTION_COMMAND_RE.test(command.command)) { + return {} + } + const result = command.result + if (result && typeof result === 'object') { + const r = result as { actual?: unknown; expected?: unknown } + if ('actual' in r || 'expected' in r) { + return { + actual: displayValue(r.actual), + expected: displayValue(r.expected) + } + } + } + if (command.args.length < 2) { return {} } return { From 47637f7d9ea209451d17478c1163a2d3f25891ea Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:41:31 +0530 Subject: [PATCH 45/91] fix(backend): escape test names in grep-style rerun filters --- packages/backend/src/runner.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/runner.ts b/packages/backend/src/runner.ts index da1104b5..95b3a14f 100644 --- a/packages/backend/src/runner.ts +++ b/packages/backend/src/runner.ts @@ -40,6 +40,14 @@ function stripNameTestNameSlot(template: string): string { return template.slice(0, leftEdge) + template.slice(idx + NAME_SLOT.length) } +// Grep-style rerun filters (mocha --grep, jest/vitest -t, cucumber --name) treat +// the value as a REGEX, so a test name containing (), [], ., etc. matches +// nothing unless escaped. The slot is double-quoted in the template, so the +// added backslashes survive shell parsing and reach the runner as literals. +function escapeRerunName(name: string): string { + return name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + class TestRunner { #child?: ChildProcess #lastPayload?: RunnerRequestBody @@ -203,7 +211,7 @@ class TestRunner { return `${stripped} ${shellQuote([featureSpec])}` } const name = payload.label || payload.fullTitle || '' - return template.replace(/\{\{testName\}\}/g, name) + return template.replace(/\{\{testName\}\}/g, escapeRerunName(name)) } #parseGenericCommand(command: string): { file: string; args: string[] } { From 527fa7c8ec1d10a5fc3b745082a7d3c0713b1351 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:41:45 +0530 Subject: [PATCH 46/91] fix(selenium): read snapshot state via unpatched driver; forward tracePolicy --- .../selenium-devtools/src/action-snapshot.ts | 30 +++++++++++-------- .../selenium-devtools/src/driverPatcher.ts | 10 +++++++ packages/selenium-devtools/src/index.ts | 5 ++++ packages/selenium-devtools/src/types.ts | 2 ++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/packages/selenium-devtools/src/action-snapshot.ts b/packages/selenium-devtools/src/action-snapshot.ts index 7644ae1d..83a81d3e 100644 --- a/packages/selenium-devtools/src/action-snapshot.ts +++ b/packages/selenium-devtools/src/action-snapshot.ts @@ -1,33 +1,37 @@ // Selenium adapter: wires SeleniumDriverLike into core's captureActionSnapshot. +// URL/title/screenshot/script are read through the UNPATCHED driver originals +// (getDriverOriginals) so the snapshot's own reads don't record as commands. +// getCurrentUrl/getTitle map to page.* actions, so capturing them would make +// every snapshot trigger another snapshot — a feedback loop that bloats the +// trace (observed: thousands of getUrl/getTitle actions in one run). import { captureActionSnapshot as coreCapture } from '@wdio/devtools-core' import type { ActionSnapshot } from '@wdio/devtools-shared' +import { getDriverOriginals } from './driverPatcher.js' import type { SeleniumDriverLike } from './types.js' -interface DriverWithUrl extends SeleniumDriverLike { - getCurrentUrl?: () => Promise<string> - getTitle?: () => Promise<string> -} - export function captureActionSnapshot( driver: SeleniumDriverLike, command: string ): Promise<ActionSnapshot | null> { - const d = driver as DriverWithUrl + const orig = getDriverOriginals() return coreCapture({ command, - runScript: (src) => driver.executeScript(`return (${src})`), + runScript: (src) => + orig.executeScript + ? orig.executeScript(driver, `return (${src})`) + : driver.executeScript(`return (${src})`), takeScreenshot: () => - d.takeScreenshot - ? d.takeScreenshot().catch(() => undefined) + orig.takeScreenshot + ? orig.takeScreenshot(driver).catch(() => undefined) : Promise.resolve(undefined), getUrl: () => - d.getCurrentUrl - ? d.getCurrentUrl().catch(() => undefined) + orig.getCurrentUrl + ? orig.getCurrentUrl(driver).catch(() => undefined) : Promise.resolve(undefined), getTitle: () => - d.getTitle - ? d.getTitle().catch(() => undefined) + orig.getTitle + ? orig.getTitle(driver).catch(() => undefined) : Promise.resolve(undefined) }) } diff --git a/packages/selenium-devtools/src/driverPatcher.ts b/packages/selenium-devtools/src/driverPatcher.ts index d4b4a1c5..d1aab68f 100644 --- a/packages/selenium-devtools/src/driverPatcher.ts +++ b/packages/selenium-devtools/src/driverPatcher.ts @@ -217,6 +217,16 @@ function stashDriverOriginals(driverProto: Patchable): void { originals.manage = (driver) => orig.call(driver) as ReturnType<typeof orig> & object } + const url = driverProto.getCurrentUrl + if (typeof url === 'function') { + const orig = url as (this: unknown) => unknown + originals.getCurrentUrl = (driver) => orig.call(driver) as Promise<string> + } + const title = driverProto.getTitle + if (typeof title === 'function') { + const orig = title as (this: unknown) => unknown + originals.getTitle = (driver) => orig.call(driver) as Promise<string> + } } // Lets onBeforeQuit flush async cleanup before runners that `process.exit()` diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 7289cd81..4c4b2406 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -64,6 +64,7 @@ import { type ScreencastOptions, type TraceFormat, type TraceGranularity, + type TraceRetentionPolicy, type SeleniumDriverLike, type TestStats } from './types.js' @@ -277,6 +278,7 @@ class SeleniumDevToolsPlugin { mode?: DevToolsMode traceFormat?: TraceFormat traceGranularity?: TraceGranularity + tracePolicy?: TraceRetentionPolicy } = {} ) { if ('rerunCommand' in opts) { @@ -301,6 +303,9 @@ class SeleniumDevToolsPlugin { if (opts.traceGranularity) { this.#options.traceGranularity = opts.traceGranularity } + if (opts.tracePolicy) { + this.#options.tracePolicy = opts.tracePolicy + } if (opts.screencast) { if (this.#options.mode === 'trace' && opts.screencast.enabled) { log.warn('trace mode: ignoring screencast option (live-mode feature)') diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index af764d69..e7aceb8f 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -101,6 +101,8 @@ export interface DriverOriginals { ...args: unknown[] ) => Promise<unknown> manage?: (driver: SeleniumDriverLike) => unknown + getCurrentUrl?: (driver: SeleniumDriverLike) => Promise<string> + getTitle?: (driver: SeleniumDriverLike) => Promise<string> } // Unwrapped WebElement methods for internal enrichment paths. From 5c46fee9cdbd4f1a1397dd57197f3f6e3a20194c Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:41:59 +0530 Subject: [PATCH 47/91] chore(selenium): bump chromedriver to ^150 for Chrome 150 --- packages/selenium-devtools/package.json | 2 +- pnpm-lock.yaml | 22 ++-------------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 62b88526..8368e44d 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -65,7 +65,7 @@ "@types/ws": "^8.18.1", "@wdio/devtools-core": "workspace:^", "@wdio/devtools-shared": "workspace:^", - "chromedriver": "^148.0.4", + "chromedriver": "^150.0.0", "jest": "^30.4.2", "mocha": "^11.7.6", "selenium-webdriver": "^4.44.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 528cbb11..a1de3557 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -499,8 +499,8 @@ importers: specifier: workspace:^ version: link:../shared chromedriver: - specifier: ^148.0.4 - version: 148.0.4 + specifier: ^150.0.0 + version: 150.0.1 jest: specifier: ^30.4.2 version: 30.4.2(@types/node@25.9.3)(ts-node@10.9.2(@types/node@25.9.3)(typescript@6.0.3)) @@ -3333,11 +3333,6 @@ packages: engines: {node: '>=12.13.0'} hasBin: true - chromedriver@148.0.4: - resolution: {integrity: sha512-3UyptFDG4YF1Pyv3fzn95s1CN4K3zCpHSmE6g+6J4f2u9KxxOYzrwN2GApVyM2z02hlbSqzo9Ajn2hMi7LnvCw==} - engines: {node: '>=22'} - hasBin: true - chromedriver@150.0.1: resolution: {integrity: sha512-vpYskeQFyZGPNMEwc/5fuvfnxzfjljCO1EvwuF0dZGQ8KE0repYQar7eThTlT1IjznwVrirYRgXrRsJts7DRIQ==} engines: {node: '>=22'} @@ -10648,19 +10643,6 @@ snapshots: transitivePeerDependencies: - supports-color - chromedriver@148.0.4: - dependencies: - '@testim/chrome-version': 1.1.4 - adm-zip: 0.5.17 - axios: 1.16.1 - compare-versions: 6.1.1 - proxy-agent: 8.0.1 - proxy-from-env: 2.1.0 - tcp-port-used: 1.0.2 - transitivePeerDependencies: - - debug - - supports-color - chromedriver@150.0.1: dependencies: '@testim/chrome-version': 1.1.4 From 609ff1dcb5ce6fd0339d8a04dde3c4865984a376 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:42:17 +0530 Subject: [PATCH 48/91] chore(examples): live/trace config-ladder harnesses + retry scripts --- examples/nightwatch/nightwatch.conf.cjs | 22 +++++---- .../nightwatch/tests/assert-capture-check.js | 18 ------- examples/nightwatch/tests/smoke-test.js | 48 +++++++++++++++++++ examples/selenium/mocha-test/test/example.js | 40 +++++++++++++++- examples/wdio/cucumber/wdio.conf.ts | 2 +- examples/wdio/mocha/wdio.conf.ts | 11 +++-- package.json | 1 + packages/nightwatch-devtools/package.json | 1 + 8 files changed, 110 insertions(+), 33 deletions(-) delete mode 100644 examples/nightwatch/tests/assert-capture-check.js create mode 100644 examples/nightwatch/tests/smoke-test.js diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index 2a969097..3ec31930 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -37,16 +37,22 @@ module.exports = { }, 'goog:loggingPrefs': { performance: 'ALL' } }, - // Simple configuration - just call the function to get globals. - // - screencast: polling-mode .webm written to cwd as - // nightwatch-video-<sessionId>.webm. - // - bidi: opt-in WebDriver BiDi capture for console + network. When - // attached, the per-command Chrome perf-log network path is gated - // off to avoid duplicate entries. + // bidi: opt-in WebDriver BiDi capture for console + network. When + // attached, the per-command Chrome perf-log network path is gated off to + // avoid duplicate entries. globals: nightwatchDevtools({ port: 3000, - mode: 'live', - screencast: { enabled: true, pollIntervalMs: 200 }, + // ── Config ladder — change ONLY this block per rung ─────────────── + // 1 live: mode: 'live' + // 2 trace: mode: 'trace' + // 3 per-test: mode: 'trace', traceGranularity: 'test' + // 4 fail: mode: 'trace', traceGranularity: 'test', tracePolicy: 'retain-on-failure' + // 5 retry: mode: 'trace', traceGranularity: 'test', tracePolicy: 'on-first-retry' + // (rung 5 needs retries → run `pnpm demo:nightwatch:retry`) + // NOTE: the BDD describe/it interface fires the plugin's beforeEach once + // per module (no per-`it` hook), so traceGranularity:'test' collapses to + // a single session-scoped slice here. See CLAUDE.md § Known debt. + mode: 'trace', bidi: true }) } diff --git a/examples/nightwatch/tests/assert-capture-check.js b/examples/nightwatch/tests/assert-capture-check.js deleted file mode 100644 index 6f8511a3..00000000 --- a/examples/nightwatch/tests/assert-capture-check.js +++ /dev/null @@ -1,18 +0,0 @@ -// Verification harness for native-assert trace capture (browser.assert / verify). -// Run `pnpm demo:nightwatch` and inspect the dashboard Actions: the PASSING -// asserts must render green and the FAILING ones RED. If a failing assert shows -// green, the classic-chained capture is mis-reporting failures (the known risk -// in nativeAssertCapture — the wrapper sees the enqueue, not the queued result). -describe('Native assert capture check', function () { - it('renders passing and failing native asserts', async function (browser) { - await browser.url('https://example.com').waitForElementVisible('body', 5000) - - // Soft verify.* first — never aborts the test, so all four always run. - browser.verify.titleContains('Example') // PASS → expect green - browser.verify.titleContains('SOFT_FAIL_ME') // FAIL → expect RED - - // Hard assert.* — the classic-chained/queued path under test. - browser.assert.titleContains('Example') // PASS → expect green - browser.assert.titleContains('HARD_FAIL_ME') // FAIL → expect RED - }) -}) diff --git a/examples/nightwatch/tests/smoke-test.js b/examples/nightwatch/tests/smoke-test.js new file mode 100644 index 00000000..3cdca005 --- /dev/null +++ b/examples/nightwatch/tests/smoke-test.js @@ -0,0 +1,48 @@ +/** + * Example + config-sweep harness for @wdio/nightwatch-devtools. + * + * Walk the live/trace ladder by editing ONLY the mode/traceGranularity/ + * tracePolicy block in ../nightwatch.conf.cjs. The suite carries a passing + * pair, an always-failing test (retain-on-failure target), and a flaky + * fail-then-pass test (on-first-retry / attempt-capture target). + * + * Native asserts (browser.assert.*) double as the assertion-capture check: + * the passing ones must render green ✓, the failing one red ✗. + * + * Run from repo root: + * pnpm demo:nightwatch (rungs 1-4) + * pnpm demo:nightwatch:retry (rung 5 — adds --retries 1) + */ + +// Survives Nightwatch's testcase retry so the flaky test fails once, then passes. +let flakyAttempts = 0 + +describe('nightwatch-devtools smoke test', function () { + it('loads example.com and reads the heading', async function (browser) { + await browser.url('https://example.com') + await browser.waitForElementVisible('body', 5000) + browser.assert.titleContains('Example') + }) + + it('navigates and reads the page title', async function (browser) { + await browser.url('https://example.org') + await browser.waitForElementVisible('body', 5000) + browser.assert.titleContains('Example') + }) + + it('fails on a wrong title (retain-on-failure target)', async function (browser) { + await browser.url('https://example.com') + await browser.waitForElementVisible('body', 5000) + browser.assert.titleContains('This Is Not The Title') + }) + + it('flaky: fails the first attempt, then passes (retry target)', async function (browser) { + await browser.url('https://example.com') + await browser.waitForElementVisible('body', 5000) + flakyAttempts += 1 + if (flakyAttempts === 1) { + throw new Error('intentional first-attempt failure — should retry') + } + browser.assert.titleContains('Example') + }) +}) diff --git a/examples/selenium/mocha-test/test/example.js b/examples/selenium/mocha-test/test/example.js index be2a79e4..26360598 100644 --- a/examples/selenium/mocha-test/test/example.js +++ b/examples/selenium/mocha-test/test/example.js @@ -1,5 +1,10 @@ /** - * Smoke test for @wdio/selenium-devtools. + * Example + config-sweep harness for @wdio/selenium-devtools (Mocha runner). + * + * Walk the live/trace ladder by editing ONLY the DevTools.configure({...}) + * block below. The suite carries a passing pair, an always-failing test (for + * retain-on-failure), and a flaky fail-then-pass test with a per-test retry + * (for on-first-retry / attempt capture). * * Run from the package root: pnpm example:mocha */ @@ -8,11 +13,22 @@ import { strict as assert } from 'node:assert' import { Builder, By, until } from 'selenium-webdriver' import { DevTools } from '@wdio/selenium-devtools' +// ── Config ladder — change this block per rung ────────────────────────────── +// 1 live: { mode: 'live' } +// 2 trace: { mode: 'trace' } +// 3 per-test:{ mode: 'trace', traceGranularity: 'test' } +// 4 fail: { mode: 'trace', traceGranularity: 'test', tracePolicy: 'retain-on-failure' } +// 5 retry: { mode: 'trace', traceGranularity: 'test', tracePolicy: 'on-first-retry' } DevTools.configure({ - screencast: { enabled: true, quality: 70, maxWidth: 1280, maxHeight: 720 }, + mode: 'trace', + // traceGranularity: 'test', + // tracePolicy: 'on-first-retry', headless: true }) +// Survives Mocha's in-process retry so the flaky test fails once, then passes. +let flakyAttempts = 0 + describe('selenium-devtools smoke test', function () { let driver @@ -40,4 +56,24 @@ describe('selenium-devtools smoke test', function () { const title = await driver.getTitle() assert.match(title, /Example/i) }) + + it('fails on a wrong heading (retain-on-failure target)', async function () { + await driver.get('https://example.com') + await driver.sleep(1000) + const heading = await driver.wait(until.elementLocated(By.css('h1')), 10000) + const text = await heading.getText() + assert.equal(text, 'This Is Not The Heading') + }) + + it('flaky: fails the first attempt, then passes (retry target)', async function () { + this.retries(1) + await driver.get('https://example.com') + await driver.sleep(1000) + flakyAttempts += 1 + if (flakyAttempts === 1) { + throw new Error('intentional first-attempt failure — should retry') + } + const title = await driver.getTitle() + assert.match(title, /Example/i) + }) }) diff --git a/examples/wdio/cucumber/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts index 4c535acc..e1a82294 100644 --- a/examples/wdio/cucumber/wdio.conf.ts +++ b/examples/wdio/cucumber/wdio.conf.ts @@ -133,7 +133,7 @@ export const config: Options.Testrunner = { [ 'devtools', { - mode: 'live' as const, + mode: 'trace' as const, screencast: { enabled: true, pollIntervalMs: 200 } } ] diff --git a/examples/wdio/mocha/wdio.conf.ts b/examples/wdio/mocha/wdio.conf.ts index 966226f9..c7680f7d 100644 --- a/examples/wdio/mocha/wdio.conf.ts +++ b/examples/wdio/mocha/wdio.conf.ts @@ -39,10 +39,13 @@ export const config: Options.Testrunner = { [ 'devtools', { - mode: 'live' as const, - traceGranularity: 'spec' as const - // tracePolicy: 'retain-on-failure' as const - // screencast: { enabled: true, pollIntervalMs: 200 } + // ── Config ladder — change ONLY this block per rung ────────────── + // 1 live: mode: 'live' + // 2 trace: mode: 'trace' + // 3 per-test: mode: 'trace', traceGranularity: 'test' + // 4 fail: mode: 'trace', traceGranularity: 'test', tracePolicy: 'retain-on-failure' + // 5 retry: use `pnpm demo:wdio:retry` (adds retries:1 + on-first-retry) + mode: 'live' as const } ] ], diff --git a/package.json b/package.json index 9c120190..3e3f2b75 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "demo:wdio:mocha": "wdio run ./examples/wdio/mocha/wdio.conf.ts", "demo:wdio:retry": "wdio run ./examples/wdio/mocha/wdio.retry.conf.ts", "demo:nightwatch": "pnpm --filter @wdio/nightwatch-devtools example", + "demo:nightwatch:retry": "pnpm --filter @wdio/nightwatch-devtools example:retry", "demo:selenium": "pnpm --filter @wdio/selenium-devtools example", "dev": "pnpm --parallel dev", "preview": "pnpm --parallel preview", diff --git a/packages/nightwatch-devtools/package.json b/packages/nightwatch-devtools/package.json index 526a6ffd..36874061 100644 --- a/packages/nightwatch-devtools/package.json +++ b/packages/nightwatch-devtools/package.json @@ -34,6 +34,7 @@ "clean": "rm -rf dist", "lint": "eslint .", "example": "nightwatch -c ../../examples/nightwatch/nightwatch.conf.cjs", + "example:retry": "nightwatch -c ../../examples/nightwatch/nightwatch.conf.cjs --retries 1", "prepublishOnly": "pnpm build" }, "keywords": [ From 530018ead2b6cf3572360d49a3e12aabc7d8a5b1 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 18:42:32 +0530 Subject: [PATCH 49/91] docs: record Nightwatch BDD per-test and WDIO assert-capture debt --- CLAUDE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 14fff1a0..1f33103e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -238,8 +238,11 @@ Documented divergences from the conventions above. They exist today as debt to b - Retry-aware trace policies aren't uniform across adapters. Selenium emits one metadata entry per attempt (counter-based `generateStableUid` uid + retry-stable tracker key), so all six policies — including `retain-on-first-failure` (needs the first attempt's outcome) — evaluate correctly. Service and Nightwatch key metadata on the retry-stable uid and overwrite/mutate per attempt, collapsing to the *final* attempt's outcome; the retry-count policies (`retain-on-failure`, `on-first-retry`, `on-all-retries`, `retain-on-failure-and-retries`) are still correct, but `retain-on-first-failure` can't see a failed-then-passed first attempt and under-retains. Proper fix rides with B5 (per-test granularity preserves per-attempt slices); until then all three set `attemptInfoAvailable: true`. - Nightwatch negated native asserts (`browser.assert.not.*` / `verify.not.*`) produce no trace row: the `not` namespace is a nested Proxy that `wrapAssertionNamespaces` passes through untouched, and the internal command it issues is dropped by the `hasUserSource` gate in `browserProxy`. Positive `assert.*`/`verify.*` still record. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. -- Nightwatch native asserts populate all at once in live mode and finalize pass/fail in one test-end batch: Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin (`client.queue.tree` / `client.reporter` aren't on `browser`), so call-time capture streams neutral pending rows and `afterEach` reconciles outcomes. Trace-timeline positions are corrected from `results.commands`. -- Service assertion-suppression self-heal is command-triggered: a swallowed soft-assert (`expect.soft` / user `try/catch`) whose matcher hard-throws leaves `#assertionDepth` stuck until the next top-level user command's stack shows no `expect-webdriverio` frame (heal) or the next `beforeAssertion` (reset). A stuck depth with no following command or assertion before test end persists until the next test's `resetStack()` — benign (no later command in that test to suppress). +- Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. +- Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. +- Service assertion-suppression self-heal is command-triggered and gated on `#matcherStarted`: a swallowed soft-assert (`expect.soft` / user `try/catch`) whose matcher hard-throws leaves `#assertionDepth` stuck; the next top-level user command with no `expect-webdriverio` frame heals it — but only once the matcher has issued at least one internal (ewdio-framed) command, because the matcher's own FIRST command (an element re-lookup) also lacks a matcher frame on its sync stack and would otherwise reset the depth mid-run (duplicating every later matcher command). A stuck depth with no following command before test end persists until the next `beforeAssertion`/`resetStack()` — benign. +- Service captures the assertion **target's** element resolution as its own row. `expect(page.el).toHaveText(x)` evaluates `page.el` → `$('#sel')`/`$$` as the ARGUMENT to `expect()`, before `beforeAssertion` fires, so `#assertionDepth` is 0 and the `$`/`$$`/`isExisting` lands as a top-level command alongside the `expect.*` row. The in-window suppression can't reach it (it runs pre-window). Proper fix: coalesce a trailing element-query into the imminent assertion, or detect query-commands whose only consumer is the next matcher. +- Service `expect.*` rows can resolve `callSource` to the service bundle (`dist/index.js`) instead of the user's `expect()` line. `#assertionCallSources` captures the spec frame in `beforeAssertion`, but the matcher runs async, so in some cases the parallel stack doesn't hold a user frame and the fallback resolves into the service dist. The failure's own error stack still points at the spec. ### File-size (raw line counts; soft cap is 500 logic lines) From 7fdc49917d0dd66696a438c716243c6accd6c7e6 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 19:28:07 +0530 Subject: [PATCH 50/91] refactor: dedupe assert-result type into shared; drop unused trace defaults --- packages/core/src/trace-action-events.ts | 18 +++++------ .../src/helpers/nativeAssertions.ts | 9 +----- packages/nightwatch-devtools/src/types.ts | 1 + packages/shared/src/types.ts | 30 ++++++++----------- 4 files changed, 23 insertions(+), 35 deletions(-) diff --git a/packages/core/src/trace-action-events.ts b/packages/core/src/trace-action-events.ts index 29eeb0b2..114233d7 100644 --- a/packages/core/src/trace-action-events.ts +++ b/packages/core/src/trace-action-events.ts @@ -1,7 +1,11 @@ // Builds the before/after action events of the exported trace stream, // including tracingGroup test boundaries and frame-snapshot ref stamping. -import type { CommandLog, TestMetadataMap } from '@wdio/devtools-shared' +import type { + CollapsedAssertResult, + CommandLog, + TestMetadataMap +} from '@wdio/devtools-shared' import { ASSERT_ACTION_CLASS, formatActionTitle, @@ -75,15 +79,9 @@ interface ActionStream { groupCallId?: string } -// Nightwatch built-in assertions collapse {passed, actual, expected, message} -// into the command result on failure — surface those over positional args. -interface CollapsedAssertResult { - passed?: unknown - actual?: unknown - expected?: unknown - message?: unknown -} - +// An adapter may attach a normalized CollapsedAssertResult (see shared) to an +// assertion command — prefer its actual/expected over the positional args, +// which are only correct for node:assert-style `[actual, expected]` calls. function collapsedAssertResult( result: unknown ): CollapsedAssertResult | undefined { diff --git a/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts index 1134047d..b86beba2 100644 --- a/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts +++ b/packages/nightwatch-devtools/src/helpers/nativeAssertions.ts @@ -26,6 +26,7 @@ import { } from '@wdio/devtools-core' import type { SessionCapturer } from '../session.js' import type { + CollapsedAssertResult, CommandLog, NativeAssertCall, NightwatchBrowser, @@ -242,14 +243,6 @@ export function pendingAssertionCommand( return entry } -/** Collapsed pass/fail result core's `collapsedAssertResult` reads. */ -interface CollapsedAssertResult { - passed: boolean - expected?: unknown - actual?: unknown - message?: string -} - /** Nightwatch failure messages end with `… but got: "<actual>"`. Pull out the * real observed value: Nightwatch passes only the EXPECTED as an arg, so the * actual lives in the message (and only on failure). Undefined when absent. diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index 203dbbdb..d4c3f01e 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -3,6 +3,7 @@ export { TraceType, type ActionSnapshot, + type CollapsedAssertResult, type CommandLog, type ConsoleLog, type DevToolsMode, diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index f90db146..1e2c7033 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -49,10 +49,6 @@ export type TraceRetentionPolicy = | 'on-all-retries' | 'retain-on-failure-and-retries' -/** Video retention — video has no separate mode switch, so `off` is valid - * (and the default). */ -export type TraceVideoPolicy = 'off' | TraceRetentionPolicy - /** One node in a test's ancestor chain, outermost first. */ export interface TestAncestor { uid: string @@ -73,19 +69,19 @@ export interface TestMetadataEntry { * title + specFile for Tracing.tracingGroup events in trace output. */ export type TestMetadataMap = Map<string, TestMetadataEntry> -/** Defaults for trace-mode options when not specified by the user. */ -export const TRACE_DEFAULTS = { - mode: 'live', - traceFormat: 'zip', - traceGranularity: 'session', - tracePolicy: 'on', - video: 'off' -} as const satisfies { - mode: DevToolsMode - traceFormat: TraceFormat - traceGranularity: TraceGranularity - tracePolicy: TraceRetentionPolicy - video: TraceVideoPolicy +/** + * Normalized assertion result an adapter may attach to `CommandLog.result` for + * an assertion command. The trace exporter's assert-param builder prefers this + * over the positional `[actual, expected]` arg convention — correct for + * frameworks whose asserts pass only an expected value (a matcher like + * `titleContains('x')`), where args[0] is the expected, not the actual. + * Cross-package contract: adapters produce it, core's exporter consumes it. + */ +export interface CollapsedAssertResult { + passed: boolean + actual?: unknown + expected?: unknown + message?: string } /** From 4a80d00627ab256742e0222d750a356e8d7348b3 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 13 Jul 2026 19:28:31 +0530 Subject: [PATCH 51/91] refactor(core): one findFlushableRange for the three adapters' slice flush --- packages/core/src/spec-trace-helpers.ts | 24 ++++++++++++++ .../core/tests/spec-trace-helpers.test.ts | 32 +++++++++++++++++++ .../nightwatch-devtools/src/trace-slices.ts | 3 +- .../src/session-lifecycle.ts | 3 +- packages/service/src/trace-slices.ts | 18 ++--------- .../service/tests/trace-granularity.test.ts | 29 ----------------- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index 2eb69298..6de04bf8 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -36,6 +36,30 @@ export interface SpecRange { snapshotCount: number } +/** + * The slice range to eager-flush for a just-ended test. When the caller knows + * the test's uid (e.g. WDIO's `afterTest`), reverse-scan for it — retries push + * a new range under the same uid, and the next test's boundary may already be + * recorded, so the last range isn't reliably this test's. When the uid isn't + * independently known (Nightwatch/Selenium discover it from the range itself), + * the last recorded range is the just-ended test's. Undefined when there is no + * range to flush. One helper for what the three adapters each open-coded. + */ +export function findFlushableRange( + ranges: readonly SpecRange[], + testUid?: string +): SpecRange | undefined { + if (testUid !== undefined) { + for (let i = ranges.length - 1; i >= 0; i--) { + if (ranges[i]!.testUid === testUid) { + return ranges[i] + } + } + return undefined + } + return ranges[ranges.length - 1] +} + // ─── Spec name sanitization ─────────────────────────────────────────────────── /** diff --git a/packages/core/tests/spec-trace-helpers.test.ts b/packages/core/tests/spec-trace-helpers.test.ts index 95257afe..27f4f694 100644 --- a/packages/core/tests/spec-trace-helpers.test.ts +++ b/packages/core/tests/spec-trace-helpers.test.ts @@ -8,6 +8,7 @@ import { buildTestSliceFolder, buildTestSliceSessionId, filterTestMetadataBySpec, + findFlushableRange, filterTestMetadataByUid, recordSliceBoundary, recordSpecBoundary, @@ -335,3 +336,34 @@ describe('recordSliceBoundary / recordSpecBoundary (spec granularity)', () => { expect(ctx.specRanges[0]!.key).toBe('/a.js') }) }) + +describe('findFlushableRange', () => { + const mk = (key: string, testUid?: string): SpecRange => ({ + specFile: 'f', + key, + testUid, + commandStartIdx: 0, + consoleStartIdx: 0, + networkStartIdx: 0, + mutationStartIdx: 0, + traceLogStartIdx: 0, + snapshotCount: 0 + }) + + it('reverse-scans for the given testUid (latest retry attempt wins)', () => { + const ranges = [mk('a', 'a'), mk('b', 'b'), mk('b-retry1', 'b')] + expect(findFlushableRange(ranges, 'b')?.key).toBe('b-retry1') + expect(findFlushableRange(ranges, 'a')?.key).toBe('a') + }) + + it('returns undefined when no range matches the testUid', () => { + expect(findFlushableRange([mk('spec.ts', undefined)], 'x')).toBeUndefined() + expect(findFlushableRange([], 'x')).toBeUndefined() + }) + + it('falls back to the last recorded range when no testUid is given', () => { + const ranges = [mk('a', 'a'), mk('b', 'b')] + expect(findFlushableRange(ranges)?.key).toBe('b') + expect(findFlushableRange([])).toBeUndefined() + }) +}) diff --git a/packages/nightwatch-devtools/src/trace-slices.ts b/packages/nightwatch-devtools/src/trace-slices.ts index bdd0cf0d..c69f8fe7 100644 --- a/packages/nightwatch-devtools/src/trace-slices.ts +++ b/packages/nightwatch-devtools/src/trace-slices.ts @@ -12,6 +12,7 @@ */ import { + findFlushableRange, recordSliceBoundary, recordSpecBoundary, type SpecBoundaryContext, @@ -93,7 +94,7 @@ export function flushTestSlice(ctx: TestSliceCtx): void { if (!sliceActive(ctx)) { return } - const range = ctx.specRanges[ctx.specRanges.length - 1] + const range = findFlushableRange(ctx.specRanges) if (!range) { return } diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 856d05ee..7716210b 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -14,6 +14,7 @@ import { errorMessage, finalizeScreencast, finalizeTraceExport, + findFlushableRange, flushRangeLogged, recordSliceBoundary as coreRecordSliceBoundary, recordSpecBoundary as coreRecordSpecBoundary, @@ -425,7 +426,7 @@ export function flushCurrentTestTrace(ctx: SessionLifecycleCtx): void { return } const sessionId = ctx.sessionCapturer.metadata?.sessionId - const currentRange = ctx.specRanges[ctx.specRanges.length - 1] + const currentRange = findFlushableRange(ctx.specRanges) if (!sessionId || currentRange?.testUid === undefined) { return } diff --git a/packages/service/src/trace-slices.ts b/packages/service/src/trace-slices.ts index fcc36962..5da66806 100644 --- a/packages/service/src/trace-slices.ts +++ b/packages/service/src/trace-slices.ts @@ -3,26 +3,12 @@ // selection and flush I/O are unit-testable and the god-file stays lean. import { + findFlushableRange, flushRangeLogged, type SpecRange, type TraceExportContext } from '@wdio/devtools-core' -/** The range for the test that just ended is the most recent slice recorded - * under this base testUid — retries push a new range under the same testUid, - * so reverse-scanning finds the attempt whose afterTest is now firing. */ -export function findCurrentTestRange( - ranges: readonly SpecRange[], - testUid: string -): SpecRange | undefined { - for (let i = ranges.length - 1; i >= 0; i--) { - if (ranges[i]!.testUid === testUid) { - return ranges[i] - } - } - return undefined -} - /** Fire-and-forget flush of the previous unflushed slice at a boundary change * (spec granularity, or a test slice whose eager flush was missed). Errors are * logged, never thrown, so a failed flush can't abort the next test. */ @@ -41,7 +27,7 @@ export async function flushTestSlice( ranges: readonly SpecRange[], testUid: string ): Promise<void> { - const range = findCurrentTestRange(ranges, testUid) + const range = findFlushableRange(ranges, testUid) if (!range) { return } diff --git a/packages/service/tests/trace-granularity.test.ts b/packages/service/tests/trace-granularity.test.ts index c43b6b66..66a3b971 100644 --- a/packages/service/tests/trace-granularity.test.ts +++ b/packages/service/tests/trace-granularity.test.ts @@ -1,7 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import type * as DevtoolsCore from '@wdio/devtools-core' import { deterministicUid, type SpecRange } from '@wdio/devtools-core' -import { findCurrentTestRange } from '../src/trace-slices.js' // Records the key/state observed at each per-slice flush and replays the real // dedupe (flushed.add) so recordSliceBoundary's prev-slice logic behaves as in @@ -87,34 +86,6 @@ vi.mock('@wdio/devtools-core', async (importOriginal) => { // Imported after the mocks are declared so the mocked core module is used. const { default: DevToolsHookService } = await import('../src/index.js') -describe('findCurrentTestRange', () => { - const mk = (key: string, testUid?: string): SpecRange => ({ - specFile: 'f', - key, - testUid, - commandStartIdx: 0, - consoleStartIdx: 0, - networkStartIdx: 0, - mutationStartIdx: 0, - traceLogStartIdx: 0, - snapshotCount: 0 - }) - - it('returns the most recent range recorded under the base testUid', () => { - const ranges = [mk('a', 'a'), mk('b', 'b'), mk('b-retry1', 'b')] - // A retry pushes a new range under the same testUid; the latest one wins. - expect(findCurrentTestRange(ranges, 'b')?.key).toBe('b-retry1') - expect(findCurrentTestRange(ranges, 'a')?.key).toBe('a') - }) - - it('returns undefined when no range matches (spec/session slices)', () => { - expect( - findCurrentTestRange([mk('spec.ts', undefined)], 'x') - ).toBeUndefined() - expect(findCurrentTestRange([], 'x')).toBeUndefined() - }) -}) - describe('DevtoolsService - trace granularity slicing', () => { const file = '/proj/specs/login.spec.ts' const mockBrowser = { From c04020fff7d080a40a37e2c74aedce67b45992c0 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 02:02:26 +0530 Subject: [PATCH 52/91] refactor(service): rebuild expect-matcher capture around before/afterAssertion --- packages/service/src/assert-capture.ts | 108 +++++---- packages/service/src/index.ts | 222 +++++++++--------- packages/service/src/session.ts | 44 ++++ packages/service/tests/assert-capture.test.ts | 73 ++---- packages/service/tests/assertion-rows.test.ts | 189 ++++++++++----- packages/service/tests/index.test.ts | 78 ------ packages/service/tests/session.test.ts | 106 +++++++++ 7 files changed, 481 insertions(+), 339 deletions(-) diff --git a/packages/service/src/assert-capture.ts b/packages/service/src/assert-capture.ts index 49725ec8..a62b4cff 100644 --- a/packages/service/src/assert-capture.ts +++ b/packages/service/src/assert-capture.ts @@ -9,42 +9,45 @@ import { stripAnsi } from '@wdio/devtools-core' import type { CommandLog, SerializedError } from '@wdio/devtools-shared' -import { parse } from 'stack-trace' -import { - resolveCallSourceFromFrame, - resolveFilePathFromFrame -} from './call-source.js' -import { isUserSpecFile } from './utils.js' import type { SessionCapturer } from './session.js' const log = logger('@wdio/devtools-service:assert-capture') /** - * Capture the user's `expect()` call site from the SYNCHRONOUS stack at matcher - * entry (call from `beforeAssertion`). The matcher then runs async, so this is - * the only point a user frame is still on the stack — reading it in - * `afterAssertion` resolves to the service bundle instead. Mirrors - * `beforeCommand`'s resolver (`parse(new Error()).reverse()` → first user-spec - * frame → `resolveCallSourceFromFrame`) so assertion rows share regular - * commands' Source-tab behaviour. Also loads that file's source via - * `captureSource` so the tab renders. Returns `undefined` when no user frame is - * present (row falls back to no callSource, exactly as before this fix). + * The WDIO element/browser query commands an expect-webdriverio matcher issues + * to read the value it asserts on (`toHaveText`→`getText`, `toExist`→ + * `isExisting`, …). The matcher's read is captured as a normal command; on + * `afterAssertion` the synthesized `expect.*` row is coalesced into it (it + * already carries the correct callSource, screenshot, and timeline position), + * so only one row remains — no timing/stack heuristics. Not exhaustive: an + * unlisted matcher just leaves its read visible plus the assertion row. */ -export function resolveAssertionCallSource( - captureSource: (filePath: string) => void -): string | undefined { - Error.stackTraceLimit = 20 - const frame = parse(new Error('')) - .reverse() - .find((f) => isUserSpecFile(f.getFileName())) - if (!frame) { - return undefined - } - const filePath = resolveFilePathFromFrame(frame) - if (filePath) { - captureSource(filePath) - } - return resolveCallSourceFromFrame(frame) +const MATCHER_READ_COMMANDS = new Set([ + 'getText', + 'getHTML', + 'isExisting', + 'isDisplayed', + 'isDisplayedInViewport', + 'getValue', + 'getAttribute', + 'getProperty', + 'getComputedRole', + 'getComputedLabel', + 'isEnabled', + 'isClickable', + 'isSelected', + 'isFocused', + 'getSize', + 'getLocation', + 'getCSSProperty', + 'getTagName', + 'getUrl', + 'getTitle' +]) + +/** True when a command is a matcher's value-read (see MATCHER_READ_COMMANDS). */ +export function isMatcherReadCommand(command: string): boolean { + return MATCHER_READ_COMMANDS.has(command) } /** @@ -131,33 +134,48 @@ export interface ExpectAssertion { result: { pass?: boolean; result?: boolean; message?: () => string } } +/** `expect.stringContaining(x)` / `objectContaining` etc. are jest asymmetric + * matchers — objects carrying a `sample` payload and an `asymmetricMatch` + * method. Surface the payload so a row reads `toHaveText("x")` instead of + * `toHaveText({"sample":"x"})`; non-matchers pass through unchanged. */ +function unwrapAsymmetricMatcher(value: unknown): unknown { + if ( + value !== null && + typeof value === 'object' && + 'sample' in value && + typeof (value as { asymmetricMatch?: unknown }).asymmetricMatch === + 'function' + ) { + return (value as { sample: unknown }).sample + } + return value +} + /** * Adapt expect-webdriverio's afterAssertion params to the shared matcher * converter. Framework-specific extraction only (matcher name, expectedValue → - * args, the runtime `pass` vs typed `result` flag); the actual CommandLog - * shaping lives once in core's `matcherAssertionToCommandLog`. `callSource` is - * the user's `expect()` call site captured in `beforeAssertion` (the matcher - * runs async, so afterAssertion's own stack no longer holds a user frame) — it - * makes the row's Source tab point at the spec, not the service bundle. + * args, the runtime `pass` vs typed `result` flag); the CommandLog shaping + * lives once in core's `matcherAssertionToCommandLog`. The row's callSource + + * screenshot come from the matcher's read command it's coalesced into (see + * `coalesceAssertionIntoLastRead`), not from a stack walk here. */ export function expectAssertionToCommandLog( params: ExpectAssertion, - testUid: string | undefined, - callSource?: string + testUid: string | undefined ): CommandLog { const { matcherName, expectedValue, result } = params + const rawArgs = + expectedValue === undefined + ? [] + : Array.isArray(expectedValue) + ? expectedValue + : [expectedValue] return matcherAssertionToCommandLog( { method: matcherName, - args: - expectedValue === undefined - ? [] - : Array.isArray(expectedValue) - ? expectedValue - : [expectedValue], + args: rawArgs.map(unwrapAsymmetricMatcher), passed: result.pass ?? result.result ?? false, - message: result.message, - callSource + message: result.message }, testUid ) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 064be3a4..0b1bcf71 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -15,7 +15,7 @@ import { import { captureExpectFailure, expectAssertionToCommandLog, - resolveAssertionCallSource, + isMatcherReadCommand, wireAssertCapture, type ExpectAssertion } from './assert-capture.js' @@ -104,35 +104,23 @@ export default class DevToolsHookService implements Services.ServiceInstance { */ #commandStack: CommandFrame[] = [] - /** Depth of the current expect-webdriverio matcher evaluation. While > 0, the - * matcher's internal WebDriver commands (getText/isExisting polling) are - * suppressed so only the `expect.<matcher>` assertion row is captured — - * matching Playwright and the Nightwatch native-assert behaviour. */ - #assertionDepth = 0 - - /** True once the current matcher has issued an internal command whose stack - * shows an expect-webdriverio frame. Gates the stuck-depth self-heal: the - * matcher's FIRST internal command (an element re-lookup) traces to the - * user's expect() line but its sync stack has no matcher frame yet, which - * would otherwise be misread as "matcher ended" and reset the depth mid-run - * — capturing every later matcher command as a duplicate row. */ - #matcherStarted = false - - /** Matcher start timestamps (stack, paired with #assertionDepth) so a captured - * assertion spans [matcher start → end] — its poll duration — instead of - * collapsing to a zero-width point at completion, which left the screencast - * tracking the preceding command during the poll. */ - #assertionStartTimes: number[] = [] - - /** User `expect()` call sites (stack, paired with #assertionStartTimes), - * captured in beforeAssertion while a user frame is still synchronous. The - * matcher runs async, so afterAssertion's own stack resolves to the service - * bundle — this parallel stack lets each row keep its spec-file callSource. */ - #assertionCallSources: (string | undefined)[] = [] - /** Current test UID, set in beforeTest(), used by afterCommand() to tag commands. */ #currentTestUid?: string + /** expect-webdriverio matcher nesting depth. Aliases fire before/afterAssertion + * twice (toBeChecked→toBeSelected), so only the outermost pair owns the row. */ + #assertionDepth = 0 + + /** Matcher armed at beforeAssertion, cleared at its afterAssertion. It survives + * to test end only when the matcher hard-threw (element never resolved, so + * waitUntil rethrew and afterAssertion never fired) — then it's synthesized as + * a failing expect.<matcher> row instead of leaving a raw read. */ + #pendingAssertion?: { + matcherName: string + expectedValue?: unknown + testUid?: string + } + /** Map of testUid → metadata for trace group events and per-spec partitioning. */ #testMetadata: TestMetadataMap = new Map() @@ -417,7 +405,54 @@ export default class DevToolsHookService implements Services.ServiceInstance { _scenario?: unknown, result?: { error?: unknown } ) { - this.#captureExpectFailure(result?.error) + this.#handleAssertionOutcome(result?.error) + } + + /** Route a test/step failure to assertion capture. A matcher that hard-threw + * (element never resolved) left an armed #pendingAssertion because + * afterAssertion never fired — synthesize its failing expect row. Any other + * failure just marks the last action with the error. */ + #handleAssertionOutcome(error: unknown): void { + if (this.#options.captureAssertions === false) { + return + } + if (this.#pendingAssertion) { + this.#finalizePendingAssertion(error) + return + } + this.#captureExpectFailure(error) + } + + /** Synthesize the failing expect.<matcher> row for a hard-thrown matcher: fold + * it into the throwing read (relabel `getText`→`expect.toHaveText`, keeping + * the error) so the assertion renders consistently whether or not the element + * resolved; fall back to a fresh row when there is no read to fold. */ + #finalizePendingAssertion(error: unknown): void { + const pending = this.#pendingAssertion + this.#pendingAssertion = undefined + this.#assertionDepth = 0 + if (!pending) { + return + } + const message = errorMessage(error) || `${pending.matcherName} failed` + const entry = expectAssertionToCommandLog( + { + matcherName: pending.matcherName, + expectedValue: pending.expectedValue, + result: { pass: false, message: () => message } + }, + pending.testUid + ) + if ( + this.#sessionCapturer.coalesceAssertionIntoLastRead( + entry, + isMatcherReadCommand, + true + ) + ) { + return + } + this.#sessionCapturer.captureAssertCommand(entry) } /** Mark the failing action from a matcher error (afterStep for Cucumber, @@ -463,7 +498,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { _context?: unknown, result?: TestOutcomeResult ) { - this.#captureExpectFailure(result?.error) + this.#handleAssertionOutcome(result?.error) const testTitle = test?.fullTitle || test?.title const uid = testTitle ? testMetadataUid(test?.file, testTitle) : undefined if (uid) { @@ -476,52 +511,63 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** expect-webdriverio fires this before each matcher evaluates. The matcher's - * internal polling commands run inside the before→after window and must not - * surface as their own rows — only the `expect.<matcher>` assertion should. */ - beforeAssertion(): void { - // Matchers don't nest, so any residual depth here is a prior matcher whose - // afterAssertion was skipped by a hard throw. Reset before pushing so the - // suppression window can't stay stuck open across assertions. - if (this.#assertionDepth > 0) { - this.#assertionDepth = 0 - this.#assertionStartTimes = [] - this.#assertionCallSources = [] + /** + * expect-webdriverio fires this after each matcher evaluates, with the matcher + * name + pass/fail + expected value. The matcher's value-read (getText / + * isExisting / …) was captured as a normal command; fold this assertion into + * that read so one `expect.<matcher>` row remains — inheriting the read's real + * callSource, screenshot, and timeline position. Deterministic and anchored to + * this reliable hook: no `#assertionDepth`, no stack-frame detection. + */ + /** + * expect-webdriverio fires this at the START of every matcher, before it polls + * — so it fires even for a matcher that later hard-throws (element never + * resolved), unlike afterAssertion. Arm the pending matcher here so test-end + * can synthesize its expect row if afterAssertion never comes. Depth-counted: + * aliases (toBeChecked→toBeSelected) fire this twice, so only the outermost + * arms the row. + */ + beforeAssertion(params: { + matcherName: string + expectedValue?: unknown + }): void { + if (this.#options.captureAssertions === false) { + return + } + if (this.#assertionDepth === 0) { + this.#pendingAssertion = { + matcherName: params.matcherName, + expectedValue: params.expectedValue, + testUid: this.#currentTestUid + } } this.#assertionDepth++ - this.#matcherStarted = false - this.#assertionStartTimes.push(Date.now()) - // Capture the user's expect() call site now — see #assertionCallSources. - this.#assertionCallSources.push( - resolveAssertionCallSource( - (file) => void this.#sessionCapturer.captureSource(file) - ) - ) } async afterAssertion(params: ExpectAssertion): Promise<void> { - // Decrement first (even when capture is off) so the window stays balanced. - const startTime = this.#assertionStartTimes.pop() - const callSource = this.#assertionCallSources.pop() - if (this.#assertionDepth > 0) { - this.#assertionDepth-- - } - this.#matcherStarted = false if (this.#options.captureAssertions === false) { return } - const entry = expectAssertionToCommandLog( - params, - this.#currentTestUid, - callSource - ) - // Span the matcher's poll window so the row's duration is real and the - // screencast tracks the assertion (not the preceding command) during it. - if (startTime !== undefined) { - entry.startTime = startTime + this.#assertionDepth = Math.max(0, this.#assertionDepth - 1) + // Inner matcher of a nested pair (toBeChecked→toBeSelected): the outer + // afterAssertion owns the row. + if (this.#assertionDepth > 0) { + return } - // The suppressed matcher-internal command carried the DOM screenshot; - // capture one here so the assertion row's Snapshot panel isn't blank. + // Reached afterAssertion → the matcher resolved (pass or value-fail), so no + // hard-throw synthesis is needed at test end. + this.#pendingAssertion = undefined + const entry = expectAssertionToCommandLog(params, this.#currentTestUid) + if ( + this.#sessionCapturer.coalesceAssertionIntoLastRead( + entry, + isMatcherReadCommand + ) + ) { + return + } + // No matcher read to fold into (a value matcher like toBe(x), or the read + // hard-threw): emit a fresh row with its own screenshot + trace snapshot. if (this.#browser && !isNativeMobile(this.#browser)) { try { entry.screenshot = await this.#browser.takeScreenshot() @@ -530,9 +576,6 @@ export default class DevToolsHookService implements Services.ServiceInstance { log.debug(`assertion screenshot skipped: ${errorMessage(err)}`) } } - // Trace mode: push a DOM action-snapshot stamped at this row's timestamp so - // the trace player's Snapshot tab renders it, exactly like a regular - // command's post-action snapshot (captureActionResult). if (this.#options.mode === 'trace' && this.#browser) { await pushActionSnapshotAt( this.#browser, @@ -573,9 +616,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { private resetStack() { this.#commandStack = [] this.#assertionDepth = 0 - this.#matcherStarted = false - this.#assertionStartTimes = [] - this.#assertionCallSources = [] + this.#pendingAssertion = undefined this.#sessionCapturer.resetLastSelector() this.#sessionCapturer.resetRetryTracker() } @@ -597,30 +638,6 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** Maintain the expect-matcher suppression window for an incoming command. - * `inMatcher` marks the matcher as started (so its trailing internal - * commands stay suppressed); otherwise self-heal a depth left stuck by a - * matcher that hard-threw (skipping afterAssertion) — but only once the - * matcher has actually started, so its own leading element re-lookup (no - * matcher frame on the sync stack yet) can't reset the depth mid-run. */ - #trackAssertionWindow(inMatcher: boolean, hasSource: boolean): void { - if (inMatcher) { - this.#matcherStarted = true - return - } - if ( - hasSource && - this.#commandStack.length === 0 && - this.#assertionDepth > 0 && - this.#matcherStarted - ) { - this.#assertionDepth = 0 - this.#assertionStartTimes = [] - this.#assertionCallSources = [] - this.#matcherStarted = false - } - } - async beforeCommand(command: string, args: string[]) { if (!this.#browser) { return @@ -641,15 +658,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { Error.stackTraceLimit = 20 const stack = parse(new Error('')).reverse() const source = stack.find((frame) => isUserSpecFile(frame.getFileName())) - const inMatcher = - this.#assertionDepth > 0 && - stack.some((frame) => frame.getFileName()?.includes('expect-webdriverio')) - this.#trackAssertionWindow(inMatcher, Boolean(source)) - if ( - source && - this.#commandStack.length === 0 && - this.#assertionDepth === 0 - ) { + // A matcher's value-read (getText/isExisting) is captured normally like any + // command; afterAssertion later folds it into the expect.<matcher> row (see + // coalesceAssertionIntoLastRead) — no suppression window needed here. + if (source && this.#commandStack.length === 0) { this.#pushTopLevelCommandFrame( command, resolveCallSourceFromFrame(source) diff --git a/packages/service/src/session.ts b/packages/service/src/session.ts index 58a90a24..6e8b2b9a 100644 --- a/packages/service/src/session.ts +++ b/packages/service/src/session.ts @@ -254,6 +254,50 @@ export class SessionCapturer extends SessionCapturerBase { this.#captureOrReplace(entry) } + /** + * Fold an expect-matcher assertion into the matcher's value-read command when + * that read is the most recent captured command (per `isRead`). The read + * already carries the correct callSource, screenshot, and timeline position — + * the DOM the matcher evaluated — so replace it in place with the assertion + * row: one row, no duplicate, and no timing/stack heuristics. WDIO's + * RetryTracker already collapses a matcher's repeated polls to that one read. + * Returns false when the last command isn't a matcher read (a value matcher), + * so the caller emits a fresh assertion row instead. + * + * `foldErrored` folds even when the read carries an error — used by the + * hard-throw path (element never resolved, so afterAssertion never fired and + * the read threw): relabel the throwing read as the failing expect row rather + * than leave a raw `getText`. The normal path keeps the guard so a value + * matcher can't accidentally swallow an unrelated errored command. + */ + coalesceAssertionIntoLastRead( + entry: CommandLog, + isRead: (command: string) => boolean, + foldErrored = false + ): boolean { + const log = this.commandsLog as (CommandLog & { _id?: number })[] + const last = log[log.length - 1] + if (!last || !isRead(last.command) || (last.error && !foldErrored)) { + return false + } + // Inherit the read's `_id` (local dedup bookkeeping) and timestamp, but do + // NOT stamp a public `id`: WDIO replaces by timestamp (like #captureOrReplace), + // and `commandCounter` resets per worker/spec, so a bare `id` collides across + // specs and the app's id-first replaceCommand would swap the wrong row. + const merged: CommandLog & { _id?: number } = { + ...entry, + _id: last._id, + timestamp: last.timestamp, + startTime: last.startTime, + callSource: entry.callSource ?? last.callSource, + screenshot: entry.screenshot ?? last.screenshot, + error: entry.error ?? last.error + } + log[log.length - 1] = merged + this.sendReplaceCommand(last.timestamp, merged) + return true + } + /** * Run the shared Performance API capture script and attach the result to * the given CommandLog entry. Same `CAPTURE_PERFORMANCE_SCRIPT` + diff --git a/packages/service/tests/assert-capture.test.ts b/packages/service/tests/assert-capture.test.ts index 534945e3..e88f3dca 100644 --- a/packages/service/tests/assert-capture.test.ts +++ b/packages/service/tests/assert-capture.test.ts @@ -7,30 +7,11 @@ import { import { captureExpectFailure, expectAssertionToCommandLog, - resolveAssertionCallSource, toCommandError, wireAssertCapture } from '../src/assert-capture.js' import type { SessionCapturer } from '../src/session.js' -const stackFrames = vi.hoisted(() => ({ - value: [] as Array<{ - getFileName: () => string | null - getLineNumber: () => number | null - getColumnNumber: () => number | null - }> -})) - -// Only resolveAssertionCallSource reads 'stack-trace'; node:assert capture uses -// core's stacktrace-parser path, so mocking here doesn't affect those tests. -vi.mock('stack-trace', () => ({ parse: () => stackFrames.value })) - -const frame = (file: string | null, line = 1, column = 1) => ({ - getFileName: () => file, - getLineNumber: () => line, - getColumnNumber: () => column -}) - describe('toCommandError', () => { it('normalizes a plain Error object (ANSI stripped)', () => { // Matcher errors are skipped now (afterAssertion owns them); a plain @@ -178,6 +159,23 @@ describe('expectAssertionToCommandLog', () => { expect(entry.error).toBeUndefined() }) + it('unwraps a jest asymmetric matcher (stringContaining) to its payload', () => { + const entry = expectAssertionToCommandLog( + { + matcherName: 'toHaveText', + // expect.stringContaining('You logged in') shape. + expectedValue: { + sample: 'You logged in', + asymmetricMatch: () => true + }, + result: { pass: true, message: () => 'ok' } + }, + 'uid-2' + ) + // Label reads toHaveText("You logged in"), not toHaveText({"sample":…}). + expect(entry.args).toEqual(['You logged in']) + }) + it('captures a failing matcher with its ANSI-stripped message as the error', () => { const entry = expectAssertionToCommandLog( { @@ -215,41 +213,4 @@ describe('expectAssertionToCommandLog', () => { ) expect(entry).toMatchObject({ command: 'expect.toBeClickable', args: [] }) }) - - it('forwards the captured callSource onto the assertion row', () => { - const entry = expectAssertionToCommandLog( - { matcherName: 'toExist', result: { pass: true } }, - 'uid-1', - '/proj/specs/login.e2e.ts:30:7' - ) - expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') - }) -}) - -describe('resolveAssertionCallSource', () => { - it('returns the outermost user-spec frame and loads its source', () => { - // innermost → outermost: service bundle, expect-webdriverio matcher, the - // user spec, then node internals. The user spec must win, not the bundle. - stackFrames.value = [ - frame('/repo/packages/service/dist/index.js', 4045, 12), - frame('/repo/node_modules/expect-webdriverio/lib/matchers.js', 9, 3), - frame('/proj/specs/login.e2e.ts', 30, 7), - frame('node:internal/process/task_queues', 95, 5) - ] - const captured: string[] = [] - const callSource = resolveAssertionCallSource((f) => captured.push(f)) - expect(callSource).toBe('/proj/specs/login.e2e.ts:30:7') - expect(callSource).not.toContain('/dist/') - expect(captured).toEqual(['/proj/specs/login.e2e.ts']) - }) - - it('returns undefined and loads nothing when only dependency frames exist', () => { - stackFrames.value = [ - frame('/repo/node_modules/expect-webdriverio/lib/matchers.js', 9, 3), - frame('node:internal/process/task_queues', 95, 5) - ] - const captured: string[] = [] - expect(resolveAssertionCallSource((f) => captured.push(f))).toBeUndefined() - expect(captured).toEqual([]) - }) }) diff --git a/packages/service/tests/assertion-rows.test.ts b/packages/service/tests/assertion-rows.test.ts index 5a5b09ac..ddd216b2 100644 --- a/packages/service/tests/assertion-rows.test.ts +++ b/packages/service/tests/assertion-rows.test.ts @@ -5,18 +5,14 @@ import { TRACKED_ASSERT_METHODS } from '@wdio/devtools-core' -// Controlled synchronous stack for the beforeAssertion call-source walk. -const stackFrames = vi.hoisted(() => ({ - value: [] as Array<{ - getFileName: () => string | null - getLineNumber: () => number | null - getColumnNumber: () => number | null - }> -})) -vi.mock('stack-trace', () => ({ parse: () => stackFrames.value })) - +// Mock the capturer: the service routes each expect matcher through +// coalesceAssertionIntoLastRead (fold into the matcher's read command) or, when +// there's no read to fold into, captureAssertCommand (a fresh row). These tests +// assert that routing; the fold itself is a capturer unit test (session.test). const capturer = vi.hoisted(() => ({ captureAssertCommand: vi.fn(), + coalesceAssertionIntoLastRead: vi.fn(), + failLastAction: vi.fn(), captureSource: vi.fn().mockResolvedValue(undefined), injectScript: vi.fn().mockResolvedValue(undefined), sendUpstream: vi.fn(), @@ -42,13 +38,6 @@ vi.mock('../src/action-snapshot.js', () => ({ })) import DevToolsHookService from '../src/index.js' -import type { ExpectAssertion } from '../src/assert-capture.js' - -const userFrame = { - getFileName: () => '/proj/specs/login.e2e.ts', - getLineNumber: () => 30, - getColumnNumber: () => 7 -} const mockBrowser = { isBidi: true, @@ -76,82 +65,172 @@ afterAll(() => { } }) -const capturedEntry = () => capturer.captureAssertCommand.mock.calls[0]![0] - describe('DevtoolsService — expect.* assertion rows', () => { beforeEach(() => { vi.clearAllMocks() - stackFrames.value = [userFrame] }) - it('trace mode: user-spec callSource, spec source loaded, DOM snapshot pushed', async () => { + it('folds the assertion into the matcher read (no fresh row or snapshot)', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(true) const service = new DevToolsHookService({ mode: 'trace' }) await service.before({} as never, [], mockBrowser) - service.beforeAssertion() - const params: ExpectAssertion = { - matcherName: 'toExist', - result: { pass: true, message: () => 'ok' } - } - await service.afterAssertion(params) - - const entry = capturedEntry() - expect(entry.command).toBe('expect.toExist') - // The regression: callSource must point at the user's spec, not the - // service bundle (…/service/dist/index.js). - expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') - expect(entry.callSource).not.toContain('/dist/') - // Source of that file is loaded so the Source tab can render it. - expect(capturer.captureSource).toHaveBeenCalledWith( - '/proj/specs/login.e2e.ts' - ) - // Live-mode screenshot kept for the CommandLog. + await service.afterAssertion({ + matcherName: 'toHaveText', + expectedValue: 'Hi', + result: { pass: false, message: () => 'nope' } + }) + + // Routed through the fold: the matcher's read becomes the expect row (it + // already carries the correct callSource + screenshot + position). + const [entry, isRead] = + capturer.coalesceAssertionIntoLastRead.mock.calls[0]! + expect(entry.command).toBe('expect.toHaveText') + expect(entry.error).toMatchObject({ message: 'nope' }) + expect(isRead('getText')).toBe(true) + expect(isRead('click')).toBe(false) + // No duplicate fresh row, no fresh screenshot/snapshot. + expect(capturer.captureAssertCommand).not.toHaveBeenCalled() + expect(pushActionSnapshotAt).not.toHaveBeenCalled() + }) + + it('trace fallback: fresh row + screenshot + DOM snapshot when there is no read to fold', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(false) + const service = new DevToolsHookService({ mode: 'trace' }) + await service.before({} as never, [], mockBrowser) + + await service.afterAssertion({ + matcherName: 'toBe', + expectedValue: 1, + result: { pass: true } + }) + + const entry = capturer.captureAssertCommand.mock.calls[0]![0] + expect(entry.command).toBe('expect.toBe') expect(entry.screenshot).toBe('SHOT') - // Trace-player Snapshot tab: a DOM snapshot stamped at the row timestamp. expect(pushActionSnapshotAt).toHaveBeenCalledWith( mockBrowser, - 'expect.toExist', + 'expect.toBe', entry.timestamp, expect.any(Array) ) - // WDIO reconciles rows by timestamp (like every regular WDIO command), - // so assertion rows carry no public id — parity with regular commands. - expect(entry.id).toBeUndefined() }) - it('live mode: keeps the screenshot but pushes no DOM snapshot', async () => { + it('live fallback: keeps the screenshot but pushes no DOM snapshot', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(false) const service = new DevToolsHookService({ mode: 'live' }) await service.before({} as never, [], mockBrowser) - service.beforeAssertion() await service.afterAssertion({ - matcherName: 'toHaveText', - expectedValue: 'Hi', - result: { pass: false, message: () => 'nope' } + matcherName: 'toBe', + expectedValue: 1, + result: { pass: true } }) - const entry = capturedEntry() - expect(entry.command).toBe('expect.toHaveText') - expect(entry.callSource).toBe('/proj/specs/login.e2e.ts:30:7') - expect(entry.error).toMatchObject({ message: 'nope' }) + const entry = capturer.captureAssertCommand.mock.calls[0]![0] expect(entry.screenshot).toBe('SHOT') expect(pushActionSnapshotAt).not.toHaveBeenCalled() }) - it('captureAssertions: false suppresses the row but keeps the window balanced', async () => { + it('captureAssertions: false emits nothing (no fold, no fresh row)', async () => { const service = new DevToolsHookService({ mode: 'trace', captureAssertions: false }) await service.before({} as never, [], mockBrowser) - service.beforeAssertion() await service.afterAssertion({ matcherName: 'toExist', result: { pass: true } }) + expect(capturer.coalesceAssertionIntoLastRead).not.toHaveBeenCalled() expect(capturer.captureAssertCommand).not.toHaveBeenCalled() expect(pushActionSnapshotAt).not.toHaveBeenCalled() }) + + it('afterAssertion clears the armed matcher so test end does not re-synthesize', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(true) + const service = new DevToolsHookService({ mode: 'live' }) + await service.before({} as never, [], mockBrowser) + + service.beforeAssertion({ matcherName: 'toHaveText', expectedValue: 'Hi' }) + await service.afterAssertion({ + matcherName: 'toHaveText', + expectedValue: 'Hi', + // a real matcher failure carries matcherResult, so failLastAction skips it + result: { pass: false, message: () => 'nope' } + }) + // afterAssertion fired → pending cleared; test end must not fold again. + await service.afterTest({ file: '/s.ts' } as never, undefined, { + error: Object.assign(new Error('nope'), { + matcherResult: { pass: false } + }) + } as never) + + expect(capturer.coalesceAssertionIntoLastRead).toHaveBeenCalledTimes(1) + expect(capturer.captureAssertCommand).not.toHaveBeenCalled() + }) + + it('hard-throw (no afterAssertion): folds the throwing read into a failing expect row', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(true) + const service = new DevToolsHookService({ mode: 'live' }) + await service.before({} as never, [], mockBrowser) + + // Matcher armed, then getText hard-threw → afterAssertion never fires. + service.beforeAssertion({ + matcherName: 'toHaveText', + expectedValue: 'Your username is invalid!' + }) + await service.afterTest({ file: '/s.ts' } as never, undefined, { + error: new Error('element ("#flash") still not existing') + } as never) + + const [entry, , foldErrored] = + capturer.coalesceAssertionIntoLastRead.mock.calls[0]! + expect(entry.command).toBe('expect.toHaveText') + expect(entry.args).toEqual(['Your username is invalid!']) + expect(entry.error.message).toContain('still not existing') + expect(foldErrored).toBe(true) // folds even though the read carries an error + expect(capturer.captureAssertCommand).not.toHaveBeenCalled() + }) + + it('hard-throw with no read to fold: emits a fresh failing expect row', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(false) + const service = new DevToolsHookService({ mode: 'live' }) + await service.before({} as never, [], mockBrowser) + + service.beforeAssertion({ matcherName: 'toBeDisplayed' }) + await service.afterTest({ file: '/s.ts' } as never, undefined, { + error: new Error('element not found') + } as never) + + const entry = capturer.captureAssertCommand.mock.calls[0]![0] + expect(entry.command).toBe('expect.toBeDisplayed') + expect(entry.error.message).toContain('element not found') + }) + + it('nested matcher aliases fold once (toBeChecked→toBeSelected)', async () => { + capturer.coalesceAssertionIntoLastRead.mockReturnValue(true) + const service = new DevToolsHookService({ mode: 'live' }) + await service.before({} as never, [], mockBrowser) + + // toBeChecked delegates to toBeSelected — before/after fire twice, nested. + service.beforeAssertion({ matcherName: 'toBeChecked' }) + service.beforeAssertion({ matcherName: 'toBeSelected' }) + await service.afterAssertion({ + matcherName: 'toBeSelected', + result: { pass: true } + }) + await service.afterAssertion({ + matcherName: 'toBeChecked', + result: { pass: true } + }) + + // Only the outer afterAssertion emits — one row, labelled by the alias. + expect(capturer.coalesceAssertionIntoLastRead).toHaveBeenCalledTimes(1) + expect( + capturer.coalesceAssertionIntoLastRead.mock.calls[0]![0].command + ).toBe('expect.toBeChecked') + }) }) diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index a4f4a229..ba881d4a 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -294,81 +294,3 @@ describe('DevtoolsService - Screencast Integration', () => { expect(mockScreencastRecorder.start).toHaveBeenCalledWith(mockBrowser) }) }) - -describe('DevtoolsService - assertion suppression self-heal', () => { - let service: DevToolsHookService - const mockBrowser = { - isBidi: true, - sessionId: 'heal-session', - scriptAddPreloadScript: vi.fn().mockResolvedValue(undefined), - takeScreenshot: vi.fn().mockResolvedValue('screenshot'), - execute: vi.fn().mockResolvedValue({ - width: 1200, - height: 800, - offsetLeft: 0, - offsetTop: 0 - }), - on: vi.fn(), - emit: vi.fn() - } as any - - const capturedCommands = () => - mockSessionCapturerInstance.afterCommand.mock.calls.map((call) => call[1]) - - beforeEach(async () => { - vi.clearAllMocks() - stackMock.frames = [stackMock.userFrame] - service = new DevToolsHookService() - await service.before({} as any, [], mockBrowser) - vi.clearAllMocks() - }) - - // expect-webdriverio doesn't wrap afterAssertion in try/finally, so a matcher - // whose internal command hard-throws runs beforeAssertion but skips - // afterAssertion, leaving #assertionDepth stuck ≥ 1. Without the self-heal - // that stuck depth suppresses every later user command in the test. - it('captures a top-level command after a matcher that started then threw left the assertion depth stuck', async () => { - service.beforeAssertion() // depth 1 - // The matcher issues an internal command (stack shows an expect-webdriverio - // frame) → marks the matcher started, then hard-throws (afterAssertion - // skipped), leaving the depth stuck. - stackMock.frames = [stackMock.userFrame, stackMock.matcherFrame] - service.beforeCommand('getText' as any, ['#el']) - // A later top-level user command with no matcher frame self-heals the depth. - stackMock.frames = [stackMock.userFrame] - service.beforeCommand('click' as any, ['.button']) - await service.afterCommand('click' as any, ['.button'], undefined) - - expect(capturedCommands()).toEqual(['click']) - }) - - it('keeps suppressing the matcher’s own leading command (no premature self-heal)', async () => { - service.beforeAssertion() // depth 1 - // The matcher's FIRST internal command is an element re-lookup whose sync - // stack has no expect-webdriverio frame yet. It must NOT reset the depth — - // otherwise the subsequent matcher commands would surface as duplicate rows. - stackMock.frames = [stackMock.userFrame] - service.beforeCommand('$' as any, ['#el']) - await service.afterCommand('$' as any, ['#el'], undefined) - - expect(capturedCommands()).toEqual([]) - }) - - it('a fresh beforeAssertion resets a stuck depth so the window stays balanced', async () => { - service.beforeAssertion() - service.beforeAssertion() // leftover depth from a prior throw - - // A new matcher starts (resetting the stuck depth) and completes normally. - service.beforeAssertion() - await service.afterAssertion({ - matcherName: 'toBeDisplayed', - result: { pass: true } - } as any) - - // Depth is balanced again, so the next top-level command is captured. - service.beforeCommand('setValue' as any, ['.input', 'hi']) - await service.afterCommand('setValue' as any, ['.input', 'hi'], undefined) - - expect(capturedCommands()).toEqual(['setValue']) - }) -}) diff --git a/packages/service/tests/session.test.ts b/packages/service/tests/session.test.ts index 43f9853b..1f1185f2 100644 --- a/packages/service/tests/session.test.ts +++ b/packages/service/tests/session.test.ts @@ -769,4 +769,110 @@ describe('SessionCapturer', () => { expect(capturer.commandsLog[0].testUid).toBeUndefined() }) }) + + describe('coalesceAssertionIntoLastRead', () => { + const isRead = (c: string) => c === 'getText' + + it('folds the assertion into the trailing matcher read, in place', () => { + const capturer = new SessionCapturer() + capturer.commandsLog.push({ + command: 'getText', + args: [], + timestamp: 100, + startTime: 90, + callSource: '/spec.ts:13:5', + screenshot: 'READ_SHOT', + _id: 7 + } as never) + + const folded = capturer.coalesceAssertionIntoLastRead( + { + command: 'expect.toHaveText', + args: ['x'], + timestamp: 999, + result: 'passed' + } as never, + isRead + ) + + expect(folded).toBe(true) + expect(capturer.commandsLog).toHaveLength(1) + const row = capturer.commandsLog[0] as Record<string, unknown> + expect(row.command).toBe('expect.toHaveText') // became the assertion + expect(row.callSource).toBe('/spec.ts:13:5') // inherited from the read + expect(row.screenshot).toBe('READ_SHOT') // inherited from the read + expect(row.timestamp).toBe(100) // kept the read's timeline position + expect(row._id).toBe(7) // local dedup bookkeeping preserved + // No public `id`: WDIO replaces by timestamp, and commandCounter resets + // per worker/spec, so a bare id would collide across specs and the app's + // id-first replaceCommand would swap the wrong row. + expect(row.id).toBeUndefined() + }) + + it('returns false and leaves the log untouched when the last command is not a matcher read', () => { + const capturer = new SessionCapturer() + capturer.commandsLog.push({ + command: 'click', + args: [], + timestamp: 100 + } as never) + + const folded = capturer.coalesceAssertionIntoLastRead( + { command: 'expect.toExist', args: [], timestamp: 999 } as never, + isRead + ) + + expect(folded).toBe(false) + expect(capturer.commandsLog).toHaveLength(1) + expect((capturer.commandsLog[0] as Record<string, unknown>).command).toBe( + 'click' + ) + }) + + it('returns false when the trailing read hard-threw (carries an error)', () => { + const capturer = new SessionCapturer() + capturer.commandsLog.push({ + command: 'getText', + args: [], + timestamp: 100, + error: { message: 'element not found' } + } as never) + + expect( + capturer.coalesceAssertionIntoLastRead( + { command: 'expect.toHaveText', args: [], timestamp: 999 } as never, + isRead + ) + ).toBe(false) + }) + + it('foldErrored=true folds a throwing read, keeping its error (hard-throw)', () => { + const capturer = new SessionCapturer() + capturer.commandsLog.push({ + command: 'getText', + args: [], + timestamp: 100, + callSource: '/spec.ts:22:5', + error: { message: 'element not found' }, + _id: 3 + } as never) + + const folded = capturer.coalesceAssertionIntoLastRead( + { command: 'expect.toHaveText', args: ['x'], timestamp: 999 } as never, + isRead, + true + ) + + expect(folded).toBe(true) + expect(capturer.commandsLog).toHaveLength(1) + const row = capturer.commandsLog[0] as Record<string, unknown> + expect(row.command).toBe('expect.toHaveText') // relabelled from the read + expect(row.callSource).toBe('/spec.ts:22:5') // inherited from the read + expect(row.timestamp).toBe(100) // kept the read's timeline position + expect((row.error as { message: string }).message).toBe( + 'element not found' + ) // the throw's error carries through + expect(row.id).toBeUndefined() // still no cross-spec-colliding public id + }) + }) }) From 2a75a4a5554501c8468cb3bcb82bf17220d3b2c5 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 02:02:42 +0530 Subject: [PATCH 53/91] fix(service): exclude the devtools bundle from user-frame detection --- packages/service/src/utils.ts | 33 +++++++++++++++++++++++++--- packages/service/tests/utils.test.ts | 16 ++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/service/src/utils.ts b/packages/service/src/utils.ts index dc58e737..ebb7c94a 100644 --- a/packages/service/src/utils.ts +++ b/packages/service/src/utils.ts @@ -2,6 +2,9 @@ // (AST parsing, source mapping, cucumber step-def lookup) lives in // utils/source-mapping.ts and utils/step-defs.ts. +import path from 'node:path' +import { fileURLToPath } from 'node:url' + export { setCurrentSpecFile, findTestLocations, @@ -11,8 +14,17 @@ export { } from './utils/source-mapping.js' export { findStepDefinitionLocation } from './utils/step-defs.js' -/** A spec file owned by the user — excludes node-builtins and node_modules, - * but keeps WDIO's expect helpers (callers may want to step into those). */ +/** The service's own bundle directory. Stack frames from here are the service's + * instrumentation, not user code — a normal install has the service under + * node_modules (already excluded below), but a monorepo/linked setup puts the + * built service outside node_modules, so exclude it explicitly. */ +const SELF_DIR = path + .dirname(fileURLToPath(import.meta.url)) + .replace(/\\/g, '/') + +/** A spec file owned by the user — excludes node-builtins, node_modules, and + * the service's own bundle, but keeps WDIO's expect helpers (callers may want + * to step into those). */ export function isUserSpecFile(file?: string | null): boolean { if (!file) { return false @@ -20,10 +32,25 @@ export function isUserSpecFile(file?: string | null): boolean { if (file.startsWith('node:')) { return false } - const normalized = file.replace(/\\/g, '/') + // ESM stack frames report a file:// URL; SELF_DIR is a plain path, so decode + // to a plain path first or the self-bundle exclusion below silently no-ops. + // Use fileURLToPath (not `new URL().pathname`) so the Windows drive-letter is + // normalized the same way SELF_DIR is — otherwise `/C:/…` vs `C:/…` mismatch. + let normalized = file + if (normalized.startsWith('file://')) { + try { + normalized = fileURLToPath(normalized) + } catch { + /* keep the raw value */ + } + } + normalized = normalized.replace(/\\/g, '/') if (normalized.includes('/@wdio/expect-webdriverio/')) { return true } + if (normalized.startsWith(SELF_DIR)) { + return false + } return !normalized.includes('/node_modules/') } diff --git a/packages/service/tests/utils.test.ts b/packages/service/tests/utils.test.ts index bb9ffb76..2583e2cb 100644 --- a/packages/service/tests/utils.test.ts +++ b/packages/service/tests/utils.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' +import { fileURLToPath } from 'node:url' import { getBrowserObject, isUserSpecFile, @@ -82,5 +83,20 @@ describe('service utils', () => { ).toBe(false) expect(isUserSpecFile('C:\\proj\\test\\login.spec.ts')).toBe(true) }) + + it('decodes file:// URLs (incl. percent-encoding) before matching', () => { + expect(isUserSpecFile('file:///proj/test/login%20spec.ts')).toBe(true) + expect(isUserSpecFile('file:///proj/node_modules/lib/index.js')).toBe( + false + ) + }) + + it("excludes the service's own bundle dir, plain and as a file:// frame", () => { + // SELF_DIR is the dir this module resolves from; at test time that's + // packages/service/src. A frame from there is instrumentation, not a spec. + const selfDir = fileURLToPath(new URL('../src/', import.meta.url)) + expect(isUserSpecFile(`${selfDir}index.js`)).toBe(false) + expect(isUserSpecFile(`file://${selfDir}session.js`)).toBe(false) + }) }) }) From 346ca3bd0e3ab9e87ae7b8fa929af831e4964b98 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 15:01:28 +0530 Subject: [PATCH 54/91] fix(nightwatch): record negated native asserts (assert.not.* / verify.not.*) --- .../src/helpers/browserProxy.ts | 64 +++++++---- .../tests/browserProxy.test.ts | 100 +++++++++++++++++- 2 files changed, 142 insertions(+), 22 deletions(-) diff --git a/packages/nightwatch-devtools/src/helpers/browserProxy.ts b/packages/nightwatch-devtools/src/helpers/browserProxy.ts index 21225506..1d41c4fb 100644 --- a/packages/nightwatch-devtools/src/helpers/browserProxy.ts +++ b/packages/nightwatch-devtools/src/helpers/browserProxy.ts @@ -109,28 +109,52 @@ export class BrowserProxy { if (!original || typeof original !== 'object') { return } - b[prefix] = new Proxy(original as object, { - get: (target, name, receiver) => { - const orig = Reflect.get(target, name, receiver) - // `not` (negation Proxy) and non-method props pass through untouched. - if (typeof orig !== 'function' || typeof name !== 'string') { - return orig - } - return (...args: unknown[]) => { - const callInfo = getCallSourceFromStack() - if (callInfo.filePath !== undefined) { - this.emitPendingAssertion({ - prefix, - method: name, - args, - callSource: callInfo.callSource, - timestamp: Date.now() - }) - } - return (orig as (...a: unknown[]) => unknown)(...args) + b[prefix] = this.recordingNamespaceProxy(original, prefix, []) + }) + } + + /** + * Recording Proxy over one assert/verify namespace. A function property + * becomes a call-time recorder keyed by its full dotted path + * (`titleContains`, `not.titleContains`); a nested namespace object recurses + * through the SAME wrapper — Nightwatch exposes `assert.not` as its own Proxy, + * so negated asserts are recorded via the identical mechanism as positive + * ones instead of a parallel path. The recorder buffers a pending row, then + * delegates to the ORIGINAL method so Nightwatch's queue, chaining, and + * abort/negate semantics stay byte-for-byte unchanged. Non-method, + * non-namespace props pass through untouched. + */ + private recordingNamespaceProxy( + target: object, + prefix: 'assert' | 'verify', + path: readonly string[] + ): object { + return new Proxy(target, { + get: (t, name, receiver) => { + const orig = Reflect.get(t, name, receiver) + if (typeof name !== 'string') { + return orig + } + if (orig !== null && typeof orig === 'object') { + return this.recordingNamespaceProxy(orig, prefix, [...path, name]) + } + if (typeof orig !== 'function') { + return orig + } + return (...args: unknown[]) => { + const callInfo = getCallSourceFromStack() + if (callInfo.filePath !== undefined) { + this.emitPendingAssertion({ + prefix, + method: [...path, name].join('.'), + args, + callSource: callInfo.callSource, + timestamp: Date.now() + }) } + return (orig as (...a: unknown[]) => unknown)(...args) } - }) + } }) } diff --git a/packages/nightwatch-devtools/tests/browserProxy.test.ts b/packages/nightwatch-devtools/tests/browserProxy.test.ts index d69ffcf3..d6b7605d 100644 --- a/packages/nightwatch-devtools/tests/browserProxy.test.ts +++ b/packages/nightwatch-devtools/tests/browserProxy.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import type { CommandLog, NightwatchBrowser } from '../src/types.js' +import type { + CommandLog, + NightwatchBrowser, + NightwatchCurrentTest +} from '../src/types.js' // browserProxy resolves the caller's source via this helper; stub it so each // test can decide whether the command looks user-issued or framework-internal. @@ -10,6 +14,7 @@ const { getCallSourceFromStack } = vi.hoisted(() => ({ vi.mock('../src/helpers/utils.js', () => ({ getCallSourceFromStack })) import { BrowserProxy } from '../src/helpers/browserProxy.js' +import { captureNativeAssertions } from '../src/helpers/nativeAssertions.js' import type { SessionCapturer } from '../src/session.js' import type { TestManager } from '../src/helpers/testManager.js' @@ -39,16 +44,24 @@ function makeCapturer() { return true } ) + const captureAssertCommand = vi.fn( + (entry: CommandLog & { _id?: number; id?: number }) => { + entry._id = counter++ + entry.id = entry._id + commandsLog.push(entry) + } + ) const capturer = { commandsLog, captureCommand, + captureAssertCommand, replaceCommand: vi.fn(), sendCommand: vi.fn(), sendReplaceCommand: vi.fn(), takeScreenshotViaHttp: vi.fn(async () => null), captureTrace: vi.fn(async () => {}) } as unknown as SessionCapturer - return { capturer, commandsLog, captureCommand } + return { capturer, commandsLog, captureCommand, captureAssertCommand } } function makeTestManager() { @@ -166,3 +179,86 @@ describe('BrowserProxy captureAssertions gating', () => { expect(browser.verify).not.toBe(originalVerify) }) }) + +describe('BrowserProxy negated native assertions (assert.not.* / verify.not.*)', () => { + beforeEach(() => { + getCallSourceFromStack.mockReset() + }) + + /** Browser whose assert/verify mirror Nightwatch's structure: a positive + * namespace plus a nested `not` namespace object (Nightwatch exposes the + * negation as its own Proxy). Each leaf method records so the test can assert + * the wrapper still delegates to the real negated method. */ + function makeNegatableAssertBrowser() { + const record: string[] = [] + const ns = (label: string) => ({ + titleContains: vi.fn(() => { + record.push(`${label}titleContains`) + }) + }) + const browser = { + assert: { ...ns(''), not: ns('not.') }, + verify: { ...ns(''), not: ns('not.') } + } as unknown as NightwatchBrowser + return { browser, record } + } + + it('buffers a negated assert with a negation-reflecting label and finalizes its outcome', async () => { + const { capturer, commandsLog } = makeCapturer() + getCallSourceFromStack.mockReturnValue({ + filePath: '/tests/spec.js', + callSource: '/tests/spec.js:9' + }) + const proxy = new BrowserProxy(capturer, makeTestManager(), () => ({ + uid: 't1' + })) + const { browser, record } = makeNegatableAssertBrowser() + proxy.wrapBrowserCommands(browser) + + // browser.assert.not.titleContains('Example') — a negated call from user code. + ;( + browser as unknown as { + assert: { not: { titleContains: (a: unknown) => unknown } } + } + ).assert.not.titleContains('Example') + + // Delegated to the ORIGINAL negated method (Nightwatch semantics unchanged). + expect(record).toEqual(['not.titleContains']) + + // Buffered exactly one recorded call, keyed by its full dotted path — so the + // negation is captured through the SAME mechanism as positive asserts. + const calls = proxy.drainNativeAssertCalls() + expect(calls).toHaveLength(1) + expect(calls[0].prefix).toBe('assert') + expect(calls[0].method).toBe('not.titleContains') + expect(calls[0].args).toEqual(['Example']) + expect(calls[0].callSource).toBe('/tests/spec.js:9') + expect(calls[0].entry?.command).toBe('assert.not.titleContains') + expect(calls[0].entry?.title).toBe("assert.not.titleContains('Example')") + + // Reconciled at test-end through the same finalize path as positive asserts: + // a failing negated entry yields one row with the negated label + fail outcome. + const results = { + assertions: [ + { + message: "Testing if the page title doesn't contain 'Example'", + fullMsg: "Testing if the page title doesn't contain 'Example'", + failure: + "Testing if the page title doesn't contain 'Example' — failed" + } + ], + commands: [] + } + const currentTest = { + name: 't', + results + } as unknown as NightwatchCurrentTest + await captureNativeAssertions(capturer, browser, currentTest, 't1', calls) + + const row = commandsLog[commandsLog.length - 1] + expect(row.command).toBe('assert.not.titleContains') + expect(row.title).toBe("assert.not.titleContains('Example')") + expect(row.result).toMatchObject({ passed: false }) + expect(row.error).toBeDefined() + }) +}) From 0ce8e11b8ee9e00df16f8596c404c6cfcd0be3ba Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 15:06:05 +0530 Subject: [PATCH 55/91] refactor(service): extract assertion capture into AssertionTracker --- packages/service/src/assertion-tracker.ts | 187 ++++++++++++++++++++++ packages/service/src/index.ts | 171 +++----------------- 2 files changed, 205 insertions(+), 153 deletions(-) create mode 100644 packages/service/src/assertion-tracker.ts diff --git a/packages/service/src/assertion-tracker.ts b/packages/service/src/assertion-tracker.ts new file mode 100644 index 00000000..b0ab2f3d --- /dev/null +++ b/packages/service/src/assertion-tracker.ts @@ -0,0 +1,187 @@ +// Owns the expect-webdriverio matcher lifecycle for the WDIO worker: the matcher +// nesting depth and the pending-matcher state machine that turns a matcher call +// into a single trace row. Pure CommandLog shaping lives in ./assert-capture. + +import logger from '@wdio/logger' +import { errorMessage } from '@wdio/devtools-core' +import type { ActionSnapshot } from '@wdio/devtools-shared' +import { + captureExpectFailure, + expectAssertionToCommandLog, + isMatcherReadCommand, + type ExpectAssertion +} from './assert-capture.js' +import { pushActionSnapshotAt } from './action-snapshot.js' +import { isNativeMobile } from './mobile.js' +import type { SessionCapturer } from './session.js' +import type { ServiceOptions } from './types.js' + +const log = logger('@wdio/devtools-service:assertion-tracker') + +/** Live accessors into the owning service's state. The capturer is replaced in + * before() and the browser/test uid change per test, so each is read lazily. */ +export interface AssertionTrackerContext { + getCapturer: () => SessionCapturer + getBrowser: () => WebdriverIO.Browser | undefined + getTestUid: () => string | undefined + options: ServiceOptions + actionSnapshots: ActionSnapshot[] +} + +interface PendingAssertion { + matcherName: string + expectedValue?: unknown + testUid?: string +} + +export class AssertionTracker { + #ctx: AssertionTrackerContext + + /** expect-webdriverio matcher nesting depth. Aliases fire before/afterAssertion + * twice (toBeChecked→toBeSelected), so only the outermost pair owns the row. */ + #assertionDepth = 0 + + /** Matcher armed at beforeAssertion, cleared at its afterAssertion. It survives + * to test end only when the matcher hard-threw (element never resolved, so + * waitUntil rethrew and afterAssertion never fired) — then it's synthesized as + * a failing expect.<matcher> row instead of leaving a raw read. */ + #pendingAssertion?: PendingAssertion + + constructor(ctx: AssertionTrackerContext) { + this.#ctx = ctx + } + + /** Clear per-test matcher state; called from the service's resetStack. */ + reset(): void { + this.#assertionDepth = 0 + this.#pendingAssertion = undefined + } + + /** + * expect-webdriverio fires this at the START of every matcher, before it polls + * — so it fires even for a matcher that later hard-throws (element never + * resolved), unlike afterAssertion. Arm the pending matcher here so test-end + * can synthesize its expect row if afterAssertion never comes. Depth-counted: + * aliases (toBeChecked→toBeSelected) fire this twice, so only the outermost + * arms the row. + */ + beforeAssertion(params: { + matcherName: string + expectedValue?: unknown + }): void { + if (this.#ctx.options.captureAssertions === false) { + return + } + if (this.#assertionDepth === 0) { + this.#pendingAssertion = { + matcherName: params.matcherName, + expectedValue: params.expectedValue, + testUid: this.#ctx.getTestUid() + } + } + this.#assertionDepth++ + } + + /** + * expect-webdriverio fires this after each matcher evaluates, with the matcher + * name + pass/fail + expected value. The matcher's value-read (getText / + * isExisting / …) was captured as a normal command; fold this assertion into + * that read so one `expect.<matcher>` row remains — inheriting the read's real + * callSource, screenshot, and timeline position. Deterministic and anchored to + * this reliable hook: no `#assertionDepth`, no stack-frame detection. + */ + async afterAssertion(params: ExpectAssertion): Promise<void> { + if (this.#ctx.options.captureAssertions === false) { + return + } + this.#assertionDepth = Math.max(0, this.#assertionDepth - 1) + // Inner matcher of a nested pair (toBeChecked→toBeSelected): the outer + // afterAssertion owns the row. + if (this.#assertionDepth > 0) { + return + } + // Reached afterAssertion → the matcher resolved (pass or value-fail), so no + // hard-throw synthesis is needed at test end. + this.#pendingAssertion = undefined + const capturer = this.#ctx.getCapturer() + const entry = expectAssertionToCommandLog(params, this.#ctx.getTestUid()) + if (capturer.coalesceAssertionIntoLastRead(entry, isMatcherReadCommand)) { + return + } + // No matcher read to fold into (a value matcher like toBe(x), or the read + // hard-threw): emit a fresh row with its own screenshot + trace snapshot. + const browser = this.#ctx.getBrowser() + if (browser && !isNativeMobile(browser)) { + try { + entry.screenshot = await browser.takeScreenshot() + } catch (err) { + // best-effort: a missing screenshot must not fail the assertion hook + log.debug(`assertion screenshot skipped: ${errorMessage(err)}`) + } + } + if (this.#ctx.options.mode === 'trace' && browser) { + await pushActionSnapshotAt( + browser, + entry.command, + entry.timestamp, + this.#ctx.actionSnapshots + ) + } + capturer.captureAssertCommand(entry) + } + + /** Route a test/step failure to assertion capture. A matcher that hard-threw + * (element never resolved) left an armed pendingAssertion because + * afterAssertion never fired — synthesize its failing expect row. Any other + * failure just marks the last action with the error. */ + handleOutcome(error: unknown): void { + if (this.#ctx.options.captureAssertions === false) { + return + } + if (this.#pendingAssertion) { + this.#finalizePendingAssertion(error) + return + } + this.#captureExpectFailure(error) + } + + /** Mark the failing action from a matcher error (afterStep for Cucumber, + * afterTest for Mocha route here). */ + #captureExpectFailure(error: unknown): void { + captureExpectFailure( + this.#ctx.getCapturer(), + this.#ctx.getTestUid(), + error, + this.#ctx.options.captureAssertions !== false + ) + } + + /** Synthesize the failing expect.<matcher> row for a hard-thrown matcher: fold + * it into the throwing read (relabel `getText`→`expect.toHaveText`, keeping + * the error) so the assertion renders consistently whether or not the element + * resolved; fall back to a fresh row when there is no read to fold. */ + #finalizePendingAssertion(error: unknown): void { + const pending = this.#pendingAssertion + this.#pendingAssertion = undefined + this.#assertionDepth = 0 + if (!pending) { + return + } + const message = errorMessage(error) || `${pending.matcherName} failed` + const entry = expectAssertionToCommandLog( + { + matcherName: pending.matcherName, + expectedValue: pending.expectedValue, + result: { pass: false, message: () => message } + }, + pending.testUid + ) + const capturer = this.#ctx.getCapturer() + if ( + capturer.coalesceAssertionIntoLastRead(entry, isMatcherReadCommand, true) + ) { + return + } + capturer.captureAssertCommand(entry) + } +} diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 0b1bcf71..e2d770c1 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -12,13 +12,8 @@ import { type SpecRange, type TraceExportContext } from '@wdio/devtools-core' -import { - captureExpectFailure, - expectAssertionToCommandLog, - isMatcherReadCommand, - wireAssertCapture, - type ExpectAssertion -} from './assert-capture.js' +import { wireAssertCapture, type ExpectAssertion } from './assert-capture.js' +import { AssertionTracker } from './assertion-tracker.js' import { cucumberScenarioUid, resolveTestAttempt, @@ -30,8 +25,7 @@ import { resolveCallSourceFromFrame } from './call-source.js' import { flushPrevSlice, flushTestSlice } from './trace-slices.js' import { captureActionResult, - captureActionSnapshot, - pushActionSnapshotAt + captureActionSnapshot } from './action-snapshot.js' import { dedupeSnapshotsByTimestamp, @@ -80,9 +74,17 @@ export default class DevToolsHookService implements Services.ServiceInstance { #screencastOptions?: ScreencastOptions #options: ServiceOptions #actionSnapshots: ActionSnapshot[] = [] + #assertionTracker: AssertionTracker constructor(serviceOptions: ServiceOptions = {}) { this.#options = serviceOptions + this.#assertionTracker = new AssertionTracker({ + getCapturer: () => this.#sessionCapturer, + getBrowser: () => this.#browser, + getTestUid: () => this.#currentTestUid, + options: this.#options, + actionSnapshots: this.#actionSnapshots + }) const policyWarning = tracePolicyModeWarning( serviceOptions.tracePolicy, serviceOptions.mode @@ -107,20 +109,6 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Current test UID, set in beforeTest(), used by afterCommand() to tag commands. */ #currentTestUid?: string - /** expect-webdriverio matcher nesting depth. Aliases fire before/afterAssertion - * twice (toBeChecked→toBeSelected), so only the outermost pair owns the row. */ - #assertionDepth = 0 - - /** Matcher armed at beforeAssertion, cleared at its afterAssertion. It survives - * to test end only when the matcher hard-threw (element never resolved, so - * waitUntil rethrew and afterAssertion never fired) — then it's synthesized as - * a failing expect.<matcher> row instead of leaving a raw read. */ - #pendingAssertion?: { - matcherName: string - expectedValue?: unknown - testUid?: string - } - /** Map of testUid → metadata for trace group events and per-spec partitioning. */ #testMetadata: TestMetadataMap = new Map() @@ -405,65 +393,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { _scenario?: unknown, result?: { error?: unknown } ) { - this.#handleAssertionOutcome(result?.error) - } - - /** Route a test/step failure to assertion capture. A matcher that hard-threw - * (element never resolved) left an armed #pendingAssertion because - * afterAssertion never fired — synthesize its failing expect row. Any other - * failure just marks the last action with the error. */ - #handleAssertionOutcome(error: unknown): void { - if (this.#options.captureAssertions === false) { - return - } - if (this.#pendingAssertion) { - this.#finalizePendingAssertion(error) - return - } - this.#captureExpectFailure(error) - } - - /** Synthesize the failing expect.<matcher> row for a hard-thrown matcher: fold - * it into the throwing read (relabel `getText`→`expect.toHaveText`, keeping - * the error) so the assertion renders consistently whether or not the element - * resolved; fall back to a fresh row when there is no read to fold. */ - #finalizePendingAssertion(error: unknown): void { - const pending = this.#pendingAssertion - this.#pendingAssertion = undefined - this.#assertionDepth = 0 - if (!pending) { - return - } - const message = errorMessage(error) || `${pending.matcherName} failed` - const entry = expectAssertionToCommandLog( - { - matcherName: pending.matcherName, - expectedValue: pending.expectedValue, - result: { pass: false, message: () => message } - }, - pending.testUid - ) - if ( - this.#sessionCapturer.coalesceAssertionIntoLastRead( - entry, - isMatcherReadCommand, - true - ) - ) { - return - } - this.#sessionCapturer.captureAssertCommand(entry) - } - - /** Mark the failing action from a matcher error (afterStep for Cucumber, - * afterTest for Mocha route here). */ - #captureExpectFailure(error: unknown): void { - captureExpectFailure( - this.#sessionCapturer, - this.#currentTestUid, - error, - this.#options.captureAssertions !== false - ) + this.#assertionTracker.handleOutcome(result?.error) } /** Stamp final state + the resolved 0-based attempt onto the test's metadata @@ -498,7 +428,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { _context?: unknown, result?: TestOutcomeResult ) { - this.#handleAssertionOutcome(result?.error) + this.#assertionTracker.handleOutcome(result?.error) const testTitle = test?.fullTitle || test?.title const uid = testTitle ? testMetadataUid(test?.file, testTitle) : undefined if (uid) { @@ -511,80 +441,16 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** - * expect-webdriverio fires this after each matcher evaluates, with the matcher - * name + pass/fail + expected value. The matcher's value-read (getText / - * isExisting / …) was captured as a normal command; fold this assertion into - * that read so one `expect.<matcher>` row remains — inheriting the read's real - * callSource, screenshot, and timeline position. Deterministic and anchored to - * this reliable hook: no `#assertionDepth`, no stack-frame detection. - */ - /** - * expect-webdriverio fires this at the START of every matcher, before it polls - * — so it fires even for a matcher that later hard-throws (element never - * resolved), unlike afterAssertion. Arm the pending matcher here so test-end - * can synthesize its expect row if afterAssertion never comes. Depth-counted: - * aliases (toBeChecked→toBeSelected) fire this twice, so only the outermost - * arms the row. - */ + /** expect-webdriverio matcher hooks — delegated to the assertion tracker. */ beforeAssertion(params: { matcherName: string expectedValue?: unknown }): void { - if (this.#options.captureAssertions === false) { - return - } - if (this.#assertionDepth === 0) { - this.#pendingAssertion = { - matcherName: params.matcherName, - expectedValue: params.expectedValue, - testUid: this.#currentTestUid - } - } - this.#assertionDepth++ + this.#assertionTracker.beforeAssertion(params) } - async afterAssertion(params: ExpectAssertion): Promise<void> { - if (this.#options.captureAssertions === false) { - return - } - this.#assertionDepth = Math.max(0, this.#assertionDepth - 1) - // Inner matcher of a nested pair (toBeChecked→toBeSelected): the outer - // afterAssertion owns the row. - if (this.#assertionDepth > 0) { - return - } - // Reached afterAssertion → the matcher resolved (pass or value-fail), so no - // hard-throw synthesis is needed at test end. - this.#pendingAssertion = undefined - const entry = expectAssertionToCommandLog(params, this.#currentTestUid) - if ( - this.#sessionCapturer.coalesceAssertionIntoLastRead( - entry, - isMatcherReadCommand - ) - ) { - return - } - // No matcher read to fold into (a value matcher like toBe(x), or the read - // hard-threw): emit a fresh row with its own screenshot + trace snapshot. - if (this.#browser && !isNativeMobile(this.#browser)) { - try { - entry.screenshot = await this.#browser.takeScreenshot() - } catch (err) { - // best-effort: a missing screenshot must not fail the assertion hook - log.debug(`assertion screenshot skipped: ${errorMessage(err)}`) - } - } - if (this.#options.mode === 'trace' && this.#browser) { - await pushActionSnapshotAt( - this.#browser, - entry.command, - entry.timestamp, - this.#actionSnapshots - ) - } - this.#sessionCapturer.captureAssertCommand(entry) + afterAssertion(params: ExpectAssertion): Promise<void> { + return this.#assertionTracker.afterAssertion(params) } async #finalizePerScenario() { @@ -615,8 +481,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { private resetStack() { this.#commandStack = [] - this.#assertionDepth = 0 - this.#pendingAssertion = undefined + this.#assertionTracker.reset() this.#sessionCapturer.resetLastSelector() this.#sessionCapturer.resetRetryTracker() } From d0f84a561cba06c52fce52c16a5bdb0b6b754e4b Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 16:37:17 +0530 Subject: [PATCH 56/91] docs: sync WDIO assert-capture debt to the fold model; drop fixed entries --- CLAUDE.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1f33103e..007cc486 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -233,22 +233,19 @@ Documented divergences from the conventions above. They exist today as debt to b ### Architecture - `replaceCommand` has two semantics — Selenium mutates in place (preserves `_id`/`id` for chained calls); Nightwatch splices and reissues. Both call the same `core/suite-helpers` factories; the storage strategy stays adapter-specific because runner integrations differ. Could be unified by parameterizing the policy if the divergence ever causes a real problem. -- `patchNodeAssert` (via `core/assert-patcher`) is now wired in all three adapters, default-on behind each adapter's `captureAssertions` option (opt out with `captureAssertions: false`). Framework matcher libraries differ: Service additionally taps expect-webdriverio's `afterAssertion` hook so passing+failing matchers render as `expect.*` actions; Selenium (chai/jest) and Nightwatch (`browser.assert`/`verify`) don't yet surface passing framework-matcher assertions as trace actions. +- `patchNodeAssert` (via `core/assert-patcher`) is now wired in all three adapters, default-on behind each adapter's `captureAssertions` option (opt out with `captureAssertions: false`). Framework matcher libraries differ: Service taps expect-webdriverio's `beforeAssertion`/`afterAssertion` hooks so passing+failing matchers render as `expect.*` actions (mechanism in the assert-capture entry below); Nightwatch native `assert`/`verify` and Selenium's `node:assert` also surface passing+failing rows via their reconcile/patch paths. The remaining gap is Selenium's jest-style `expect()` (and chai): jest/vitest expose no pass+fail assertion hook, so only failing matchers surface there. - BiDi is auto-attached in Service and Selenium; Nightwatch is opt-in via `bidi: true` and requires `webSocketUrl: true` in capabilities. - Retry-aware trace policies aren't uniform across adapters. Selenium emits one metadata entry per attempt (counter-based `generateStableUid` uid + retry-stable tracker key), so all six policies — including `retain-on-first-failure` (needs the first attempt's outcome) — evaluate correctly. Service and Nightwatch key metadata on the retry-stable uid and overwrite/mutate per attempt, collapsing to the *final* attempt's outcome; the retry-count policies (`retain-on-failure`, `on-first-retry`, `on-all-retries`, `retain-on-failure-and-retries`) are still correct, but `retain-on-first-failure` can't see a failed-then-passed first attempt and under-retains. Proper fix rides with B5 (per-test granularity preserves per-attempt slices); until then all three set `attemptInfoAvailable: true`. -- Nightwatch negated native asserts (`browser.assert.not.*` / `verify.not.*`) produce no trace row: the `not` namespace is a nested Proxy that `wrapAssertionNamespaces` passes through untouched, and the internal command it issues is dropped by the `hasUserSource` gate in `browserProxy`. Positive `assert.*`/`verify.*` still record. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. - Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. -- Service assertion-suppression self-heal is command-triggered and gated on `#matcherStarted`: a swallowed soft-assert (`expect.soft` / user `try/catch`) whose matcher hard-throws leaves `#assertionDepth` stuck; the next top-level user command with no `expect-webdriverio` frame heals it — but only once the matcher has issued at least one internal (ewdio-framed) command, because the matcher's own FIRST command (an element re-lookup) also lacks a matcher frame on its sync stack and would otherwise reset the depth mid-run (duplicating every later matcher command). A stuck depth with no following command before test end persists until the next `beforeAssertion`/`resetStack()` — benign. -- Service captures the assertion **target's** element resolution as its own row. `expect(page.el).toHaveText(x)` evaluates `page.el` → `$('#sel')`/`$$` as the ARGUMENT to `expect()`, before `beforeAssertion` fires, so `#assertionDepth` is 0 and the `$`/`$$`/`isExisting` lands as a top-level command alongside the `expect.*` row. The in-window suppression can't reach it (it runs pre-window). Proper fix: coalesce a trailing element-query into the imminent assertion, or detect query-commands whose only consumer is the next matcher. -- Service `expect.*` rows can resolve `callSource` to the service bundle (`dist/index.js`) instead of the user's `expect()` line. `#assertionCallSources` captures the spec frame in `beforeAssertion`, but the matcher runs async, so in some cases the parallel stack doesn't hold a user frame and the fallback resolves into the service dist. The failure's own error stack still points at the spec. +- Service renders expect-webdriverio matchers as single `expect.<matcher>` rows by **folding**, not stack/depth suppression (the old `#assertionDepth`/`#matcherStarted`/self-heal machinery is gone). The matcher's value-read (`toHaveText`→`getText`, `toExist`→`isExisting`, …) is captured as a normal command; `afterAssertion` then coalesces the synthesized `expect.*` row into that read in place — inheriting its callSource, screenshot, and timeline position — and the fold replaces **by timestamp, never a public `id`**: `id` is the per-worker `commandCounter`, which resets per spec, so stamping one lets the app's id-first `replaceCommand` swap a same-id row from another spec (duplicate rows + a fold from another spec vanishing, in multi-spec live mode). `beforeAssertion` arms the pending matcher (depth-counted so aliases like `toBeChecked`→`toBeSelected` fold once); a matcher that **hard-throws** — element never resolves, so expect-webdriverio's `waitUntil` rethrows and `afterAssertion` never fires — is synthesized at `afterTest`/`afterStep` from the throwing read, so a failing assertion renders as `expect.<matcher>` whether or not the element existed. Two limits: its error is then the read's (`Can't call getText on … element wasn't found`), not an assertion-phrased message; and `MATCHER_READ_COMMANDS` is a hand-maintained allowlist, so a matcher whose read isn't listed leaves its raw read visible alongside the `expect.*` row. Plain-value jest matchers (`expect(x).toBe(y)`) don't fire the ewdio hooks, so they aren't captured as rows. ### File-size (raw line counts; soft cap is 500 logic lines) Most entries below don't trigger the `max-lines` lint rule after `skipBlankLines`/`skipComments`; they're documented because their raw line count is over 500, and the next substantive change to any of them should still look for an extraction opportunity. The service plugin is the exception — it's now over the *logic*-line cap. -- `packages/service/src/index.ts` — over the 500-**logic**-line soft cap (the WDIO plugin god-file: session/screencast/command/assertion/trace-slice wiring in one class). Split candidate; the trace-slice, assertion-capture, and screencast concerns are the natural extraction seams. +- `packages/service/src/index.ts` — over the 500-**logic**-line soft cap and ~620 raw (the WDIO plugin god-file: session/screencast/command/assertion/trace-slice wiring in one class). The assertion-capture block (`beforeAssertion`/`afterAssertion`/`#handleAssertionOutcome`/`#finalizePendingAssertion`, plus the `#assertionDepth`/`#pendingAssertion` state) is now the largest self-contained seam and the prime extraction candidate; trace-slice and screencast are the next two. - `packages/nightwatch-devtools/src/index.ts` (~536 raw). Cucumber/test/run-lifecycle, session-init, event-hub modules already extracted; remainder is the `PluginInternals` accessor bag plus per-method delegators plus the factory. Accept-as-is. - `packages/selenium-devtools/src/index.ts` (~560 raw). Session/test-lifecycle extracted; remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Same situation as nightwatch. From e6cd94909fc4b0763435a71bbdd71e75cd99514d Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 17:35:59 +0530 Subject: [PATCH 57/91] feat(core): per-attempt outcome ledger + group-by-test retention --- packages/core/src/attempt-tracker.ts | 97 +++++++++++++++++---- packages/core/src/trace-finalizer.ts | 32 ++++++- packages/core/src/trace-retention.ts | 60 +++++++++++-- packages/core/tests/attempt-tracker.test.ts | 75 ++++++++++++++++ packages/core/tests/trace-finalizer.test.ts | 44 ++++++++++ packages/core/tests/trace-retention.test.ts | 44 ++++++++++ 6 files changed, 326 insertions(+), 26 deletions(-) diff --git a/packages/core/src/attempt-tracker.ts b/packages/core/src/attempt-tracker.ts index e4915584..60bbdfa4 100644 --- a/packages/core/src/attempt-tracker.ts +++ b/packages/core/src/attempt-tracker.ts @@ -1,29 +1,71 @@ +import type { TestStatus } from '@wdio/devtools-shared' +import type { TestOutcome } from './trace-retention.js' + /** - * Framework-agnostic per-test attempt counting. Every supported runner - * re-enters its per-test start hook when a test is retried, so recording the - * same uid again yields an incremented attempt number (0-based: the first run - * is attempt 0, the first retry is attempt 1). This is the primary, - * runner-independent retry signal feeding `TestOutcome.attempt` for the - * retry-aware trace policies (see trace-retention.ts). + * Framework-agnostic per-test attempt ledger. Every supported runner re-enters + * its per-test start hook when a test is retried, so recording the same uid + * again appends a new attempt slot (0-based: first run is attempt 0, first retry + * is attempt 1). Once the outcome is known the adapter stamps it onto the slot. + * This is the primary, runner-independent retry signal feeding the retry-aware + * trace policies — the finalizer reads the scoped views to evaluate retention + * per test/spec/session with real per-attempt outcomes (see trace-retention.ts). */ -export class TestAttemptTracker { - #attempts = new Map<string, number>() +export interface RetryOutcomeView { + /** Every attempt of every test — for session-scope retention. */ + all(): TestOutcome[] + /** Every attempt of the tests recorded against `specFile` — for spec scope. */ + forSpec(specFile: string): TestOutcome[] + /** A single test's attempts; pass `attempt` to scope to one attempt's slice. */ + forTest(uid: string, attempt?: number): TestOutcome[] +} + +export class TestAttemptTracker implements RetryOutcomeView { + #ledger = new Map<string, { specFile?: string; attempts: TestOutcome[] }>() #sawRetry = false - /** Record a starting test; returns its attempt number (0 first, +1 per rerun). */ - recordStart(uid: string): number { - const prior = this.#attempts.get(uid) - const attempt = prior === undefined ? 0 : prior + 1 - this.#attempts.set(uid, attempt) + /** Record a starting test; returns its attempt number (0 first, +1 per rerun). + * `specFile` (optional) enables spec-scoped retention lookups. */ + recordStart(uid: string, specFile?: string): number { + const entry = this.#ledger.get(uid) + const attempt = entry ? entry.attempts.length : 0 if (attempt > 0) { this.#sawRetry = true } + const slot: TestOutcome = { uid, attempt } + if (entry) { + entry.attempts.push(slot) + if (specFile !== undefined) { + entry.specFile = specFile + } + } else { + this.#ledger.set(uid, { specFile, attempts: [slot] }) + } return attempt } + /** Stamp the resolved state onto uid's most recent attempt slot once the + * outcome is known. `attempt` overrides the slot's number when the adapter + * resolved a more authoritative value (e.g. WDIO's result.retries). */ + recordOutcome( + uid: string, + state: TestStatus | undefined, + attempt?: number + ): void { + const attempts = this.#ledger.get(uid)?.attempts + const slot = attempts?.[attempts.length - 1] + if (!slot) { + return + } + slot.state = state + if (attempt !== undefined) { + slot.attempt = attempt + } + } + /** Latest attempt recorded for `uid`, or undefined if it never started. */ attemptFor(uid: string): number | undefined { - return this.#attempts.get(uid) + const entry = this.#ledger.get(uid) + return entry ? entry.attempts.length - 1 : undefined } /** True once any test has started more than once (a retry occurred). */ @@ -31,9 +73,34 @@ export class TestAttemptTracker { return this.#sawRetry } + all(): TestOutcome[] { + const out: TestOutcome[] = [] + for (const entry of this.#ledger.values()) { + out.push(...entry.attempts) + } + return out + } + + forSpec(specFile: string): TestOutcome[] { + const out: TestOutcome[] = [] + for (const entry of this.#ledger.values()) { + if (entry.specFile === specFile) { + out.push(...entry.attempts) + } + } + return out + } + + forTest(uid: string, attempt?: number): TestOutcome[] { + const attempts = this.#ledger.get(uid)?.attempts ?? [] + return attempt === undefined + ? attempts + : attempts.filter((a) => a.attempt === attempt) + } + /** Clear all state — used at session boundaries and reuse-mode reconnects. */ reset(): void { - this.#attempts.clear() + this.#ledger.clear() this.#sawRetry = false } } diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index f0459590..a07286af 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -23,6 +23,7 @@ import { type SpecRange } from './spec-trace-helpers.js' import { shouldRetainTrace, type TestOutcome } from './trace-retention.js' +import type { RetryOutcomeView } from './attempt-tracker.js' import { writeTraceZip, type TraceCapturer } from './trace-exporter.js' /** One artifact produced (or, when `retained` is false, decided-against) by a @@ -45,6 +46,13 @@ export interface TraceExportContext { /** True when the adapter fed real per-test attempt numbers (B4); retry-aware * policies degrade to retain-on-failure when this is false. */ attemptInfoAvailable?: boolean + /** Per-attempt outcome ledger. When present, retention is evaluated against + * real per-attempt outcomes (scoped per session/spec/test) instead of the + * collapsed final-attempt state in `testMetadata` — this is what lets + * retain-on-first-failure see a failed-then-passed first attempt and stops + * retain-on-failure over-retaining it. Absent → falls back to `testMetadata` + * (unchanged behavior). */ + outcomes?: RetryOutcomeView granularity?: TraceGranularity format?: TraceFormat capturer: TraceCapturer @@ -153,14 +161,22 @@ function toOutcomes(metadata: TestMetadataMap): TestOutcome[] { */ function shouldRetain( ctx: TraceExportContext, - metadata: TestMetadataMap + metadata: TestMetadataMap, + ledgerOutcomes?: TestOutcome[] ): boolean { return shouldRetainTrace(ctx.policy, { - outcomes: toOutcomes(metadata), + outcomes: ledgerOutcomes ?? toOutcomes(metadata), attemptInfoAvailable: ctx.attemptInfoAvailable ?? false }).retain } +/** Attempt number encoded in a test-slice key (`<uid>-retry<n>`); 0 when the + * key is the base uid (the first attempt / no retry suffix). */ +function attemptFromKey(key: string): number { + const match = /-retry(\d+)$/.exec(key) + return match ? Number(match[1]) : 0 +} + /** * Policy-aware single-range flush: dedupes via `ctx.flushed` on the slice * `key`, applies the retention decision, and delegates the byte-level @@ -182,13 +198,21 @@ export async function flushRangeTrace( const sliceMetadata = isTestSlice ? filterTestMetadataByUid(ctx.testMetadata, range.testUid!) : filterTestMetadataBySpec(ctx.testMetadata, range.specFile) + // Scope the per-attempt ledger to this slice: a test slice sees only its own + // attempt (so a passing retry's slice isn't retained on first-failure), a spec + // slice sees every attempt of its tests. + const sliceOutcomes = ctx.outcomes + ? isTestSlice + ? ctx.outcomes.forTest(range.testUid!, attemptFromKey(range.key)) + : ctx.outcomes.forSpec(range.specFile) + : undefined const artifact: TraceArtifact = { kind: 'trace', path: '', scope: isTestSlice ? 'test' : 'spec', key: range.key, testUids: Array.from(sliceMetadata.keys()), - retained: shouldRetain(ctx, sliceMetadata) + retained: shouldRetain(ctx, sliceMetadata, sliceOutcomes) } if (!artifact.retained) { ctx.onArtifact?.(artifact) @@ -255,7 +279,7 @@ async function writeSessionTrace( path: '', scope: 'session', testUids: Array.from(ctx.testMetadata.keys()), - retained: shouldRetain(ctx, ctx.testMetadata) + retained: shouldRetain(ctx, ctx.testMetadata, ctx.outcomes?.all()) } if (!artifact.retained) { ctx.onArtifact?.(artifact) diff --git a/packages/core/src/trace-retention.ts b/packages/core/src/trace-retention.ts index 6688dfcb..cdc35013 100644 --- a/packages/core/src/trace-retention.ts +++ b/packages/core/src/trace-retention.ts @@ -23,6 +23,10 @@ const KNOWN_POLICIES = new Set<TraceRetentionPolicy>([ ]) export interface TestOutcome { + /** Retry-stable test identity. Outcomes sharing a uid are one test's attempts + * (attempt 0, 1, …); an outcome with no uid is its own single-attempt group, + * so a flat/uid-less feed evaluates exactly as one-outcome-per-test. */ + uid?: string state?: TestStatus attempt?: number } @@ -56,26 +60,68 @@ export function shouldRetainTrace( if (!input.attemptInfoAvailable && policy !== 'retain-on-failure') { return { retain: anyFailed, degradedToFailure: true } } + // Group a test's attempts so failure policies key on the RIGHT attempt: the + // *final* attempt (retain-on-failure — a fail-then-pass ends passed) vs the + // *first* attempt (retain-on-first-failure). A uid-less feed makes every + // outcome its own group, so this reduces to the flat one-outcome-per-test + // logic and is byte-identical for callers that don't supply per-attempt uids. + const groups = groupByTest(outcomes) switch (policy) { case 'retain-on-failure': - return { retain: anyFailed } + return { retain: groups.some((g) => finalAttempt(g).state === 'failed') } case 'retain-on-first-failure': return { - retain: outcomes.some( - (o) => o.state === 'failed' && (o.attempt ?? 0) === 0 - ) + retain: groups.some((g) => firstAttempt(g)?.state === 'failed') } case 'on-first-retry': - return { retain: outcomes.some((o) => o.attempt === 1) } + return { retain: groups.some((g) => g.some((o) => o.attempt === 1)) } case 'on-all-retries': - return { retain: outcomes.some((o) => (o.attempt ?? 0) >= 1) } + return { retain: groups.some((g) => g.some((o) => attemptOf(o) >= 1)) } case 'retain-on-failure-and-retries': return { - retain: anyFailed || outcomes.some((o) => (o.attempt ?? 0) >= 1) + retain: groups.some( + (g) => + finalAttempt(g).state === 'failed' || + g.some((o) => attemptOf(o) >= 1) + ) } } } +const attemptOf = (o: TestOutcome): number => o.attempt ?? 0 + +/** Group outcomes by `uid`; a uid-less outcome becomes its own singleton group + * (preserving flat, one-outcome-per-test evaluation for callers without uids). */ +function groupByTest(outcomes: TestOutcome[]): TestOutcome[][] { + const byUid = new Map<string, TestOutcome[]>() + const groups: TestOutcome[][] = [] + for (const outcome of outcomes) { + if (outcome.uid === undefined) { + groups.push([outcome]) + continue + } + const existing = byUid.get(outcome.uid) + if (existing) { + existing.push(outcome) + } else { + const group = [outcome] + byUid.set(outcome.uid, group) + groups.push(group) + } + } + return groups +} + +/** The highest-numbered attempt's outcome — the test's final result. */ +function finalAttempt(group: TestOutcome[]): TestOutcome { + return group.reduce((best, o) => (attemptOf(o) >= attemptOf(best) ? o : best)) +} + +/** The attempt-0 outcome, if the group recorded one. */ +function firstAttempt(group: TestOutcome[]): TestOutcome | undefined { + return group.find((o) => attemptOf(o) === 0) +} + /** * Warning text when a retention policy is configured outside trace mode, where * it has no effect (the finalizer no-ops in live mode). Returns undefined when diff --git a/packages/core/tests/attempt-tracker.test.ts b/packages/core/tests/attempt-tracker.test.ts index 5d9a44b4..012fbaf7 100644 --- a/packages/core/tests/attempt-tracker.test.ts +++ b/packages/core/tests/attempt-tracker.test.ts @@ -43,4 +43,79 @@ describe('TestAttemptTracker', () => { expect(t.attemptFor('a')).toBeUndefined() expect(t.recordStart('a')).toBe(0) }) + + describe('outcome ledger', () => { + it('records per-attempt outcomes with uid + attempt for a retried test', () => { + const t = new TestAttemptTracker() + t.recordStart('a', 'login.spec.ts') + t.recordOutcome('a', 'failed') + t.recordStart('a', 'login.spec.ts') + t.recordOutcome('a', 'passed') + expect(t.forTest('a')).toEqual([ + { uid: 'a', attempt: 0, state: 'failed' }, + { uid: 'a', attempt: 1, state: 'passed' } + ]) + }) + + it('recordOutcome stamps only the latest attempt slot', () => { + const t = new TestAttemptTracker() + t.recordStart('a') + t.recordOutcome('a', 'passed') + t.recordStart('a') + t.recordOutcome('a', 'failed') + expect(t.forTest('a').map((o) => o.state)).toEqual(['passed', 'failed']) + }) + + it('recordOutcome can override the slot attempt (authoritative retry #)', () => { + const t = new TestAttemptTracker() + t.recordStart('a') + t.recordOutcome('a', 'failed', 3) + expect(t.forTest('a')).toEqual([ + { uid: 'a', attempt: 3, state: 'failed' } + ]) + }) + + it('forTest(uid, attempt) scopes to one attempt (per-slice retention)', () => { + const t = new TestAttemptTracker() + t.recordStart('a') + t.recordOutcome('a', 'failed') + t.recordStart('a') + t.recordOutcome('a', 'passed') + expect(t.forTest('a', 0)).toEqual([ + { uid: 'a', attempt: 0, state: 'failed' } + ]) + expect(t.forTest('a', 1)).toEqual([ + { uid: 'a', attempt: 1, state: 'passed' } + ]) + }) + + it('all() and forSpec() gather attempts across tests', () => { + const t = new TestAttemptTracker() + t.recordStart('a', 'one.spec.ts') + t.recordOutcome('a', 'failed') + t.recordStart('a', 'one.spec.ts') + t.recordOutcome('a', 'passed') + t.recordStart('b', 'two.spec.ts') + t.recordOutcome('b', 'passed') + expect(t.all()).toHaveLength(3) + expect(t.forSpec('one.spec.ts').map((o) => o.uid)).toEqual(['a', 'a']) + expect(t.forSpec('two.spec.ts')).toEqual([ + { uid: 'b', attempt: 0, state: 'passed' } + ]) + }) + + it('recordOutcome is a safe no-op for an unstarted uid', () => { + const t = new TestAttemptTracker() + expect(() => t.recordOutcome('ghost', 'failed')).not.toThrow() + expect(t.forTest('ghost')).toEqual([]) + }) + + it('reset clears the ledger too', () => { + const t = new TestAttemptTracker() + t.recordStart('a') + t.recordOutcome('a', 'failed') + t.reset() + expect(t.all()).toEqual([]) + }) + }) }) diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index 6d2b5013..a73df5d8 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -7,6 +7,7 @@ import { buildTestSliceFolder, finalizeTraceExport, flushRangeTrace, + TestAttemptTracker, type SpecRange, type TraceArtifact, type TraceCapturer, @@ -374,6 +375,49 @@ describe('finalizeTraceExport', () => { ) }) + // The ledger fix: collapsed testMetadata carries only the final (passed) + // attempt, so a metadata-only feed can't see the failed first attempt. The + // per-attempt ledger supplies both, so the failure policies key correctly. + function failThenPassLedger(): TestAttemptTracker { + const ledger = new TestAttemptTracker() + ledger.recordStart('u1', '/a.js') + ledger.recordOutcome('u1', 'failed') + ledger.recordStart('u1', '/a.js') + ledger.recordOutcome('u1', 'passed') + return ledger + } + const finalPassedMeta = meta([ + ['u1', { title: 'T', specFile: '/a.js', state: 'passed', attempt: 1 }] + ]) + + it('retain-on-first-failure retains a fail-then-pass via the ledger', async () => { + const result = await finalizeTraceExport( + baseCtx({ + policy: 'retain-on-first-failure', + attemptInfoAvailable: true, + outcomes: failThenPassLedger(), + testMetadata: finalPassedMeta + }) + ) + expect(result[0]!.retained).toBe(true) + expect(await exists(path.join(outputDir, 'trace-abcd1234.zip'))).toBe( + true + ) + }) + + it('retain-on-failure does NOT retain a fail-then-pass (final attempt passed)', async () => { + const result = await finalizeTraceExport( + baseCtx({ + policy: 'retain-on-failure', + attemptInfoAvailable: true, + outcomes: failThenPassLedger(), + testMetadata: finalPassedMeta + }) + ) + expect(result[0]!.retained).toBe(false) + expect(result[0]!.path).toBe('') + }) + // Session slice = OR over every test: one failure keeps the whole trace. it('session slice retains when ANY test failed under retain-on-failure', async () => { const result = await finalizeTraceExport( diff --git a/packages/core/tests/trace-retention.test.ts b/packages/core/tests/trace-retention.test.ts index f2e34f82..00982ea2 100644 --- a/packages/core/tests/trace-retention.test.ts +++ b/packages/core/tests/trace-retention.test.ts @@ -216,6 +216,50 @@ describe('shouldRetainTrace unknown policy (fail open)', () => { }) }) +describe('shouldRetainTrace groups a test’s attempts by uid', () => { + // One test, failed then passed on retry. + const failThenPass: TestOutcome[] = [ + { uid: 't', state: 'failed', attempt: 0 }, + { uid: 't', state: 'passed', attempt: 1 } + ] + // One test, passed then failed on retry. + const passThenFail: TestOutcome[] = [ + { uid: 't', state: 'passed', attempt: 0 }, + { uid: 't', state: 'failed', attempt: 1 } + ] + + it('retain-on-failure keys on the FINAL attempt, not any attempt', () => { + // Ends passed → not retained (flat any-failed logic over-retained here). + expect(retain('retain-on-failure', failThenPass)).toBe(false) + // Ends failed → retained. + expect(retain('retain-on-failure', passThenFail)).toBe(true) + }) + + it('retain-on-first-failure keys on attempt 0', () => { + // First attempt failed → retained even though it later passed. + expect(retain('retain-on-first-failure', failThenPass)).toBe(true) + // First attempt passed → not retained. + expect(retain('retain-on-first-failure', passThenFail)).toBe(false) + }) + + it('treats distinct uids as independent groups', () => { + const mixed: TestOutcome[] = [ + { uid: 'a', state: 'failed', attempt: 0 }, + { uid: 'a', state: 'passed', attempt: 1 }, + { uid: 'b', state: 'passed', attempt: 0 } + ] + // a ends passed, b passed → nothing to retain on final-attempt failure. + expect(retain('retain-on-failure', mixed)).toBe(false) + // a's first attempt failed → retained. + expect(retain('retain-on-first-failure', mixed)).toBe(true) + }) + + it('retry-count policies see the grouped attempts', () => { + expect(retain('on-first-retry', failThenPass)).toBe(true) + expect(retain('on-all-retries', passThenFail)).toBe(true) + }) +}) + describe('tracePolicyModeWarning', () => { it('warns when a policy is set outside trace mode', () => { expect(tracePolicyModeWarning('retain-on-failure', 'live')).toMatch( From a06669f4a94dc3fac311930ef8f72c49074607f4 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 17:36:16 +0530 Subject: [PATCH 58/91] fix(service): feed the retention ledger for correct retry-aware policies --- packages/service/src/index.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index e2d770c1..be45c5f3 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -185,6 +185,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { capabilities: browser.capabilities, testMetadata: this.#testMetadata, attemptInfoAvailable: true, + outcomes: this.#attemptTracker, ranges: this.#specRanges, flushed: this.#flushedSpecs, resolveOutputDir: () => this.#outputDir, @@ -354,7 +355,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { // ── Test identity for command tagging ── if (uid && scenarioName && featureFile) { this.#currentTestUid = uid - this.#attemptTracker.recordStart(uid) + this.#attemptTracker.recordStart(uid, featureFile) this.#testMetadata.set(uid, { title: scenarioName, specFile: featureFile @@ -378,7 +379,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (uid && testTitle) { this.#currentTestUid = uid - this.#attemptTracker.recordStart(uid) + this.#attemptTracker.recordStart(uid, test?.file) this.#testMetadata.set(uid, { title: testTitle, specFile: test?.file ?? '' @@ -402,6 +403,13 @@ export default class DevToolsHookService implements Services.ServiceInstance { const fallback = this.#attemptTracker.attemptFor(uid) ?? 0 const attempt = resolveTestAttempt(result, fallback) stampTestState(this.#testMetadata, uid, result, attempt) + // Feed the per-attempt ledger so session/spec retention sees this attempt's + // real outcome, not just the final state that overwrites #testMetadata. + this.#attemptTracker.recordOutcome( + uid, + this.#testMetadata.get(uid)?.state, + attempt + ) } async afterScenario( From 8a3d609ef184d329f15a47f7a1e972427d9ec612 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 17:45:52 +0530 Subject: [PATCH 59/91] fix(core): fall back to metadata when a scoped retention ledger view is empty --- packages/core/src/trace-finalizer.ts | 5 ++++- packages/core/tests/trace-finalizer.test.ts | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index a07286af..d69a2898 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -164,8 +164,11 @@ function shouldRetain( metadata: TestMetadataMap, ledgerOutcomes?: TestOutcome[] ): boolean { + // An empty scoped ledger view (adapter fed outcomes but not for this scope) + // falls back to metadata rather than fail-opening the evaluator into retaining + // everything — only a genuinely empty metadata slice fails open. return shouldRetainTrace(ctx.policy, { - outcomes: ledgerOutcomes ?? toOutcomes(metadata), + outcomes: ledgerOutcomes?.length ? ledgerOutcomes : toOutcomes(metadata), attemptInfoAvailable: ctx.attemptInfoAvailable ?? false }).retain } diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index a73df5d8..d25e30b8 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -418,6 +418,27 @@ describe('finalizeTraceExport', () => { expect(result[0]!.path).toBe('') }) + it('falls back to metadata when the scoped ledger view is empty', async () => { + // outcomes present but this scope's view is empty (e.g. an adapter that + // didn't feed this scope): must use metadata, not fail-open into retaining + // a passing test. + const emptyView = { all: () => [], forSpec: () => [], forTest: () => [] } + const result = await finalizeTraceExport( + baseCtx({ + policy: 'retain-on-failure', + attemptInfoAvailable: true, + outcomes: emptyView, + testMetadata: meta([ + [ + 'u1', + { title: 'T', specFile: '/a.js', state: 'passed', attempt: 0 } + ] + ]) + }) + ) + expect(result[0]!.retained).toBe(false) + }) + // Session slice = OR over every test: one failure keeps the whole trace. it('session slice retains when ANY test failed under retain-on-failure', async () => { const result = await finalizeTraceExport( From 1c9b00652a44c8d5a13112857d7c6666cf0a3864 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 19:11:52 +0530 Subject: [PATCH 60/91] fix(core): infer the failed attempt from a retry in the outcome ledger (message given earlier) --- packages/core/src/attempt-tracker.ts | 9 +++++++-- packages/core/tests/attempt-tracker.test.ts | 20 +++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/core/src/attempt-tracker.ts b/packages/core/src/attempt-tracker.ts index 60bbdfa4..6f6f8bc1 100644 --- a/packages/core/src/attempt-tracker.ts +++ b/packages/core/src/attempt-tracker.ts @@ -27,10 +27,15 @@ export class TestAttemptTracker implements RetryOutcomeView { * `specFile` (optional) enables spec-scoped retention lookups. */ recordStart(uid: string, specFile?: string): number { const entry = this.#ledger.get(uid) - const attempt = entry ? entry.attempts.length : 0 - if (attempt > 0) { + if (entry && entry.attempts.length > 0) { this.#sawRetry = true + // A retry only follows a failure. Some runners (e.g. Mocha through a + // --require plugin) can't observe the failed attempt's state from their + // outcome hook and report it as passed — the retry starting IS the + // reliable failure signal, so stamp the prior attempt failed here. + entry.attempts[entry.attempts.length - 1].state = 'failed' } + const attempt = entry ? entry.attempts.length : 0 const slot: TestOutcome = { uid, attempt } if (entry) { entry.attempts.push(slot) diff --git a/packages/core/tests/attempt-tracker.test.ts b/packages/core/tests/attempt-tracker.test.ts index 012fbaf7..845e6f8b 100644 --- a/packages/core/tests/attempt-tracker.test.ts +++ b/packages/core/tests/attempt-tracker.test.ts @@ -57,13 +57,27 @@ describe('TestAttemptTracker', () => { ]) }) - it('recordOutcome stamps only the latest attempt slot', () => { + it('recordOutcome stamps the latest slot', () => { const t = new TestAttemptTracker() t.recordStart('a') t.recordOutcome('a', 'passed') + expect(t.forTest('a')).toEqual([ + { uid: 'a', attempt: 0, state: 'passed' } + ]) + }) + + it('a retry stamps the prior attempt failed (swallowed-failure runners)', () => { + // Mocha via a --require plugin reports the retried attempt as passed; the + // retry starting is the reliable failure signal, so attempt 0 is corrected. + const t = new TestAttemptTracker() t.recordStart('a') - t.recordOutcome('a', 'failed') - expect(t.forTest('a').map((o) => o.state)).toEqual(['passed', 'failed']) + t.recordOutcome('a', 'passed') // outcome hook couldn't see the failure + t.recordStart('a') // retry ⟹ attempt 0 must have failed + t.recordOutcome('a', 'passed') // attempt 1 genuinely passed + expect(t.forTest('a')).toEqual([ + { uid: 'a', attempt: 0, state: 'failed' }, + { uid: 'a', attempt: 1, state: 'passed' } + ]) }) it('recordOutcome can override the slot attempt (authoritative retry #)', () => { From 25754bec5c74db5126d3b9348fac8c1d6a0deef7 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 19:12:25 +0530 Subject: [PATCH 61/91] fix(nightwatch): feed the retention ledger for correct retry-aware policies --- .../src/cucumber-lifecycle.ts | 8 ++- .../src/helpers/cucumberScenarioBuilder.ts | 8 +-- .../src/helpers/testManager.ts | 7 ++- packages/nightwatch-devtools/src/index.ts | 6 +- .../src/plugin-internals.ts | 10 +++- .../nightwatch-devtools/src/session-init.ts | 8 ++- .../nightwatch-devtools/src/test-lifecycle.ts | 4 +- .../nightwatch-devtools/src/trace-context.ts | 3 + .../tests/attemptTracking.test.ts | 55 ++++++++++++++++++- 9 files changed, 93 insertions(+), 16 deletions(-) diff --git a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts index 44197c0e..46684570 100644 --- a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts +++ b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts @@ -61,7 +61,8 @@ export interface CucumberLifecycleCtx extends TestSliceCtx { setCurrentStep(s: unknown): void getCurrentStep(): unknown setCurrentTest(t: unknown): void - recordAttempt(uid: string): number + recordAttempt(uid: string, specFile?: string): number + recordOutcome(uid: string, state: TestStats['state']): void } type MutStep = { @@ -182,7 +183,7 @@ export async function initCucumberScenario( stepKeywords, scenarioLine, parentFeatureSuiteUid: featureSuite.uid, - recordAttempt: (uid) => ctx.recordAttempt(uid) + recordAttempt: (uid, specFile) => ctx.recordAttempt(uid, specFile) }) attachScenarioToFeature(ctx, featureSuite, scenarioSuite) // The scenario is the `test` unit; its steps are the leaf metadata entries. @@ -211,6 +212,9 @@ export async function finalizeCucumberScenario( scenario.state = scenarioState scenario.end = now scenario._duration = duration + // Stamp this attempt's real outcome so spec/session retention doesn't + // collapse to the retry-stable suite's last-overwritten state. + ctx.recordOutcome(scenario.uid, scenarioState) closeOpenSteps(scenario, scenarioState, now) const featureUri: string = pickle?.uri ?? 'unknown.feature' diff --git a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts index b60d5ad4..af617f91 100644 --- a/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts +++ b/packages/nightwatch-devtools/src/helpers/cucumberScenarioBuilder.ts @@ -21,9 +21,9 @@ export interface CucumberScenarioBuildInput { scenarioLine: number /** Parent feature-suite uid — scenarios nest under this. */ parentFeatureSuiteUid: string - /** Records the scenario start under its (retry-stable) uid and returns the - * 0-based attempt number stamped on every step. Omitted → attempt 0. */ - recordAttempt?: (scenarioUid: string) => number + /** Records the scenario start under its (retry-stable) uid + feature file and + * returns the 0-based attempt number stamped on every step. Omitted → 0. */ + recordAttempt?: (scenarioUid: string, specFile?: string) => number } /** @@ -101,7 +101,7 @@ export function buildCucumberScenarioSuite( featureUri, `scenario:${scenarioName}:${scenarioLine}` ) - const attempt = input.recordAttempt?.(scenarioUid) ?? 0 + const attempt = input.recordAttempt?.(scenarioUid, featureUri) ?? 0 const scenarioSuite: SuiteStats = { uid: scenarioUid, cid: DEFAULTS.CID, diff --git a/packages/nightwatch-devtools/src/helpers/testManager.ts b/packages/nightwatch-devtools/src/helpers/testManager.ts index 0edc5ead..2b98c5d3 100644 --- a/packages/nightwatch-devtools/src/helpers/testManager.ts +++ b/packages/nightwatch-devtools/src/helpers/testManager.ts @@ -13,7 +13,11 @@ export class TestManager { private processedTests = new Map<string, Set<string>>() private lastKnownTestName: string | null = null - constructor(private testReporter: TestReporter) {} + constructor( + private testReporter: TestReporter, + /** Stamps a test's resolved terminal state onto the retry ledger. */ + private recordOutcome?: (uid: string, state: TestStats['state']) => void + ) {} /** * Update test state and report to UI @@ -42,6 +46,7 @@ export class TestManager { if (state !== TEST_STATE.RUNNING) { this.testReporter.onTestEnd(test) + this.recordOutcome?.(test.uid, state) } } diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 53b1a3dd..a4084321 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -299,7 +299,10 @@ class NightwatchDevToolsPlugin { self.suiteManager.clearExecutionData() self.#attemptTracker.reset() }, - recordAttempt: (uid) => self.#attemptTracker.recordStart(uid), + recordAttempt: (uid, specFile) => + self.#attemptTracker.recordStart(uid, specFile), + recordOutcome: (uid, state) => + self.#attemptTracker.recordOutcome(uid, state), attemptFor: (uid) => self.#attemptTracker.attemptFor(uid), buildMetadataOptions: () => self.#buildMetadataOptions(), ensureSessionInitialized: (b) => self.#ensureSessionInitialized(b), @@ -586,6 +589,7 @@ class NightwatchDevToolsPlugin { format: this.options.traceFormat, capturer: this.sessionCapturer, suites: this.suiteManager.getAllSuites().values(), + outcomes: this.#attemptTracker, ranges: this.#specRanges, flushed: this.#flushedSpecs, configPath: this.#configPath, diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 1753d09c..4279d360 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -77,9 +77,13 @@ export interface PluginInternals { setCucumberRunner(v: boolean): void getRerunLabel(): string | undefined - /** Records a test/scenario start under its retry-stable uid; returns the - * 0-based attempt number (0 first run, +1 per rerun). */ - recordAttempt(uid: string): number + /** Records a test/scenario start under its retry-stable uid; `specFile` + * (when known) enables spec-scoped retention. Returns the 0-based attempt + * number (0 first run, +1 per rerun). */ + recordAttempt(uid: string, specFile?: string): number + /** Stamps the resolved terminal state onto uid's latest attempt slot so the + * retry-aware retention policies see real per-attempt outcomes. */ + recordOutcome(uid: string, state: TestStats['state']): void /** Latest attempt recorded for `uid`, or undefined if it never started. */ attemptFor(uid: string): number | undefined diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index e9d1004a..8e485a21 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -28,7 +28,8 @@ import type { DevToolsMode, NightwatchBrowser, ScreencastOptions, - SuiteStats + SuiteStats, + TestStats } from './types.js' const log = logger('@wdio/nightwatch-devtools:session-init') @@ -59,6 +60,7 @@ export interface SessionInitCtx { getCurrentScenarioSuite(): SuiteStats | null buildMetadataOptions(): unknown attemptFor(uid: string): number | undefined + recordOutcome(uid: string, state: TestStats['state']): void } async function handleSessionChange( @@ -94,7 +96,9 @@ function initReporterChain(ctx: SessionInitCtx): void { }, (uid) => ctx.attemptFor(uid) ) - ctx.testManager = new TestManager(ctx.testReporter) + ctx.testManager = new TestManager(ctx.testReporter, (uid, state) => + ctx.recordOutcome(uid, state) + ) ctx.suiteManager = new SuiteManager(ctx.testReporter) ctx.browserProxy = new BrowserProxy( ctx.sessionCapturer, diff --git a/packages/nightwatch-devtools/src/test-lifecycle.ts b/packages/nightwatch-devtools/src/test-lifecycle.ts index 7192c12a..e8e0c60d 100644 --- a/packages/nightwatch-devtools/src/test-lifecycle.ts +++ b/packages/nightwatch-devtools/src/test-lifecycle.ts @@ -40,7 +40,7 @@ export interface TestLifecycleCtx extends TestSliceCtx { incrementCount(state: TestStats['state']): void testIcon(state: TestStats['state']): string setCurrentTest(t: unknown): void - recordAttempt(uid: string): number + recordAttempt(uid: string, specFile?: string): number } interface SuiteMetadata { @@ -126,7 +126,7 @@ export async function startNextTest( const test = ctx.testManager.findTestInSuite(currentSuite, currentTestName) if (test) { // Nightwatch has no per-test retry index; the tracker is the retry signal. - test.retries = ctx.recordAttempt(test.uid) + test.retries = ctx.recordAttempt(test.uid, specFile ?? undefined) if (specFile) { recordTestSliceBoundary(ctx, specFile, test.uid) } diff --git a/packages/nightwatch-devtools/src/trace-context.ts b/packages/nightwatch-devtools/src/trace-context.ts index 359737b9..bf217fb2 100644 --- a/packages/nightwatch-devtools/src/trace-context.ts +++ b/packages/nightwatch-devtools/src/trace-context.ts @@ -9,6 +9,7 @@ import { collectSuiteTestMetadata, resolveAdapterOutputDir, + type RetryOutcomeView, type SpecRange, type TraceExportContext } from '@wdio/devtools-core' @@ -28,6 +29,7 @@ export interface TraceContextInput { format: TraceFormat capturer: SessionCapturer suites: Iterable<SuiteStats> + outcomes?: RetryOutcomeView ranges: SpecRange[] flushed: Set<string> configPath: string | undefined @@ -48,6 +50,7 @@ export function buildTraceContext( actionSnapshots: input.capturer.actionSnapshots, sessionId, testMetadata: collectSuiteTestMetadata(input.suites), + outcomes: input.outcomes, ranges: input.ranges, flushed: input.flushed, resolveOutputDir: () => diff --git a/packages/nightwatch-devtools/tests/attemptTracking.test.ts b/packages/nightwatch-devtools/tests/attemptTracking.test.ts index 30df38e2..7d5eacd3 100644 --- a/packages/nightwatch-devtools/tests/attemptTracking.test.ts +++ b/packages/nightwatch-devtools/tests/attemptTracking.test.ts @@ -10,7 +10,9 @@ import { TEST_STATE } from '../src/constants.js' import type { SessionCapturer } from '../src/session.js' import type { SuiteStats, TestStats } from '../src/types.js' -function scenarioInput(recordAttempt: (uid: string) => number) { +function scenarioInput( + recordAttempt: (uid: string, specFile?: string) => number +) { return { featureUri: 'login.feature', scenarioName: 'Valid login', @@ -71,6 +73,25 @@ describe('cucumber scenario retry attempt', () => { true ) }) + + it('feeds the ledger per attempt with specFile so forSpec sees real outcomes', () => { + const tracker = new TestAttemptTracker() + const record = (uid: string, specFile?: string) => + tracker.recordStart(uid, specFile) + + // Attempt 0 fails, then the retry (same uid) passes — each attempt's real + // outcome must survive, not collapse to the retry-stable suite's last state. + const first = buildCucumberScenarioSuite(scenarioInput(record)) + tracker.recordOutcome(first.uid, TEST_STATE.FAILED as TestStats['state']) + const retried = buildCucumberScenarioSuite(scenarioInput(record)) + tracker.recordOutcome(retried.uid, TEST_STATE.PASSED as TestStats['state']) + + expect(first.uid).toBe(retried.uid) + expect(tracker.forSpec('login.feature')).toEqual([ + { uid: first.uid, attempt: 0, state: 'failed' }, + { uid: first.uid, attempt: 1, state: 'passed' } + ]) + }) }) describe('regular test retry attempt via startNextTest', () => { @@ -131,4 +152,36 @@ describe('buildTraceContext', () => { expect(ctx.sessionId).toBe('session-1') expect(ctx.testMetadata.get('t1')?.attempt).toBe(2) }) + + it('exposes the attempt-outcome ledger on the trace context', () => { + const capturer = { + actionSnapshots: [], + snapshotCaptures: [] + } as unknown as SessionCapturer + const tracker = new TestAttemptTracker() + tracker.recordStart('t1', '/spec.ts') + tracker.recordOutcome('t1', TEST_STATE.FAILED as TestStats['state']) + + const ctx = buildTraceContext( + { + mode: 'trace', + policy: 'retain-on-failure', + granularity: 'spec', + format: 'zip', + capturer, + suites: [], + outcomes: tracker, + ranges: [], + flushed: new Set(), + configPath: undefined, + log: () => {} + }, + 'session-1' + ) + + expect(ctx.outcomes).toBe(tracker) + expect(ctx.outcomes?.forSpec('/spec.ts')).toEqual([ + { uid: 't1', attempt: 0, state: 'failed' } + ]) + }) }) From 2ac7f251eb9cfe82c74757141370a9a517e1e1fe Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 19:12:42 +0530 Subject: [PATCH 62/91] fix(selenium): feed the retention ledger for correct retry-aware policies --- .../src/helpers/testManager.ts | 33 +++++++- .../src/session-lifecycle.ts | 5 ++ .../tests/attempt-capture.test.ts | 81 ++++++++++++++++++- 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/packages/selenium-devtools/src/helpers/testManager.ts b/packages/selenium-devtools/src/helpers/testManager.ts index 70fcb14b..09c741c7 100644 --- a/packages/selenium-devtools/src/helpers/testManager.ts +++ b/packages/selenium-devtools/src/helpers/testManager.ts @@ -2,7 +2,8 @@ import logger from '@wdio/logger' import { createTestStats, stampTestEnd, - TestAttemptTracker + TestAttemptTracker, + type RetryOutcomeView } from '@wdio/devtools-core' import { DEFAULTS, TEST_STATE } from '../constants.js' import type { SuiteStats, TestStats } from '../types.js' @@ -23,9 +24,14 @@ export class TestManager { #currentTest: TestStats | null = null #lastMarkedTest: TestStats | null = null #mode: 'session' | 'marked' = 'session' - // Per-test attempt counter. Persists for the whole in-process session so + // Per-test attempt ledger. Persists for the whole in-process session so // same-process retries (Mocha/Jest/etc re-entering the start hook) accumulate. + // Retry-stable-keyed so a test's attempts group under one entry; the trace + // finalizer reads it (attemptOutcomes) for retry-aware retention. #attemptTracker = new TestAttemptTracker() + // Retry-stable uid of the in-progress marked test, so endCurrent can stamp + // this attempt's outcome onto the same ledger entry recordStart opened. + #currentRetryStableUid: string | undefined /** Set true the first time the user calls startMarkedTest. Once true we * never auto-create the synthetic session test — orphan commands attach * to the most-recently-marked test instead. */ @@ -134,9 +140,12 @@ export class TestManager { // deterministicUid is retry-stable (no counter), so re-entering with the // same logical test increments the heuristic. Mocha supplies an // authoritative per-test retry index via opts.attempt; prefer it. + const retryStableUid = deterministicUid(file, signature) const heuristicAttempt = this.#attemptTracker.recordStart( - deterministicUid(file, signature) + retryStableUid, + file ) + this.#currentRetryStableUid = retryStableUid test.retries = opts.attempt ?? heuristicAttempt log.info( `Started marked test "${name}" (callSource: ${opts.callSource || 'n/a'})` @@ -155,6 +164,18 @@ export class TestManager { } test.state = state stampTestEnd(test) + // Stamp this attempt's real outcome onto its retry-stable ledger slot so a + // fail-then-pass groups under one test: retain-on-failure keys on the final + // (passed) attempt and stops over-retaining, retain-on-first-failure on + // attempt 0. Skipped for the synthetic session test (no recordStart). + if (this.#currentRetryStableUid !== undefined) { + this.#attemptTracker.recordOutcome( + this.#currentRetryStableUid, + state, + test.retries + ) + this.#currentRetryStableUid = undefined + } this.testReporter.onTestEnd(test) this.#currentTest = null } @@ -163,6 +184,12 @@ export class TestManager { return this.#currentTest } + /** Read-only per-attempt outcome ledger for the trace finalizer's retry-aware + * retention evaluation (retry-stable-keyed, so a test's attempts group). */ + get attemptOutcomes(): RetryOutcomeView { + return this.#attemptTracker + } + /** Called when the driver session is closing (process exit / quit). */ finalizeSession(): void { if (this.#currentTest) { diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 7716210b..ca4bab76 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -315,6 +315,11 @@ export function buildTraceExportContext( // TestStats.retries carries the per-test attempt (Mocha authoritative, // other runners heuristic), so retry-aware policies can trust it. attemptInfoAvailable: true, + // Per-attempt outcome ledger, retry-stable-keyed so a test's attempts + // group as one test. Without it, session/spec retention reads the + // per-attempt suite-node metadata — which sees a fail-then-pass as two + // separate one-shot tests and over-retains it under retain-on-failure. + outcomes: ctx.testManager?.attemptOutcomes, ranges: ctx.specRanges, flushed: ctx.flushedSpecs, resolveOutputDir: (range) => diff --git a/packages/selenium-devtools/tests/attempt-capture.test.ts b/packages/selenium-devtools/tests/attempt-capture.test.ts index 2f2840ff..173acfc8 100644 --- a/packages/selenium-devtools/tests/attempt-capture.test.ts +++ b/packages/selenium-devtools/tests/attempt-capture.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect, vi } from 'vitest' -import { collectSuiteTestMetadata } from '@wdio/devtools-core' +import { + collectSuiteTestMetadata, + shouldRetainTrace +} from '@wdio/devtools-core' import { resetSignatureCounters } from '../src/helpers/utils.js' import { TestManager } from '../src/helpers/testManager.js' @@ -94,3 +97,79 @@ describe('retry/attempt capture', () => { } }) }) + +describe('retry outcome ledger feeds retry-aware retention', () => { + it('groups a fail-then-pass retry under one retry-stable uid', () => { + const { mgr } = makeManager() + + const first = mgr.startMarkedTest('flaky') + mgr.endCurrent('failed') + const retry = mgr.startMarkedTest('flaky') + mgr.endCurrent('passed') + + // Selenium gives each attempt its own suite-node uid… + expect(first.uid).not.toBe(retry.uid) + + // …but the ledger records both attempts under ONE retry-stable uid. + const ledger = mgr.attemptOutcomes.all() + expect(ledger).toHaveLength(2) + expect(ledger[0]).toMatchObject({ attempt: 0, state: 'failed' }) + expect(ledger[1]).toMatchObject({ attempt: 1, state: 'passed' }) + expect(ledger[0].uid).toBe(ledger[1].uid) + }) + + it('exposes outcomes on the ctx so retain-on-failure drops a fail-then-pass but retain-on-first-failure keeps it', () => { + const { mgr, suiteManager } = makeManager() + + mgr.startMarkedTest('flaky') + mgr.endCurrent('failed') + mgr.startMarkedTest('flaky') + mgr.endCurrent('passed') + + const capturer = new SessionCapturer() + try { + const ctx = { + options: { + mode: 'trace', + traceGranularity: 'session', + traceFormat: 'zip' + }, + testManager: mgr, + suiteManager, + actionSnapshots: [], + specRanges: [], + flushedSpecs: new Set<string>(), + traceFlushes: [], + snapshotCaptures: [] + } as unknown as SessionLifecycleCtx + + const traceCtx = buildTraceExportContext( + ctx, + capturer, + 'sess-1', + '/spec.ts' + ) + expect(traceCtx.outcomes).toBe(mgr.attemptOutcomes) + + const outcomes = [...traceCtx.outcomes!.all()] + const retainOnFailure = shouldRetainTrace('retain-on-failure', { + outcomes, + attemptInfoAvailable: true + }) + const retainOnFirstFailure = shouldRetainTrace( + 'retain-on-first-failure', + { + outcomes, + attemptInfoAvailable: true + } + ) + + // Final attempt passed → retain-on-failure drops it (no over-retention). + expect(retainOnFailure.retain).toBe(false) + // Attempt 0 failed → retain-on-first-failure still keeps it. + expect(retainOnFirstFailure.retain).toBe(true) + } finally { + capturer.cleanup() + } + }) +}) From 440014de21eee898b20466aede925c68dbb44619 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Tue, 14 Jul 2026 19:12:57 +0530 Subject: [PATCH 63/91] docs: retry-aware retention scope + assert-capture debt sync --- CLAUDE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 007cc486..59009622 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -235,7 +235,10 @@ Documented divergences from the conventions above. They exist today as debt to b - `replaceCommand` has two semantics — Selenium mutates in place (preserves `_id`/`id` for chained calls); Nightwatch splices and reissues. Both call the same `core/suite-helpers` factories; the storage strategy stays adapter-specific because runner integrations differ. Could be unified by parameterizing the policy if the divergence ever causes a real problem. - `patchNodeAssert` (via `core/assert-patcher`) is now wired in all three adapters, default-on behind each adapter's `captureAssertions` option (opt out with `captureAssertions: false`). Framework matcher libraries differ: Service taps expect-webdriverio's `beforeAssertion`/`afterAssertion` hooks so passing+failing matchers render as `expect.*` actions (mechanism in the assert-capture entry below); Nightwatch native `assert`/`verify` and Selenium's `node:assert` also surface passing+failing rows via their reconcile/patch paths. The remaining gap is Selenium's jest-style `expect()` (and chai): jest/vitest expose no pass+fail assertion hook, so only failing matchers surface there. - BiDi is auto-attached in Service and Selenium; Nightwatch is opt-in via `bidi: true` and requires `webSocketUrl: true` in capabilities. -- Retry-aware trace policies aren't uniform across adapters. Selenium emits one metadata entry per attempt (counter-based `generateStableUid` uid + retry-stable tracker key), so all six policies — including `retain-on-first-failure` (needs the first attempt's outcome) — evaluate correctly. Service and Nightwatch key metadata on the retry-stable uid and overwrite/mutate per attempt, collapsing to the *final* attempt's outcome; the retry-count policies (`retain-on-failure`, `on-first-retry`, `on-all-retries`, `retain-on-failure-and-retries`) are still correct, but `retain-on-first-failure` can't see a failed-then-passed first attempt and under-retains. Proper fix rides with B5 (per-test granularity preserves per-attempt slices); until then all three set `attemptInfoAvailable: true`. +- Retry-aware trace policies share one mechanism: adapters feed a per-attempt **outcome ledger** (`core/attempt-tracker.ts` `TestAttemptTracker.recordStart(uid, specFile)` + `recordOutcome(uid, state, attempt)`) keyed by the **retry-stable** uid, and the finalizer reads the scoped views (`all`/`forSpec`/`forTest`) so `trace-retention.ts` evaluates **group-by-test** — `retain-on-failure` keys on each test's *final* attempt (no over-retaining a fail-then-pass), `retain-on-first-failure` on *attempt 0*. `recordStart` on a second attempt stamps the prior attempt `failed` (a retry only follows a failure), which corrects runners that swallow the intermediate failure — e.g. Mocha via a `--require` plugin never surfaces the retried attempt's failure, so the ledger would otherwise see `[passed, passed]`. An empty scoped view falls back to `testMetadata` (never fail-open-retains). + - **WDIO + Selenium: verified end-to-end** (manual fail-then-pass runs: retained under `retain-on-first-failure`, dropped under `retain-on-failure`). + - **Nightwatch: `retain-on-failure` works; the other retry-aware policies degrade.** Its `--retries` re-runs the testcase *internally* without re-firing the plugin's per-test hooks, and the per-testcase results carry no attempt/retry field (retries live only in undocumented, version-varying Nightwatch internals — `suiteRetries.testRetriesCount` / `reporter.testResults.retryTest`), so the ledger sees only the final attempt for the `describe/it` and exports-object interfaces. Cucumber scenarios expose per-scenario hooks, so the feed captures their attempts. Not cleanly fixable without depending on those internals. + - WDIO `specFileRetries` spawns a fresh worker per retry, so cross-process attempts aren't in the (process-scoped) ledger. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. - Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. From 6794687913fbf6ba16971993e07872965d18c969 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Wed, 15 Jul 2026 02:16:32 +0530 Subject: [PATCH 64/91] feat(core): artifacts manifest + WDIO Allure trace/video attach --- packages/core/src/artifacts-manifest.ts | 81 ++++++++++++++ packages/core/src/index.ts | 1 + packages/core/src/trace-finalizer.ts | 67 +++++++++++- .../core/tests/artifacts-manifest.test.ts | 103 ++++++++++++++++++ packages/core/tests/trace-finalizer.test.ts | 64 +++++++++++ packages/nightwatch-devtools/src/index.ts | 16 ++- .../nightwatch-devtools/src/trace-context.ts | 8 +- .../tests/attemptTracking.test.ts | 4 + packages/selenium-devtools/src/index.ts | 9 +- .../selenium-devtools/src/plugin-internals.ts | 4 +- .../src/session-lifecycle.ts | 8 +- packages/service/package.json | 6 + packages/service/src/allure.ts | 90 +++++++++++++++ packages/service/src/index.ts | 40 +++++-- packages/service/src/trace-slices.ts | 10 +- packages/service/tests/allure.test.ts | 91 ++++++++++++++++ 16 files changed, 578 insertions(+), 24 deletions(-) create mode 100644 packages/core/src/artifacts-manifest.ts create mode 100644 packages/core/tests/artifacts-manifest.test.ts create mode 100644 packages/service/src/allure.ts create mode 100644 packages/service/tests/allure.test.ts diff --git a/packages/core/src/artifacts-manifest.ts b/packages/core/src/artifacts-manifest.ts new file mode 100644 index 00000000..0853e401 --- /dev/null +++ b/packages/core/src/artifacts-manifest.ts @@ -0,0 +1,81 @@ +/** + * The generic artifacts manifest: a single JSON side-file + * (`devtools-artifacts-<sessionId>.json`) written next to the trace/video + * artifacts at end-of-run. It enumerates every artifact a trace-mode session + * produced — including the ones a retention policy decided against + * (`retained: false`) — alongside each test's final state and attempt. Any + * ecosystem reporter (Allure, a CI collector, a custom dashboard) can read this + * one file to discover what to attach and how each test fared, instead of + * re-deriving it from framework internals. The WDIO Allure glue consumes it + * directly; Selenium/Nightwatch document reader recipes against it. + */ + +import fs from 'node:fs/promises' +import path from 'node:path' +import type { + TestMetadataEntry, + TestMetadataMap, + TraceFormat +} from '@wdio/devtools-shared' +import type { TraceArtifact } from './trace-finalizer.js' + +/** One test's outcome as recorded at export time. Field types track + * TestMetadataEntry in shared; adds the keying uid, drops trace-only ancestry. */ +export interface ArtifactManifestTest extends Pick< + TestMetadataEntry, + 'title' | 'specFile' | 'state' | 'attempt' +> { + uid: string +} + +export interface ArtifactsManifest { + sessionId: string + format: TraceFormat + artifacts: TraceArtifact[] + tests: ArtifactManifestTest[] +} + +/** Deterministic manifest filename for a session (used by writers and readers). */ +export function artifactsManifestFilename(sessionId: string): string { + return `devtools-artifacts-${sessionId}.json` +} + +/** Assemble the manifest from the artifacts collected across the run and the + * test-metadata map. Pure — the write is separate so it stays unit-testable. */ +export function buildArtifactsManifest(input: { + sessionId: string + format: TraceFormat + artifacts: readonly TraceArtifact[] + testMetadata: TestMetadataMap +}): ArtifactsManifest { + const tests: ArtifactManifestTest[] = [] + for (const [uid, meta] of input.testMetadata) { + tests.push({ + uid, + title: meta.title, + specFile: meta.specFile, + state: meta.state, + attempt: meta.attempt + }) + } + return { + sessionId: input.sessionId, + format: input.format, + artifacts: [...input.artifacts], + tests + } +} + +/** Write the manifest to `outputDir`, returning its absolute path. */ +export async function writeArtifactsManifest( + outputDir: string, + manifest: ArtifactsManifest +): Promise<string> { + await fs.mkdir(outputDir, { recursive: true }) + const filePath = path.join( + outputDir, + artifactsManifestFilename(manifest.sessionId) + ) + await fs.writeFile(filePath, JSON.stringify(manifest, null, 2), 'utf8') + return filePath +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 89c6c301..9d37bbf0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ export * from './action-mapping.js' export * from './action-snapshot.js' +export * from './artifacts-manifest.js' export * from './attempt-tracker.js' export * from './with-timeout.js' export * from './assert-patcher.js' diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index d69a2898..9cb57af2 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -15,6 +15,10 @@ import type { TraceRetentionPolicy } from '@wdio/devtools-shared' import { errorMessage } from './error.js' +import { + buildArtifactsManifest, + writeArtifactsManifest +} from './artifacts-manifest.js' import { filterTestMetadataBySpec, filterTestMetadataByUid, @@ -72,6 +76,15 @@ export interface TraceExportContext { awaitPending?: Promise<unknown>[] log?: (level: 'info' | 'warn', msg: string) => void onArtifact?: (a: TraceArtifact) => void + /** When true, finalize writes a `devtools-artifacts-<sessionId>.json` manifest + * enumerating every artifact (retained or not) plus per-test states, for + * ecosystem reporters (Allure, CI collectors) to consume. */ + emitManifest?: boolean + /** Every artifact seen via `onArtifact` across the run (including eager + * mid-run test-slice flushes, which the end-of-run fan-out dedupes away). + * Pass the same list the adapter accumulates so the manifest is complete; + * finalize falls back to its own fan-out results when this is omitted. */ + collectedArtifacts?: readonly TraceArtifact[] } const SPEC_WITHOUT_BOUNDARIES_WARNING = @@ -336,10 +349,11 @@ async function flushAllRanges( } /** - * Entry point for the after/end-of-run hook. No-op outside trace mode. Awaits - * any pending snapshot captures, then fans out to per-spec, per-test, or - * session writes. `spec`/`test` granularity with no recorded boundaries warns - * and falls back to a single session-level trace. + * Entry point for the after/end-of-run hook. No-op outside trace mode. Settles + * in-flight captures and eager flushes, then fans out to per-spec, per-test, or + * session writes and finally the artifacts manifest. `spec`/`test` granularity + * with no recorded boundaries warns and falls back to a single session-level + * trace. */ export async function finalizeTraceExport( ctx: TraceExportContext @@ -347,7 +361,22 @@ export async function finalizeTraceExport( if (ctx.mode !== 'trace') { return [] } + // Settle in-flight snapshot captures AND the adapters' tracked fire-and-forget + // eager flushes (Selenium/Nightwatch put both in awaitPending) before writing + // anything: every eager-flushed range is a synchronous dedupe-hit in the + // fan-out below, so this is the one place that awaits those eager-flush + // promises. Without it the last eager write can race teardown and its artifact + // can miss the manifest. A no-op for WDIO (awaits eager flushes inline). await awaitPendingCaptures(ctx) + const artifacts = await fanOutTraceWrites(ctx) + await maybeWriteManifest(ctx, artifacts) + return artifacts +} + +/** Session vs per-slice fan-out (with the no-boundaries fallback). */ +async function fanOutTraceWrites( + ctx: TraceExportContext +): Promise<TraceArtifact[]> { const sliced = ctx.granularity === 'spec' || ctx.granularity === 'test' if (sliced && ctx.ranges.length > 0) { if (ctx.ranges.length > SLICE_COUNT_WARN_THRESHOLD) { @@ -363,3 +392,33 @@ export async function finalizeTraceExport( const artifact = await safely(ctx, () => writeSessionTrace(ctx)) return artifact ? [artifact] : [] } + +/** Write the artifacts manifest when enabled, preferring the adapter's full + * collected list (includes eager slices) over this pass's fan-out results. + * Never throws — a manifest failure must not abort the run. */ +async function maybeWriteManifest( + ctx: TraceExportContext, + fanOutArtifacts: readonly TraceArtifact[] +): Promise<void> { + if (!ctx.emitManifest) { + return + } + const artifacts = ctx.collectedArtifacts ?? fanOutArtifacts + try { + const path = await writeArtifactsManifest( + ctx.resolveOutputDir(), + buildArtifactsManifest({ + sessionId: ctx.sessionId, + format: ctx.format ?? 'zip', + artifacts, + testMetadata: ctx.testMetadata + }) + ) + ctx.log?.('info', `Artifacts manifest written to ${path}`) + } catch (err) { + ctx.log?.( + 'warn', + `Failed to write artifacts manifest: ${errorMessage(err)}` + ) + } +} diff --git a/packages/core/tests/artifacts-manifest.test.ts b/packages/core/tests/artifacts-manifest.test.ts new file mode 100644 index 00000000..0c7e969b --- /dev/null +++ b/packages/core/tests/artifacts-manifest.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, afterEach } from 'vitest' +import { rm, readFile, mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + buildArtifactsManifest, + writeArtifactsManifest, + artifactsManifestFilename, + type ArtifactsManifest +} from '../src/artifacts-manifest.js' +import type { TraceArtifact } from '../src/trace-finalizer.js' +import type { TestMetadataMap } from '@wdio/devtools-shared' + +function metadata(): TestMetadataMap { + return new Map([ + [ + 'u1', + { title: 'passes', specFile: '/a.spec.ts', state: 'passed', attempt: 0 } + ], + [ + 'u2', + { title: 'fails', specFile: '/a.spec.ts', state: 'failed', attempt: 1 } + ] + ]) +} + +const artifacts: TraceArtifact[] = [ + { + kind: 'trace', + path: '/out/u1.zip', + scope: 'test', + key: 'u1', + testUids: ['u1'], + retained: false + }, + { + kind: 'trace', + path: '/out/u2.zip', + scope: 'test', + key: 'u2', + testUids: ['u2'], + retained: true + } +] + +describe('artifacts manifest', () => { + const dirs: string[] = [] + afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) + }) + + it('buildArtifactsManifest carries every artifact (retained + not) and per-test states', () => { + const m = buildArtifactsManifest({ + sessionId: 'sess-1', + format: 'zip', + artifacts, + testMetadata: metadata() + }) + expect(m.sessionId).toBe('sess-1') + expect(m.format).toBe('zip') + expect(m.artifacts).toEqual(artifacts) + expect(m.artifacts).not.toBe(artifacts) // copied, not aliased + expect(m.tests).toEqual([ + { + uid: 'u1', + title: 'passes', + specFile: '/a.spec.ts', + state: 'passed', + attempt: 0 + }, + { + uid: 'u2', + title: 'fails', + specFile: '/a.spec.ts', + state: 'failed', + attempt: 1 + } + ]) + }) + + it('filename is deterministic per session', () => { + expect(artifactsManifestFilename('abc')).toBe('devtools-artifacts-abc.json') + }) + + it('writeArtifactsManifest round-trips the JSON to disk', async () => { + const dir = await mkdtemp(join(tmpdir(), 'manifest-')) + dirs.push(dir) + const manifest = buildArtifactsManifest({ + sessionId: 'sess-2', + format: 'zip', + artifacts, + testMetadata: metadata() + }) + const filePath = await writeArtifactsManifest(dir, manifest) + expect(filePath).toBe(join(dir, 'devtools-artifacts-sess-2.json')) + const parsed = JSON.parse( + await readFile(filePath, 'utf8') + ) as ArtifactsManifest + expect(parsed).toEqual(manifest) + }) +}) diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index d25e30b8..70f655fe 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -6,6 +6,7 @@ import { buildSpecSessionId, buildTestSliceFolder, finalizeTraceExport, + flushRangeLogged, flushRangeTrace, TestAttemptTracker, type SpecRange, @@ -315,6 +316,69 @@ describe('finalizeTraceExport', () => { expect(await exists(path.join(outputDir, 'trace-abcd1234.zip'))).toBe(true) }) + describe('artifacts manifest', () => { + const manifestPath = () => + path.join(outputDir, 'devtools-artifacts-abcd1234.json') + + it('is not written unless emitManifest is set', async () => { + await finalizeTraceExport( + baseCtx({ + testMetadata: meta([['u1', { title: 'T1', specFile: '/a.js' }]]) + }) + ) + expect(await exists(manifestPath())).toBe(false) + }) + + it('enumerates collected artifacts and per-test states when enabled', async () => { + await finalizeTraceExport( + baseCtx({ + emitManifest: true, + collectedArtifacts: artifacts, + testMetadata: meta([ + [ + 'u1', + { title: 'T1', specFile: '/a.js', state: 'passed', attempt: 0 } + ] + ]) + }) + ) + const manifest = JSON.parse(await fs.readFile(manifestPath(), 'utf8')) + expect(manifest.sessionId).toBe('abcd1234') + expect(manifest.format).toBe('zip') + // The session write pushed one artifact through onArtifact into the list. + expect(manifest.artifacts).toHaveLength(1) + expect(manifest.artifacts[0].scope).toBe('session') + expect(manifest.tests).toEqual([ + { + uid: 'u1', + title: 'T1', + specFile: '/a.js', + state: 'passed', + attempt: 0 + } + ]) + }) + + it('includes eager-flushed slices the fan-out deduped away', async () => { + const ctx = baseCtx({ + granularity: 'test', + emitManifest: true, + collectedArtifacts: artifacts, + ranges: [testRange('/a.js', 'u1', 0)], + testMetadata: meta([['u1', { title: 'T1', specFile: '/a.js' }]]) + }) + // Eager mid-run flush: writes the slice and records it via onArtifact. + await flushRangeLogged(ctx, ctx.ranges[0]!) + // Finalize: the range is already flushed, so fan-out returns nothing new, + // but the manifest must still list the eager artifact. + await finalizeTraceExport(ctx) + const manifest = JSON.parse(await fs.readFile(manifestPath(), 'utf8')) + expect(manifest.artifacts).toHaveLength(1) + expect(manifest.artifacts[0].scope).toBe('test') + expect(manifest.artifacts[0].key).toBe('u1') + }) + }) + describe('retention wiring', () => { it("writes everything under the default 'on' policy", async () => { const result = await finalizeTraceExport( diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index a4084321..fad32b4f 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -108,6 +108,14 @@ class NightwatchDevToolsPlugin { /** Set of spec files already flushed to disk. */ #flushedSpecs = new Set<string>() + /** Every trace/video artifact seen this run (retained or not), for the + * end-of-run artifacts manifest. Populated via the context's onArtifact. */ + #artifacts: TraceArtifact[] = [] + + /** In-flight fire-and-forget eager/boundary slice flushes, awaited at finalize + * so the last slice's write + its manifest entry land before teardown. */ + #traceFlushes: Promise<unknown>[] = [] + #getRerunLabel() { return process.env[REUSE_ENV.RERUN_ENTRY_TYPE] === 'test' ? process.env[REUSE_ENV.RERUN_LABEL]?.trim() @@ -575,7 +583,11 @@ class NightwatchDevToolsPlugin { if (!sessionId) { return Promise.resolve(undefined) } - return flushRangeLogged(this.#traceContext(sessionId), range) + // Track the promise (before it's added to awaitPending) so finalize awaits + // this fire-and-forget flush — its context snapshot already excludes it. + const flush = flushRangeLogged(this.#traceContext(sessionId), range) + this.#traceFlushes.push(flush) + return flush } /** Assemble the framework-agnostic trace-export context from plugin state. @@ -592,6 +604,8 @@ class NightwatchDevToolsPlugin { outcomes: this.#attemptTracker, ranges: this.#specRanges, flushed: this.#flushedSpecs, + artifacts: this.#artifacts, + traceFlushes: this.#traceFlushes, configPath: this.#configPath, testFilePath: this.browserProxy?.getCurrentTestFullPath?.() ?? undefined, diff --git a/packages/nightwatch-devtools/src/trace-context.ts b/packages/nightwatch-devtools/src/trace-context.ts index bf217fb2..4c4666be 100644 --- a/packages/nightwatch-devtools/src/trace-context.ts +++ b/packages/nightwatch-devtools/src/trace-context.ts @@ -11,6 +11,7 @@ import { resolveAdapterOutputDir, type RetryOutcomeView, type SpecRange, + type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' import type { SessionCapturer } from './session.js' @@ -32,6 +33,8 @@ export interface TraceContextInput { outcomes?: RetryOutcomeView ranges: SpecRange[] flushed: Set<string> + artifacts: TraceArtifact[] + traceFlushes: Promise<unknown>[] configPath: string | undefined testFilePath: string | undefined log: (level: 'info' | 'warn', msg: string) => void @@ -58,10 +61,13 @@ export function buildTraceContext( testFilePath: input.testFilePath, configPath: input.configPath }), - awaitPending: input.capturer.snapshotCaptures, + awaitPending: [...input.capturer.snapshotCaptures, ...input.traceFlushes], // Nightwatch feeds real per-test attempt numbers via TestAttemptTracker // (B4), so retry-aware policies use per-test attempts, not the fallback. attemptInfoAvailable: true, + emitManifest: true, + collectedArtifacts: input.artifacts, + onArtifact: (a) => input.artifacts.push(a), log: input.log } } diff --git a/packages/nightwatch-devtools/tests/attemptTracking.test.ts b/packages/nightwatch-devtools/tests/attemptTracking.test.ts index 7d5eacd3..fe49f502 100644 --- a/packages/nightwatch-devtools/tests/attemptTracking.test.ts +++ b/packages/nightwatch-devtools/tests/attemptTracking.test.ts @@ -142,6 +142,8 @@ describe('buildTraceContext', () => { suites: [suite], ranges: [], flushed: new Set(), + artifacts: [], + traceFlushes: [], configPath: undefined, log: () => {} }, @@ -173,6 +175,8 @@ describe('buildTraceContext', () => { outcomes: tracker, ranges: [], flushed: new Set(), + artifacts: [], + traceFlushes: [], configPath: undefined, log: () => {} }, diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 4c4b2406..91af73f9 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -24,7 +24,7 @@ import { recordTraceBoundary, flushCurrentTestTrace } from './session-lifecycle.js' -import type { SpecRange } from '@wdio/devtools-core' +import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' import { startTest as tmStartTest, endTest as tmEndTest, @@ -127,6 +127,10 @@ class SeleniumDevToolsPlugin { /** In-flight per-test eager flushes (test granularity), awaited at finalize. */ #traceFlushes: Promise<unknown>[] = [] + /** Every trace/video artifact seen this run (retained or not), for the + * end-of-run artifacts manifest. Populated via the context's onArtifact. */ + #artifacts: TraceArtifact[] = [] + constructor(options: DevToolsOptions = {}) { this.#options = { port: options.port ?? 3000, @@ -439,6 +443,9 @@ class SeleniumDevToolsPlugin { get traceFlushes() { return self.#traceFlushes }, + get artifacts() { + return self.#artifacts + }, setFinalized: (v) => { self.#finalized = v }, diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index a9672e50..36d1f98e 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -22,7 +22,7 @@ import type { } from './types.js' import type { RetryTracker } from '@wdio/devtools-core' import type { PendingTestAction, PendingScenario } from './test-management.js' -import type { SpecRange } from '@wdio/devtools-core' +import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' export interface PluginInternals { // Config @@ -69,6 +69,8 @@ export interface PluginInternals { readonly flushedSpecs: Set<string> // In-flight per-test eager flushes (test granularity), awaited at finalize. readonly traceFlushes: Promise<unknown>[] + // Every trace/video artifact seen this run, for the end-of-run manifest. + readonly artifacts: TraceArtifact[] // Plugin-side delegates setFinalized(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index ca4bab76..fabe3895 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -21,6 +21,7 @@ import { resolveAdapterOutputDir, type SpecBoundaryContext, type SpecRange, + type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' import { TIMING } from './constants.js' @@ -85,6 +86,8 @@ export interface SessionLifecycleCtx { // In-flight per-test eager flushes (test granularity); awaited at finalize // so the last test's write completes before the process exits. readonly traceFlushes: Promise<unknown>[] + // Every trace/video artifact seen this run, for the end-of-run manifest. + readonly artifacts: TraceArtifact[] setFinalized(v: boolean): void ensureBackendStarted(): Promise<void> @@ -327,7 +330,10 @@ export function buildTraceExportContext( testFilePath: range ? range.specFile : testFilePath }), awaitPending: [...ctx.snapshotCaptures, ...ctx.traceFlushes], - log: (level, msg) => log[level](msg) + log: (level, msg) => log[level](msg), + emitManifest: true, + collectedArtifacts: ctx.artifacts, + onArtifact: (a) => ctx.artifacts.push(a) } } diff --git a/packages/service/package.json b/packages/service/package.json index 3a938db1..fcfc1885 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -73,8 +73,14 @@ "vite-plugin-dts": "^5.0.2" }, "peerDependencies": { + "@wdio/allure-reporter": "^9.0.0", "@wdio/protocols": "9.28.0", "devtools": "^8.42.0", "webdriverio": "^9.19.1" + }, + "peerDependenciesMeta": { + "@wdio/allure-reporter": { + "optional": true + } } } diff --git a/packages/service/src/allure.ts b/packages/service/src/allure.ts new file mode 100644 index 00000000..ce0e678d --- /dev/null +++ b/packages/service/src/allure.ts @@ -0,0 +1,90 @@ +// WDIO Allure glue: attach retained trace/video artifacts to the current Allure +// test so a failed run's trace travels with the report. Isolated here so +// index.ts stays free of the optional-dependency dance and the content-type +// convention lives in one place. + +import fs from 'node:fs/promises' +import { basename } from 'node:path' +import logger from '@wdio/logger' +import type { TraceArtifact } from '@wdio/devtools-core' + +const log = logger('@wdio/devtools-service') + +// Attach the trace as a plain zip download — the viewer stays the user's choice +// (our `show-trace`, or any compatible viewer), rather than routing them into a +// third-party viewer Allure would open for a trace-specific content type. Video +// uses video/webm so Allure renders it inline. +const TRACE_CONTENT_TYPE = 'application/zip' +const VIDEO_CONTENT_TYPE = 'video/webm' + +/** The one @wdio/allure-reporter method we use. Typed locally so the optional + * peer dependency never becomes a build-time type dependency. */ +interface AllureReporterModule { + addAttachment(name: string, content: Buffer, type: string): void +} + +// undefined = not yet probed; null = probed and unavailable. +let cachedReporter: AllureReporterModule | null | undefined + +// A non-literal specifier keeps TypeScript and the bundler from resolving this +// optional peer dependency at build time — `import()` of a variable is typed +// `any` and left as a runtime import that no-ops (caught below) when absent. +const ALLURE_REPORTER_SPECIFIER = '@wdio/allure-reporter' + +async function loadAllureReporter(): Promise<AllureReporterModule | null> { + if (cachedReporter !== undefined) { + return cachedReporter + } + try { + const mod = await import(/* @vite-ignore */ ALLURE_REPORTER_SPECIFIER) + const candidate = ( + typeof mod.addAttachment === 'function' ? mod : mod.default + ) as AllureReporterModule | undefined + cachedReporter = + candidate && typeof candidate.addAttachment === 'function' + ? candidate + : null + } catch { + // Optional peer dependency not installed — attaching is a no-op. + cachedReporter = null + } + return cachedReporter +} + +/** + * Attach one retained artifact to the current Allure test. No-op when the + * artifact wasn't retained, has no path yet, or @wdio/allure-reporter isn't + * installed. Must be called from within a per-test hook (afterTest) so Allure + * associates the attachment with the right test. + */ +export async function attachArtifactToAllure( + artifact: TraceArtifact +): Promise<void> { + if (!artifact.retained || !artifact.path) { + return + } + const reporter = await loadAllureReporter() + if (!reporter) { + return + } + try { + // Allure attaches a single file; the ndjson-directory trace format yields a + // directory path, which can't be attached — skip it. + const stat = await fs.stat(artifact.path) + if (!stat.isFile()) { + return + } + const content = await fs.readFile(artifact.path) + const type = + artifact.kind === 'video' ? VIDEO_CONTENT_TYPE : TRACE_CONTENT_TYPE + reporter.addAttachment(basename(artifact.path), content, type) + } catch (err) { + // A missing/unreadable artifact must never reject the test hook. + log.warn(`Allure attach skipped for ${artifact.path}: ${String(err)}`) + } +} + +/** Reset the memoized reporter — test seam only. */ +export function resetAllureReporterCache(): void { + cachedReporter = undefined +} diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index be45c5f3..fd24e32a 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -10,8 +10,10 @@ import { TestAttemptTracker, tracePolicyModeWarning, type SpecRange, + type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' +import { attachArtifactToAllure } from './allure.js' import { wireAssertCapture, type ExpectAssertion } from './assert-capture.js' import { AssertionTracker } from './assertion-tracker.js' import { @@ -123,6 +125,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Set of spec files already flushed to disk. */ #flushedSpecs = new Set<string>() + /** Every trace/video artifact seen this run (retained or not), for the + * end-of-run artifacts manifest. Populated via the context's onArtifact. */ + #artifacts: TraceArtifact[] = [] + /** Build the boundary context for recordSliceBoundary — the same shape is * needed in both beforeTest and beforeScenario. */ get #boundaryContext() { @@ -156,15 +162,17 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Eager per-test flush at test end (test granularity only), run after the * outcome is stamped so this attempt's metadata is written before a retry * overwrites it; the end-of-run finalizer then dedupes it via the key set. */ - async #eagerFlushTestSlice(testUid: string): Promise<void> { + async #eagerFlushTestSlice( + testUid: string + ): Promise<TraceArtifact | undefined> { if ( this.#options.traceGranularity !== 'test' || this.#options.mode !== 'trace' || !this.#browser ) { - return + return undefined } - await flushTestSlice( + return flushTestSlice( this.#traceContext(this.#browser), this.#specRanges, testUid @@ -190,7 +198,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { flushed: this.#flushedSpecs, resolveOutputDir: () => this.#outputDir, prepareSnapshots: dedupeSnapshotsByTimestamp, - log: (level, msg) => log[level](msg) + log: (level, msg) => log[level](msg), + emitManifest: true, + collectedArtifacts: this.#artifacts, + onArtifact: (a) => this.#artifacts.push(a) } } @@ -425,10 +436,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#stampOutcome(uid, result) } await this.#finalizePerScenario() - // Flush now so this slice includes the final snapshot and stamped outcome. - if (uid) { - await this.#eagerFlushTestSlice(uid) - } + await this.#eagerFlushAndAttach(uid) } async afterTest( @@ -443,9 +451,19 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#stampOutcome(uid, result) } await this.#finalizePerScenario() - // Flush now so this slice includes the final snapshot and stamped outcome. - if (uid) { - await this.#eagerFlushTestSlice(uid) + await this.#eagerFlushAndAttach(uid) + } + + /** Flush this test's slice (so it captures the final snapshot + stamped + * outcome), then attach the retained artifact to Allure while the per-test + * hook is still open. No-op outside `test`-granularity trace mode. */ + async #eagerFlushAndAttach(uid: string | undefined): Promise<void> { + if (!uid) { + return + } + const artifact = await this.#eagerFlushTestSlice(uid) + if (artifact) { + await attachArtifactToAllure(artifact) } } diff --git a/packages/service/src/trace-slices.ts b/packages/service/src/trace-slices.ts index 5da66806..24533abb 100644 --- a/packages/service/src/trace-slices.ts +++ b/packages/service/src/trace-slices.ts @@ -6,6 +6,7 @@ import { findFlushableRange, flushRangeLogged, type SpecRange, + type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' @@ -21,15 +22,16 @@ export function flushPrevSlice( /** Awaited flush of the just-ended test's slice (test granularity), so this * attempt's just-stamped metadata is written before a retry's beforeTest - * overwrites the entry. No-op when the test recorded no range. */ + * overwrites the entry. Returns the produced artifact (for same-hook Allure + * attach); undefined when the test recorded no range. */ export async function flushTestSlice( ctx: TraceExportContext, ranges: readonly SpecRange[], testUid: string -): Promise<void> { +): Promise<TraceArtifact | undefined> { const range = findFlushableRange(ranges, testUid) if (!range) { - return + return undefined } - await flushRangeLogged(ctx, range) + return flushRangeLogged(ctx, range) } diff --git a/packages/service/tests/allure.test.ts b/packages/service/tests/allure.test.ts new file mode 100644 index 00000000..e8305f6a --- /dev/null +++ b/packages/service/tests/allure.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { TraceArtifact } from '@wdio/devtools-core' + +const addAttachment = vi.fn() +vi.mock('@wdio/allure-reporter', () => ({ + addAttachment, + default: { addAttachment } +})) + +import { + attachArtifactToAllure, + resetAllureReporterCache +} from '../src/allure.js' + +function artifact(over: Partial<TraceArtifact> = {}): TraceArtifact { + return { + kind: 'trace', + path: '', + scope: 'test', + testUids: ['u1'], + retained: true, + ...over + } +} + +describe('attachArtifactToAllure', () => { + const dirs: string[] = [] + beforeEach(() => { + addAttachment.mockReset() + resetAllureReporterCache() + }) + afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) + }) + + async function tempZip(name: string): Promise<string> { + const dir = await mkdtemp(join(tmpdir(), 'allure-')) + dirs.push(dir) + const p = join(dir, name) + await writeFile(p, 'zip-bytes') + return p + } + + it('is a no-op for a non-retained artifact', async () => { + await attachArtifactToAllure(artifact({ retained: false, path: '/x.zip' })) + expect(addAttachment).not.toHaveBeenCalled() + }) + + it('is a no-op for an artifact with no path yet', async () => { + await attachArtifactToAllure(artifact({ path: '' })) + expect(addAttachment).not.toHaveBeenCalled() + }) + + it('attaches a trace zip with the Allure Playwright content type', async () => { + const path = await tempZip('trace-abc.zip') + await attachArtifactToAllure(artifact({ path })) + expect(addAttachment).toHaveBeenCalledOnce() + const [name, content, type] = addAttachment.mock.calls[0]! + expect(name).toBe('trace-abc.zip') + expect(Buffer.isBuffer(content)).toBe(true) + expect(type).toBe('application/zip') + }) + + it('attaches a video artifact as video/webm', async () => { + const path = await tempZip('rec-abc.webm') + await attachArtifactToAllure(artifact({ kind: 'video', path })) + const [, , type] = addAttachment.mock.calls[0]! + expect(type).toBe('video/webm') + }) + + it('skips a directory-format artifact without throwing (ndjson-directory)', async () => { + const dir = await mkdtemp(join(tmpdir(), 'allure-dir-')) + dirs.push(dir) + await expect( + attachArtifactToAllure(artifact({ path: dir })) + ).resolves.toBeUndefined() + expect(addAttachment).not.toHaveBeenCalled() + }) + + it('never rejects when the artifact path is missing', async () => { + await expect( + attachArtifactToAllure(artifact({ path: '/no/such/trace.zip' })) + ).resolves.toBeUndefined() + expect(addAttachment).not.toHaveBeenCalled() + }) +}) From 82cff0cf757d9f5e71f01fcdf05ad9204af8725e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Wed, 15 Jul 2026 17:08:22 +0530 Subject: [PATCH 65/91] feat: per-test screenshot + video options with inline Allure attach --- CLAUDE.md | 1 + examples/nightwatch/nightwatch.conf.cjs | 2 + examples/selenium/mocha-test/test/example.js | 4 +- examples/wdio/cucumber/features/login.feature | 2 +- examples/wdio/cucumber/wdio.conf.ts | 26 +- examples/wdio/mocha/wdio.conf.ts | 5 +- package.json | 1 + packages/core/src/artifact-naming.ts | 10 + packages/core/src/index.ts | 3 + packages/core/src/screenshot-artifact.ts | 53 ++++ packages/core/src/trace-finalizer.ts | 5 +- packages/core/src/video-slice.ts | 69 ++++++ packages/core/tests/output-dir.test.ts | 2 +- .../core/tests/screenshot-artifact.test.ts | 53 ++++ packages/core/tests/video-slice.test.ts | 100 ++++++++ packages/service/README.md | 52 +++- packages/service/src/allure.ts | 10 +- packages/service/src/index.ts | 164 +++++++++++-- packages/service/src/screenshot-capture.ts | 67 +++++ packages/service/src/test-metadata.ts | 8 + packages/service/src/video-capture.ts | 80 ++++++ packages/service/tests/allure.test.ts | 9 +- .../service/tests/screenshot-capture.test.ts | 86 +++++++ packages/service/tests/video-capture.test.ts | 75 ++++++ packages/shared/src/types.ts | 19 ++ pnpm-lock.yaml | 231 +++++++++++++++--- 26 files changed, 1065 insertions(+), 72 deletions(-) create mode 100644 packages/core/src/artifact-naming.ts create mode 100644 packages/core/src/screenshot-artifact.ts create mode 100644 packages/core/src/video-slice.ts create mode 100644 packages/core/tests/screenshot-artifact.test.ts create mode 100644 packages/core/tests/video-slice.test.ts create mode 100644 packages/service/src/screenshot-capture.ts create mode 100644 packages/service/src/video-capture.ts create mode 100644 packages/service/tests/screenshot-capture.test.ts create mode 100644 packages/service/tests/video-capture.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 59009622..5a2d88ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,6 +240,7 @@ Documented divergences from the conventions above. They exist today as debt to b - **Nightwatch: `retain-on-failure` works; the other retry-aware policies degrade.** Its `--retries` re-runs the testcase *internally* without re-firing the plugin's per-test hooks, and the per-testcase results carry no attempt/retry field (retries live only in undocumented, version-varying Nightwatch internals — `suiteRetries.testRetriesCount` / `reporter.testResults.retryTest`), so the ledger sees only the final attempt for the `describe/it` and exports-object interfaces. Cucumber scenarios expose per-scenario hooks, so the feed captures their attempts. Not cleanly fixable without depending on those internals. - WDIO `specFileRetries` spawns a fresh worker per retry, so cross-process attempts aren't in the (process-scoped) ledger. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. +- The per-test `screenshot` and `video` options live on `BaseDevToolsOptions` (shared) so all three adapters accept them, but only the **WDIO service** reads them today. The capture/slice/encode logic is framework-agnostic (`core/screenshot-artifact.ts`, `core/video-slice.ts`), so Selenium/Nightwatch adoption is wiring-only (capture the base64 + feed `captureAndAttach*`); pending. Both are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the frame buffer is unbounded for the session (a decimation cap is a noted follow-up), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. - Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. - Service renders expect-webdriverio matchers as single `expect.<matcher>` rows by **folding**, not stack/depth suppression (the old `#assertionDepth`/`#matcherStarted`/self-heal machinery is gone). The matcher's value-read (`toHaveText`→`getText`, `toExist`→`isExisting`, …) is captured as a normal command; `afterAssertion` then coalesces the synthesized `expect.*` row into that read in place — inheriting its callSource, screenshot, and timeline position — and the fold replaces **by timestamp, never a public `id`**: `id` is the per-worker `commandCounter`, which resets per spec, so stamping one lets the app's id-first `replaceCommand` swap a same-id row from another spec (duplicate rows + a fold from another spec vanishing, in multi-spec live mode). `beforeAssertion` arms the pending matcher (depth-counted so aliases like `toBeChecked`→`toBeSelected` fold once); a matcher that **hard-throws** — element never resolves, so expect-webdriverio's `waitUntil` rethrows and `afterAssertion` never fires — is synthesized at `afterTest`/`afterStep` from the throwing read, so a failing assertion renders as `expect.<matcher>` whether or not the element existed. Two limits: its error is then the read's (`Can't call getText on … element wasn't found`), not an assertion-phrased message; and `MATCHER_READ_COMMANDS` is a hand-maintained allowlist, so a matcher whose read isn't listed leaves its raw read visible alongside the `expect.*` row. Plain-value jest matchers (`expect(x).toBe(y)`) don't fire the ewdio hooks, so they aren't captured as rows. diff --git a/examples/nightwatch/nightwatch.conf.cjs b/examples/nightwatch/nightwatch.conf.cjs index 3ec31930..b0d1134d 100644 --- a/examples/nightwatch/nightwatch.conf.cjs +++ b/examples/nightwatch/nightwatch.conf.cjs @@ -53,6 +53,8 @@ module.exports = { // per module (no per-`it` hook), so traceGranularity:'test' collapses to // a single session-scoped slice here. See CLAUDE.md § Known debt. mode: 'trace', + traceGranularity: 'session', + tracePolicy: 'retain-on-first-failure', bidi: true }) } diff --git a/examples/selenium/mocha-test/test/example.js b/examples/selenium/mocha-test/test/example.js index 26360598..25fe5a28 100644 --- a/examples/selenium/mocha-test/test/example.js +++ b/examples/selenium/mocha-test/test/example.js @@ -21,8 +21,8 @@ import { DevTools } from '@wdio/selenium-devtools' // 5 retry: { mode: 'trace', traceGranularity: 'test', tracePolicy: 'on-first-retry' } DevTools.configure({ mode: 'trace', - // traceGranularity: 'test', - // tracePolicy: 'on-first-retry', + traceGranularity: 'session', + tracePolicy: 'retain-on-first-failure', headless: true }) diff --git a/examples/wdio/cucumber/features/login.feature b/examples/wdio/cucumber/features/login.feature index 8b9bcf9a..e6dd0f87 100644 --- a/examples/wdio/cucumber/features/login.feature +++ b/examples/wdio/cucumber/features/login.feature @@ -8,5 +8,5 @@ Feature: The Internet Guinea Pig Website Examples: | username | password | message | - | tomsmith | SuperSecretPassword1! | You logged into a secure area! | + | tomsmith | SuperSecretPassword! | You logged into a secure area! | | foobar | barfoo | Your username is invalid! | diff --git a/examples/wdio/cucumber/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts index e1a82294..75a1f5b9 100644 --- a/examples/wdio/cucumber/wdio.conf.ts +++ b/examples/wdio/cucumber/wdio.conf.ts @@ -69,7 +69,6 @@ export const config: Options.Testrunner = { 'goog:chromeOptions': { args: [ '--headless', - '--disable-gpu', '--remote-allow-origins=*', '--window-size=1600,900' ] @@ -134,7 +133,13 @@ export const config: Options.Testrunner = { 'devtools', { mode: 'trace' as const, - screencast: { enabled: true, pollIntervalMs: 200 } + // tracePolicy: 'retain-on-failure', + traceGranularity: 'test' as const, + // NEW: per-test screenshot + video, attached inline to Allure. + // Both require traceGranularity:'test' (the uniform rule). + screenshot: 'only-on-failure' as const, // 'off' | 'on' | 'only-on-failure' + video: 'on' as const // 'off' | retention policy + // screencast: { enabled: true, pollIntervalMs: 200 } } ] ], @@ -160,7 +165,22 @@ export const config: Options.Testrunner = { // Test reporter for stdout. // The only one supported by default is 'dot' // see also: https://webdriver.io/docs/dot-reporter - reporters: ['spec'], + reporters: [ + 'spec', + [ + 'allure', + { + outputDir: path.resolve(__dirname, 'allure-results'), + // Trace mode issues a takeScreenshot (+ reads) per action to build the + // trace snapshots; @wdio/allure-reporter logs every WebDriver command as + // a step and attaches a screenshot per takeScreenshot, which floods the + // report. Silence both — the trace.zip attachment is unaffected, and the + // gherkin Given/When/Then steps still show. + disableWebdriverStepsReporting: true, + disableWebdriverScreenshotsReporting: true + } + ] + ], // If you are using Cucumber you need to specify the location of your step definitions. cucumberOpts: { diff --git a/examples/wdio/mocha/wdio.conf.ts b/examples/wdio/mocha/wdio.conf.ts index c7680f7d..f6d9e779 100644 --- a/examples/wdio/mocha/wdio.conf.ts +++ b/examples/wdio/mocha/wdio.conf.ts @@ -19,6 +19,7 @@ export const config: Options.Testrunner = { capabilities: [ { browserName: 'chrome', + // browserVersion: '147.0.7727.56', 'goog:chromeOptions': { args: [ '--headless', @@ -45,7 +46,9 @@ export const config: Options.Testrunner = { // 3 per-test: mode: 'trace', traceGranularity: 'test' // 4 fail: mode: 'trace', traceGranularity: 'test', tracePolicy: 'retain-on-failure' // 5 retry: use `pnpm demo:wdio:retry` (adds retries:1 + on-first-retry) - mode: 'live' as const + mode: 'live' as const, + traceGranularity: 'test' as const, + tracePolicy: 'retain-on-failure' as const } ] ], diff --git a/package.json b/package.json index 3e3f2b75..45e4af9f 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@typescript-eslint/utils": "^8.60.1", "@vitest/browser": "^4.1.8", "@vitest/coverage-v8": "^4.1.8", + "@wdio/allure-reporter": "^9.29.1", "autoprefixer": "^10.5.0", "eslint": "^10.4.1", "eslint-config-prettier": "^10.1.8", diff --git a/packages/core/src/artifact-naming.ts b/packages/core/src/artifact-naming.ts new file mode 100644 index 00000000..dcf597b6 --- /dev/null +++ b/packages/core/src/artifact-naming.ts @@ -0,0 +1,10 @@ +/** + * Filename helpers for per-test artifacts (screenshots, videos). Shared so the + * screenshot and video writers slug a test uid identically. + */ + +/** File-safe slug of a value: keep alphanumerics, `_` and `-`; collapse the + * rest to a single `-`; trim leading/trailing dashes. */ +export function fileSlug(value: string): string { + return value.replace(/[^a-zA-Z0-9_-]+/g, '-').replace(/^-+|-+$/g, '') +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9d37bbf0..e6f4a7ba 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,8 +3,11 @@ export * from './action-mapping.js' export * from './action-snapshot.js' +export * from './artifact-naming.js' export * from './artifacts-manifest.js' export * from './attempt-tracker.js' +export * from './screenshot-artifact.js' +export * from './video-slice.js' export * from './with-timeout.js' export * from './assert-patcher.js' export * from './element-snapshot.js' diff --git a/packages/core/src/screenshot-artifact.ts b/packages/core/src/screenshot-artifact.ts new file mode 100644 index 00000000..881ff6e3 --- /dev/null +++ b/packages/core/src/screenshot-artifact.ts @@ -0,0 +1,53 @@ +/** + * Per-test screenshot artifact: the policy decision and the file write, kept + * framework-agnostic so every adapter feeds it the same way (the adapter only + * supplies the base64 capture from its own driver). Mirrors Playwright's + * `screenshot` option — `on` after every test, `only-on-failure` after a + * failing one — and produces a TraceArtifact the manifest and the Allure glue + * consume like any other artifact. + */ + +import fs from 'node:fs/promises' +import path from 'node:path' +import type { TraceScreenshotPolicy } from '@wdio/devtools-shared' +import { fileSlug } from './artifact-naming.js' +import type { TraceArtifact } from './trace-finalizer.js' + +/** Whether a per-test screenshot should be captured for this outcome. */ +export function shouldCaptureScreenshot( + policy: TraceScreenshotPolicy | undefined, + failed: boolean +): boolean { + if (policy === 'on') { + return true + } + if (policy === 'only-on-failure') { + return failed + } + return false +} + +/** + * Write a base64 PNG (from the driver's screenshot command) next to the trace + * output and return the artifact describing it. Retained is always true — a + * screenshot is only ever written when the policy already decided to capture. + */ +export async function writeScreenshotArtifact(input: { + outputDir: string + testUid: string + sessionId: string + base64: string +}): Promise<TraceArtifact> { + await fs.mkdir(input.outputDir, { recursive: true }) + const filename = `screenshot-${fileSlug(input.testUid)}-${input.sessionId.slice(0, 8)}.png` + const filePath = path.join(input.outputDir, filename) + await fs.writeFile(filePath, Buffer.from(input.base64, 'base64')) + return { + kind: 'screenshot', + path: filePath, + scope: 'test', + key: input.testUid, + testUids: [input.testUid], + retained: true + } +} diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index 9cb57af2..ff2b3b99 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -31,9 +31,10 @@ import type { RetryOutcomeView } from './attempt-tracker.js' import { writeTraceZip, type TraceCapturer } from './trace-exporter.js' /** One artifact produced (or, when `retained` is false, decided-against) by a - * trace-mode finalize pass. */ + * trace-mode run — a trace slice, a screencast video, or a per-test + * screenshot. */ export interface TraceArtifact { - kind: 'trace' | 'video' + kind: 'trace' | 'video' | 'screenshot' path: string scope: 'session' | 'spec' | 'test' /** specFile for spec scope, testUid for test scope. */ diff --git a/packages/core/src/video-slice.ts b/packages/core/src/video-slice.ts new file mode 100644 index 00000000..d20e71bb --- /dev/null +++ b/packages/core/src/video-slice.ts @@ -0,0 +1,69 @@ +/** + * Per-test video: slice the continuous screencast frame buffer to one test's + * wall-clock window and encode that slice to a `.webm`. Framework-agnostic — + * the adapter supplies the recorder's frames and the test's start time; the + * recorder itself (CDP push on Chrome, screenshot polling elsewhere) lives in + * each adapter. Best-effort: too few frames or a missing ffmpeg yields no + * artifact rather than an error, so video never aborts a run. + */ + +import fs from 'node:fs/promises' +import path from 'node:path' +import type { ScreencastFrame } from '@wdio/devtools-shared' +import { fileSlug } from './artifact-naming.js' +import { encodeToVideo } from './video-encoder.js' +import { errorMessage } from './error.js' +import type { TraceArtifact } from './trace-finalizer.js' + +/** Frames captured at or after `startWallTime` (ms) — one test's window, since + * the buffer runs continuously and the next test's frames arrive later. */ +export function sliceFramesFrom( + frames: readonly ScreencastFrame[], + startWallTime: number +): ScreencastFrame[] { + return frames.filter((f) => f.timestamp >= startWallTime) +} + +/** + * Encode a test's frame slice to `<outputDir>/video-<uid>-<session>.webm` and + * return the artifact. Returns undefined when there are too few frames to make + * a video or when encoding fails (e.g. fluent-ffmpeg absent) — video is + * best-effort and must never abort the run. + */ +export async function encodePerTestVideo(input: { + frames: readonly ScreencastFrame[] + outputDir: string + testUid: string + sessionId: string + /** 0-based attempt; a `-retry<n>` suffix for n>0 keeps retries from + * overwriting each other's video (mirrors the trace slice's retry keys). */ + attempt?: number + captureFormat?: 'jpeg' | 'png' + minFrames?: number + onLog?: (level: 'info' | 'warn', message: string) => void +}): Promise<TraceArtifact | undefined> { + const minFrames = input.minFrames ?? 2 + if (input.frames.length < minFrames) { + return undefined + } + await fs.mkdir(input.outputDir, { recursive: true }) + const retrySuffix = input.attempt ? `-retry${input.attempt}` : '' + const filename = `video-${fileSlug(input.testUid)}-${input.sessionId.slice(0, 8)}${retrySuffix}.webm` + const filePath = path.join(input.outputDir, filename) + try { + await encodeToVideo([...input.frames], filePath, { + captureFormat: input.captureFormat + }) + } catch (err) { + input.onLog?.('warn', `Per-test video encode failed: ${errorMessage(err)}`) + return undefined + } + return { + kind: 'video', + path: filePath, + scope: 'test', + key: input.testUid, + testUids: [input.testUid], + retained: true + } +} diff --git a/packages/core/tests/output-dir.test.ts b/packages/core/tests/output-dir.test.ts index f44ba3ef..a3226cfa 100644 --- a/packages/core/tests/output-dir.test.ts +++ b/packages/core/tests/output-dir.test.ts @@ -4,7 +4,7 @@ import path from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { resolveAdapterOutputDir } from '../src/output-dir.js' -/** Every resolved dir is grouped under this subfolder (Playwright-style). */ +/** Every resolved dir is grouped under this subfolder */ const grouped = (base: string) => path.join(base, 'test-results') describe('resolveAdapterOutputDir', () => { diff --git a/packages/core/tests/screenshot-artifact.test.ts b/packages/core/tests/screenshot-artifact.test.ts new file mode 100644 index 00000000..a24a1efe --- /dev/null +++ b/packages/core/tests/screenshot-artifact.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + shouldCaptureScreenshot, + writeScreenshotArtifact +} from '../src/screenshot-artifact.js' + +describe('shouldCaptureScreenshot', () => { + it('captures for `on` regardless of outcome', () => { + expect(shouldCaptureScreenshot('on', false)).toBe(true) + expect(shouldCaptureScreenshot('on', true)).toBe(true) + }) + + it('captures for `only-on-failure` only when failed', () => { + expect(shouldCaptureScreenshot('only-on-failure', true)).toBe(true) + expect(shouldCaptureScreenshot('only-on-failure', false)).toBe(false) + }) + + it('never captures for `off` or undefined', () => { + expect(shouldCaptureScreenshot('off', true)).toBe(false) + expect(shouldCaptureScreenshot(undefined, true)).toBe(false) + }) +}) + +describe('writeScreenshotArtifact', () => { + const dirs: string[] = [] + afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) + }) + + it('writes the decoded PNG and returns a test-scoped screenshot artifact', async () => { + const dir = await mkdtemp(join(tmpdir(), 'shot-')) + dirs.push(dir) + const base64 = Buffer.from('png-bytes').toString('base64') + const artifact = await writeScreenshotArtifact({ + outputDir: dir, + testUid: 'suite > a test/name', + sessionId: 'abcd1234ef', + base64 + }) + expect(artifact.kind).toBe('screenshot') + expect(artifact.scope).toBe('test') + expect(artifact.key).toBe('suite > a test/name') + expect(artifact.retained).toBe(true) + // Filename is slugged (no spaces/slashes) and session-scoped. + expect(artifact.path).toMatch(/screenshot-suite-a-test-name-abcd1234\.png$/) + expect(await readFile(artifact.path, 'utf8')).toBe('png-bytes') + }) +}) diff --git a/packages/core/tests/video-slice.test.ts b/packages/core/tests/video-slice.test.ts new file mode 100644 index 00000000..c2de429a --- /dev/null +++ b/packages/core/tests/video-slice.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ScreencastFrame } from '@wdio/devtools-shared' + +const { encodeToVideo } = vi.hoisted(() => ({ encodeToVideo: vi.fn() })) +vi.mock('../src/video-encoder.js', () => ({ encodeToVideo })) + +import { sliceFramesFrom, encodePerTestVideo } from '../src/video-slice.js' + +const frames = (ts: number[]): ScreencastFrame[] => + ts.map((timestamp) => ({ data: 'AAAA', timestamp })) + +describe('sliceFramesFrom', () => { + it('keeps only frames at or after the window start', () => { + const all = frames([100, 200, 300, 400]) + expect(sliceFramesFrom(all, 250).map((f) => f.timestamp)).toEqual([ + 300, 400 + ]) + }) + + it('returns all frames when start precedes them', () => { + expect(sliceFramesFrom(frames([100, 200]), 0)).toHaveLength(2) + }) +}) + +describe('encodePerTestVideo', () => { + const dirs: string[] = [] + beforeEach(() => encodeToVideo.mockReset()) + afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) + }) + async function dir() { + const d = await mkdtemp(join(tmpdir(), 'vid-')) + dirs.push(d) + return d + } + + it('returns undefined below the min-frames threshold (no encode)', async () => { + const artifact = await encodePerTestVideo({ + frames: frames([1]), + outputDir: await dir(), + testUid: 'u1', + sessionId: 'sess1234' + }) + expect(artifact).toBeUndefined() + expect(encodeToVideo).not.toHaveBeenCalled() + }) + + it('encodes and returns a test-scoped video artifact', async () => { + const d = await dir() + encodeToVideo.mockResolvedValueOnce(undefined) + const artifact = await encodePerTestVideo({ + frames: frames([1, 2, 3]), + outputDir: d, + testUid: 'my/test', + sessionId: 'sess1234ef' + }) + expect(artifact).toBeDefined() + expect(artifact!.kind).toBe('video') + expect(artifact!.scope).toBe('test') + expect(artifact!.key).toBe('my/test') + expect(artifact!.path).toMatch(/video-my-test-sess1234\.webm$/) + expect(encodeToVideo).toHaveBeenCalledOnce() + }) + + it('suffixes the filename with the attempt so retries do not overwrite', async () => { + encodeToVideo.mockResolvedValue(undefined) + const a0 = await encodePerTestVideo({ + frames: frames([1, 2]), + outputDir: await dir(), + testUid: 'u1', + sessionId: 'sess1234', + attempt: 0 + }) + const a1 = await encodePerTestVideo({ + frames: frames([1, 2]), + outputDir: await dir(), + testUid: 'u1', + sessionId: 'sess1234', + attempt: 1 + }) + expect(a0!.path).toMatch(/video-u1-sess1234\.webm$/) + expect(a1!.path).toMatch(/video-u1-sess1234-retry1\.webm$/) + }) + + it('returns undefined (no throw) when the encoder fails', async () => { + encodeToVideo.mockRejectedValueOnce(new Error('ffmpeg missing')) + const artifact = await encodePerTestVideo({ + frames: frames([1, 2, 3]), + outputDir: await dir(), + testUid: 'u1', + sessionId: 'sess1234' + }) + expect(artifact).toBeUndefined() + }) +}) diff --git a/packages/service/README.md b/packages/service/README.md index d5532798..bfd4002f 100644 --- a/packages/service/README.md +++ b/packages/service/README.md @@ -46,8 +46,58 @@ services: [['devtools', options]] | `port` | `number` | random | Port the DevTools UI server listens on | | `hostname` | `string` | `'localhost'` | Hostname the DevTools UI server binds to | | `devtoolsCapabilities` | `Capabilities` | Chrome 1600×1200 | Capabilities used to open the DevTools UI window | -| `screencast` | `ScreencastOptions` | — | Session video recording (live mode only — see below) | +| `screencast` | `ScreencastOptions` | — | Session video recording (live mode only — see below; for trace mode use `video`) | | `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-<sessionId>.zip` at session end. See [Trace mode](../../README.md#-trace-mode-tracezip) | +| `traceGranularity` | `'session' \| 'spec' \| 'test'` | `'session'` | Trace mode only. How traces are partitioned. `'test'` is required for per-test Allure attachments (trace, screenshot, video). | +| `tracePolicy` | `TraceRetentionPolicy` | `'on'` | Trace mode only. Which traces to keep — e.g. `'retain-on-failure'`, `'retain-on-first-failure'`. | +| `screenshot` | `'off' \| 'on' \| 'only-on-failure'` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screenshot, attached inline to Allure (`image/png`). | +| `video` | `'off' \| TraceRetentionPolicy` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screencast video, retained per the given policy, attached inline to Allure (`video/webm`). | + +## Allure integration + +When `@wdio/allure-reporter` is installed, trace-mode artifacts are attached to +the Allure report automatically: + +- **`traceGranularity: 'test'`** — each test's trace (`application/zip`, a + download that opens in `pnpm show-trace`), screenshot (`image/png`, inline) + and video (`video/webm`, inline) attach to that test's card. This is the mode + to use for a per-test Allure report. +- **`traceGranularity: 'session'` / `'spec'`** — a session/spec-spanning + `trace.zip` is written to disk and enumerated in + `devtools-artifacts-<sessionId>.json` (the artifacts manifest, listing every + artifact + each test's state), but it is **not** attached to individual test + cards. + +### Why session/spec traces aren't attached per test + +The reporter's `addAttachment` targets the **currently-running test**. A +session/spec trace is only finalized **after** all its tests have run — by which +point their Allure cards are already closed — so there is no open test to attach +it to. Per-test attachment therefore requires `traceGranularity: 'test'`, where +each slice is written during its own test hook while the card is still open. + +To surface a session/spec trace in Allure anyway, post-process the manifest in +your **own** `onComplete` hook (copying the `trace.zip` into `allure-results/` +and appending it to the result files). This is deliberately left to userland — +baking it into the adapter would couple it to Allure's on-disk result format. + +### Report noise + +In trace mode the service captures a per-action snapshot (a `takeScreenshot` +WebDriver command) to build the trace timeline; `@wdio/allure-reporter` logs +every WebDriver command as a step and attaches a screenshot per `takeScreenshot`. +Silence that flood with the reporter's own options — the `trace.zip` / +screenshot / video attachments are unaffected: + +```ts +reporters: [ + ['allure', { + outputDir: 'allure-results', + disableWebdriverStepsReporting: true, + disableWebdriverScreenshotsReporting: true + }] +] +``` ## Screencast Recording diff --git a/packages/service/src/allure.ts b/packages/service/src/allure.ts index ce0e678d..ce415529 100644 --- a/packages/service/src/allure.ts +++ b/packages/service/src/allure.ts @@ -16,6 +16,13 @@ const log = logger('@wdio/devtools-service') // uses video/webm so Allure renders it inline. const TRACE_CONTENT_TYPE = 'application/zip' const VIDEO_CONTENT_TYPE = 'video/webm' +const SCREENSHOT_CONTENT_TYPE = 'image/png' + +const CONTENT_TYPE_BY_KIND: Record<TraceArtifact['kind'], string> = { + trace: TRACE_CONTENT_TYPE, + video: VIDEO_CONTENT_TYPE, + screenshot: SCREENSHOT_CONTENT_TYPE +} /** The one @wdio/allure-reporter method we use. Typed locally so the optional * peer dependency never becomes a build-time type dependency. */ @@ -75,8 +82,7 @@ export async function attachArtifactToAllure( return } const content = await fs.readFile(artifact.path) - const type = - artifact.kind === 'video' ? VIDEO_CONTENT_TYPE : TRACE_CONTENT_TYPE + const type = CONTENT_TYPE_BY_KIND[artifact.kind] reporter.addAttachment(basename(artifact.path), content, type) } catch (err) { // A missing/unreadable artifact must never reject the test hook. diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index fd24e32a..dc9cea69 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -14,10 +14,13 @@ import { type TraceExportContext } from '@wdio/devtools-core' import { attachArtifactToAllure } from './allure.js' +import { captureAndAttachScreenshot } from './screenshot-capture.js' +import { captureAndAttachVideo } from './video-capture.js' import { wireAssertCapture, type ExpectAssertion } from './assert-capture.js' import { AssertionTracker } from './assertion-tracker.js' import { cucumberScenarioUid, + isFailedResult, resolveTestAttempt, stampTestState, testMetadataUid, @@ -33,7 +36,11 @@ import { dedupeSnapshotsByTimestamp, upsertRichestSnapshot } from './snapshot-dedupe.js' -import type { ActionSnapshot, TestMetadataMap } from '@wdio/devtools-shared' +import type { + ActionSnapshot, + ScreencastFrame, + TestMetadataMap +} from '@wdio/devtools-shared' import { SevereServiceError } from 'webdriverio' import type { Services, Capabilities, Options, Reporters } from '@wdio/types' import type { WebDriverCommands } from '@wdio/protocols' @@ -95,11 +102,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { log.warn(policyWarning) } if (serviceOptions.mode === 'trace' && serviceOptions.screencast?.enabled) { - log.warn('trace mode: ignoring screencast option (live-mode feature)') - this.#screencastOptions = undefined - } else { - this.#screencastOptions = serviceOptions.screencast + log.warn( + 'trace mode: `screencast.enabled` is ignored — use `video` to record; ' + + 'the tuning fields (quality/interval) still apply' + ) } + // Tuning is kept for both modes; whether we actually record is decided by + // #shouldRecordScreencast (screencast.enabled in live, `video` in trace). + this.#screencastOptions = serviceOptions.screencast } /** @@ -111,6 +121,19 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Current test UID, set in beforeTest(), used by afterCommand() to tag commands. */ #currentTestUid?: string + /** Wall-clock ms at the current test's start, set in beforeTest/beforeScenario; + * the lower bound of that test's video frame window (per-test slicing). */ + #currentTestStartWallTime = 0 + + /** Recorder frames snapshotted in onReload before reloadSession replaces the + * recorder — so the ending test's per-test video can still be sliced in + * afterScenario (which runs AFTER the cucumber After hook's reloadSession). */ + #pendingVideoFrames?: { + testUid: string | undefined + startWallTime: number + frames: ScreencastFrame[] + } + /** Map of testUid → metadata for trace group events and per-spec partitioning. */ #testMetadata: TestMetadataMap = new Map() @@ -159,6 +182,34 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } + /** Base64 of the last rendered action snapshot for the current test, skipping + * the end-of-scenario `__final__` frame — that one is captured post-teardown + * and can come back blank when a reloadSession runs before afterScenario. + * Used as the per-test screenshot so it's the real failure-moment frame, + * reload-immune. Scoped to this test's window (>= its start) so a test that + * captured nothing doesn't borrow the previous test's frame. */ + #lastRenderedScreenshot(): string | undefined { + for (let i = this.#actionSnapshots.length - 1; i >= 0; i--) { + const snap = this.#actionSnapshots[i]! + if (snap.timestamp < this.#currentTestStartWallTime) { + return undefined + } + if (snap.command !== '__final__' && snap.screenshot) { + return snap.screenshot + } + } + return undefined + } + + /** Record a screencast this session? Live mode: `screencast.enabled`. Trace + * mode: a non-`off` `video` policy (frames are sliced per test at flush). */ + #shouldRecordScreencast(): boolean { + if (this.#options.mode === 'trace') { + return !!this.#options.video && this.#options.video !== 'off' + } + return !!this.#screencastOptions?.enabled + } + /** Eager per-test flush at test end (test granularity only), run after the * outcome is stamped so this attempt's metadata is written before a retry * overwrites it; the end-of-run finalizer then dedupes it via the key set. */ @@ -254,12 +305,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { } /** - * Start screencast recording if the user has enabled it. - * Options come from the service constructor (services: [['devtools', { screencast: { enabled: true } }]]). - * Failures are non-fatal — a warning is logged and the session continues. + * Start screencast recording when enabled — `screencast.enabled` in live + * mode, or a non-`off` `video` policy in trace mode (per-test slicing at + * flush time). Failures are non-fatal — logged, session continues. */ - if (this.#screencastOptions?.enabled) { - this.#screencastRecorder = new ScreencastRecorder(this.#screencastOptions) + if (this.#shouldRecordScreencast()) { + this.#screencastRecorder = new ScreencastRecorder( + this.#screencastOptions ?? {} + ) await this.#screencastRecorder.start(browser) } @@ -347,6 +400,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { pickle?: { uri?: string; name?: string; astNodeIds?: readonly string[] } }) { this.resetStack() + this.#currentTestStartWallTime = Date.now() const featureFile = world?.pickle?.uri const scenarioName = world?.pickle?.name @@ -377,6 +431,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Mocha/Jasmine hook: the beforeScenario equivalent for file-based specs. */ beforeTest(test?: { file?: string; title?: string; fullTitle?: string }) { this.resetStack() + this.#currentTestStartWallTime = Date.now() // Track test identity for command tagging. Generate a stable UID // from file + title so commands can be partitioned across reruns. @@ -436,7 +491,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#stampOutcome(uid, result) } await this.#finalizePerScenario() - await this.#eagerFlushAndAttach(uid) + await this.#emitTestArtifacts(uid, isFailedResult(result)) } async afterTest( @@ -451,20 +506,60 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#stampOutcome(uid, result) } await this.#finalizePerScenario() - await this.#eagerFlushAndAttach(uid) + await this.#emitTestArtifacts(uid, isFailedResult(result)) } - /** Flush this test's slice (so it captures the final snapshot + stamped - * outcome), then attach the retained artifact to Allure while the per-test - * hook is still open. No-op outside `test`-granularity trace mode. */ - async #eagerFlushAndAttach(uid: string | undefined): Promise<void> { - if (!uid) { - return - } - const artifact = await this.#eagerFlushTestSlice(uid) - if (artifact) { - await attachArtifactToAllure(artifact) + /** At test end, while the per-test hook is still open: eager-flush this test's + * slice (so it captures the final snapshot + stamped outcome) and attach the + * retained trace to Allure, then capture the per-test screenshot per policy + * and attach it too. Each part no-ops when its feature is off. */ + async #emitTestArtifacts( + uid: string | undefined, + failed: boolean + ): Promise<void> { + if (uid) { + const artifact = await this.#eagerFlushTestSlice(uid) + if (artifact) { + await attachArtifactToAllure(artifact) + } } + await captureAndAttachScreenshot({ + mode: this.#options.mode, + granularity: this.#options.traceGranularity, + policy: this.#options.screenshot, + failed, + screenshotBase64: this.#lastRenderedScreenshot(), + sessionId: this.#browser?.sessionId, + outputDir: this.#outputDir, + testUid: uid, + onArtifact: (a) => this.#artifacts.push(a) + }) + // Authoritative attempt for this test (stamped into metadata by + // #stampOutcome, which ran just before this). Scopes retention + the video + // filename to this attempt so retries don't overwrite each other. + const attempt = uid ? this.#testMetadata.get(uid)?.attempt : undefined + // Prefer frames snapshotted in onReload (reloadSession tears the recorder + // down before this hook); fall back to the live recorder otherwise. + const pending = + this.#pendingVideoFrames?.testUid === uid + ? this.#pendingVideoFrames + : undefined + this.#pendingVideoFrames = undefined + await captureAndAttachVideo({ + mode: this.#options.mode, + granularity: this.#options.traceGranularity, + policy: this.#options.video, + frames: pending?.frames ?? this.#screencastRecorder?.frames, + startWallTime: pending?.startWallTime ?? this.#currentTestStartWallTime, + outcomes: uid ? this.#attemptTracker.forTest(uid, attempt) : [], + attempt, + outputDir: this.#outputDir, + testUid: uid, + sessionId: this.#browser?.sessionId, + captureFormat: this.#screencastOptions?.captureFormat, + onArtifact: (a) => this.#artifacts.push(a), + onLog: (level, msg) => log[level](msg) + }) } /** expect-webdriverio matcher hooks — delegated to the assertion tracker. */ @@ -663,16 +758,30 @@ export default class DevToolsHookService implements Services.ServiceInstance { * on the new session so the second scenario is also covered. */ async onReload(oldSessionId: string, _newSessionId: string) { - if (!this.#screencastOptions?.enabled || !this.#browser) { + if (!this.#shouldRecordScreencast() || !this.#browser) { return } + // Trace mode: the ending test's afterScenario runs AFTER this reload (a + // cucumber `After(() => reloadSession())` is WDIO boilerplate), by which + // point the recorder below has replaced these frames. Snapshot them now, + // keyed to the ending test, so afterScenario can still slice its video. + if (this.#options.mode === 'trace' && this.#screencastRecorder) { + this.#pendingVideoFrames = { + testUid: this.#currentTestUid, + startWallTime: this.#currentTestStartWallTime, + frames: [...this.#screencastRecorder.frames] + } + } + // Finalize the recording from the old session (CDP is already gone, so // stop() will fail gracefully and we encode whatever frames arrived). await this.#finalizeScreencast(oldSessionId) // Start a new recorder for the new session. - this.#screencastRecorder = new ScreencastRecorder(this.#screencastOptions) + this.#screencastRecorder = new ScreencastRecorder( + this.#screencastOptions ?? {} + ) await this.#screencastRecorder.start(this.#browser) } @@ -708,6 +817,13 @@ export default class DevToolsHookService implements Services.ServiceInstance { if (!this.#screencastRecorder) { return } + // Trace mode: the video is emitted per-test (sliced in #emitTestArtifacts), + // and there's no dashboard to receive a session recording — so just stop the + // recorder to release resources; never encode an orphan session-wide webm. + if (this.#options.mode === 'trace') { + await this.#screencastRecorder.stop() + return + } // Skip ghost sessions: browser.reloadSession() creates a new session at // the end of a test run that has no steps — it captures at most a handful // of frames before teardown. Require at least 5 frames so we don't produce diff --git a/packages/service/src/screenshot-capture.ts b/packages/service/src/screenshot-capture.ts new file mode 100644 index 00000000..9aca639c --- /dev/null +++ b/packages/service/src/screenshot-capture.ts @@ -0,0 +1,67 @@ +// Per-test failure/always screenshot for the WDIO adapter: the policy gate and +// the Allure attach. Kept out of index.ts so the god-file stays lean and the +// capture is unit-testable. The policy decision and file write live in core +// (framework-agnostic). +// +// The image is NOT a fresh screenshot taken here — it's the last rendered action +// snapshot the service already captured during the test. A fresh end-of-test +// takeScreenshot() is unreliable: a cucumber `After(() => reloadSession())` hook +// (WDIO boilerplate) runs before the service afterScenario hook and blanks the +// page, so an end-of-test capture comes back empty. The last action snapshot was +// captured mid-command, before any teardown, so it's the real failure-moment +// frame — and reusing it also drops an extra WebDriver command from the report. + +import { + shouldCaptureScreenshot, + writeScreenshotArtifact, + type TraceArtifact +} from '@wdio/devtools-core' +import type { + DevToolsMode, + TraceGranularity, + TraceScreenshotPolicy +} from '@wdio/devtools-shared' +import { attachArtifactToAllure } from './allure.js' + +/** + * Write the per-test screenshot (per the policy) from an already-captured base64 + * frame and attach it to Allure. No-op outside `test`-granularity trace mode, + * without a uid/session/frame, or when the policy declines. Gated to `test` + * granularity so the whole per-test inline story (trace + screenshot + video) is + * one rule; coarser granularities keep their artifacts in the manifest. The + * artifact is handed to `onArtifact` so it lands in the manifest too. + */ +export async function captureAndAttachScreenshot(input: { + mode: DevToolsMode | undefined + granularity: TraceGranularity | undefined + policy: TraceScreenshotPolicy | undefined + failed: boolean + /** Base64 of the last rendered action snapshot (reload-immune, not a fresh + * end-of-test capture). Undefined when the test recorded no snapshot. */ + screenshotBase64: string | undefined + sessionId: string | undefined + outputDir: string + testUid: string | undefined + onArtifact: (artifact: TraceArtifact) => void +}): Promise<void> { + const { mode, granularity, policy, failed, screenshotBase64, sessionId } = + input + if ( + mode !== 'trace' || + granularity !== 'test' || + !screenshotBase64 || + !sessionId || + !input.testUid || + !shouldCaptureScreenshot(policy, failed) + ) { + return + } + const artifact = await writeScreenshotArtifact({ + outputDir: input.outputDir, + testUid: input.testUid, + sessionId, + base64: screenshotBase64 + }) + input.onArtifact(artifact) + await attachArtifactToAllure(artifact) +} diff --git a/packages/service/src/test-metadata.ts b/packages/service/src/test-metadata.ts index dc2babad..5e37ba20 100644 --- a/packages/service/src/test-metadata.ts +++ b/packages/service/src/test-metadata.ts @@ -46,6 +46,14 @@ export function resultToState(result: { return result.passed ? 'passed' : 'failed' } +/** True when a result represents a failed test (not passed, not skipped). + * Absent result → not failed (nothing ran to fail). */ +export function isFailedResult( + result?: Pick<TestOutcomeResult, 'passed' | 'skipped'> +): boolean { + return result ? resultToState(result) === 'failed' : false +} + /** Stamp the final state (and, when known, the 0-based attempt) onto the * metadata entry beforeTest/beforeScenario created, so retention can gate its * trace per attempt. No-op when there's no entry. */ diff --git a/packages/service/src/video-capture.ts b/packages/service/src/video-capture.ts new file mode 100644 index 00000000..ff8b1d06 --- /dev/null +++ b/packages/service/src/video-capture.ts @@ -0,0 +1,80 @@ +// Per-test video for the WDIO adapter: the granularity/policy gate, the +// retention decision, the frame-window slice, the encode, and the Allure +// attach. Kept out of index.ts so the god-file stays lean. The slice + encode +// live in core (framework-agnostic); only the recorder is WDIO-specific. + +import { + encodePerTestVideo, + shouldRetainTrace, + sliceFramesFrom, + type TestOutcome, + type TraceArtifact +} from '@wdio/devtools-core' +import type { + DevToolsMode, + ScreencastFrame, + TraceGranularity, + TraceVideoPolicy +} from '@wdio/devtools-shared' +import { attachArtifactToAllure } from './allure.js' + +/** + * Slice the continuous screencast to this test's window, encode a `.webm`, and + * attach it to Allure — when trace mode + `test` granularity + a non-`off` + * `video` policy that retains this test's outcome. No-op otherwise. The encoded + * artifact is handed to `onArtifact` so it lands in the manifest. + */ +export async function captureAndAttachVideo(input: { + mode: DevToolsMode | undefined + granularity: TraceGranularity | undefined + policy: TraceVideoPolicy | undefined + frames: readonly ScreencastFrame[] | undefined + startWallTime: number + outcomes: TestOutcome[] + attempt: number | undefined + outputDir: string + testUid: string | undefined + sessionId: string | undefined + captureFormat?: 'jpeg' | 'png' + onArtifact: (artifact: TraceArtifact) => void + onLog?: (level: 'info' | 'warn', message: string) => void +}): Promise<void> { + const { mode, granularity, policy, frames, testUid, sessionId } = input + if ( + mode !== 'trace' || + granularity !== 'test' || + !policy || + policy === 'off' || + !frames || + !testUid || + !sessionId || + // No recorded attempt for this uid — the test never started (e.g. a + // skipped/pending test whose afterTest fires without a beforeTest). Skip + // rather than fail-open on empty outcomes, which would slice the PREVIOUS + // test's frames into a video attributed to this one. + input.outcomes.length === 0 + ) { + return + } + const decision = shouldRetainTrace(policy, { + outcomes: input.outcomes, + attemptInfoAvailable: true + }) + if (!decision.retain) { + return + } + const artifact = await encodePerTestVideo({ + frames: sliceFramesFrom(frames, input.startWallTime), + outputDir: input.outputDir, + testUid, + sessionId, + attempt: input.attempt, + captureFormat: input.captureFormat, + onLog: input.onLog + }) + if (!artifact) { + return + } + input.onArtifact(artifact) + await attachArtifactToAllure(artifact) +} diff --git a/packages/service/tests/allure.test.ts b/packages/service/tests/allure.test.ts index e8305f6a..aa269eed 100644 --- a/packages/service/tests/allure.test.ts +++ b/packages/service/tests/allure.test.ts @@ -56,7 +56,7 @@ describe('attachArtifactToAllure', () => { expect(addAttachment).not.toHaveBeenCalled() }) - it('attaches a trace zip with the Allure Playwright content type', async () => { + it('attaches a trace zip as a plain application/zip download', async () => { const path = await tempZip('trace-abc.zip') await attachArtifactToAllure(artifact({ path })) expect(addAttachment).toHaveBeenCalledOnce() @@ -73,6 +73,13 @@ describe('attachArtifactToAllure', () => { expect(type).toBe('video/webm') }) + it('attaches a screenshot artifact as image/png', async () => { + const path = await tempZip('screenshot-u1.png') + await attachArtifactToAllure(artifact({ kind: 'screenshot', path })) + const [, , type] = addAttachment.mock.calls[0]! + expect(type).toBe('image/png') + }) + it('skips a directory-format artifact without throwing (ndjson-directory)', async () => { const dir = await mkdtemp(join(tmpdir(), 'allure-dir-')) dirs.push(dir) diff --git a/packages/service/tests/screenshot-capture.test.ts b/packages/service/tests/screenshot-capture.test.ts new file mode 100644 index 00000000..7c3bdb1d --- /dev/null +++ b/packages/service/tests/screenshot-capture.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtemp, rm, readdir } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { TraceArtifact } from '@wdio/devtools-core' + +const addAttachment = vi.fn() +vi.mock('@wdio/allure-reporter', () => ({ + addAttachment, + default: { addAttachment } +})) + +import { captureAndAttachScreenshot } from '../src/screenshot-capture.js' +import { resetAllureReporterCache } from '../src/allure.js' + +const SHOT = Buffer.from('png-bytes').toString('base64') + +describe('captureAndAttachScreenshot', () => { + const dirs: string[] = [] + const collected: TraceArtifact[] = [] + beforeEach(() => { + addAttachment.mockReset() + resetAllureReporterCache() + collected.length = 0 + }) + afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) + }) + + async function run( + over: Partial<Parameters<typeof captureAndAttachScreenshot>[0]> + ) { + const dir = await mkdtemp(join(tmpdir(), 'shot-cap-')) + dirs.push(dir) + await captureAndAttachScreenshot({ + mode: 'trace', + granularity: 'test', + policy: 'only-on-failure', + failed: true, + screenshotBase64: SHOT, + sessionId: 'sess1234', + outputDir: dir, + testUid: 'u1', + onArtifact: (a) => collected.push(a), + ...over + }) + return { dir } + } + + it('writes + attaches the reused snapshot on failure under only-on-failure', async () => { + const { dir } = await run({}) + expect(collected).toHaveLength(1) + expect(collected[0]!.kind).toBe('screenshot') + expect(addAttachment).toHaveBeenCalledOnce() + expect(await readdir(dir)).toHaveLength(1) + }) + + it('does nothing on a passing test under only-on-failure', async () => { + await run({ failed: false }) + expect(collected).toHaveLength(0) + expect(addAttachment).not.toHaveBeenCalled() + }) + + it('does nothing outside trace mode', async () => { + await run({ mode: 'live', policy: 'on' }) + expect(collected).toHaveLength(0) + }) + + it('does nothing outside test granularity (uniform rule)', async () => { + await run({ granularity: 'session', policy: 'on' }) + expect(collected).toHaveLength(0) + }) + + it('does nothing without a captured frame (no snapshot to reuse)', async () => { + await run({ policy: 'on', screenshotBase64: undefined }) + expect(collected).toHaveLength(0) + }) + + it('does nothing without a sessionId or uid', async () => { + await run({ sessionId: undefined }) + await run({ testUid: undefined }) + expect(collected).toHaveLength(0) + }) +}) diff --git a/packages/service/tests/video-capture.test.ts b/packages/service/tests/video-capture.test.ts new file mode 100644 index 00000000..25c1c404 --- /dev/null +++ b/packages/service/tests/video-capture.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import type { ScreencastFrame } from '@wdio/devtools-shared' +import type { TraceArtifact, TestOutcome } from '@wdio/devtools-core' + +// Every case here is designed to no-op BEFORE the ffmpeg encode (failed gate or +// non-retaining policy), so no encoder/reporter mock is needed — the real +// (pure) shouldRetainTrace decides retention. The encode path itself is covered +// by core's video-slice.test.ts. +import { captureAndAttachVideo } from '../src/video-capture.js' + +const frames: ScreencastFrame[] = [ + { data: 'AAAA', timestamp: 10 }, + { data: 'AAAA', timestamp: 20 }, + { data: 'AAAA', timestamp: 30 } +] + +const failedOutcome: TestOutcome[] = [ + { uid: 'u1', attempt: 0, state: 'failed' } +] +const passedOutcome: TestOutcome[] = [ + { uid: 'u1', attempt: 0, state: 'passed' } +] + +describe('captureAndAttachVideo gating', () => { + const collected: TraceArtifact[] = [] + beforeEach(() => { + collected.length = 0 + }) + + const base = { + mode: 'trace' as const, + granularity: 'test' as const, + policy: 'retain-on-failure' as const, + frames, + startWallTime: 0, + outcomes: failedOutcome, + attempt: 0, + outputDir: '/tmp/does-not-encode', + testUid: 'u1', + sessionId: 'sess1234', + onArtifact: (a: TraceArtifact) => collected.push(a) + } + + it('no-ops outside test granularity', async () => { + await captureAndAttachVideo({ ...base, granularity: 'session' }) + expect(collected).toHaveLength(0) + }) + + it('no-ops when the test never started (empty outcomes — no fail-open)', async () => { + await captureAndAttachVideo({ ...base, outcomes: [] }) + expect(collected).toHaveLength(0) + }) + + it('no-ops when video policy is off/undefined', async () => { + await captureAndAttachVideo({ ...base, policy: 'off' }) + await captureAndAttachVideo({ ...base, policy: undefined }) + expect(collected).toHaveLength(0) + }) + + it('no-ops when there are no frames', async () => { + await captureAndAttachVideo({ ...base, frames: undefined }) + expect(collected).toHaveLength(0) + }) + + it('no-ops when the policy does not retain this outcome (passing + retain-on-failure)', async () => { + await captureAndAttachVideo({ ...base, outcomes: passedOutcome }) + expect(collected).toHaveLength(0) + }) + + it('no-ops without a sessionId or uid', async () => { + await captureAndAttachVideo({ ...base, sessionId: undefined }) + await captureAndAttachVideo({ ...base, testUid: undefined }) + expect(collected).toHaveLength(0) + }) +}) diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 1e2c7033..b360936d 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -49,6 +49,17 @@ export type TraceRetentionPolicy = | 'on-all-retries' | 'retain-on-failure-and-retries' +/** Per-test screenshot capture policy, mirroring Playwright's `screenshot` + * option. `only-on-failure` shoots after a failing test; `on` after every + * test; `off` (default) never. Only applies in trace mode. */ +export type TraceScreenshotPolicy = 'off' | 'on' | 'only-on-failure' + +/** Per-test video capture policy. `off` (default) records nothing; any other + * value records the screencast and keeps each test's video slice per the same + * retention semantics as `tracePolicy`. Only applies in trace mode at + * `traceGranularity: 'test'` (the per-test scope videos attach to). */ +export type TraceVideoPolicy = 'off' | TraceRetentionPolicy + /** One node in a test's ancestor chain, outermost first. */ export interface TestAncestor { uid: string @@ -306,6 +317,14 @@ export interface BaseDevToolsOptions { /** Trace retention policy — gates which traces are kept (e.g. * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ tracePolicy?: TraceRetentionPolicy + /** Per-test screenshot capture, attached to the trace artifacts and to Allure + * inline. `off` (default) | `on` | `only-on-failure`. Only applies in trace + * mode. */ + screenshot?: TraceScreenshotPolicy + /** Per-test video (screencast) capture, retained per the given policy and + * attached to Allure inline. `off` (default) or a retention policy. Only + * applies in trace mode at `traceGranularity: 'test'`. */ + video?: TraceVideoPolicy } /** Minimal Cucumber pickle-step shape — only the fields the adapters read. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1de3557..8f03e235 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: '@vitest/coverage-v8': specifier: ^4.1.8 version: 4.1.9(@vitest/browser@4.1.9)(vitest@4.1.9) + '@wdio/allure-reporter': + specifier: ^9.29.1 + version: 9.29.1 autoprefixer: specifier: ^10.5.0 version: 10.5.0(postcss@8.5.15) @@ -152,7 +155,7 @@ importers: version: 9.28.0 expect-webdriverio: specifier: ^5.6.7 - version: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.18.0)(webdriverio@9.28.0(puppeteer-core@21.11.0)) + version: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.29.1)(webdriverio@9.28.0(puppeteer-core@21.11.0)) ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@25.9.3)(typescript@6.0.3) @@ -368,7 +371,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.0.16 - version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) packages/nightwatch-devtools: dependencies: @@ -538,6 +541,9 @@ importers: '@types/yazl': specifier: ^2.4.6 version: 2.4.6 + '@wdio/allure-reporter': + specifier: ^9.0.0 + version: 9.29.1 '@wdio/devtools-backend': specifier: workspace:^ version: link:../backend @@ -2711,6 +2717,10 @@ packages: '@volar/typescript@2.4.28': resolution: {integrity: sha512-Ja6yvWrbis2QtN4ClAKreeUZPVYMARDYZl9LMEv1iQ1QdepB6wn0jTRxA9MftYmYa4DQ4k/DaSZpFPUfxl8giw==} + '@wdio/allure-reporter@9.29.1': + resolution: {integrity: sha512-NroTqEN+l0WHVbqS6CH7sTrFtd3d0mgxjPFvFcbzCER1+GA0chTzFT/GCwl4WvNrbbX6a/z7QWT+UwWdXJQJvQ==} + engines: {node: '>=18.20.0'} + '@wdio/cli@9.28.0': resolution: {integrity: sha512-jEKYdCvZ9ST8YQ4EvyV9lsEoRxhWenplGJppbiH9SKHiwPqrUapi/EE7f6CBDwkWP7NIlzj2PyTe+JRmkXILLw==} engines: {node: '>=18.20.0'} @@ -2751,6 +2761,10 @@ packages: resolution: {integrity: sha512-HdzDrRs+ywAqbXGKqe1i/bLtCv47plz4TvsHFH3j729OooT5VH38ctFn5aLXgECmiAKDkmH/A6kOq2Zh5DIxww==} engines: {node: '>=18.20.0'} + '@wdio/logger@9.29.1': + resolution: {integrity: sha512-0ZAEIo6PNyMIJPlOGkIgyOJUjcd0pC8/QHlVAAe1c91/IcjZ1X+k0yidXHaboJdN7dq1XPUacmhRdtua0U5EZg==} + engines: {node: '>=18.20.0'} + '@wdio/mocha-framework@9.28.0': resolution: {integrity: sha512-UdkgigdyxJu0Rpx2ZRSNuB8u8B5nmLBeRjn5JHYTXBp8Ubi+wgFR1+EdLg8k2z+CtuYe3bNHBw7vPpBYuQMLZw==} engines: {node: '>=18.20.0'} @@ -2769,6 +2783,10 @@ packages: resolution: {integrity: sha512-q9gG6SXNTn/9cKF6EJ+aa5sGZM5HAVNsDZ3YU5B0IHg9ufdBuJgfT0LiAsnehLiceEuivuzPyz85vbDb0SFiVA==} engines: {node: '>=18.20.0'} + '@wdio/reporter@9.29.1': + resolution: {integrity: sha512-CKAcVy9BGwvufokMcl3H+yNcvD11Klku/1BPz8bKSRv7/KzueXR06s1cPbJH3tv9r9zxPErxgdVMpulN8I0qAg==} + engines: {node: '>=18.20.0'} + '@wdio/runner@9.28.0': resolution: {integrity: sha512-i6Zj9IKvHqNrRAuYoj56dhI6dXy5IkAxvsxuMih4R+EHLEihDoIwDRouJ9wOme1ZyHZ0Wpc6XDy8Igf1KnqWvQ==} engines: {node: '>=18.20.0'} @@ -2788,6 +2806,10 @@ packages: resolution: {integrity: sha512-75JPq39gifkPNqOSn5C4/A5ZSyXwF+dGr5jfsCubFN9Lk9dKBXfjdbWueSQNpJg0jmE6dVrbT7+9mnDNnO0HdQ==} engines: {node: '>=18.20.0'} + '@wdio/types@9.29.1': + resolution: {integrity: sha512-jp8jgMv6TS35G96YzHZxw3PVN0Dz6xQ6tnMAicndAJ8Jt9AIXb0ywIse4TjaFywakO3dLoEhlyA3ZtR26vmt+w==} + engines: {node: '>=18.20.0'} + '@wdio/utils@8.41.0': resolution: {integrity: sha512-0TcTjBiax1VxtJQ/iQA0ZyYOSHjjX2ARVmEI0AMo9+AuIq+xBfnY561+v8k9GqOMPKsiH/HrK3xwjx8xCVS03g==} engines: {node: ^16.13 || >=18} @@ -2875,6 +2897,14 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + allure-js-commons@3.10.2: + resolution: {integrity: sha512-5nAjUF1iNPSQMwKtEsadclSta8C3L705/k/pIzGb9M6P9krgfRaCyaEHt8pkLlCP9NFj5xk3grjtnAhiLeT+Kg==} + peerDependencies: + allure-playwright: 3.10.2 + peerDependenciesMeta: + allure-playwright: + optional: true + ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} @@ -3310,6 +3340,9 @@ packages: chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} + charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + check-error@1.0.2: resolution: {integrity: sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==} @@ -3546,6 +3579,9 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + css-functions-list@3.3.3: resolution: {integrity: sha512-8HFEBPKhOpJPEPu70wJJetjKta86Gw9+CCyCnB3sui2qQfOvRyqBy4IKLKKAwdMpWb2lHXWk9Wb4Z6AmaUT1Pg==} engines: {node: '>=12'} @@ -3576,6 +3612,9 @@ packages: resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==} engines: {node: '>=18'} + csv-stringify@6.8.1: + resolution: {integrity: sha512-tZ6X6TKQyQgCo5OptXcyAbfN1pwmoxEqELPQ7KFazNErx7kiVsDK8o+VYRXhfMl4N9vvOOLXuioquR2MeP847A==} + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} @@ -4559,6 +4598,9 @@ packages: resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} engines: {node: '>=18'} + html-entities@2.6.0: + resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==} + html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} @@ -4735,6 +4777,9 @@ packages: resolution: {integrity: sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==} engines: {node: '>= 0.4'} + is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + is-builtin-module@5.0.0: resolution: {integrity: sha512-f4RqJKBUe5rQkJ2eJEJBXSticB3hGbN9j0yxxMQFqIW89Jp9WYFtzfTcRlstDKVUTRzSOTLKRfO9vIztenwtxA==} engines: {node: '>=18.20'} @@ -5431,6 +5476,9 @@ packages: mathml-tag-names@4.0.0: resolution: {integrity: sha512-aa6AU2Pcx0VP/XWnh8IGL0SYSgQHDT6Ucror2j2mXeFAlN3ahaNs8EZtG1YiticMkSLj3Gt6VPFfZogt7G5iFQ==} + md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} @@ -7610,7 +7658,7 @@ snapshots: '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -7769,7 +7817,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/template': 7.29.7 '@babel/types': 7.29.7 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -8450,7 +8498,7 @@ snapshots: '@eslint/config-array@0.23.5': dependencies: '@eslint/object-schema': 3.0.5 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) minimatch: 10.2.5 transitivePeerDependencies: - supports-color @@ -9134,7 +9182,7 @@ snapshots: '@puppeteer/browsers@2.13.2': dependencies: - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.5.0 @@ -9552,7 +9600,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) eslint: 10.5.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: @@ -9562,7 +9610,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -9581,7 +9629,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) eslint: 10.5.0(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 @@ -9596,7 +9644,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) minimatch: 10.2.5 semver: 7.8.4 tinyglobby: 0.2.17 @@ -9735,6 +9783,14 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 + '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': + dependencies: + '@vitest/spy': 4.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) + '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 @@ -9789,6 +9845,18 @@ snapshots: path-browserify: 1.0.1 vscode-uri: 3.1.0 + '@wdio/allure-reporter@9.29.1': + dependencies: + '@types/node': 25.9.3 + '@wdio/reporter': 9.29.1 + '@wdio/types': 9.29.1 + allure-js-commons: 3.10.2 + csv-stringify: 6.8.1 + html-entities: 2.6.0 + strip-ansi: 7.2.0 + transitivePeerDependencies: + - allure-playwright + '@wdio/cli@9.28.0(@types/node@25.9.3)(expect-webdriverio@5.6.8)(puppeteer-core@21.11.0)': dependencies: '@vitest/snapshot': 2.1.9 @@ -9878,7 +9946,7 @@ snapshots: '@wdio/globals@9.28.0(expect-webdriverio@5.6.8)(webdriverio@9.28.0(puppeteer-core@21.11.0))': dependencies: - expect-webdriverio: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.18.0)(webdriverio@9.28.0(puppeteer-core@21.11.0)) + expect-webdriverio: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.29.1)(webdriverio@9.28.0(puppeteer-core@21.11.0)) webdriverio: 9.28.0(puppeteer-core@21.11.0) '@wdio/local-runner@9.28.0(@wdio/globals@9.28.0)(webdriverio@9.28.0(puppeteer-core@21.11.0))': @@ -9918,6 +9986,14 @@ snapshots: safe-regex2: 5.1.1 strip-ansi: 7.2.0 + '@wdio/logger@9.29.1': + dependencies: + chalk: 5.6.2 + loglevel: 1.9.2 + loglevel-plugin-prefix: 0.8.4 + safe-regex2: 5.1.1 + strip-ansi: 7.2.0 + '@wdio/mocha-framework@9.28.0': dependencies: '@types/mocha': 10.0.10 @@ -9948,6 +10024,14 @@ snapshots: diff: 8.0.4 object-inspect: 1.13.4 + '@wdio/reporter@9.29.1': + dependencies: + '@types/node': 25.9.3 + '@wdio/logger': 9.29.1 + '@wdio/types': 9.29.1 + diff: 8.0.4 + object-inspect: 1.13.4 + '@wdio/runner@9.28.0(expect-webdriverio@5.6.8)(webdriverio@9.28.0(puppeteer-core@21.11.0))': dependencies: '@types/node': 25.9.3 @@ -9958,7 +10042,7 @@ snapshots: '@wdio/types': 9.28.0 '@wdio/utils': 9.28.0 deepmerge-ts: 7.1.5 - expect-webdriverio: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.18.0)(webdriverio@9.28.0(puppeteer-core@21.11.0)) + expect-webdriverio: 5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.29.1)(webdriverio@9.28.0(puppeteer-core@21.11.0)) webdriver: 9.28.0 webdriverio: 9.28.0(puppeteer-core@21.11.0) transitivePeerDependencies: @@ -9985,6 +10069,10 @@ snapshots: dependencies: '@types/node': 25.9.3 + '@wdio/types@9.29.1': + dependencies: + '@types/node': 25.9.3 + '@wdio/utils@8.41.0': dependencies: '@puppeteer/browsers': 1.9.1 @@ -10058,7 +10146,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -10099,6 +10187,10 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + allure-js-commons@3.10.2: + dependencies: + md5: 2.3.0 + ansi-align@3.0.1: dependencies: string-width: 4.2.3 @@ -10593,6 +10685,8 @@ snapshots: chardet@2.1.1: {} + charenc@0.0.2: {} + check-error@1.0.2: {} cheerio-select@2.1.0: @@ -10860,6 +10954,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crypt@0.0.2: {} + css-functions-list@3.3.3: {} css-select@5.2.2: @@ -10888,6 +10984,8 @@ snapshots: '@asamuzakjp/css-color': 3.2.0 rrweb-cssom: 0.8.0 + csv-stringify@6.8.1: {} + data-uri-to-buffer@4.0.1: {} data-uri-to-buffer@6.0.2: {} @@ -11489,7 +11587,7 @@ snapshots: '@types/estree': 1.0.9 ajv: 6.15.0 cross-spawn: 7.0.6 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) escape-string-regexp: 4.0.0 eslint-scope: 9.1.2 eslint-visitor-keys: 5.0.1 @@ -11591,6 +11689,16 @@ snapshots: jest-matcher-utils: 30.4.1 webdriverio: 9.28.0(puppeteer-core@21.11.0) + expect-webdriverio@5.6.8(@wdio/globals@9.28.0)(@wdio/logger@9.29.1)(webdriverio@9.28.0(puppeteer-core@21.11.0)): + dependencies: + '@vitest/snapshot': 4.1.9 + '@wdio/globals': 9.28.0(expect-webdriverio@5.6.8)(webdriverio@9.28.0(puppeteer-core@21.11.0)) + '@wdio/logger': 9.29.1 + deep-eql: 5.0.2 + expect: 30.4.1 + jest-matcher-utils: 30.4.1 + webdriverio: 9.28.0(puppeteer-core@21.11.0) + expect@30.4.1: dependencies: '@jest/expect-utils': 30.4.1 @@ -11606,7 +11714,7 @@ snapshots: extract-zip@2.0.1: dependencies: - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -11937,7 +12045,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 6.0.2 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -11945,7 +12053,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 8.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -12117,6 +12225,8 @@ snapshots: dependencies: whatwg-encoding: 3.1.1 + html-entities@2.6.0: {} + html-escaper@2.0.2: {} html-tags@5.1.0: {} @@ -12141,35 +12251,35 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color http-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color https-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -12290,6 +12400,8 @@ snapshots: call-bound: 1.0.4 has-tostringtag: 1.0.2 + is-buffer@1.1.6: {} + is-builtin-module@5.0.0: dependencies: builtin-modules: 5.2.0 @@ -12466,7 +12578,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -12929,7 +13041,7 @@ snapshots: lighthouse-logger@2.0.2: dependencies: - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) marky: 1.3.0 transitivePeerDependencies: - supports-color @@ -13131,6 +13243,12 @@ snapshots: mathml-tag-names@4.0.0: {} + md5@2.3.0: + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + mdn-data@2.27.1: {} memorystream@0.3.1: {} @@ -13552,7 +13670,7 @@ snapshots: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.4 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) get-uri: 6.0.5 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 @@ -13564,7 +13682,7 @@ snapshots: pac-proxy-agent@9.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) get-uri: 8.0.0 http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 @@ -13854,7 +13972,7 @@ snapshots: proxy-agent@6.5.0: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 @@ -13867,7 +13985,7 @@ snapshots: proxy-agent@8.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 lru-cache: 7.18.3 @@ -14387,7 +14505,7 @@ snapshots: socks-proxy-agent@10.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14395,7 +14513,7 @@ snapshots: socks-proxy-agent@8.0.5: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14604,7 +14722,7 @@ snapshots: cosmiconfig: 9.0.2(typescript@6.0.3) css-functions-list: 3.3.3 css-tree: 3.2.1 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) fast-glob: 3.3.3 fastest-levenshtein: 1.0.16 file-entry-cache: 11.1.3 @@ -14867,7 +14985,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) esbuild: 0.27.7 fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 @@ -15000,7 +15118,7 @@ snapshots: '@rollup/pluginutils': 5.4.0(rollup@4.62.0) '@volar/typescript': 2.4.28 compare-versions: 6.1.1 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) kolorist: 1.8.0 local-pkg: 1.2.1 magic-string: 0.30.21 @@ -15143,6 +15261,21 @@ snapshots: optionalDependencies: rollup: 4.62.0 + vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): + dependencies: + lightningcss: 1.32.0 + picomatch: 4.0.4 + postcss: 8.5.15 + rolldown: 1.0.3 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 25.9.3 + esbuild: 0.27.7 + fsevents: 2.3.3 + jiti: 2.7.0 + tsx: 4.22.4 + yaml: 2.9.0 + vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 @@ -15158,6 +15291,36 @@ snapshots: tsx: 4.22.4 yaml: 2.9.0 + vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.9 + '@vitest/mocker': 4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.9 + '@vitest/runner': 4.1.9 + '@vitest/snapshot': 4.1.9 + '@vitest/spy': 4.1.9 + '@vitest/utils': 4.1.9 + es-module-lexer: 2.1.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.3 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 4.1.0 + tinybench: 2.9.0 + tinyexec: 1.2.4 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 25.9.3 + '@vitest/coverage-v8': 4.1.9(@vitest/browser@4.1.9)(vitest@4.1.9) + happy-dom: 20.10.4 + jsdom: 24.1.3 + transitivePeerDependencies: + - msw + vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 @@ -15200,7 +15363,7 @@ snapshots: dependencies: chalk: 4.1.2 commander: 9.5.0 - debug: 4.4.3(supports-color@5.5.0) + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color From cd75b72b7b14e0628034c4d46aa6d3f1e547754e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Wed, 15 Jul 2026 17:27:41 +0530 Subject: [PATCH 66/91] fix(core): replace polynomial-ReDoS edge-trims with a linear trimChar --- packages/core/src/artifact-naming.ts | 18 +++++++++- packages/core/src/spec-trace-helpers.ts | 20 ++++++----- packages/core/tests/artifact-naming.test.ts | 39 +++++++++++++++++++++ 3 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 packages/core/tests/artifact-naming.test.ts diff --git a/packages/core/src/artifact-naming.ts b/packages/core/src/artifact-naming.ts index dcf597b6..fdb968d7 100644 --- a/packages/core/src/artifact-naming.ts +++ b/packages/core/src/artifact-naming.ts @@ -3,8 +3,24 @@ * screenshot and video writers slug a test uid identically. */ +/** Strip leading and trailing occurrences of `char` via a linear scan. A regex + * like `/^c+|c+$/` is O(n²) on long runs of `c` (the anchored `+` is retried + * at each position) — flagged by CodeQL's js/polynomial-redos — so this avoids + * regex entirely. `char` must be a single character. */ +export function trimChar(value: string, char: string): string { + let start = 0 + let end = value.length + while (start < end && value[start] === char) { + start++ + } + while (end > start && value[end - 1] === char) { + end-- + } + return value.slice(start, end) +} + /** File-safe slug of a value: keep alphanumerics, `_` and `-`; collapse the * rest to a single `-`; trim leading/trailing dashes. */ export function fileSlug(value: string): string { - return value.replace(/[^a-zA-Z0-9_-]+/g, '-').replace(/^-+|-+$/g, '') + return trimChar(value.replace(/[^a-zA-Z0-9_-]+/g, '-'), '-') } diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index 6de04bf8..4b37fe80 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -16,6 +16,7 @@ import type { import type { TraceCapturer } from './trace-exporter.js' import { writeTraceZip } from './trace-exporter.js' import { deterministicUid } from './uid.js' +import { trimChar } from './artifact-naming.js' // ─── SpecRange ──────────────────────────────────────────────────────────────── @@ -68,13 +69,14 @@ export function findFlushableRange( * Falls back to `'unknown-spec'` when the result is empty. */ export function sanitizeSpecName(specFile: string): string { - return ( + const cleaned = trimChar( specFile .replace(/^.*[/\\]/, '') .replace(/\.[^.]+$/, '') - .replace(/[^a-zA-Z0-9_-]/g, '_') - .replace(/^_+|_+$/g, '') || 'unknown-spec' + .replace(/[^a-zA-Z0-9_-]/g, '_'), + '_' ) + return cleaned || 'unknown-spec' } // ─── Spec session ID ────────────────────────────────────────────────────────── @@ -121,12 +123,12 @@ const MAX_SLUG_LENGTH = 60 /** Lowercase, collapse runs of non-alphanumerics to `-`, trim edge dashes, * and cap length (trimming a dash the cut may leave behind). */ function slugify(text: string): string { - return text - .toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .replace(/^-+|-+$/g, '') - .slice(0, MAX_SLUG_LENGTH) - .replace(/-+$/g, '') + const collapsed = trimChar( + text.toLowerCase().replace(/[^a-z0-9]+/g, '-'), + '-' + ) + // Trim again after the length cap in case it sliced mid-dash-run. + return trimChar(collapsed.slice(0, MAX_SLUG_LENGTH), '-') } /** diff --git a/packages/core/tests/artifact-naming.test.ts b/packages/core/tests/artifact-naming.test.ts new file mode 100644 index 00000000..a8436f00 --- /dev/null +++ b/packages/core/tests/artifact-naming.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest' +import { trimChar, fileSlug } from '../src/artifact-naming.js' + +describe('trimChar', () => { + it('strips leading and trailing occurrences only', () => { + expect(trimChar('--a-b--', '-')).toBe('a-b') + expect(trimChar('__x__', '_')).toBe('x') + }) + + it('returns empty for an all-char string', () => { + expect(trimChar('-----', '-')).toBe('') + }) + + it('is a no-op when there is nothing to trim', () => { + expect(trimChar('abc', '-')).toBe('abc') + expect(trimChar('', '-')).toBe('') + }) + + it('handles long runs without catastrophic backtracking', () => { + const long = '-'.repeat(100000) + 'x' + '-'.repeat(100000) + // A polynomial /^-+|-+$/ would stall here; the linear scan returns fast. + expect(trimChar(long, '-')).toBe('x') + }) +}) + +describe('fileSlug', () => { + it('collapses disallowed runs to a single dash and trims edges', () => { + expect(fileSlug('a/b c!!d')).toBe('a-b-c-d') + expect(fileSlug(' spaced ')).toBe('spaced') + }) + + it('keeps alphanumerics, underscore and dash', () => { + expect(fileSlug('Test_Case-1')).toBe('Test_Case-1') + }) + + it('stays fast + correct on a long disallowed run', () => { + expect(fileSlug('a' + ' '.repeat(100000) + 'b')).toBe('a-b') + }) +}) From 92aff8955ce6637f462e553787e58346678707b1 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Wed, 15 Jul 2026 20:31:16 +0530 Subject: [PATCH 67/91] fix: scope screenshot/video options to the WDIO ServiceOptions --- CLAUDE.md | 2 +- packages/service/src/types.ts | 14 +++++++++++++- packages/shared/src/types.ts | 8 -------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5a2d88ce..f97151a3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,7 +240,7 @@ Documented divergences from the conventions above. They exist today as debt to b - **Nightwatch: `retain-on-failure` works; the other retry-aware policies degrade.** Its `--retries` re-runs the testcase *internally* without re-firing the plugin's per-test hooks, and the per-testcase results carry no attempt/retry field (retries live only in undocumented, version-varying Nightwatch internals — `suiteRetries.testRetriesCount` / `reporter.testResults.retryTest`), so the ledger sees only the final attempt for the `describe/it` and exports-object interfaces. Cucumber scenarios expose per-scenario hooks, so the feed captures their attempts. Not cleanly fixable without depending on those internals. - WDIO `specFileRetries` spawns a fresh worker per retry, so cross-process attempts aren't in the (process-scoped) ledger. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. -- The per-test `screenshot` and `video` options live on `BaseDevToolsOptions` (shared) so all three adapters accept them, but only the **WDIO service** reads them today. The capture/slice/encode logic is framework-agnostic (`core/screenshot-artifact.ts`, `core/video-slice.ts`), so Selenium/Nightwatch adoption is wiring-only (capture the base64 + feed `captureAndAttach*`); pending. Both are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the frame buffer is unbounded for the session (a decimation cap is a noted follow-up), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). +- The per-test `screenshot` and `video` options live on the **WDIO `ServiceOptions` only** — not `BaseDevToolsOptions` — because only the service implements them (an option belongs on an adapter until a second adapter consumes it, mirroring the core-helper rule; putting them on the shared base made them appear available in Selenium/Nightwatch and broke those adapters' `Required<>` option types). The policy *types* (`TraceScreenshotPolicy`/`TraceVideoPolicy`) and the capture/slice/encode logic (`core/screenshot-artifact.ts`, `core/video-slice.ts`) are framework-agnostic, so Selenium/Nightwatch adoption is wiring-only (capture the base64 + feed `captureAndAttach*`) plus adding the option to their own options type; pending. Both are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the frame buffer is unbounded for the session (a decimation cap is a noted follow-up), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. - Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. - Service renders expect-webdriverio matchers as single `expect.<matcher>` rows by **folding**, not stack/depth suppression (the old `#assertionDepth`/`#matcherStarted`/self-heal machinery is gone). The matcher's value-read (`toHaveText`→`getText`, `toExist`→`isExisting`, …) is captured as a normal command; `afterAssertion` then coalesces the synthesized `expect.*` row into that read in place — inheriting its callSource, screenshot, and timeline position — and the fold replaces **by timestamp, never a public `id`**: `id` is the per-worker `commandCounter`, which resets per spec, so stamping one lets the app's id-first `replaceCommand` swap a same-id row from another spec (duplicate rows + a fold from another spec vanishing, in multi-spec live mode). `beforeAssertion` arms the pending matcher (depth-counted so aliases like `toBeChecked`→`toBeSelected` fold once); a matcher that **hard-throws** — element never resolves, so expect-webdriverio's `waitUntil` rethrows and `afterAssertion` never fires — is synthesized at `afterTest`/`afterStep` from the throwing read, so a failing assertion renders as `expect.<matcher>` whether or not the element existed. Two limits: its error is then the read's (`Can't call getText on … element wasn't found`), not an assertion-phrased message; and `MATCHER_READ_COMMANDS` is a hand-maintained allowlist, so a matcher whose read isn't listed leaves its raw read visible alongside the `expect.*` row. Plain-value jest matchers (`expect(x).toBe(y)`) don't fire the ewdio hooks, so they aren't captured as rows. diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index d21dcff3..84ef345f 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -23,7 +23,11 @@ export { type Viewport } from '@wdio/devtools-shared' -import type { BaseDevToolsOptions } from '@wdio/devtools-shared' +import type { + BaseDevToolsOptions, + TraceScreenshotPolicy, + TraceVideoPolicy +} from '@wdio/devtools-shared' // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing service-internal imports. @@ -41,6 +45,14 @@ export interface ExtendedCapabilities extends WebdriverIO.Capabilities { } export interface ServiceOptions extends BaseDevToolsOptions { + /** Per-test screenshot capture, attached to the trace artifacts and inline to + * Allure. `off` (default) | `on` | `only-on-failure`. Trace mode + + * `traceGranularity: 'test'` only. WDIO-service-specific for now. */ + screenshot?: TraceScreenshotPolicy + /** Per-test video (screencast) capture, retained per the given policy and + * attached inline to Allure. `off` (default) or a retention policy. Trace + * mode + `traceGranularity: 'test'` only. WDIO-service-specific for now. */ + video?: TraceVideoPolicy /** * capabilities used to launch the devtools application * @default diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index b360936d..583ca3e7 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -317,14 +317,6 @@ export interface BaseDevToolsOptions { /** Trace retention policy — gates which traces are kept (e.g. * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ tracePolicy?: TraceRetentionPolicy - /** Per-test screenshot capture, attached to the trace artifacts and to Allure - * inline. `off` (default) | `on` | `only-on-failure`. Only applies in trace - * mode. */ - screenshot?: TraceScreenshotPolicy - /** Per-test video (screencast) capture, retained per the given policy and - * attached to Allure inline. `off` (default) or a retention policy. Only - * applies in trace mode at `traceGranularity: 'test'`. */ - video?: TraceVideoPolicy } /** Minimal Cucumber pickle-step shape — only the fields the adapters read. From c8380782b6877369e8bf33d83236eb2676ce6109 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Wed, 15 Jul 2026 20:31:38 +0530 Subject: [PATCH 68/91] fix(app): de-duplicate live-mode Errors tab (command + reworded test error) --- .../components/workbench/errors/collect.ts | 49 +++++++++++- packages/app/tests/errors-collect.test.ts | 74 +++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/packages/app/src/components/workbench/errors/collect.ts b/packages/app/src/components/workbench/errors/collect.ts index f28d8065..27d7d0ec 100644 --- a/packages/app/src/components/workbench/errors/collect.ts +++ b/packages/app/src/components/workbench/errors/collect.ts @@ -212,24 +212,65 @@ function commandErrors(commands: CommandLog[] | undefined): CollectedError[] { .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)) } +/** Collapse whitespace and drop a leading `…Error:` name prefix so a failed + * test that only re-reports a command failure compares equal despite framework + * wrapping: `@wdio/cucumber-framework` rebuilds a failed step's error from the + * first line of the error's *stack* (`new Error(stack.split('\n')[0])`), which + * prefixes `Error: `, while the command carries the same headline without it. */ +function normalizeMessage(message: string): string { + return message + .replace(/^[A-Za-z]*Error:\s*/, '') + .replace(/\s+/g, ' ') + .trim() +} + +/** True when a command's assertion diff is echoed inside a failed test's raw + * text. Cucumber keeps the full matcher output (`Expected:`/`Received:`) in the + * step error's stack even when its headline was truncated to the first line, so + * matching on both distinctive values catches the duplicate without depending + * on the headline wording — which diverges from the command's. */ +function isAssertionEcho( + command: CollectedError, + test: { message: string; stack?: string } +): boolean { + if (!command.expected || !command.actual) { + return false + } + const haystack = `${test.message}\n${test.stack ?? ''}` + return ( + haystack.includes(command.expected) && haystack.includes(command.actual) + ) +} + /** * Build the Errors-tab list from the live/player contexts. * * Command failures come first (time-ordered) because they carry the clickable - * action; a failed test that only echoes a command's message is dropped so the + * action; a failed test that only echoes a command's failure is dropped so the * same failure isn't listed twice (e.g. a Cucumber `Then` fails as both the - * assertion command and the scenario). + * assertion command and the step). The echo is detected two ways because the + * frameworks reword the test-level message: a normalized-message match (robust + * to the `Error:` prefix Cucumber adds), and — for assertions — the command's + * expected+actual both appearing in the test error's raw text. */ export function collectErrors( commands: CommandLog[] | undefined, suites: Record<string, SuiteStatsFragment>[] | undefined ): CollectedError[] { const fromCommands = commandErrors(commands) - const seenMessages = new Set(fromCommands.map((e) => e.message)) + const seenMessages = new Set( + fromCommands.map((e) => normalizeMessage(e.message)) + ) const fromTests = collectFailedTests(suites).flatMap((test) => { const read = readError(test.error ?? test.errors?.[0]) - if (!read || seenMessages.has(read.message)) { + if (!read) { + return [] + } + if (seenMessages.has(normalizeMessage(read.message))) { + return [] + } + if (fromCommands.some((command) => isAssertionEcho(command, read))) { return [] } return [ diff --git a/packages/app/tests/errors-collect.test.ts b/packages/app/tests/errors-collect.test.ts index c92e3680..0b482fca 100644 --- a/packages/app/tests/errors-collect.test.ts +++ b/packages/app/tests/errors-collect.test.ts @@ -193,6 +193,80 @@ describe('collectErrors', () => { expect(errors[0].command?.command).toBe('expect') }) + it('dedupes a Cucumber assertion listed as both a command and a reworded test error', () => { + // The matcher failure that expect-webdriverio reports: headline + diff. + const matcherMessage = + 'Expect $(`#flash`) to have text\n\n' + + 'Expected: StringContaining "You logged into a secure area!"\n' + + 'Received: "Your username is invalid!"' + const errors = collectErrors( + [ + command({ + command: 'expect.toHaveText', + title: 'expect.toHaveText', + callSource: 'steps.ts:34:3', + error: { name: 'Error', message: matcherMessage }, + timestamp: 50 + }) + ], + suiteMap({ + uid: 'scenario', + state: 'failed', + tests: [ + failedTest({ + uid: 'step', + title: 'Then I should see a flash message', + callSource: 'steps.ts:31', + // Cucumber rebuilds the failed step's error from the first line of + // the error's stack (so the headline gains an `Error:` prefix and + // loses the diff) but keeps the full matcher output in `.stack`. + error: { + name: 'Error', + message: 'Error: Expect to have text', + stack: `Error: ${matcherMessage}\n at World.<anonymous> (/steps.ts:34:3)` + } as unknown as Error + }) + ] + }) + ) + expect(errors).toHaveLength(1) + expect(errors[0].command?.command).toBe('expect.toHaveText') + expect(errors[0].actual).toBe('"Your username is invalid!"') + expect(errors[0].expected).toBe( + 'StringContaining "You logged into a secure area!"' + ) + }) + + it('dedupes a Cucumber command failure whose test error only adds an Error: prefix', () => { + const errors = collectErrors( + [ + command({ + command: 'click', + error: { + name: 'Error', + message: "Can't call click on element, it wasn't found" + }, + timestamp: 5 + }) + ], + suiteMap({ + uid: 'scenario', + state: 'failed', + tests: [ + failedTest({ + uid: 'step', + error: { + name: 'Error', + message: "Error: Can't call click on element, it wasn't found" + } + }) + ] + }) + ) + expect(errors).toHaveLength(1) + expect(errors[0].command?.command).toBe('click') + }) + it('keeps a distinct test failure alongside command failures', () => { const errors = collectErrors( [ From f57ab54cb512ce393e57873ae0423f69786dfcdd Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 01:27:03 +0530 Subject: [PATCH 69/91] feat: dense screencast filmstrip in trace mode, and fix per-test slicing --- CLAUDE.md | 1 + packages/core/src/index.ts | 1 + packages/core/src/screencast-trace.ts | 110 ++++++++++++++ packages/core/src/spec-trace-helpers.ts | 65 ++++++++- packages/core/src/trace-exporter.ts | 25 +++- packages/core/src/trace-finalizer.ts | 38 +++-- packages/core/tests/screencast-trace.test.ts | 137 ++++++++++++++++++ .../core/tests/spec-trace-helpers.test.ts | 80 +++++++++- packages/core/tests/trace-exporter.test.ts | 53 +++++++ packages/core/tests/trace-finalizer.test.ts | 13 ++ packages/nightwatch-devtools/src/index.ts | 32 +++- .../nightwatch-devtools/src/trace-context.ts | 3 + packages/selenium-devtools/src/index.ts | 11 +- .../selenium-devtools/src/plugin-internals.ts | 3 + .../src/session-lifecycle.ts | 23 ++- packages/service/README.md | 1 + packages/service/src/index.ts | 41 +++++- packages/service/tests/index.test.ts | 14 ++ packages/shared/src/types.ts | 10 ++ 19 files changed, 622 insertions(+), 39 deletions(-) create mode 100644 packages/core/src/screencast-trace.ts create mode 100644 packages/core/tests/screencast-trace.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index f97151a3..36896d87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -241,6 +241,7 @@ Documented divergences from the conventions above. They exist today as debt to b - WDIO `specFileRetries` spawns a fresh worker per retry, so cross-process attempts aren't in the (process-scoped) ledger. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. - The per-test `screenshot` and `video` options live on the **WDIO `ServiceOptions` only** — not `BaseDevToolsOptions` — because only the service implements them (an option belongs on an adapter until a second adapter consumes it, mirroring the core-helper rule; putting them on the shared base made them appear available in Selenium/Nightwatch and broke those adapters' `Required<>` option types). The policy *types* (`TraceScreenshotPolicy`/`TraceVideoPolicy`) and the capture/slice/encode logic (`core/screenshot-artifact.ts`, `core/video-slice.ts`) are framework-agnostic, so Selenium/Nightwatch adoption is wiring-only (capture the base64 + feed `captureAndAttach*`) plus adding the option to their own options type; pending. Both are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the frame buffer is unbounded for the session (a decimation cap is a noted follow-up), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). +- The `filmstrip` option (dense screencast into the trace) is on **`BaseDevToolsOptions`** — the counterexample to the screenshot/video entry above — because all three adapters implement it (the "second consumer → base" rule realized). Core owns the work (`core/screencast-trace.ts` `thinScreencastFrames`/`buildDenseScreencast`; slice windowing in `spec-trace-helpers.ts`); adapters only default the option, un-gate the recorder in trace mode when it's set, and feed `recorder.frames` into the finalize context. Each adapter captures frames while the recorder is still alive (service `onReload` → `#filmstripFrames`; Selenium `onDriverEnd` drain before nulling; Nightwatch `#finalizeCurrentScreencast` snapshot before delegating), and each finalize context spreads `[...accumulated, ...(live recorder frames)]` so a **mid-run** per-spec/per-test slice flush (which fires before the recorder is drained) isn't blank. Two known limits: (1) the dense frames are written **alongside** the sparse per-action frames (union), not replacing them — the sparse frames carry the DOM `elements`/`snapshot` refs, so suppressing them (attaching those refs to the nearest dense frame) is a deferred follow-up; near each action the filmstrip therefore shows the action's own frame plus adjacent dense frames. (2) Thinning is applied at **export**, so the raw session frame buffer is unbounded in memory during the run (same decimation-cap follow-up as video). Per-test filmstrip slicing follows the same per-test-hook availability as `traceGranularity:'test'` (works for WDIO mocha/cucumber, Selenium mocha, Nightwatch exports-object/cucumber; Nightwatch BDD `describe/it` degrades to session scope per the entry below), and non-Chrome polling carries the same reporter-noise caveat. - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. - Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. - Service renders expect-webdriverio matchers as single `expect.<matcher>` rows by **folding**, not stack/depth suppression (the old `#assertionDepth`/`#matcherStarted`/self-heal machinery is gone). The matcher's value-read (`toHaveText`→`getText`, `toExist`→`isExisting`, …) is captured as a normal command; `afterAssertion` then coalesces the synthesized `expect.*` row into that read in place — inheriting its callSource, screenshot, and timeline position — and the fold replaces **by timestamp, never a public `id`**: `id` is the per-worker `commandCounter`, which resets per spec, so stamping one lets the app's id-first `replaceCommand` swap a same-id row from another spec (duplicate rows + a fold from another spec vanishing, in multi-spec live mode). `beforeAssertion` arms the pending matcher (depth-counted so aliases like `toBeChecked`→`toBeSelected` fold once); a matcher that **hard-throws** — element never resolves, so expect-webdriverio's `waitUntil` rethrows and `afterAssertion` never fires — is synthesized at `afterTest`/`afterStep` from the throwing read, so a failing assertion renders as `expect.<matcher>` whether or not the element existed. Two limits: its error is then the read's (`Can't call getText on … element wasn't found`), not an assertion-phrased message; and `MATCHER_READ_COMMANDS` is a hand-maintained allowlist, so a matcher whose read isn't listed leaves its raw read visible alongside the `expect.*` row. Plain-value jest matchers (`expect(x).toBe(y)`) don't fire the ewdio hooks, so they aren't captured as rows. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e6f4a7ba..a73cb7bf 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,6 +36,7 @@ export * from './output-dir.js' export * from './performance-capture.js' export * from './retry-tracker.js' export * from './screencast.js' +export * from './screencast-trace.js' export * from './script-loader.js' export * from './session-capturer.js' export * from './spec-trace-helpers.js' diff --git a/packages/core/src/screencast-trace.ts b/packages/core/src/screencast-trace.ts new file mode 100644 index 00000000..c823ef52 --- /dev/null +++ b/packages/core/src/screencast-trace.ts @@ -0,0 +1,110 @@ +/** + * Dense screencast filmstrip for trace mode. Turns the recorder's continuous + * frame buffer into `screencast-frame` events plus content-addressed image + * resources, so the trace player scrubs smooth playback instead of one frame + * per action. + * + * Two independent bounds keep the cost in check: thinning caps the number of + * emitted events (min inter-frame gap + drop consecutive byte-identical frames + * + even downsample to a hard cap), and content-addressing dedupes the image + * bytes (identical frames — a static wait polled repeatedly — collapse to one + * resource). Framework-agnostic: adapters feed `recorder.frames`; the recorder + * itself (CDP push on Chrome, screenshot polling elsewhere) lives per-adapter. + */ + +import { createHash } from 'node:crypto' +import type { ScreencastFrame } from '@wdio/devtools-shared' +import type { TraceZipResource } from './trace-zip-writer.js' +import type { ScreencastFrameEvent } from './trace-snapshots.js' + +export interface DenseScreencastOptions { + /** Hard cap on emitted frame events (default 600). */ + maxFrames?: number + /** Minimum ms between kept frames (default 100 ≈ 10 fps). */ + minFrameIntervalMs?: number +} + +const DEFAULT_MAX_FRAMES = 600 +const DEFAULT_MIN_INTERVAL_MS = 100 + +/** + * Thin a continuous frame buffer down to a bounded event set. Keeps a frame + * only when it is at least `minFrameIntervalMs` after the last kept frame and + * its bytes differ from the last kept frame (a static run collapses to its + * first frame — the player holds it until the next change, which is the big win + * for polling mode where every poll re-shoots an unchanged page). A final even + * downsample enforces the hard cap while always keeping the first and last + * frames so the timeline still spans the full run. + */ +export function thinScreencastFrames( + frames: readonly ScreencastFrame[], + options: DenseScreencastOptions = {} +): ScreencastFrame[] { + const minInterval = options.minFrameIntervalMs ?? DEFAULT_MIN_INTERVAL_MS + const maxFrames = options.maxFrames ?? DEFAULT_MAX_FRAMES + + const kept: ScreencastFrame[] = [] + let lastKeptTs = -Infinity + let lastData: string | undefined + for (const frame of frames) { + if (frame.timestamp - lastKeptTs < minInterval) { + continue + } + if (frame.data === lastData) { + continue + } + kept.push(frame) + lastKeptTs = frame.timestamp + lastData = frame.data + } + + if (kept.length <= maxFrames) { + return kept + } + if (maxFrames <= 1) { + return kept.length ? [kept[0]!] : [] + } + const out: ScreencastFrame[] = [] + const step = (kept.length - 1) / (maxFrames - 1) + for (let i = 0; i < maxFrames; i++) { + out.push(kept[Math.round(i * step)]!) + } + return out +} + +/** + * Build the dense filmstrip: `screencast-frame` events (offsets rebased against + * `wallTime`, matching the sparse filmstrip) and their image resources named by + * content hash so byte-identical frames share a single resource. Adapters that + * don't record frames pass an empty array and get an empty result — byte-stable + * with today's output. + */ +export function buildDenseScreencast( + frames: readonly ScreencastFrame[], + pageId: string, + wallTime: number, + viewport: { width: number; height: number }, + options: DenseScreencastOptions = {} +): { events: ScreencastFrameEvent[]; resources: TraceZipResource[] } { + const events: ScreencastFrameEvent[] = [] + const resources: TraceZipResource[] = [] + const seen = new Set<string>() + for (const frame of thinScreencastFrames(frames, options)) { + const data = Buffer.from(frame.data, 'base64') + const sha1 = createHash('sha1').update(data).digest('hex') + const resourceName = `${sha1}.jpeg` + if (!seen.has(sha1)) { + seen.add(sha1) + resources.push({ resourceName, data }) + } + events.push({ + type: 'screencast-frame', + pageId, + sha1: resourceName, + width: viewport.width, + height: viewport.height, + timestamp: Math.max(0, frame.timestamp - wallTime) + }) + } + return { events, resources } +} diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index 4b37fe80..33d51bc0 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -9,6 +9,8 @@ import path from 'node:path' import type { ActionSnapshot, + CommandLog, + ScreencastFrame, TestMetadataMap, TraceFormat, TraceGranularity @@ -144,13 +146,16 @@ export function buildTestSliceFolder( key: string ): string { const specBase = sanitizeSpecName(path.basename(specFile)) - const titleSlug = - slugify(testTitle ?? '') || - deterministicUid(key).split('-').pop()!.slice(0, 8) + // A short hash of the slice key disambiguates two tests that share a title + // (Scenario Outline examples with a placeholder-free name, or duplicate `it` + // titles) — without it their folders collide and one trace overwrites the + // other. Mirrors buildTestSliceSessionId, which hashes the key for the id. + const keyHash = deterministicUid(key).split('-').pop()!.slice(0, 8) + const titleSlug = slugify(testTitle ?? '') || keyHash const browserSlug = slugify(browser ?? '') || 'browser' const retryMatch = key.match(/-retry(\d+)$/) const retrySuffix = retryMatch ? `-retry${retryMatch[1]}` : '' - return `${specBase}-${titleSlug}-${browserSlug}${retrySuffix}` + return `${specBase}-${titleSlug}-${browserSlug}-${keyHash}${retrySuffix}` } // ─── TraceCapturer slice ───────────────────────────────────────────────────── @@ -343,6 +348,8 @@ export interface WriteSpecTraceInput { /** Shape-compatible with `buildSpecCapturer`'s first parameter. */ capturer: Parameters<typeof buildSpecCapturer>[0] actionSnapshots: ActionSnapshot[] + /** Full session frame buffer; windowed to this slice's wall-clock span. */ + screencastFrames?: readonly ScreencastFrame[] sessionId: string outputDir: string format?: TraceFormat @@ -351,6 +358,24 @@ export interface WriteSpecTraceInput { capabilities?: unknown } +/** Timestamped items inside a slice's wall-clock window `[start, end)`. An + * undefined `start` (a slice with no commands to anchor on) yields nothing; an + * undefined `end` (the last slice) is open-ended, so a trailing assertion-wait + * stays with its test. Used for both action snapshots and screencast frames so + * they partition on the same reliable boundary as the sliced commands. */ +function sliceByWindow<T extends { timestamp: number }>( + items: readonly T[], + start: number | undefined, + end: number | undefined +): T[] { + if (start === undefined) { + return [] + } + return items.filter( + (i) => i.timestamp >= start && (end === undefined || i.timestamp < end) + ) +} + /** Slice the parent capturer/snapshots for one range and write the artifact * under `sliceSessionId` with the pre-filtered `testMetadata`. Shared by the * spec and test write paths so both slice identically. `overrides` lets the @@ -367,10 +392,35 @@ async function writeSliceTrace( input.nextRange ) - const sliceSnapshots = input.actionSnapshots.slice( - input.range.snapshotCount, - input.nextRange?.snapshotCount ?? input.actionSnapshots.length + // Anchor the slice on its own commands, which buildSpecCapturer index-slices + // reliably — parent-array index lookups desync under reloadSession and would + // sweep an earlier test's snapshots/frames in. Snapshots and frames are then + // windowed by that wall-clock span, and the slice is rebased to its own start + // (start === wallTime) so a per-test trace begins at 0, not the session + // offset. `windowEnd` is the next slice's start; open for the last. Anchor on + // the command's `startTime` (invocation), not `timestamp` (completion), so the + // frames captured *during* the first action — e.g. the page load of an opening + // `url()` — fall inside this slice rather than being dropped or bleeding into + // the previous one. + const commandStart = (c: CommandLog | undefined): number | undefined => + c ? (c.startTime ?? c.timestamp) : undefined + const windowStart = commandStart(sliceCapturer.commandsLog[0]) + const windowEnd = input.nextRange + ? commandStart(input.capturer.commandsLog[input.nextRange.commandStartIdx]) + : undefined + + const sliceSnapshots = sliceByWindow( + input.actionSnapshots, + windowStart, + windowEnd ) + const sliceFrames = input.screencastFrames + ? sliceByWindow(input.screencastFrames, windowStart, windowEnd) + : undefined + + if (windowStart !== undefined) { + sliceCapturer.startWallTime = windowStart + } return writeTraceZip(sliceCapturer, { outputDir: overrides.outputDir ?? input.outputDir, @@ -378,6 +428,7 @@ async function writeSliceTrace( fileStem: overrides.fileStem, capabilities: input.capabilities, actionSnapshots: sliceSnapshots.length > 0 ? sliceSnapshots : undefined, + screencastFrames: sliceFrames?.length ? sliceFrames : undefined, format: input.format, testMetadata }) diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index 40079b9a..6a43755e 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -9,6 +9,7 @@ import type { ConsoleLog, Metadata, NetworkRequest, + ScreencastFrame, TestMetadataMap, TraceFormat, TraceLog, @@ -30,6 +31,7 @@ import { buildSnapshotResources, type ScreencastFrameEvent } from './trace-snapshots.js' +import { buildDenseScreencast } from './screencast-trace.js' import { buildImageFrameSnapshots, FrameSnapshotIndex, @@ -320,7 +322,8 @@ function buildEventStream( ctxOptions: ContextOptionsEvent, pageId: string, wallTime: number, - testMetadata?: TestMetadataMap + testMetadata?: TestMetadataMap, + denseFrameEvents: ScreencastFrameEvent[] = [] ): TraceEvent[] { const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } const snapshots = trace.actionSnapshots ?? [] @@ -328,6 +331,7 @@ function buildEventStream( const events: TraceEvent[] = [ ctxOptions, ...buildFilmstripEvents(snapshots, pageId, wallTime, viewport), + ...denseFrameEvents, ...buildActionEvents( trace.commands, pageId, @@ -363,12 +367,19 @@ function buildTraceBundle( const contextId = `context@${idPrefix}` const pageId = `page@${idPrefix}` const ctxOptions = buildContextOptions(trace, contextId, wallTime) + const dense = buildDenseScreencast( + trace.screencastFrames ?? [], + pageId, + wallTime, + trace.metadata.viewport ?? { width: 1280, height: 720 } + ) const events = buildEventStream( trace, ctxOptions, pageId, wallTime, - opts.testMetadata + opts.testMetadata, + dense.events ) const networkBodies = buildNetworkBodyResources(trace.networkRequests) return { @@ -386,6 +397,7 @@ function buildTraceBundle( ), resources: [ ...buildSnapshotResources(trace.actionSnapshots ?? [], pageId), + ...dense.resources, ...buildSourceResources(trace.sources), ...networkBodies.resources ] @@ -461,6 +473,10 @@ export interface WriteTraceZipOptions { * viewer still renders thumbnails for adapters without an action hook. */ actionSnapshots?: ActionSnapshot[] + /** Dense screencast frames for the filmstrip. Thinned + content-addressed at + * export time; adapters pass the slice's windowed frames (or all, session + * scope). Omitted → no dense filmstrip (byte-stable with today's output). */ + screencastFrames?: readonly ScreencastFrame[] /** Output layout — `zip` (default) writes a single archive, `directory` * unpacks the same files into `trace-<id>/`. */ format?: TraceFormat @@ -496,7 +512,10 @@ export async function writeTraceZip( }, commands: capturer.commandsLog, sources: Object.fromEntries(capturer.sources), - ...(actionSnapshots.length ? { actionSnapshots } : {}) + ...(actionSnapshots.length ? { actionSnapshots } : {}), + ...(opts.screencastFrames?.length + ? { screencastFrames: [...opts.screencastFrames] } + : {}) } await fs.mkdir(opts.outputDir, { recursive: true }) const exportOpts = { diff --git a/packages/core/src/trace-finalizer.ts b/packages/core/src/trace-finalizer.ts index ff2b3b99..1ed8c937 100644 --- a/packages/core/src/trace-finalizer.ts +++ b/packages/core/src/trace-finalizer.ts @@ -9,6 +9,7 @@ import type { ActionSnapshot, DevToolsMode, + ScreencastFrame, TestMetadataMap, TraceFormat, TraceGranularity, @@ -62,6 +63,9 @@ export interface TraceExportContext { format?: TraceFormat capturer: TraceCapturer actionSnapshots?: ActionSnapshot[] + /** Continuous screencast frame buffer for the dense filmstrip (filmstrip + * option on). Session write uses all; slice writes window per test. */ + screencastFrames?: readonly ScreencastFrame[] sessionId: string capabilities?: unknown testMetadata: TestMetadataMap @@ -194,6 +198,22 @@ function attemptFromKey(key: string): number { return match ? Number(match[1]) : 0 } +/** Scope the per-attempt ledger to a slice: a test slice sees only its own + * attempt (so a passing retry isn't retained on first-failure); a spec slice + * sees every attempt of its tests. */ +function sliceOutcomeView( + ctx: TraceExportContext, + range: SpecRange, + isTestSlice: boolean +): TestOutcome[] | undefined { + if (!ctx.outcomes) { + return undefined + } + return isTestSlice + ? ctx.outcomes.forTest(range.testUid!, attemptFromKey(range.key)) + : ctx.outcomes.forSpec(range.specFile) +} + /** * Policy-aware single-range flush: dedupes via `ctx.flushed` on the slice * `key`, applies the retention decision, and delegates the byte-level @@ -206,6 +226,11 @@ export async function flushRangeTrace( range: SpecRange, nextRange?: SpecRange ): Promise<TraceArtifact | undefined> { + // Eager mid-run flushes reach here directly, bypassing finalizeTraceExport's + // gate, so enforce trace-mode here too — a live run must not write slices. + if (ctx.mode !== 'trace') { + return undefined + } if (ctx.flushed.has(range.key)) { return undefined } @@ -215,21 +240,14 @@ export async function flushRangeTrace( const sliceMetadata = isTestSlice ? filterTestMetadataByUid(ctx.testMetadata, range.testUid!) : filterTestMetadataBySpec(ctx.testMetadata, range.specFile) - // Scope the per-attempt ledger to this slice: a test slice sees only its own - // attempt (so a passing retry's slice isn't retained on first-failure), a spec - // slice sees every attempt of its tests. - const sliceOutcomes = ctx.outcomes - ? isTestSlice - ? ctx.outcomes.forTest(range.testUid!, attemptFromKey(range.key)) - : ctx.outcomes.forSpec(range.specFile) - : undefined + const outcomes = sliceOutcomeView(ctx, range, isTestSlice) const artifact: TraceArtifact = { kind: 'trace', path: '', scope: isTestSlice ? 'test' : 'spec', key: range.key, testUids: Array.from(sliceMetadata.keys()), - retained: shouldRetain(ctx, sliceMetadata, sliceOutcomes) + retained: shouldRetain(ctx, sliceMetadata, outcomes) } if (!artifact.retained) { ctx.onArtifact?.(artifact) @@ -242,6 +260,7 @@ export async function flushRangeTrace( nextRange, capturer: ctx.capturer, actionSnapshots: ctx.actionSnapshots ?? [], + screencastFrames: ctx.screencastFrames, sessionId: ctx.sessionId, outputDir: ctx.resolveOutputDir(range), format: ctx.format, @@ -308,6 +327,7 @@ async function writeSessionTrace( sessionId: ctx.sessionId, capabilities: ctx.capabilities, actionSnapshots: snapshots.length ? snapshots : undefined, + screencastFrames: ctx.screencastFrames, format: ctx.format, testMetadata: ctx.testMetadata }) diff --git a/packages/core/tests/screencast-trace.test.ts b/packages/core/tests/screencast-trace.test.ts new file mode 100644 index 00000000..79ef0adb --- /dev/null +++ b/packages/core/tests/screencast-trace.test.ts @@ -0,0 +1,137 @@ +import { describe, it, expect } from 'vitest' +import { createHash } from 'node:crypto' +import type { ScreencastFrame } from '@wdio/devtools-shared' +import { + thinScreencastFrames, + buildDenseScreencast +} from '../src/screencast-trace.js' + +/** base64 of a distinct payload per label, so identical labels are byte-equal. */ +function frame(label: string, timestamp: number): ScreencastFrame { + return { data: Buffer.from(label).toString('base64'), timestamp } +} + +function sha1Name(label: string): string { + const hex = createHash('sha1').update(Buffer.from(label)).digest('hex') + return `${hex}.jpeg` +} + +describe('thinScreencastFrames', () => { + it('keeps frames at least the min interval apart', () => { + const frames = [ + frame('a', 0), + frame('b', 40), + frame('c', 90), + frame('d', 150), + frame('e', 210) + ] + const kept = thinScreencastFrames(frames, { minFrameIntervalMs: 100 }) + // 0 kept; 40,90 dropped (<100 since last kept 0); 150 kept; 210 dropped. + expect(kept.map((f) => f.timestamp)).toEqual([0, 150]) + }) + + it('drops consecutive byte-identical frames (static run collapses)', () => { + const frames = [ + frame('same', 0), + frame('same', 200), + frame('same', 400), + frame('changed', 600) + ] + const kept = thinScreencastFrames(frames, { minFrameIntervalMs: 100 }) + expect(kept.map((f) => f.timestamp)).toEqual([0, 600]) + }) + + it('keeps a repeated frame that is not consecutive (A B A)', () => { + const frames = [frame('a', 0), frame('b', 200), frame('a', 400)] + const kept = thinScreencastFrames(frames, { minFrameIntervalMs: 100 }) + expect(kept.map((f) => f.timestamp)).toEqual([0, 200, 400]) + }) + + it('downsamples to the cap while keeping first and last', () => { + const frames = Array.from({ length: 50 }, (_, i) => frame(`f${i}`, i * 200)) + const kept = thinScreencastFrames(frames, { + maxFrames: 10, + minFrameIntervalMs: 100 + }) + expect(kept).toHaveLength(10) + expect(kept[0]!.timestamp).toBe(0) + expect(kept[kept.length - 1]!.timestamp).toBe(49 * 200) + }) + + it('returns empty for empty input', () => { + expect(thinScreencastFrames([])).toEqual([]) + }) + + it('handles maxFrames<=1 without producing undefined holes', () => { + const frames = [frame('a', 0), frame('b', 200), frame('c', 400)] + const kept = thinScreencastFrames(frames, { + maxFrames: 1, + minFrameIntervalMs: 100 + }) + expect(kept).toEqual([frames[0]]) + // and buildDenseScreencast must not throw on that result + expect(() => + buildDenseScreencast( + frames, + 'page@x', + 0, + { width: 1, height: 1 }, + { + maxFrames: 1, + minFrameIntervalMs: 100 + } + ) + ).not.toThrow() + }) +}) + +describe('buildDenseScreencast', () => { + const viewport = { width: 800, height: 600 } + + it('rebases timestamps against wallTime and never goes negative', () => { + const frames = [frame('a', 1000), frame('b', 1300)] + const { events } = buildDenseScreencast(frames, 'page@x', 1000, viewport, { + minFrameIntervalMs: 100 + }) + expect(events.map((e) => e.timestamp)).toEqual([0, 300]) + expect(events.every((e) => e.type === 'screencast-frame')).toBe(true) + expect(events[0]!.pageId).toBe('page@x') + expect(events[0]!.width).toBe(800) + }) + + it('content-addresses frames and dedupes identical bytes to one resource', () => { + // Two distinct frames plus one that repeats the first's bytes non-adjacently. + const frames = [frame('img-a', 0), frame('img-b', 200), frame('img-a', 400)] + const { events, resources } = buildDenseScreencast( + frames, + 'page@x', + 0, + viewport, + { minFrameIntervalMs: 100 } + ) + // three events, but only two unique resources (img-a shared). + expect(events).toHaveLength(3) + expect(resources).toHaveLength(2) + expect(events[0]!.sha1).toBe(sha1Name('img-a')) + expect(events[2]!.sha1).toBe(sha1Name('img-a')) + expect(events[0]!.sha1).toBe(events[2]!.sha1) + expect(new Set(resources.map((r) => r.resourceName)).size).toBe(2) + }) + + it('returns empty events and resources for no frames (byte-stable)', () => { + expect(buildDenseScreencast([], 'page@x', 0, viewport)).toEqual({ + events: [], + resources: [] + }) + }) + + it('resource bytes decode back to the source frame data', () => { + const { resources } = buildDenseScreencast( + [frame('hello', 0)], + 'page@x', + 0, + viewport + ) + expect(resources[0]!.data.toString()).toBe('hello') + }) +}) diff --git a/packages/core/tests/spec-trace-helpers.test.ts b/packages/core/tests/spec-trace-helpers.test.ts index 27f4f694..a3b8c8d2 100644 --- a/packages/core/tests/spec-trace-helpers.test.ts +++ b/packages/core/tests/spec-trace-helpers.test.ts @@ -206,7 +206,7 @@ describe('recordSliceBoundary (test granularity)', () => { }) describe('buildTestSliceFolder', () => { - it('combines sanitized spec, title slug, and browser slug', () => { + it('combines sanitized spec, title slug, browser slug, and key hash', () => { expect( buildTestSliceFolder( '/tests/login.e2e.js', @@ -214,24 +214,32 @@ describe('buildTestSliceFolder', () => { 'chrome', 'u1' ) - ).toBe('login_e2e-shows-an-error-message-for-an-invalid-username-chrome') + ).toMatch( + /^login_e2e-shows-an-error-message-for-an-invalid-username-chrome-[a-z0-9]{1,8}$/ + ) + }) + + it('gives two same-title tests distinct folders (no collision)', () => { + const a = buildTestSliceFolder('/login.feature', 'As a user', 'chrome', 'A') + const b = buildTestSliceFolder('/login.feature', 'As a user', 'chrome', 'B') + expect(a).not.toBe(b) }) - it('appends a -retry<N> suffix when the key is a retry key', () => { + it('appends a -retry<N> suffix after the key hash on a retry key', () => { expect( buildTestSliceFolder('/a.js', 'My Test', 'chrome', 'u1-retry2') - ).toBe('a-my-test-chrome-retry2') + ).toMatch(/^a-my-test-chrome-[a-z0-9]{1,8}-retry2$/) }) it('defaults the browser slug to "browser" when the browser is absent', () => { - expect(buildTestSliceFolder('/a.js', 'My Test', undefined, 'u1')).toBe( - 'a-my-test-browser' + expect(buildTestSliceFolder('/a.js', 'My Test', undefined, 'u1')).toMatch( + /^a-my-test-browser-[a-z0-9]{1,8}$/ ) }) it('falls back to a stable short hash of the key when the title is empty', () => { const folder = buildTestSliceFolder('/a.js', '', 'chrome', 'u1') - expect(folder).toMatch(/^a-[a-z0-9]+-chrome$/) + expect(folder).toMatch(/^a-[a-z0-9]+-chrome-[a-z0-9]+$/) expect(buildTestSliceFolder('/a.js', undefined, 'chrome', 'u1')).toBe( folder ) @@ -296,11 +304,67 @@ describe('writeTestSliceTrace / writeSpecTrace output layout', () => { 'firefox', 'u1' ) - expect(folder).toBe('login-my-test-firefox') + expect(folder).toMatch(/^login-my-test-firefox-[a-z0-9]{1,8}$/) expect(written).toBe(path.join(outputDir, folder, 'trace.zip')) await expect(fs.access(written)).resolves.toBeUndefined() }) + it('windows a test slice to its own commands and rebases to its start', async () => { + // Two tests in one session; the second's slice must exclude the first's + // frames (the reloadSession-desync bug) and rebase to its own start, not + // the session start (the huge-empty-prefix bug). + const capturer: TraceCapturer = { + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { command: 'url', args: ['a'], timestamp: 1000 }, + { command: 'click', args: [], timestamp: 2000 }, + // B's opening url: invoked at 4800, completes at 5000 — its load frames + // land in [4800, 5000) and must belong to B, not A. + { command: 'url', args: ['b'], timestamp: 5000, startTime: 4800 }, + { command: 'click', args: [], timestamp: 6000 } + ], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + } + const frames = [ + { data: Buffer.from('a1').toString('base64'), timestamp: 1500 }, + { data: Buffer.from('a2').toString('base64'), timestamp: 2500 }, + { data: Buffer.from('bload').toString('base64'), timestamp: 4900 }, + { data: Buffer.from('b1').toString('base64'), timestamp: 5200 }, + { data: Buffer.from('b2').toString('base64'), timestamp: 6200 } + ] + const dir = await writeTestSliceTrace( + input({ + range: range({ + specFile: '/t.js', + key: 'B', + testUid: 'B', + commandStartIdx: 2 + }), + capturer, + screencastFrames: frames, + format: 'ndjson-directory', + testMetadata: new Map([['B', { title: 'Test B', specFile: '/t.js' }]]) + }) + ) + const events = (await fs.readFile(path.join(dir, 'trace.trace'), 'utf8')) + .trim() + .split('\n') + .map((l) => JSON.parse(l) as Record<string, unknown>) + const frameOffsets = events + .filter((e) => e.type === 'screencast-frame') + .map((e) => e.timestamp as number) + .sort((x, y) => x - y) + // Rebased against B's first-command invocation (startTime 4800), and B's + // in-flight load frame is kept: 4900→100, 5200→400, 6200→1400. Test A's + // frames (1500/2500) are excluded, not bled in. + expect(frameOffsets).toEqual([100, 400, 1400]) + }) + it('keeps the spec write flat as trace-<id>.zip (unchanged layout)', async () => { const written = await writeSpecTrace( input({ diff --git a/packages/core/tests/trace-exporter.test.ts b/packages/core/tests/trace-exporter.test.ts index 3dd16c49..537d3c4c 100644 --- a/packages/core/tests/trace-exporter.test.ts +++ b/packages/core/tests/trace-exporter.test.ts @@ -404,3 +404,56 @@ describe('exported trace stream — frame-snapshot events', () => { await fs.rm(outputDir, { recursive: true, force: true }) }) }) + +describe('exported trace stream — dense filmstrip', () => { + it('emits content-addressed screencast-frame events + resources', async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'trace-dense-')) + const capturer: TraceCapturer = { + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { + command: 'url', + args: ['https://example.test'], + timestamp: 1200, + startTime: 1150 + } + ], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + } + const dir = await writeTraceZip(capturer, { + outputDir, + sessionId: 'abc12345', + format: 'ndjson-directory', + screencastFrames: [ + { data: Buffer.from('paint-1').toString('base64'), timestamp: 1000 }, + { data: Buffer.from('paint-1').toString('base64'), timestamp: 1050 }, + { data: Buffer.from('paint-2').toString('base64'), timestamp: 1400 } + ] + }) + const raw = await fs.readFile(path.join(dir, 'trace.trace'), 'utf8') + const frames = raw + .trim() + .split('\n') + .map((l) => JSON.parse(l) as Record<string, unknown>) + .filter((l) => l.type === 'screencast-frame') + + // Two identical bytes 50ms apart collapse to one kept frame; the second + // distinct paint is the other. Both rebased against wallTime 1000. + expect(frames).toHaveLength(2) + expect(frames.map((f) => f.timestamp)).toEqual([0, 400]) + for (const f of frames) { + expect(String(f.sha1)).toMatch(/^[0-9a-f]{40}\.jpeg$/) + } + + const resourceNames = await fs.readdir(path.join(dir, 'resources')) + for (const f of frames) { + expect(resourceNames).toContain(f.sha1) + } + await fs.rm(outputDir, { recursive: true, force: true }) + }) +}) diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index 70f655fe..7510de03 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -611,6 +611,19 @@ describe('flushRangeTrace', () => { ).resolves.toBeUndefined() }) + it('is a no-op in live mode — a granularity set outside trace never writes', async () => { + const flushed = new Set<string>() + const seen: unknown[] = [] + const artifact = await flushRangeTrace( + ctx({ mode: 'live', flushed, onArtifact: (a) => seen.push(a) }), + range('/spec.js', 0) + ) + expect(artifact).toBeUndefined() + expect(seen).toEqual([]) + expect(flushed.size).toBe(0) + await expect(fs.readdir(outputDir)).resolves.toEqual([]) + }) + it('returns undefined for an already-flushed spec', async () => { const flushed = new Set<string>(['/spec.js']) const artifact = await flushRangeTrace( diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index fad32b4f..5f93d23b 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -23,7 +23,8 @@ import { REUSE_ENV, SCREENCAST_DEFAULTS, type CucumberPickle, - type CucumberPickleStep + type CucumberPickleStep, + type ScreencastFrame } from '@wdio/devtools-shared' import logger from '@wdio/logger' import { @@ -125,6 +126,9 @@ class NightwatchDevToolsPlugin { #screencastOptions: ScreencastOptions #screencastRecorder?: ScreencastRecorder #screencastSessionId?: string + + // Snapshotted before each recorder is nulled, so the export isn't blank. + #filmstripFrames: ScreencastFrame[] = [] #bidiEnabled = false #bidiAttachAttempted = false @@ -134,11 +138,17 @@ class NightwatchDevToolsPlugin { constructor(options: DevToolsOptions = {}) { const mode = options.mode ?? 'live' - const ignore = mode === 'trace' && options.screencast?.enabled === true + // Filmstrip drives the recorder in trace mode; bare screencast stays live-only. + const filmstrip = mode === 'trace' && options.filmstrip === true + const ignore = + mode === 'trace' && !filmstrip && options.screencast?.enabled === true if (ignore) { log.warn('trace mode: ignoring screencast option (live-mode feature)') } - const screencast = ignore ? {} : (options.screencast ?? {}) + let screencast = ignore ? {} : (options.screencast ?? {}) + if (filmstrip) { + screencast = { ...(options.screencast ?? {}), enabled: true } + } this.options = { port: options.port ?? 3000, hostname: options.hostname ?? 'localhost', @@ -148,7 +158,8 @@ class NightwatchDevToolsPlugin { mode, traceFormat: options.traceFormat ?? 'zip', traceGranularity: options.traceGranularity ?? 'session', - tracePolicy: options.tracePolicy ?? 'on' + tracePolicy: options.tracePolicy ?? 'on', + filmstrip: options.filmstrip ?? false } const policyWarning = tracePolicyModeWarning(options.tracePolicy, mode) if (policyWarning) { @@ -375,6 +386,9 @@ class NightwatchDevToolsPlugin { } async #finalizeCurrentScreencast(): Promise<void> { + if (this.options.filmstrip && this.#screencastRecorder) { + this.#filmstripFrames.push(...this.#screencastRecorder.frames) + } await finalizeCurrentScreencast(this.#getInternals()) } @@ -606,6 +620,16 @@ class NightwatchDevToolsPlugin { flushed: this.#flushedSpecs, artifacts: this.#artifacts, traceFlushes: this.#traceFlushes, + // Accumulated (finalized-session) frames plus the live recorder's — a + // mid-run per-spec/per-test flush fires before the recorder is drained + // into #filmstripFrames, so its frames live only on the recorder still; + // at the final write the recorder is already nulled, so no double-count. + screencastFrames: this.options.filmstrip + ? [ + ...this.#filmstripFrames, + ...(this.#screencastRecorder?.frames ?? []) + ] + : undefined, configPath: this.#configPath, testFilePath: this.browserProxy?.getCurrentTestFullPath?.() ?? undefined, diff --git a/packages/nightwatch-devtools/src/trace-context.ts b/packages/nightwatch-devtools/src/trace-context.ts index 4c4666be..cf0a9baf 100644 --- a/packages/nightwatch-devtools/src/trace-context.ts +++ b/packages/nightwatch-devtools/src/trace-context.ts @@ -17,6 +17,7 @@ import { import type { SessionCapturer } from './session.js' import type { DevToolsMode, + ScreencastFrame, SuiteStats, TraceFormat, TraceGranularity, @@ -35,6 +36,7 @@ export interface TraceContextInput { flushed: Set<string> artifacts: TraceArtifact[] traceFlushes: Promise<unknown>[] + screencastFrames?: readonly ScreencastFrame[] configPath: string | undefined testFilePath: string | undefined log: (level: 'info' | 'warn', msg: string) => void @@ -56,6 +58,7 @@ export function buildTraceContext( outcomes: input.outcomes, ranges: input.ranges, flushed: input.flushed, + screencastFrames: input.screencastFrames, resolveOutputDir: () => resolveAdapterOutputDir({ testFilePath: input.testFilePath, diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index 91af73f9..dcc19fd3 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -61,6 +61,7 @@ import { type CapturedCommand, type DevToolsMode, type DevToolsOptions, + type ScreencastFrame, type ScreencastOptions, type TraceFormat, type TraceGranularity, @@ -131,6 +132,10 @@ class SeleniumDevToolsPlugin { * end-of-run artifacts manifest. Populated via the context's onArtifact. */ #artifacts: TraceArtifact[] = [] + /** Filmstrip frames accumulated across drivers, appended at each driver end + * before the recorder is nulled so the finalize context is never blank. */ + #filmstripFrames: ScreencastFrame[] = [] + constructor(options: DevToolsOptions = {}) { this.#options = { port: options.port ?? 3000, @@ -143,7 +148,8 @@ class SeleniumDevToolsPlugin { mode: options.mode ?? 'live', traceFormat: options.traceFormat ?? 'zip', traceGranularity: options.traceGranularity ?? 'session', - tracePolicy: options.tracePolicy ?? 'on' + tracePolicy: options.tracePolicy ?? 'on', + filmstrip: options.filmstrip ?? false } const policyWarning = tracePolicyModeWarning( options.tracePolicy, @@ -446,6 +452,9 @@ class SeleniumDevToolsPlugin { get artifacts() { return self.#artifacts }, + get filmstripFrames() { + return self.#filmstripFrames + }, setFinalized: (v) => { self.#finalized = v }, diff --git a/packages/selenium-devtools/src/plugin-internals.ts b/packages/selenium-devtools/src/plugin-internals.ts index 36d1f98e..2893b0e9 100644 --- a/packages/selenium-devtools/src/plugin-internals.ts +++ b/packages/selenium-devtools/src/plugin-internals.ts @@ -15,6 +15,7 @@ import type { ScreencastRecorder } from './screencast.js' import type { ActionSnapshot, DevToolsMode, + ScreencastFrame, ScreencastOptions, SeleniumDriverLike, TraceFormat, @@ -71,6 +72,8 @@ export interface PluginInternals { readonly traceFlushes: Promise<unknown>[] // Every trace/video artifact seen this run, for the end-of-run manifest. readonly artifacts: TraceArtifact[] + // Dense filmstrip frames accumulated across drivers (filmstrip option only). + readonly filmstripFrames: ScreencastFrame[] // Plugin-side delegates setFinalized(v: boolean): void diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index fabe3895..d00b4030 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -36,6 +36,7 @@ import type { ActionSnapshot, DevToolsMode, Metadata, + ScreencastFrame, ScreencastOptions, SeleniumDriverLike, TraceFormat, @@ -57,6 +58,7 @@ export interface SessionLifecycleCtx { traceFormat?: TraceFormat traceGranularity?: TraceGranularity tracePolicy?: TraceRetentionPolicy + filmstrip?: boolean } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -88,6 +90,8 @@ export interface SessionLifecycleCtx { readonly traceFlushes: Promise<unknown>[] // Every trace/video artifact seen this run, for the end-of-run manifest. readonly artifacts: TraceArtifact[] + // Dense filmstrip frames accumulated across drivers (filmstrip option only). + readonly filmstripFrames: ScreencastFrame[] setFinalized(v: boolean): void ensureBackendStarted(): Promise<void> @@ -183,8 +187,12 @@ async function initPerDriverCapture( ctx.sessionCapturer.sendUpstream('metadata', metadata) } - // Parallel — serial attach misses frames on fast tests. - const screencastPromise = ctx.screencastOptions.enabled + // Parallel — serial attach misses frames on fast tests. Trace-mode filmstrip + // needs the same recorder even though live screencast is off in trace mode. + const wantScreencast = + ctx.screencastOptions.enabled || + (ctx.options.mode === 'trace' && !!ctx.options.filmstrip) + const screencastPromise = wantScreencast ? (async () => { try { ctx.screencast = new ScreencastRecorder(ctx.screencastOptions) @@ -228,6 +236,11 @@ export async function onDriverEnd(ctx: SessionLifecycleCtx): Promise<void> { onLog: (level, message) => log[level](message) }) } + // Drain this driver's frames into the run-wide buffer while the recorder is + // still alive — the finalize context is built after screencast is nulled. + if (ctx.options.filmstrip && ctx.screencast) { + ctx.filmstripFrames.push(...ctx.screencast.frames) + } ctx.driver = undefined ctx.screencast = undefined ctx.scriptInjected = false @@ -313,6 +326,12 @@ export function buildTraceExportContext( format: ctx.options.traceFormat, capturer, actionSnapshots: ctx.actionSnapshots, + // Accumulated (ended-driver) frames plus the live recorder's — a mid-run + // per-spec/per-test flush fires before onDriverEnd drains the recorder, so + // its frames aren't in filmstripFrames yet; the core windows per slice. + screencastFrames: ctx.options.filmstrip + ? [...ctx.filmstripFrames, ...(ctx.screencast?.frames ?? [])] + : undefined, sessionId, testMetadata: collectSuiteTestMetadata(root ? [root] : []), // TestStats.retries carries the per-test attempt (Mocha authoritative, diff --git a/packages/service/README.md b/packages/service/README.md index bfd4002f..c5a28e80 100644 --- a/packages/service/README.md +++ b/packages/service/README.md @@ -52,6 +52,7 @@ services: [['devtools', options]] | `tracePolicy` | `TraceRetentionPolicy` | `'on'` | Trace mode only. Which traces to keep — e.g. `'retain-on-failure'`, `'retain-on-first-failure'`. | | `screenshot` | `'off' \| 'on' \| 'only-on-failure'` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screenshot, attached inline to Allure (`image/png`). | | `video` | `'off' \| TraceRetentionPolicy` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screencast video, retained per the given policy, attached inline to Allure (`video/webm`). | +| `filmstrip` | `boolean` | `false` | Trace mode only. Records a dense, continuous screencast filmstrip *into* the trace so the player scrubs smooth playback — dense frames are added alongside the per-action frames (not one frame per action). Frames are thinned (≥100 ms apart, ~600 max) and content-addressed (identical frames — a static wait — collapse to one resource); windowed per slice at any `traceGranularity`. Runs the screencast recorder (CDP push on Chrome, polling elsewhere). | ## Allure integration diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index dc9cea69..012382d6 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -134,6 +134,12 @@ export default class DevToolsHookService implements Services.ServiceInstance { frames: ScreencastFrame[] } + /** Filmstrip frames accumulated across reloadSession() boundaries — the + * recorder's buffer resets per session, so this persists earlier sessions' + * frames (like #actionSnapshots) and is concatenated with the live recorder's + * frames at export, then windowed per slice in core. Filmstrip mode only. */ + #filmstripFrames: ScreencastFrame[] = [] + /** Map of testUid → metadata for trace group events and per-spec partitioning. */ #testMetadata: TestMetadataMap = new Map() @@ -202,14 +208,31 @@ export default class DevToolsHookService implements Services.ServiceInstance { } /** Record a screencast this session? Live mode: `screencast.enabled`. Trace - * mode: a non-`off` `video` policy (frames are sliced per test at flush). */ + * mode: a non-`off` `video` policy (frames sliced per test at flush) or + * `filmstrip` (dense frames written into the trace itself). */ #shouldRecordScreencast(): boolean { if (this.#options.mode === 'trace') { - return !!this.#options.video && this.#options.video !== 'off' + return ( + (!!this.#options.video && this.#options.video !== 'off') || + !!this.#options.filmstrip + ) } return !!this.#screencastOptions?.enabled } + /** Whole-run filmstrip frames for the export context: earlier sessions' + * accumulated frames plus the live recorder's, or undefined when filmstrip + * is off (so the trace stays byte-stable with today's output). */ + #filmstripFramesForExport(): ScreencastFrame[] | undefined { + if (!this.#options.filmstrip) { + return undefined + } + return [ + ...this.#filmstripFrames, + ...(this.#screencastRecorder?.frames ?? []) + ] + } + /** Eager per-test flush at test end (test granularity only), run after the * outcome is stamped so this attempt's metadata is written before a retry * overwrites it; the end-of-run finalizer then dedupes it via the key set. */ @@ -240,6 +263,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { format: this.#options.traceFormat, capturer: this.#sessionCapturer, actionSnapshots: this.#actionSnapshots, + screencastFrames: this.#filmstripFramesForExport(), sessionId: browser.sessionId, capabilities: browser.capabilities, testMetadata: this.#testMetadata, @@ -306,8 +330,9 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** * Start screencast recording when enabled — `screencast.enabled` in live - * mode, or a non-`off` `video` policy in trace mode (per-test slicing at - * flush time). Failures are non-fatal — logged, session continues. + * mode, or a non-`off` `video` policy (per-test slicing at flush) or + * `filmstrip` (dense frames into the trace) in trace mode. Failures are + * non-fatal — logged, session continues. */ if (this.#shouldRecordScreencast()) { this.#screencastRecorder = new ScreencastRecorder( @@ -767,10 +792,16 @@ export default class DevToolsHookService implements Services.ServiceInstance { // point the recorder below has replaced these frames. Snapshot them now, // keyed to the ending test, so afterScenario can still slice its video. if (this.#options.mode === 'trace' && this.#screencastRecorder) { + const frames = [...this.#screencastRecorder.frames] this.#pendingVideoFrames = { testUid: this.#currentTestUid, startWallTime: this.#currentTestStartWallTime, - frames: [...this.#screencastRecorder.frames] + frames + } + // Persist for the filmstrip too — the recorder below resets the buffer, + // so a session/spec trace spanning this reload keeps its earlier frames. + if (this.#options.filmstrip) { + this.#filmstripFrames.push(...frames) } } diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index ba881d4a..37fc2c86 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -277,6 +277,20 @@ describe('DevtoolsService - Screencast Integration', () => { // helper itself (covered in core/tests). Service just needs to invoke it. }) + it('trace mode: filmstrip starts the recorder; no filmstrip/video leaves it off', async () => { + // filmstrip on → recorder runs so its frames become the dense trace filmstrip + service = new DevToolsHookService({ mode: 'trace', filmstrip: true }) + await service.before({} as any, [], mockBrowser) + expect(mockScreencastRecorder.start).toHaveBeenCalledWith(mockBrowser) + + vi.clearAllMocks() + + // trace mode, neither filmstrip nor video → no recorder (byte-stable output) + service = new DevToolsHookService({ mode: 'trace' }) + await service.before({} as any, [], mockBrowser) + expect(mockScreencastRecorder.start).not.toHaveBeenCalled() + }) + it('onReload finalizes old session and starts fresh recorder', async () => { const { ScreencastRecorder } = await import('../src/screencast.js') service = new DevToolsHookService({ screencast: { enabled: true } }) diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 583ca3e7..793c7b8c 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -317,6 +317,12 @@ export interface BaseDevToolsOptions { /** Trace retention policy — gates which traces are kept (e.g. * `retain-on-failure`). Default `on` (keep all). Only applies in trace mode. */ tracePolicy?: TraceRetentionPolicy + /** Record a dense, continuous screencast filmstrip into the trace for + * scrubbable playback in the trace player — not just one frame per action. + * The dense frames are added alongside the per-action frames (which carry the + * DOM snapshots). Runs the screencast recorder (CDP push on Chrome, polling + * elsewhere). Default false. Only applies in trace mode. */ + filmstrip?: boolean } /** Minimal Cucumber pickle-step shape — only the fields the adapters read. @@ -408,6 +414,10 @@ export interface TraceLog { config?: { configFile?: string } /** Per-action snapshots captured in `mode: 'trace'` for the trace.zip exporter. */ actionSnapshots?: ActionSnapshot[] + /** Dense screencast frames (continuous recorder buffer) for the trace-mode + * filmstrip. Present only when `filmstrip` is enabled; thinned + content- + * addressed at export time. */ + screencastFrames?: ScreencastFrame[] } // ─── Preserve-and-rerun ───────────────────────────────────────────────────── From b21adf8bb3724400e3fe4f060d6e677cbc63c620 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 03:14:34 +0530 Subject: [PATCH 70/91] fix(core): window command-less trace slices; drop dead SpecRange.snapshotCount --- packages/core/src/spec-trace-helpers.ts | 24 +++++-- .../core/tests/spec-trace-helpers.test.ts | 70 +++++++++++++++++-- packages/core/tests/trace-finalizer.test.ts | 3 +- .../nightwatch-devtools/src/trace-slices.ts | 3 +- .../src/session-lifecycle.ts | 3 +- packages/service/src/index.ts | 3 +- 6 files changed, 85 insertions(+), 21 deletions(-) diff --git a/packages/core/src/spec-trace-helpers.ts b/packages/core/src/spec-trace-helpers.ts index 33d51bc0..8271d341 100644 --- a/packages/core/src/spec-trace-helpers.ts +++ b/packages/core/src/spec-trace-helpers.ts @@ -36,7 +36,6 @@ export interface SpecRange { networkStartIdx: number mutationStartIdx: number traceLogStartIdx: number - snapshotCount: number } /** @@ -256,7 +255,6 @@ export interface SpecBoundaryContext { mutations: ArrayLike<unknown> traceLogs: ArrayLike<unknown> } - actionSnapshots: ArrayLike<unknown> } /** Push a new slice range and return the previous (unflushed) range to flush. @@ -285,8 +283,7 @@ function pushSliceRange( consoleStartIdx: ctx.capturer.consoleLogs.length, networkStartIdx: ctx.capturer.networkRequests.length, mutationStartIdx: ctx.capturer.mutations.length, - traceLogStartIdx: ctx.capturer.traceLogs.length, - snapshotCount: ctx.actionSnapshots.length + traceLogStartIdx: ctx.capturer.traceLogs.length }) return prevRange @@ -359,7 +356,7 @@ export interface WriteSpecTraceInput { } /** Timestamped items inside a slice's wall-clock window `[start, end)`. An - * undefined `start` (a slice with no commands to anchor on) yields nothing; an + * undefined `start` (a slice with nothing to anchor on) yields nothing; an * undefined `end` (the last slice) is open-ended, so a trailing assertion-wait * stays with its test. Used for both action snapshots and screencast frames so * they partition on the same reliable boundary as the sliced commands. */ @@ -404,7 +401,22 @@ async function writeSliceTrace( // the previous one. const commandStart = (c: CommandLog | undefined): number | undefined => c ? (c.startTime ?? c.timestamp) : undefined - const windowStart = commandStart(sliceCapturer.commandsLog[0]) + // A command-less slice (assertion-only test) has no command to anchor on; fall + // back to the earliest of its own index-sliced console/network timestamps so + // its snapshots/frames still window+rebase rather than drop to the session + // offset. Empty of both keeps today's undefined → no-window behavior. + // Both are epoch ms (Date.now), matching the command anchor and the frame/ + // snapshot clock — NOT NetworkRequest.startTime, which is performance.now + // (relative ms) and would anchor the window ~54,000 years off. + const fallbackStart = (): number | undefined => { + const times = [ + ...sliceCapturer.consoleLogs.map((c) => c.timestamp), + ...sliceCapturer.networkRequests.map((n) => n.timestamp) + ] + return times.length ? Math.min(...times) : undefined + } + const windowStart = + commandStart(sliceCapturer.commandsLog[0]) ?? fallbackStart() const windowEnd = input.nextRange ? commandStart(input.capturer.commandsLog[input.nextRange.commandStartIdx]) : undefined diff --git a/packages/core/tests/spec-trace-helpers.test.ts b/packages/core/tests/spec-trace-helpers.test.ts index a3b8c8d2..c0e4ea91 100644 --- a/packages/core/tests/spec-trace-helpers.test.ts +++ b/packages/core/tests/spec-trace-helpers.test.ts @@ -49,7 +49,6 @@ const range = (over: Partial<SpecRange> = {}): SpecRange => ({ networkStartIdx: 0, mutationStartIdx: 0, traceLogStartIdx: 0, - snapshotCount: 0, ...over }) @@ -66,7 +65,6 @@ function boundaryCtx( mutations: [], traceLogs: [] }, - actionSnapshots: [], ...over } } @@ -162,15 +160,13 @@ describe('recordSliceBoundary (test granularity)', () => { networkRequests: [], mutations: [1, 2, 3], traceLogs: [] - }, - actionSnapshots: [1, 1] + } }) expect(recordSliceBoundary(ctx, 'test', '/a.js', 'u1')).toBeNull() expect(ctx.specRanges).toHaveLength(1) expect(ctx.specRanges[0]!.key).toBe('u1') expect(ctx.specRanges[0]!.testUid).toBe('u1') expect(ctx.specRanges[0]!.commandStartIdx).toBe(2) - expect(ctx.specRanges[0]!.snapshotCount).toBe(2) const prev = recordSliceBoundary(ctx, 'test', '/a.js', 'u2') expect(prev?.key).toBe('u1') @@ -365,6 +361,67 @@ describe('writeTestSliceTrace / writeSpecTrace output layout', () => { expect(frameOffsets).toEqual([100, 400, 1400]) }) + it('windows a command-less slice by its own console/network and rebases', async () => { + // An assertion-only test records no commands, so there is no command to + // anchor the wall-clock window on. The slice must anchor on the earliest of + // its own console/network *epoch* timestamps and rebase to it. The network + // request carries a small performance.now `startTime` (50) alongside its + // epoch `timestamp` (2000) — the fallback MUST use `.timestamp`; using + // `.startTime` would anchor the window at 50 and sweep the pre-window 1500 + // snapshot in with a garbage cross-clock offset. + const capturer: TraceCapturer = { + mutations: [], + traceLogs: [], + consoleLogs: [{ type: 'info', args: ['hi'], timestamp: 2600 }], + networkRequests: [ + { + id: 'r1', + url: 'https://example.test', + method: 'GET', + type: 'fetch', + startTime: 50, + timestamp: 2000 + } + ], + commandsLog: [], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + } + const snapshots = [ + { command: 'assert', timestamp: 1500, screenshot: 'QQ==' }, + { command: 'assert', timestamp: 2200, screenshot: 'QkI=' }, + { command: 'assert', timestamp: 2500, screenshot: 'Q0M=' } + ] + const dir = await writeTestSliceTrace( + input({ + range: range({ specFile: '/t.js', key: 'A', testUid: 'A' }), + capturer, + actionSnapshots: snapshots, + format: 'ndjson-directory', + testMetadata: new Map([['A', { title: 'Test A', specFile: '/t.js' }]]) + }) + ) + const events = (await fs.readFile(path.join(dir, 'trace.trace'), 'utf8')) + .trim() + .split('\n') + .map((l) => JSON.parse(l) as Record<string, unknown>) + // Window anchors on the earliest epoch timestamp (network .timestamp 2000): + // the pre-window 1500 snapshot is excluded; the sparse filmstrip re-anchors + // its first in-window frame (2200) to 0 and 2500 lands at 500. (Using the + // perf.now .startTime of 50 would instead admit 1500 → [0, 2150, 2450].) + const frameOffsets = events + .filter((e) => e.type === 'screencast-frame') + .map((e) => e.timestamp as number) + .sort((x, y) => x - y) + expect(frameOffsets).toEqual([0, 500]) + // Console (2600) rebased to the same window start (2000), not the session offset. + const consoleTimes = events + .filter((e) => e.type === 'console') + .map((e) => e.time as number) + expect(consoleTimes).toEqual([600]) + }) + it('keeps the spec write flat as trace-<id>.zip (unchanged layout)', async () => { const written = await writeSpecTrace( input({ @@ -410,8 +467,7 @@ describe('findFlushableRange', () => { consoleStartIdx: 0, networkStartIdx: 0, mutationStartIdx: 0, - traceLogStartIdx: 0, - snapshotCount: 0 + traceLogStartIdx: 0 }) it('reverse-scans for the given testUid (latest retry attempt wins)', () => { diff --git a/packages/core/tests/trace-finalizer.test.ts b/packages/core/tests/trace-finalizer.test.ts index 7510de03..6fe66b9d 100644 --- a/packages/core/tests/trace-finalizer.test.ts +++ b/packages/core/tests/trace-finalizer.test.ts @@ -59,8 +59,7 @@ function range(specFile: string, startIdx: number): SpecRange { consoleStartIdx: 0, networkStartIdx: 0, mutationStartIdx: 0, - traceLogStartIdx: 0, - snapshotCount: 0 + traceLogStartIdx: 0 } } diff --git a/packages/nightwatch-devtools/src/trace-slices.ts b/packages/nightwatch-devtools/src/trace-slices.ts index c69f8fe7..33111723 100644 --- a/packages/nightwatch-devtools/src/trace-slices.ts +++ b/packages/nightwatch-devtools/src/trace-slices.ts @@ -39,8 +39,7 @@ function boundaryContext(ctx: TestSliceCtx): SpecBoundaryContext { return { specRanges: ctx.specRanges, flushedSpecs: ctx.flushedSpecs, - capturer: ctx.sessionCapturer, - actionSnapshots: ctx.sessionCapturer.actionSnapshots + capturer: ctx.sessionCapturer } } diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index d00b4030..2f203dce 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -364,8 +364,7 @@ function boundaryContext( return { specRanges: ctx.specRanges, flushedSpecs: ctx.flushedSpecs, - capturer, - actionSnapshots: ctx.actionSnapshots + capturer } } diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 012382d6..af6bcf68 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -164,8 +164,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { return { specRanges: this.#specRanges, flushedSpecs: this.#flushedSpecs, - capturer: this.#sessionCapturer, - actionSnapshots: this.#actionSnapshots + capturer: this.#sessionCapturer } } From 37ddf4567185b73640b7660172920cabd60d8989 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 03:14:48 +0530 Subject: [PATCH 71/91] fix(core): drop the sparse per-action filmstrip when a dense one is present --- packages/core/src/trace-exporter.ts | 9 ++- packages/core/tests/trace-exporter.test.ts | 66 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index 6a43755e..c8ae443d 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -328,10 +328,15 @@ function buildEventStream( const viewport = trace.metadata.viewport ?? { width: 1280, height: 720 } const snapshots = trace.actionSnapshots ?? [] const snapshotIndex = new FrameSnapshotIndex(snapshots) + // Dense frames supersede the sparse per-action filmstrip; per-action DOM is + // carried by the frame-snapshot events, so sparse frames would only duplicate. + const filmstrip = + denseFrameEvents.length > 0 + ? denseFrameEvents + : buildFilmstripEvents(snapshots, pageId, wallTime, viewport) const events: TraceEvent[] = [ ctxOptions, - ...buildFilmstripEvents(snapshots, pageId, wallTime, viewport), - ...denseFrameEvents, + ...filmstrip, ...buildActionEvents( trace.commands, pageId, diff --git a/packages/core/tests/trace-exporter.test.ts b/packages/core/tests/trace-exporter.test.ts index 537d3c4c..365eb74c 100644 --- a/packages/core/tests/trace-exporter.test.ts +++ b/packages/core/tests/trace-exporter.test.ts @@ -456,4 +456,70 @@ describe('exported trace stream — dense filmstrip', () => { } await fs.rm(outputDir, { recursive: true, force: true }) }) + + it('drops the sparse per-action filmstrip when dense frames are present', async () => { + const outputDir = await fs.mkdtemp( + path.join(os.tmpdir(), 'trace-dense-sparse-') + ) + const capturer: TraceCapturer = { + mutations: [], + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { + command: 'url', + args: ['https://example.test'], + timestamp: 1200, + startTime: 1150 + }, + { command: 'click', args: ['#go'], timestamp: 1500, startTime: 1450 } + ], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + } + const dir = await writeTraceZip(capturer, { + outputDir, + sessionId: 'abc12345', + format: 'ndjson-directory', + // Action snapshots carry the DOM refs the sparse filmstrip used to fold in. + actionSnapshots: [ + { + timestamp: 1200, + command: 'url', + screenshot: 'AAAA', + elements: [{ selector: '#go' }], + snapshotText: '<html></html>' + }, + { timestamp: 1500, command: 'click', screenshot: 'BBBB' } + ], + screencastFrames: [ + { data: Buffer.from('paint-1').toString('base64'), timestamp: 1000 }, + { data: Buffer.from('paint-2').toString('base64'), timestamp: 1400 } + ] + }) + const lines = (await fs.readFile(path.join(dir, 'trace.trace'), 'utf8')) + .trim() + .split('\n') + .map((l) => JSON.parse(l) as Record<string, unknown>) + + // Every filmstrip frame is a content-addressed dense frame; no sparse + // `page@…-<ts>.jpeg` frames survive, so there is no duplication. + const frames = lines.filter((l) => l.type === 'screencast-frame') + expect(frames).toHaveLength(2) + for (const f of frames) { + expect(String(f.sha1)).toMatch(/^[0-9a-f]{40}\.jpeg$/) + } + + // Per-action DOM survives: frame-snapshot events + after-snapshot refs stay. + const frameSnaps = lines.filter((l) => l.type === 'frame-snapshot') + expect(frameSnaps).toHaveLength(2) + const afterSnaps = lines + .filter((l) => l.type === 'after') + .map((l) => l.afterSnapshot) + .filter(Boolean) + expect(afterSnaps).toEqual(['after@call@1', 'after@call@2']) + await fs.rm(outputDir, { recursive: true, force: true }) + }) }) From b275b3973a8b84655e95504a8d2e3a75d6735078 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 03:15:08 +0530 Subject: [PATCH 72/91] fix(core): cap the screencast frame buffer to bound memory --- packages/core/src/screencast.ts | 41 +++++++++++++-- packages/core/tests/screencast.test.ts | 71 ++++++++++++++++++++++++++ packages/shared/src/types.ts | 5 +- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/packages/core/src/screencast.ts b/packages/core/src/screencast.ts index 892d1bcd..e2c8b4ea 100644 --- a/packages/core/src/screencast.ts +++ b/packages/core/src/screencast.ts @@ -130,7 +130,42 @@ export abstract class ScreencastRecorderBase<TDriver = unknown> { typeof timestampSeconds === 'number' ? Math.round(timestampSeconds * 1000) : Date.now() - this.buffer.push({ data, timestamp }) + this.#appendFrame({ data, timestamp }) + } + + /** + * Append a frame, decimating the buffer in place once it exceeds + * `maxBufferFrames` so a long session can't grow it without bound. + */ + #appendFrame(frame: ScreencastFrame): void { + this.buffer.push(frame) + // Decimation always keeps the first and last frame, so 2 is the hard floor; + // clamp so a nonsensical sub-2 cap is honored as 2 rather than never firing. + if (this.buffer.length > Math.max(2, this.options.maxBufferFrames)) { + this.#decimateBuffer() + } + } + + /** + * Halve the buffer by keeping every other frame plus the first and last, so + * memory is bounded while the temporal spread survives (never tail-truncated). + * `#startIndex` is remapped to the count of surviving pre-marker frames, which + * keeps `frames`/`duration` excluding pre-marker frames across decimation. + */ + #decimateBuffer(): void { + const lastIndex = this.buffer.length - 1 + const kept: ScreencastFrame[] = [] + let preMarkerKept = 0 + for (let i = 0; i < this.buffer.length; i++) { + if (i % 2 === 0 || i === lastIndex) { + if (i < this.#startIndex) { + preMarkerKept++ + } + kept.push(this.buffer[i]) + } + } + this.buffer = kept + this.#startIndex = preMarkerKept } /** Whether `setStartMarker` (or `markStartAtLatest`) has fired yet. */ @@ -180,14 +215,14 @@ export abstract class ScreencastRecorderBase<TDriver = unknown> { this.onUnavailable(new Error('first screenshot returned null')) return } - this.buffer.push({ data: first, timestamp: Date.now() }) + this.#appendFrame({ data: first, timestamp: Date.now() }) const intervalMs = this.options.pollIntervalMs this.#pollTimer = setInterval(async () => { try { const data = await this.takeScreenshot() if (data !== null) { - this.buffer.push({ data, timestamp: Date.now() }) + this.#appendFrame({ data, timestamp: Date.now() }) } } catch { // Session ended mid-interval — stop polling gracefully. diff --git a/packages/core/tests/screencast.test.ts b/packages/core/tests/screencast.test.ts index dc57bdb6..d1c1de47 100644 --- a/packages/core/tests/screencast.test.ts +++ b/packages/core/tests/screencast.test.ts @@ -148,3 +148,74 @@ describe('ScreencastRecorderBase — CDP override path', () => { await r.stop() }) }) + +describe('ScreencastRecorderBase — buffer cap / decimation', () => { + class PushRecorder extends ScreencastRecorderBase<{ name: string }> { + protected override async takeScreenshot() { + return null + } + protected override async tryStartCdp() { + return true + } + push(d: string, t: number) { + this.pushCdpFrame(d, t) + } + get len() { + return this.buffer.length + } + } + + it('never lets the buffer exceed maxBufferFrames across many appends', async () => { + const cap = 10 + const r = new PushRecorder({ maxBufferFrames: cap }) + await r.start({ name: 'd' }) + for (let i = 0; i < 1000; i++) { + r.push(`f-${i}`, i) + expect(r.len).toBeLessThanOrEqual(cap) + } + await r.stop() + }) + + it('preserves the first and last frame and the temporal spread (not tail-truncated)', async () => { + const cap = 100 + const total = 350 + const r = new PushRecorder({ maxBufferFrames: cap }) + await r.start({ name: 'd' }) + for (let i = 0; i < total; i++) { + r.push(`f-${i}`, i) // pushCdpFrame stores seconds*1000, so timestamp == i*1000 + } + const frames = r.frames + expect(frames[0].data).toBe('f-0') // first kept — a tail-truncated buffer would drop it + expect(frames[frames.length - 1].data).toBe(`f-${total - 1}`) // last kept + expect(r.duration).toBe((total - 1) * 1000) // spans the whole session + + // Spread survives: frames land in the early, middle, and late thirds of the + // timeline rather than clustering at either end. + const times = frames.map((f) => f.timestamp / 1000) + expect(times.some((t) => t < total / 4)).toBe(true) + expect(times.some((t) => t > total / 4 && t < (total * 3) / 4)).toBe(true) + expect(times.some((t) => t > (total * 3) / 4)).toBe(true) + await r.stop() + }) + + it('keeps setStartMarker semantics and a sane duration across decimation', async () => { + const cap = 10 + const r = new PushRecorder({ maxBufferFrames: cap }) + await r.start({ name: 'd' }) + for (let i = 0; i < 5; i++) { + r.push(`pre-${i}`, i) // ts 0..4, before the marker + } + r.setStartMarker() + for (let i = 0; i < 200; i++) { + r.push(`post-${i}`, 100 + i) // ts 100..299, after the marker — forces many decimations + } + const frames = r.frames + expect(frames.length).toBeGreaterThan(1) + // No pre-marker frame ever leaks into the public getter. + expect(frames.every((f) => f.data.startsWith('post-'))).toBe(true) + // Duration stays within the post-marker time window (ts 100..299 → 199s span in ms). + expect(r.duration).toBeGreaterThan(0) + expect(r.duration).toBeLessThanOrEqual(199 * 1000) + await r.stop() + }) +}) diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 793c7b8c..b6d37d30 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -278,6 +278,8 @@ export interface ScreencastOptions { * (default: 200 ms ≈ 5 fps). Lower = smoother, more WebDriver round-trips. */ pollIntervalMs?: number + /** Cap on frames held in memory (default: 2000 ≈ several minutes at ~5 fps); the buffer is decimated in place past this. */ + maxBufferFrames?: number } /** Defaults applied to ScreencastOptions when not specified by the user. */ @@ -287,7 +289,8 @@ export const SCREENCAST_DEFAULTS: Required<ScreencastOptions> = { quality: 70, maxWidth: 1280, maxHeight: 720, - pollIntervalMs: 200 + pollIntervalMs: 200, + maxBufferFrames: 2000 } /** From cd32ba1f8b692a530b8cefeb6119632ce2d3934d Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 19:37:00 +0530 Subject: [PATCH 73/91] fix(core): render node:assert failures as clean Expected/Received --- packages/core/src/assert-patcher.ts | 77 ++++++++++++++++++- packages/core/tests/assert-patcher.test.ts | 48 ++++++++++-- .../tests/assertCapture.test.ts | 2 +- packages/service/tests/assert-capture.test.ts | 2 +- 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index c60f21ab..c171c82a 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -1,5 +1,9 @@ import { createRequire } from 'node:module' -import { TRACKED_ASSERT_METHODS, type CommandLog } from '@wdio/devtools-shared' +import { + TRACKED_ASSERT_METHODS, + type CollapsedAssertResult, + type CommandLog +} from '@wdio/devtools-shared' import { getCallSourceFromStack } from './stack.js' import { toError } from './error.js' import { stripAnsi } from './console.js' @@ -21,7 +25,10 @@ export const ASSERT_PATCHED_SYMBOL = Symbol.for( export interface CapturedAssert { command: string args: unknown[] - result: 'passed' | undefined + /** `'passed'` for a passing assert; a `CollapsedAssertResult` for a failing + * node:assert (its clean `actual`/`expected` props), so the trace shows + * labelled rows rather than node's ANSI-stripped char-diff. */ + result: 'passed' | CollapsedAssertResult | undefined error: Error | undefined callSource: string | undefined timestamp: number @@ -52,6 +59,65 @@ export function safeSerializeAssertArg(value: unknown): unknown { return value } +interface NodeAssertionError { + actual?: unknown + expected?: unknown + generatedMessage?: boolean +} + +/** Human-readable rendering of an assert value for the message body. */ +function displayAssertValue(value: unknown): string { + if (typeof value === 'string') { + return `'${value}'` + } + try { + return JSON.stringify(value) ?? String(value) + } catch { + return String(value) + } +} + +/** + * Turn a thrown assert error into the captured shape. node:assert's + * AssertionError carries clean `actual`/`expected` props plus, when it + * auto-generated the message, a per-character COLORED diff whose body becomes + * unreadable noise once ANSI is stripped (`'ExampleThis DIs Nomt...'`). Pull the + * clean values into a CollapsedAssertResult AND rebuild an auto-generated + * message (and its echo in the stack) as a value-bearing `Expected:/Received:` + * block, so every consumer — the trace's Errors tab, the runner console, and + * allure-mocha's error box (which strips the diff's ANSI to mush) — shows + * readable values. A user-supplied message and errors without `actual`/ + * `expected` (e.g. `assert.ok`, a thrown non-assert error) pass through. + */ +function describeAssertFailure(err: unknown): { + result: CollapsedAssertResult | undefined + error: Error +} { + const error = toError(err) + const e = (typeof err === 'object' && err ? err : {}) as NodeAssertionError + if (!('actual' in e) && !('expected' in e)) { + return { result: undefined, error } + } + const rawMessage = stripAnsi(error.message) + const cleanMessage = e.generatedMessage + ? `${rawMessage.split('\n')[0] ?? 'Assertion failed'}\n\n` + + `Expected: ${displayAssertValue(e.expected)}\n` + + `Received: ${displayAssertValue(e.actual)}` + : rawMessage + error.message = cleanMessage + if (error.stack) { + error.stack = stripAnsi(error.stack).replace(rawMessage, cleanMessage) + } + return { + result: { + passed: false, + actual: safeSerializeAssertArg(e.actual), + expected: safeSerializeAssertArg(e.expected) + }, + error + } +} + function makeAssertEmitters( methodName: string, args: unknown[], @@ -65,7 +131,7 @@ function makeAssertEmitters( } const startedAt = Date.now() const sanitizedArgs = args.map(safeSerializeAssertArg) - const emit = (result: 'passed' | undefined, error: Error | undefined) => + const emit = (result: CapturedAssert['result'], error: Error | undefined) => onCommand({ command: `assert.${methodName}`, args: sanitizedArgs, @@ -76,7 +142,10 @@ function makeAssertEmitters( }) return { passed: () => emit('passed', undefined), - failed: (err: unknown) => emit(undefined, toError(err)) + failed: (err: unknown) => { + const { result, error } = describeAssertFailure(err) + emit(result, error) + } } } diff --git a/packages/core/tests/assert-patcher.test.ts b/packages/core/tests/assert-patcher.test.ts index 5eda95da..9fb79d9b 100644 --- a/packages/core/tests/assert-patcher.test.ts +++ b/packages/core/tests/assert-patcher.test.ts @@ -78,17 +78,47 @@ describe('patchNodeAssert', () => { const captured: CapturedAssert[] = [] patchNodeAssert((c) => captured.push(c)) expect(() => assert.equal(1, 2)).toThrow() - expect(captured[0].result).toBeUndefined() + // A failed node:assert carries its clean actual/expected (not node's + // ANSI-stripped char-diff) so the trace renders labelled rows. + expect(captured[0].result).toMatchObject({ + passed: false, + actual: 1, + expected: 2 + }) expect(captured[0].error).toBeInstanceOf(Error) }) + it('carries clean expected/actual and a header-only message for a strict string diff', () => { + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + // strict.equal is what produces node's colored char-diff (the source of the + // 'ExampleThis DIs Nomt…' mush once ANSI is stripped downstream). + expect(() => + assert.strict.equal('Example Domain', 'This Is Not The Heading') + ).toThrow() + // node's clean props, not its char-diff, drive the rows. + expect(captured[0].result).toMatchObject({ + passed: false, + actual: 'Example Domain', + expected: 'This Is Not The Heading' + }) + // The auto-generated diff body is rebuilt as a value-bearing Expected/ + // Received block — never the ANSI-stripped char-diff mush. + const message = captured[0].error?.message ?? '' + expect(message).toContain('Expected values to be strictly equal:') + expect(message).toContain("Expected: 'This Is Not The Heading'") + expect(message).toContain("Received: 'Example Domain'") + expect(message).not.toContain('ExampleThis') + }) + it('handles Promise-returning asserts (rejects/doesNotReject)', async () => { const captured: CapturedAssert[] = [] patchNodeAssert((c) => captured.push(c)) await assert.doesNotReject(async () => 1) await expect(assert.rejects(async () => 1)).rejects.toThrow() const results = captured.map((c) => c.result) - expect(results).toEqual(['passed', undefined]) // first resolved, second rejected + expect(results[0]).toBe('passed') // first resolved + expect(results[1]).not.toBe('passed') // second rejected → failed capture }) // `import { strict as assert }` (and `assert.strict.*`) reference a separate @@ -98,10 +128,12 @@ describe('patchNodeAssert', () => { patchNodeAssert((c) => captured.push(c)) assert.strict.equal(1, 1) expect(() => assert.strict.equal(1, 2)).toThrow() - expect(captured.map((c) => [c.command, c.result])).toEqual([ - ['assert.equal', 'passed'], - ['assert.equal', undefined] + expect(captured.map((c) => c.command)).toEqual([ + 'assert.equal', + 'assert.equal' ]) + expect(captured[0].result).toBe('passed') + expect(captured[1].result).toMatchObject({ passed: false }) }) // Only assertions that originate in user test code should reach the trace. @@ -144,7 +176,7 @@ describe('patchNodeAssert', () => { result: 'passed', callSource: USER_FRAME.callSource }) - expect(captured[1].result).toBeUndefined() + expect(captured[1].result).toMatchObject({ passed: false }) expect(captured[1].error).toBeInstanceOf(Error) }) }) @@ -179,7 +211,7 @@ describe('patchNodeAssert', () => { patchNodeAssert((c) => captured.push(c)) expect(() => assert.match('a', /b/)).toThrow() expect(captured[0].command).toBe('assert.match') - expect(captured[0].result).toBeUndefined() + expect(captured[0].result).toMatchObject({ passed: false }) expect(captured[0].error).toBeInstanceOf(Error) }) @@ -196,7 +228,7 @@ describe('patchNodeAssert', () => { patchNodeAssert((c) => captured.push(c)) expect(() => assert.doesNotMatch('a', /a/)).toThrow() expect(captured[0].command).toBe('assert.doesNotMatch') - expect(captured[0].result).toBeUndefined() + expect(captured[0].result).toMatchObject({ passed: false }) expect(captured[0].error).toBeInstanceOf(Error) }) diff --git a/packages/nightwatch-devtools/tests/assertCapture.test.ts b/packages/nightwatch-devtools/tests/assertCapture.test.ts index 61602964..7c498b0b 100644 --- a/packages/nightwatch-devtools/tests/assertCapture.test.ts +++ b/packages/nightwatch-devtools/tests/assertCapture.test.ts @@ -87,7 +87,7 @@ describe('wireAssertCapture', () => { expect(() => assert.strictEqual('a', 'b')).toThrow() const failed = fake.commandsLog[1] expect(failed.command).toBe('assert.strictEqual') - expect(failed.result).toBeUndefined() + expect(failed.result).toMatchObject({ passed: false }) expect(failed.error).toBeInstanceOf(Error) expect(fake.sendCommand).toHaveBeenCalledTimes(2) }) diff --git a/packages/service/tests/assert-capture.test.ts b/packages/service/tests/assert-capture.test.ts index e88f3dca..58edd9d8 100644 --- a/packages/service/tests/assert-capture.test.ts +++ b/packages/service/tests/assert-capture.test.ts @@ -124,9 +124,9 @@ describe('wireAssertCapture', () => { expect(() => assert.strictEqual('a', 'b')).toThrow() const failed = entries[1] expect(failed.command).toBe('assert.strictEqual') + expect(failed.result).toMatchObject({ passed: false }) expect(failed.testUid).toBe('uid-2') expect(failed.error).toMatchObject({ name: 'AssertionError' }) - expect(failed.result).toBeUndefined() }) it('is a no-op wiring when invoked twice (per-process patch guard)', () => { From 712faf757a96dce7b9204daf3acb6109f2a91a6e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 19:37:44 +0530 Subject: [PATCH 74/91] feat: cross-adapter Allure artifact attachment via pluggable sink --- .gitignore | 11 +- eslint.config.cjs | 8 +- packages/core/src/allure-artifacts.ts | 206 ++++++++++++++++ packages/core/src/index.ts | 1 + packages/core/tests/allure-artifacts.test.ts | 224 ++++++++++++++++++ packages/nightwatch-devtools/README.md | 4 + .../src/cucumber-lifecycle.ts | 7 + packages/nightwatch-devtools/src/index.ts | 75 +++++- .../src/plugin-internals.ts | 4 + .../nightwatch-devtools/src/session-init.ts | 33 ++- .../nightwatch-devtools/src/test-artifacts.ts | 92 +++++++ packages/nightwatch-devtools/src/types.ts | 19 +- .../tests/testArtifacts.test.ts | 134 +++++++++++ packages/selenium-devtools/README.md | 2 + packages/selenium-devtools/package.json | 9 + packages/selenium-devtools/src/allure.ts | 62 +++++ .../src/helpers/testManager.ts | 19 +- packages/selenium-devtools/src/index.ts | 144 ++++++++++- .../src/runnerHooks/cucumber.ts | 12 +- .../selenium-devtools/src/runnerHooks/jest.ts | 6 +- .../src/runnerHooks/mocha.ts | 12 +- .../src/session-lifecycle.ts | 66 ++++-- packages/selenium-devtools/src/types.ts | 27 ++- .../selenium-devtools/tests/allure.test.ts | 52 ++++ packages/service/src/allure.ts | 92 ++----- packages/service/src/index.ts | 40 ++-- packages/service/src/screenshot-capture.ts | 67 ------ packages/service/src/video-capture.ts | 80 ------- packages/service/tests/allure.test.ts | 99 ++------ .../service/tests/screenshot-capture.test.ts | 86 ------- packages/service/tests/video-capture.test.ts | 75 ------ pnpm-lock.yaml | 133 ++++------- 32 files changed, 1263 insertions(+), 638 deletions(-) create mode 100644 packages/core/src/allure-artifacts.ts create mode 100644 packages/core/tests/allure-artifacts.test.ts create mode 100644 packages/nightwatch-devtools/src/test-artifacts.ts create mode 100644 packages/nightwatch-devtools/tests/testArtifacts.test.ts create mode 100644 packages/selenium-devtools/src/allure.ts create mode 100644 packages/selenium-devtools/tests/allure.test.ts delete mode 100644 packages/service/src/screenshot-capture.ts delete mode 100644 packages/service/src/video-capture.ts delete mode 100644 packages/service/tests/screenshot-capture.test.ts delete mode 100644 packages/service/tests/video-capture.test.ts diff --git a/.gitignore b/.gitignore index e8a7a934..4bcee08d 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,7 @@ trace-*.zip examples/**/trace-*/ # test results -examples/**/test-results/ +examples/**/test-results*/ # vitest --coverage output coverage/ @@ -49,3 +49,12 @@ coverage/ /packages/**/*.mjs .claude + +# Local Allure exploration / trace-parity comparison (throwaway — not committed) +examples/**/allure-results*/ +examples/**/allure-report*/ +packages/**/allure-results*/ +packages/**/allure-report*/ +examples/playwright-allure/ +examples/wdio/cucumber/wdio.allure.conf.ts + diff --git a/eslint.config.cjs b/eslint.config.cjs index b7c00ca1..7ef666a2 100644 --- a/eslint.config.cjs +++ b/eslint.config.cjs @@ -9,7 +9,13 @@ const security = require('eslint-plugin-security') module.exports = [ { - ignores: ['node_modules/**', '**/dist/**', '**/.tsup/**'] + ignores: [ + 'node_modules/**', + '**/dist/**', + '**/.tsup/**', + '**/allure-report/**', + '**/allure-results/**' + ] }, // Base JS config { diff --git a/packages/core/src/allure-artifacts.ts b/packages/core/src/allure-artifacts.ts new file mode 100644 index 00000000..a07d2595 --- /dev/null +++ b/packages/core/src/allure-artifacts.ts @@ -0,0 +1,206 @@ +// Framework-agnostic capture-and-attach of per-test trace artifacts to Allure. +// The *produce* half (screenshot/video write, retention decision) already lives +// in core; this module adds the orchestration + the attach, behind a pluggable +// `AllureAttachSink` so each adapter supplies only its reporter binding: +// - WDIO service → @wdio/allure-reporter's addAttachment +// - Selenium → allure-js-commons' attachment() (runtime-agnostic) +// - Nightwatch → no sink (produce-only; its Allure path is post-hoc, with no +// live attach API), so artifacts land in the manifest only. +// A missing/undefined sink means "produce but don't attach" — never an error. + +import fs from 'node:fs/promises' +import { basename } from 'node:path' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastFrame, + TraceGranularity, + TraceScreenshotPolicy, + TraceVideoPolicy +} from '@wdio/devtools-shared' +import { + shouldCaptureScreenshot, + writeScreenshotArtifact +} from './screenshot-artifact.js' +import { encodePerTestVideo, sliceFramesFrom } from './video-slice.js' +import { shouldRetainTrace, type TestOutcome } from './trace-retention.js' +import type { TraceArtifact } from './trace-finalizer.js' + +/** Adapter-supplied binding to the active Allure reporter. Attaches one file to + * the currently-executing test. Undefined = attach unsupported (produce-only). */ +export type AllureAttachSink = ( + name: string, + content: Buffer, + contentType: string +) => void | Promise<void> + +type LogFn = (level: 'info' | 'warn', message: string) => void + +// Trace stays a plain zip download (the viewer is the user's choice, not a +// third-party viewer Allure would open for a trace-specific content type); +// video/image render inline. +const CONTENT_TYPE_BY_KIND: Record<TraceArtifact['kind'], string> = { + trace: 'application/zip', + video: 'video/webm', + screenshot: 'image/png' +} + +/** + * Read a retained artifact and hand it to the sink for attachment to the current + * Allure test. No-op when the artifact wasn't retained, has no path, the sink is + * absent, or the path is a directory (the ndjson-directory trace format). Never + * throws — a missing/unreadable artifact must not reject the caller's hook. + */ +export async function attachTraceArtifact( + artifact: TraceArtifact, + sink: AllureAttachSink | undefined, + onLog?: LogFn +): Promise<void> { + if (!sink || !artifact.retained || !artifact.path) { + return + } + try { + const stat = await fs.stat(artifact.path) + if (!stat.isFile()) { + return + } + const content = await fs.readFile(artifact.path) + await sink( + basename(artifact.path), + content, + CONTENT_TYPE_BY_KIND[artifact.kind] + ) + } catch (err) { + onLog?.( + 'warn', + `Allure attach skipped for ${artifact.path}: ${String(err)}` + ) + } +} + +/** + * The base64 of the last rendered action snapshot for the current test, skipping + * the end-of-scenario `__final__` frame (captured post-teardown, often blank when + * a reloadSession runs before the after-hook). Scoped to `>= startWallTime` so a + * test that captured nothing doesn't borrow the previous test's frame. Reused as + * the per-test screenshot — reload-immune and one fewer WebDriver command than a + * fresh end-of-test capture. + */ +export function lastRenderedScreenshot( + snapshots: readonly ActionSnapshot[], + startWallTime: number +): string | undefined { + for (let i = snapshots.length - 1; i >= 0; i--) { + const snap = snapshots[i]! + if (snap.timestamp < startWallTime) { + return undefined + } + if (snap.command !== '__final__' && snap.screenshot) { + return snap.screenshot + } + } + return undefined +} + +/** + * Write the per-test screenshot (per policy) from an already-captured base64 + * frame, record it in the manifest, and attach it via the sink. No-op outside + * `test`-granularity trace mode, without a uid/session/frame, or when the policy + * declines. Gated to `test` granularity so the whole per-test inline story + * (trace + screenshot + video) is one rule. + */ +export async function captureAndAttachScreenshot(input: { + mode: DevToolsMode | undefined + granularity: TraceGranularity | undefined + policy: TraceScreenshotPolicy | undefined + failed: boolean + /** Base64 of the last rendered action snapshot (reload-immune). */ + screenshotBase64: string | undefined + sessionId: string | undefined + outputDir: string + testUid: string | undefined + attach: AllureAttachSink | undefined + onArtifact: (artifact: TraceArtifact) => void + onLog?: LogFn +}): Promise<void> { + const { mode, granularity, policy, failed, screenshotBase64, sessionId } = + input + if ( + mode !== 'trace' || + granularity !== 'test' || + !screenshotBase64 || + !sessionId || + !input.testUid || + !shouldCaptureScreenshot(policy, failed) + ) { + return + } + const artifact = await writeScreenshotArtifact({ + outputDir: input.outputDir, + testUid: input.testUid, + sessionId, + base64: screenshotBase64 + }) + input.onArtifact(artifact) + await attachTraceArtifact(artifact, input.attach, input.onLog) +} + +/** + * Slice the continuous screencast to this test's window, encode a `.webm`, + * record it in the manifest, and attach it via the sink — when trace mode + + * `test` granularity + a non-`off` `video` policy retains this test's outcome. + * Empty outcomes (test never started) skip rather than fail-open onto the + * previous test's frames. + */ +export async function captureAndAttachVideo(input: { + mode: DevToolsMode | undefined + granularity: TraceGranularity | undefined + policy: TraceVideoPolicy | undefined + frames: readonly ScreencastFrame[] | undefined + startWallTime: number + outcomes: TestOutcome[] + attempt: number | undefined + outputDir: string + testUid: string | undefined + sessionId: string | undefined + captureFormat?: 'jpeg' | 'png' + attach: AllureAttachSink | undefined + onArtifact: (artifact: TraceArtifact) => void + onLog?: LogFn +}): Promise<void> { + const { mode, granularity, policy, frames, testUid, sessionId } = input + if ( + mode !== 'trace' || + granularity !== 'test' || + !policy || + policy === 'off' || + !frames || + !testUid || + !sessionId || + input.outcomes.length === 0 + ) { + return + } + if ( + !shouldRetainTrace(policy, { + outcomes: input.outcomes, + attemptInfoAvailable: true + }).retain + ) { + return + } + const artifact = await encodePerTestVideo({ + frames: sliceFramesFrom(frames, input.startWallTime), + outputDir: input.outputDir, + testUid, + sessionId, + attempt: input.attempt, + captureFormat: input.captureFormat, + onLog: input.onLog + }) + if (!artifact) { + return + } + input.onArtifact(artifact) + await attachTraceArtifact(artifact, input.attach, input.onLog) +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a73cb7bf..cee1a38e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,6 +5,7 @@ export * from './action-mapping.js' export * from './action-snapshot.js' export * from './artifact-naming.js' export * from './artifacts-manifest.js' +export * from './allure-artifacts.js' export * from './attempt-tracker.js' export * from './screenshot-artifact.js' export * from './video-slice.js' diff --git a/packages/core/tests/allure-artifacts.test.ts b/packages/core/tests/allure-artifacts.test.ts new file mode 100644 index 00000000..7ce26252 --- /dev/null +++ b/packages/core/tests/allure-artifacts.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { mkdtemp, rm, readdir, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ActionSnapshot, ScreencastFrame } from '@wdio/devtools-shared' +import { + attachTraceArtifact, + captureAndAttachScreenshot, + captureAndAttachVideo, + lastRenderedScreenshot, + type AllureAttachSink, + type TestOutcome, + type TraceArtifact +} from '../src/index.js' + +function artifact(over: Partial<TraceArtifact> = {}): TraceArtifact { + return { + kind: 'trace', + path: '', + scope: 'test', + testUids: ['u1'], + retained: true, + ...over + } +} + +const dirs: string[] = [] +afterEach(async () => { + await Promise.all( + dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) + ) +}) +async function tempFile(name: string, bytes = 'bytes'): Promise<string> { + const dir = await mkdtemp(join(tmpdir(), 'allure-art-')) + dirs.push(dir) + const p = join(dir, name) + await writeFile(p, bytes) + return p +} + +describe('attachTraceArtifact', () => { + let sink: ReturnType<typeof vi.fn> + const withSink = (a: TraceArtifact) => + attachTraceArtifact(a, sink as unknown as AllureAttachSink) + beforeEach(() => { + sink = vi.fn() + }) + + it('routes a trace zip to the sink as application/zip', async () => { + const path = await tempFile('trace-abc.zip') + await withSink(artifact({ path })) + expect(sink).toHaveBeenCalledOnce() + const [name, content, type] = sink.mock.calls[0]! + expect(name).toBe('trace-abc.zip') + expect(Buffer.isBuffer(content)).toBe(true) + expect(type).toBe('application/zip') + }) + + it('maps video → video/webm and screenshot → image/png', async () => { + await withSink(artifact({ kind: 'video', path: await tempFile('r.webm') })) + await withSink( + artifact({ kind: 'screenshot', path: await tempFile('s.png') }) + ) + expect(sink.mock.calls[0]![2]).toBe('video/webm') + expect(sink.mock.calls[1]![2]).toBe('image/png') + }) + + it('no-ops for a non-retained artifact, an empty path, or an absent sink', async () => { + await withSink(artifact({ retained: false, path: '/x.zip' })) + await withSink(artifact({ path: '' })) + expect(sink).not.toHaveBeenCalled() + // undefined sink is produce-only, not an error + await expect( + attachTraceArtifact(artifact({ path: '/x.zip' }), undefined) + ).resolves.toBeUndefined() + }) + + it('skips a directory path (ndjson-directory) without throwing', async () => { + const dir = await mkdtemp(join(tmpdir(), 'allure-dir-')) + dirs.push(dir) + await expect(withSink(artifact({ path: dir }))).resolves.toBeUndefined() + expect(sink).not.toHaveBeenCalled() + }) + + it('never rejects when the path is missing', async () => { + await expect( + withSink(artifact({ path: '/no/such/trace.zip' })) + ).resolves.toBeUndefined() + expect(sink).not.toHaveBeenCalled() + }) +}) + +describe('lastRenderedScreenshot', () => { + const snap = ( + command: string, + timestamp: number, + screenshot?: string + ): ActionSnapshot => ({ command, timestamp, screenshot }) as ActionSnapshot + + it('returns the last non-final screenshot at/after the test start', () => { + const snaps = [ + snap('click', 100, 'AA'), + snap('setValue', 200, 'BB'), + snap('__final__', 300, 'CC') + ] + expect(lastRenderedScreenshot(snaps, 100)).toBe('BB') + }) + + it('returns undefined when the only snapshots predate the test start', () => { + expect( + lastRenderedScreenshot([snap('click', 50, 'AA')], 100) + ).toBeUndefined() + }) + + it('returns undefined when there are no snapshots', () => { + expect(lastRenderedScreenshot([], 0)).toBeUndefined() + }) +}) + +describe('captureAndAttachScreenshot', () => { + const collected: TraceArtifact[] = [] + let sink: ReturnType<typeof vi.fn> + beforeEach(() => { + collected.length = 0 + sink = vi.fn() + }) + const SHOT = Buffer.from('png-bytes').toString('base64') + + async function run( + over: Partial<Parameters<typeof captureAndAttachScreenshot>[0]> + ) { + const dir = await mkdtemp(join(tmpdir(), 'shot-cap-')) + dirs.push(dir) + await captureAndAttachScreenshot({ + mode: 'trace', + granularity: 'test', + policy: 'only-on-failure', + failed: true, + screenshotBase64: SHOT, + sessionId: 'sess1234', + outputDir: dir, + testUid: 'u1', + attach: sink as unknown as AllureAttachSink, + onArtifact: (a) => collected.push(a), + ...over + }) + return dir + } + + it('writes + attaches the reused snapshot on failure under only-on-failure', async () => { + const dir = await run({}) + expect(collected).toHaveLength(1) + expect(collected[0]!.kind).toBe('screenshot') + expect(sink).toHaveBeenCalledOnce() + expect(await readdir(dir)).toHaveLength(1) + }) + + it('produces without attaching when the sink is absent', async () => { + await run({ attach: undefined }) + expect(collected).toHaveLength(1) + expect(sink).not.toHaveBeenCalled() + }) + + it('no-ops on a passing test under only-on-failure', async () => { + await run({ failed: false }) + expect(collected).toHaveLength(0) + expect(sink).not.toHaveBeenCalled() + }) + + it('no-ops outside trace mode / test granularity / without frame / without ids', async () => { + await run({ mode: 'live', policy: 'on' }) + await run({ granularity: 'session', policy: 'on' }) + await run({ policy: 'on', screenshotBase64: undefined }) + await run({ sessionId: undefined }) + await run({ testUid: undefined }) + expect(collected).toHaveLength(0) + }) +}) + +describe('captureAndAttachVideo gating', () => { + // Every case no-ops BEFORE the ffmpeg encode (gate or non-retaining policy), + // so no encoder/reporter mock is needed; the encode path is covered by + // video-slice.test.ts. + const collected: TraceArtifact[] = [] + beforeEach(() => { + collected.length = 0 + }) + const frames: ScreencastFrame[] = [ + { data: 'AAAA', timestamp: 10 }, + { data: 'AAAA', timestamp: 20 } + ] + const failedOutcome: TestOutcome[] = [ + { uid: 'u1', attempt: 0, state: 'failed' } + ] + const passedOutcome: TestOutcome[] = [ + { uid: 'u1', attempt: 0, state: 'passed' } + ] + const base = { + mode: 'trace' as const, + granularity: 'test' as const, + policy: 'retain-on-failure' as const, + frames, + startWallTime: 0, + outcomes: failedOutcome, + attempt: 0, + outputDir: '/tmp/does-not-encode', + testUid: 'u1', + sessionId: 'sess1234', + attach: undefined, + onArtifact: (a: TraceArtifact) => collected.push(a) + } + + it('no-ops for granularity/outcomes/policy/frames/ids gates and non-retaining outcome', async () => { + await captureAndAttachVideo({ ...base, granularity: 'session' }) + await captureAndAttachVideo({ ...base, outcomes: [] }) + await captureAndAttachVideo({ ...base, policy: 'off' }) + await captureAndAttachVideo({ ...base, policy: undefined }) + await captureAndAttachVideo({ ...base, frames: undefined }) + await captureAndAttachVideo({ ...base, sessionId: undefined }) + await captureAndAttachVideo({ ...base, testUid: undefined }) + await captureAndAttachVideo({ ...base, outcomes: passedOutcome }) + expect(collected).toHaveLength(0) + }) +}) diff --git a/packages/nightwatch-devtools/README.md b/packages/nightwatch-devtools/README.md index 5f404f5d..59acab32 100644 --- a/packages/nightwatch-devtools/README.md +++ b/packages/nightwatch-devtools/README.md @@ -85,6 +85,10 @@ module.exports = { | `screencast` | `ScreencastOptions` | `{ enabled: false }` | Session video recording — live mode only (see [Screencast](#screencast)). | | `bidi` | `boolean` | `false` | Opt into WebDriver BiDi capture for browser console + JS exceptions + network. Requires `webSocketUrl: true` in your capabilities and a BiDi-capable chromedriver. When attached, the per-command Chrome perf-log network path is gated off so requests don't duplicate. | | `mode` | `'live' \| 'trace'` | `'live'` | `'live'` opens the DevTools UI window; `'trace'` skips the UI and writes a `trace-<sessionId>.zip` next to your `nightwatch.conf.cjs` at run end. See [Trace mode](../../README.md#-trace-mode-tracezip). | +| `screenshot` | `'off' \| 'on' \| 'only-on-failure'` | `'off'` | Per-test screenshot. Trace mode + `traceGranularity: 'test'` only. **Produce-only** — the PNG is written to the trace output dir and listed in the artifacts manifest; it is NOT attached inline to Allure (see note below). | +| `video` | `'off' \| TraceRetentionPolicy` | `'off'` | Per-test video slice, retained per the given policy (e.g. `'retain-on-failure'`). Trace mode + `traceGranularity: 'test'` only. Setting a non-`off` policy starts the screencast recorder itself — you do **not** also need `filmstrip` or `screencast.enabled`; the recorder runs continuously for the session and each test's slice is cut from it by wall-clock window. **Produce-only** — the `.webm` is written to the trace output dir and listed in the manifest; NOT attached inline to Allure. | + +> **Inline Allure attachment is not supported for Nightwatch.** Its official `nightwatch-allure` reporter is post-hoc (no live attach API), and `allure-js-commons`' `attachment()` no-ops in a Nightwatch run. So `screenshot`/`video` artifacts are *produced* (files + manifest) in the trace output dir but not attached to an Allure test. Per-test slicing (and therefore these artifacts) is meaningful for the cucumber and exports-object interfaces; the BDD `describe/it` interface collapses to session granularity, so the gate no-ops there. ```javascript globals: nightwatchDevtools({ diff --git a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts index 46684570..6ed8f26c 100644 --- a/packages/nightwatch-devtools/src/cucumber-lifecycle.ts +++ b/packages/nightwatch-devtools/src/cucumber-lifecycle.ts @@ -63,6 +63,7 @@ export interface CucumberLifecycleCtx extends TestSliceCtx { setCurrentTest(t: unknown): void recordAttempt(uid: string, specFile?: string): number recordOutcome(uid: string, state: TestStats['state']): void + emitTestArtifacts(uid: string | undefined, failed: boolean): Promise<void> } type MutStep = { @@ -241,6 +242,12 @@ export async function finalizeCucumberScenario( // Flush before the next attempt's attachScenarioToFeature overwrites this // scenario's suite (and thus its outcome) in the tree. flushTestSlice(ctx) + // Produce this scenario's per-test screenshot/video (core-gated to trace + + // `test`). `scenario` is the retry-stable test unit; null-scenario no-ops. + await ctx.emitTestArtifacts( + scenario?.uid, + scenarioState === TEST_STATE.FAILED + ) } catch (err) { log.error(`Failed to finalize Cucumber scenario: ${errorMessage(err)}`) } diff --git a/packages/nightwatch-devtools/src/index.ts b/packages/nightwatch-devtools/src/index.ts index 5f93d23b..c30f9fc4 100644 --- a/packages/nightwatch-devtools/src/index.ts +++ b/packages/nightwatch-devtools/src/index.ts @@ -10,6 +10,7 @@ import { errorMessage, finalizeTraceExport, flushRangeLogged, + resolveAdapterOutputDir, TestAttemptTracker, tracePolicyModeWarning, type SpecRange, @@ -17,6 +18,7 @@ import { type TraceExportContext } from '@wdio/devtools-core' import { buildTraceContext } from './trace-context.js' +import { emitTestArtifacts } from './test-artifacts.js' import { wireAssertCapture } from './helpers/assertCapture.js' import { stop as stopBackend } from '@wdio/devtools-backend' import { @@ -132,21 +134,29 @@ class NightwatchDevToolsPlugin { #bidiEnabled = false #bidiAttachAttempted = false + /** Wall-clock ms at the current test/scenario start — the lower bound of that + * test's action-snapshot and video-frame window (per-test artifact slicing). */ + #currentTestStartWallTime = 0 + // Nightwatch `--retries` and cross-worker reruns may reset this in-process // tracker; only retries that re-enter this process's start hook are counted. #attemptTracker = new TestAttemptTracker() constructor(options: DevToolsOptions = {}) { const mode = options.mode ?? 'live' - // Filmstrip drives the recorder in trace mode; bare screencast stays live-only. - const filmstrip = mode === 'trace' && options.filmstrip === true + // Filmstrip OR a produce-only per-test video drives the recorder in trace + // mode; bare screencast (with neither on) stays a live-only feature. + const wantRecorder = + mode === 'trace' && + (options.filmstrip === true || + (options.video !== undefined && options.video !== 'off')) const ignore = - mode === 'trace' && !filmstrip && options.screencast?.enabled === true + mode === 'trace' && !wantRecorder && options.screencast?.enabled === true if (ignore) { log.warn('trace mode: ignoring screencast option (live-mode feature)') } let screencast = ignore ? {} : (options.screencast ?? {}) - if (filmstrip) { + if (wantRecorder) { screencast = { ...(options.screencast ?? {}), enabled: true } } this.options = { @@ -159,7 +169,9 @@ class NightwatchDevToolsPlugin { traceFormat: options.traceFormat ?? 'zip', traceGranularity: options.traceGranularity ?? 'session', tracePolicy: options.tracePolicy ?? 'on', - filmstrip: options.filmstrip ?? false + filmstrip: options.filmstrip ?? false, + screenshot: options.screenshot ?? 'off', + video: options.video ?? 'off' } const policyWarning = tracePolicyModeWarning(options.tracePolicy, mode) if (policyWarning) { @@ -344,7 +356,8 @@ class NightwatchDevToolsPlugin { get flushedSpecs() { return self.#flushedSpecs }, - flushTraceRange: (range) => self.#flushSpecTrace(range) + flushTraceRange: (range) => self.#flushSpecTrace(range), + emitTestArtifacts: (uid, failed) => self.#emitTestArtifacts(uid, failed) } return this.#internals } @@ -386,13 +399,19 @@ class NightwatchDevToolsPlugin { } async #finalizeCurrentScreencast(): Promise<void> { - if (this.options.filmstrip && this.#screencastRecorder) { + // Preserve pre-reload frames for the trace filmstrip AND the produce-only + // per-test video — either can drive the recorder in trace mode now. + if ( + (this.options.filmstrip || this.options.video !== 'off') && + this.#screencastRecorder + ) { this.#filmstripFrames.push(...this.#screencastRecorder.frames) } await finalizeCurrentScreencast(this.#getInternals()) } async cucumberBefore(browser: NightwatchBrowser, pickle: CucumberPickle) { + this.#currentTestStartWallTime = Date.now() await cucumberLifecycleBefore(this.#getInternals(), browser, pickle) } @@ -480,6 +499,7 @@ class NightwatchDevToolsPlugin { if (this.#isCucumberRunner) { return } + this.#currentTestStartWallTime = Date.now() await this.#ensureSessionInitialized(browser) const currentTest = browser.currentTest as NightwatchCurrentTest | undefined @@ -545,6 +565,10 @@ class NightwatchDevToolsPlugin { await this.sessionCapturer.captureTrace(browser) // Flush this test's slice before the next test overwrites its outcome. flushTestSlice(this.#getInternals()) + const results = (browser.currentTest as NightwatchCurrentTest)?.results + const failed = + !!results && ((results.errors ?? 0) > 0 || (results.failed ?? 0) > 0) + await this.#emitTestArtifacts(this.#currentTestUid(), failed) } catch (err) { log.error(`Failed to capture trace: ${errorMessage(err)}`) } @@ -555,6 +579,43 @@ class NightwatchDevToolsPlugin { await closeOutTestcases(this.#getInternals(), browser) } + /** Directory for produced artifacts — next to the test file, else the config, + * else cwd (mirrors #traceContext's resolution). */ + get #outputDir(): string { + return resolveAdapterOutputDir({ + testFilePath: this.browserProxy?.getCurrentTestFullPath?.() ?? undefined, + configPath: this.#configPath + }) + } + + /** Produce (never attach — no live Allure API) this test's per-test screenshot + * and video slice. No-op outside trace mode + `test` granularity (core-gated). + * Shared by the per-test `afterEach` and the cucumber per-scenario finalize. */ + async #emitTestArtifacts( + uid: string | undefined, + failed: boolean + ): Promise<void> { + const attempt = uid ? this.#attemptTracker.attemptFor(uid) : undefined + await emitTestArtifacts({ + mode: this.options.mode, + granularity: this.options.traceGranularity, + screenshotPolicy: this.options.screenshot, + videoPolicy: this.options.video, + failed, + actionSnapshots: this.sessionCapturer.actionSnapshots, + frames: this.#screencastRecorder?.frames ?? this.#filmstripFrames, + startWallTime: this.#currentTestStartWallTime, + outcomes: uid ? this.#attemptTracker.forTest(uid, attempt) : [], + uid, + attempt, + sessionId: this.sessionCapturer?.metadata?.sessionId, + outputDir: this.#outputDir, + captureFormat: this.#screencastOptions.captureFormat, + onArtifact: (a) => this.#artifacts.push(a), + onLog: (level, msg) => log[level](msg) + }) + } + async after(browser?: NightwatchBrowser) { await this.#finalizeCurrentScreencast() try { diff --git a/packages/nightwatch-devtools/src/plugin-internals.ts b/packages/nightwatch-devtools/src/plugin-internals.ts index 4279d360..1d239fb4 100644 --- a/packages/nightwatch-devtools/src/plugin-internals.ts +++ b/packages/nightwatch-devtools/src/plugin-internals.ts @@ -94,4 +94,8 @@ export interface PluginInternals { readonly specRanges: SpecRange[] readonly flushedSpecs: Set<string> flushTraceRange(range: SpecRange): Promise<TraceArtifact | undefined> + + /** Produce (no attach) this test's per-test screenshot + video artifacts. + * Core-gated to trace mode + `test` granularity. */ + emitTestArtifacts(uid: string | undefined, failed: boolean): Promise<void> } diff --git a/packages/nightwatch-devtools/src/session-init.ts b/packages/nightwatch-devtools/src/session-init.ts index 8e485a21..9e5138cd 100644 --- a/packages/nightwatch-devtools/src/session-init.ts +++ b/packages/nightwatch-devtools/src/session-init.ts @@ -277,19 +277,26 @@ export async function finalizeCurrentScreencast( if (!ctx.screencastRecorder || !ctx.screencastSessionId) { return } - await finalizeScreencast({ - recorder: ctx.screencastRecorder, - sessionId: ctx.screencastSessionId, - filenamePrefix: 'nightwatch-video', - outputDir: resolveAdapterOutputDir({ - testFilePath: ctx.browserProxy?.getCurrentTestFullPath?.() ?? undefined, - configPath: ctx.configPath - }), - captureFormat: ctx.screencastOptions.captureFormat, - sendUpstream: (scope, data) => - ctx.sessionCapturer?.sendUpstream(scope, data), - onLog: (level, message) => log[level](message) - }) + if (ctx.mode === 'trace') { + // Trace mode: the per-test video is written by the produce path and the + // filmstrip frames are embedded in the trace itself, so stop the recorder + // without encoding a session .webm nothing references (orphan file). + await ctx.screencastRecorder.stop() + } else { + await finalizeScreencast({ + recorder: ctx.screencastRecorder, + sessionId: ctx.screencastSessionId, + filenamePrefix: 'nightwatch-video', + outputDir: resolveAdapterOutputDir({ + testFilePath: ctx.browserProxy?.getCurrentTestFullPath?.() ?? undefined, + configPath: ctx.configPath + }), + captureFormat: ctx.screencastOptions.captureFormat, + sendUpstream: (scope, data) => + ctx.sessionCapturer?.sendUpstream(scope, data), + onLog: (level, message) => log[level](message) + }) + } ctx.screencastRecorder = undefined ctx.screencastSessionId = undefined } diff --git a/packages/nightwatch-devtools/src/test-artifacts.ts b/packages/nightwatch-devtools/src/test-artifacts.ts new file mode 100644 index 00000000..b81af5e8 --- /dev/null +++ b/packages/nightwatch-devtools/src/test-artifacts.ts @@ -0,0 +1,92 @@ +/** + * Produce-only per-test artifacts (screenshot + video) for the Nightwatch + * plugin. + * + * Nightwatch has no live Allure attach API — its official `nightwatch-allure` + * reporter is post-hoc, and `allure-js-commons`' `attachment()` no-ops in a + * Nightwatch run (nothing wires the global test runtime). So we pass NO sink + * (`attach: undefined`), which the core treats as produce-only: it writes the + * file and records the manifest artifact via `onArtifact`, but skips the attach. + * + * Both the per-test (Mocha/Jasmine `afterEach`) and cucumber (per-scenario + * finalize) paths funnel through here so the two produce calls live once. Core + * gates both to trace mode + `test` granularity, so this no-ops otherwise. + */ + +import { + captureAndAttachScreenshot, + captureAndAttachVideo, + lastRenderedScreenshot, + type TestOutcome, + type TraceArtifact +} from '@wdio/devtools-core' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastFrame, + TraceGranularity, + TraceScreenshotPolicy, + TraceVideoPolicy +} from '@wdio/devtools-shared' + +export interface EmitTestArtifactsInput { + mode: DevToolsMode + granularity: TraceGranularity + screenshotPolicy: TraceScreenshotPolicy + videoPolicy: TraceVideoPolicy + failed: boolean + actionSnapshots: readonly ActionSnapshot[] + frames: readonly ScreencastFrame[] + startWallTime: number + /** This test's attempt slots (already scoped via `forTest`) for retention. */ + outcomes: TestOutcome[] + uid: string | undefined + attempt: number | undefined + sessionId: string | undefined + outputDir: string + captureFormat?: 'jpeg' | 'png' + onArtifact: (artifact: TraceArtifact) => void + onLog?: (level: 'info' | 'warn', message: string) => void +} + +/** + * Produce this test's screenshot then its video slice, both with `attach: + * undefined` (produce-only). Each core call self-gates on mode/granularity/ + * policy, so a non-trace run or a non-`test` granularity is a no-op. + */ +export async function emitTestArtifacts( + input: EmitTestArtifactsInput +): Promise<void> { + await captureAndAttachScreenshot({ + mode: input.mode, + granularity: input.granularity, + policy: input.screenshotPolicy, + failed: input.failed, + screenshotBase64: lastRenderedScreenshot( + input.actionSnapshots, + input.startWallTime + ), + sessionId: input.sessionId, + outputDir: input.outputDir, + testUid: input.uid, + attach: undefined, + onArtifact: input.onArtifact, + onLog: input.onLog + }) + await captureAndAttachVideo({ + mode: input.mode, + granularity: input.granularity, + policy: input.videoPolicy, + frames: input.frames, + startWallTime: input.startWallTime, + outcomes: input.outcomes, + attempt: input.attempt, + outputDir: input.outputDir, + testUid: input.uid, + sessionId: input.sessionId, + captureFormat: input.captureFormat, + attach: undefined, + onArtifact: input.onArtifact, + onLog: input.onLog + }) +} diff --git a/packages/nightwatch-devtools/src/types.ts b/packages/nightwatch-devtools/src/types.ts index d4c3f01e..89b2a9f1 100644 --- a/packages/nightwatch-devtools/src/types.ts +++ b/packages/nightwatch-devtools/src/types.ts @@ -21,10 +21,17 @@ export { type TestMetadataMap, type TraceGranularity, type TraceRetentionPolicy, + type TraceScreenshotPolicy, + type TraceVideoPolicy, type TraceLog } from '@wdio/devtools-shared' -import type { BaseDevToolsOptions, CommandLog } from '@wdio/devtools-shared' +import type { + BaseDevToolsOptions, + CommandLog, + TraceScreenshotPolicy, + TraceVideoPolicy +} from '@wdio/devtools-shared' export interface CommandStackFrame { command: string @@ -105,6 +112,16 @@ export interface DevToolsOptions extends BaseDevToolsOptions { * entries. Defaults to `false` — opt-in. */ bidi?: boolean + /** Per-test screenshot capture, produced to the trace output dir + manifest. + * `off` (default) | `on` | `only-on-failure`. Trace mode + + * `traceGranularity: 'test'` only. Produce-only: Nightwatch has no live + * Allure attach API, so artifacts are not attached inline. */ + screenshot?: TraceScreenshotPolicy + /** Per-test video (screencast slice) capture, produced to the trace output + * dir + manifest per the given retention policy. `off` (default) or a + * retention policy. Trace mode + `traceGranularity: 'test'` only. + * Produce-only (no inline Allure attach). */ + video?: TraceVideoPolicy } export interface NightwatchBrowser { diff --git a/packages/nightwatch-devtools/tests/testArtifacts.test.ts b/packages/nightwatch-devtools/tests/testArtifacts.test.ts new file mode 100644 index 00000000..32703392 --- /dev/null +++ b/packages/nightwatch-devtools/tests/testArtifacts.test.ts @@ -0,0 +1,134 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import type { TraceArtifact } from '@wdio/devtools-core' + +// Mock only the three core entry points the helper touches — we verify the +// produce-only WIRING (attach: undefined, onArtifact collection, gate values +// forwarded), not core internals. +vi.mock('@wdio/devtools-core', () => ({ + lastRenderedScreenshot: vi.fn(() => 'SCREENSHOT_B64'), + captureAndAttachScreenshot: vi.fn( + async (input: { + testUid?: string + onArtifact: (a: TraceArtifact) => void + }) => { + input.onArtifact({ + kind: 'screenshot', + path: '/out/screenshot.png', + scope: 'test', + key: input.testUid ?? '', + testUids: input.testUid ? [input.testUid] : [], + retained: true + }) + } + ), + captureAndAttachVideo: vi.fn( + async (input: { + testUid?: string + onArtifact: (a: TraceArtifact) => void + }) => { + input.onArtifact({ + kind: 'video', + path: '/out/video.webm', + scope: 'test', + key: input.testUid ?? '', + testUids: input.testUid ? [input.testUid] : [], + retained: true + }) + } + ) +})) + +import { + captureAndAttachScreenshot, + captureAndAttachVideo, + lastRenderedScreenshot +} from '@wdio/devtools-core' +import { + emitTestArtifacts, + type EmitTestArtifactsInput +} from '../src/test-artifacts.js' + +function makeInput(overrides: Partial<EmitTestArtifactsInput> = {}): { + input: EmitTestArtifactsInput + artifacts: TraceArtifact[] +} { + const artifacts: TraceArtifact[] = [] + const input: EmitTestArtifactsInput = { + mode: 'trace', + granularity: 'test', + screenshotPolicy: 'on', + videoPolicy: 'on', + failed: false, + actionSnapshots: [], + frames: [], + startWallTime: 1000, + outcomes: [{ uid: 'uid-1', attempt: 0, state: 'passed' }], + uid: 'uid-1', + attempt: 0, + sessionId: 'session-abcdef', + outputDir: '/out', + captureFormat: 'jpeg', + onArtifact: (a) => artifacts.push(a), + ...overrides + } + return { input, artifacts } +} + +describe('emitTestArtifacts — produce-only per-test artifacts', () => { + beforeEach(() => vi.clearAllMocks()) + + it('drives both core produce fns with NO sink (attach: undefined)', async () => { + const { input } = makeInput() + await emitTestArtifacts(input) + + expect(captureAndAttachScreenshot).toHaveBeenCalledTimes(1) + expect(captureAndAttachVideo).toHaveBeenCalledTimes(1) + expect(captureAndAttachScreenshot).toHaveBeenCalledWith( + expect.objectContaining({ attach: undefined }) + ) + expect(captureAndAttachVideo).toHaveBeenCalledWith( + expect.objectContaining({ attach: undefined }) + ) + }) + + it('collects the produced artifacts via onArtifact (manifest path)', async () => { + const { input, artifacts } = makeInput() + await emitTestArtifacts(input) + + expect(artifacts.map((a) => a.kind)).toEqual(['screenshot', 'video']) + expect(artifacts.every((a) => a.scope === 'test')).toBe(true) + expect(artifacts.every((a) => a.key === 'uid-1')).toBe(true) + }) + + it('forwards the gate + policy values and the sliced outcomes', async () => { + const { input } = makeInput({ + screenshotPolicy: 'only-on-failure', + failed: true, + attempt: 1 + }) + await emitTestArtifacts(input) + + expect(lastRenderedScreenshot).toHaveBeenCalledWith([], 1000) + expect(captureAndAttachScreenshot).toHaveBeenCalledWith( + expect.objectContaining({ + mode: 'trace', + granularity: 'test', + policy: 'only-on-failure', + failed: true, + screenshotBase64: 'SCREENSHOT_B64', + sessionId: 'session-abcdef', + outputDir: '/out', + testUid: 'uid-1' + }) + ) + expect(captureAndAttachVideo).toHaveBeenCalledWith( + expect.objectContaining({ + policy: 'on', + outcomes: [{ uid: 'uid-1', attempt: 0, state: 'passed' }], + attempt: 1, + captureFormat: 'jpeg', + testUid: 'uid-1' + }) + ) + }) +}) diff --git a/packages/selenium-devtools/README.md b/packages/selenium-devtools/README.md index 89d099a9..ad8cfd22 100644 --- a/packages/selenium-devtools/README.md +++ b/packages/selenium-devtools/README.md @@ -346,6 +346,8 @@ See [Trace mode](../../README.md#-trace-mode-tracezip) in the root README for th | `headless` | `boolean` | `false` | Run the **test** browser headless (injects `--headless=old`). The DevTools UI window is unaffected. | | `screencast` | `ScreencastOptions` | `{ enabled: false }` | Per-session `.webm` video recording. See sub-options below. | | `rerunCommand` | `string` | auto | Command template for per-test rerun. `{{testName}}` is substituted. Auto-derived from runner argv if omitted. | +| `screenshot` | `'off' \| 'on' \| 'only-on-failure'` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screenshot, attached inline to Allure (`image/png`) via `allure-js-commons` when an Allure runner adapter is active. | +| `video` | `'off' \| TraceRetentionPolicy` | `'off'` | Trace mode + `traceGranularity: 'test'`. Per-test screencast video, retained per the given policy, attached inline to Allure (`video/webm`) via `allure-js-commons` when an Allure runner adapter is active. | `ScreencastOptions`: diff --git a/packages/selenium-devtools/package.json b/packages/selenium-devtools/package.json index 8368e44d..99a61f9f 100644 --- a/packages/selenium-devtools/package.json +++ b/packages/selenium-devtools/package.json @@ -33,6 +33,7 @@ "prepublishOnly": "pnpm build", "example": "pnpm example:cucumber", "example:mocha": "mocha --require @wdio/selenium-devtools --timeout 60000 ../../examples/selenium/mocha-test/test/example.js", + "example:mocha:allure": "mocha --require @wdio/selenium-devtools --reporter allure-mocha --reporter-options resultsDir=../../examples/selenium/mocha-test/allure-results --timeout 60000 ../../examples/selenium/mocha-test/test/example.js", "example:jest": "NODE_OPTIONS=--experimental-vm-modules jest --config ../../examples/selenium/jest-test/jest.config.json", "example:cucumber": "cucumber-js --config ../../examples/selenium/cucumber-test/cucumber.json" }, @@ -65,6 +66,8 @@ "@types/ws": "^8.18.1", "@wdio/devtools-core": "workspace:^", "@wdio/devtools-shared": "workspace:^", + "allure-js-commons": "^3.0.0", + "allure-mocha": "^3.0.0", "chromedriver": "^150.0.0", "jest": "^30.4.2", "mocha": "^11.7.6", @@ -74,6 +77,12 @@ "vitest": "^4.1.8" }, "peerDependencies": { + "allure-js-commons": "^3.0.0", "selenium-webdriver": ">=4.8.0" + }, + "peerDependenciesMeta": { + "allure-js-commons": { + "optional": true + } } } diff --git a/packages/selenium-devtools/src/allure.ts b/packages/selenium-devtools/src/allure.ts new file mode 100644 index 00000000..92f4ed24 --- /dev/null +++ b/packages/selenium-devtools/src/allure.ts @@ -0,0 +1,62 @@ +// Selenium Allure binding: resolve the optional `allure-js-commons` into a core +// `AllureAttachSink`. The capture/attach orchestration + content-type convention +// live in `@wdio/devtools-core`; this file owns only the runtime-agnostic +// optional-dependency dance and `allure-js-commons`' functional `attachment()`. + +import type { AllureAttachSink } from '@wdio/devtools-core' + +/** The one `allure-js-commons` function we use. Typed locally so the optional + * peer dependency never becomes a build-time type dependency. */ +interface AllureCommonsModule { + attachment( + name: string, + content: Buffer | string, + contentType: string + ): void | Promise<void> +} + +// undefined = unresolved; null = runtime present but allure-js-commons absent. +let cachedSink: AllureAttachSink | null | undefined + +// A non-literal specifier keeps TypeScript and tsup/esbuild from resolving this +// optional peer dependency at build time — `import()` of a variable is typed +// `any` and left as a runtime import that no-ops (caught below) when absent. +const ALLURE_COMMONS_SPECIFIER = 'allure-js-commons' + +/** + * Resolve the Allure attach sink, or undefined when Allure isn't active. Only + * binds a sink when a runner adapter has installed `globalThis.allureTestRuntime`: + * without it `allure-js-commons`' `attachment()` is a noop that logs a warning on + * every call, so a plain (non-Allure) run would be spammed. The resolved sink is + * memoized once found; while the runtime is absent this returns undefined + * WITHOUT caching, so a later call after Allure activates can still resolve. The + * sink must be called from within a per-test hook so Allure attaches to the + * right test. + */ +export async function getAllureSink(): Promise<AllureAttachSink | undefined> { + if (cachedSink !== undefined) { + return cachedSink ?? undefined + } + if (!(globalThis as { allureTestRuntime?: unknown }).allureTestRuntime) { + return undefined + } + try { + const mod = await import(/* @vite-ignore */ ALLURE_COMMONS_SPECIFIER) + const commons = ( + typeof mod.attachment === 'function' ? mod : mod.default + ) as AllureCommonsModule | undefined + cachedSink = + commons && typeof commons.attachment === 'function' + ? (name, content, type) => commons.attachment(name, content, type) + : null + } catch { + // Optional peer dependency not installed — attaching is a no-op. + cachedSink = null + } + return cachedSink ?? undefined +} + +/** Reset the memoized sink — test seam only. */ +export function resetAllureSinkCache(): void { + cachedSink = undefined +} diff --git a/packages/selenium-devtools/src/helpers/testManager.ts b/packages/selenium-devtools/src/helpers/testManager.ts index 09c741c7..08d6e687 100644 --- a/packages/selenium-devtools/src/helpers/testManager.ts +++ b/packages/selenium-devtools/src/helpers/testManager.ts @@ -3,7 +3,8 @@ import { createTestStats, stampTestEnd, TestAttemptTracker, - type RetryOutcomeView + type RetryOutcomeView, + type TestOutcome } from '@wdio/devtools-core' import { DEFAULTS, TEST_STATE } from '../constants.js' import type { SuiteStats, TestStats } from '../types.js' @@ -32,6 +33,10 @@ export class TestManager { // Retry-stable uid of the in-progress marked test, so endCurrent can stamp // this attempt's outcome onto the same ledger entry recordStart opened. #currentRetryStableUid: string | undefined + // Retry-stable uid of the most-recently-started marked test, retained past + // endCurrent so the per-test artifact path (which runs after the test ends) + // can read this test's attempts from the ledger for video retention. + #lastRetryStableUid: string | undefined /** Set true the first time the user calls startMarkedTest. Once true we * never auto-create the synthetic session test — orphan commands attach * to the most-recently-marked test instead. */ @@ -146,6 +151,7 @@ export class TestManager { file ) this.#currentRetryStableUid = retryStableUid + this.#lastRetryStableUid = retryStableUid test.retries = opts.attempt ?? heuristicAttempt log.info( `Started marked test "${name}" (callSource: ${opts.callSource || 'n/a'})` @@ -190,6 +196,17 @@ export class TestManager { return this.#attemptTracker } + /** The just-ended marked test's attempts, keyed by its retry-stable uid (the + * ledger key), for the per-test video retention decision. The artifact + * filename uses the marked uid, but retention groups a test's attempts under + * the retry-stable key — this bridges the two. Empty when no marked test ran. */ + lastTestOutcomes(): TestOutcome[] { + if (this.#lastRetryStableUid === undefined) { + return [] + } + return this.#attemptTracker.forTest(this.#lastRetryStableUid) + } + /** Called when the driver session is closing (process exit / quit). */ finalizeSession(): void { if (this.#currentTest) { diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index dcc19fd3..a622b695 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -24,7 +24,11 @@ import { recordTraceBoundary, flushCurrentTestTrace } from './session-lifecycle.js' -import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' +import type { + AllureAttachSink, + SpecRange, + TraceArtifact +} from '@wdio/devtools-core' import { startTest as tmStartTest, endTest as tmEndTest, @@ -45,9 +49,15 @@ import { import { findFreePort } from './helpers/utils.js' import { RetryTracker, + attachTraceArtifact, + captureAndAttachScreenshot, + captureAndAttachVideo, errorMessage, + lastRenderedScreenshot, + resolveAdapterOutputDir, tracePolicyModeWarning } from '@wdio/devtools-core' +import { getAllureSink } from './allure.js' import { tryRegisterRunnerHooks } from './runnerHooks.js' import { patchNodeAssert } from './assertPatcher.js' import { @@ -66,6 +76,8 @@ import { type TraceFormat, type TraceGranularity, type TraceRetentionPolicy, + type TraceScreenshotPolicy, + type TraceVideoPolicy, type SeleniumDriverLike, type TestStats } from './types.js' @@ -136,6 +148,13 @@ class SeleniumDevToolsPlugin { * before the recorder is nulled so the finalize context is never blank. */ #filmstripFrames: ScreencastFrame[] = [] + /** Wall-clock ms at the current test/scenario start — the lower bound of that + * test's video frame window and screenshot lookup (per-test slicing). */ + #currentTestStartWallTime = Date.now() + + /** Allure attach sink, resolved once lazily at the first per-test end. */ + #allureSinkResolved?: Promise<AllureAttachSink | undefined> + constructor(options: DevToolsOptions = {}) { this.#options = { port: options.port ?? 3000, @@ -149,7 +168,9 @@ class SeleniumDevToolsPlugin { traceFormat: options.traceFormat ?? 'zip', traceGranularity: options.traceGranularity ?? 'session', tracePolicy: options.tracePolicy ?? 'on', - filmstrip: options.filmstrip ?? false + filmstrip: options.filmstrip ?? false, + screenshot: options.screenshot ?? 'off', + video: options.video ?? 'off' } const policyWarning = tracePolicyModeWarning( options.tracePolicy, @@ -278,6 +299,9 @@ class SeleniumDevToolsPlugin { return this.#uiReadyPromise } + // Declarative option-application mapper — one guarded assignment per option; + // splitting it purely to satisfy the line rule hurts readability. + // eslint-disable-next-line max-lines-per-function configure( opts: { rerunCommand?: string @@ -289,6 +313,8 @@ class SeleniumDevToolsPlugin { traceFormat?: TraceFormat traceGranularity?: TraceGranularity tracePolicy?: TraceRetentionPolicy + screenshot?: TraceScreenshotPolicy + video?: TraceVideoPolicy } = {} ) { if ('rerunCommand' in opts) { @@ -316,6 +342,12 @@ class SeleniumDevToolsPlugin { if (opts.tracePolicy) { this.#options.tracePolicy = opts.tracePolicy } + if (opts.screenshot) { + this.#options.screenshot = opts.screenshot + } + if (opts.video) { + this.#options.video = opts.video + } if (opts.screencast) { if (this.#options.mode === 'trace' && opts.screencast.enabled) { log.warn('trace mode: ignoring screencast option (live-mode feature)') @@ -478,22 +510,110 @@ class SeleniumDevToolsPlugin { /** Public API: start a marked test. */ startTest(name: string, meta: StartTestMeta = {}) { + this.#currentTestStartWallTime = Date.now() tmStartTest(this.#getInternals(), name, meta) recordTraceBoundary(this.#getInternals(), meta.file) } - endTest(state: TestStats['state'] = 'passed') { + // Async + awaited by the runner's afterEach so the per-test artifact emit + // (trace slice + screenshot + video attach) completes while the test is still + // open; otherwise allure closes the test before attachment() runs. + async endTest(state: TestStats['state'] = 'passed') { + const ended = this.#testManager?.getCurrentTest() ?? null tmEndTest(this.#getInternals(), state) - flushCurrentTestTrace(this.#getInternals()) + const flushed = flushCurrentTestTrace(this.#getInternals()) + await this.#emitTestArtifacts(ended, state === 'failed', flushed) } startScenario(name: string, meta: StartScenarioMeta = {}) { + this.#currentTestStartWallTime = Date.now() tmStartScenario(this.#getInternals(), name, meta) recordTraceBoundary(this.#getInternals(), meta.file) } - endScenario(state: TestStats['state'] = 'passed') { + async endScenario(state: TestStats['state'] = 'passed') { + const ended = this.#testManager?.getCurrentTest() ?? null tmEndScenario(this.#getInternals(), state) + const flushed = flushCurrentTestTrace(this.#getInternals()) + await this.#emitTestArtifacts(ended, state === 'failed', flushed) + } + + /** Lazily resolve the Allure attach sink once (produce-only when Allure is + * inactive), reused across every per-test attach this run. */ + #allureSink(): Promise<AllureAttachSink | undefined> { + return (this.#allureSinkResolved ??= getAllureSink()) + } + + /** Snapshot every per-test capture input synchronously at test end. Selenium's + * runner hooks fire-and-forget the emit, so reading these after an await would + * race the next test overwriting the fields; frames/outcomes are copied + * because their backing arrays keep mutating after this test ends. */ + #perTestArtifactInputs(endedTest: TestStats | null) { + const startWallTime = this.#currentTestStartWallTime + return { + startWallTime, + sessionId: this.#sessionId, + testUid: endedTest?.uid, + attempt: endedTest?.retries, + screenshotBase64: lastRenderedScreenshot( + this.#actionSnapshots, + startWallTime + ), + frames: this.#screencast ? [...this.#screencast.frames] : undefined, + outcomes: (this.#testManager?.lastTestOutcomes() ?? []).map((o) => ({ + ...o + })), + captureFormat: this.#screencastOptions.captureFormat, + outputDir: resolveAdapterOutputDir({ testFilePath: this.#testFilePath }) + } + } + + /** At a test/scenario end, while the runner's per-test hook is still open: + * attach the just-flushed trace slice to Allure, then capture + attach the + * per-test screenshot and video per their policies. Each part no-ops when its + * feature is off (all gate on trace mode + `test` granularity in core). */ + async #emitTestArtifacts( + endedTest: TestStats | null, + failed: boolean, + flushed: Promise<TraceArtifact | undefined> + ): Promise<void> { + const inp = this.#perTestArtifactInputs(endedTest) + const onArtifact = (a: TraceArtifact) => this.#artifacts.push(a) + const onLog = (level: 'info' | 'warn', msg: string) => log[level](msg) + const attach = await this.#allureSink() + const artifact = await flushed + if (artifact) { + await attachTraceArtifact(artifact, attach, onLog) + } + await captureAndAttachScreenshot({ + mode: this.#options.mode, + granularity: this.#options.traceGranularity, + policy: this.#options.screenshot, + failed, + screenshotBase64: inp.screenshotBase64, + sessionId: inp.sessionId, + outputDir: inp.outputDir, + testUid: inp.testUid, + attach, + onArtifact, + onLog + }) + await captureAndAttachVideo({ + mode: this.#options.mode, + granularity: this.#options.traceGranularity, + policy: this.#options.video, + frames: inp.frames, + startWallTime: inp.startWallTime, + outcomes: inp.outcomes, + attempt: inp.attempt, + outputDir: inp.outputDir, + testUid: inp.testUid, + sessionId: inp.sessionId, + captureFormat: inp.captureFormat, + attach, + onArtifact, + onLog + }) } #flushPendingTestActions() { @@ -637,9 +757,9 @@ function registerHooks() { attempt ) ), - onTestEnd: (state) => { - plugin.endTest(state === 'pending' ? 'skipped' : state) - }, + // Return the promise so the runner hook awaits the full attach chain. + onTestEnd: (state) => + plugin.endTest(state === 'pending' ? 'skipped' : state), onScenarioStart: ( name, file, @@ -654,9 +774,8 @@ function registerHooks() { featureCallSource }) }, - onScenarioEnd: (state) => { - plugin.endScenario(state === 'pending' ? 'skipped' : state) - }, + onScenarioEnd: (state) => + plugin.endScenario(state === 'pending' ? 'skipped' : state), onTestRunComplete: () => { void plugin.finalizeTestRun() } @@ -684,6 +803,9 @@ export const DevTools = { mode?: DevToolsMode traceFormat?: TraceFormat traceGranularity?: TraceGranularity + tracePolicy?: TraceRetentionPolicy + screenshot?: TraceScreenshotPolicy + video?: TraceVideoPolicy }) => plugin.configure(opts), startTest: (name: string, meta?: { file?: string }) => plugin.startTest(name, meta), diff --git a/packages/selenium-devtools/src/runnerHooks/cucumber.ts b/packages/selenium-devtools/src/runnerHooks/cucumber.ts index 86f8d3c2..31febfc6 100644 --- a/packages/selenium-devtools/src/runnerHooks/cucumber.ts +++ b/packages/selenium-devtools/src/runnerHooks/cucumber.ts @@ -12,7 +12,7 @@ const log = logger('@wdio/selenium-devtools:runnerHooks:cucumber') type CucumberModule = Record<string, unknown> & { Before?: (fn: (testCase: unknown) => void) => void - After?: (fn: (testCase: unknown) => void) => void + After?: (fn: (testCase: unknown) => void | Promise<void>) => void BeforeAll?: (fn: () => void) => void AfterAll?: (fn: () => void) => void BeforeStep?: (fn: (arg: unknown) => void) => void @@ -309,11 +309,11 @@ function handleScenarioStart( ) } -function handleScenarioEnd( +async function handleScenarioEnd( testCase: CucumberTestCase, counters: RunCounters, callbacks: RunnerHookCallbacks -): void { +): Promise<void> { const state = mapCucumberStatus(String(testCase?.result?.status ?? '')) const scenarioState: ScenarioState = state === 'skipped' ? 'pending' : state const icon = @@ -326,7 +326,7 @@ function handleScenarioEnd( } else { counters.pending++ } - callbacks.onScenarioEnd?.(scenarioState) + await callbacks.onScenarioEnd?.(scenarioState) } function registerScenarioHooks( @@ -347,7 +347,9 @@ function registerScenarioHooks( callbacks ) ) - After((testCase) => + // Async so cucumber awaits the per-scenario artifact emit while the scenario + // is still open — a fire-and-forget attach would land after allure closed it. + After(async (testCase) => handleScenarioEnd(testCase as CucumberTestCase, counters, callbacks) ) } diff --git a/packages/selenium-devtools/src/runnerHooks/jest.ts b/packages/selenium-devtools/src/runnerHooks/jest.ts index 24477e06..8dabd583 100644 --- a/packages/selenium-devtools/src/runnerHooks/jest.ts +++ b/packages/selenium-devtools/src/runnerHooks/jest.ts @@ -225,7 +225,9 @@ function registerJestAfterEach( state: JestState, callbacks: RunnerHookCallbacks ): void { - g.afterEach!(() => { + // Async so jest/vitest awaits the per-test artifact emit while the test is + // still open — a fire-and-forget attach would land after allure closed it. + g.afterEach!(async () => { const expectState = g.expect!.getState!() as { suppressedErrors?: unknown[] currentTestName?: string @@ -258,7 +260,7 @@ function registerJestAfterEach( } else { state.testsFailed++ } - callbacks.onTestEnd(finalState) + await callbacks.onTestEnd(finalState) }) } diff --git a/packages/selenium-devtools/src/runnerHooks/mocha.ts b/packages/selenium-devtools/src/runnerHooks/mocha.ts index e7b8a616..2f90eeca 100644 --- a/packages/selenium-devtools/src/runnerHooks/mocha.ts +++ b/packages/selenium-devtools/src/runnerHooks/mocha.ts @@ -7,7 +7,9 @@ const log = logger('@wdio/selenium-devtools:runnerHooks:mocha') type MochaGlobals = { beforeEach?: (fn: (this: { currentTest?: MochaTestCtx }) => void) => void - afterEach?: (fn: (this: { currentTest?: MochaTestCtx }) => void) => void + afterEach?: ( + fn: (this: { currentTest?: MochaTestCtx }) => void | Promise<void> + ) => void before?: (fn: () => void) => void after?: (fn: () => void) => void } @@ -108,8 +110,10 @@ function resolveMochaState( function makeMochaAfterEach( counters: MochaCounters, callbacks: RunnerHookCallbacks -): (this: { currentTest?: MochaTestCtx }) => void { - return function (this: { currentTest?: MochaTestCtx }) { +): (this: { currentTest?: MochaTestCtx }) => Promise<void> { + // Async so mocha awaits the per-test artifact emit while the test is still + // open — a fire-and-forget attach would land after allure closed the test. + return async function (this: { currentTest?: MochaTestCtx }) { const test = this?.currentTest const state = resolveMochaState(test) const icon = state === 'passed' ? '✓' : state === 'failed' ? '✗' : '○' @@ -123,7 +127,7 @@ function makeMochaAfterEach( } else { counters.pending++ } - callbacks.onTestEnd(state) + await callbacks.onTestEnd(state) } } diff --git a/packages/selenium-devtools/src/session-lifecycle.ts b/packages/selenium-devtools/src/session-lifecycle.ts index 2f203dce..c426e687 100644 --- a/packages/selenium-devtools/src/session-lifecycle.ts +++ b/packages/selenium-devtools/src/session-lifecycle.ts @@ -41,7 +41,8 @@ import type { SeleniumDriverLike, TraceFormat, TraceGranularity, - TraceRetentionPolicy + TraceRetentionPolicy, + TraceVideoPolicy } from './types.js' import type { TestManager } from './helpers/testManager.js' @@ -59,6 +60,7 @@ export interface SessionLifecycleCtx { traceGranularity?: TraceGranularity tracePolicy?: TraceRetentionPolicy filmstrip?: boolean + video?: TraceVideoPolicy } readonly screencastOptions: ScreencastOptions readonly runner: string @@ -161,6 +163,19 @@ function ctxPluginRef(ctx: SessionLifecycleCtx): unknown { return (ctx as unknown as Record<symbol, unknown>)[PLUGIN_REF] } +/** Record a screencast this driver? Live mode: `screencast.enabled`. Trace mode: + * a non-`off` per-test `video` policy or `filmstrip` — both feed the recorder + * even though live screencast is off in trace mode (parity with the service). */ +function shouldRecordScreencast(ctx: SessionLifecycleCtx): boolean { + if (ctx.options.mode === 'trace') { + return ( + (!!ctx.options.video && ctx.options.video !== 'off') || + !!ctx.options.filmstrip + ) + } + return !!ctx.screencastOptions.enabled +} + async function initPerDriverCapture( ctx: SessionLifecycleCtx, driver: SeleniumDriverLike, @@ -187,11 +202,8 @@ async function initPerDriverCapture( ctx.sessionCapturer.sendUpstream('metadata', metadata) } - // Parallel — serial attach misses frames on fast tests. Trace-mode filmstrip - // needs the same recorder even though live screencast is off in trace mode. - const wantScreencast = - ctx.screencastOptions.enabled || - (ctx.options.mode === 'trace' && !!ctx.options.filmstrip) + // Parallel — serial attach misses frames on fast tests. + const wantScreencast = shouldRecordScreencast(ctx) const screencastPromise = wantScreencast ? (async () => { try { @@ -223,18 +235,25 @@ async function initPerDriverCapture( export async function onDriverEnd(ctx: SessionLifecycleCtx): Promise<void> { if (ctx.screencast && ctx.sessionId) { - await finalizeScreencast({ - recorder: ctx.screencast, - sessionId: ctx.sessionId, - filenamePrefix: 'selenium-video', - outputDir: resolveAdapterOutputDir({ - testFilePath: ctx.testFilePath - }), - captureFormat: ctx.screencastOptions.captureFormat, - sendUpstream: (scope, data) => - ctx.sessionCapturer?.sendUpstream(scope, data), - onLog: (level, message) => log[level](message) - }) + if (ctx.options.mode === 'trace') { + // Trace mode: the video is emitted per-test (sliced in #emitTestArtifacts) + // and there's no dashboard to receive a session recording — just stop the + // recorder to release resources; never encode an orphan session webm. + await ctx.screencast.stop() + } else { + await finalizeScreencast({ + recorder: ctx.screencast, + sessionId: ctx.sessionId, + filenamePrefix: 'selenium-video', + outputDir: resolveAdapterOutputDir({ + testFilePath: ctx.testFilePath + }), + captureFormat: ctx.screencastOptions.captureFormat, + sendUpstream: (scope, data) => + ctx.sessionCapturer?.sendUpstream(scope, data), + onLog: (level, message) => log[level](message) + }) + } } // Drain this driver's frames into the run-wide buffer while the recorder is // still alive — the finalize context is built after screencast is nulled. @@ -449,15 +468,19 @@ function recordTestBoundary( * endTest has finalized its state so collectSuiteTestMetadata sees the final * outcome. flushRangeTrace dedupes by key, so finalizeTraceExport won't * re-write it; the promise is tracked so finalize awaits the last write. + * Returns the produced artifact so the plugin can attach the retained slice to + * Allure within the still-open per-test hook; undefined when nothing flushed. */ -export function flushCurrentTestTrace(ctx: SessionLifecycleCtx): void { +export function flushCurrentTestTrace( + ctx: SessionLifecycleCtx +): Promise<TraceArtifact | undefined> { if (ctx.options.traceGranularity !== 'test' || !ctx.sessionCapturer) { - return + return Promise.resolve(undefined) } const sessionId = ctx.sessionCapturer.metadata?.sessionId const currentRange = findFlushableRange(ctx.specRanges) if (!sessionId || currentRange?.testUid === undefined) { - return + return Promise.resolve(undefined) } const flush = flushRangeLogged( buildTraceExportContext( @@ -469,6 +492,7 @@ export function flushCurrentTestTrace(ctx: SessionLifecycleCtx): void { currentRange ) ctx.traceFlushes.push(flush) + return flush } async function writeTraceIfNeeded( diff --git a/packages/selenium-devtools/src/types.ts b/packages/selenium-devtools/src/types.ts index e7aceb8f..925ec86b 100644 --- a/packages/selenium-devtools/src/types.ts +++ b/packages/selenium-devtools/src/types.ts @@ -17,7 +17,9 @@ export { type TestMetadataMap, type TraceFormat, type TraceGranularity, - type TraceRetentionPolicy + type TraceRetentionPolicy, + type TraceScreenshotPolicy, + type TraceVideoPolicy } from '@wdio/devtools-shared' export interface DevToolsOptions extends BaseDevToolsOptions { @@ -33,11 +35,24 @@ export interface DevToolsOptions extends BaseDevToolsOptions { * is unaffected. Defaults to false to preserve user-supplied options. */ headless?: boolean + /** Per-test screenshot capture, recorded in the trace artifacts and attached + * inline to Allure. `off` (default) | `on` | `only-on-failure`. Trace mode + + * `traceGranularity: 'test'` only. */ + screenshot?: TraceScreenshotPolicy + /** Per-test video (screencast) capture, retained per the given policy and + * attached inline to Allure. `off` (default) or a retention policy. Trace + * mode + `traceGranularity: 'test'` only. */ + video?: TraceVideoPolicy } // ScreencastFrame, ScreencastOptions hoisted to @wdio/devtools-shared; re-exported // here for backwards compatibility with existing selenium-internal imports. -import type { BaseDevToolsOptions, TestStatus } from '@wdio/devtools-shared' +import type { + BaseDevToolsOptions, + TestStatus, + TraceScreenshotPolicy, + TraceVideoPolicy +} from '@wdio/devtools-shared' export type { ScreencastFrame, ScreencastOptions } from '@wdio/devtools-shared' /** @@ -134,7 +149,9 @@ export interface RunnerHookCallbacks { // `_currentRetry`); undefined leaves the heuristic tracker to decide. attempt?: number ) => void - onTestEnd: (state: Exclude<TestStatus, 'running'>) => void + // Async so the runner's afterEach/After awaits the full per-test artifact + // emit (trace slice + screenshot + video attach) before Allure closes the test. + onTestEnd: (state: Exclude<TestStatus, 'running'>) => void | Promise<void> // Cucumber-only: scenario boundary creates a sub-suite under the feature // rootSuite; subsequent onTestStart/onTestEnd attach as Gherkin steps inside. onScenarioStart?: ( @@ -144,7 +161,9 @@ export interface RunnerHookCallbacks { featureName?: string, featureCallSource?: string ) => void - onScenarioEnd?: (state: Exclude<TestStatus, 'running'>) => void + onScenarioEnd?: ( + state: Exclude<TestStatus, 'running'> + ) => void | Promise<void> // Fires from the runner's after-all hook so the dashboard suite header // updates without waiting for process exit. onTestRunComplete?: (summary: { diff --git a/packages/selenium-devtools/tests/allure.test.ts b/packages/selenium-devtools/tests/allure.test.ts new file mode 100644 index 00000000..f8374e35 --- /dev/null +++ b/packages/selenium-devtools/tests/allure.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' + +const attachment = vi.fn() +vi.mock('allure-js-commons', () => ({ + attachment, + default: { attachment } +})) + +import { getAllureSink, resetAllureSinkCache } from '../src/allure.js' + +type GlobalWithRuntime = { allureTestRuntime?: unknown } + +function clearRuntime() { + delete (globalThis as GlobalWithRuntime).allureTestRuntime +} + +describe('getAllureSink (selenium)', () => { + beforeEach(() => { + attachment.mockReset() + resetAllureSinkCache() + clearRuntime() + }) + + afterEach(clearRuntime) + + it('resolves a sink that forwards to allure-js-commons.attachment when Allure is active', async () => { + ;(globalThis as GlobalWithRuntime).allureTestRuntime = {} + const sink = await getAllureSink() + expect(sink).toBeTypeOf('function') + const content = Buffer.from('zip-bytes') + await sink!('trace-abc.zip', content, 'application/zip') + expect(attachment).toHaveBeenCalledOnce() + expect(attachment).toHaveBeenCalledWith( + 'trace-abc.zip', + content, + 'application/zip' + ) + }) + + it('returns undefined (produce-only, no attachment call) when no Allure runtime is active', async () => { + const sink = await getAllureSink() + expect(sink).toBeUndefined() + expect(attachment).not.toHaveBeenCalled() + }) + + it('memoizes the resolved sink across calls', async () => { + ;(globalThis as GlobalWithRuntime).allureTestRuntime = {} + const first = await getAllureSink() + const second = await getAllureSink() + expect(first).toBe(second) + }) +}) diff --git a/packages/service/src/allure.ts b/packages/service/src/allure.ts index ce415529..5fdf32c7 100644 --- a/packages/service/src/allure.ts +++ b/packages/service/src/allure.ts @@ -1,28 +1,9 @@ -// WDIO Allure glue: attach retained trace/video artifacts to the current Allure -// test so a failed run's trace travels with the report. Isolated here so -// index.ts stays free of the optional-dependency dance and the content-type -// convention lives in one place. +// WDIO Allure binding: resolve the optional @wdio/allure-reporter into a core +// `AllureAttachSink`. The capture/attach orchestration + content-type convention +// now live in `@wdio/devtools-core`; this file owns only the WDIO-specific +// optional-dependency dance and the reporter's `addAttachment` shape. -import fs from 'node:fs/promises' -import { basename } from 'node:path' -import logger from '@wdio/logger' -import type { TraceArtifact } from '@wdio/devtools-core' - -const log = logger('@wdio/devtools-service') - -// Attach the trace as a plain zip download — the viewer stays the user's choice -// (our `show-trace`, or any compatible viewer), rather than routing them into a -// third-party viewer Allure would open for a trace-specific content type. Video -// uses video/webm so Allure renders it inline. -const TRACE_CONTENT_TYPE = 'application/zip' -const VIDEO_CONTENT_TYPE = 'video/webm' -const SCREENSHOT_CONTENT_TYPE = 'image/png' - -const CONTENT_TYPE_BY_KIND: Record<TraceArtifact['kind'], string> = { - trace: TRACE_CONTENT_TYPE, - video: VIDEO_CONTENT_TYPE, - screenshot: SCREENSHOT_CONTENT_TYPE -} +import type { AllureAttachSink } from '@wdio/devtools-core' /** The one @wdio/allure-reporter method we use. Typed locally so the optional * peer dependency never becomes a build-time type dependency. */ @@ -31,66 +12,39 @@ interface AllureReporterModule { } // undefined = not yet probed; null = probed and unavailable. -let cachedReporter: AllureReporterModule | null | undefined +let cachedSink: AllureAttachSink | null | undefined // A non-literal specifier keeps TypeScript and the bundler from resolving this // optional peer dependency at build time — `import()` of a variable is typed // `any` and left as a runtime import that no-ops (caught below) when absent. const ALLURE_REPORTER_SPECIFIER = '@wdio/allure-reporter' -async function loadAllureReporter(): Promise<AllureReporterModule | null> { - if (cachedReporter !== undefined) { - return cachedReporter +/** + * Resolve the Allure attach sink, or undefined when @wdio/allure-reporter isn't + * installed. Memoized. The returned sink must be called from within a per-test + * hook so Allure associates the attachment with the right test. + */ +export async function getAllureSink(): Promise<AllureAttachSink | undefined> { + if (cachedSink !== undefined) { + return cachedSink ?? undefined } try { const mod = await import(/* @vite-ignore */ ALLURE_REPORTER_SPECIFIER) - const candidate = ( + const reporter = ( typeof mod.addAttachment === 'function' ? mod : mod.default ) as AllureReporterModule | undefined - cachedReporter = - candidate && typeof candidate.addAttachment === 'function' - ? candidate + cachedSink = + reporter && typeof reporter.addAttachment === 'function' + ? (name, content, type) => reporter.addAttachment(name, content, type) : null } catch { // Optional peer dependency not installed — attaching is a no-op. - cachedReporter = null - } - return cachedReporter -} - -/** - * Attach one retained artifact to the current Allure test. No-op when the - * artifact wasn't retained, has no path yet, or @wdio/allure-reporter isn't - * installed. Must be called from within a per-test hook (afterTest) so Allure - * associates the attachment with the right test. - */ -export async function attachArtifactToAllure( - artifact: TraceArtifact -): Promise<void> { - if (!artifact.retained || !artifact.path) { - return - } - const reporter = await loadAllureReporter() - if (!reporter) { - return - } - try { - // Allure attaches a single file; the ndjson-directory trace format yields a - // directory path, which can't be attached — skip it. - const stat = await fs.stat(artifact.path) - if (!stat.isFile()) { - return - } - const content = await fs.readFile(artifact.path) - const type = CONTENT_TYPE_BY_KIND[artifact.kind] - reporter.addAttachment(basename(artifact.path), content, type) - } catch (err) { - // A missing/unreadable artifact must never reject the test hook. - log.warn(`Allure attach skipped for ${artifact.path}: ${String(err)}`) + cachedSink = null } + return cachedSink ?? undefined } -/** Reset the memoized reporter — test seam only. */ -export function resetAllureReporterCache(): void { - cachedReporter = undefined +/** Reset the memoized sink — test seam only. */ +export function resetAllureSinkCache(): void { + cachedSink = undefined } diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index af6bcf68..ef5320d6 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -1,9 +1,13 @@ /// <reference types="../../script/types.d.ts" /> import logger from '@wdio/logger' import { + attachTraceArtifact, + captureAndAttachScreenshot, + captureAndAttachVideo, errorMessage, finalizeScreencast, finalizeTraceExport, + lastRenderedScreenshot, mapCommandToAction, recordSliceBoundary, resolveAdapterOutputDir, @@ -13,9 +17,7 @@ import { type TraceArtifact, type TraceExportContext } from '@wdio/devtools-core' -import { attachArtifactToAllure } from './allure.js' -import { captureAndAttachScreenshot } from './screenshot-capture.js' -import { captureAndAttachVideo } from './video-capture.js' +import { getAllureSink } from './allure.js' import { wireAssertCapture, type ExpectAssertion } from './assert-capture.js' import { AssertionTracker } from './assertion-tracker.js' import { @@ -187,25 +189,6 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } - /** Base64 of the last rendered action snapshot for the current test, skipping - * the end-of-scenario `__final__` frame — that one is captured post-teardown - * and can come back blank when a reloadSession runs before afterScenario. - * Used as the per-test screenshot so it's the real failure-moment frame, - * reload-immune. Scoped to this test's window (>= its start) so a test that - * captured nothing doesn't borrow the previous test's frame. */ - #lastRenderedScreenshot(): string | undefined { - for (let i = this.#actionSnapshots.length - 1; i >= 0; i--) { - const snap = this.#actionSnapshots[i]! - if (snap.timestamp < this.#currentTestStartWallTime) { - return undefined - } - if (snap.command !== '__final__' && snap.screenshot) { - return snap.screenshot - } - } - return undefined - } - /** Record a screencast this session? Live mode: `screencast.enabled`. Trace * mode: a non-`off` `video` policy (frames sliced per test at flush) or * `filmstrip` (dense frames written into the trace itself). */ @@ -541,10 +524,12 @@ export default class DevToolsHookService implements Services.ServiceInstance { uid: string | undefined, failed: boolean ): Promise<void> { + const attach = await getAllureSink() + const onLog = (level: 'info' | 'warn', msg: string) => log[level](msg) if (uid) { const artifact = await this.#eagerFlushTestSlice(uid) if (artifact) { - await attachArtifactToAllure(artifact) + await attachTraceArtifact(artifact, attach, onLog) } } await captureAndAttachScreenshot({ @@ -552,10 +537,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { granularity: this.#options.traceGranularity, policy: this.#options.screenshot, failed, - screenshotBase64: this.#lastRenderedScreenshot(), + screenshotBase64: lastRenderedScreenshot( + this.#actionSnapshots, + this.#currentTestStartWallTime + ), sessionId: this.#browser?.sessionId, outputDir: this.#outputDir, testUid: uid, + attach, onArtifact: (a) => this.#artifacts.push(a) }) // Authoritative attempt for this test (stamped into metadata by @@ -581,8 +570,9 @@ export default class DevToolsHookService implements Services.ServiceInstance { testUid: uid, sessionId: this.#browser?.sessionId, captureFormat: this.#screencastOptions?.captureFormat, + attach, onArtifact: (a) => this.#artifacts.push(a), - onLog: (level, msg) => log[level](msg) + onLog }) } diff --git a/packages/service/src/screenshot-capture.ts b/packages/service/src/screenshot-capture.ts deleted file mode 100644 index 9aca639c..00000000 --- a/packages/service/src/screenshot-capture.ts +++ /dev/null @@ -1,67 +0,0 @@ -// Per-test failure/always screenshot for the WDIO adapter: the policy gate and -// the Allure attach. Kept out of index.ts so the god-file stays lean and the -// capture is unit-testable. The policy decision and file write live in core -// (framework-agnostic). -// -// The image is NOT a fresh screenshot taken here — it's the last rendered action -// snapshot the service already captured during the test. A fresh end-of-test -// takeScreenshot() is unreliable: a cucumber `After(() => reloadSession())` hook -// (WDIO boilerplate) runs before the service afterScenario hook and blanks the -// page, so an end-of-test capture comes back empty. The last action snapshot was -// captured mid-command, before any teardown, so it's the real failure-moment -// frame — and reusing it also drops an extra WebDriver command from the report. - -import { - shouldCaptureScreenshot, - writeScreenshotArtifact, - type TraceArtifact -} from '@wdio/devtools-core' -import type { - DevToolsMode, - TraceGranularity, - TraceScreenshotPolicy -} from '@wdio/devtools-shared' -import { attachArtifactToAllure } from './allure.js' - -/** - * Write the per-test screenshot (per the policy) from an already-captured base64 - * frame and attach it to Allure. No-op outside `test`-granularity trace mode, - * without a uid/session/frame, or when the policy declines. Gated to `test` - * granularity so the whole per-test inline story (trace + screenshot + video) is - * one rule; coarser granularities keep their artifacts in the manifest. The - * artifact is handed to `onArtifact` so it lands in the manifest too. - */ -export async function captureAndAttachScreenshot(input: { - mode: DevToolsMode | undefined - granularity: TraceGranularity | undefined - policy: TraceScreenshotPolicy | undefined - failed: boolean - /** Base64 of the last rendered action snapshot (reload-immune, not a fresh - * end-of-test capture). Undefined when the test recorded no snapshot. */ - screenshotBase64: string | undefined - sessionId: string | undefined - outputDir: string - testUid: string | undefined - onArtifact: (artifact: TraceArtifact) => void -}): Promise<void> { - const { mode, granularity, policy, failed, screenshotBase64, sessionId } = - input - if ( - mode !== 'trace' || - granularity !== 'test' || - !screenshotBase64 || - !sessionId || - !input.testUid || - !shouldCaptureScreenshot(policy, failed) - ) { - return - } - const artifact = await writeScreenshotArtifact({ - outputDir: input.outputDir, - testUid: input.testUid, - sessionId, - base64: screenshotBase64 - }) - input.onArtifact(artifact) - await attachArtifactToAllure(artifact) -} diff --git a/packages/service/src/video-capture.ts b/packages/service/src/video-capture.ts deleted file mode 100644 index ff8b1d06..00000000 --- a/packages/service/src/video-capture.ts +++ /dev/null @@ -1,80 +0,0 @@ -// Per-test video for the WDIO adapter: the granularity/policy gate, the -// retention decision, the frame-window slice, the encode, and the Allure -// attach. Kept out of index.ts so the god-file stays lean. The slice + encode -// live in core (framework-agnostic); only the recorder is WDIO-specific. - -import { - encodePerTestVideo, - shouldRetainTrace, - sliceFramesFrom, - type TestOutcome, - type TraceArtifact -} from '@wdio/devtools-core' -import type { - DevToolsMode, - ScreencastFrame, - TraceGranularity, - TraceVideoPolicy -} from '@wdio/devtools-shared' -import { attachArtifactToAllure } from './allure.js' - -/** - * Slice the continuous screencast to this test's window, encode a `.webm`, and - * attach it to Allure — when trace mode + `test` granularity + a non-`off` - * `video` policy that retains this test's outcome. No-op otherwise. The encoded - * artifact is handed to `onArtifact` so it lands in the manifest. - */ -export async function captureAndAttachVideo(input: { - mode: DevToolsMode | undefined - granularity: TraceGranularity | undefined - policy: TraceVideoPolicy | undefined - frames: readonly ScreencastFrame[] | undefined - startWallTime: number - outcomes: TestOutcome[] - attempt: number | undefined - outputDir: string - testUid: string | undefined - sessionId: string | undefined - captureFormat?: 'jpeg' | 'png' - onArtifact: (artifact: TraceArtifact) => void - onLog?: (level: 'info' | 'warn', message: string) => void -}): Promise<void> { - const { mode, granularity, policy, frames, testUid, sessionId } = input - if ( - mode !== 'trace' || - granularity !== 'test' || - !policy || - policy === 'off' || - !frames || - !testUid || - !sessionId || - // No recorded attempt for this uid — the test never started (e.g. a - // skipped/pending test whose afterTest fires without a beforeTest). Skip - // rather than fail-open on empty outcomes, which would slice the PREVIOUS - // test's frames into a video attributed to this one. - input.outcomes.length === 0 - ) { - return - } - const decision = shouldRetainTrace(policy, { - outcomes: input.outcomes, - attemptInfoAvailable: true - }) - if (!decision.retain) { - return - } - const artifact = await encodePerTestVideo({ - frames: sliceFramesFrom(frames, input.startWallTime), - outputDir: input.outputDir, - testUid, - sessionId, - attempt: input.attempt, - captureFormat: input.captureFormat, - onLog: input.onLog - }) - if (!artifact) { - return - } - input.onArtifact(artifact) - await attachArtifactToAllure(artifact) -} diff --git a/packages/service/tests/allure.test.ts b/packages/service/tests/allure.test.ts index aa269eed..498b3d31 100644 --- a/packages/service/tests/allure.test.ts +++ b/packages/service/tests/allure.test.ts @@ -1,8 +1,4 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { mkdtemp, writeFile, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type { TraceArtifact } from '@wdio/devtools-core' +import { describe, it, expect, vi, beforeEach } from 'vitest' const addAttachment = vi.fn() vi.mock('@wdio/allure-reporter', () => ({ @@ -10,89 +6,30 @@ vi.mock('@wdio/allure-reporter', () => ({ default: { addAttachment } })) -import { - attachArtifactToAllure, - resetAllureReporterCache -} from '../src/allure.js' +import { getAllureSink, resetAllureSinkCache } from '../src/allure.js' -function artifact(over: Partial<TraceArtifact> = {}): TraceArtifact { - return { - kind: 'trace', - path: '', - scope: 'test', - testUids: ['u1'], - retained: true, - ...over - } -} - -describe('attachArtifactToAllure', () => { - const dirs: string[] = [] +describe('getAllureSink', () => { beforeEach(() => { addAttachment.mockReset() - resetAllureReporterCache() - }) - afterEach(async () => { - await Promise.all( - dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) - ) - }) - - async function tempZip(name: string): Promise<string> { - const dir = await mkdtemp(join(tmpdir(), 'allure-')) - dirs.push(dir) - const p = join(dir, name) - await writeFile(p, 'zip-bytes') - return p - } - - it('is a no-op for a non-retained artifact', async () => { - await attachArtifactToAllure(artifact({ retained: false, path: '/x.zip' })) - expect(addAttachment).not.toHaveBeenCalled() - }) - - it('is a no-op for an artifact with no path yet', async () => { - await attachArtifactToAllure(artifact({ path: '' })) - expect(addAttachment).not.toHaveBeenCalled() + resetAllureSinkCache() }) - it('attaches a trace zip as a plain application/zip download', async () => { - const path = await tempZip('trace-abc.zip') - await attachArtifactToAllure(artifact({ path })) + it('resolves a sink that forwards to @wdio/allure-reporter.addAttachment', async () => { + const sink = await getAllureSink() + expect(sink).toBeTypeOf('function') + const content = Buffer.from('zip-bytes') + await sink!('trace-abc.zip', content, 'application/zip') expect(addAttachment).toHaveBeenCalledOnce() - const [name, content, type] = addAttachment.mock.calls[0]! - expect(name).toBe('trace-abc.zip') - expect(Buffer.isBuffer(content)).toBe(true) - expect(type).toBe('application/zip') - }) - - it('attaches a video artifact as video/webm', async () => { - const path = await tempZip('rec-abc.webm') - await attachArtifactToAllure(artifact({ kind: 'video', path })) - const [, , type] = addAttachment.mock.calls[0]! - expect(type).toBe('video/webm') - }) - - it('attaches a screenshot artifact as image/png', async () => { - const path = await tempZip('screenshot-u1.png') - await attachArtifactToAllure(artifact({ kind: 'screenshot', path })) - const [, , type] = addAttachment.mock.calls[0]! - expect(type).toBe('image/png') - }) - - it('skips a directory-format artifact without throwing (ndjson-directory)', async () => { - const dir = await mkdtemp(join(tmpdir(), 'allure-dir-')) - dirs.push(dir) - await expect( - attachArtifactToAllure(artifact({ path: dir })) - ).resolves.toBeUndefined() - expect(addAttachment).not.toHaveBeenCalled() + expect(addAttachment).toHaveBeenCalledWith( + 'trace-abc.zip', + content, + 'application/zip' + ) }) - it('never rejects when the artifact path is missing', async () => { - await expect( - attachArtifactToAllure(artifact({ path: '/no/such/trace.zip' })) - ).resolves.toBeUndefined() - expect(addAttachment).not.toHaveBeenCalled() + it('memoizes the resolved sink across calls (single reporter probe)', async () => { + const first = await getAllureSink() + const second = await getAllureSink() + expect(first).toBe(second) }) }) diff --git a/packages/service/tests/screenshot-capture.test.ts b/packages/service/tests/screenshot-capture.test.ts deleted file mode 100644 index 7c3bdb1d..00000000 --- a/packages/service/tests/screenshot-capture.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' -import { mkdtemp, rm, readdir } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type { TraceArtifact } from '@wdio/devtools-core' - -const addAttachment = vi.fn() -vi.mock('@wdio/allure-reporter', () => ({ - addAttachment, - default: { addAttachment } -})) - -import { captureAndAttachScreenshot } from '../src/screenshot-capture.js' -import { resetAllureReporterCache } from '../src/allure.js' - -const SHOT = Buffer.from('png-bytes').toString('base64') - -describe('captureAndAttachScreenshot', () => { - const dirs: string[] = [] - const collected: TraceArtifact[] = [] - beforeEach(() => { - addAttachment.mockReset() - resetAllureReporterCache() - collected.length = 0 - }) - afterEach(async () => { - await Promise.all( - dirs.splice(0).map((d) => rm(d, { recursive: true, force: true })) - ) - }) - - async function run( - over: Partial<Parameters<typeof captureAndAttachScreenshot>[0]> - ) { - const dir = await mkdtemp(join(tmpdir(), 'shot-cap-')) - dirs.push(dir) - await captureAndAttachScreenshot({ - mode: 'trace', - granularity: 'test', - policy: 'only-on-failure', - failed: true, - screenshotBase64: SHOT, - sessionId: 'sess1234', - outputDir: dir, - testUid: 'u1', - onArtifact: (a) => collected.push(a), - ...over - }) - return { dir } - } - - it('writes + attaches the reused snapshot on failure under only-on-failure', async () => { - const { dir } = await run({}) - expect(collected).toHaveLength(1) - expect(collected[0]!.kind).toBe('screenshot') - expect(addAttachment).toHaveBeenCalledOnce() - expect(await readdir(dir)).toHaveLength(1) - }) - - it('does nothing on a passing test under only-on-failure', async () => { - await run({ failed: false }) - expect(collected).toHaveLength(0) - expect(addAttachment).not.toHaveBeenCalled() - }) - - it('does nothing outside trace mode', async () => { - await run({ mode: 'live', policy: 'on' }) - expect(collected).toHaveLength(0) - }) - - it('does nothing outside test granularity (uniform rule)', async () => { - await run({ granularity: 'session', policy: 'on' }) - expect(collected).toHaveLength(0) - }) - - it('does nothing without a captured frame (no snapshot to reuse)', async () => { - await run({ policy: 'on', screenshotBase64: undefined }) - expect(collected).toHaveLength(0) - }) - - it('does nothing without a sessionId or uid', async () => { - await run({ sessionId: undefined }) - await run({ testUid: undefined }) - expect(collected).toHaveLength(0) - }) -}) diff --git a/packages/service/tests/video-capture.test.ts b/packages/service/tests/video-capture.test.ts deleted file mode 100644 index 25c1c404..00000000 --- a/packages/service/tests/video-capture.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { describe, it, expect, beforeEach } from 'vitest' -import type { ScreencastFrame } from '@wdio/devtools-shared' -import type { TraceArtifact, TestOutcome } from '@wdio/devtools-core' - -// Every case here is designed to no-op BEFORE the ffmpeg encode (failed gate or -// non-retaining policy), so no encoder/reporter mock is needed — the real -// (pure) shouldRetainTrace decides retention. The encode path itself is covered -// by core's video-slice.test.ts. -import { captureAndAttachVideo } from '../src/video-capture.js' - -const frames: ScreencastFrame[] = [ - { data: 'AAAA', timestamp: 10 }, - { data: 'AAAA', timestamp: 20 }, - { data: 'AAAA', timestamp: 30 } -] - -const failedOutcome: TestOutcome[] = [ - { uid: 'u1', attempt: 0, state: 'failed' } -] -const passedOutcome: TestOutcome[] = [ - { uid: 'u1', attempt: 0, state: 'passed' } -] - -describe('captureAndAttachVideo gating', () => { - const collected: TraceArtifact[] = [] - beforeEach(() => { - collected.length = 0 - }) - - const base = { - mode: 'trace' as const, - granularity: 'test' as const, - policy: 'retain-on-failure' as const, - frames, - startWallTime: 0, - outcomes: failedOutcome, - attempt: 0, - outputDir: '/tmp/does-not-encode', - testUid: 'u1', - sessionId: 'sess1234', - onArtifact: (a: TraceArtifact) => collected.push(a) - } - - it('no-ops outside test granularity', async () => { - await captureAndAttachVideo({ ...base, granularity: 'session' }) - expect(collected).toHaveLength(0) - }) - - it('no-ops when the test never started (empty outcomes — no fail-open)', async () => { - await captureAndAttachVideo({ ...base, outcomes: [] }) - expect(collected).toHaveLength(0) - }) - - it('no-ops when video policy is off/undefined', async () => { - await captureAndAttachVideo({ ...base, policy: 'off' }) - await captureAndAttachVideo({ ...base, policy: undefined }) - expect(collected).toHaveLength(0) - }) - - it('no-ops when there are no frames', async () => { - await captureAndAttachVideo({ ...base, frames: undefined }) - expect(collected).toHaveLength(0) - }) - - it('no-ops when the policy does not retain this outcome (passing + retain-on-failure)', async () => { - await captureAndAttachVideo({ ...base, outcomes: passedOutcome }) - expect(collected).toHaveLength(0) - }) - - it('no-ops without a sessionId or uid', async () => { - await captureAndAttachVideo({ ...base, sessionId: undefined }) - await captureAndAttachVideo({ ...base, testUid: undefined }) - expect(collected).toHaveLength(0) - }) -}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f03e235..57b9978d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -371,7 +371,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.0.16 - version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) + version: 4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) packages/nightwatch-devtools: dependencies: @@ -501,6 +501,12 @@ importers: '@wdio/devtools-shared': specifier: workspace:^ version: link:../shared + allure-js-commons: + specifier: ^3.0.0 + version: 3.10.2 + allure-mocha: + specifier: ^3.0.0 + version: 3.10.2(mocha@11.7.6) chromedriver: specifier: ^150.0.0 version: 150.0.1 @@ -2905,6 +2911,11 @@ packages: allure-playwright: optional: true + allure-mocha@3.10.2: + resolution: {integrity: sha512-HtqRIhfRRvDQ2xnSTUDxgKPA+rXid3GetRPX4EIC8fDzY4FIlOwNcPcq2lX40NC5ApkPc4XS2Xv18K/oVNhxnA==} + peerDependencies: + mocha: '>=6.2.0' + ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} @@ -7658,7 +7669,7 @@ snapshots: '@babel/types': 7.29.7 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -7817,7 +7828,7 @@ snapshots: '@babel/parser': 7.29.7 '@babel/template': 7.29.7 '@babel/types': 7.29.7 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -8498,7 +8509,7 @@ snapshots: '@eslint/config-array@0.23.5': dependencies: '@eslint/object-schema': 3.0.5 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) minimatch: 10.2.5 transitivePeerDependencies: - supports-color @@ -9182,7 +9193,7 @@ snapshots: '@puppeteer/browsers@2.13.2': dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) extract-zip: 2.0.1 progress: 2.0.3 proxy-agent: 6.5.0 @@ -9600,7 +9611,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) eslint: 10.5.0(jiti@2.7.0) typescript: 6.0.3 transitivePeerDependencies: @@ -9610,7 +9621,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -9629,7 +9640,7 @@ snapshots: '@typescript-eslint/types': 8.61.1 '@typescript-eslint/typescript-estree': 8.61.1(typescript@6.0.3) '@typescript-eslint/utils': 8.61.1(eslint@10.5.0(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) eslint: 10.5.0(jiti@2.7.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 @@ -9644,7 +9655,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.61.1(typescript@6.0.3) '@typescript-eslint/types': 8.61.1 '@typescript-eslint/visitor-keys': 8.61.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) minimatch: 10.2.5 semver: 7.8.4 tinyglobby: 0.2.17 @@ -9783,14 +9794,6 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': - dependencies: - '@vitest/spy': 4.1.9 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) - '@vitest/mocker@4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.9 @@ -10146,7 +10149,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -10191,6 +10194,13 @@ snapshots: dependencies: md5: 2.3.0 + allure-mocha@3.10.2(mocha@11.7.6): + dependencies: + allure-js-commons: 3.10.2 + mocha: 11.7.6 + transitivePeerDependencies: + - allure-playwright + ansi-align@3.0.1: dependencies: string-width: 4.2.3 @@ -11587,7 +11597,7 @@ snapshots: '@types/estree': 1.0.9 ajv: 6.15.0 cross-spawn: 7.0.6 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) escape-string-regexp: 4.0.0 eslint-scope: 9.1.2 eslint-visitor-keys: 5.0.1 @@ -11714,7 +11724,7 @@ snapshots: extract-zip@2.0.1: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -12045,7 +12055,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 6.0.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12053,7 +12063,7 @@ snapshots: dependencies: basic-ftp: 5.3.1 data-uri-to-buffer: 8.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12251,35 +12261,35 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color http-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@9.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -12578,7 +12588,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -13041,7 +13051,7 @@ snapshots: lighthouse-logger@2.0.2: dependencies: - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) marky: 1.3.0 transitivePeerDependencies: - supports-color @@ -13670,7 +13680,7 @@ snapshots: dependencies: '@tootallnate/quickjs-emscripten': 0.23.0 agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-uri: 6.0.5 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 @@ -13682,7 +13692,7 @@ snapshots: pac-proxy-agent@9.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) get-uri: 8.0.0 http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 @@ -13972,7 +13982,7 @@ snapshots: proxy-agent@6.5.0: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 lru-cache: 7.18.3 @@ -13985,7 +13995,7 @@ snapshots: proxy-agent@8.0.1: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) http-proxy-agent: 9.0.0 https-proxy-agent: 9.0.0 lru-cache: 7.18.3 @@ -14505,7 +14515,7 @@ snapshots: socks-proxy-agent@10.0.0: dependencies: agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14513,7 +14523,7 @@ snapshots: socks-proxy-agent@8.0.5: dependencies: agent-base: 7.1.4 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) socks: 2.8.9 transitivePeerDependencies: - supports-color @@ -14722,7 +14732,7 @@ snapshots: cosmiconfig: 9.0.2(typescript@6.0.3) css-functions-list: 3.3.3 css-tree: 3.2.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) fast-glob: 3.3.3 fastest-levenshtein: 1.0.16 file-entry-cache: 11.1.3 @@ -14985,7 +14995,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) esbuild: 0.27.7 fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 @@ -15118,7 +15128,7 @@ snapshots: '@rollup/pluginutils': 5.4.0(rollup@4.62.0) '@volar/typescript': 2.4.28 compare-versions: 6.1.1 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) kolorist: 1.8.0 local-pkg: 1.2.1 magic-string: 0.30.21 @@ -15261,21 +15271,6 @@ snapshots: optionalDependencies: rollup: 4.62.0 - vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): - dependencies: - lightningcss: 1.32.0 - picomatch: 4.0.4 - postcss: 8.5.15 - rolldown: 1.0.3 - tinyglobby: 0.2.17 - optionalDependencies: - '@types/node': 25.9.3 - esbuild: 0.27.7 - fsevents: 2.3.3 - jiti: 2.7.0 - tsx: 4.22.4 - yaml: 2.9.0 - vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 @@ -15291,36 +15286,6 @@ snapshots: tsx: 4.22.4 yaml: 2.9.0 - vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): - dependencies: - '@vitest/expect': 4.1.9 - '@vitest/mocker': 4.1.9(vite@8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) - '@vitest/pretty-format': 4.1.9 - '@vitest/runner': 4.1.9 - '@vitest/snapshot': 4.1.9 - '@vitest/spy': 4.1.9 - '@vitest/utils': 4.1.9 - es-module-lexer: 2.1.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.3 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.1.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 - tinyglobby: 0.2.17 - tinyrainbow: 3.1.0 - vite: 8.0.16(@types/node@25.9.3)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 25.9.3 - '@vitest/coverage-v8': 4.1.9(@vitest/browser@4.1.9)(vitest@4.1.9) - happy-dom: 20.10.4 - jsdom: 24.1.3 - transitivePeerDependencies: - - msw - vitest@4.1.9(@types/node@25.9.3)(@vitest/coverage-v8@4.1.9)(happy-dom@20.10.4)(jsdom@24.1.3)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.9 @@ -15363,7 +15328,7 @@ snapshots: dependencies: chalk: 4.1.2 commander: 9.5.0 - debug: 4.4.3(supports-color@10.2.2) + debug: 4.4.3(supports-color@5.5.0) transitivePeerDependencies: - supports-color From 602a62e1162a6fa282c305d599c966d6137bb57b Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Thu, 16 Jul 2026 19:38:00 +0530 Subject: [PATCH 75/91] docs: record Nightwatch BDD per-test artifact limitation as known debt --- CLAUDE.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 36896d87..2cfbe770 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -240,10 +240,13 @@ Documented divergences from the conventions above. They exist today as debt to b - **Nightwatch: `retain-on-failure` works; the other retry-aware policies degrade.** Its `--retries` re-runs the testcase *internally* without re-firing the plugin's per-test hooks, and the per-testcase results carry no attempt/retry field (retries live only in undocumented, version-varying Nightwatch internals — `suiteRetries.testRetriesCount` / `reporter.testResults.retryTest`), so the ledger sees only the final attempt for the `describe/it` and exports-object interfaces. Cucumber scenarios expose per-scenario hooks, so the feed captures their attempts. Not cleanly fixable without depending on those internals. - WDIO `specFileRetries` spawns a fresh worker per retry, so cross-process attempts aren't in the (process-scoped) ledger. - Eager per-test trace slice (Nightwatch + Selenium) can drop an action snapshot whose fire-and-forget capture hasn't resolved by `afterEach` / scenario end — the slice is written from whatever snapshots exist at flush time. The WDIO service is immune because it awaits each snapshot inline before flushing. -- The per-test `screenshot` and `video` options live on the **WDIO `ServiceOptions` only** — not `BaseDevToolsOptions` — because only the service implements them (an option belongs on an adapter until a second adapter consumes it, mirroring the core-helper rule; putting them on the shared base made them appear available in Selenium/Nightwatch and broke those adapters' `Required<>` option types). The policy *types* (`TraceScreenshotPolicy`/`TraceVideoPolicy`) and the capture/slice/encode logic (`core/screenshot-artifact.ts`, `core/video-slice.ts`) are framework-agnostic, so Selenium/Nightwatch adoption is wiring-only (capture the base64 + feed `captureAndAttach*`) plus adding the option to their own options type; pending. Both are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the frame buffer is unbounded for the session (a decimation cap is a noted follow-up), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). -- The `filmstrip` option (dense screencast into the trace) is on **`BaseDevToolsOptions`** — the counterexample to the screenshot/video entry above — because all three adapters implement it (the "second consumer → base" rule realized). Core owns the work (`core/screencast-trace.ts` `thinScreencastFrames`/`buildDenseScreencast`; slice windowing in `spec-trace-helpers.ts`); adapters only default the option, un-gate the recorder in trace mode when it's set, and feed `recorder.frames` into the finalize context. Each adapter captures frames while the recorder is still alive (service `onReload` → `#filmstripFrames`; Selenium `onDriverEnd` drain before nulling; Nightwatch `#finalizeCurrentScreencast` snapshot before delegating), and each finalize context spreads `[...accumulated, ...(live recorder frames)]` so a **mid-run** per-spec/per-test slice flush (which fires before the recorder is drained) isn't blank. Two known limits: (1) the dense frames are written **alongside** the sparse per-action frames (union), not replacing them — the sparse frames carry the DOM `elements`/`snapshot` refs, so suppressing them (attaching those refs to the nearest dense frame) is a deferred follow-up; near each action the filmstrip therefore shows the action's own frame plus adjacent dense frames. (2) Thinning is applied at **export**, so the raw session frame buffer is unbounded in memory during the run (same decimation-cap follow-up as video). Per-test filmstrip slicing follows the same per-test-hook availability as `traceGranularity:'test'` (works for WDIO mocha/cucumber, Selenium mocha, Nightwatch exports-object/cucumber; Nightwatch BDD `describe/it` degrades to session scope per the entry below), and non-Chrome polling carries the same reporter-noise caveat. +- The per-test `screenshot` and `video` options live on the **WDIO `ServiceOptions` only** — not `BaseDevToolsOptions` — because only the service implements them (an option belongs on an adapter until a second adapter consumes it, mirroring the core-helper rule; putting them on the shared base made them appear available in Selenium/Nightwatch and broke those adapters' `Required<>` option types). The policy *types* (`TraceScreenshotPolicy`/`TraceVideoPolicy`) and the capture/slice/encode logic (`core/screenshot-artifact.ts`, `core/video-slice.ts`) are framework-agnostic, so Selenium/Nightwatch adoption was wiring-only — now done (Selenium adds the options on its own `DevToolsOptions` with full inline attach; Nightwatch adds them produce-only — see the Allure-attach entry below). All are gated to `traceGranularity:'test'` (per-test inline Allure); coarser granularities keep artifacts in the manifest. Video records the screencast continuously and slices per-test by wall-time — the session frame buffer is bounded by `maxBufferFrames` (default 2000; decimates keeping first/last), and on non-Chrome the polling recorder issues many `takeScreenshot`s that flood `@wdio/allure-reporter` (pair with `disableWebdriverStepsReporting`). +- The `filmstrip` option (dense screencast into the trace) is on **`BaseDevToolsOptions`** — the counterexample to the screenshot/video entry above — because all three adapters implement it (the "second consumer → base" rule realized). Core owns the work (`core/screencast-trace.ts` `thinScreencastFrames`/`buildDenseScreencast`; slice windowing in `spec-trace-helpers.ts`); adapters only default the option, un-gate the recorder in trace mode when it's set, and feed `recorder.frames` into the finalize context. Each adapter captures frames while the recorder is still alive (service `onReload` → `#filmstripFrames`; Selenium `onDriverEnd` drain before nulling; Nightwatch `#finalizeCurrentScreencast` snapshot before delegating), and each finalize context spreads `[...accumulated, ...(live recorder frames)]` so a **mid-run** per-spec/per-test slice flush (which fires before the recorder is drained) isn't blank. When dense frames are present they **supersede** the sparse per-action filmstrip (the per-action DOM `elements`/`snapshot` are carried independently by the `frame-snapshot` events, so no DOM data is lost); a run without dense frames keeps the sparse filmstrip, byte-stable with before. Thinning is applied at export; the live session frame buffer is bounded by `maxBufferFrames` (default 2000; see the screenshot/video entry above). Per-test filmstrip slicing follows the same per-test-hook availability as `traceGranularity:'test'` (works for WDIO mocha/cucumber, Selenium mocha, Nightwatch exports-object/cucumber; Nightwatch BDD `describe/it` degrades to session scope per the entry below), and non-Chrome polling carries the same reporter-noise caveat. +- Per-test artifact **Allure attachment** is cross-adapter via a pluggable sink in `core/allure-artifacts.ts` (`AllureAttachSink`; `captureAndAttachScreenshot`/`captureAndAttachVideo`/`attachTraceArtifact`/`lastRenderedScreenshot` — moved out of the WDIO service). The service supplies a `@wdio/allure-reporter` sink; **Selenium** supplies an `allure-js-commons` `attachment()` sink (runtime-agnostic — attaches under any allure runner adapter — gated on `globalThis.allureTestRuntime`, dynamic-imported as an optional peer dep). An **undefined sink = produce-only** (write file + manifest, skip attach). Caveats: (a) **Selenium inline-attach needs awaited runner hooks** — its mocha `afterEach`/jest `afterEach`/cucumber `After` are now async + awaited so the async produce+attach lands while the allure adapter still holds the current test open (fire-and-forget attaches to the next test or drops); Gherkin `AfterStep`→`onTestEnd` stays fire-and-forget. (b) **Nightwatch inline-attach is unsupported** — `nightwatch-allure` is post-hoc with no live attach API, and `allure-js-commons` no-ops in a Nightwatch run (nothing calls `setGlobalTestRuntime`); Nightwatch is produce-only. Revisit if an `allure-js-commons`-based Nightwatch runtime adapter appears (then it's a sink swap). (c) **video is standalone** in both adapters (recorder starts for `video != off` OR filmstrip in trace mode; the orphan session-webm encode is trace-gated to stop-only). +- **node:assert error display** (`core/assert-patcher.ts` `describeAssertFailure`): node auto-generates a per-character COLORED diff as the AssertionError message; once any consumer strips the ANSI (allure-mocha, the app console filter, plain terminals) the actual/expected interleave into mush (`'ExampleThis DIs Nomt…'`). The patcher pulls node's clean `.actual`/`.expected` into a `CollapsedAssertResult` AND **rewrites the auto-generated message** (and its echo in the stack) as a value-bearing `Expected: … / Received: …` block. Because `toError` returns the thrown error itself, this rewrite reaches every consumer — the trace's Errors tab, the runner console, and **allure-mocha's error box** (which would otherwise show the ANSI-stripped mush). It only touches node's auto-generated messages; a user-supplied message and errors without `actual`/`expected` (e.g. `assert.ok`) pass through unchanged. This is a deliberate mutation of the thrown assert error — node's auto-message is a display artifact, and the rewrite is strictly more readable and matches `@wdio/allure-reporter`'s `Expected/Received` shape. - Nightwatch native asserts are buffered at call time and emitted in one batch at test-end (Nightwatch exposes no per-assertion execution/outcome hook reachable from the plugin — `client.queue.tree` / `client.reporter` aren't on `browser`). Streaming mid-run would flash every assert green before its outcome is known, so rows are held and reconciled against `results.testcases[*].assertions` at `afterEach` (the flat `results.assertions` only reflects the last testcase). The exporter re-sorts commands by timestamp (`buildActionEvents`) so the batched rows land at their real timeline positions rather than clustering at flush time. `actual` is parsed from the failure message (`but got: …`, failures only) into a collapsed `{passed, expected, actual}` result; `expected` is the assertion arg — the label still mirrors the call args, not the derived values. -- Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. +- Nightwatch's BDD `describe/it` interface fires the plugin's global `beforeEach`/`afterEach` **once per module** (with an empty `currentTest.name`), not per `it` — Nightwatch runs the individual `it`s internally with no per-testcase hook or event reachable from the plugin (the lib emits only transport-level events). So `traceGranularity:'test'` records a single slice boundary (the first test) and collapses to one session-scoped slice; retry-aware/per-test retention (`tracePolicy`) degrades to session scope for this interface. Per-test slicing works for adapters/styles that expose per-test hooks (WDIO mocha/cucumber, Selenium mocha; likely Nightwatch's exports-object style, unverified). Session-granularity trace is unaffected. **No per-test support has been built for this interface** — real per-`it` slicing needs a hook Nightwatch doesn't surface; deferred to a full-picture pass, not attempted yet. + - **Empirically confirmed** (BDD example, `mode:'trace'` + `traceGranularity:'test'` + `tracePolicy:'retain-on-failure'` + `screenshot:'on'` + `video:'retain-on-failure'`): the one collapsed slice is keyed to the **first** test's uid (the artifacts manifest's `tests[]` still carries every testcase with correct `state`, so metadata capture is fine — it's the *slice/artifact* keying that collapses). Consequence: with a passing first test and a failing later test, the screenshot **produces** (policy `'on'` captures regardless of outcome, keyed to the first test) but the **trace zip and video are dropped** — `retain-on-failure` evaluates the first (passing) test's outcome, and the actually-failing test never gets its own slice to retain. The per-test produce-only path itself (screenshot/video write + manifest) is correct; only the BDD slice-keying limits it. - Service renders expect-webdriverio matchers as single `expect.<matcher>` rows by **folding**, not stack/depth suppression (the old `#assertionDepth`/`#matcherStarted`/self-heal machinery is gone). The matcher's value-read (`toHaveText`→`getText`, `toExist`→`isExisting`, …) is captured as a normal command; `afterAssertion` then coalesces the synthesized `expect.*` row into that read in place — inheriting its callSource, screenshot, and timeline position — and the fold replaces **by timestamp, never a public `id`**: `id` is the per-worker `commandCounter`, which resets per spec, so stamping one lets the app's id-first `replaceCommand` swap a same-id row from another spec (duplicate rows + a fold from another spec vanishing, in multi-spec live mode). `beforeAssertion` arms the pending matcher (depth-counted so aliases like `toBeChecked`→`toBeSelected` fold once); a matcher that **hard-throws** — element never resolves, so expect-webdriverio's `waitUntil` rethrows and `afterAssertion` never fires — is synthesized at `afterTest`/`afterStep` from the throwing read, so a failing assertion renders as `expect.<matcher>` whether or not the element existed. Two limits: its error is then the read's (`Can't call getText on … element wasn't found`), not an assertion-phrased message; and `MATCHER_READ_COMMANDS` is a hand-maintained allowlist, so a matcher whose read isn't listed leaves its raw read visible alongside the `expect.*` row. Plain-value jest matchers (`expect(x).toBe(y)`) don't fire the ewdio hooks, so they aren't captured as rows. ### File-size (raw line counts; soft cap is 500 logic lines) From c29607ab6f2d82d41b2852b52aeea8f66a3187cf Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 17 Jul 2026 15:06:36 +0530 Subject: [PATCH 76/91] docs: record Nightwatch BDD per-test limitation + Selenium seam extraction as debt --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2cfbe770..24f95abe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -256,7 +256,7 @@ Most entries below don't trigger the `max-lines` lint rule after `skipBlankLines - `packages/service/src/index.ts` — over the 500-**logic**-line soft cap and ~620 raw (the WDIO plugin god-file: session/screencast/command/assertion/trace-slice wiring in one class). The assertion-capture block (`beforeAssertion`/`afterAssertion`/`#handleAssertionOutcome`/`#finalizePendingAssertion`, plus the `#assertionDepth`/`#pendingAssertion` state) is now the largest self-contained seam and the prime extraction candidate; trace-slice and screencast are the next two. - `packages/nightwatch-devtools/src/index.ts` (~536 raw). Cucumber/test/run-lifecycle, session-init, event-hub modules already extracted; remainder is the `PluginInternals` accessor bag plus per-method delegators plus the factory. Accept-as-is. -- `packages/selenium-devtools/src/index.ts` (~560 raw). Session/test-lifecycle extracted; remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Same situation as nightwatch. +- `packages/selenium-devtools/src/index.ts` (~817 raw). Session/test-lifecycle extracted; remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Grew past ~560 when the per-test-artifact seam (`#allureSink`/`#perTestArtifactInputs`/`#emitTestArtifacts`) landed inline with the cross-adapter Allure work; Nightwatch extracted the twin concern into `test-artifacts.ts`, so the next Selenium change to this seam should mirror that (a `selenium-devtools/src/test-artifacts.ts` threading the sink + flushed-trace promise through an input bag). Prime extraction candidate. - `packages/nightwatch-devtools/src/session.ts` (~468). `captureNetworkFromPerformanceLogs` + `captureBrowserLogs` + `captureTrace` are tightly coupled to NightwatchBrowser state. Coverage at 78% after recent backfill; further extraction would need rewriting the browser-coupling. ### Test coverage gaps (worst-risk-first) From fa5de7b20da9b29cfd992167cf779c65decff263 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 17 Jul 2026 18:26:13 +0530 Subject: [PATCH 77/91] feat: DOM time-travel in the trace player (mutations + field state) --- .../app/src/components/browser/snapshot.ts | 62 ++++++++++--- .../backend/src/trace-reader-constants.ts | 3 + packages/backend/src/trace-reader.ts | 58 ++++++++++--- packages/backend/tests/trace-reader.test.ts | 40 +++++++++ packages/core/src/index.ts | 1 + packages/core/src/trace-exporter.ts | 24 ++++- packages/core/src/trace-mutations.ts | 62 +++++++++++++ packages/core/src/trace-zip-writer.ts | 5 ++ packages/core/tests/trace-exporter.test.ts | 67 ++++++++++++++ packages/core/tests/trace-mutations.test.ts | 87 +++++++++++++++++++ packages/script/src/index.ts | 34 ++++++++ packages/shared/src/types.ts | 23 +++++ 12 files changed, 439 insertions(+), 27 deletions(-) create mode 100644 packages/core/src/trace-mutations.ts create mode 100644 packages/core/tests/trace-mutations.test.ts diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts index 9fe5f7b4..fe4cef66 100644 --- a/packages/app/src/components/browser/snapshot.ts +++ b/packages/app/src/components/browser/snapshot.ts @@ -257,9 +257,36 @@ export class DevtoolsBrowser extends Element { commandPageUrl(command, this.commands ?? [], this.mutations ?? []) ?? this.#activeUrl } - // Switch to snapshot mode so the command screenshot is visible instead of the video. + // Switch to snapshot mode so the command snapshot is visible instead of the video. this.#viewMode = 'snapshot' - this.requestUpdate() + // DOM time-travel: rebuild the iframe DOM as of the selected command's time. + // #renderBrowserState requestUpdates internally, so only request one here + // when there's no mutation stream to replay (screenshot-only fallback). + const target = this.#mutationForCommand(command) + if (target) { + await this.#renderBrowserState(target) + } else { + this.requestUpdate() + } + } + + /** The last mutation captured at or before the command's time — the DOM state + * the command observed. Falls back to the first mutation when every mutation + * is later (command precedes the slice's initial full-DOM snapshot). */ + #mutationForCommand(command?: CommandLog): TraceMutation | undefined { + const mutations = this.mutations + if (!command?.timestamp || !mutations?.length) { + return undefined + } + let best: TraceMutation | undefined + for (const mutation of mutations) { + if (mutation.timestamp <= command.timestamp) { + best = mutation + } else { + break + } + } + return best ?? mutations[0] } // View-mode flips swap the iframe with <img>/<video> and don't fire resize. @@ -328,7 +355,7 @@ export class DevtoolsBrowser extends Element { } #handleAttributeMutation(mutation: TraceMutation) { - if (!mutation.attributeName || !mutation.attributeValue) { + if (!mutation.attributeName) { return } @@ -337,7 +364,16 @@ export class DevtoolsBrowser extends Element { return } - el.setAttribute(mutation.attributeName, mutation.attributeValue || '') + const value = mutation.attributeValue ?? '' + el.setAttribute(mutation.attributeName, value) + // Form-field state lives on the PROPERTY, not just the attribute — mirror it + // so a replayed input shows the captured value / checked state, including a + // field cleared back to empty. + if (mutation.attributeName === 'value' && 'value' in el) { + ;(el as HTMLInputElement).value = value + } else if (mutation.attributeName === 'checked' && 'checked' in el) { + ;(el as HTMLInputElement).checked = value === 'true' + } } #handleChildListMutation(mutation: TraceMutation) { @@ -554,6 +590,17 @@ export class DevtoolsBrowser extends Element { ></wdio-devtools-screencast-player> </div>` } + // DOM replay is the primary snapshot whenever the trace carries mutations: + // #renderBrowserState reconstructs the iframe DOM at the selected command's + // time, so points without a captured frame (assertions, static waits) still + // show the real page instead of a blank/stale screenshot. + if (hasMutations) { + return html`<div class="iframe-wrapper"> + <iframe class="origin-top-left"></iframe> + </div>` + } + // No mutation stream (DOM-less / foreign trace): fall back to the selected + // command's screenshot, then the latest available frame. if (this.#screenshotData) { return html`<div class="iframe-wrapper"> <div @@ -567,12 +614,7 @@ export class DevtoolsBrowser extends Element { </div> </div>` } - if (hasMutations) { - return html`<div class="iframe-wrapper"> - <iframe class="origin-top-left"></iframe> - </div>` - } - const autoScreenshot = hasMutations ? null : this.#latestAutoScreenshot + const autoScreenshot = this.#latestAutoScreenshot if (autoScreenshot) { return html`<div class="iframe-wrapper"> <div diff --git a/packages/backend/src/trace-reader-constants.ts b/packages/backend/src/trace-reader-constants.ts index 3cd37bd4..737f0903 100644 --- a/packages/backend/src/trace-reader-constants.ts +++ b/packages/backend/src/trace-reader-constants.ts @@ -17,6 +17,9 @@ export const NETWORK_STREAM_SUFFIX = '.network' /** Sidecar entries holding call stacks keyed by numeric call id. */ export const STACKS_STREAM_SUFFIX = '.stacks' +/** Every zip entry ending in this suffix is an NDJSON DOM-mutation stream. */ +export const MUTATIONS_STREAM_SUFFIX = '.mutations' + /** Foreign screencast refs may be a bare sha1; probe image extensions too. */ export const FRAME_RESOURCE_SUFFIXES = ['', '.jpeg', '.png'] as const diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index 204fd464..3b4f9ef2 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -5,22 +5,26 @@ // is a HAR stream, and `.stacks` sidecars carry call stacks. Commands come // from before/after events, the filmstrip from screencast-frame events + // resources/*, console logs from console/stdio events, sources from stack -// frames + src@ resources. Fields the zip never carried (mutations, suites) -// come back empty. Constants, event types, and pure helpers live in the -// sibling trace-reader-{constants,types,utils}.ts files. +// frames + src@ resources, and DOM mutations from the `.mutations` stream when +// present. Fields the zip never carried (suites) come back empty. Constants, +// event types, and pure helpers live in the sibling +// trace-reader-{constants,types,utils}.ts files. import fs from 'node:fs/promises' import { unzipSync, strFromU8 } from 'fflate' import { + isMutationsTruncationMarker, type CommandLog, type NetworkRequest, type TraceLog, + type TraceMutation, type TracePlayerData, type TracePlayerFrame } from '@wdio/devtools-shared' import { FRAME_RESOURCE_SUFFIXES, + MUTATIONS_STREAM_SUFFIX, NETWORK_STREAM_SUFFIX, REVERSE_ACTION_MAP, STACKS_STREAM_SUFFIX, @@ -270,6 +274,39 @@ function parseNetworkStreams( ) } +function parseMutationStreams( + files: Record<string, Uint8Array>, + names: string[] +): TraceMutation[] { + // The `.mutations` NDJSON is TraceMutation JSON, minus the trailing + // truncation marker; cast at this boundary like the HAR/action parsers. + return names + .flatMap((name) => parseNdjson(strFromU8(files[name]))) + .filter( + (entry) => !isMutationsTruncationMarker(entry) + ) as unknown as TraceMutation[] +} + +/** Categorize + rebase every `.trace` action stream, merge them, and fold in + * any `.stacks` sidecars — the event-stream prelude for `parseTraceZip`. */ +function parseAndMergeEventStreams( + files: Record<string, Uint8Array>, + names: string[] +): MergedEvents { + const streams = names + .filter((name) => name.endsWith(TRACE_STREAM_SUFFIX)) + .map((name) => { + const stream = categorizeEvents(parseNdjson(strFromU8(files[name]))) + rebaseToEpoch(stream) + return stream + }) + const merged = mergeStreams(streams) + for (const name of names.filter((n) => n.endsWith(STACKS_STREAM_SUFFIX))) { + attachSidecarStacks(merged.befores, strFromU8(files[name])) + } + return merged +} + function earliestWallTime(ctxs: ContextOptionsEvent[]): number { const wallTimes = ctxs .map((ctx) => ctx.wallTime) @@ -283,15 +320,7 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { const names = Object.keys(files).sort() const entriesWith = (suffix: string) => names.filter((name) => name.endsWith(suffix)) - const streams = entriesWith(TRACE_STREAM_SUFFIX).map((name) => { - const stream = categorizeEvents(parseNdjson(strFromU8(files[name]))) - rebaseToEpoch(stream) - return stream - }) - const merged = mergeStreams(streams) - for (const name of entriesWith(STACKS_STREAM_SUFFIX)) { - attachSidecarStacks(merged.befores, strFromU8(files[name])) - } + const merged = parseAndMergeEventStreams(files, names) const { frames, maxTime: frameMax } = buildFrames(files, merged.frameEvents) const { commands, @@ -306,7 +335,10 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { ) const startTime = earliestWallTime(merged.ctxs) const trace: TraceLog = { - mutations: [], + mutations: parseMutationStreams( + files, + entriesWith(MUTATIONS_STREAM_SUFFIX) + ), logs: [], consoleLogs: buildConsoleLogs(merged.consoleEvents), networkRequests: parseNetworkStreams( diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index 25f9a2f9..2dfc7a7b 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -212,6 +212,46 @@ describe('parseTraceZip', () => { expect(trace.suites).toEqual([]) }) + it('restores DOM mutations from a trace.mutations stream, dropping the marker', () => { + const mutations = [ + { + type: 'childList', + addedNodes: [{ tag: 'html' }], + removedNodes: [], + timestamp: 1000 + }, + { + type: 'attributes', + target: 'body', + attributeName: 'class', + addedNodes: [], + removedNodes: [], + timestamp: 1100 + } + ] + const zip = zipSync({ + 'trace.trace': toNdjson([ + { + type: 'context-options', + wallTime: WALL_TIME, + browserName: 'chrome', + contextId: 'context@abcd1234', + options: { viewport: { width: 1024, height: 768 } } + } + ]), + 'trace.mutations': toNdjson([ + ...mutations, + { __truncated__: true, dropped: 7 } + ]) + }) + const { trace } = parseTraceZip(zip) + // Two mutations from three NDJSON lines proves the trailing truncation + // marker was dropped rather than surfaced as a mutation. + expect(trace.mutations).toHaveLength(2) + expect(trace.mutations[0]).toMatchObject({ type: 'childList' }) + expect(trace.mutations[1]).toMatchObject({ target: 'body' }) + }) + it('restores callSource and sources from stack frames + src resources', () => { const { trace } = parseTraceZip(fixtureZip()) const click = trace.commands.find((c) => c.command === 'click') diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index cee1a38e..ee4126a0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -23,6 +23,7 @@ export * from './trace-frame-snapshots.js' export * from './trace-retention.js' export * from './trace-sources.js' export * from './trace-har.js' +export * from './trace-mutations.js' export * from './trace-snapshots.js' export * from './trace-zip-writer.js' export * from './bidi.js' diff --git a/packages/core/src/trace-exporter.ts b/packages/core/src/trace-exporter.ts index c8ae443d..31299af6 100644 --- a/packages/core/src/trace-exporter.ts +++ b/packages/core/src/trace-exporter.ts @@ -41,6 +41,7 @@ import { buildActionEvents, type ActionEvent } from './trace-action-events.js' import { buildSourceResources } from './trace-sources.js' import { networkRequestToHar } from './trace-har.js' import { buildTraceZip, type TraceZipResource } from './trace-zip-writer.js' +import { buildMutationsNdjson } from './trace-mutations.js' import { sha1Hex } from './sha1.js' const TRACE_VERSION = 8 @@ -127,6 +128,14 @@ function shortId(sessionId?: string): string { return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) } +function allocateTraceIds(sessionId?: string): { + contextId: string + pageId: string +} { + const idPrefix = shortId(sessionId) + return { contextId: `context@${idPrefix}`, pageId: `page@${idPrefix}` } +} + function resolveContextNaming(caps: Record<string, unknown> | undefined): { browserName: string title: string @@ -314,6 +323,7 @@ interface TraceBundle { traceNdjson: string networkNdjson: Buffer transcriptMd: string + mutationsNdjson: Buffer resources: TraceZipResource[] } @@ -368,9 +378,7 @@ function buildTraceBundle( // subsequent actions render at positive deltas in the trace viewer. const firstCommandTs = trace.commands[0]?.timestamp const wallTime = opts.wallTimeOverride ?? firstCommandTs ?? Date.now() - const idPrefix = shortId(opts.sessionId) - const contextId = `context@${idPrefix}` - const pageId = `page@${idPrefix}` + const { contextId, pageId } = allocateTraceIds(opts.sessionId) const ctxOptions = buildContextOptions(trace, contextId, wallTime) const dense = buildDenseScreencast( trace.screencastFrames ?? [], @@ -400,6 +408,7 @@ function buildTraceBundle( wallTime, ctxOptions.title ), + mutationsNdjson: buildMutationsNdjson(trace.mutations ?? []).ndjson, resources: [ ...buildSnapshotResources(trace.actionSnapshots ?? [], pageId), ...dense.resources, @@ -422,7 +431,8 @@ export async function exportTraceZip( traceNdjson: bundle.traceNdjson, networkNdjson: bundle.networkNdjson, resources: bundle.resources, - transcriptMd: bundle.transcriptMd + transcriptMd: bundle.transcriptMd, + mutationsNdjson: bundle.mutationsNdjson }) } @@ -450,6 +460,12 @@ async function exportTraceDirectory( bundle.networkNdjson ) : Promise.resolve(), + bundle.mutationsNdjson.length + ? fs.writeFile( + path.join(targetDir, 'trace.mutations'), + bundle.mutationsNdjson + ) + : Promise.resolve(), ...bundle.resources.map((r) => fs.writeFile(path.join(targetDir, 'resources', r.resourceName), r.data) ) diff --git a/packages/core/src/trace-mutations.ts b/packages/core/src/trace-mutations.ts new file mode 100644 index 00000000..d63ce094 --- /dev/null +++ b/packages/core/src/trace-mutations.ts @@ -0,0 +1,62 @@ +// Serialize the captured DOM mutation stream into the trace zip's +// `trace.mutations` NDJSON entry so the offline player can replay DOM +// time-travel. Standard trace viewers ignore the unknown entry, so it's +// compat-safe. Keep-earliest under the byte cap — the initial full-DOM +// childList and early diffs survive so replay-from-start stays intact; only +// late mutations drop, and a trailing sentinel records how many. + +import type { + MutationsTruncationMarker, + TraceMutation +} from '@wdio/devtools-shared' + +/** Ceiling on the serialized `trace.mutations` payload — keeps archives bounded + * on mutation-heavy SPAs. Late mutations drop first (replay-from-start holds). */ +export const MAX_MUTATIONS_NDJSON_BYTES = 50 * 1024 * 1024 + +export interface MutationsNdjsonResult { + /** NDJSON payload (one mutation per line, optional trailing marker). Empty + * buffer when there are no mutations. */ + ndjson: Buffer + truncated: boolean + /** Count actually written (excludes the dropped tail and the marker line). */ + written: number +} + +/** + * Serialize mutations to NDJSON under `cap` bytes, keeping the earliest. The + * first mutation is always emitted (even if it alone exceeds the cap) so the + * initial full-DOM snapshot is never lost; when any are dropped a + * `MutationsTruncationMarker` line is appended. + */ +export function buildMutationsNdjson( + mutations: readonly TraceMutation[], + cap: number = MAX_MUTATIONS_NDJSON_BYTES +): MutationsNdjsonResult { + if (!mutations.length) { + return { ndjson: Buffer.alloc(0), truncated: false, written: 0 } + } + const lines: string[] = [] + let bytes = 0 + for (const mutation of mutations) { + const line = JSON.stringify(mutation) + // +1 for the '\n' that will join this line to the previous one. + const add = Buffer.byteLength(line, 'utf8') + (lines.length ? 1 : 0) + if (lines.length > 0 && bytes + add > cap) { + break + } + lines.push(line) + bytes += add + } + const written = lines.length + const dropped = mutations.length - written + if (dropped > 0) { + const marker: MutationsTruncationMarker = { __truncated__: true, dropped } + lines.push(JSON.stringify(marker)) + } + return { + ndjson: Buffer.from(lines.join('\n'), 'utf8'), + truncated: dropped > 0, + written + } +} diff --git a/packages/core/src/trace-zip-writer.ts b/packages/core/src/trace-zip-writer.ts index 13849f13..8e3e524e 100644 --- a/packages/core/src/trace-zip-writer.ts +++ b/packages/core/src/trace-zip-writer.ts @@ -16,6 +16,8 @@ export interface TraceZipInputs { networkNdjson: Buffer /** Human/LLM-readable Markdown transcript. */ transcriptMd?: string + /** NDJSON DOM mutation stream (one mutation per line). Omitted/empty → no entry. */ + mutationsNdjson?: Buffer /** Files written under `resources/` — typically screenshots + element snapshots. */ resources: TraceZipResource[] } @@ -31,6 +33,9 @@ export function buildTraceZip(inputs: TraceZipInputs): Promise<Buffer> { 'transcript.md' ) } + if (inputs.mutationsNdjson?.length) { + zipFile.addBuffer(inputs.mutationsNdjson, 'trace.mutations') + } for (const resource of inputs.resources) { zipFile.addBuffer(resource.data, `resources/${resource.resourceName}`) } diff --git a/packages/core/tests/trace-exporter.test.ts b/packages/core/tests/trace-exporter.test.ts index 365eb74c..a6a36503 100644 --- a/packages/core/tests/trace-exporter.test.ts +++ b/packages/core/tests/trace-exporter.test.ts @@ -523,3 +523,70 @@ describe('exported trace stream — dense filmstrip', () => { await fs.rm(outputDir, { recursive: true, force: true }) }) }) + +describe('exported trace stream — DOM mutations', () => { + const baseCapturer = ( + mutations: TraceCapturer['mutations'] + ): TraceCapturer => ({ + mutations, + traceLogs: [], + consoleLogs: [], + networkRequests: [], + commandsLog: [ + { + command: 'url', + args: ['https://example.test'], + timestamp: 1200, + startTime: 1150 + } + ], + sources: new Map(), + metadata: { type: TraceType.Standalone }, + startWallTime: 1000 + }) + + it('writes captured mutations to a trace.mutations entry, one per line', async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'trace-mut-')) + const dir = await writeTraceZip( + baseCapturer([ + { + type: 'childList', + addedNodes: [{ tag: 'html' }], + removedNodes: [], + timestamp: 1000 + }, + { + type: 'attributes', + target: 'body', + attributeName: 'class', + attributeValue: 'ready', + addedNodes: [], + removedNodes: [], + timestamp: 1100 + } + ]), + { outputDir, sessionId: 'abc12345', format: 'ndjson-directory' } + ) + const lines = (await fs.readFile(path.join(dir, 'trace.mutations'), 'utf8')) + .trim() + .split('\n') + .map((l) => JSON.parse(l) as Record<string, unknown>) + expect(lines).toHaveLength(2) + expect(lines[0]!.type).toBe('childList') + expect(lines[1]!.target).toBe('body') + await fs.rm(outputDir, { recursive: true, force: true }) + }) + + it('writes no trace.mutations entry when there are none', async () => { + const outputDir = await fs.mkdtemp( + path.join(os.tmpdir(), 'trace-mut-none-') + ) + const dir = await writeTraceZip(baseCapturer([]), { + outputDir, + sessionId: 'abc12345', + format: 'ndjson-directory' + }) + await expect(fs.access(path.join(dir, 'trace.mutations'))).rejects.toThrow() + await fs.rm(outputDir, { recursive: true, force: true }) + }) +}) diff --git a/packages/core/tests/trace-mutations.test.ts b/packages/core/tests/trace-mutations.test.ts new file mode 100644 index 00000000..9dfd5bdd --- /dev/null +++ b/packages/core/tests/trace-mutations.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect } from 'vitest' +import { buildMutationsNdjson } from '@wdio/devtools-core' +import { + isMutationsTruncationMarker, + type TraceMutation +} from '@wdio/devtools-shared' + +function mutation(overrides: Partial<TraceMutation> = {}): TraceMutation { + return { + type: 'childList', + addedNodes: [], + removedNodes: [], + timestamp: 1000, + ...overrides + } +} + +describe('buildMutationsNdjson', () => { + it('returns an empty buffer for no mutations', () => { + const result = buildMutationsNdjson([]) + expect(result.ndjson.byteLength).toBe(0) + expect(result.truncated).toBe(false) + expect(result.written).toBe(0) + }) + + it('serializes one JSON mutation per line with no marker under the cap', () => { + const mutations = [ + mutation({ type: 'childList', addedNodes: [{ tag: 'html' }] }), + mutation({ type: 'attributes', target: 'body', timestamp: 1100 }) + ] + const result = buildMutationsNdjson(mutations) + const lines = result.ndjson.toString('utf8').split('\n') + expect(lines).toHaveLength(2) + expect(result.written).toBe(2) + expect(result.truncated).toBe(false) + expect(lines.some((l) => l.includes('__truncated__'))).toBe(false) + expect((JSON.parse(lines[0]!) as TraceMutation).addedNodes).toEqual([ + { tag: 'html' } + ]) + }) + + it('keeps the earliest under the cap and appends a truncation marker', () => { + const mutations = Array.from({ length: 5 }, (_, i) => + mutation({ timestamp: 1000 + i, target: `n${i}` }) + ) + // Cap sized to fit exactly the first two mutation lines. + const cap = buildMutationsNdjson(mutations.slice(0, 2)).ndjson.byteLength + const result = buildMutationsNdjson(mutations, cap) + const lines = result.ndjson.toString('utf8').split('\n') + expect(result.written).toBe(2) + expect(result.truncated).toBe(true) + expect(JSON.parse(lines.at(-1)!)).toEqual({ + __truncated__: true, + dropped: 3 + }) + // Earliest retained, latest dropped. + expect(result.ndjson.toString('utf8')).toContain('"n0"') + expect(result.ndjson.toString('utf8')).not.toContain('"n4"') + }) + + it('always emits the first mutation even when it alone exceeds the cap', () => { + const big = mutation({ addedNodes: [{ html: 'x'.repeat(200) }] }) + const result = buildMutationsNdjson( + [big, mutation({ timestamp: 2000 })], + 10 + ) + const lines = result.ndjson.toString('utf8').split('\n') + expect(result.written).toBe(1) + expect(result.truncated).toBe(true) + expect(JSON.parse(lines.at(-1)!)).toEqual({ + __truncated__: true, + dropped: 1 + }) + }) +}) + +describe('isMutationsTruncationMarker', () => { + it('recognizes the sentinel and rejects mutations / non-objects', () => { + expect( + isMutationsTruncationMarker({ __truncated__: true, dropped: 3 }) + ).toBe(true) + expect(isMutationsTruncationMarker(mutation())).toBe(false) + expect(isMutationsTruncationMarker({ __truncated__: false })).toBe(false) + expect(isMutationsTruncationMarker(null)).toBe(false) + expect(isMutationsTruncationMarker('x')).toBe(false) + }) +}) diff --git a/packages/script/src/index.ts b/packages/script/src/index.ts index e1a0bc2a..82b2bae9 100644 --- a/packages/script/src/index.ts +++ b/packages/script/src/index.ts @@ -79,6 +79,40 @@ try { } }) observer.observe(document.body, config) + + // Form-field state (value / checked) lives on element PROPERTIES, which the + // MutationObserver never reports — so a replayed page shows empty inputs even + // after a fill. Capture input/change and emit a synthetic attribute mutation + // carrying the live value so the replay shows what was typed or selected. + const captureFieldState = (target: EventTarget | null) => { + if (!(target instanceof Element)) { + return + } + const tag = target.tagName + if (tag !== 'INPUT' && tag !== 'TEXTAREA' && tag !== 'SELECT') { + return + } + const ref = getRef(target) + if (!ref) { + return + } + const el = target as HTMLInputElement + const checkable = + tag === 'INPUT' && (el.type === 'checkbox' || el.type === 'radio') + collector.captureMutation([ + { + type: 'attributes', + target: ref, + attributeName: checkable ? 'checked' : 'value', + attributeValue: checkable ? String(el.checked) : String(el.value), + addedNodes: [], + removedNodes: [], + timestamp: Date.now() + } as TraceMutation + ]) + } + document.addEventListener('input', (e) => captureFieldState(e.target), true) + document.addEventListener('change', (e) => captureFieldState(e.target), true) } catch (err) { collector.captureError(err as Error) } diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index b6d37d30..8598cb04 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -390,6 +390,29 @@ export interface TraceMutation { url?: string } +/** + * Trailing sentinel line in a `trace.mutations` NDJSON stream, marking that + * `dropped` late mutations were discarded under the size cap. A distinct shape + * (not a `TraceMutation.type`) so it never collides with a real mutation. This + * is the writer↔reader contract shared by core's exporter and the backend + * reader, so both agree on the key. + */ +export interface MutationsTruncationMarker { + __truncated__: true + dropped: number +} + +/** True when an NDJSON entry is the truncation sentinel, not a mutation. */ +export function isMutationsTruncationMarker( + entry: unknown +): entry is MutationsTruncationMarker { + return ( + typeof entry === 'object' && + entry !== null && + (entry as { __truncated__?: unknown }).__truncated__ === true + ) +} + /** * Captured at each user-facing action boundary in `trace` mode. Feeds the * downstream trace.zip exporter (Phase 4). `screenshot` is base64-encoded JPEG. From ada2adf284053083be46859237751198901eedc7 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 17 Jul 2026 18:26:40 +0530 Subject: [PATCH 78/91] fix(service): re-capture DOM across reloadSession and before navigation --- packages/service/src/index.ts | 23 ++++++++++++++++- packages/service/src/session.ts | 17 +++++++++++-- packages/service/tests/index.test.ts | 1 + packages/service/tests/session.test.ts | 35 ++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index ef5320d6..3161e4f2 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -59,7 +59,11 @@ import { type ServiceOptions, TraceType } from './types.js' -import { CONTEXT_CHANGE_COMMANDS, INTERNAL_COMMANDS } from './constants.js' +import { + CONTEXT_CHANGE_COMMANDS, + INTERNAL_COMMANDS, + PAGE_TRANSITION_COMMANDS +} from './constants.js' import { isNativeMobile } from './mobile.js' import { detectInvocationConfigPath } from './standalone.js' @@ -654,6 +658,15 @@ export default class DevToolsHookService implements Services.ServiceInstance { this.#screencastRecorder?.setStartMarker() this.#sessionCapturer.sendUpstream('metadata', { url: args[0] }) } + // Flush the outgoing page's buffered mutations (e.g. field edits from prior + // fills — value/checked changes fire no page transition) BEFORE a navigating + // command discards its collector, else the replay shows empty inputs. + if ( + this.#options.mode === 'trace' && + PAGE_TRANSITION_COMMANDS.includes(command) + ) { + await this.#sessionCapturer.captureTrace(this.#browser) + } // Smart stack filtering to detect top-level user commands. Error.stackTraceLimit = 20 const stack = parse(new Error('')).reverse() @@ -772,6 +785,14 @@ export default class DevToolsHookService implements Services.ServiceInstance { * on the new session so the second scenario is also covered. */ async onReload(oldSessionId: string, _newSessionId: string) { + // reloadSession starts a fresh session with no preload script (BiDi preload + // scripts are per-session), so DOM-mutation capture would silently stop + // after the first session — every post-reload scenario would replay the + // prior session's last DOM. Re-arm capture for the new session here, + // independent of screencast, so it runs before the early-return below. + this.#sessionCapturer.resetScriptInjection() + await this.#ensureInjected('reloadSession') + if (!this.#shouldRecordScreencast() || !this.#browser) { return } diff --git a/packages/service/src/session.ts b/packages/service/src/session.ts index 6e8b2b9a..51632586 100644 --- a/packages/service/src/session.ts +++ b/packages/service/src/session.ts @@ -203,7 +203,7 @@ export class SessionCapturer extends SessionCapturerBase { ) { await Promise.all([ this.#capturePerformance(browser, commandLogEntry, args), - this.#captureTrace(browser) + this.captureTrace(browser) ]) } } @@ -357,7 +357,20 @@ export class SessionCapturer extends SessionCapturerBase { log.info('✓ Script injected successfully') } - async #captureTrace(browser: WebdriverIO.Browser) { + /** Clear the per-session injection guard so the next `injectScript` re-adds + * the preload script. BiDi preload scripts are scoped to one session, so + * after `reloadSession()` the new session has none — without this, DOM + * capture silently stops after the first session. The guard itself still + * prevents double-adding within a single session. */ + resetScriptInjection() { + this.#isScriptInjected = false + } + + /** Drain the current page's buffered trace data (mutations/console/network) + * into the capturer. Public so the plugin can flush BEFORE a navigating + * command, capturing the outgoing page's field edits (value/checked + * mutations fire no page transition) before its collector is discarded. */ + async captureTrace(browser: WebdriverIO.Browser) { if (!this.#isScriptInjected) { log.warn('Script not injected, skipping trace capture') return diff --git a/packages/service/tests/index.test.ts b/packages/service/tests/index.test.ts index 37fc2c86..99ba04da 100644 --- a/packages/service/tests/index.test.ts +++ b/packages/service/tests/index.test.ts @@ -27,6 +27,7 @@ const mockSessionCapturerInstance = { afterCommand: vi.fn(), sendUpstream: vi.fn(), injectScript: vi.fn().mockResolvedValue(undefined), + resetScriptInjection: vi.fn(), captureSource: vi.fn(), captureAssertCommand: vi.fn(), cleanup: vi.fn(), diff --git a/packages/service/tests/session.test.ts b/packages/service/tests/session.test.ts index 1f1185f2..e3a28bba 100644 --- a/packages/service/tests/session.test.ts +++ b/packages/service/tests/session.test.ts @@ -387,6 +387,41 @@ describe('SessionCapturer', () => { }) }) + describe('resetScriptInjection', () => { + // node:fs/promises is auto-mocked at module scope, so stub the preload-file + // read injectScript performs (otherwise readFile → undefined → throws). + beforeEach(() => { + vi.mocked(fs.readFile).mockResolvedValue( + '// preload' as unknown as Buffer + ) + }) + + // Minimal BiDi browser stub — injectScript only needs isBidi + the preload + // hook; the script body comes from the mocked readFile above. + const bidiBrowser = () => + ({ + isBidi: true, + scriptAddPreloadScript: vi.fn().mockResolvedValue(undefined) + }) as unknown as WebdriverIO.Browser + + it('guards against double preload injection within one session', async () => { + const capturer = new SessionCapturer() + const browser = bidiBrowser() + await capturer.injectScript(browser) + await capturer.injectScript(browser) + expect(browser.scriptAddPreloadScript).toHaveBeenCalledTimes(1) + }) + + it('re-adds the preload script for the new session after a reload reset', async () => { + const capturer = new SessionCapturer() + const browser = bidiBrowser() + await capturer.injectScript(browser) + capturer.resetScriptInjection() + await capturer.injectScript(browser) + expect(browser.scriptAddPreloadScript).toHaveBeenCalledTimes(2) + }) + }) + describe('integration', () => { it('should handle complete session capture workflow', async () => { const capturer = new SessionCapturer() From 69dd363bf7b6906ee4f44a2cb9ae3d07ba1541ff Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 17 Jul 2026 18:27:44 +0530 Subject: [PATCH 79/91] fix(core): only capture asserts fired directly by user code --- packages/core/src/assert-patcher.ts | 25 +++++++---- packages/core/src/stack.ts | 31 +++++++++++--- packages/core/tests/assert-patcher.test.ts | 23 +++++++++-- packages/core/tests/stack.test.ts | 48 ++++++++++++++++++++++ 4 files changed, 111 insertions(+), 16 deletions(-) create mode 100644 packages/core/tests/stack.test.ts diff --git a/packages/core/src/assert-patcher.ts b/packages/core/src/assert-patcher.ts index c171c82a..0c6b2753 100644 --- a/packages/core/src/assert-patcher.ts +++ b/packages/core/src/assert-patcher.ts @@ -4,7 +4,7 @@ import { type CollapsedAssertResult, type CommandLog } from '@wdio/devtools-shared' -import { getCallSourceFromStack } from './stack.js' +import { getCallSourceFromStack, isAssertFromUserCode } from './stack.js' import { toError } from './error.js' import { stripAnsi } from './console.js' @@ -121,12 +121,15 @@ function describeAssertFailure(err: unknown): { function makeAssertEmitters( methodName: string, args: unknown[], - onCommand: (cmd: CapturedAssert) => void + onCommand: (cmd: CapturedAssert) => void, + callerStack: string | undefined ): { passed: () => void; failed: (err: unknown) => void } { - const callInfo = getCallSourceFromStack() - // No user-code frame means the assert came from a dependency or framework - // internal, not the user's test — drop it so it never reaches the trace. - if (callInfo.filePath === undefined) { + const callInfo = getCallSourceFromStack(callerStack) + // Drop asserts the user's test didn't fire directly: either no user-code frame + // on the stack at all, or the assert's immediate caller is a dependency (a + // framework/library assert firing during a user operation) — both are noise. + // callerStack is captured in patchedAssert so the frame offsets line up. + if (callInfo.filePath === undefined || !isAssertFromUserCode(callerStack)) { return { passed: () => {}, failed: () => {} } } const startedAt = Date.now() @@ -156,7 +159,15 @@ function makePatchedAssertMethod( onCommand: (cmd: CapturedAssert) => void ): (...args: unknown[]) => unknown { return function patchedAssert(this: unknown, ...args: unknown[]) { - const { passed, failed } = makeAssertEmitters(methodName, args, onCommand) + // Captured HERE so frames[0] is this wrapper and frames[1] is the assert's + // caller — a fixed offset isAssertFromUserCode reads (minification-robust). + const callerStack = new Error().stack + const { passed, failed } = makeAssertEmitters( + methodName, + args, + onCommand, + callerStack + ) let result: unknown // Node's internalMatch dispatches on `fn === assert.match` (Node ≤20), so // a wrapper installed on that property silently inverts `match` into diff --git a/packages/core/src/stack.ts b/packages/core/src/stack.ts index b414d27c..9813a5fb 100644 --- a/packages/core/src/stack.ts +++ b/packages/core/src/stack.ts @@ -33,17 +33,38 @@ export function normalizeFilePath(filePath: string): string { } } +/** + * True when a tracked assert was called DIRECTLY by user code. `stack` MUST be + * captured inside the `patchedAssert` wrapper (`new Error().stack` on its first + * line), so frames[0] is the wrapper and frames[1] is whoever called the + * assert. A non-user immediate caller means a framework/dependency assert fired + * *during* a user operation (which `getCallSourceFromStack` would otherwise + * mis-attribute to the far-off user frame and surface as a noisy row) — drop it. + * Uses the fixed frame offset rather than the wrapper's name, so it survives a + * bundler minifying/renaming the wrapper. No stack → keep (never lose a real + * assert to an absent stack). + */ +export function isAssertFromUserCode(stack: string | undefined): boolean { + if (!stack) { + return true + } + const caller = parseStackTrace(stack)[1] + return !!caller && isUserCodeFrame(caller) +} + /** * Capture `{ filePath, callSource }` for the first user-code frame on the - * current stack. `callSource` is `<file>:<line>` for the UI's source-location - * displays; returns `'unknown:0'` (and `undefined` filePath) when no user - * frame can be found. + * stack. `callSource` is `<file>:<line>` for the UI's source-location displays; + * returns `'unknown:0'` (and `undefined` filePath) when no user frame can be + * found. `stack` defaults to the live stack; callers with a pre-captured stack + * (e.g. the assert wrapper) pass it so the frame offsets line up. */ -export function getCallSourceFromStack(): { +export function getCallSourceFromStack( + stack: string | undefined = new Error().stack +): { filePath: string | undefined callSource: string } { - const stack = new Error().stack if (!stack) { return { filePath: undefined, callSource: 'unknown:0' } } diff --git a/packages/core/tests/assert-patcher.test.ts b/packages/core/tests/assert-patcher.test.ts index 9fb79d9b..5eccc4c7 100644 --- a/packages/core/tests/assert-patcher.test.ts +++ b/packages/core/tests/assert-patcher.test.ts @@ -9,12 +9,14 @@ import { safeSerializeAssertArg, type CapturedAssert } from '../src/assert-patcher.js' -import { getCallSourceFromStack } from '../src/stack.js' +import { getCallSourceFromStack, isAssertFromUserCode } from '../src/stack.js' -// Stub the call-source resolver so tests choose whether an assert looks like -// it originated in user code (a frame) or a dependency/internal (no frame). +// Stub the stack resolvers so tests choose whether an assert looks like it +// originated in user code (a frame + a direct user caller) or a +// dependency/internal (no frame, or an indirect framework caller). vi.mock('../src/stack.js', () => ({ - getCallSourceFromStack: vi.fn() + getCallSourceFromStack: vi.fn(), + isAssertFromUserCode: vi.fn() })) const USER_FRAME = { @@ -44,6 +46,7 @@ beforeEach(() => { } // Default: every assert looks like it came from user code so captures fire. vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) + vi.mocked(isAssertFromUserCode).mockReturnValue(true) }) describe('safeSerializeAssertArg', () => { @@ -165,6 +168,18 @@ describe('patchNodeAssert', () => { expect(captured).toHaveLength(0) }) + it('drops a framework assert whose immediate caller is a dependency', () => { + // A user frame exists deep in the stack, but the assert was fired by a + // dependency during a user operation — isAssertFromUserCode says no. + vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) + vi.mocked(isAssertFromUserCode).mockReturnValue(false) + const captured: CapturedAssert[] = [] + patchNodeAssert((c) => captured.push(c)) + assert.equal(1, 1) + expect(() => assert.equal(1, 2)).toThrow() + expect(captured).toHaveLength(0) + }) + it('keeps a user-origin assert and carries its callSource through', () => { vi.mocked(getCallSourceFromStack).mockReturnValue(USER_FRAME) const captured: CapturedAssert[] = [] diff --git a/packages/core/tests/stack.test.ts b/packages/core/tests/stack.test.ts new file mode 100644 index 00000000..b13d8185 --- /dev/null +++ b/packages/core/tests/stack.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest' +import { isAssertFromUserCode } from '@wdio/devtools-core' + +// Synthetic V8 stacks as captured INSIDE patchedAssert: frames[0] is the +// wrapper (any name — may be minified), frames[1] is whoever called the assert. +const stackWithCaller = (callerFile: string, wrapperName = 'patchedAssert') => + [ + 'Error', + ` at ${wrapperName} (/repo/packages/core/src/assert-patcher.ts:158:30)`, + ` at Object.<anonymous> (${callerFile}:5:10)`, + ' at Module._compile (node:internal/modules/cjs/loader:1234:14)' + ].join('\n') + +describe('isAssertFromUserCode', () => { + it('keeps an assert whose immediate caller is user code', () => { + expect( + isAssertFromUserCode(stackWithCaller('/repo/tests/login.spec.ts')) + ).toBe(true) + }) + + it('drops an assert fired by a dependency during a user operation', () => { + expect( + isAssertFromUserCode( + stackWithCaller('/repo/node_modules/webdriverio/build/session.js') + ) + ).toBe(false) + }) + + it('drops one whose immediate caller is bundled dist output', () => { + expect( + isAssertFromUserCode(stackWithCaller('/repo/packages/core/dist/x.js')) + ).toBe(false) + }) + + it('works regardless of the wrapper frame name (minified bundle)', () => { + // The wrapper is renamed by the bundler; the fixed frame offset still holds. + expect( + isAssertFromUserCode(stackWithCaller('/repo/tests/login.spec.ts', 'n2')) + ).toBe(true) + expect( + isAssertFromUserCode(stackWithCaller('/repo/node_modules/x/i.js', 'n2')) + ).toBe(false) + }) + + it('returns true when there is no stack', () => { + expect(isAssertFromUserCode(undefined)).toBe(true) + }) +}) From 8c47aac757e5cbff3a9b9fac1c1fa6b8ef5279dd Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Fri, 17 Jul 2026 19:01:39 +0530 Subject: [PATCH 80/91] feat(app): trace player Transcript tab + Copy-for-LLM --- packages/app/src/components/workbench.ts | 6 + .../src/components/workbench/transcript.ts | 138 ++++++++++++++++++ packages/app/src/controller/DataManager.ts | 6 +- packages/app/src/controller/context.ts | 6 + packages/backend/src/trace-reader.ts | 6 +- packages/backend/tests/trace-reader.test.ts | 20 +++ packages/shared/src/trace-player.ts | 2 + 7 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 packages/app/src/components/workbench/transcript.ts diff --git a/packages/app/src/components/workbench.ts b/packages/app/src/components/workbench.ts index a97b20c9..c3f43bc0 100644 --- a/packages/app/src/components/workbench.ts +++ b/packages/app/src/components/workbench.ts @@ -33,6 +33,7 @@ import './workbench/console.js' import './workbench/metadata.js' import './workbench/network.js' import './workbench/errors.js' +import './workbench/transcript.js' import './workbench/compare.js' import './browser/snapshot.js' import './browser/trace-timeline.js' @@ -368,6 +369,11 @@ export class DevtoolsWorkbench extends Element { > <wdio-devtools-errors></wdio-devtools-errors> </wdio-devtools-tab> + ${this.playerMode + ? html`<wdio-devtools-tab label="Transcript"> + <wdio-devtools-transcript></wdio-devtools-transcript> + </wdio-devtools-tab>` + : nothing} ${this.#renderCompareTabIfAvailable()} ` } diff --git a/packages/app/src/components/workbench/transcript.ts b/packages/app/src/components/workbench/transcript.ts new file mode 100644 index 00000000..61227814 --- /dev/null +++ b/packages/app/src/components/workbench/transcript.ts @@ -0,0 +1,138 @@ +import { Element } from '@core/element' +import { html, css, nothing } from 'lit' +import { customElement, state } from 'lit/decorators.js' +import { consume } from '@lit/context' + +import type { CommandLog } from '@wdio/devtools-shared' +import { transcriptContext, commandContext } from '../../controller/context.js' + +import '../placeholder.js' + +const COMPONENT = 'wdio-devtools-transcript' + +/** Player-only panel: renders the run's `transcript.md` and offers a one-click + * "Copy prompt" that bundles the transcript with any failing-command errors — + * paste-ready context for an LLM. */ +@customElement(COMPONENT) +export class DevtoolsTranscript extends Element { + @consume({ context: transcriptContext, subscribe: true }) + transcript: string | undefined = undefined + + @consume({ context: commandContext, subscribe: true }) + commands: CommandLog[] = [] + + @state() + private copied = false + + static styles = [ + ...Element.styles, + css` + :host { + display: flex; + flex-direction: column; + width: 100%; + height: 100%; + overflow: hidden; + } + .bar { + display: flex; + justify-content: flex-end; + padding: 8px 12px; + border-bottom: 1px solid var(--vscode-panel-border); + } + button { + font-family: inherit; + font-size: 11px; + padding: 5px 12px; + border: 1px solid var(--vscode-panel-border); + border-radius: 8px; + background: var(--vscode-input-background); + color: var(--vscode-foreground); + cursor: pointer; + line-height: 1; + } + button:hover { + border-color: var(--accent); + } + button.copied { + color: var(--vscode-charts-green); + border-color: var(--vscode-charts-green); + } + pre { + margin: 0; + padding: 14px; + flex: 1; + overflow: auto; + font-family: var(--vscode-editor-font-family); + font-size: 12px; + line-height: 1.6; + color: var(--vscode-foreground); + white-space: pre-wrap; + word-break: break-word; + } + ` + ] + + #errorMessage(command: CommandLog): string { + const err = command.error + if (err && typeof err === 'object' && 'message' in err) { + return String((err as { message: unknown }).message) + } + return String(err) + } + + /** transcript + a Failures section built from commands carrying an error. */ + #buildPrompt(): string { + const parts: string[] = [] + if (this.transcript) { + parts.push(this.transcript.trim()) + } + const failures = (this.commands ?? []).filter((c) => c.error) + if (failures.length) { + parts.push( + '## Failures\n' + + failures + .map((f) => `- ${f.title ?? f.command}: ${this.#errorMessage(f)}`) + .join('\n') + ) + } + return parts.join('\n\n') + } + + async #copy() { + try { + await navigator.clipboard.writeText(this.#buildPrompt()) + this.copied = true + setTimeout(() => { + this.copied = false + }, 1500) + } catch { + /* clipboard blocked (no user gesture / permissions) — no-op */ + } + } + + render() { + if (!this.transcript) { + return html`<wdio-devtools-placeholder></wdio-devtools-placeholder>` + } + return html` + <div class="bar"> + <button + class=${this.copied ? 'copied' : ''} + @click=${() => this.#copy()} + title="Copy the transcript + failures as an LLM prompt" + > + ${this.copied ? 'Copied' : 'Copy prompt'} + </button> + </div> + <pre>${this.transcript}</pre> + ${nothing} + ` + } +} + +declare global { + interface HTMLElementTagNameMap { + [COMPONENT]: DevtoolsTranscript + } +} diff --git a/packages/app/src/controller/DataManager.ts b/packages/app/src/controller/DataManager.ts index 2af04a25..e5e40972 100644 --- a/packages/app/src/controller/DataManager.ts +++ b/packages/app/src/controller/DataManager.ts @@ -22,7 +22,8 @@ import { baselineContext, selectedTestUidContext, framesContext, - actionGroupsContext + actionGroupsContext, + transcriptContext } from './context.js' import { BASELINE_WS_SCOPE, TRACE_API, WS_SCOPE } from '@wdio/devtools-shared' import { CACHE_ID } from './constants.js' @@ -66,6 +67,7 @@ export class DataManagerController implements ReactiveController { selectedTestUidContextProvider: ContextProvider<typeof selectedTestUidContext> framesContextProvider: ContextProvider<typeof framesContext> actionGroupsContextProvider: ContextProvider<typeof actionGroupsContext> + transcriptContextProvider: ContextProvider<typeof transcriptContext> #playerMode = false @@ -100,6 +102,7 @@ export class DataManagerController implements ReactiveController { actionGroupsContext, undefined ) + this.transcriptContextProvider = this.#provide(transcriptContext, undefined) } #provide<T extends Context<unknown, unknown>>( @@ -216,6 +219,7 @@ export class DataManagerController implements ReactiveController { this.#playerMode = true this.framesContextProvider.setValue(data.frames) this.actionGroupsContextProvider.setValue(data.groups) + this.transcriptContextProvider.setValue(data.transcript) this.loadTraceFile(data.trace) this.hasConnectionProvider.setValue(true) this.#host.requestUpdate() diff --git a/packages/app/src/controller/context.ts b/packages/app/src/controller/context.ts index 151822ea..91fe35de 100644 --- a/packages/app/src/controller/context.ts +++ b/packages/app/src/controller/context.ts @@ -58,3 +58,9 @@ export const framesContext = createContext<TracePlayerFrame[]>( export const actionGroupsContext = createContext< TraceActionChild[] | undefined >(Symbol('actionGroupsContext')) + +/** Markdown run transcript from a loaded trace (`transcript.md`); undefined in + * live mode or when the zip carried none. */ +export const transcriptContext = createContext<string | undefined>( + Symbol('transcriptContext') +) diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index 3b4f9ef2..439bc779 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -334,6 +334,9 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { indexByCallId ) const startTime = earliestWallTime(merged.ctxs) + const transcript = files['transcript.md'] + ? strFromU8(files['transcript.md']) + : undefined const trace: TraceLog = { mutations: parseMutationStreams( files, @@ -357,7 +360,8 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { frames, startTime, duration: Math.max(0, Math.max(frameMax, cmdMax) - startTime), - ...(groups ? { groups } : {}) + ...(groups ? { groups } : {}), + ...(transcript ? { transcript } : {}) } } diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index 2dfc7a7b..858f56a6 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -252,6 +252,26 @@ describe('parseTraceZip', () => { expect(trace.mutations[1]).toMatchObject({ target: 'body' }) }) + it('reads transcript.md into the player payload when present', () => { + const zip = zipSync({ + 'trace.trace': toNdjson([ + { + type: 'context-options', + wallTime: WALL_TIME, + browserName: 'chrome', + contextId: 'context@abcd1234', + options: { viewport: { width: 1024, height: 768 } } + } + ]), + 'transcript.md': strToU8('# Session\n\n1. Page.navigate("https://x")') + }) + expect(parseTraceZip(zip).transcript).toContain('Page.navigate') + }) + + it('leaves transcript undefined when the zip carries none', () => { + expect(parseTraceZip(fixtureZip()).transcript).toBeUndefined() + }) + it('restores callSource and sources from stack frames + src resources', () => { const { trace } = parseTraceZip(fixtureZip()) const click = trace.commands.find((c) => c.command === 'click') diff --git a/packages/shared/src/trace-player.ts b/packages/shared/src/trace-player.ts index ef0eaafc..ec02fd29 100644 --- a/packages/shared/src/trace-player.ts +++ b/packages/shared/src/trace-player.ts @@ -49,4 +49,6 @@ export interface TracePlayerData { /** Root children of the action tree, chronological. Absent when the zip * carried no structural steps — the player then renders the flat list. */ groups?: TraceActionChild[] + /** Markdown run transcript (`transcript.md`), when the zip carried one. */ + transcript?: string } From a063f4cfebb910f264c48d663fab369e2733ef8f Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 00:55:26 +0530 Subject: [PATCH 81/91] feat(app): trace player A11y tab + element overlay (pick-locator) --- examples/wdio/cucumber/wdio.conf.ts | 1 + .../src/components/browser/element-overlay.ts | 64 ++++++ .../app/src/components/browser/snapshot.ts | 137 ++++++++++-- packages/app/src/components/workbench.ts | 10 +- .../app/src/components/workbench/a11y-tree.ts | 207 ++++++++++++++++++ packages/backend/src/trace-reader-types.ts | 10 + packages/backend/src/trace-reader.ts | 51 ++++- packages/backend/tests/trace-reader.test.ts | 33 +++ packages/shared/src/types.ts | 4 + 9 files changed, 493 insertions(+), 24 deletions(-) create mode 100644 packages/app/src/components/browser/element-overlay.ts create mode 100644 packages/app/src/components/workbench/a11y-tree.ts diff --git a/examples/wdio/cucumber/wdio.conf.ts b/examples/wdio/cucumber/wdio.conf.ts index 75a1f5b9..814d0b03 100644 --- a/examples/wdio/cucumber/wdio.conf.ts +++ b/examples/wdio/cucumber/wdio.conf.ts @@ -133,6 +133,7 @@ export const config: Options.Testrunner = { 'devtools', { mode: 'trace' as const, + filmstrip: true, // tracePolicy: 'retain-on-failure', traceGranularity: 'test' as const, // NEW: per-test screenshot + video, attached inline to Allure. diff --git a/packages/app/src/components/browser/element-overlay.ts b/packages/app/src/components/browser/element-overlay.ts new file mode 100644 index 00000000..3ce1129e --- /dev/null +++ b/packages/app/src/components/browser/element-overlay.ts @@ -0,0 +1,64 @@ +// Draws the "element overlay" — a labeled, click-to-copy box over each locator +// the test interacted with — INSIDE the replayed iframe's document, so the +// boxes inherit the iframe's scale transform (no manual coordinate math). Kept +// out of snapshot.ts so that file stays focused on capture/replay. + +const OVERLAY_CLASS = '__wdio-el-overlay__' + +export function clearElementOverlay( + iframe: HTMLIFrameElement | null | undefined +): void { + iframe?.contentDocument + ?.querySelectorAll(`.${OVERLAY_CLASS}`) + .forEach((node) => node.remove()) +} + +/** + * Outline each selector that resolves in the replayed iframe, labelled with the + * locator and copying it on click. Selectors that querySelector can't parse + * (WDIO-style, e.g. `button*=Login`) are skipped rather than throwing. + */ +export function drawElementOverlay( + iframe: HTMLIFrameElement | null | undefined, + selectors: string[], + onPick: (selector: string) => void +): void { + const docEl = iframe?.contentDocument + if (!docEl?.body) { + return + } + clearElementOverlay(iframe) + const scrollY = iframe?.contentWindow?.scrollY || 0 + const scrollX = iframe?.contentWindow?.scrollX || 0 + for (const selector of selectors) { + let el: Element | null = null + try { + el = docEl.querySelector(selector) + } catch { + continue // non-CSS (WDIO) locator — can't resolve in the DOM + } + if (!el) { + continue + } + const rect = el.getBoundingClientRect() + const box = docEl.createElement('div') + box.className = OVERLAY_CLASS + box.setAttribute( + 'style', + `position:absolute;box-sizing:border-box;top:${scrollY + rect.top}px;left:${scrollX + rect.left}px;width:${rect.width}px;height:${rect.height}px;outline:1.5px solid #38bdf8;background:rgba(56,189,248,0.12);z-index:9999;cursor:pointer;` + ) + box.title = `Copy locator: ${selector}` + const label = docEl.createElement('div') + label.textContent = selector + label.setAttribute( + 'style', + 'position:absolute;top:-15px;left:-1px;font:10px/1.4 ui-monospace,monospace;background:#38bdf8;color:#06222e;padding:0 4px;white-space:nowrap;border-radius:3px 3px 0 0;' + ) + box.appendChild(label) + box.addEventListener('click', (e) => { + e.stopPropagation() + onPick(selector) + }) + docEl.body.appendChild(box) + } +} diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts index fe4cef66..35d88a51 100644 --- a/packages/app/src/components/browser/snapshot.ts +++ b/packages/app/src/components/browser/snapshot.ts @@ -3,6 +3,7 @@ import { html, nothing } from 'lit' import { consume } from '@lit/context' import { snapshotStyles } from './snapshot-styles.js' import { renderBrowserChrome } from './browser-chrome.js' +import { drawElementOverlay, clearElementOverlay } from './element-overlay.js' import { commandPageUrl } from './url-at-timestamp.js' import { imageMime } from './trace-timeline-utils.js' @@ -22,6 +23,7 @@ import type { Metadata, MetadataBySession } from '@wdio/devtools-shared' import '../placeholder.js' import './screencast-player.js' +import '~icons/mdi/cursor-default-click-outline.js' const MUTATION_SELECTOR = '__mutation-highlight__' @@ -62,6 +64,9 @@ export class DevtoolsBrowser extends Element { * 'snapshot' — show DOM mutations replay and per-command screenshots */ #viewMode: 'snapshot' | 'video' = 'snapshot' + /** When on, outline the elements the test interacted with (their command + * target selectors) on the replayed page; click a box to copy its locator. */ + #overlayOn = false @consume({ context: metadataContext, subscribe: true }) metadata: Metadata | undefined = undefined @@ -97,6 +102,7 @@ export class DevtoolsBrowser extends Element { window.addEventListener('app-mutation-select', (ev) => this.#renderBrowserState(ev.detail) ) + window.addEventListener('a11y-highlight', this.#highlightBySelector) window.addEventListener( 'show-command', this.#handleShowCommand as EventListener @@ -186,6 +192,9 @@ export class DevtoolsBrowser extends Element { this.iframe.style.left = `${Math.max(0, (availW - viewportWidth * scale) / 2)}px` this.iframe.style.top = '0px' } + // Layout is now settled — the one moment element rects are reliable, so + // the overlay boxes track the replayed DOM after every step and resize. + this.#redrawOverlay() }) } @@ -421,24 +430,18 @@ export class DevtoolsBrowser extends Element { return rootElement.querySelector(`*[data-wdio-ref="${ref}"]`) as HTMLElement } - #highlightMutation(ev: CustomEvent<TraceMutation | null>) { - if (!ev.detail) { - this.iframe?.contentDocument - ?.querySelector(`.${MUTATION_SELECTOR}`) - ?.remove() - return - } + #clearHighlight() { + this.iframe?.contentDocument + ?.querySelector(`.${MUTATION_SELECTOR}`) + ?.remove() + } - const mutation = ev.detail + /** Draw the outline box over an element in the replayed iframe. */ + #outline(el: HTMLElement) { const docEl = this.iframe?.contentDocument - if (!docEl || !mutation.target) { - return - } - const el = this.#queryElement(mutation.target) - if (!el) { + if (!docEl) { return } - el.scrollIntoView({ block: 'center', inline: 'center' }) const rect = el.getBoundingClientRect() const scrollY = this.iframe?.contentWindow?.scrollY || 0 @@ -450,10 +453,83 @@ export class DevtoolsBrowser extends Element { 'style', `position: absolute; background: #38bdf8; outline: 2px dotted red; opacity: .2; top: ${scrollY + rect.top}px; left: ${scrollX + rect.left}px; width: ${rect.width}px; height: ${rect.height}px; z-index: 10000;` ) - docEl.querySelector(`.${MUTATION_SELECTOR}`)?.remove() + this.#clearHighlight() docEl.body.appendChild(highlight) } + #highlightMutation(ev: CustomEvent<TraceMutation | null>) { + if (!ev.detail) { + this.#clearHighlight() + return + } + const el = ev.detail.target + ? this.#queryElement(ev.detail.target) + : undefined + if (el) { + this.#outline(el) + } + } + + /** Outline the element for an a11y-tree locator (CSS selectors only; WDIO + * locators like `button*=Login` can't resolve via querySelector → no-op). */ + #highlightBySelector = (ev: Event) => { + const detail = (ev as CustomEvent<{ selector?: string } | null>).detail + const docEl = this.iframe?.contentDocument + if (!docEl) { + return + } + this.#clearHighlight() + if (!detail?.selector) { + return + } + let el: HTMLElement | null = null + try { + el = docEl.querySelector(detail.selector) + } catch { + return + } + if (el) { + this.#outline(el) + } + } + + /** Distinct target selectors the test interacted with (each command's first + * arg), in order. querySelector filters non-element args (URLs, matcher + * strings) at draw time, so this stays permissive. */ + #testSelectors(): string[] { + const seen = new Set<string>() + for (const command of this.commands ?? []) { + const arg = command.args?.[0] + if (typeof arg === 'string' && arg) { + seen.add(arg) + } + } + return [...seen] + } + + // Draws immediately — call only once the iframe has laid out (the sizing rAF + // in #sizeSnapshotToViewport is the one point that holds for both a replay and + // a resize; reading element rects any earlier yields pre-layout positions). + #redrawOverlay() { + if (!this.#overlayOn) { + clearElementOverlay(this.iframe) + return + } + drawElementOverlay(this.iframe, this.#testSelectors(), (selector) => + this.#copyLocator(selector) + ) + } + + #toggleOverlay() { + this.#overlayOn = !this.#overlayOn + this.#redrawOverlay() + this.requestUpdate() + } + + #copyLocator(selector: string) { + navigator.clipboard?.writeText(selector).catch(() => {}) + } + async #renderBrowserState(mutationEntry?: TraceMutation) { const mutations = this.mutations if (!mutations || !mutations.length) { @@ -499,6 +575,8 @@ export class DevtoolsBrowser extends Element { } } + // The replay wiped any overlay boxes; the requestUpdate below runs updated() + // → #sizeSnapshotToViewport, whose settled rAF redraws them post-layout. this.requestUpdate() } @@ -535,6 +613,28 @@ export class DevtoolsBrowser extends Element { return null } + /** Compact "element overlay" toggle in the browser chrome. Shown only when + * the DOM is replayable; boxes the elements the test interacted with. */ + #renderOverlayToggle(hasMutations: number | null) { + if (!hasMutations) { + return nothing + } + const on = this.#overlayOn + return html`<button + title="Element overlay — outline what the test interacted with" + @click=${() => this.#toggleOverlay()} + style="display:inline-grid;place-items:center;width:24px;height:24px;margin:0 8px;flex:none;border-radius:6px;cursor:pointer;border:1px solid var(--vscode-panel-border, #2a2a31);background:${on + ? 'var(--accent, #ff6a3d)' + : 'transparent'};color:${on + ? '#1a0d06' + : 'var(--vscode-descriptionForeground, #8b8b96)'};" + > + <icon-mdi-cursor-default-click-outline + style="width:14px;height:14px;" + ></icon-mdi-cursor-default-click-outline> + </button>` + } + #renderViewToggle() { if (this.#videos.length === 0) { return nothing @@ -643,7 +743,12 @@ export class DevtoolsBrowser extends Element { <section class="w-full h-full bg-sideBarBackground rounded-[14px] border-2 border-panelBorder" > - ${renderBrowserChrome(this.#displayUrl, this.#renderViewToggle())} + ${renderBrowserChrome( + this.#displayUrl, + html`${this.#renderOverlayToggle( + hasMutations + )}${this.#renderViewToggle()}` + )} ${this.#renderViewport(hasMutations)} </section> ` diff --git a/packages/app/src/components/workbench.ts b/packages/app/src/components/workbench.ts index c3f43bc0..c4fdad53 100644 --- a/packages/app/src/components/workbench.ts +++ b/packages/app/src/components/workbench.ts @@ -33,6 +33,7 @@ import './workbench/console.js' import './workbench/metadata.js' import './workbench/network.js' import './workbench/errors.js' +import './workbench/a11y-tree.js' import './workbench/transcript.js' import './workbench/compare.js' import './browser/snapshot.js' @@ -370,9 +371,12 @@ export class DevtoolsWorkbench extends Element { <wdio-devtools-errors></wdio-devtools-errors> </wdio-devtools-tab> ${this.playerMode - ? html`<wdio-devtools-tab label="Transcript"> - <wdio-devtools-transcript></wdio-devtools-transcript> - </wdio-devtools-tab>` + ? html`<wdio-devtools-tab label="A11y"> + <wdio-devtools-a11y></wdio-devtools-a11y> + </wdio-devtools-tab> + <wdio-devtools-tab label="Transcript"> + <wdio-devtools-transcript></wdio-devtools-transcript> + </wdio-devtools-tab>` : nothing} ${this.#renderCompareTabIfAvailable()} ` diff --git a/packages/app/src/components/workbench/a11y-tree.ts b/packages/app/src/components/workbench/a11y-tree.ts new file mode 100644 index 00000000..e3fd0458 --- /dev/null +++ b/packages/app/src/components/workbench/a11y-tree.ts @@ -0,0 +1,207 @@ +import { Element } from '@core/element' +import { html, css, nothing, type TemplateResult } from 'lit' +import { customElement, state } from 'lit/decorators.js' + +import type { CommandLog } from '@wdio/devtools-shared' + +import '../placeholder.js' + +const COMPONENT = 'wdio-devtools-a11y' +const NAME_MAX = 64 + +interface A11yNode { + depth: number + role: string + name: string + /** Captured locator (from the serialized `→ …` suffix), when the node is an + * element — hovering highlights it in the snapshot; clicking copies it. */ + selector?: string +} + +/** Player dock tab: the accessibility tree (roles + accessible names) captured + * for the selected command — the semantic view a screen reader sees, distinct + * from the raw DOM the snapshot pane replays. Follows the active command via + * the same `show-command` event the browser pane listens to. */ +@customElement(COMPONENT) +export class DevtoolsA11yTree extends Element { + @state() + private active?: CommandLog + + /** Locator just copied — the row shows a brief "copied" flash. */ + @state() + private copiedSel?: string + + #onShow = (e: Event) => { + this.active = (e as CustomEvent<{ command?: CommandLog }>).detail?.command + } + + connectedCallback() { + super.connectedCallback() + window.addEventListener('show-command', this.#onShow as EventListener) + } + disconnectedCallback() { + window.removeEventListener('show-command', this.#onShow as EventListener) + this.#highlight(undefined) + super.disconnectedCallback() + } + + /** Ask the snapshot pane to outline (or clear) the element for this locator. */ + #highlight(selector?: string) { + window.dispatchEvent( + new CustomEvent('a11y-highlight', { + detail: selector ? { selector } : null + }) + ) + } + + async #copy(selector?: string) { + if (!selector) { + return + } + try { + await navigator.clipboard.writeText(selector) + this.copiedSel = selector + setTimeout(() => { + this.copiedSel = undefined + }, 1200) + } catch { + /* clipboard blocked — no-op */ + } + } + + static styles = [ + ...Element.styles, + css` + :host { + display: block; + width: 100%; + height: 100%; + overflow: auto; + } + .tree { + font-family: var(--vscode-editor-font-family, monospace); + font-size: 12px; + padding: 10px 12px; + } + .hdr { + color: var(--vscode-descriptionForeground); + font-size: 11px; + margin: 0 0 8px 4px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .node { + display: flex; + align-items: baseline; + gap: 7px; + padding: 2px 6px; + border-radius: 6px; + line-height: 1.35; + white-space: nowrap; + } + .node:hover { + background: var( + --vscode-list-hoverBackground, + rgba(255, 255, 255, 0.05) + ); + } + .twig { + color: var(--vscode-descriptionForeground); + opacity: 0.5; + flex: none; + } + .role { + color: var(--accent, #ff6a3d); + flex: none; + } + .nm { + color: var(--vscode-charts-green, #46c96a); + overflow: hidden; + text-overflow: ellipsis; + } + .node.pick { + cursor: pointer; + } + .sel { + margin-left: auto; + padding-left: 12px; + color: var(--pick, #38bdf8); + opacity: 0; + font-size: 11px; + flex: none; + } + .node.pick:hover .sel { + opacity: 0.85; + } + ` + ] + + #trunc(name: string): string { + return name.length > NAME_MAX ? name.slice(0, NAME_MAX - 1) + '…' : name + } + + /** Parse one serialized line — `<indent>role[level] "name" [∈ …] [→ …]` — into + * a node, dropping the purpose/selector suffixes for a clean tree. Header and + * blank lines return null. */ + #parse(line: string): A11yNode | null { + const trimmed = line.trimStart() + if (!trimmed || trimmed.startsWith('[Page')) { + return null + } + const indent = line.length - trimmed.length + const depth = Math.max(0, Math.floor(indent / 2) - 1) + const role = /^\S+/.exec(trimmed)?.[0] ?? trimmed + const name = /"([^"]*)"/.exec(trimmed)?.[1] ?? '' + // The `→ <locator>` suffix (last one wins; skips any `∈ "purpose"` before it). + const selector = trimmed.includes('→') + ? trimmed.split('→').pop()?.trim() || undefined + : undefined + return { depth, role, name, selector } + } + + #row(node: A11yNode): TemplateResult { + const sel = node.selector + return html`<div + class="node ${sel ? 'pick' : ''}" + style="padding-left:${8 + node.depth * 16}px" + @mouseenter=${() => this.#highlight(sel)} + @mouseleave=${() => this.#highlight(undefined)} + @click=${() => this.#copy(sel)} + title=${sel ? `Click to copy locator: ${sel}` : nothing} + > + <span class="twig">•</span> + <span class="role">${node.role}</span> + ${node.name + ? html`<span class="nm">"${this.#trunc(node.name)}"</span>` + : nothing} + ${sel + ? html`<span class="sel" + >${this.copiedSel === sel ? 'copied ✓' : sel}</span + >` + : nothing} + </div>` + } + + render() { + const text = this.active?.snapshotText + if (!text) { + return html`<wdio-devtools-placeholder></wdio-devtools-placeholder>` + } + const lines = text.split('\n') + const header = lines[0]?.startsWith('[Page') ? lines[0] : undefined + const nodes = lines + .map((l) => this.#parse(l)) + .filter((n): n is A11yNode => n !== null) + return html`<div class="tree"> + ${header ? html`<div class="hdr">${header}</div>` : nothing} + ${nodes.map((n) => this.#row(n))} + </div>` + } +} + +declare global { + interface HTMLElementTagNameMap { + [COMPONENT]: DevtoolsA11yTree + } +} diff --git a/packages/backend/src/trace-reader-types.ts b/packages/backend/src/trace-reader-types.ts index 0afc3a9b..098b43b6 100644 --- a/packages/backend/src/trace-reader-types.ts +++ b/packages/backend/src/trace-reader-types.ts @@ -32,6 +32,15 @@ export interface ScreencastFrameEvent { timestamp: number } +/** Per-action DOM snapshot event. Always emitted (independent of the sparse/ + * dense filmstrip), so it's the reliable anchor for per-command data: `callId` + * maps to the command and `wallTime` (the original snapshot ts) names the + * `-snapshot.txt` / `-elements.json` resources. */ +export interface FrameSnapshotEvent { + type: 'frame-snapshot' + snapshot: { callId: string; pageId: string; wallTime: number } +} + export interface ConsoleEvent { type: 'console' time: number @@ -97,6 +106,7 @@ export interface CategorizedEvents { befores: Map<string, BeforeEvent> afters: Map<string, AfterEvent> frameEvents: ScreencastFrameEvent[] + frameSnapshots: FrameSnapshotEvent[] consoleEvents: (ConsoleEvent | StdioEvent)[] } diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index 439bc779..4cb987a5 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -36,6 +36,7 @@ import type { CategorizedEvents, ConsoleEvent, ContextOptionsEvent, + FrameSnapshotEvent, HarSnapshot, MergedEvents, ScreencastFrameEvent, @@ -65,6 +66,7 @@ function categorizeEvents( const befores = new Map<string, BeforeEvent>() const afters = new Map<string, AfterEvent>() const frameEvents: ScreencastFrameEvent[] = [] + const frameSnapshots: FrameSnapshotEvent[] = [] const consoleEvents: (ConsoleEvent | StdioEvent)[] = [] let ctx: ContextOptionsEvent | undefined for (const event of events) { @@ -85,6 +87,9 @@ function categorizeEvents( case 'screencast-frame': frameEvents.push(event as unknown as ScreencastFrameEvent) break + case 'frame-snapshot': + frameSnapshots.push(event as unknown as FrameSnapshotEvent) + break case 'console': case 'stdout': case 'stderr': @@ -92,7 +97,7 @@ function categorizeEvents( break } } - return { ctx, befores, afters, frameEvents, consoleEvents } + return { ctx, befores, afters, frameEvents, frameSnapshots, consoleEvents } } // Anchors both encodings: our offsets (monotonicTime 0 → anchor = wallTime) @@ -127,6 +132,7 @@ function mergeStreams(streams: CategorizedEvents[]): MergedEvents { befores: new Map(), afters: new Map(), frameEvents: [], + frameSnapshots: [], consoleEvents: [] } for (const stream of streams) { @@ -140,6 +146,7 @@ function mergeStreams(streams: CategorizedEvents[]): MergedEvents { merged.afters.set(callId, after) } merged.frameEvents.push(...stream.frameEvents) + merged.frameSnapshots.push(...stream.frameSnapshots) merged.consoleEvents.push(...stream.consoleEvents) } return merged @@ -200,7 +207,8 @@ function reconstructCommand( before: BeforeEvent, after: AfterEvent | undefined, afters: Map<string, AfterEvent>, - frames: TracePlayerFrame[] + frames: TracePlayerFrame[], + snapshotText?: string ): CommandLog { const endTime = after?.endTime ?? before.startTime const command: CommandLog = { @@ -229,12 +237,35 @@ function reconstructCommand( if (frame) { command.screenshot = frame.screenshot } + if (snapshotText) { + command.snapshotText = snapshotText + } return command } +/** Resolve each per-action `-snapshot.txt` (a11y tree) resource, keyed by the + * command's callId. The frame-snapshot event names the resource via its pageId + * + original timestamp (wallTime), so this works regardless of whether the zip + * used the sparse or dense filmstrip. */ +function buildSnapshotTextByCallId( + files: Record<string, Uint8Array>, + frameSnapshots: FrameSnapshotEvent[] +): Map<string, string> { + const byCallId = new Map<string, string>() + for (const { snapshot } of frameSnapshots) { + const data = + files[`resources/${snapshot.pageId}-${snapshot.wallTime}-snapshot.txt`] + if (data) { + byCallId.set(snapshot.callId, strFromU8(data)) + } + } + return byCallId +} + function buildCommands( events: MergedEvents, - frames: TracePlayerFrame[] + frames: TracePlayerFrame[], + files: Record<string, Uint8Array> ): { commands: CommandLog[] maxTime: number @@ -242,6 +273,10 @@ function buildCommands( } { const entries: { callId: string; command: CommandLog }[] = [] const structural = collectStructuralIds(events.befores) + const snapshotByCallId = buildSnapshotTextByCallId( + files, + events.frameSnapshots + ) let maxTime = 0 for (const [callId, before] of events.befores) { // Group markers are structure, not actions — as command rows their end @@ -250,7 +285,13 @@ function buildCommands( continue } const after = events.afters.get(callId) - const command = reconstructCommand(before, after, events.afters, frames) + const command = reconstructCommand( + before, + after, + events.afters, + frames, + snapshotByCallId.get(callId) + ) maxTime = Math.max(maxTime, command.timestamp) entries.push({ callId, command }) } @@ -326,7 +367,7 @@ export function parseTraceZip(zip: Uint8Array): TracePlayerData { commands, maxTime: cmdMax, indexByCallId - } = buildCommands(merged, frames) + } = buildCommands(merged, frames, files) const groups = buildActionTree( merged.befores, merged.afters, diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index 858f56a6..0f34c23c 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -272,6 +272,39 @@ describe('parseTraceZip', () => { expect(parseTraceZip(fixtureZip()).transcript).toBeUndefined() }) + it('attaches per-command a11y snapshotText via the frame-snapshot resource', () => { + const zip = zipSync({ + 'trace.trace': toNdjson([ + { + type: 'context-options', + wallTime: WALL_TIME, + browserName: 'chrome', + contextId: 'context@abcd1234', + options: { viewport: { width: 1024, height: 768 } } + }, + { + type: 'before', + callId: 'call@1', + startTime: 0, + class: 'Element', + method: 'fill', + params: { selector: '#username', value: 'tom' } + }, + { type: 'after', callId: 'call@1', endTime: 50 }, + // wallTime here names the resource, independent of the filmstrip mode. + { + type: 'frame-snapshot', + snapshot: { callId: 'call@1', pageId: 'page@abcd1234', wallTime: 999 } + } + ]), + 'resources/page@abcd1234-999-snapshot.txt': strToU8( + '[Page: Login]\n textbox "Username"\n button "Login"' + ) + }) + const { trace } = parseTraceZip(zip) + expect(trace.commands[0]?.snapshotText).toContain('textbox "Username"') + }) + it('restores callSource and sources from stack frames + src resources', () => { const { trace } = parseTraceZip(fixtureZip()) const click = trace.commands.find((c) => c.command === 'click') diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 8598cb04..40cd45b7 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -173,6 +173,10 @@ export interface CommandLog { cookies?: string documentInfo?: DocumentInfo id?: number + /** Depth-indented accessibility-tree text for the page state at this command + * (from the per-action `-snapshot.txt` resource). Reconstructed by the trace + * reader; drives the player's A11y tab. */ + snapshotText?: string } /** From ef0bc69f64a6759fa046b877324b2e8fa8778560 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 00:59:19 +0530 Subject: [PATCH 82/91] polish(app): float the transcript copy control as an in-corner icon --- .../src/components/workbench/transcript.ts | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/packages/app/src/components/workbench/transcript.ts b/packages/app/src/components/workbench/transcript.ts index 61227814..4b0805d8 100644 --- a/packages/app/src/components/workbench/transcript.ts +++ b/packages/app/src/components/workbench/transcript.ts @@ -7,6 +7,8 @@ import type { CommandLog } from '@wdio/devtools-shared' import { transcriptContext, commandContext } from '../../controller/context.js' import '../placeholder.js' +import '~icons/mdi/content-copy.js' +import '~icons/mdi/check.js' const COMPONENT = 'wdio-devtools-transcript' @@ -28,31 +30,32 @@ export class DevtoolsTranscript extends Element { ...Element.styles, css` :host { - display: flex; - flex-direction: column; + display: block; + position: relative; width: 100%; height: 100%; - overflow: hidden; - } - .bar { - display: flex; - justify-content: flex-end; - padding: 8px 12px; - border-bottom: 1px solid var(--vscode-panel-border); + overflow: auto; } button { - font-family: inherit; - font-size: 11px; - padding: 5px 12px; + position: absolute; + top: 10px; + right: 14px; + z-index: 2; + display: inline-grid; + place-items: center; + width: 28px; + height: 28px; + padding: 0; border: 1px solid var(--vscode-panel-border); border-radius: 8px; background: var(--vscode-input-background); - color: var(--vscode-foreground); + color: var(--vscode-descriptionForeground); cursor: pointer; - line-height: 1; + font-size: 14px; } button:hover { border-color: var(--accent); + color: var(--vscode-foreground); } button.copied { color: var(--vscode-charts-green); @@ -60,9 +63,7 @@ export class DevtoolsTranscript extends Element { } pre { margin: 0; - padding: 14px; - flex: 1; - overflow: auto; + padding: 14px 52px 14px 14px; font-family: var(--vscode-editor-font-family); font-size: 12px; line-height: 1.6; @@ -116,15 +117,15 @@ export class DevtoolsTranscript extends Element { return html`<wdio-devtools-placeholder></wdio-devtools-placeholder>` } return html` - <div class="bar"> - <button - class=${this.copied ? 'copied' : ''} - @click=${() => this.#copy()} - title="Copy the transcript + failures as an LLM prompt" - > - ${this.copied ? 'Copied' : 'Copy prompt'} - </button> - </div> + <button + class=${this.copied ? 'copied' : ''} + @click=${() => this.#copy()} + title="Copy the transcript + failures as an LLM prompt" + > + ${this.copied + ? html`<icon-mdi-check></icon-mdi-check>` + : html`<icon-mdi-content-copy></icon-mdi-content-copy>`} + </button> <pre>${this.transcript}</pre> ${nothing} ` From 2d503bc8b3c8e8d9ab5a20d3887f65ca5763d63a Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 13:44:07 +0530 Subject: [PATCH 83/91] =?UTF-8?q?feat(app):=20bidirectional=20element-over?= =?UTF-8?q?lay=20=E2=86=94=20A11y-tree=20linking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + .../src/components/browser/element-overlay.ts | 31 +++++++- .../app/src/components/browser/snapshot.ts | 26 ++++++- packages/app/src/components/tabs.ts | 14 ++++ .../app/src/components/workbench/a11y-tree.ts | 75 ++++++++++++++++++- 5 files changed, 142 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 45e4af9f..1ee1994c 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "build": "pnpm -r build", + "show-trace": "node packages/backend/dist/show-trace.js", "demo:wdio": "wdio run ./examples/wdio/cucumber/wdio.conf.ts", "demo:wdio:mocha": "wdio run ./examples/wdio/mocha/wdio.conf.ts", "demo:wdio:retry": "wdio run ./examples/wdio/mocha/wdio.retry.conf.ts", diff --git a/packages/app/src/components/browser/element-overlay.ts b/packages/app/src/components/browser/element-overlay.ts index 3ce1129e..c83cb0db 100644 --- a/packages/app/src/components/browser/element-overlay.ts +++ b/packages/app/src/components/browser/element-overlay.ts @@ -5,6 +5,16 @@ const OVERLAY_CLASS = '__wdio-el-overlay__' +export interface OverlayHandlers { + /** Click a box — copy its locator + jump to the A11y row (selector + name). */ + onPick: (selector: string, label: string) => void + /** Hover a box — reveal the matching a11y-tree row (by selector, else by the + * element's accessible name for locators the serializer captured a different + * way, e.g. the test's `button[type=submit]` vs the tree's `button*=Login`). */ + onHover?: (selector: string, label: string) => void + onLeave?: () => void +} + export function clearElementOverlay( iframe: HTMLIFrameElement | null | undefined ): void { @@ -13,6 +23,20 @@ export function clearElementOverlay( .forEach((node) => node.remove()) } +/** Cheap accessible-name approximation for cross-referencing the a11y tree — + * the visible label a screen reader would announce, not the raw value. */ +function elementLabel(el: Element): string { + const aria = el.getAttribute('aria-label')?.trim() + if (aria) { + return aria + } + const text = el.textContent?.trim() + if (text) { + return text + } + return el.getAttribute('placeholder')?.trim() ?? '' +} + /** * Outline each selector that resolves in the replayed iframe, labelled with the * locator and copying it on click. Selectors that querySelector can't parse @@ -21,7 +45,7 @@ export function clearElementOverlay( export function drawElementOverlay( iframe: HTMLIFrameElement | null | undefined, selectors: string[], - onPick: (selector: string) => void + handlers: OverlayHandlers ): void { const docEl = iframe?.contentDocument if (!docEl?.body) { @@ -40,6 +64,7 @@ export function drawElementOverlay( if (!el) { continue } + const name = elementLabel(el) const rect = el.getBoundingClientRect() const box = docEl.createElement('div') box.className = OVERLAY_CLASS @@ -57,8 +82,10 @@ export function drawElementOverlay( box.appendChild(label) box.addEventListener('click', (e) => { e.stopPropagation() - onPick(selector) + handlers.onPick(selector, name) }) + box.addEventListener('mouseenter', () => handlers.onHover?.(selector, name)) + box.addEventListener('mouseleave', () => handlers.onLeave?.()) docEl.body.appendChild(box) } } diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts index 35d88a51..c647fdad 100644 --- a/packages/app/src/components/browser/snapshot.ts +++ b/packages/app/src/components/browser/snapshot.ts @@ -515,8 +515,30 @@ export class DevtoolsBrowser extends Element { clearElementOverlay(this.iframe) return } - drawElementOverlay(this.iframe, this.#testSelectors(), (selector) => - this.#copyLocator(selector) + drawElementOverlay(this.iframe, this.#testSelectors(), { + onPick: (selector, label) => this.#pickElement(selector, label), + onHover: (selector, label) => this.#revealA11yRow(selector, label), + onLeave: () => this.#revealA11yRow() + }) + } + + /** Clicking a box: copy the locator, open the A11y tab, and pin its row. */ + #pickElement(selector: string, label: string) { + this.#copyLocator(selector) + window.dispatchEvent( + new CustomEvent('open-dock-tab', { detail: { label: 'A11y' } }) + ) + this.#revealA11yRow(selector, label, true) + } + + /** Reverse link: ask the A11y tab to highlight the matching row — by selector, + * falling back to the element's accessible name. `pin` keeps it highlighted + * (click) rather than clearing on mouse-out (hover). */ + #revealA11yRow(selector?: string, label?: string, pin = false) { + window.dispatchEvent( + new CustomEvent('a11y-reveal', { + detail: selector ? { selector, label, pin } : null + }) ) } diff --git a/packages/app/src/components/tabs.ts b/packages/app/src/components/tabs.ts index 316c6bb8..9b4d2249 100644 --- a/packages/app/src/components/tabs.ts +++ b/packages/app/src/components/tabs.ts @@ -12,6 +12,15 @@ export class DevtoolsTabs extends Element { #tabList: string[] = [] #badgeCheckInterval?: number + // Programmatic tab open (e.g. clicking an element overlay box jumps to A11y). + // Guarded by activateTab, so only the tabs instance that owns the label reacts. + #onOpenTab = (e: Event) => { + const label = (e as CustomEvent<{ label?: string }>).detail?.label + if (label) { + this.activateTab(label) + } + } + @property({ type: String }) cacheId?: string @@ -142,6 +151,7 @@ export class DevtoolsTabs extends Element { connectedCallback() { super.connectedCallback() + window.addEventListener('open-dock-tab', this.#onOpenTab as EventListener) setTimeout(() => { // wait till innerHTML is parsed this.#refreshTabList() @@ -188,6 +198,10 @@ export class DevtoolsTabs extends Element { disconnectedCallback() { super.disconnectedCallback() + window.removeEventListener( + 'open-dock-tab', + this.#onOpenTab as EventListener + ) if (this.#badgeCheckInterval) { clearInterval(this.#badgeCheckInterval) } diff --git a/packages/app/src/components/workbench/a11y-tree.ts b/packages/app/src/components/workbench/a11y-tree.ts index e3fd0458..cdbd8f31 100644 --- a/packages/app/src/components/workbench/a11y-tree.ts +++ b/packages/app/src/components/workbench/a11y-tree.ts @@ -31,20 +31,85 @@ export class DevtoolsA11yTree extends Element { @state() private copiedSel?: string + /** Reverse of #highlight — the row the snapshot pane points at. `revealed` is + * transient (overlay-box hover); `pinned` persists (box click, which also + * opens this tab). Both matched by selector then accessible name. */ + @state() + private revealed?: { selector?: string; label?: string } + + @state() + private pinned?: { selector?: string; label?: string } + #onShow = (e: Event) => { this.active = (e as CustomEvent<{ command?: CommandLog }>).detail?.command + // A pin belongs to one page state; drop it when the command changes. + this.pinned = undefined + this.revealed = undefined + } + + #onReveal = (e: Event) => { + const detail = ( + e as CustomEvent<{ + selector?: string + label?: string + pin?: boolean + } | null> + ).detail + if (detail?.pin) { + this.pinned = { selector: detail.selector, label: detail.label } + } else { + this.revealed = detail ?? undefined + } } connectedCallback() { super.connectedCallback() window.addEventListener('show-command', this.#onShow as EventListener) + window.addEventListener('a11y-reveal', this.#onReveal as EventListener) } disconnectedCallback() { window.removeEventListener('show-command', this.#onShow as EventListener) + window.removeEventListener('a11y-reveal', this.#onReveal as EventListener) this.#highlight(undefined) super.disconnectedCallback() } + /** Scroll the highlighted row into view when the snapshot pane points at it. */ + updated() { + if (this.revealed || this.pinned) { + this.renderRoot + ?.querySelector('.node.hot') + ?.scrollIntoView({ block: 'nearest' }) + } + } + + /** A node matches a reveal target by exact selector, else by accessible name + * (covers locators the serializer captured differently, e.g. the test's + * `button[type=submit]` vs `button*=Login`). */ + #matches(node: A11yNode, target?: { selector?: string; label?: string }) { + if (!target) { + return false + } + if ( + target.selector && + node.selector && + node.selector.trim() === target.selector.trim() + ) { + return true + } + return !!( + target.label && + node.name && + node.name.trim() === target.label.trim() + ) + } + + #isRevealed(node: A11yNode): boolean { + return ( + this.#matches(node, this.revealed) || this.#matches(node, this.pinned) + ) + } + /** Ask the snapshot pane to outline (or clear) the element for this locator. */ #highlight(selector?: string) { window.dispatchEvent( @@ -134,6 +199,13 @@ export class DevtoolsA11yTree extends Element { .node.pick:hover .sel { opacity: 0.85; } + .node.hot { + background: color-mix(in srgb, var(--pick, #38bdf8) 22%, transparent); + box-shadow: inset 0 0 0 1px var(--pick, #38bdf8); + } + .node.hot .sel { + opacity: 0.85; + } ` ] @@ -162,8 +234,9 @@ export class DevtoolsA11yTree extends Element { #row(node: A11yNode): TemplateResult { const sel = node.selector + const hot = this.#isRevealed(node) return html`<div - class="node ${sel ? 'pick' : ''}" + class="node ${sel ? 'pick' : ''} ${hot ? 'hot' : ''}" style="padding-left:${8 + node.depth * 16}px" @mouseenter=${() => this.#highlight(sel)} @mouseleave=${() => this.#highlight(undefined)} From 1f4064d06066cefebe9013ba5f799d961d01973e Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 14:29:12 +0530 Subject: [PATCH 84/91] refactor(shared): single-source the a11y snapshot-text format tokens --- .../app/src/components/workbench/a11y-tree.ts | 20 +++++++--- packages/core/src/element-snapshot.ts | 38 +++++++++++++------ packages/shared/src/index.ts | 1 + packages/shared/src/snapshot-format.ts | 17 +++++++++ 4 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 packages/shared/src/snapshot-format.ts diff --git a/packages/app/src/components/workbench/a11y-tree.ts b/packages/app/src/components/workbench/a11y-tree.ts index cdbd8f31..32ee0252 100644 --- a/packages/app/src/components/workbench/a11y-tree.ts +++ b/packages/app/src/components/workbench/a11y-tree.ts @@ -2,6 +2,11 @@ import { Element } from '@core/element' import { html, css, nothing, type TemplateResult } from 'lit' import { customElement, state } from 'lit/decorators.js' +import { + SNAPSHOT_INDENT_UNIT, + SNAPSHOT_PAGE_HEADER, + SNAPSHOT_LOCATOR_DELIM +} from '@wdio/devtools-shared' import type { CommandLog } from '@wdio/devtools-shared' import '../placeholder.js' @@ -218,16 +223,19 @@ export class DevtoolsA11yTree extends Element { * blank lines return null. */ #parse(line: string): A11yNode | null { const trimmed = line.trimStart() - if (!trimmed || trimmed.startsWith('[Page')) { + if (!trimmed || trimmed.startsWith(SNAPSHOT_PAGE_HEADER)) { return null } const indent = line.length - trimmed.length - const depth = Math.max(0, Math.floor(indent / 2) - 1) + const depth = Math.max( + 0, + Math.floor(indent / SNAPSHOT_INDENT_UNIT.length) - 1 + ) const role = /^\S+/.exec(trimmed)?.[0] ?? trimmed const name = /"([^"]*)"/.exec(trimmed)?.[1] ?? '' // The `→ <locator>` suffix (last one wins; skips any `∈ "purpose"` before it). - const selector = trimmed.includes('→') - ? trimmed.split('→').pop()?.trim() || undefined + const selector = trimmed.includes(SNAPSHOT_LOCATOR_DELIM) + ? trimmed.split(SNAPSHOT_LOCATOR_DELIM).pop()?.trim() || undefined : undefined return { depth, role, name, selector } } @@ -262,7 +270,9 @@ export class DevtoolsA11yTree extends Element { return html`<wdio-devtools-placeholder></wdio-devtools-placeholder>` } const lines = text.split('\n') - const header = lines[0]?.startsWith('[Page') ? lines[0] : undefined + const header = lines[0]?.startsWith(SNAPSHOT_PAGE_HEADER) + ? lines[0] + : undefined const nodes = lines .map((l) => this.#parse(l)) .filter((n): n is A11yNode => n !== null) diff --git a/packages/core/src/element-snapshot.ts b/packages/core/src/element-snapshot.ts index e94e2c55..2070c61d 100644 --- a/packages/core/src/element-snapshot.ts +++ b/packages/core/src/element-snapshot.ts @@ -11,6 +11,14 @@ import type { SnapshotElement, SnapshotResult } from './element-types.js' + +import { + SNAPSHOT_INDENT_UNIT, + SNAPSHOT_PAGE_HEADER, + SNAPSHOT_LOCATOR_DELIM, + SNAPSHOT_PURPOSE_TOKEN +} from '@wdio/devtools-shared' + import type { JSONElement } from './locators/types.js' import { parseAndroidBounds, parseIOSBounds } from './locators/xml-parsing.js' import { @@ -81,7 +89,7 @@ export function serializeWebSnapshot( ): string { const { inViewportOnly = true } = options - let header = '[Page' + let header = SNAPSHOT_PAGE_HEADER if (context?.title) { header += `: ${context.title}` } @@ -103,7 +111,7 @@ export function serializeWebSnapshot( continue } - const indent = ' '.repeat(node.depth + 1) // +1 indents everything under the header + const indent = SNAPSHOT_INDENT_UNIT.repeat(node.depth + 1) // +1 indents everything under the header const isInteractive = INTERACTIVE_ROLES.has(node.role) if (isStatictextEchoedByParent(nodes, i)) { @@ -127,13 +135,17 @@ export function serializeWebSnapshot( // duplicate selectors like six "Add to Wishlist" buttons. lines.push( purpose - ? `${indent}${roleLabel} "${node.name}" ∈ "${purpose}" → ${node.selector}` - : `${indent}${roleLabel} "${node.name}" → ${node.selector}` + ? `${indent}${roleLabel} "${node.name}" ${SNAPSHOT_PURPOSE_TOKEN} "${purpose}" ${SNAPSHOT_LOCATOR_DELIM} ${node.selector}` + : `${indent}${roleLabel} "${node.name}" ${SNAPSHOT_LOCATOR_DELIM} ${node.selector}` ) } else if (purpose) { - lines.push(`${indent}${roleLabel} ∈ "${purpose}" → ${node.selector}`) + lines.push( + `${indent}${roleLabel} ${SNAPSHOT_PURPOSE_TOKEN} "${purpose}" ${SNAPSHOT_LOCATOR_DELIM} ${node.selector}` + ) } else { - lines.push(`${indent}${roleLabel} → ${node.selector}`) + lines.push( + `${indent}${roleLabel} ${SNAPSHOT_LOCATOR_DELIM} ${node.selector}` + ) } } else { // Container / structural: show role + name when present, no selector @@ -633,7 +645,7 @@ function renderMobileNodes(nodes: MobileFlatNode[]): string[] { for (let i = 0; i < nodes.length; i++) { const node = nodes[i] - const indent = ' '.repeat(node.depth + 1) + const indent = SNAPSHOT_INDENT_UNIT.repeat(node.depth + 1) // Collapse anonymous layout containers at depth ≥ 2. // Keep depth 0-1 structural chrome and any named container. @@ -686,13 +698,17 @@ function renderMobileNodes(nodes: MobileFlatNode[]): string[] { if (node.name) { lines.push( purpose - ? `${indent}${node.role} "${node.name}" ∈ "${purpose}" → ${selector}` - : `${indent}${node.role} "${node.name}" → ${selector}` + ? `${indent}${node.role} "${node.name}" ${SNAPSHOT_PURPOSE_TOKEN} "${purpose}" ${SNAPSHOT_LOCATOR_DELIM} ${selector}` + : `${indent}${node.role} "${node.name}" ${SNAPSHOT_LOCATOR_DELIM} ${selector}` ) } else if (purpose) { - lines.push(`${indent}${node.role} ∈ "${purpose}" → ${selector}`) + lines.push( + `${indent}${node.role} ${SNAPSHOT_PURPOSE_TOKEN} "${purpose}" ${SNAPSHOT_LOCATOR_DELIM} ${selector}` + ) } else { - lines.push(`${indent}${node.role} → ${selector}`) + lines.push( + `${indent}${node.role} ${SNAPSHOT_LOCATOR_DELIM} ${selector}` + ) } } else { // Container / structural / non-locatable diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index c0320b2a..e2071680 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -5,6 +5,7 @@ export * from './baseline.js' export * from './files.js' export * from './routes.js' export * from './runner.js' +export * from './snapshot-format.js' export * from './timing.js' export * from './trace-actions.js' export * from './trace-player.js' diff --git a/packages/shared/src/snapshot-format.ts b/packages/shared/src/snapshot-format.ts new file mode 100644 index 00000000..534cc420 --- /dev/null +++ b/packages/shared/src/snapshot-format.ts @@ -0,0 +1,17 @@ +// Single source of truth for the accessibility-snapshot text format (the +// `-snapshot.txt` trace resource). The producer (core's element-snapshot +// serializers) and the consumer (the app's A11y-tree parser) both reference +// these, so the written and parsed grammar can't drift. Keep one canonical form +// per concept — see CLAUDE.md § "One source of truth per concept". + +/** Indent step per tree depth (the header sits at one unit; nodes at depth+1). */ +export const SNAPSHOT_INDENT_UNIT = ' ' + +/** Prefix of the web page-header line (`[Page: <title> — <url>]`). */ +export const SNAPSHOT_PAGE_HEADER = '[Page' + +/** Separator between a node and its captured locator (rendered space-padded). */ +export const SNAPSHOT_LOCATOR_DELIM = '→' + +/** Marks an inferred purpose before the locator (`<role> ∈ "<purpose>"`). */ +export const SNAPSHOT_PURPOSE_TOKEN = '∈' From abc75485c87becd6a65386f429e4628593d29eb8 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 15:11:49 +0530 Subject: [PATCH 85/91] refactor(selenium): extract per-test artifact emit into test-artifacts.ts --- CLAUDE.md | 2 +- packages/selenium-devtools/src/index.ts | 103 ++++----------- .../selenium-devtools/src/test-artifacts.ts | 117 ++++++++++++++++++ 3 files changed, 140 insertions(+), 82 deletions(-) create mode 100644 packages/selenium-devtools/src/test-artifacts.ts diff --git a/CLAUDE.md b/CLAUDE.md index 24f95abe..8750159a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -256,7 +256,7 @@ Most entries below don't trigger the `max-lines` lint rule after `skipBlankLines - `packages/service/src/index.ts` — over the 500-**logic**-line soft cap and ~620 raw (the WDIO plugin god-file: session/screencast/command/assertion/trace-slice wiring in one class). The assertion-capture block (`beforeAssertion`/`afterAssertion`/`#handleAssertionOutcome`/`#finalizePendingAssertion`, plus the `#assertionDepth`/`#pendingAssertion` state) is now the largest self-contained seam and the prime extraction candidate; trace-slice and screencast are the next two. - `packages/nightwatch-devtools/src/index.ts` (~536 raw). Cucumber/test/run-lifecycle, session-init, event-hub modules already extracted; remainder is the `PluginInternals` accessor bag plus per-method delegators plus the factory. Accept-as-is. -- `packages/selenium-devtools/src/index.ts` (~817 raw). Session/test-lifecycle extracted; remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Grew past ~560 when the per-test-artifact seam (`#allureSink`/`#perTestArtifactInputs`/`#emitTestArtifacts`) landed inline with the cross-adapter Allure work; Nightwatch extracted the twin concern into `test-artifacts.ts`, so the next Selenium change to this seam should mirror that (a `selenium-devtools/src/test-artifacts.ts` threading the sink + flushed-trace promise through an input bag). Prime extraction candidate. +- `packages/selenium-devtools/src/index.ts` (~758 raw). Session/test-lifecycle **and** the per-test-artifact seam are now extracted: the sink cache + input snapshot + produce/attach flow live in `selenium-devtools/src/test-artifacts.ts` as `SeleniumTestArtifacts` (mirrors Nightwatch's twin — a typed input bag threading the Allure sink + flushed-trace promise), and the plugin keeps only a thin bag-building delegator. Remainder is the `PluginInternals` accessor bag plus onCommand/onDriverCreated wiring. Still over the 500 **raw** soft cap (under the logic-line cap after `skipBlankLines`/`skipComments`); the accessor bag / command wiring is the next extraction candidate if it grows. - `packages/nightwatch-devtools/src/session.ts` (~468). `captureNetworkFromPerformanceLogs` + `captureBrowserLogs` + `captureTrace` are tightly coupled to NightwatchBrowser state. Coverage at 78% after recent backfill; further extraction would need rewriting the browser-coupling. ### Test coverage gaps (worst-risk-first) diff --git a/packages/selenium-devtools/src/index.ts b/packages/selenium-devtools/src/index.ts index a622b695..6bd8298e 100644 --- a/packages/selenium-devtools/src/index.ts +++ b/packages/selenium-devtools/src/index.ts @@ -24,11 +24,7 @@ import { recordTraceBoundary, flushCurrentTestTrace } from './session-lifecycle.js' -import type { - AllureAttachSink, - SpecRange, - TraceArtifact -} from '@wdio/devtools-core' +import type { SpecRange, TraceArtifact } from '@wdio/devtools-core' import { startTest as tmStartTest, endTest as tmEndTest, @@ -49,15 +45,10 @@ import { import { findFreePort } from './helpers/utils.js' import { RetryTracker, - attachTraceArtifact, - captureAndAttachScreenshot, - captureAndAttachVideo, errorMessage, - lastRenderedScreenshot, - resolveAdapterOutputDir, tracePolicyModeWarning } from '@wdio/devtools-core' -import { getAllureSink } from './allure.js' +import { SeleniumTestArtifacts } from './test-artifacts.js' import { tryRegisterRunnerHooks } from './runnerHooks.js' import { patchNodeAssert } from './assertPatcher.js' import { @@ -152,8 +143,9 @@ class SeleniumDevToolsPlugin { * test's video frame window and screenshot lookup (per-test slicing). */ #currentTestStartWallTime = Date.now() - /** Allure attach sink, resolved once lazily at the first per-test end. */ - #allureSinkResolved?: Promise<AllureAttachSink | undefined> + /** Per-test artifact emit (trace-slice attach + screenshot + video); owns its + * own lazily-resolved Allure attach sink. */ + #testArtifacts = new SeleniumTestArtifacts() constructor(options: DevToolsOptions = {}) { this.#options = { @@ -538,81 +530,30 @@ class SeleniumDevToolsPlugin { await this.#emitTestArtifacts(ended, state === 'failed', flushed) } - /** Lazily resolve the Allure attach sink once (produce-only when Allure is - * inactive), reused across every per-test attach this run. */ - #allureSink(): Promise<AllureAttachSink | undefined> { - return (this.#allureSinkResolved ??= getAllureSink()) - } - - /** Snapshot every per-test capture input synchronously at test end. Selenium's - * runner hooks fire-and-forget the emit, so reading these after an await would - * race the next test overwriting the fields; frames/outcomes are copied - * because their backing arrays keep mutating after this test ends. */ - #perTestArtifactInputs(endedTest: TestStats | null) { - const startWallTime = this.#currentTestStartWallTime - return { - startWallTime, - sessionId: this.#sessionId, - testUid: endedTest?.uid, - attempt: endedTest?.retries, - screenshotBase64: lastRenderedScreenshot( - this.#actionSnapshots, - startWallTime - ), - frames: this.#screencast ? [...this.#screencast.frames] : undefined, - outcomes: (this.#testManager?.lastTestOutcomes() ?? []).map((o) => ({ - ...o - })), - captureFormat: this.#screencastOptions.captureFormat, - outputDir: resolveAdapterOutputDir({ testFilePath: this.#testFilePath }) - } - } - - /** At a test/scenario end, while the runner's per-test hook is still open: - * attach the just-flushed trace slice to Allure, then capture + attach the - * per-test screenshot and video per their policies. Each part no-ops when its - * feature is off (all gate on trace mode + `test` granularity in core). */ - async #emitTestArtifacts( + /** Build the per-test artifact bag from live plugin state and delegate to the + * emitter (which copies the mutable inputs synchronously before awaiting). */ + #emitTestArtifacts( endedTest: TestStats | null, failed: boolean, flushed: Promise<TraceArtifact | undefined> ): Promise<void> { - const inp = this.#perTestArtifactInputs(endedTest) - const onArtifact = (a: TraceArtifact) => this.#artifacts.push(a) - const onLog = (level: 'info' | 'warn', msg: string) => log[level](msg) - const attach = await this.#allureSink() - const artifact = await flushed - if (artifact) { - await attachTraceArtifact(artifact, attach, onLog) - } - await captureAndAttachScreenshot({ + return this.#testArtifacts.emit({ mode: this.#options.mode, granularity: this.#options.traceGranularity, - policy: this.#options.screenshot, + screenshotPolicy: this.#options.screenshot, + videoPolicy: this.#options.video, failed, - screenshotBase64: inp.screenshotBase64, - sessionId: inp.sessionId, - outputDir: inp.outputDir, - testUid: inp.testUid, - attach, - onArtifact, - onLog - }) - await captureAndAttachVideo({ - mode: this.#options.mode, - granularity: this.#options.traceGranularity, - policy: this.#options.video, - frames: inp.frames, - startWallTime: inp.startWallTime, - outcomes: inp.outcomes, - attempt: inp.attempt, - outputDir: inp.outputDir, - testUid: inp.testUid, - sessionId: inp.sessionId, - captureFormat: inp.captureFormat, - attach, - onArtifact, - onLog + flushed, + startWallTime: this.#currentTestStartWallTime, + sessionId: this.#sessionId, + endedTest, + actionSnapshots: this.#actionSnapshots, + frames: this.#screencast?.frames, + outcomes: this.#testManager?.lastTestOutcomes() ?? [], + captureFormat: this.#screencastOptions.captureFormat, + testFilePath: this.#testFilePath, + onArtifact: (a) => this.#artifacts.push(a), + onLog: (level, msg) => log[level](msg) }) } diff --git a/packages/selenium-devtools/src/test-artifacts.ts b/packages/selenium-devtools/src/test-artifacts.ts new file mode 100644 index 00000000..04a69585 --- /dev/null +++ b/packages/selenium-devtools/src/test-artifacts.ts @@ -0,0 +1,117 @@ +// Per-test artifact emit (trace-slice attach + screenshot + video) for the +// Selenium plugin. Mirrors nightwatch's `test-artifacts.ts`, plus the two +// Selenium-only concerns: a live Allure attach sink (nightwatch is produce-only) +// and the just-flushed trace slice threaded through the input bag. +// +// Selenium's runner hooks fire-and-forget some emits (Gherkin `AfterStep`), so +// the mutable capture arrays are copied synchronously before the first await — +// the next test would otherwise overwrite their backing arrays mid-flight. Each +// core call self-gates on mode + `test` granularity + policy, so a non-trace run +// or a coarser granularity is a no-op. + +import { + attachTraceArtifact, + captureAndAttachScreenshot, + captureAndAttachVideo, + lastRenderedScreenshot, + resolveAdapterOutputDir, + type AllureAttachSink, + type TestOutcome, + type TraceArtifact +} from '@wdio/devtools-core' +import { getAllureSink } from './allure.js' +import type { + ActionSnapshot, + DevToolsMode, + ScreencastFrame, + TestStats, + TraceGranularity, + TraceScreenshotPolicy, + TraceVideoPolicy +} from './types.js' + +export interface EmitTestArtifactsInput { + mode: DevToolsMode + granularity: TraceGranularity + screenshotPolicy: TraceScreenshotPolicy + videoPolicy: TraceVideoPolicy + failed: boolean + /** The just-flushed per-test trace slice, attached to Allure when present. */ + flushed: Promise<TraceArtifact | undefined> + startWallTime: number + sessionId: string | undefined + /** Ended test — its `uid`/`retries` key the artifact and retry attempt. */ + endedTest: TestStats | null + actionSnapshots: readonly ActionSnapshot[] + frames: readonly ScreencastFrame[] | undefined + /** This test's attempt slots (already scoped via `lastTestOutcomes`). */ + outcomes: readonly TestOutcome[] + captureFormat?: 'jpeg' | 'png' + testFilePath: string | undefined + onArtifact: (artifact: TraceArtifact) => void + onLog?: (level: 'info' | 'warn', message: string) => void +} + +export class SeleniumTestArtifacts { + /** Allure attach sink, resolved once lazily at the first per-test end + * (produce-only when Allure is inactive), reused across every attach. */ + #sinkResolved?: Promise<AllureAttachSink | undefined> + + #sink(): Promise<AllureAttachSink | undefined> { + return (this.#sinkResolved ??= getAllureSink()) + } + + /** At a test/scenario end, while the runner's per-test hook is still open: + * attach the just-flushed trace slice to Allure, then capture + attach the + * per-test screenshot and video per their policies. */ + async emit(input: EmitTestArtifactsInput): Promise<void> { + // Snapshot the mutable capture inputs synchronously before any await — see + // the file header on Selenium's fire-and-forget hooks. + const frames = input.frames ? [...input.frames] : undefined + const outcomes = input.outcomes.map((o) => ({ ...o })) + const screenshotBase64 = lastRenderedScreenshot( + input.actionSnapshots, + input.startWallTime + ) + const outputDir = resolveAdapterOutputDir({ + testFilePath: input.testFilePath + }) + const testUid = input.endedTest?.uid + const { onArtifact, onLog } = input + + const attach = await this.#sink() + const artifact = await input.flushed + if (artifact) { + await attachTraceArtifact(artifact, attach, onLog) + } + await captureAndAttachScreenshot({ + mode: input.mode, + granularity: input.granularity, + policy: input.screenshotPolicy, + failed: input.failed, + screenshotBase64, + sessionId: input.sessionId, + outputDir, + testUid, + attach, + onArtifact, + onLog + }) + await captureAndAttachVideo({ + mode: input.mode, + granularity: input.granularity, + policy: input.videoPolicy, + frames, + startWallTime: input.startWallTime, + outcomes, + attempt: input.endedTest?.retries, + outputDir, + testUid, + sessionId: input.sessionId, + captureFormat: input.captureFormat, + attach, + onArtifact, + onLog + }) + } +} From e9a2cb7d1584a45a377ccb4b55a1e13ae7959b1b Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 17:36:25 +0530 Subject: [PATCH 86/91] feat(trace): input markers + Cucumber step nesting --- .../src/components/browser/trace-timeline.ts | 18 ++- packages/backend/src/trace-reader-types.ts | 2 + packages/backend/src/trace-reader.ts | 3 + packages/backend/tests/trace-reader.test.ts | 30 ++++ packages/core/src/action-snapshot.ts | 3 +- packages/core/src/index.ts | 1 + packages/core/src/trace-action-events.ts | 139 +++++++++++++----- packages/core/src/trace-frame-snapshots.ts | 14 ++ packages/core/src/trace-hierarchy.ts | 30 ++++ .../core/tests/trace-frame-snapshots.test.ts | 68 ++++++++- packages/core/tests/trace-hierarchy.test.ts | 63 ++++++++ packages/service/src/assertion-tracker.ts | 7 +- packages/service/src/index.ts | 31 +++- packages/service/src/session.ts | 6 +- packages/shared/src/trace-actions.ts | 31 ++++ packages/shared/src/types.ts | 7 + 16 files changed, 404 insertions(+), 49 deletions(-) create mode 100644 packages/core/src/trace-hierarchy.ts create mode 100644 packages/core/tests/trace-hierarchy.test.ts diff --git a/packages/app/src/components/browser/trace-timeline.ts b/packages/app/src/components/browser/trace-timeline.ts index 07cb81a1..1eb7e351 100644 --- a/packages/app/src/components/browser/trace-timeline.ts +++ b/packages/app/src/components/browser/trace-timeline.ts @@ -3,6 +3,7 @@ import { html, type TemplateResult } from 'lit' import { customElement, state, query } from 'lit/decorators.js' import { consume } from '@lit/context' import type { CommandLog, TracePlayerFrame } from '@wdio/devtools-shared' +import { isKeyboardCommand } from '@wdio/devtools-shared' import { commandContext, framesContext } from '../../controller/context.js' import { activeSpanAt } from '../workbench/active-entry.js' @@ -347,7 +348,9 @@ export class TraceTimeline extends Element { const active = frame.timestamp === activeFrame return html`<button class="absolute top-0.5 bottom-0.5 aspect-video border rounded overflow-hidden hover:border-chartsBlue hover:z-10 ${active - ? 'border-chartsBlue ring-1 ring-chartsBlue z-10' + ? `border-chartsBlue ring-1 ring-chartsBlue${ + this.playing ? '' : ' z-10' + }` : 'border-panelBorder'}" style="left:${fraction * 100}%; transform:translateX(-${fraction * 100}%);" @@ -375,10 +378,17 @@ export class TraceTimeline extends Element { class="absolute left-0 right-0 top-1/2 -translate-y-1/2 h-0.5 bg-chartsBlue/50 rounded" ></div> ${this.#sortedCommands.map((command) => { - const tickFraction = this.#fraction(command.timestamp ?? 0) + const pct = this.#fraction(command.timestamp ?? 0) * 100 + // Keyboard = green mark; pointer (has a hit point) = blue dot; else a + // plain white tick. Glyphs are illegible at this scale, so shape+colour. + const mark = isKeyboardCommand(command.command) + ? 'width:6px;height:10px;border-radius:2px;background:#46c96a;' + : command.point + ? 'width:8px;height:8px;border-radius:50%;background:#38bdf8;box-shadow:0 0 0 1px rgba(255,255,255,0.5);' + : 'width:1px;height:10px;background:rgba(255,255,255,0.8);' return html`<div - class="absolute top-1/2 -translate-y-1/2 h-2.5 w-px bg-white/80" - style="left:${tickFraction * 100}%;" + class="absolute top-1/2 -translate-y-1/2" + style="left:${pct}%;${mark}" title="${command.title ?? command.command}" ></div>` })} diff --git a/packages/backend/src/trace-reader-types.ts b/packages/backend/src/trace-reader-types.ts index 098b43b6..237b37f0 100644 --- a/packages/backend/src/trace-reader-types.ts +++ b/packages/backend/src/trace-reader-types.ts @@ -24,6 +24,8 @@ export interface AfterEvent { error?: { message: string } /** Command return value, restored onto CommandLog.result. */ result?: unknown + /** Pointer hit point, restored onto CommandLog.point (A8 input marker). */ + point?: { x: number; y: number } } export interface ScreencastFrameEvent { diff --git a/packages/backend/src/trace-reader.ts b/packages/backend/src/trace-reader.ts index 4cb987a5..163004f3 100644 --- a/packages/backend/src/trace-reader.ts +++ b/packages/backend/src/trace-reader.ts @@ -240,6 +240,9 @@ function reconstructCommand( if (snapshotText) { command.snapshotText = snapshotText } + if (after?.point) { + command.point = after.point + } return command } diff --git a/packages/backend/tests/trace-reader.test.ts b/packages/backend/tests/trace-reader.test.ts index 0f34c23c..4244c261 100644 --- a/packages/backend/tests/trace-reader.test.ts +++ b/packages/backend/tests/trace-reader.test.ts @@ -305,6 +305,36 @@ describe('parseTraceZip', () => { expect(trace.commands[0]?.snapshotText).toContain('textbox "Username"') }) + it('restores the pointer hit point from after.point (A8)', () => { + const zip = zipSync({ + 'trace.trace': toNdjson([ + { + type: 'context-options', + wallTime: WALL_TIME, + browserName: 'chrome', + contextId: 'context@abcd1234', + options: { viewport: { width: 1024, height: 768 } } + }, + { + type: 'before', + callId: 'call@1', + startTime: 0, + class: 'Element', + method: 'click', + params: { selector: '#go' } + }, + { + type: 'after', + callId: 'call@1', + endTime: 50, + point: { x: 30, y: 25 } + } + ]) + }) + const { trace } = parseTraceZip(zip) + expect(trace.commands[0]?.point).toEqual({ x: 30, y: 25 }) + }) + it('restores callSource and sources from stack frames + src resources', () => { const { trace } = parseTraceZip(fixtureZip()) const click = trace.commands.find((c) => c.command === 'click') diff --git a/packages/core/src/action-snapshot.ts b/packages/core/src/action-snapshot.ts index 19060f46..e589f743 100644 --- a/packages/core/src/action-snapshot.ts +++ b/packages/core/src/action-snapshot.ts @@ -68,7 +68,8 @@ export async function captureActionSnapshot( ), runWith<BrowserElementInfo[]>( input.runScript, - elementsScript(false, true), + // includeBounds: the per-action element rects drive A8 input points. + elementsScript(true, true), [] ) ]) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ee4126a0..758625cb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,6 +17,7 @@ export * from './element-types.js' export * from './sha1.js' export * from './trace-action-events.js' export * from './trace-console.js' +export * from './trace-hierarchy.js' export * from './trace-exporter.js' export * from './trace-finalizer.js' export * from './trace-frame-snapshots.js' diff --git a/packages/core/src/trace-action-events.ts b/packages/core/src/trace-action-events.ts index 114233d7..c6f9c712 100644 --- a/packages/core/src/trace-action-events.ts +++ b/packages/core/src/trace-action-events.ts @@ -6,6 +6,7 @@ import type { CommandLog, TestMetadataMap } from '@wdio/devtools-shared' +import { POINTABLE_METHODS } from '@wdio/devtools-shared' import { ASSERT_ACTION_CLASS, formatActionTitle, @@ -15,6 +16,7 @@ import { } from './action-mapping.js' import { callSourceToStack, type StackFrame } from './trace-sources.js' import type { FrameSnapshotIndex } from './trace-frame-snapshots.js' +import { buildGroupPath } from './trace-hierarchy.js' export interface BeforeEvent { type: 'before' @@ -46,6 +48,9 @@ export interface AfterEvent { parentId?: string /** Frame-snapshot name rendered as the action's after state. */ afterSnapshot?: string + /** Page-coordinate hit point (centre of the matched element) for pointer + * actions — drives the player's click marker + timeline pointer glyph. */ + point?: { x: number; y: number } } // Serialized command results over this size are dropped from the trace — a @@ -75,8 +80,9 @@ interface ActionStream { events: ActionEvent[] prevEndMs: number callCounter: number - lastTestUid?: string - groupCallId?: string + /** Currently-open tracingGroup path, outermost first (feature→scenario→step). + * Actions nest under the innermost entry. */ + openGroups: { uid: string; callId: string }[] } // An adapter may attach a normalized CollapsedAssertResult (see shared) to an @@ -139,49 +145,64 @@ function buildActionParams( return Object.fromEntries(rawArgs.map((a, i) => [String(i), a])) } -function closeGroup(stream: ActionStream): void { - if (!stream.lastTestUid || !stream.groupCallId) { - return +function closeGroupsFrom(stream: ActionStream, from: number): void { + // Innermost-first so nested after events stay balanced. + for (let i = stream.openGroups.length - 1; i >= from; i--) { + stream.events.push({ + type: 'after', + callId: stream.openGroups[i].callId, + endTime: stream.prevEndMs + }) } - stream.callCounter++ - stream.events.push({ - type: 'after', - callId: stream.groupCallId, - endTime: stream.prevEndMs - }) + stream.openGroups.length = Math.max(0, from) } -// When the testUid changes, close the previous Tracing.tracingGroup and open -// a new one; child actions reference it via parentId to render as spans. -function handleTestBoundary( +// Diff the command's desired group path (feature→scenario→step, or just the +// test) against the open stack: close the diverged tail, then open the new tail +// with parentId chaining so the reader nests them. Actions then reference the +// innermost open group. A path of length ≤1 with no ancestry/step reproduces +// the previous single-group-per-test output. +function syncGroups( stream: ActionStream, cmd: CommandLog, pageId: string, wallTime: number, testMetadata?: TestMetadataMap ): void { - if (!cmd.testUid || cmd.testUid === stream.lastTestUid) { - return + const desired = buildGroupPath(cmd, testMetadata) + let common = 0 + while ( + common < stream.openGroups.length && + common < desired.length && + stream.openGroups[common].uid === desired[common].uid + ) { + common++ + } + closeGroupsFrom(stream, common) + for (let i = common; i < desired.length; i++) { + stream.callCounter++ + const callId = `call@${stream.callCounter}` + const parentId = stream.openGroups[i - 1]?.callId + const groupBefore: BeforeEvent = { + type: 'before', + callId, + startTime: Math.max( + stream.prevEndMs, + (cmd.startTime ?? cmd.timestamp) - wallTime + ), + class: 'Tracing', + method: 'tracingGroup', + pageId, + params: { name: desired[i].title }, + title: desired[i].title, + apiName: 'tracing.tracingGroup' + } + if (parentId) { + groupBefore.parentId = parentId + } + stream.events.push(groupBefore) + stream.openGroups.push({ uid: desired[i].uid, callId }) } - closeGroup(stream) - stream.callCounter++ - stream.groupCallId = `call@${stream.callCounter}` - const groupName = testMetadata?.get(cmd.testUid)?.title ?? cmd.testUid - stream.events.push({ - type: 'before', - callId: stream.groupCallId, - startTime: Math.max( - stream.prevEndMs, - (cmd.startTime ?? cmd.timestamp) - wallTime - ), - class: 'Tracing', - method: 'tracingGroup', - pageId, - params: { name: groupName }, - title: groupName, - apiName: 'tracing.tracingGroup' - }) - stream.lastTestUid = cmd.testUid } function buildParamsAndTitle( @@ -221,6 +242,37 @@ function actionError( return undefined } +/** Centre of the element a pointer action matched, from the captured element + * rects at the command's completion. Undefined for non-pointer actions, a + * non-string selector, or a selector absent from the captured elements (e.g. a + * WDIO text/xpath locator `getSelector` didn't reproduce). */ +function resolveActionPoint( + action: TraceAction, + cmd: CommandLog, + snapshotIndex?: FrameSnapshotIndex +): { x: number; y: number } | undefined { + if (action.class !== 'Element' || !POINTABLE_METHODS.has(action.method)) { + return undefined + } + const selector = cmd.args?.[0] + if (typeof selector !== 'string') { + return undefined + } + const elements = snapshotIndex?.elementsAt(cmd.timestamp) + for (const el of elements ?? []) { + // Narrow the unknown element record at the boundary (element-scripts shape). + const e = el as { + selector?: string + boundingBox?: { x: number; y: number; width: number; height: number } + } + if (e.selector === selector && e.boundingBox) { + const bb = e.boundingBox + return { x: bb.x + bb.width / 2, y: bb.y + bb.height / 2 } + } + } + return undefined +} + function buildAfterEvent( cmd: CommandLog, action: TraceAction, @@ -241,6 +293,10 @@ function buildAfterEvent( if (afterName) { afterEvent.afterSnapshot = afterName } + const point = resolveActionPoint(action, cmd, snapshotIndex) + if (point) { + afterEvent.point = point + } return afterEvent } @@ -272,7 +328,7 @@ function pushActionPair( params, title, apiName: `${action.class.toLowerCase()}.${action.method}`, - parentId: stream.groupCallId + parentId: stream.openGroups[stream.openGroups.length - 1]?.callId } const stack = callSourceToStack(cmd.callSource) if (stack) { @@ -294,7 +350,12 @@ export function buildActionEvents( testMetadata?: TestMetadataMap, snapshotIndex?: FrameSnapshotIndex ): ActionEvent[] { - const stream: ActionStream = { events: [], prevEndMs: 0, callCounter: 0 } + const stream: ActionStream = { + events: [], + prevEndMs: 0, + callCounter: 0, + openGroups: [] + } // Process in chronological order, not insertion order. Deferred rows (e.g. // Nightwatch native asserts, finalized in one batch at test-end) are appended // late but carry their real call-time `startTime`; since pushActionPair floors @@ -311,9 +372,9 @@ export function buildActionEvents( if (!action) { continue } - handleTestBoundary(stream, cmd, pageId, wallTime, testMetadata) + syncGroups(stream, cmd, pageId, wallTime, testMetadata) pushActionPair(stream, cmd, action, pageId, wallTime, snapshotIndex) } - closeGroup(stream) + closeGroupsFrom(stream, 0) return stream.events } diff --git a/packages/core/src/trace-frame-snapshots.ts b/packages/core/src/trace-frame-snapshots.ts index 618b612a..4f91f208 100644 --- a/packages/core/src/trace-frame-snapshots.ts +++ b/packages/core/src/trace-frame-snapshots.ts @@ -50,11 +50,20 @@ export interface FrameSnapshotRef { /** Correlates captured screenshots to callIds as the exporter assigns them. */ export class FrameSnapshotIndex { #byTimestamp = new Map<number, ActionSnapshot>() + // Element rects by timestamp, indexed independently of the screenshot gate so + // A8 input points resolve even when the frame carries no screenshot. + #elementsByTimestamp = new Map<number, unknown[]>() #refs: FrameSnapshotRef[] = [] #lastName?: string constructor(snapshots: ActionSnapshot[]) { for (const snap of snapshots) { + if (snap.elements) { + const currentEls = this.#elementsByTimestamp.get(snap.timestamp) + if (!currentEls || snap.elements.length > currentEls.length) { + this.#elementsByTimestamp.set(snap.timestamp, snap.elements) + } + } if (!snap.screenshot) { continue } @@ -69,6 +78,11 @@ export class FrameSnapshotIndex { } } + /** Captured element rects at a command's completion timestamp, if any. */ + elementsAt(timestamp: number): unknown[] | undefined { + return this.#elementsByTimestamp.get(timestamp) + } + /** Snapshot name representing the page state before the next action. */ beforeName(): string | undefined { return this.#lastName diff --git a/packages/core/src/trace-hierarchy.ts b/packages/core/src/trace-hierarchy.ts new file mode 100644 index 00000000..9ed4188c --- /dev/null +++ b/packages/core/src/trace-hierarchy.ts @@ -0,0 +1,30 @@ +// The ordered open-group path for a command — outermost first — that the action +// exporter's group stack turns into balanced nested Tracing.tracingGroup +// markers. The path is the test's ancestry (feature/scenario/suite, from the +// suite-metadata walk) + the test itself + its step, when a stepUid is set. + +import type { CommandLog, TestMetadataMap } from '@wdio/devtools-shared' + +export interface GroupNode { + uid: string + title: string +} + +export function buildGroupPath( + cmd: CommandLog, + testMetadata?: TestMetadataMap +): GroupNode[] { + const path: GroupNode[] = [] + if (cmd.testUid) { + const entry = testMetadata?.get(cmd.testUid) + for (const ancestor of entry?.ancestry ?? []) { + path.push({ uid: ancestor.uid, title: ancestor.title }) + } + path.push({ uid: cmd.testUid, title: entry?.title ?? cmd.testUid }) + } + if (cmd.stepUid) { + const stepEntry = testMetadata?.get(cmd.stepUid) + path.push({ uid: cmd.stepUid, title: stepEntry?.title ?? cmd.stepUid }) + } + return path +} diff --git a/packages/core/tests/trace-frame-snapshots.test.ts b/packages/core/tests/trace-frame-snapshots.test.ts index 81bdfebd..6d23cd05 100644 --- a/packages/core/tests/trace-frame-snapshots.test.ts +++ b/packages/core/tests/trace-frame-snapshots.test.ts @@ -1,10 +1,12 @@ import { describe, it, expect } from 'vitest' import { + buildActionEvents, buildImageFrameSnapshots, FrameSnapshotIndex, + type AfterEvent, type FrameSnapshotRef } from '@wdio/devtools-core' -import type { ActionSnapshot } from '@wdio/devtools-shared' +import type { ActionSnapshot, CommandLog } from '@wdio/devtools-shared' function snap(overrides: Partial<ActionSnapshot> = {}): ActionSnapshot { return { timestamp: 2000, command: 'click', screenshot: 'AAAA', ...overrides } @@ -57,6 +59,70 @@ describe('FrameSnapshotIndex', () => { index.claimAfter(3000, 'call@3') expect(index.beforeName()).toBe('after@call@3') }) + + it('exposes element rects by timestamp regardless of screenshot', () => { + const index = new FrameSnapshotIndex([ + snap({ screenshot: undefined, elements: [{ selector: '#go' }] }) + ]) + expect(index.elementsAt(2000)).toEqual([{ selector: '#go' }]) + expect(index.elementsAt(9999)).toBeUndefined() + }) + + it('keeps the richest element set for duplicate timestamps', () => { + const index = new FrameSnapshotIndex([ + snap({ elements: [{ selector: 'a' }] }), + snap({ elements: [{ selector: 'a' }, { selector: 'b' }] }) + ]) + expect(index.elementsAt(2000)).toHaveLength(2) + }) +}) + +describe('input point synthesis (A8)', () => { + const wallTime = 1000 + const box = { x: 10, y: 20, width: 40, height: 10 } + + function pointOf(commands: CommandLog[], snaps: ActionSnapshot[]) { + const events = buildActionEvents( + commands, + 'page@abc', + wallTime, + undefined, + new FrameSnapshotIndex(snaps) + ) + const after = events.find((e) => e.type === 'after') as + | AfterEvent + | undefined + return after?.point + } + + it('sets after.point to the matched element rect centre for a pointer action', () => { + const point = pointOf( + [{ command: 'click', args: ['#go'], timestamp: 2000, startTime: 1950 }], + [snap({ elements: [{ selector: '#go', boundingBox: box }] })] + ) + expect(point).toEqual({ x: 30, y: 25 }) + }) + + it('omits the point when the selector is absent from captured elements', () => { + const point = pointOf( + [{ command: 'click', args: ['#go'], timestamp: 2000, startTime: 1950 }], + [snap({ elements: [{ selector: '#other', boundingBox: box }] })] + ) + expect(point).toBeUndefined() + }) + + it('omits the point for non-pointer commands', () => { + const point = pointOf( + [{ command: 'getText', args: ['#go'], timestamp: 2000, startTime: 1950 }], + [ + snap({ + command: 'getText', + elements: [{ selector: '#go', boundingBox: box }] + }) + ] + ) + expect(point).toBeUndefined() + }) }) describe('buildImageFrameSnapshots', () => { diff --git a/packages/core/tests/trace-hierarchy.test.ts b/packages/core/tests/trace-hierarchy.test.ts new file mode 100644 index 00000000..fc10cd3f --- /dev/null +++ b/packages/core/tests/trace-hierarchy.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect } from 'vitest' +import { buildGroupPath } from '@wdio/devtools-core' +import type { CommandLog, TestMetadataMap } from '@wdio/devtools-shared' + +function cmd(overrides: Partial<CommandLog> = {}): CommandLog { + return { command: 'click', args: ['#go'], timestamp: 1, ...overrides } +} + +describe('buildGroupPath', () => { + it('is empty when the command has no testUid', () => { + expect(buildGroupPath(cmd())).toEqual([]) + }) + + it('is a single node (the test) when there is no ancestry or step', () => { + const meta: TestMetadataMap = new Map([ + ['t1', { title: 'logs in', specFile: 'a.js' }] + ]) + expect(buildGroupPath(cmd({ testUid: 't1' }), meta)).toEqual([ + { uid: 't1', title: 'logs in' } + ]) + }) + + it('prepends the ancestry chain outermost-first', () => { + const meta: TestMetadataMap = new Map([ + [ + 't1', + { + title: 'valid login', + specFile: 'login.feature', + ancestry: [ + { uid: 'f1', title: 'Feature: Login', kind: 'feature' }, + { uid: 's1', title: 'Scenario: valid', kind: 'scenario' } + ] + } + ] + ]) + expect(buildGroupPath(cmd({ testUid: 't1' }), meta)).toEqual([ + { uid: 'f1', title: 'Feature: Login' }, + { uid: 's1', title: 'Scenario: valid' }, + { uid: 't1', title: 'valid login' } + ]) + }) + + it('appends the step below the test when stepUid is set', () => { + const meta: TestMetadataMap = new Map([ + ['sc1', { title: 'Scenario', specFile: 'x.feature' }], + ['st1', { title: 'When I log in', specFile: 'x.feature' }] + ]) + expect( + buildGroupPath(cmd({ testUid: 'sc1', stepUid: 'st1' }), meta) + ).toEqual([ + { uid: 'sc1', title: 'Scenario' }, + { uid: 'st1', title: 'When I log in' } + ]) + }) + + it('falls back to the raw uid when a title is missing', () => { + expect(buildGroupPath(cmd({ testUid: 'sc1', stepUid: 'st1' }))).toEqual([ + { uid: 'sc1', title: 'sc1' }, + { uid: 'st1', title: 'st1' } + ]) + }) +}) diff --git a/packages/service/src/assertion-tracker.ts b/packages/service/src/assertion-tracker.ts index b0ab2f3d..416d6e5d 100644 --- a/packages/service/src/assertion-tracker.ts +++ b/packages/service/src/assertion-tracker.ts @@ -24,6 +24,7 @@ export interface AssertionTrackerContext { getCapturer: () => SessionCapturer getBrowser: () => WebdriverIO.Browser | undefined getTestUid: () => string | undefined + getStepUid: () => string | undefined options: ServiceOptions actionSnapshots: ActionSnapshot[] } @@ -32,6 +33,7 @@ interface PendingAssertion { matcherName: string expectedValue?: unknown testUid?: string + stepUid?: string } export class AssertionTracker { @@ -76,7 +78,8 @@ export class AssertionTracker { this.#pendingAssertion = { matcherName: params.matcherName, expectedValue: params.expectedValue, - testUid: this.#ctx.getTestUid() + testUid: this.#ctx.getTestUid(), + stepUid: this.#ctx.getStepUid() } } this.#assertionDepth++ @@ -105,6 +108,7 @@ export class AssertionTracker { this.#pendingAssertion = undefined const capturer = this.#ctx.getCapturer() const entry = expectAssertionToCommandLog(params, this.#ctx.getTestUid()) + entry.stepUid = this.#ctx.getStepUid() if (capturer.coalesceAssertionIntoLastRead(entry, isMatcherReadCommand)) { return } @@ -176,6 +180,7 @@ export class AssertionTracker { }, pending.testUid ) + entry.stepUid = pending.stepUid const capturer = this.#ctx.getCapturer() if ( capturer.coalesceAssertionIntoLastRead(entry, isMatcherReadCommand, true) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 3161e4f2..739f8f5b 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -97,6 +97,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { getCapturer: () => this.#sessionCapturer, getBrowser: () => this.#browser, getTestUid: () => this.#currentTestUid, + getStepUid: () => this.#currentStepUid, options: this.#options, actionSnapshots: this.#actionSnapshots }) @@ -126,6 +127,11 @@ export default class DevToolsHookService implements Services.ServiceInstance { /** Current test UID, set in beforeTest(), used by afterCommand() to tag commands. */ #currentTestUid?: string + /** Current Cucumber step UID, set in beforeStep(), used by afterCommand() to + * nest commands under the step in the trace group tree (C2). */ + #currentStepUid?: string + /** Per-scenario step counter for stable, collision-free step uids. */ + #currentStepIndex = 0 /** Wall-clock ms at the current test's start, set in beforeTest/beforeScenario; * the lower bound of that test's video frame window (per-test slicing). */ @@ -412,6 +418,8 @@ export default class DevToolsHookService implements Services.ServiceInstance { }) { this.resetStack() this.#currentTestStartWallTime = Date.now() + this.#currentStepIndex = 0 + this.#currentStepUid = undefined const featureFile = world?.pickle?.uri const scenarioName = world?.pickle?.name @@ -464,6 +472,25 @@ export default class DevToolsHookService implements Services.ServiceInstance { } } + // Tag the scenario's commands with a stable per-step uid so the trace nests + // them under the step (Feature→Scenario→Step). The uid combines the scenario + // uid with a per-scenario counter, so repeated step text can't collide. + beforeStep(step?: { text?: string; keyword?: string }) { + if (!this.#currentTestUid) { + return + } + this.#currentStepIndex += 1 + const uid = `${this.#currentTestUid}:step:${this.#currentStepIndex}` + const title = + [step?.keyword, step?.text].filter(Boolean).join('').trim() || + `Step ${this.#currentStepIndex}` + this.#currentStepUid = uid + this.#testMetadata.set(uid, { + title, + specFile: this.#testMetadata.get(this.#currentTestUid)?.specFile ?? '' + }) + } + // afterStep fires right after each step, so the failing assertion lands next // to the step's actions rather than after reloadSession at scenario end. afterStep( @@ -471,6 +498,7 @@ export default class DevToolsHookService implements Services.ServiceInstance { _scenario?: unknown, result?: { error?: unknown } ) { + this.#currentStepUid = undefined this.#assertionTracker.handleOutcome(result?.error) } @@ -737,7 +765,8 @@ export default class DevToolsHookService implements Services.ServiceInstance { error, frame.callSource, frame.startTimestamp, - this.#currentTestUid + this.#currentTestUid, + this.#currentStepUid ) if (this.#options.mode === 'trace') { await captureActionResult( diff --git a/packages/service/src/session.ts b/packages/service/src/session.ts index 51632586..d679e105 100644 --- a/packages/service/src/session.ts +++ b/packages/service/src/session.ts @@ -112,7 +112,8 @@ export class SessionCapturer extends SessionCapturerBase { error: Error | undefined, callSource?: string, commandStartTime?: number, - testUid?: string + testUid?: string, + stepUid?: string ) { const { sourceFileLocation, absolutePath } = this.#resolveUserStackFrame() const sourceFilePath = absolutePath.split(':')[0] @@ -127,7 +128,8 @@ export class SessionCapturer extends SessionCapturerBase { timestamp: Date.now(), startTime: commandStartTime, callSource: callSource ?? absolutePath, - testUid + testUid, + stepUid } if (!isNativeMobile(browser)) { try { diff --git a/packages/shared/src/trace-actions.ts b/packages/shared/src/trace-actions.ts index c3bd7462..75eebeb2 100644 --- a/packages/shared/src/trace-actions.ts +++ b/packages/shared/src/trace-actions.ts @@ -105,3 +105,34 @@ export const ACTION_MAP: Record<string, TraceAction> = { getRect: { class: 'Element', method: 'getRect' }, getCurrentUrl: { class: 'Page', method: 'getUrl' } } + +/** Trace methods (ACTION_MAP values) that act at a point on the page — the + * exporter emits a hit `point` for these so the player can mark where the + * action landed. `fill` is included (typing focuses the element first). */ +export const POINTABLE_METHODS: ReadonlySet<string> = new Set([ + 'click', + 'dblclick', + 'hover', + 'tap', + 'dragTo', + 'scrollIntoViewIfNeeded', + 'selectOption', + 'fill' +]) + +/** Runner command names (native or trace method) that type into an element — + * drives the timeline's keyboard glyph vs the pointer glyph. */ +const KEYBOARD_COMMANDS: ReadonlySet<string> = new Set([ + 'setValue', + 'sendKeys', + 'addValue', + 'clearValue', + 'clear', + 'keys', + 'fill', + 'press' +]) + +export function isKeyboardCommand(command: string): boolean { + return KEYBOARD_COMMANDS.has(command) +} diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 40cd45b7..98dc2604 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -173,10 +173,17 @@ export interface CommandLog { cookies?: string documentInfo?: DocumentInfo id?: number + /** Cucumber step this command ran under — nests below testUid in the trace's + * group tree (Feature → Scenario → Step). Set by the adapter step hooks. */ + stepUid?: string /** Depth-indented accessibility-tree text for the page state at this command * (from the per-action `-snapshot.txt` resource). Reconstructed by the trace * reader; drives the player's A11y tab. */ snapshotText?: string + /** Page-coordinate hit point for pointer actions — the centre of the matched + * element at capture time. Synthesized in the exporter; drives the snapshot + * click marker and the timeline pointer glyph. */ + point?: { x: number; y: number } } /** From 33e590cef83059ec43bd9b7079f2530bc40a8e31 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 17:36:41 +0530 Subject: [PATCH 87/91] fix(trace): drop replayed comment nodes + settle layout before overlay measure --- packages/app/src/components/browser/element-overlay.ts | 6 ++++++ packages/script/src/utils.ts | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/browser/element-overlay.ts b/packages/app/src/components/browser/element-overlay.ts index c83cb0db..dbf00d42 100644 --- a/packages/app/src/components/browser/element-overlay.ts +++ b/packages/app/src/components/browser/element-overlay.ts @@ -52,6 +52,12 @@ export function drawElementOverlay( return } clearElementOverlay(iframe) + // Force a synchronous layout flush before measuring. #sizeSnapshotToViewport + // strips + restores the iframe's inline size in the same frame that draws the + // overlay, so the content reflow to full width is still pending — reading + // rects now would capture the transient narrow-breakpoint layout (boxes end up + // low + oversized). Reading offsetHeight settles layout first. + void docEl.documentElement.offsetHeight const scrollY = iframe?.contentWindow?.scrollY || 0 const scrollX = iframe?.contentWindow?.scrollX || 0 for (const selector of selectors) { diff --git a/packages/script/src/utils.ts b/packages/script/src/utils.ts index 258b1358..c6d86cd0 100644 --- a/packages/script/src/utils.ts +++ b/packages/script/src/utils.ts @@ -25,7 +25,10 @@ export function parseNode( const props: Record<string, unknown> = {} if (fragment.nodeName === '#comment') { - return (fragment as vComment).data + // Drop comment content — returning its data rendered the comment as visible + // text on replay (e.g. an IE conditional comment's `<![endif]` showed as + // text and added a line box that shifted the whole page layout down). + return '' } if (fragment.nodeName === '#text') { return (fragment as vText).value From c46be9830ee79c2257e358e6e6efb31b52779caa Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 18:13:19 +0530 Subject: [PATCH 88/91] =?UTF-8?q?fix(trace):=20correct=20replay=20fidelity?= =?UTF-8?q?=20=E2=80=94=20comments,=20overlay=20alignment,=20zoom=20flicke?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/app/src/components/browser/snapshot.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/app/src/components/browser/snapshot.ts b/packages/app/src/components/browser/snapshot.ts index c647fdad..3193057d 100644 --- a/packages/app/src/components/browser/snapshot.ts +++ b/packages/app/src/components/browser/snapshot.ts @@ -148,9 +148,12 @@ export class DevtoolsBrowser extends Element { return } - this.iframe?.removeAttribute('style') - // Defer to next frame so we read post-reflow dimensions on resize events. + // NB: we deliberately do NOT clear the iframe's inline style first — the rAF + // below overwrites every property it sets, so the iframe keeps its prior + // (correct) transform until then. Clearing synchronously here made it paint + // one frame un-scaled → a zoom flicker on every replayed frame during + // playback, and let the overlay measure a collapsed/narrow layout. requestAnimationFrame(() => { if (!this.section || !this.header) { return From 4a651ecef24820eaba7817e0c4528ad5f6eb1689 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 18:15:10 +0530 Subject: [PATCH 89/91] fix(service): Gate the devtools-artifacts manifest behind an emitArtifactsManifest option (default off) --- packages/service/src/index.ts | 13 ++++++++++++- packages/service/src/types.ts | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index 739f8f5b..d9946b33 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -132,6 +132,10 @@ export default class DevToolsHookService implements Services.ServiceInstance { #currentStepUid?: string /** Per-scenario step counter for stable, collision-free step uids. */ #currentStepIndex = 0 + /** True when @wdio/allure-reporter is in the WDIO config (detected in + * beforeSession) — auto-enables the artifacts manifest even if the option + * isn't set, since session/spec-scoped Allure attach reads it. */ + #allureReporterConfigured = false /** Wall-clock ms at the current test's start, set in beforeTest/beforeScenario; * the lower bound of that test's video frame window (per-test slicing). */ @@ -266,7 +270,8 @@ export default class DevToolsHookService implements Services.ServiceInstance { resolveOutputDir: () => this.#outputDir, prepareSnapshots: dedupeSnapshotsByTimestamp, log: (level, msg) => log[level](msg), - emitManifest: true, + emitManifest: + this.#options.emitArtifactsManifest ?? this.#allureReporterConfigured, collectedArtifacts: this.#artifacts, onArtifact: (a) => this.#artifacts.push(a) } @@ -389,6 +394,12 @@ export default class DevToolsHookService implements Services.ServiceInstance { } if ('reporters' in config) { + // Detect the Allure reporter (before we append our own) so the artifacts + // manifest auto-enables for session/spec-scoped Allure attach. + this.#allureReporterConfigured = (config.reporters || []).some((r) => { + const name = Array.isArray(r) ? r[0] : r + return typeof name === 'string' && name.includes('allure') + }) const self = this config.reporters = [ ...(config.reporters || []), diff --git a/packages/service/src/types.ts b/packages/service/src/types.ts index 84ef345f..695eafa8 100644 --- a/packages/service/src/types.ts +++ b/packages/service/src/types.ts @@ -53,6 +53,12 @@ export interface ServiceOptions extends BaseDevToolsOptions { * attached inline to Allure. `off` (default) or a retention policy. Trace * mode + `traceGranularity: 'test'` only. WDIO-service-specific for now. */ video?: TraceVideoPolicy + /** Write the `devtools-artifacts-<sessionId>.json` manifest next to the trace + * — the generic index reporters/CI consume to discover produced artifacts. + * Off by default (WDIO auto-attaches per-test traces to Allure directly); + * auto-enabled when `@wdio/allure-reporter` is in the config, since + * session/spec-scoped Allure attach reads the manifest. */ + emitArtifactsManifest?: boolean /** * capabilities used to launch the devtools application * @default From 39ec0a65d238249b7dc1776e5685178262b823c5 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 19:57:18 +0530 Subject: [PATCH 90/91] test(service): add addCommand to browser mocks for main's getSnapshot hook --- packages/service/tests/assertion-rows.test.ts | 3 ++- packages/service/tests/trace-granularity.test.ts | 1 + packages/service/tests/trace-metadata.test.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/service/tests/assertion-rows.test.ts b/packages/service/tests/assertion-rows.test.ts index ddd216b2..9f045f8a 100644 --- a/packages/service/tests/assertion-rows.test.ts +++ b/packages/service/tests/assertion-rows.test.ts @@ -49,7 +49,8 @@ const mockBrowser = { .fn() .mockResolvedValue({ width: 1, height: 1, offsetLeft: 0, offsetTop: 0 }), on: vi.fn(), - emit: vi.fn() + emit: vi.fn(), + addCommand: vi.fn() } as unknown as WebdriverIO.Browser // before() wires node:assert capture; restore the real methods afterwards. diff --git a/packages/service/tests/trace-granularity.test.ts b/packages/service/tests/trace-granularity.test.ts index 66a3b971..537c0281 100644 --- a/packages/service/tests/trace-granularity.test.ts +++ b/packages/service/tests/trace-granularity.test.ts @@ -98,6 +98,7 @@ describe('DevtoolsService - trace granularity slicing', () => { .mockResolvedValue({ width: 1, height: 1, offsetLeft: 0, offsetTop: 0 }), on: vi.fn(), emit: vi.fn(), + addCommand: vi.fn(), options: { rootDir: '/proj' }, capabilities: { browserName: 'chrome' } } as never diff --git a/packages/service/tests/trace-metadata.test.ts b/packages/service/tests/trace-metadata.test.ts index 9b2ee653..e34527f0 100644 --- a/packages/service/tests/trace-metadata.test.ts +++ b/packages/service/tests/trace-metadata.test.ts @@ -99,6 +99,7 @@ describe('DevtoolsService - afterTest state stamping', () => { .mockResolvedValue({ width: 1, height: 1, offsetLeft: 0, offsetTop: 0 }), on: vi.fn(), emit: vi.fn(), + addCommand: vi.fn(), options: { rootDir: '/proj' }, capabilities: { browserName: 'chrome' } } as never From c1a27c7654b9d27586d1d7824221cdcec288ceec Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan <vishnu.p@browserstack.com> Date: Mon, 20 Jul 2026 19:57:29 +0530 Subject: [PATCH 91/91] fix: green the branch after integrating main --- packages/core/src/element-snapshot.ts | 21 +++++++++++++++------ packages/service/src/index.ts | 19 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/packages/core/src/element-snapshot.ts b/packages/core/src/element-snapshot.ts index 2070c61d..9a03f476 100644 --- a/packages/core/src/element-snapshot.ts +++ b/packages/core/src/element-snapshot.ts @@ -798,7 +798,9 @@ export function serializeMobileSnapshot( function extractTagFromSelector(selector: string, fallback: string): string { // Matches tag name followed by a CSS selector combinator or operator. // Supports hyphenated custom elements (my-component) and pseudo-classes (:nth-of-type). - const match = selector.match(/^([a-z][a-z0-9]*(?:-[a-z][a-z0-9]*)*)[*.#\[:=(^$~]/) + const match = selector.match( + /^([a-z][a-z0-9]*(?:-[a-z][a-z0-9]*)*)[*.#\[:=(^$~]/ + ) if (match) { return match[1] } @@ -810,7 +812,10 @@ function extractTagFromSelector(selector: string, fallback: string): string { } /** Walk backwards to find the nearest structural container name for ∈ context. */ -function findContextName(nodes: SnapshotNode[], index: number): string | undefined { +function findContextName( + nodes: SnapshotNode[], + index: number +): string | undefined { const myDepth = nodes[index].depth for (let i = index - 1; i >= 0; i--) { if (nodes[i].depth <= myDepth && nodes[i].name) { @@ -844,7 +849,10 @@ export function buildSnapshot( const selectorCounts = new Map<string, number>() for (const node of nodes) { if (node.isInteractive && node.selector) { - selectorCounts.set(node.selector, (selectorCounts.get(node.selector) ?? 0) + 1) + selectorCounts.set( + node.selector, + (selectorCounts.get(node.selector) ?? 0) + 1 + ) } } @@ -945,9 +953,10 @@ export function accessibilityNodesToSnapshotNodes( continue } - const tagName = isInteractive && node.selector - ? extractTagFromSelector(node.selector, node.role) - : node.role + const tagName = + isInteractive && node.selector + ? extractTagFromSelector(node.selector, node.role) + : node.role result.push({ role: node.role, diff --git a/packages/service/src/index.ts b/packages/service/src/index.ts index d9946b33..ab3dce36 100644 --- a/packages/service/src/index.ts +++ b/packages/service/src/index.ts @@ -362,15 +362,18 @@ export default class DevToolsHookService implements Services.ServiceInstance { * * Returns { text, elements } — see @wdio/elements SnapshotResult. */ - browser.addCommand('getSnapshot', async (options?: { inViewportOnly?: boolean }) => { - try { - const { getSnapshot } = await import('@wdio/elements') - return await getSnapshot(browser, options ?? { inViewportOnly: true }) - } catch (err) { - log.warn(`getSnapshot failed: ${errorMessage(err)}`) - return { text: '[Snapshot unavailable]', elements: {} } + browser.addCommand( + 'getSnapshot', + async (options?: { inViewportOnly?: boolean }) => { + try { + const { getSnapshot } = await import('@wdio/elements') + return await getSnapshot(browser, options ?? { inViewportOnly: true }) + } catch (err) { + log.warn(`getSnapshot failed: ${errorMessage(err)}`) + return { text: '[Snapshot unavailable]', elements: {} } + } } - }) + ) } // The method signature is corrected to use W3CCapabilities