diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 841cdf5e7..3a2e95bfc 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -319,6 +319,11 @@ function summarizeProviderScenarioFlagExclusions() { 'proxyPort', ], }, + { + name: 'daemon lifecycle control', + owner: 'daemon CLI lifecycle tests', + keys: ['clean'], + }, { name: 'platform boot fallback without provider seam', owner: 'handler and Android platform unit tests', diff --git a/src/cli-schema/command-overrides.ts b/src/cli-schema/command-overrides.ts index 8b4dc6b47..ec4e3e3b7 100644 --- a/src/cli-schema/command-overrides.ts +++ b/src/cli-schema/command-overrides.ts @@ -29,6 +29,16 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { positionalArgs: ['status|login|logout'], supportedFlags: ['remoteConfig', 'stateDir'], }, + daemon: { + usageOverride: 'daemon stop [--state-dir ] [--clean]', + listUsageOverride: 'daemon stop', + helpDescription: + 'Stop a local daemon after verifying its PID/start-time identity. Use --clean to remove retained Apple runner processes and leases owned by that daemon.', + summary: 'Safely stop a local daemon and optionally clean retained runner resources', + positionalArgs: ['stop'], + allowedFlags: ['clean'], + supportedFlags: ['stateDir'], + }, connect: { usageOverride: 'connect [cloud|proxy|limrun|browserstack|aws-device-farm] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', diff --git a/src/cli.ts b/src/cli.ts index c9acd341b..82cb2dee9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -75,6 +75,7 @@ const REMOTE_MATERIALIZATION_DEFERRED_COMMANDS = new Set([ 'connect', 'connection', 'close', + 'daemon', 'disconnect', 'metro', 'proxy', @@ -554,6 +555,7 @@ function resolveActiveConnectionDefaults(options: { if ( options.command === 'connect' || options.command === 'connection' || + options.command === 'daemon' || options.command === 'proxy' ) { return null; @@ -577,7 +579,9 @@ function shouldMaterializeRemoteConnection(command: string): boolean { } function shouldResolveRemoteAuth(command: string): boolean { - return command !== 'auth' && command !== 'connection' && command !== 'proxy'; + return ( + command !== 'auth' && command !== 'connection' && command !== 'daemon' && command !== 'proxy' + ); } function shouldWarnOpenMayMissRemoteRuntime(options: { diff --git a/src/cli/commands/__tests__/daemon.test.ts b/src/cli/commands/__tests__/daemon.test.ts new file mode 100644 index 000000000..ad0aeda54 --- /dev/null +++ b/src/cli/commands/__tests__/daemon.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; +import type { DaemonStopResult } from '../../../daemon/daemon-stop.ts'; + +const mocks = vi.hoisted(() => ({ + cleanupRunnerLeasesForOwner: vi.fn(async () => undefined), + readDaemonShutdownReport: vi.fn(), + readDaemonStopIdentity: vi.fn(), + stopDaemon: vi.fn(), + writeCommandOutput: vi.fn(), +})); + +vi.mock('../../../daemon/daemon-stop.ts', () => ({ + readDaemonStopIdentity: mocks.readDaemonStopIdentity, + stopDaemon: mocks.stopDaemon, +})); +vi.mock('../../../daemon/daemon-shutdown-report.ts', () => ({ + readDaemonShutdownReport: mocks.readDaemonShutdownReport, +})); +vi.mock('../../../platforms/apple/core/runner/runner-lease.ts', () => ({ + cleanupRunnerLeasesForOwner: mocks.cleanupRunnerLeasesForOwner, +})); +vi.mock('../../../platforms/apple/core/runner/runner-disposal.ts', () => ({ + runnerLeaseCleanupAdapter: {}, +})); +vi.mock('../shared.ts', () => ({ writeCommandOutput: mocks.writeCommandOutput })); + +import { daemonCommand } from '../daemon.ts'; + +const GRACEFUL_RESULT: DaemonStopResult = { + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + claimsReleased: [], + claimsOrphaned: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [], +}; + +afterEach(() => { + vi.clearAllMocks(); +}); + +test('accepts only daemon stop', async () => { + await assert.rejects( + async () => + await daemonCommand({ + positionals: [], + flags: { help: false, json: false, version: false }, + client: {} as never, + }), + (error: { code?: string }) => error.code === 'INVALID_ARGS', + ); + await assert.rejects( + async () => + await daemonCommand({ + positionals: ['stop', 'extra'], + flags: { help: false, json: false, version: false }, + client: {} as never, + }), + (error: { code?: string }) => error.code === 'INVALID_ARGS', + ); +}); + +test('merges a graceful shutdown report and cleans runner leases with the start-time identity', async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-command-')); + mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); + mocks.readDaemonShutdownReport.mockReturnValue({ + providerReleases: { + released: [{ leaseId: 'lease-1', provider: 'limrun' }], + pending: [], + }, + }); + + try { + await daemonCommand({ + positionals: ['stop'], + flags: { clean: true, help: false, json: false, stateDir, version: false }, + client: {} as never, + }); + + expect(mocks.cleanupRunnerLeasesForOwner).toHaveBeenCalledWith( + { pid: 123, startTime: 'start-time' }, + {}, + ); + expect(mocks.writeCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ clean: true, json: false }), + expect.objectContaining({ + clean: true, + providerReleases: { + status: 'completed', + released: [{ leaseId: 'lease-1', provider: 'limrun' }], + pending: [], + }, + }), + expect.any(Function), + ); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('reports graceful provider cleanup as unknown when the shutdown report is unavailable', async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-command-')); + mocks.readDaemonStopIdentity.mockReturnValue(null); + mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); + mocks.readDaemonShutdownReport.mockReturnValue(null); + + try { + await daemonCommand({ + positionals: ['stop'], + flags: { help: false, json: false, stateDir, version: false }, + client: {} as never, + }); + + expect(mocks.writeCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ json: false }), + expect.objectContaining({ + clean: false, + cleanupConfidence: 'unknown', + providerReleases: { status: 'unknown', released: [], pending: null }, + warnings: [expect.stringContaining('provider cleanup state is unknown')], + }), + expect.any(Function), + ); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts new file mode 100644 index 000000000..d5ffd3a46 --- /dev/null +++ b/src/cli/commands/daemon.ts @@ -0,0 +1,67 @@ +import { resolveDaemonPaths } from '../../daemon/config.ts'; +import { + readDaemonStopIdentity, + stopDaemon, + type DaemonStopResult, +} from '../../daemon/daemon-stop.ts'; +import { readDaemonShutdownReport } from '../../daemon/daemon-shutdown-report.ts'; +import { AppError } from '../../kernel/errors.ts'; +import { writeCommandOutput } from './shared.ts'; +import type { ClientCommandHandler } from './router-types.ts'; + +export const daemonCommand: ClientCommandHandler = async ({ positionals, flags }) => { + const subcommand = positionals[0]; + if (subcommand !== 'stop' || positionals.length !== 1) { + throw new AppError('INVALID_ARGS', 'daemon accepts only: stop'); + } + const paths = resolveDaemonPaths(flags.stateDir); + const identity = readDaemonStopIdentity(paths.infoPath); + const stopped = await stopDaemon({ paths }); + const report = stopped.mode === 'graceful' ? readDaemonShutdownReport(paths.baseDir) : null; + const result = mergeShutdownReport(stopped, report); + const shouldClean = flags.clean === true && identity !== null && result.stopped; + if (shouldClean) { + const [{ cleanupRunnerLeasesForOwner }, { runnerLeaseCleanupAdapter }] = await Promise.all([ + import('../../platforms/apple/core/runner/runner-lease.ts'), + import('../../platforms/apple/core/runner/runner-disposal.ts'), + ]); + await cleanupRunnerLeasesForOwner( + { pid: identity.pid, startTime: identity.processStartTime }, + runnerLeaseCleanupAdapter, + ); + } + const data = { ...result, clean: shouldClean }; + writeCommandOutput(flags, data, () => renderDaemonStop(data)); + return true; +}; + +function mergeShutdownReport( + stopped: DaemonStopResult, + report: ReturnType, +): DaemonStopResult { + if (stopped.mode !== 'graceful' || report) { + return report + ? { ...stopped, providerReleases: { status: 'completed', ...report.providerReleases } } + : stopped; + } + return { + ...stopped, + cleanupConfidence: 'unknown', + providerReleases: { status: 'unknown', released: [], pending: null }, + warnings: [ + ...stopped.warnings, + 'The graceful shutdown report is unavailable, so provider cleanup state is unknown. Provider allocations may remain active.', + ], + }; +} + +function renderDaemonStop( + result: Pick & { + clean: boolean; + }, +): string { + const headline = result.stopped ? `Daemon stopped (${result.mode}).` : 'No running daemon found.'; + return [headline, result.clean ? 'Retained runner cleanup completed.' : null, ...result.warnings] + .filter((line): line is string => Boolean(line)) + .join('\n'); +} diff --git a/src/cli/commands/router.ts b/src/cli/commands/router.ts index f47cf15fd..cefbd4aa3 100644 --- a/src/cli/commands/router.ts +++ b/src/cli/commands/router.ts @@ -3,6 +3,7 @@ import type { AgentDeviceClient } from '../../agent-device-client.ts'; import { isClientBackedCliCommandName, type ClientBackedCliCommandName } from './client-backed.ts'; import { connectCommand, connectionCommand, disconnectCommand } from './connection.ts'; import { authCommand } from './auth.ts'; +import { daemonCommand } from './daemon.ts'; import { proxyCommand } from './proxy.ts'; import { replayCommand } from './replay.ts'; import { screenshotCommand, diffCommand } from './screenshot.ts'; @@ -19,6 +20,7 @@ const dedicatedCliCommandHandlers = { disconnect: disconnectCommand, connection: connectionCommand, auth: authCommand, + daemon: daemonCommand, proxy: proxyCommand, replay: replayCommand, screenshot: screenshotCommand, diff --git a/src/commands/cli-grammar/flag-definitions-connection.ts b/src/commands/cli-grammar/flag-definitions-connection.ts index 612a93d19..e2976da40 100644 --- a/src/commands/cli-grammar/flag-definitions-connection.ts +++ b/src/commands/cli-grammar/flag-definitions-connection.ts @@ -201,6 +201,13 @@ export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageDescription: 'Force replacement of existing state (connect: replace an active connection; --save-script: overwrite an existing target instead of refusing)', }, + { + key: 'clean', + names: ['--clean'], + type: 'boolean', + usageLabel: '--clean', + usageDescription: 'Daemon stop: remove retained runner processes and leases after stopping', + }, { key: 'noLogin', names: ['--no-login'], diff --git a/src/contracts/cli-flags.ts b/src/contracts/cli-flags.ts index 445b9033c..330a8494b 100644 --- a/src/contracts/cli-flags.ts +++ b/src/contracts/cli-flags.ts @@ -42,6 +42,7 @@ export type CliFlags = CloudProviderProfileFields & provider?: string; providerSessionId?: string; force?: boolean; + clean?: boolean; noLogin?: boolean; kind?: string; perfTemplate?: string; diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index a079f32cd..e45b44168 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -1075,6 +1075,15 @@ export const RAW_COMMAND_DESCRIPTORS = [ timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: false, }, + { + name: 'daemon', + ...(ownerFilesEnabled ? { ownerFiles: ['src/cli/commands/daemon.ts'] as const } : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: false, + mcpExposed: false, + }, { name: 'metro', ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/metro/index.ts'] as const } : {}), diff --git a/src/daemon/__tests__/daemon-shutdown-report.test.ts b/src/daemon/__tests__/daemon-shutdown-report.test.ts new file mode 100644 index 000000000..559b453bb --- /dev/null +++ b/src/daemon/__tests__/daemon-shutdown-report.test.ts @@ -0,0 +1,54 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { + clearDaemonShutdownReport, + readDaemonShutdownReport, + writeDaemonShutdownReport, +} from '../daemon-shutdown-report.ts'; +import { LeaseRegistry } from '../lease-registry.ts'; + +test('round-trips provider release records without persisting lease credentials', () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-shutdown-report-')); + const lease = new LeaseRegistry().allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseProvider: 'limrun', + }); + + try { + writeDaemonShutdownReport(stateDir, { released: [lease], pending: [lease] }); + + expect(readDaemonShutdownReport(stateDir)).toEqual({ + providerReleases: { + released: [{ leaseId: lease.leaseId, provider: 'limrun' }], + pending: [{ leaseId: lease.leaseId, provider: 'limrun' }], + }, + }); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('ignores malformed shutdown reports and clear removes a prior report', () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-shutdown-report-')); + const reportPath = path.join(stateDir, 'daemon-shutdown.json'); + + try { + expect(readDaemonShutdownReport(stateDir)).toBeNull(); + fs.writeFileSync(reportPath, JSON.stringify({ providerReleases: { released: [] } })); + expect(readDaemonShutdownReport(stateDir)).toBeNull(); + + fs.writeFileSync( + reportPath, + JSON.stringify({ providerReleases: { released: [{}], pending: [] } }), + ); + expect(readDaemonShutdownReport(stateDir)).toBeNull(); + + clearDaemonShutdownReport(stateDir); + expect(fs.existsSync(reportPath)).toBe(false); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); diff --git a/src/daemon/__tests__/daemon-stop.test.ts b/src/daemon/__tests__/daemon-stop.test.ts new file mode 100644 index 000000000..4309d304c --- /dev/null +++ b/src/daemon/__tests__/daemon-stop.test.ts @@ -0,0 +1,161 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + isAgentDeviceDaemonProcess: vi.fn(), + isProcessAlive: vi.fn(), + sleep: vi.fn(async () => undefined), + trySignalProcess: vi.fn(), + waitForProcessExit: vi.fn(), +})); + +vi.mock('../daemon-process.ts', () => ({ + isAgentDeviceDaemonProcess: mocks.isAgentDeviceDaemonProcess, + trySignalProcess: mocks.trySignalProcess, +})); +vi.mock('../../utils/host-process.ts', () => ({ + isProcessAlive: mocks.isProcessAlive, + waitForProcessExit: mocks.waitForProcessExit, +})); +vi.mock('../../utils/timeouts.ts', () => ({ sleep: mocks.sleep })); + +import { resolveDaemonPaths } from '../config.ts'; +import { stopDaemon } from '../daemon-stop.ts'; + +afterEach(() => { + vi.clearAllMocks(); +}); + +function createDaemonPaths(): ReturnType { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-stop-')); + const paths = resolveDaemonPaths(stateDir); + fs.mkdirSync(paths.baseDir, { recursive: true }); + fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: 'start-time' })); + return paths; +} + +function removeDaemonPaths(paths: ReturnType): void { + fs.rmSync(paths.baseDir, { recursive: true, force: true }); +} + +test('reports not-running when daemon metadata is absent', async () => { + const paths = resolveDaemonPaths( + fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-stop-')), + ); + + try { + const result = await stopDaemon({ paths }); + expect(result).toMatchObject({ stopped: false, mode: 'not-running' }); + } finally { + removeDaemonPaths(paths); + } +}); + +test('refuses to signal a live process whose daemon identity cannot be verified', async () => { + const paths = createDaemonPaths(); + mocks.isAgentDeviceDaemonProcess.mockReturnValue(false); + mocks.isProcessAlive.mockReturnValue(true); + + try { + await assert.rejects( + async () => await stopDaemon({ paths }), + (error: { code?: string }) => error.code === 'COMMAND_FAILED', + ); + expect(mocks.trySignalProcess).not.toHaveBeenCalled(); + } finally { + removeDaemonPaths(paths); + } +}); + +test('refuses to signal a live process when daemon metadata lacks a non-empty start-time identity', async () => { + const paths = createDaemonPaths(); + fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: ' ' })); + mocks.isProcessAlive.mockReturnValue(true); + + try { + await assert.rejects( + async () => await stopDaemon({ paths }), + (error: { code?: string }) => error.code === 'COMMAND_FAILED', + ); + expect(mocks.isAgentDeviceDaemonProcess).not.toHaveBeenCalled(); + expect(mocks.trySignalProcess).not.toHaveBeenCalled(); + } finally { + removeDaemonPaths(paths); + } +}); + +test('treats an exited daemon between identity verification and SIGTERM as not-running', async () => { + const paths = createDaemonPaths(); + mocks.isAgentDeviceDaemonProcess.mockReturnValue(true); + mocks.trySignalProcess.mockReturnValue(false); + mocks.isProcessAlive.mockReturnValue(false); + + try { + const result = await stopDaemon({ paths }); + expect(result).toMatchObject({ stopped: false, mode: 'not-running' }); + } finally { + removeDaemonPaths(paths); + } +}); + +test('reports graceful cleanup after SIGTERM exits the verified daemon', async () => { + const paths = createDaemonPaths(); + mocks.isAgentDeviceDaemonProcess.mockReturnValue(true); + mocks.trySignalProcess.mockReturnValue(true); + mocks.waitForProcessExit.mockImplementation(async () => { + fs.rmSync(paths.infoPath, { force: true }); + return true; + }); + + try { + const result = await stopDaemon({ paths }); + expect(result).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + providerReleases: { pending: [] }, + }); + expect(mocks.trySignalProcess).toHaveBeenCalledWith(123, 'SIGTERM'); + } finally { + removeDaemonPaths(paths); + } +}); + +test('re-verifies identity before SIGKILL and reports forced cleanup as unknown', async () => { + const paths = createDaemonPaths(); + mocks.isAgentDeviceDaemonProcess.mockReturnValue(true); + mocks.trySignalProcess.mockReturnValue(true); + mocks.waitForProcessExit.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + + try { + const result = await stopDaemon({ paths }); + expect(result).toMatchObject({ + stopped: true, + mode: 'forced', + cleanupConfidence: 'unknown', + providerReleases: { pending: null }, + }); + expect(mocks.trySignalProcess).toHaveBeenNthCalledWith(1, 123, 'SIGTERM'); + expect(mocks.trySignalProcess).toHaveBeenNthCalledWith(2, 123, 'SIGKILL'); + } finally { + removeDaemonPaths(paths); + } +}); + +test('does not send SIGKILL if the daemon identity changes during the graceful wait', async () => { + const paths = createDaemonPaths(); + mocks.isAgentDeviceDaemonProcess.mockReturnValueOnce(true).mockReturnValueOnce(false); + mocks.trySignalProcess.mockReturnValue(true); + mocks.waitForProcessExit.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + + try { + await stopDaemon({ paths }); + expect(mocks.trySignalProcess).toHaveBeenCalledTimes(1); + expect(mocks.trySignalProcess).toHaveBeenCalledWith(123, 'SIGTERM'); + } finally { + removeDaemonPaths(paths); + } +}); diff --git a/src/daemon/__tests__/provider-lease-expiry.test.ts b/src/daemon/__tests__/provider-lease-expiry.test.ts index 87b823594..e07332783 100644 --- a/src/daemon/__tests__/provider-lease-expiry.test.ts +++ b/src/daemon/__tests__/provider-lease-expiry.test.ts @@ -120,3 +120,47 @@ test('does not release until the expired lease record is durable', async () => { fs.rmSync(stateDir, { recursive: true, force: true }); } }); + +test('shutdown drain reports releases completed before and during its final retry', async () => { + const leaseRegistry = new LeaseRegistry(); + const releasedBeforeShutdown = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-0', + leaseProvider: 'browserstack', + }); + const releasedDuringShutdown = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseProvider: 'browserstack', + }); + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-2', + leaseProvider: 'browserstack', + }); + const release = vi + .fn() + .mockResolvedValueOnce({}) + .mockRejectedValueOnce(new Error('temporary outage')) + .mockResolvedValue({}); + const releaser = createExpiredProviderLeaseReleaser({ + leaseLifecycleProvider: { release }, + providerRuntimeIds: ['browserstack'], + }); + + try { + await releaser.release(releasedBeforeShutdown); + releaser.beginShutdown(); + await releaser.release(releasedDuringShutdown); + await releaser.release(lease); + const drained = await releaser.drain(10); + + expect(release).toHaveBeenCalledTimes(4); + expect(release).toHaveBeenCalledWith(lease); + expect(drained.released).toHaveLength(2); + expect(drained.released).toEqual(expect.arrayContaining([releasedDuringShutdown, lease])); + expect(drained.pending).toEqual([]); + } finally { + releaser.shutdown(); + } +}); diff --git a/src/daemon/daemon-process.ts b/src/daemon/daemon-process.ts index 5e130ce29..b81670875 100644 --- a/src/daemon/daemon-process.ts +++ b/src/daemon/daemon-process.ts @@ -28,7 +28,7 @@ export function isAgentDeviceDaemonProcess(pid: number, expectedStartTime?: stri return isAgentDeviceDaemonCommand(command); } -function trySignalProcess(pid: number, signal: NodeJS.Signals): boolean { +export function trySignalProcess(pid: number, signal: NodeJS.Signals): boolean { try { process.kill(pid, signal); return true; diff --git a/src/daemon/daemon-shutdown-report.ts b/src/daemon/daemon-shutdown-report.ts new file mode 100644 index 000000000..8982fb8aa --- /dev/null +++ b/src/daemon/daemon-shutdown-report.ts @@ -0,0 +1,89 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { DeviceLease } from './lease-registry.ts'; + +const SHUTDOWN_REPORT_FILE = 'daemon-shutdown.json'; + +export type ProviderReleaseRecord = { + leaseId: string; + provider?: string; +}; + +export type DaemonShutdownReport = { + providerReleases: { + released: ProviderReleaseRecord[]; + pending: ProviderReleaseRecord[]; + }; +}; + +export function writeDaemonShutdownReport( + stateDir: string, + providerReleases: { released: readonly DeviceLease[]; pending: readonly DeviceLease[] }, +): void { + const report: DaemonShutdownReport = { + providerReleases: { + released: providerReleases.released.map(toProviderReleaseRecord), + pending: providerReleases.pending.map(toProviderReleaseRecord), + }, + }; + const filePath = shutdownReportPath(stateDir); + const temporaryPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; + try { + fs.writeFileSync(temporaryPath, `${JSON.stringify(report)}\n`, { mode: 0o600 }); + fs.renameSync(temporaryPath, filePath); + fs.chmodSync(filePath, 0o600); + } catch { + try { + fs.rmSync(temporaryPath, { force: true }); + } catch {} + } +} + +export function readDaemonShutdownReport(stateDir: string): DaemonShutdownReport | null { + try { + const parsed = JSON.parse(fs.readFileSync(shutdownReportPath(stateDir), 'utf8')) as unknown; + return isDaemonShutdownReport(parsed) ? parsed : null; + } catch { + return null; + } +} + +export function clearDaemonShutdownReport(stateDir: string): void { + try { + fs.rmSync(shutdownReportPath(stateDir), { force: true }); + } catch {} +} + +function shutdownReportPath(stateDir: string): string { + return path.join(stateDir, SHUTDOWN_REPORT_FILE); +} + +function toProviderReleaseRecord(lease: DeviceLease): ProviderReleaseRecord { + return { + leaseId: lease.leaseId, + ...(lease.leaseProvider ? { provider: lease.leaseProvider } : {}), + }; +} + +function isDaemonShutdownReport(value: unknown): value is DaemonShutdownReport { + if (!value || typeof value !== 'object') return false; + const releases = (value as { providerReleases?: unknown }).providerReleases; + if (!releases || typeof releases !== 'object') return false; + const records = releases as { released?: unknown; pending?: unknown }; + return ( + Array.isArray(records.released) && + Array.isArray(records.pending) && + records.released.every(isProviderReleaseRecord) && + records.pending.every(isProviderReleaseRecord) + ); +} + +function isProviderReleaseRecord(value: unknown): value is ProviderReleaseRecord { + if (!value || typeof value !== 'object') return false; + const record = value as { leaseId?: unknown; provider?: unknown }; + return ( + typeof record.leaseId === 'string' && + record.leaseId.trim().length > 0 && + (record.provider === undefined || typeof record.provider === 'string') + ); +} diff --git a/src/daemon/daemon-stop.ts b/src/daemon/daemon-stop.ts new file mode 100644 index 000000000..815405916 --- /dev/null +++ b/src/daemon/daemon-stop.ts @@ -0,0 +1,154 @@ +import fs from 'node:fs'; +import { AppError } from '../kernel/errors.ts'; +import { isAgentDeviceDaemonProcess, trySignalProcess } from './daemon-process.ts'; +import { isProcessAlive, waitForProcessExit } from '../utils/host-process.ts'; +import { sleep } from '../utils/timeouts.ts'; +import type { DaemonPaths } from './config.ts'; +import type { ProviderReleaseRecord } from './daemon-shutdown-report.ts'; + +const DAEMON_STOP_GRACE_TIMEOUT_MS = 10_000; +const DAEMON_STOP_KILL_TIMEOUT_MS = 2_000; +const DAEMON_STOP_METADATA_WAIT_MS = 1_000; + +type DaemonInfo = { + pid?: unknown; + processStartTime?: unknown; +}; + +export type DaemonStopResult = { + stopped: boolean; + mode: 'graceful' | 'forced' | 'not-running'; + cleanupConfidence: 'known' | 'unknown'; + claimsReleased: []; + claimsOrphaned: []; + providerReleases: { + status: 'completed' | 'unknown'; + released: ProviderReleaseRecord[]; + pending: ProviderReleaseRecord[] | null; + }; + warnings: string[]; +}; + +export async function stopDaemon(params: { + paths: DaemonPaths; + graceTimeoutMs?: number; + killTimeoutMs?: number; +}): Promise { + const info = readDaemonInfo(params.paths.infoPath); + if (!info) return notRunningResult(); + if (!info.processStartTime) { + if (!isProcessAlive(info.pid)) return notRunningResult(); + throw new AppError( + 'COMMAND_FAILED', + 'Refusing to stop a daemon without a verified process start-time identity.', + { pid: info.pid }, + ); + } + if (!isAgentDeviceDaemonProcess(info.pid, info.processStartTime)) { + if (!isProcessAlive(info.pid)) return notRunningResult(); + throw new AppError( + 'COMMAND_FAILED', + 'Refusing to stop a daemon whose PID or start-time identity could not be verified.', + { pid: info.pid, processStartTime: info.processStartTime }, + ); + } + + if (!signalDaemonProcess(info.pid, 'SIGTERM')) return notRunningResult(); + const graceful = await waitForProcessExit( + info.pid, + params.graceTimeoutMs ?? DAEMON_STOP_GRACE_TIMEOUT_MS, + ); + if (graceful) { + await waitForDaemonMetadataRemoval(params.paths, DAEMON_STOP_METADATA_WAIT_MS); + return { + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + claimsReleased: [], + claimsOrphaned: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [], + }; + } + + // Re-verify immediately before escalation so a PID cannot be reused between + // the graceful wait and SIGKILL. + if (isAgentDeviceDaemonProcess(info.pid, info.processStartTime)) { + signalDaemonProcess(info.pid, 'SIGKILL'); + } + const stopped = await waitForProcessExit( + info.pid, + params.killTimeoutMs ?? DAEMON_STOP_KILL_TIMEOUT_MS, + ); + if (!stopped) { + throw new AppError('COMMAND_FAILED', 'Daemon did not exit after SIGKILL.', { pid: info.pid }); + } + return { + stopped: true, + mode: 'forced', + cleanupConfidence: 'unknown', + claimsReleased: [], + claimsOrphaned: [], + providerReleases: { status: 'unknown', released: [], pending: null }, + warnings: [ + 'The daemon was force-killed before provider lease state could be finalized. Provider allocations may remain active.', + ], + }; +} + +function signalDaemonProcess(pid: number, signal: NodeJS.Signals): boolean { + if (trySignalProcess(pid, signal)) return true; + if (!isProcessAlive(pid)) return false; + throw new AppError('COMMAND_FAILED', `Daemon could not be signaled with ${signal}.`, { + pid, + signal, + }); +} + +export function readDaemonStopIdentity( + infoPath: string, +): { pid: number; processStartTime: string } | null { + const info = readDaemonInfo(infoPath); + if (!info?.processStartTime) return null; + return { pid: info.pid, processStartTime: info.processStartTime }; +} + +function readDaemonInfo(infoPath: string): { pid: number; processStartTime?: string } | null { + try { + const parsed = JSON.parse(fs.readFileSync(infoPath, 'utf8')) as DaemonInfo; + const pid = parsed.pid; + if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return null; + return { + pid, + ...(hasProcessStartTime(parsed.processStartTime) + ? { processStartTime: parsed.processStartTime } + : {}), + }; + } catch { + return null; + } +} + +function hasProcessStartTime(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + +async function waitForDaemonMetadataRemoval(paths: DaemonPaths, timeoutMs: number): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (!fs.existsSync(paths.infoPath) && !fs.existsSync(paths.lockPath)) return; + await sleep(25); + } +} + +function notRunningResult(): DaemonStopResult { + return { + stopped: false, + mode: 'not-running', + cleanupConfidence: 'known', + claimsReleased: [], + claimsOrphaned: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [], + }; +} diff --git a/src/daemon/provider-lease-expiry.ts b/src/daemon/provider-lease-expiry.ts index e271ac950..bf9b06a2b 100644 --- a/src/daemon/provider-lease-expiry.ts +++ b/src/daemon/provider-lease-expiry.ts @@ -5,6 +5,7 @@ import type { LeaseLifecycleProvider } from './handlers/lease.ts'; import { releaseExpiredProviderLease } from './lease-lifecycle.ts'; import type { DeviceLease } from './lease-registry.ts'; import { emitDiagnostic } from '../utils/diagnostics.ts'; +import { sleep } from '../utils/timeouts.ts'; const DEFAULT_RETRY_DELAY_MS = 1_000; const PENDING_RELEASES_FILE = 'expired-provider-leases.json'; @@ -26,8 +27,10 @@ type PersistedExpiredProviderLeases = { }; export type ExpiredProviderLeaseReleaser = { + beginShutdown: () => void; release: (lease: DeviceLease) => Promise; retryPending: () => Promise; + drain: (timeoutMs: number) => Promise<{ pending: DeviceLease[]; released: DeviceLease[] }>; shutdown: () => void; }; @@ -44,7 +47,13 @@ export function createExpiredProviderLeaseReleaser(options: { const persistedRecoveryLeaseIds = new Set(pendingRecoveryLeases.keys()); const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS; let retryTimer: Timer | undefined; - let retrying = false; + const activeReleaseAttempts = new Map>(); + let trackingShutdownReleases = false; + const shutdownReleasedLeases = new Map(); + + const recordReleasedLease = (lease: DeviceLease): void => { + if (trackingShutdownReleases) shutdownReleasedLeases.set(lease.leaseId, lease); + }; const persistPendingLeases = (): boolean => { if (!options.stateDir) { @@ -104,50 +113,74 @@ export function createExpiredProviderLeaseReleaser(options: { }); }; - const retryPendingLiveLeases = async (): Promise => { - for (const lease of pendingLiveLeases.values()) { - if (!isProviderRuntimeAvailable(lease, options.providerRuntimeIds)) continue; + const trackReleaseAttempt = (key: string, attempt: Promise): Promise => { + activeReleaseAttempts.set(key, attempt); + const finish = (): void => { + if (activeReleaseAttempts.get(key) === attempt) activeReleaseAttempts.delete(key); + if (pendingLiveLeases.size === 0 && pendingRecoveryLeases.size === 0 && retryTimer) { + clearTimeout(retryTimer); + retryTimer = undefined; + } else { + scheduleRetry(); + } + }; + void attempt.then(finish, finish); + return attempt; + }; + + const ensureLiveReleaseAttempt = (lease: DeviceLease): Promise | undefined => { + if (!isProviderRuntimeAvailable(lease, options.providerRuntimeIds)) return undefined; + const key = `live:${lease.leaseId}`; + const activeAttempt = activeReleaseAttempts.get(key); + if (activeAttempt) return activeAttempt; + const attempt = (async (): Promise => { if (await releaseLiveProviderLease(options.leaseLifecycleProvider, lease)) { pendingLiveLeases.delete(lease.leaseId); + recordReleasedLease(lease); } - } + })(); + return trackReleaseAttempt(key, attempt); }; - const retryPendingRecoveryLeases = async (): Promise => { - for (const lease of pendingRecoveryLeases.values()) { - if (!isRecoverableProviderAvailable(lease, options.recoverableProviderIds)) continue; - if (!persistedRecoveryLeaseIds.has(lease.leaseId) && !persistPendingLeases()) continue; + const ensureRecoveryReleaseAttempt = (lease: DeviceLease): Promise | undefined => { + if (!isRecoverableProviderAvailable(lease, options.recoverableProviderIds)) return undefined; + if (!persistedRecoveryLeaseIds.has(lease.leaseId) && !persistPendingLeases()) return undefined; + const key = `recovery:${lease.leaseId}`; + const activeAttempt = activeReleaseAttempts.get(key); + if (activeAttempt) return activeAttempt; + const attempt = (async (): Promise => { if (await releaseExpiredProviderLease(options.recoverExpiredLease, lease)) { pendingRecoveryLeases.delete(lease.leaseId); + recordReleasedLease(lease); persistPendingLeases(); } - } + })(); + return trackReleaseAttempt(key, attempt); }; const retryPending = async (): Promise => { - if (retrying) return; - retrying = true; - try { - await retryPendingLiveLeases(); - await retryPendingRecoveryLeases(); - } finally { - retrying = false; - if (pendingLiveLeases.size === 0 && pendingRecoveryLeases.size === 0 && retryTimer) { - clearTimeout(retryTimer); - retryTimer = undefined; - } else { - scheduleRetry(); - } + const attempts = [ + ...[...pendingLiveLeases.values()].map(ensureLiveReleaseAttempt), + ...[...pendingRecoveryLeases.values()].map(ensureRecoveryReleaseAttempt), + ].filter((attempt): attempt is Promise => attempt !== undefined); + if (attempts.length === 0) { + scheduleRetry(); + return; } + await Promise.all(attempts); }; return { + beginShutdown: () => { + shutdownReleasedLeases.clear(); + trackingShutdownReleases = true; + }, release: async (lease) => { if (!lease.leaseProvider) return; if (isRecoverableProviderAvailable(lease, options.recoverableProviderIds)) { pendingRecoveryLeases.set(lease.leaseId, lease); if (persistPendingLeases()) { - await retryPending(); + await ensureRecoveryReleaseAttempt(lease); } else { scheduleRetry(); } @@ -156,12 +189,24 @@ export function createExpiredProviderLeaseReleaser(options: { if (!isProviderRuntimeAvailable(lease, options.providerRuntimeIds)) return; if (!options.leaseLifecycleProvider?.release) return; pendingLiveLeases.set(lease.leaseId, lease); - await retryPending(); + await ensureLiveReleaseAttempt(lease); if (pendingLiveLeases.has(lease.leaseId)) { scheduleRetry(); } }, retryPending, + drain: async (timeoutMs) => { + // Shutdown gets one final, bounded attempt at every pending release. A + // provider call that hangs must not keep the daemon alive indefinitely. + await Promise.race([retryPending(), sleep(Math.max(0, timeoutMs))]); + const result = { + pending: [...pendingLiveLeases.values(), ...pendingRecoveryLeases.values()], + released: [...shutdownReleasedLeases.values()], + }; + trackingShutdownReleases = false; + shutdownReleasedLeases.clear(); + return result; + }, shutdown: () => { if (retryTimer) clearTimeout(retryTimer); retryTimer = undefined; diff --git a/src/daemon/server/daemon-runtime.ts b/src/daemon/server/daemon-runtime.ts index 601747c1f..e421d6784 100644 --- a/src/daemon/server/daemon-runtime.ts +++ b/src/daemon/server/daemon-runtime.ts @@ -16,12 +16,14 @@ import { } from '../../provider-device-runtimes.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { createExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts'; +import { clearDaemonShutdownReport, writeDaemonShutdownReport } from '../daemon-shutdown-report.ts'; import { createRequestHandler } from '../request-router.ts'; import { teardownSessionResources } from '../session-teardown.ts'; import { IOS_SIMULATOR_RECORDING_STOP_ESCALATION_BUDGET_MS } from '../handlers/record-trace-ios-simulator.ts'; import { closeDaemonServers } from './server-shutdown.ts'; import type { DaemonInvokeFn, SessionState } from '../types.ts'; import { createDaemonIdleReap } from './daemon-idle-reap.ts'; +import { finalizeDaemonSessionLease } from './daemon-session-lease-finalizer.ts'; import { emitDiagnostic, flushDiagnosticsToSessionFile, @@ -56,7 +58,9 @@ import { } from '../../platforms/android/ime-lifecycle.ts'; const DAEMON_SESSION_TEARDOWN_TIMEOUT_MS = 5_000; +const DAEMON_SESSION_LEASE_RELEASE_TIMEOUT_MS = 1_000; const DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS = 1_000; +const DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS = 2_000; type WritableOutput = { write: (chunk: string) => unknown; @@ -90,8 +94,9 @@ export async function teardownDaemonSessionForShutdown(params: { sessionStore: SessionStore; stateDir?: string; stderr: WritableOutput; + beforeDelete?: (session: SessionState) => Promise; }): Promise { - const { session, sessionStore, stateDir, stderr } = params; + const { session, sessionStore, stateDir, stderr, beforeDelete } = params; const timeoutMs = resolveDaemonSessionTeardownTimeoutMs(session); const teardown = teardownSessionResources(session, session.name, stateDir).catch((error) => { stderr.write( @@ -110,6 +115,7 @@ export async function teardownDaemonSessionForShutdown(params: { // `.ad` iff the repair transaction completed, else leave a bounded // `REPAIR_SESSION_EXPIRED` tombstone for the reaped-before-finalize case. sessionStore.finalizeRepairTeardown(session); + await beforeDelete?.(session); sessionStore.delete(session.name); } @@ -207,7 +213,19 @@ export async function startDaemonRuntime( }; const teardownDaemonSession = async (session: SessionState): Promise => - await teardownDaemonSessionForShutdown({ session, sessionStore, stateDir: baseDir, stderr }); + await teardownDaemonSessionForShutdown({ + session, + sessionStore, + stateDir: baseDir, + stderr, + beforeDelete: async (sessionToFinalize) => + await finalizeDaemonSessionLease({ + session: sessionToFinalize, + leaseRegistry, + expiredProviderLeaseReleaser, + timeoutMs: DAEMON_SESSION_LEASE_RELEASE_TIMEOUT_MS, + }), + }); const teardownDaemonSessions = async (): Promise => { const sessionsToStop = sessionStore.toArray(); @@ -300,6 +318,7 @@ export async function startDaemonRuntime( exit(0); return null; } + clearDaemonShutdownReport(baseDir); let servers: DaemonServer[] = []; let socketPort: number | undefined; @@ -341,7 +360,6 @@ export async function startDaemonRuntime( idleReap.cancel(); if (shuttingDown) return; shuttingDown = true; - expiredProviderLeaseReleaser.shutdown(); if (shutdownOptions.cause) { await emitFatalDiagnostic(shutdownOptions.cause); } @@ -354,7 +372,21 @@ export async function startDaemonRuntime( try { await detachIosSimulatorRunnerSessionsForShutdown(); } catch {} + expiredProviderLeaseReleaser.beginShutdown(); await teardownDaemonSessions(); + const providerReleaseDrain = await expiredProviderLeaseReleaser.drain( + DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS, + ); + writeDaemonShutdownReport(baseDir, providerReleaseDrain); + emitDiagnostic({ + level: providerReleaseDrain.pending.length === 0 ? 'info' : 'warn', + phase: 'daemon_shutdown_provider_release_drain', + data: { + releasedLeaseIds: providerReleaseDrain.released.map((lease) => lease.leaseId), + pendingLeaseIds: providerReleaseDrain.pending.map((lease) => lease.leaseId), + }, + }); + expiredProviderLeaseReleaser.shutdown(); await Promise.allSettled( providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), ); diff --git a/src/daemon/server/daemon-session-lease-finalizer.test.ts b/src/daemon/server/daemon-session-lease-finalizer.test.ts new file mode 100644 index 000000000..319a840da --- /dev/null +++ b/src/daemon/server/daemon-session-lease-finalizer.test.ts @@ -0,0 +1,179 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; +import { LeaseRegistry, type DeviceLease } from '../lease-registry.ts'; +import { createExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts'; +import { finalizeDaemonSessionLease } from './daemon-session-lease-finalizer.ts'; + +test('journals and bounds a hung recoverable session lease release before the final drain', async () => { + vi.useFakeTimers(); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-lease-finalizer-')); + const leaseRegistry = new LeaseRegistry(); + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseProvider: 'limrun', + }); + const recoverExpiredLease = vi.fn(() => new Promise(() => {})); + const expiredProviderLeaseReleaser = createExpiredProviderLeaseReleaser({ + recoverExpiredLease, + recoverableProviderIds: ['limrun'], + stateDir, + }); + const session = makeIosSession('default', { + lease: { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + expiresAt: lease.expiresAt, + }, + }); + + try { + expiredProviderLeaseReleaser.beginShutdown(); + const finalization = finalizeDaemonSessionLease({ + session, + leaseRegistry, + expiredProviderLeaseReleaser, + timeoutMs: 10, + }); + + await vi.advanceTimersByTimeAsync(10); + await finalization; + + expect(recoverExpiredLease).toHaveBeenCalledWith(lease); + expect(leaseRegistry.listActiveLeases()).toEqual([]); + expect(fs.existsSync(path.join(stateDir, 'expired-provider-leases.json'))).toBe(true); + const drain = expiredProviderLeaseReleaser.drain(10); + await vi.advanceTimersByTimeAsync(10); + await expect(drain).resolves.toEqual({ + pending: [lease], + released: [], + }); + } finally { + expiredProviderLeaseReleaser.shutdown(); + vi.useRealTimers(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('final drain joins a release that completes after the session timeout', async () => { + vi.useFakeTimers(); + const leaseRegistry = new LeaseRegistry(); + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseProvider: 'browserstack', + }); + const release = vi.fn( + () => + new Promise>((resolve) => { + setTimeout(() => resolve({}), 1_500); + }), + ); + const expiredProviderLeaseReleaser = createExpiredProviderLeaseReleaser({ + leaseLifecycleProvider: { release }, + providerRuntimeIds: ['browserstack'], + }); + const session = makeIosSession('default', { + lease: { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + expiresAt: lease.expiresAt, + }, + }); + + try { + expiredProviderLeaseReleaser.beginShutdown(); + const finalization = finalizeDaemonSessionLease({ + session, + leaseRegistry, + expiredProviderLeaseReleaser, + timeoutMs: 1_000, + }); + + await vi.advanceTimersByTimeAsync(1_000); + await finalization; + const drain = expiredProviderLeaseReleaser.drain(2_000); + + await vi.advanceTimersByTimeAsync(500); + await expect(drain).resolves.toEqual({ pending: [], released: [lease] }); + } finally { + expiredProviderLeaseReleaser.shutdown(); + vi.useRealTimers(); + } +}); + +test('a hung provider release does not starve a later session during shutdown', async () => { + vi.useFakeTimers(); + const leaseRegistry = new LeaseRegistry(); + const hungLease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseProvider: 'browserstack', + }); + const releasedLease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-2', + leaseProvider: 'browserstack', + }); + const release = vi.fn((lease: DeviceLease) => + lease.leaseId === hungLease.leaseId + ? new Promise>(() => {}) + : Promise.resolve({}), + ); + const expiredProviderLeaseReleaser = createExpiredProviderLeaseReleaser({ + leaseLifecycleProvider: { release }, + providerRuntimeIds: ['browserstack'], + }); + const sessions = [ + makeIosSession('hung', { lease: sessionLease(hungLease) }), + makeIosSession('released', { lease: sessionLease(releasedLease) }), + ]; + + try { + expiredProviderLeaseReleaser.beginShutdown(); + const finalization = Promise.all( + sessions.map( + async (session) => + await finalizeDaemonSessionLease({ + session, + leaseRegistry, + expiredProviderLeaseReleaser, + timeoutMs: 1_000, + }), + ), + ); + + await vi.advanceTimersByTimeAsync(1_000); + await finalization; + + expect(release).toHaveBeenCalledWith(hungLease); + expect(release).toHaveBeenCalledWith(releasedLease); + expect(leaseRegistry.listActiveLeases()).toEqual([]); + const drain = expiredProviderLeaseReleaser.drain(2_000); + await vi.advanceTimersByTimeAsync(2_000); + await expect(drain).resolves.toEqual({ pending: [hungLease], released: [releasedLease] }); + } finally { + expiredProviderLeaseReleaser.shutdown(); + vi.useRealTimers(); + } +}); + +function sessionLease(lease: DeviceLease) { + return { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + expiresAt: lease.expiresAt, + }; +} diff --git a/src/daemon/server/daemon-session-lease-finalizer.ts b/src/daemon/server/daemon-session-lease-finalizer.ts new file mode 100644 index 000000000..4e48b1029 --- /dev/null +++ b/src/daemon/server/daemon-session-lease-finalizer.ts @@ -0,0 +1,70 @@ +import { leaseScopeToReleaseRequest } from '../../core/lease-scope.ts'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import type { ExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts'; +import type { LeaseRegistry } from '../lease-registry.ts'; +import type { SessionState } from '../types.ts'; + +export async function finalizeDaemonSessionLease(params: { + session: SessionState; + leaseRegistry: LeaseRegistry; + expiredProviderLeaseReleaser: ExpiredProviderLeaseReleaser; + timeoutMs: number; +}): Promise { + const { session, leaseRegistry, expiredProviderLeaseReleaser, timeoutMs } = params; + if (!session.lease) return; + try { + const releaseRequest = leaseScopeToReleaseRequest({ + leaseId: session.lease.leaseId, + tenantId: session.lease.tenantId, + runId: session.lease.runId, + leaseBackend: session.lease.leaseBackend, + leaseProvider: session.lease.leaseProvider, + deviceKey: session.lease.deviceKey, + clientId: session.lease.clientId, + }); + const activeLease = leaseRegistry.getLease(releaseRequest); + if (!activeLease) return; + const completed = await releaseWithinTimeout( + expiredProviderLeaseReleaser.release(activeLease), + timeoutMs, + ); + leaseRegistry.releaseLease(releaseRequest); + if (!completed) { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_shutdown_session_lease_release_timed_out', + data: { + session: session.name, + leaseId: session.lease.leaseId, + timeoutMs, + }, + }); + } + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_shutdown_session_lease_release_failed', + data: { + session: session.name, + leaseId: session.lease.leaseId, + error: error instanceof Error ? error.message : String(error), + }, + }); + } +} + +async function releaseWithinTimeout(release: Promise, timeoutMs: number): Promise { + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve(false), timeoutMs); + void release.then( + () => { + clearTimeout(timeout); + resolve(true); + }, + (error: unknown) => { + clearTimeout(timeout); + reject(error); + }, + ); + }); +} diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d0b9b62dd..e05fe5223 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -58,6 +58,7 @@ agent-device app-switcher - `boot` requires either an active session or an explicit device selector. - `shutdown` turns off the selected Apple simulator or Android emulator. - `shutdown` must not target an active session device; use `close --shutdown` to end the session and turn it off. +- `daemon stop --state-dir ` verifies the daemon PID/start-time identity, requests graceful shutdown, and reports whether provider-release state is known. Use `daemon stop --clean` to also remove retained Apple runner processes and leases owned by that daemon. - `--platform apple` is an alias for the Apple automation backend (`ios`, `tvOS`, `macOS` selection). - Use `--target mobile|tv|desktop` with `--platform` (required) to select phone/tablet vs TV-class vs desktop-class targets. - `boot` is mainly needed when starting a new session and `open` fails because no booted simulator/emulator is available.