diff --git a/.changeset/config.json b/.changeset/config.json index 115f54fefee..caecef79cd2 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -14,9 +14,6 @@ "updateInternalDependencies": "patch", "ignore": [ "webapp", - "coordinator", - "docker-provider", - "kubernetes-provider", "supervisor" ], "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": { diff --git a/.changeset/wild-v3-provider-helpers.md b/.changeset/wild-v3-provider-helpers.md new file mode 100644 index 00000000000..c5b3fcf9a76 --- /dev/null +++ b/.changeset/wild-v3-provider-helpers.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Removed internal helpers that were only used by the end-of-life v3 self-hosted compute providers. diff --git a/.claude/rules/server-apps.md b/.claude/rules/server-apps.md index 4d46789701c..9229a0b2f32 100644 --- a/.claude/rules/server-apps.md +++ b/.claude/rules/server-apps.md @@ -5,7 +5,7 @@ paths: # Server App Changes -When modifying server apps (webapp, supervisor, coordinator, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset: +When modifying server apps (webapp, supervisor, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset: ```bash cat > .server-changes/descriptive-name.md << 'EOF' @@ -18,6 +18,6 @@ Brief description of what changed and why. EOF ``` -- **area**: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider` +- **area**: `webapp` | `supervisor` - **type**: `feature` | `fix` | `improvement` | `breaking` - If the PR also touches `packages/`, just the changeset is sufficient (no `.server-changes/` needed). diff --git a/.cursorignore b/.cursorignore index 8430ce365fb..b1033e7a688 100644 --- a/.cursorignore +++ b/.cursorignore @@ -1,7 +1,4 @@ -apps/docker-provider/ -apps/kubernetes-provider/ apps/proxy/ -apps/coordinator/ packages/rsc/ .changeset .zed diff --git a/.github/workflows/publish-worker.yml b/.github/workflows/publish-worker.yml deleted file mode 100644 index e43334b58f6..00000000000 --- a/.github/workflows/publish-worker.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: "⚒️ Publish Worker" - -on: - workflow_call: - inputs: - image_tag: - description: The image tag to publish - type: string - required: false - default: "" - image_registry: - description: The registry namespace to publish under (e.g. ghcr.io/) - type: string - required: false - default: "" - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - push: - tags: - - "infra-dev-*" - - "infra-test-*" - - "infra-prod-*" - -permissions: - packages: write - contents: read - -jobs: - build: - strategy: - matrix: - package: [coordinator, docker-provider, kubernetes-provider] - runs-on: warp-ubuntu-latest-x64-8x - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - steps: - - name: ⬇️ Checkout git repo - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: 📦 Get image repo - id: get_repository - env: - PACKAGE: ${{ matrix.package }} - run: | - if [[ "$PACKAGE" == *-provider ]]; then - repo="provider/${PACKAGE%-provider}" - else - repo="$PACKAGE" - fi - echo "repo=${repo}" >> "$GITHUB_OUTPUT" - - - id: get_tag - uses: ./.github/actions/get-image-tag - with: - tag: ${{ inputs.image_tag }} - - - name: 🐋 Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - - # ..to avoid rate limits when pulling images - - name: 🐳 Login to DockerHub - if: ${{ env.DOCKERHUB_USERNAME }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: 🚢 Build Container Image - run: | - docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile . - - # ..to push image - - name: 🐙 Login to GitHub Container Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: 🐙 Push to GitHub Container Registry - run: | - docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - env: - # Resolved by the caller when invoked from publish.yml; falls back to the - # IMAGE_REGISTRY repository variable (or ghcr.io/) for the direct - # push triggers above, so a fork publishes to its own namespace. - REGISTRY: ${{ inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} - REPOSITORY: ${{ steps.get_repository.outputs.repo }} - IMAGE_TAG: ${{ steps.get_tag.outputs.tag }} - - # - name: 🐙 Push 'v3' tag to GitHub Container Registry - # if: steps.get_tag.outputs.is_semver == 'true' - # run: | - # docker tag infra_image "$REGISTRY/$REPOSITORY:v3" - # docker push "$REGISTRY/$REPOSITORY:v3" - # env: - # REGISTRY: ghcr.io/triggerdotdev - # REPOSITORY: ${{ steps.get_repository.outputs.repo }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ca5e98399a9..a0f32b02632 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -30,7 +30,6 @@ on: - ".github/workflows/unit-tests.yml" - ".github/workflows/e2e.yml" - ".github/workflows/publish-webapp.yml" - - ".github/workflows/publish-worker.yml" - "packages/**" - "!packages/**/*.md" - "!packages/**/*.eslintrc" @@ -80,19 +79,6 @@ jobs: # to its own namespace; set the IMAGE_REGISTRY repository variable to override. image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} - publish-worker: - needs: [typecheck] - permissions: - contents: read - packages: write - uses: ./.github/workflows/publish-worker.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - image_tag: ${{ inputs.image_tag }} - image_registry: ${{ vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} - publish-worker-v4: needs: [typecheck] permissions: diff --git a/.server-changes/README.md b/.server-changes/README.md index 1ecfe3ee23b..12815039aca 100644 --- a/.server-changes/README.md +++ b/.server-changes/README.md @@ -1,10 +1,10 @@ # Server Changes -This directory tracks changes to server-only components (webapp, supervisor, coordinator, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented. +This directory tracks changes to server-only components (webapp, supervisor, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented. ## When to add a file -**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file. +**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file. **Mixed PRs** (both packages and server): Just add a changeset as usual. No `.server-changes/` file needed — the changeset covers it. @@ -31,7 +31,7 @@ Speed up batch queue processing by removing stalls and fixing retry race ### Fields -- **area** (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider` +- **area** (required): `webapp` | `supervisor` - **type** (required): `feature` | `fix` | `improvement` | `breaking` ### Description diff --git a/CHANGESETS.md b/CHANGESETS.md index 2e225b9ad34..db9d5719d98 100644 --- a/CHANGESETS.md +++ b/CHANGESETS.md @@ -21,7 +21,7 @@ Speed up batch queue processing by removing stalls and fixing retry race EOF ``` -- `area`: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider` +- `area`: `webapp` | `supervisor` - `type`: `feature` | `fix` | `improvement` | `breaking` For **mixed PRs** (both packages and server): just add a changeset. No `.server-changes/` file needed. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f8bf5459c2..16d143395ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -265,7 +265,7 @@ Most of the time the changes you'll make are likely to be categorized as patch r ## Adding server changes -Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes. +Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes. Create a markdown file with a descriptive name: @@ -281,7 +281,7 @@ EOF ``` **Fields:** -- `area` (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider` +- `area` (required): `webapp` | `supervisor` - `type` (required): `feature` | `fix` | `improvement` | `breaking` The body text (below the frontmatter) is a one-line description of the change. Keep it concise — it will appear in release notes. diff --git a/apps/coordinator/.env.example b/apps/coordinator/.env.example deleted file mode 100644 index 77377ab3cfd..00000000000 --- a/apps/coordinator/.env.example +++ /dev/null @@ -1,4 +0,0 @@ -HTTP_SERVER_PORT=8020 -PLATFORM_ENABLED=true -PLATFORM_WS_PORT=3030 -SECURE_CONNECTION=false \ No newline at end of file diff --git a/apps/coordinator/.gitignore b/apps/coordinator/.gitignore deleted file mode 100644 index 5c84119d635..00000000000 --- a/apps/coordinator/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/coordinator/Containerfile b/apps/coordinator/Containerfile deleted file mode 100644 index 4941ff665bf..00000000000 --- a/apps/coordinator/Containerfile +++ /dev/null @@ -1,60 +0,0 @@ -# syntax=docker/dockerfile:labs - -FROM node:22.23.1-bookworm-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d058f9dff4 AS node-22 - -WORKDIR /app - -FROM node-22 AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-22 AS base - -RUN apt-get update \ - && apt-get install -y buildah ca-certificates dumb-init docker.io busybox \ - && rm -rf /var/lib/apt/lists/* - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle - -FROM alpine AS cri-tools - -WORKDIR /cri-tools - -ARG CRICTL_VERSION=v1.29.0 -ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb -ADD --checksum=${CRICTL_CHECKSUM} \ - https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz . -RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin -COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/coordinator/README.md b/apps/coordinator/README.md deleted file mode 100644 index fa6da2462bd..00000000000 --- a/apps/coordinator/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Coordinator - -Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things. diff --git a/apps/coordinator/package.json b/apps/coordinator/package.json deleted file mode 100644 index 446606a44c8..00000000000 --- a/apps/coordinator/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "coordinator", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t coordinator", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@trigger.dev/core": "workspace:*", - "nanoid": "^5.0.6", - "prom-client": "^15.1.0", - "socket.io": "4.7.4", - "tinyexec": "^0.3.0" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} diff --git a/apps/coordinator/src/chaosMonkey.ts b/apps/coordinator/src/chaosMonkey.ts deleted file mode 100644 index e2bc147674f..00000000000 --- a/apps/coordinator/src/chaosMonkey.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { setTimeout as timeout } from "node:timers/promises"; - -class ChaosMonkeyError extends Error { - constructor(message: string) { - super(message); - this.name = "ChaosMonkeyError"; - } -} - -export class ChaosMonkey { - private chaosEventRate = 0.2; - private delayInSeconds = 45; - - constructor( - private enabled = false, - private disableErrors = false, - private disableDelays = false - ) { - if (this.enabled) { - console.log("🍌 Chaos monkey enabled"); - } - } - - static Error = ChaosMonkeyError; - - enable() { - this.enabled = true; - console.log("🍌 Chaos monkey enabled"); - } - - disable() { - this.enabled = false; - console.log("🍌 Chaos monkey disabled"); - } - - async call({ - throwErrors = !this.disableErrors, - addDelays = !this.disableDelays, - }: { - throwErrors?: boolean; - addDelays?: boolean; - } = {}) { - if (!this.enabled) { - return; - } - - const random = Math.random(); - - if (random > this.chaosEventRate) { - // Don't interfere with normal operation - return; - } - - const chaosEvents: Array<() => Promise> = []; - - if (addDelays) { - chaosEvents.push(async () => { - console.log("🍌 Chaos monkey: Add delay"); - - await timeout(this.delayInSeconds * 1000); - }); - } - - if (throwErrors) { - chaosEvents.push(async () => { - console.log("🍌 Chaos monkey: Throw error"); - - throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error"); - }); - } - - if (chaosEvents.length === 0) { - console.error("🍌 Chaos monkey: No events selected"); - return; - } - - const randomIndex = Math.floor(Math.random() * chaosEvents.length); - - const chaosEvent = chaosEvents[randomIndex]; - - if (!chaosEvent) { - console.error("🍌 Chaos monkey: No event found"); - return; - } - - await chaosEvent(); - } -} diff --git a/apps/coordinator/src/checkpointer.ts b/apps/coordinator/src/checkpointer.ts deleted file mode 100644 index b864d78095d..00000000000 --- a/apps/coordinator/src/checkpointer.ts +++ /dev/null @@ -1,709 +0,0 @@ -import { ExponentialBackoff } from "@trigger.dev/core/v3/apps"; -import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly"; -import { nanoid } from "nanoid"; -import fs from "node:fs/promises"; -import { ChaosMonkey } from "./chaosMonkey"; -import { Buildah, Crictl, Exec } from "./exec"; -import { setTimeout } from "node:timers/promises"; -import { TempFileCleaner } from "./cleaner"; -import { numFromEnv, boolFromEnv } from "./util"; -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; - -type CheckpointerInitializeReturn = { - canCheckpoint: boolean; - willSimulate: boolean; -}; - -type CheckpointAndPushOptions = { - runId: string; - leaveRunning?: boolean; - projectRef: string; - deploymentVersion: string; - shouldHeartbeat?: boolean; - attemptNumber?: number; -}; - -type CheckpointAndPushResult = - | { success: true; checkpoint: CheckpointData } - | { - success: false; - reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING"; - }; - -type CheckpointData = { - location: string; - docker: boolean; -}; - -type CheckpointerOptions = { - dockerMode: boolean; - forceSimulate: boolean; - heartbeat: (runId: string) => void; - registryHost?: string; - registryNamespace?: string; - registryTlsVerify?: boolean; - disableCheckpointSupport?: boolean; - checkpointPath?: string; - simulateCheckpointFailure?: boolean; - simulateCheckpointFailureSeconds?: number; - simulatePushFailure?: boolean; - simulatePushFailureSeconds?: number; - chaosMonkey?: ChaosMonkey; -}; - -async function getFileSize(filePath: string): Promise { - try { - const stats = await fs.stat(filePath); - return stats.size; - } catch (error) { - console.error("Error getting file size:", error); - return -1; - } -} - -async function getParsedFileSize(filePath: string) { - const sizeInBytes = await getFileSize(filePath); - - let message = `Size in bytes: ${sizeInBytes}`; - - if (sizeInBytes > 1024 * 1024) { - const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2); - message = `Size in MB (rounded): ${sizeInMB}`; - } else if (sizeInBytes > 1024) { - const sizeInKB = (sizeInBytes / 1024).toFixed(2); - message = `Size in KB (rounded): ${sizeInKB}`; - } - - return { - path: filePath, - sizeInBytes, - message, - }; -} - -export class Checkpointer { - #initialized = false; - #canCheckpoint = false; - #dockerMode: boolean; - - #logger = new SimpleStructuredLogger("checkpointer"); - - #failedCheckpoints = new Map(); - - // Indexed by run ID - #runAbortControllers = new Map< - string, - { signal: AbortSignal; abort: AbortController["abort"] } - >(); - - private registryHost: string; - private registryNamespace: string; - private registryTlsVerify: boolean; - - private disableCheckpointSupport: boolean; - - private simulateCheckpointFailure: boolean; - private simulateCheckpointFailureSeconds: number; - private simulatePushFailure: boolean; - private simulatePushFailureSeconds: number; - - private chaosMonkey: ChaosMonkey; - private tmpCleaner?: TempFileCleaner; - - constructor(private opts: CheckpointerOptions) { - this.#dockerMode = opts.dockerMode; - - this.registryHost = opts.registryHost ?? "localhost:5000"; - this.registryNamespace = opts.registryNamespace ?? "trigger"; - this.registryTlsVerify = opts.registryTlsVerify ?? true; - - this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false; - - this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false; - this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300; - this.simulatePushFailure = opts.simulatePushFailure ?? false; - this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300; - - this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED); - this.tmpCleaner = this.#createTmpCleaner(); - } - - async init(): Promise { - if (this.#initialized) { - return this.#getInitReturn(this.#canCheckpoint); - } - - this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`); - - if (this.#dockerMode) { - const testCheckpoint = await testDockerCheckpoint(); - - if (testCheckpoint.ok) { - return this.#getInitReturn(true); - } - - this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error }); - return this.#getInitReturn(false); - } - - const canLogin = await Buildah.canLogin(this.registryHost); - - if (!canLogin) { - this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`); - } - - return this.#getInitReturn(canLogin); - } - - #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn { - this.#canCheckpoint = canCheckpoint; - - if (canCheckpoint) { - if (!this.#initialized) { - this.#logger.log("Full checkpoint support!"); - } - } - - this.#initialized = true; - - const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate); - - if (willSimulate) { - this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { - forceSimulate: this.opts.forceSimulate, - }); - } - - return { - canCheckpoint, - willSimulate, - }; - } - - #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) { - return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`; - } - - #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) { - const basename = `${projectRef}-${deploymentVersion}-${shortCode}`; - - if (this.#dockerMode) { - return basename; - } else { - return Crictl.getExportLocation(basename); - } - } - - async checkpointAndPush( - opts: CheckpointAndPushOptions, - delayMs?: number - ): Promise { - const start = performance.now(); - this.#logger.log(`checkpointAndPush() start`, { start, opts }); - - const { runId } = opts; - - let interval: NodeJS.Timer | undefined; - if (opts.shouldHeartbeat) { - interval = setInterval(() => { - this.#logger.log("Sending heartbeat", { runId }); - this.opts.heartbeat(runId); - }, 20_000); - } - - const controller = new AbortController(); - const signal = controller.signal; - const abort = controller.abort.bind(controller); - - const onAbort = () => { - this.#logger.error("Checkpoint aborted", { runId, options: opts }); - }; - - signal.addEventListener("abort", onAbort, { once: true }); - - const removeCurrentAbortController = () => { - const controller = this.#runAbortControllers.get(runId); - - // Ensure only the current controller is removed - if (controller && controller.signal === signal) { - this.#runAbortControllers.delete(runId); - } - - // Remove the abort listener in case it hasn't fired - signal.removeEventListener("abort", onAbort); - }; - - if (!this.#dockerMode && !this.#canCheckpoint) { - this.#logger.error("No checkpoint support. Simulation requires docker."); - this.#failCheckpoint(runId, "NO_SUPPORT"); - return; - } - - if (this.#isRunCheckpointing(runId)) { - this.#logger.error("Checkpoint procedure already in progress", { options: opts }); - this.#failCheckpoint(runId, "IN_PROGRESS"); - return; - } - - // This is a new checkpoint, clear any last failure for this run - this.#clearFailedCheckpoint(runId); - - if (this.disableCheckpointSupport) { - this.#logger.error("Checkpoint support disabled", { options: opts }); - this.#failCheckpoint(runId, "DISABLED"); - return; - } - - this.#runAbortControllers.set(runId, { signal, abort }); - - try { - const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal }); - - const end = performance.now(); - this.#logger.log(`checkpointAndPush() end`, { - start, - end, - diff: end - start, - diffWithoutDelay: end - start - (delayMs ?? 0), - opts, - success: result.success, - delayMs, - }); - - if (!result.success) { - return; - } - - return result.checkpoint; - } finally { - if (opts.shouldHeartbeat) { - // @ts-ignore - Some kind of node incompatible type issue - clearInterval(interval); - } - removeCurrentAbortController(); - } - } - - #isRunCheckpointing(runId: string) { - return this.#runAbortControllers.has(runId); - } - - cancelAllCheckpointsForRun(runId: string): boolean { - this.#logger.log("cancelAllCheckpointsForRun: call", { runId }); - - // If the last checkpoint failed, pretend we canceled it - // This ensures tasks don't wait for external resume messages to continue - if (this.#hasFailedCheckpoint(runId)) { - this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId }); - this.#clearFailedCheckpoint(runId); - return true; - } - - const controller = this.#runAbortControllers.get(runId); - - if (!controller) { - this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId }); - return false; - } - - const { abort, signal } = controller; - - if (signal.aborted) { - this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId }); - return false; - } - - abort("cancelCheckpoint()"); - this.#runAbortControllers.delete(runId); - - return true; - } - - async #checkpointAndPushWithBackoff( - { - runId, - leaveRunning = true, // This mirrors kubernetes behaviour more accurately - projectRef, - deploymentVersion, - attemptNumber, - }: CheckpointAndPushOptions, - { delayMs, signal }: { delayMs?: number; signal: AbortSignal } - ): Promise { - if (delayMs && delayMs > 0) { - this.#logger.log("Delaying checkpoint", { runId, delayMs }); - - try { - await setTimeout(delayMs, undefined, { signal }); - } catch (_error) { - this.#logger.log("Checkpoint canceled during initial delay", { runId }); - return { success: false, reason: "CANCELED" }; - } - } - - this.#logger.log("Checkpointing with backoff", { - runId, - leaveRunning, - projectRef, - deploymentVersion, - }); - - const backoff = new ExponentialBackoff() - .type("EqualJitter") - .base(3) - .max(3 * 3600) - .maxElapsed(48 * 3600); - - for await (const { delay, retry } of backoff) { - try { - if (retry > 0) { - this.#logger.error("Retrying checkpoint", { - runId, - retry, - delay, - }); - - try { - await setTimeout(delay.milliseconds, undefined, { signal }); - } catch (_error) { - this.#logger.log("Checkpoint canceled during retry delay", { runId }); - return { success: false, reason: "CANCELED" }; - } - } - - const result = await this.#checkpointAndPush( - { - runId, - leaveRunning, - projectRef, - deploymentVersion, - attemptNumber, - }, - { signal } - ); - - if (result.success) { - return result; - } - - if (result.reason === "CANCELED") { - this.#logger.log("Checkpoint canceled, won't retry", { runId }); - // Don't fail the checkpoint, as it was canceled - return result; - } - - if (result.reason === "SKIP_RETRYING") { - this.#logger.log("Skipping retrying", { runId }); - return result; - } - - continue; - } catch (error) { - this.#logger.error("Checkpoint error", { - retry, - runId, - delay, - error: error instanceof Error ? error.message : error, - }); - } - } - - this.#logger.error(`Checkpoint failed after exponential backoff`, { - runId, - leaveRunning, - projectRef, - deploymentVersion, - }); - this.#failCheckpoint(runId, "ERROR"); - - return { success: false, reason: "ERROR" }; - } - - async #checkpointAndPush( - { - runId, - leaveRunning = true, // This mirrors kubernetes behaviour more accurately - projectRef, - deploymentVersion, - attemptNumber, - }: CheckpointAndPushOptions, - { signal }: { signal: AbortSignal } - ): Promise { - await this.init(); - - const options = { - runId, - leaveRunning, - projectRef, - deploymentVersion, - attemptNumber, - }; - - const shortCode = nanoid(8); - const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode); - const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode); - - const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal }); - const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal }); - - const cleanup = async () => { - const metadata = { - runId, - exportLocation, - imageRef, - }; - - if (this.#dockerMode) { - this.#logger.debug("Skipping cleanup in docker mode", metadata); - return; - } - - this.#logger.log("Cleaning up", metadata); - - try { - await buildah.cleanup(); - await crictl.cleanup(); - } catch (error) { - this.#logger.error("Error during cleanup", { ...metadata, error }); - } - }; - - try { - await this.chaosMonkey.call(); - - this.#logger.log("checkpointAndPush: checkpointing", { options }); - - const containterName = this.#getRunContainerName(runId); - - // Create checkpoint (docker) - if (this.#dockerMode) { - await this.#createDockerCheckpoint( - signal, - runId, - exportLocation, - leaveRunning, - attemptNumber - ); - - this.#logger.log("checkpointAndPush: checkpoint created", { - runId, - location: exportLocation, - }); - - return { - success: true, - checkpoint: { - location: exportLocation, - docker: true, - }, - }; - } - - // Create checkpoint (CRI) - if (!this.#canCheckpoint) { - this.#logger.error("No checkpoint support in kubernetes mode."); - return { success: false, reason: "SKIP_RETRYING" }; - } - - const containerId = await crictl.ps(containterName, true); - - if (!containerId.stdout) { - this.#logger.error("could not find container id", { options, containterName }); - return { success: false, reason: "SKIP_RETRYING" }; - } - - const start = performance.now(); - - if (this.simulateCheckpointFailure) { - if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) { - this.#logger.error("Simulating checkpoint failure", { options }); - throw new Error("SIMULATE_CHECKPOINT_FAILURE"); - } - } - - // Create checkpoint - await crictl.checkpoint(containerId.stdout, exportLocation); - const postCheckpoint = performance.now(); - - // Print checkpoint size - const size = await getParsedFileSize(exportLocation); - this.#logger.log("checkpoint archive created", { size, options }); - - // Create image from checkpoint - const workingContainer = await buildah.from("scratch"); - const postFrom = performance.now(); - - await buildah.add(workingContainer.stdout, exportLocation, "/"); - const postAdd = performance.now(); - - await buildah.config(workingContainer.stdout, [ - `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`, - ]); - const postConfig = performance.now(); - - await buildah.commit(workingContainer.stdout, imageRef); - const postCommit = performance.now(); - - if (this.simulatePushFailure) { - if (performance.now() < this.simulatePushFailureSeconds * 1000) { - this.#logger.error("Simulating push failure", { options }); - throw new Error("SIMULATE_PUSH_FAILURE"); - } - } - - // Push checkpoint image - await buildah.push(imageRef, this.registryTlsVerify); - const postPush = performance.now(); - - const perf = { - "crictl checkpoint": postCheckpoint - start, - "buildah from": postFrom - postCheckpoint, - "buildah add": postAdd - postFrom, - "buildah config": postConfig - postAdd, - "buildah commit": postCommit - postConfig, - "buildah push": postPush - postCommit, - }; - - this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf }); - - return { - success: true, - checkpoint: { - location: imageRef, - docker: false, - }, - }; - } catch (error) { - if (error instanceof Exec.Result) { - if (error.aborted) { - this.#logger.error("Checkpoint canceled: Exec", { options }); - - return { success: false, reason: "CANCELED" }; - } else { - this.#logger.error("Checkpoint command error", { options, error }); - - return { success: false, reason: "ERROR" }; - } - } - - this.#logger.error("Unhandled checkpoint error", { - options, - error: error instanceof Error ? error.message : error, - }); - - return { success: false, reason: "ERROR" }; - } finally { - await cleanup(); - - if (signal.aborted) { - this.#logger.error("Checkpoint canceled: Cleanup", { options }); - - // Overrides any prior return value (intentional use of return-in-finally) - // eslint-disable-next-line no-unsafe-finally - return { success: false, reason: "CANCELED" }; - } - } - } - - async unpause(runId: string, attemptNumber?: number): Promise { - try { - const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber); - const exec = new Exec({ logger: this.#logger }); - await exec.x("docker", ["unpause", containterNameWithAttempt]); - } catch (error) { - this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error }); - } - } - - async #createDockerCheckpoint( - abortSignal: AbortSignal, - runId: string, - exportLocation: string, - leaveRunning: boolean, - attemptNumber?: number - ) { - const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber); - const exec = new Exec({ logger: this.#logger, abortSignal }); - - try { - if (this.opts.forceSimulate || !this.#canCheckpoint) { - this.#logger.log("Simulating checkpoint"); - - await exec.x("docker", ["pause", containterNameWithAttempt]); - - return; - } - - if (this.simulateCheckpointFailure) { - if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) { - this.#logger.error("Simulating checkpoint failure", { - runId, - exportLocation, - leaveRunning, - attemptNumber, - }); - - throw new Error("SIMULATE_CHECKPOINT_FAILURE"); - } - } - - const args = ["checkpoint", "create"]; - - if (leaveRunning) { - args.push("--leave-running"); - } - - args.push(containterNameWithAttempt, exportLocation); - - await exec.x("docker", args); - } catch (error) { - this.#logger.error("Failed while creating docker checkpoint", { exportLocation }); - throw error; - } - } - - #failCheckpoint(runId: string, error: unknown) { - this.#failedCheckpoints.set(runId, error); - } - - #clearFailedCheckpoint(runId: string) { - this.#failedCheckpoints.delete(runId); - } - - #hasFailedCheckpoint(runId: string) { - return this.#failedCheckpoints.has(runId); - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - #createTmpCleaner() { - if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) { - return; - } - - const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean); - const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? []; - const paths = pathsOverride.length ? pathsOverride : defaultPaths; - - if (paths.length === 0) { - this.#logger.error("TempFileCleaner enabled but no paths to clean", { - defaultPaths, - pathsOverride, - TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE, - }); - - return; - } - const cleaner = new TempFileCleaner({ - paths, - maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60), - intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300), - leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false), - }); - - cleaner.start(); - - return cleaner; - } -} diff --git a/apps/coordinator/src/cleaner.ts b/apps/coordinator/src/cleaner.ts deleted file mode 100644 index 58cfd24bb70..00000000000 --- a/apps/coordinator/src/cleaner.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { Exec } from "./exec"; -import { setTimeout } from "timers/promises"; - -interface TempFileCleanerOptions { - paths: string[]; - maxAgeMinutes: number; - intervalSeconds: number; - leadingEdge?: boolean; -} - -export class TempFileCleaner { - private enabled = false; - - private logger: SimpleStructuredLogger; - private exec: Exec; - - constructor(private opts: TempFileCleanerOptions) { - this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts }); - this.exec = new Exec({ logger: this.logger }); - } - - async start() { - this.logger.log("TempFileCleaner.start"); - this.enabled = true; - - if (!this.opts.leadingEdge) { - await this.wait(); - } - - while (this.enabled) { - try { - await this.clean(); - } catch (error) { - this.logger.error("error during tick", { error }); - } - - await this.wait(); - } - } - - stop() { - this.logger.log("TempFileCleaner.stop"); - this.enabled = false; - } - - private wait() { - return setTimeout(this.opts.intervalSeconds * 1000); - } - - private async clean() { - for (const path of this.opts.paths) { - try { - await this.cleanSingle(path); - } catch (error) { - this.logger.error("error while cleaning", { path, error }); - } - } - } - - private async cleanSingle(startingPoint: string) { - const maxAgeMinutes = this.opts.maxAgeMinutes; - - const ignoreStartingPoint = ["!", "-path", startingPoint]; - const onlyDirectDescendants = ["-maxdepth", "1"]; - const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`]; - - const baseArgs = [ - startingPoint, - ...ignoreStartingPoint, - ...onlyDirectDescendants, - ...onlyOldFiles, - ]; - - const duArgs = ["-exec", "du", "-ch", "{}", "+"]; - const rmArgs = ["-exec", "rm", "-rf", "{}", "+"]; - - const du = this.x("find", [...baseArgs, ...duArgs]); - const duOutput = await du; - - const duLines = duOutput.stdout.trim().split("\n"); - const fileCount = duLines.length - 1; // last line is the total - const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0]; - - if (fileCount === 0) { - this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes }); - return; - } - - this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes }); - - const rm = this.x("find", [...baseArgs, ...rmArgs]); - const rmOutput = await rm; - - if (rmOutput.stderr.length > 0) { - this.logger.error("delete unsuccessful", { rmOutput }); - return; - } - - this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } -} diff --git a/apps/coordinator/src/exec.ts b/apps/coordinator/src/exec.ts deleted file mode 100644 index ca6d3dcd101..00000000000 --- a/apps/coordinator/src/exec.ts +++ /dev/null @@ -1,293 +0,0 @@ -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -import { randomUUID } from "crypto"; -import { homedir } from "os"; -import { type Result, x } from "tinyexec"; - -class TinyResult { - pid?: number; - exitCode?: number; - aborted: boolean; - killed: boolean; - - constructor(result: Result) { - this.pid = result.pid; - this.exitCode = result.exitCode; - this.aborted = result.aborted; - this.killed = result.killed; - } -} - -interface ExecOptions { - logger?: SimpleStructuredLogger; - abortSignal?: AbortSignal; - logOutput?: boolean; - trimArgs?: boolean; - neverThrow?: boolean; -} - -export class Exec { - private logger: SimpleStructuredLogger; - private abortSignal: AbortSignal | undefined; - - private logOutput: boolean; - private trimArgs: boolean; - private neverThrow: boolean; - - constructor(opts: ExecOptions) { - this.logger = opts.logger ?? new SimpleStructuredLogger("exec"); - this.abortSignal = opts.abortSignal; - - this.logOutput = opts.logOutput ?? true; - this.trimArgs = opts.trimArgs ?? true; - this.neverThrow = opts.neverThrow ?? false; - } - - async x( - command: string, - args?: string[], - opts?: { neverThrow?: boolean; ignoreAbort?: boolean } - ) { - const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args; - - const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`; - this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed }); - - const result = x(command, argsTrimmed, { - signal: opts?.ignoreAbort ? undefined : this.abortSignal, - // We don't use this as it doesn't cover killed and aborted processes - // throwOnError: true, - }); - - const output = await result; - - const metadata = { - command, - argsRaw: args, - argsTrimmed, - globalOpts: { - trimArgs: this.trimArgs, - neverThrow: this.neverThrow, - hasAbortSignal: !!this.abortSignal, - }, - localOpts: opts, - stdout: output.stdout, - stderr: output.stderr, - pid: result.pid, - exitCode: result.exitCode, - aborted: result.aborted, - killed: result.killed, - }; - - if (this.logOutput) { - this.logger.debug(`output: ${commandWithFirstArg}`, metadata); - } - - if (this.neverThrow || opts?.neverThrow) { - return output; - } - - if (result.aborted) { - this.logger.error(`aborted: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - if (result.killed) { - this.logger.error(`killed: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - if (result.exitCode !== 0) { - this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata); - throw new TinyResult(result); - } - - return output; - } - - static Result = TinyResult; -} - -interface BuildahOptions { - id?: string; - abortSignal?: AbortSignal; -} - -export class Buildah { - private id: string; - private logger: SimpleStructuredLogger; - private exec: Exec; - - private containers = new Set(); - private images = new Set(); - - constructor(opts: BuildahOptions) { - this.id = opts.id ?? randomUUID(); - this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id }); - - this.exec = new Exec({ - logger: this.logger, - abortSignal: opts.abortSignal, - }); - - this.logger.log("initiaized", { opts }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } - - async from(baseImage: string) { - const output = await this.x("buildah", ["from", baseImage]); - this.containers.add(output.stdout); - return output; - } - - async add(container: string, src: string, dest: string) { - return await this.x("buildah", ["add", container, src, dest]); - } - - async config(container: string, annotations: string[]) { - const args = ["config"]; - - for (const annotation of annotations) { - args.push(`--annotation=${annotation}`); - } - - args.push(container); - - return await this.x("buildah", args); - } - - async commit(container: string, imageRef: string) { - const output = await this.x("buildah", ["commit", container, imageRef]); - this.images.add(output.stdout); - return output; - } - - async push(imageRef: string, registryTlsVerify?: boolean) { - return await this.x("buildah", [ - "push", - `--tls-verify=${String(!!registryTlsVerify)}`, - imageRef, - ]); - } - - async cleanup() { - if (this.containers.size > 0) { - try { - const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true }); - this.containers.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some containers", { output }); - } - } catch (error) { - this.logger.error("failed to clean up containers", { error, containers: this.containers }); - } - } else { - this.logger.debug("no containers to clean up"); - } - - if (this.images.size > 0) { - try { - const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true }); - this.images.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some images", { output }); - } - } catch (error) { - this.logger.error("failed to clean up images", { error, images: this.images }); - } - } else { - this.logger.debug("no images to clean up"); - } - } - - static async canLogin(registryHost: string) { - try { - await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true }); - return true; - } catch (_error) { - return false; - } - } - - static get tmpDir() { - return process.env.TMPDIR ?? "/var/tmp"; - } - - static get storageRootDir() { - return process.getuid?.() === 0 - ? "/var/lib/containers/storage" - : `${homedir()}/.local/share/containers/storage`; - } -} - -interface CrictlOptions { - id?: string; - abortSignal?: AbortSignal; -} - -export class Crictl { - private id: string; - private logger: SimpleStructuredLogger; - private exec: Exec; - - private archives = new Set(); - - constructor(opts: CrictlOptions) { - this.id = opts.id ?? randomUUID(); - this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id }); - - this.exec = new Exec({ - logger: this.logger, - abortSignal: opts.abortSignal, - }); - - this.logger.log("initiaized", { opts }); - } - - private get x() { - return this.exec.x.bind(this.exec); - } - - async ps(containerName: string, quiet?: boolean) { - return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]); - } - - async checkpoint(containerId: string, exportLocation: string) { - const output = await this.x("crictl", [ - "checkpoint", - `--export=${exportLocation}`, - containerId, - ]); - this.archives.add(exportLocation); - return output; - } - - async cleanup() { - if (this.archives.size > 0) { - try { - const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true }); - this.archives.clear(); - - if (output.stderr.length > 0) { - this.logger.error("failed to remove some archives", { output }); - } - } catch (error) { - this.logger.error("failed to clean up archives", { error, archives: this.archives }); - } - } else { - this.logger.debug("no archives to clean up"); - } - } - - static getExportLocation(identifier: string) { - return `${this.checkpointDir}/${identifier}.tar`; - } - - static get checkpointDir() { - return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints"; - } -} diff --git a/apps/coordinator/src/index.ts b/apps/coordinator/src/index.ts deleted file mode 100644 index fb72d5ddf12..00000000000 --- a/apps/coordinator/src/index.ts +++ /dev/null @@ -1,1781 +0,0 @@ -import { createServer } from "node:http"; -import { Server } from "socket.io"; -import type { WaitReason } from "@trigger.dev/core/v3"; -import { - CoordinatorToPlatformMessages, - CoordinatorToProdWorkerMessages, - omit, - PlatformToCoordinatorMessages, - ProdWorkerSocketData, - ProdWorkerToCoordinatorMessages, -} from "@trigger.dev/core/v3"; -import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace"; -import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket"; -import { ExponentialBackoff, HttpReply, getTextBody } from "@trigger.dev/core/v3/apps"; -import { ChaosMonkey } from "./chaosMonkey"; -import { Checkpointer } from "./checkpointer"; -import { boolFromEnv, numFromEnv, safeJsonParse } from "./util"; - -import { collectDefaultMetrics, register, Gauge } from "prom-client"; -import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; -collectDefaultMetrics(); - -const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020); -const NODE_NAME = process.env.NODE_NAME || "coordinator"; -const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000; - -const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ?? "true"); -const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; -const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030; -const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret"; -const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false"); - -const TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS = - parseInt(process.env.TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS || "") || 30_000; -const TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES = - parseInt(process.env.TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES || "") || 7; - -const WAIT_FOR_TASK_CHECKPOINT_DELAY_MS = - parseInt(process.env.WAIT_FOR_TASK_CHECKPOINT_DELAY_MS || "") || 0; -const WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS = - parseInt(process.env.WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS || "") || 0; - -const logger = new SimpleStructuredLogger("coordinator", undefined, { nodeName: NODE_NAME }); -const chaosMonkey = new ChaosMonkey( - !!process.env.CHAOS_MONKEY_ENABLED, - !!process.env.CHAOS_MONKEY_DISABLE_ERRORS, - !!process.env.CHAOS_MONKEY_DISABLE_DELAYS -); - -class CheckpointReadinessTimeoutError extends Error {} -class CheckpointCancelError extends Error {} - -class TaskCoordinator { - #httpServer: ReturnType; - #internalHttpServer: ReturnType; - - #checkpointer = new Checkpointer({ - dockerMode: !process.env.KUBERNETES_PORT, - forceSimulate: boolFromEnv("FORCE_CHECKPOINT_SIMULATION", false), - heartbeat: this.#sendRunHeartbeat.bind(this), - registryHost: process.env.REGISTRY_HOST, - registryNamespace: process.env.REGISTRY_NAMESPACE, - registryTlsVerify: boolFromEnv("REGISTRY_TLS_VERIFY", true), - disableCheckpointSupport: boolFromEnv("DISABLE_CHECKPOINT_SUPPORT", false), - simulatePushFailure: boolFromEnv("SIMULATE_PUSH_FAILURE", false), - simulatePushFailureSeconds: numFromEnv("SIMULATE_PUSH_FAILURE_SECONDS", 300), - simulateCheckpointFailure: boolFromEnv("SIMULATE_CHECKPOINT_FAILURE", false), - simulateCheckpointFailureSeconds: numFromEnv("SIMULATE_CHECKPOINT_FAILURE_SECONDS", 300), - chaosMonkey, - }); - - #prodWorkerNamespace?: ZodNamespace< - typeof ProdWorkerToCoordinatorMessages, - typeof CoordinatorToProdWorkerMessages, - typeof ProdWorkerSocketData - >; - #platformSocket?: ZodSocketConnection< - typeof CoordinatorToPlatformMessages, - typeof PlatformToCoordinatorMessages - >; - - #checkpointableTasks = new Map< - string, - { resolve: (value: void) => void; reject: (err?: any) => void } - >(); - - #delayThresholdInMs: number = DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS; - - constructor( - private port: number, - private host = "0.0.0.0" - ) { - this.#httpServer = this.#createHttpServer(); - this.#internalHttpServer = this.#createInternalHttpServer(); - - this.#checkpointer.init(); - this.#platformSocket = this.#createPlatformSocket(); - - const connectedTasksTotal = new Gauge({ - name: "daemon_connected_tasks_total", // don't change this without updating dashboard config - help: "The number of tasks currently connected.", - collect: () => { - connectedTasksTotal.set(this.#prodWorkerNamespace?.namespace.sockets.size ?? 0); - }, - }); - register.registerMetric(connectedTasksTotal); - } - - #returnValidatedExtraHeaders(headers: Record) { - for (const [key, value] of Object.entries(headers)) { - if (value === undefined) { - throw new Error(`Extra header is undefined: ${key}`); - } - } - - return headers; - } - - // MARK: SOCKET: PLATFORM - #createPlatformSocket() { - if (!PLATFORM_ENABLED) { - logger.log("INFO: platform connection disabled"); - return; - } - - const extraHeaders = this.#returnValidatedExtraHeaders({ - "x-supports-dynamic-config": "yes", - }); - - const host = PLATFORM_HOST; - const port = Number(PLATFORM_WS_PORT); - - const platformLogger = new SimpleStructuredLogger("socket-platform", undefined, { - namespace: "coordinator", - }); - - platformLogger.log("connecting", { host, port }); - platformLogger.debug("connecting with extra headers", { extraHeaders }); - - const platformConnection = new ZodSocketConnection({ - namespace: "coordinator", - host, - port, - secure: SECURE_CONNECTION, - extraHeaders, - clientMessages: CoordinatorToPlatformMessages, - serverMessages: PlatformToCoordinatorMessages, - authToken: PLATFORM_SECRET, - logHandlerPayloads: false, - handlers: { - // This is used by resumeAttempt - RESUME_AFTER_DEPENDENCY: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DEPENDENCY", - ...omit(message, "completions", "executions"), - completions: message.completions.map((c) => ({ - id: c.id, - ok: c.ok, - })), - executions: message.executions.length, - }); - - log.log("Handling RESUME_AFTER_DEPENDENCY"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DEPENDENCY"); - - await chaosMonkey.call(); - - // In case the task resumes before the checkpoint is created - this.#cancelCheckpoint(message.runId, { - event: "RESUME_AFTER_DEPENDENCY", - completions: message.completions.length, - }); - - taskSocket.emit("RESUME_AFTER_DEPENDENCY", message); - }, - // This is used by sharedQueueConsumer - RESUME_AFTER_DEPENDENCY_WITH_ACK: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DEPENDENCY_WITH_ACK", - ...omit(message, "completions", "executions"), - completions: message.completions.map((c) => ({ - id: c.id, - ok: c.ok, - })), - executions: message.executions.length, - }); - - log.log("Handling RESUME_AFTER_DEPENDENCY_WITH_ACK"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return { - success: false, - error: { - name: "SocketNotFoundError", - message: "Socket for attempt not found", - }, - }; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DEPENDENCY_WITH_ACK"); - - //if this is set, we want to kill the process because it will be resumed with the checkpoint from the queue - if (taskSocket.data.requiresCheckpointResumeWithMessage) { - log.log("RESUME_AFTER_DEPENDENCY_WITH_ACK: Checkpoint is set so going to nack"); - - return { - success: false, - error: { - name: "CheckpointMessagePresentError", - message: - "Checkpoint message is present, so we need to kill the process and resume from the queue.", - }, - }; - } - - await chaosMonkey.call(); - - // In case the task resumes before the checkpoint is created - this.#cancelCheckpoint(message.runId, { - event: "RESUME_AFTER_DEPENDENCY_WITH_ACK", - completions: message.completions.length, - }); - - taskSocket.emit("RESUME_AFTER_DEPENDENCY", message); - - return { - success: true, - }; - }, - RESUME_AFTER_DURATION: async (message) => { - const log = platformLogger.child({ - eventName: "RESUME_AFTER_DURATION", - ...message, - }); - - log.log("Handling RESUME_AFTER_DURATION"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - log.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for RESUME_AFTER_DURATION"); - - await chaosMonkey.call(); - - taskSocket.emit("RESUME_AFTER_DURATION", message); - }, - REQUEST_ATTEMPT_CANCELLATION: async (message) => { - const log = platformLogger.child({ - eventName: "REQUEST_ATTEMPT_CANCELLATION", - ...message, - }); - - log.log("Handling REQUEST_ATTEMPT_CANCELLATION"); - - const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId); - - if (!taskSocket) { - logger.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for REQUEST_ATTEMPT_CANCELLATION"); - - taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message); - }, - REQUEST_RUN_CANCELLATION: async (message) => { - const log = platformLogger.child({ - eventName: "REQUEST_RUN_CANCELLATION", - ...message, - }); - - log.log("Handling REQUEST_RUN_CANCELLATION"); - - const taskSocket = await this.#getRunSocket(message.runId); - - if (!taskSocket) { - logger.debug("Socket for run not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for REQUEST_RUN_CANCELLATION"); - - this.#cancelCheckpoint(message.runId, { event: "REQUEST_RUN_CANCELLATION", ...message }); - - if (message.delayInMs) { - taskSocket.emit("REQUEST_EXIT", { - version: "v2", - delayInMs: message.delayInMs, - }); - } else { - // If there's no delay, assume the worker doesn't support non-v1 messages - taskSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - } - }, - READY_FOR_RETRY: async (message) => { - const log = platformLogger.child({ - eventName: "READY_FOR_RETRY", - ...message, - }); - - const taskSocket = await this.#getRunSocket(message.runId); - - if (!taskSocket) { - logger.debug("Socket for attempt not found"); - return; - } - - log.addFields({ socketId: taskSocket.id, socketData: taskSocket.data }); - log.log("Found task socket for READY_FOR_RETRY"); - - await chaosMonkey.call(); - - taskSocket.emit("READY_FOR_RETRY", message); - }, - DYNAMIC_CONFIG: async (message) => { - const log = platformLogger.child({ - eventName: "DYNAMIC_CONFIG", - ...message, - }); - - log.log("Handling DYNAMIC_CONFIG"); - - this.#delayThresholdInMs = message.checkpointThresholdInMs; - - // The first time we receive a dynamic config, the worker namespace will be created - if (!this.#prodWorkerNamespace) { - const io = new Server(this.#httpServer); - this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io); - } - }, - }, - }); - - return platformConnection; - } - - async #getRunSocket(runId: string) { - const sockets = (await this.#prodWorkerNamespace?.fetchSockets()) ?? []; - - for (const socket of sockets) { - if (socket.data.runId === runId) { - return socket; - } - } - } - - async #getAttemptSocket(attemptFriendlyId: string) { - const sockets = (await this.#prodWorkerNamespace?.fetchSockets()) ?? []; - - for (const socket of sockets) { - if (socket.data.attemptFriendlyId === attemptFriendlyId) { - return socket; - } - } - } - - // MARK: SOCKET: WORKERS - #createProdWorkerNamespace(io: Server) { - const provider = new ZodNamespace({ - io, - name: "prod-worker", - clientMessages: ProdWorkerToCoordinatorMessages, - serverMessages: CoordinatorToProdWorkerMessages, - socketData: ProdWorkerSocketData, - postAuth: async (socket, next, logger) => { - function setSocketDataFromHeader( - dataKey: keyof typeof socket.data, - headerName: string, - required: boolean = true - ) { - const value = socket.handshake.headers[headerName]; - - if (value) { - socket.data[dataKey] = Array.isArray(value) ? value[0] : value; - return; - } - - if (required) { - logger.error("missing required header", { headerName }); - throw new Error("missing header"); - } - } - - try { - setSocketDataFromHeader("podName", "x-pod-name"); - setSocketDataFromHeader("contentHash", "x-trigger-content-hash"); - setSocketDataFromHeader("projectRef", "x-trigger-project-ref"); - setSocketDataFromHeader("runId", "x-trigger-run-id"); - setSocketDataFromHeader("attemptFriendlyId", "x-trigger-attempt-friendly-id", false); - setSocketDataFromHeader("attemptNumber", "x-trigger-attempt-number", false); - setSocketDataFromHeader("envId", "x-trigger-env-id"); - setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id"); - setSocketDataFromHeader("deploymentVersion", "x-trigger-deployment-version"); - } catch (error) { - logger.error("setSocketDataFromHeader error", { error }); - socket.disconnect(true); - return; - } - - logger.debug("success", socket.data); - - next(); - }, - onConnection: async (socket, handler, sender) => { - const logger = new SimpleStructuredLogger("ns-prod-worker", undefined, { - namespace: "prod-worker", - socketId: socket.id, - socketData: socket.data, - }); - - const getSocketMetadata = () => { - return { - attemptFriendlyId: socket.data.attemptFriendlyId, - attemptNumber: socket.data.attemptNumber, - requiresCheckpointResumeWithMessage: socket.data.requiresCheckpointResumeWithMessage, - }; - }; - - const getAttemptNumber = () => { - return socket.data.attemptNumber ? parseInt(socket.data.attemptNumber) : undefined; - }; - - const exitRun = () => { - logger.log("exitRun", getSocketMetadata()); - - socket.emit("REQUEST_EXIT", { - version: "v1", - }); - }; - - const crashRun = async (error: { name: string; message: string; stack?: string }) => { - logger.error("crashRun", { ...getSocketMetadata(), error }); - - try { - this.#platformSocket?.send("RUN_CRASHED", { - version: "v1", - runId: socket.data.runId, - error, - }); - } finally { - exitRun(); - } - }; - - const checkpointInProgress = () => { - return this.#checkpointableTasks.has(socket.data.runId); - }; - - const readyToCheckpoint = async ( - reason: WaitReason | "RETRY" - ): Promise< - | { - success: true; - } - | { - success: false; - reason?: string; - } - > => { - const log = logger.child(getSocketMetadata()); - - log.log("readyToCheckpoint", { runId: socket.data.runId, reason }); - - if (checkpointInProgress()) { - return { - success: false, - reason: "checkpoint in progress", - }; - } - - let timeout: NodeJS.Timeout | undefined = undefined; - - const CHECKPOINTABLE_TIMEOUT_SECONDS = 20; - - const isCheckpointable = new Promise((resolve, reject) => { - // We set a reasonable timeout to prevent waiting forever - timeout = setTimeout( - () => reject(new CheckpointReadinessTimeoutError()), - CHECKPOINTABLE_TIMEOUT_SECONDS * 1000 - ); - - this.#checkpointableTasks.set(socket.data.runId, { resolve, reject }); - }); - - try { - await isCheckpointable; - this.#checkpointableTasks.delete(socket.data.runId); - - return { - success: true, - }; - } catch (error) { - log.error("Error while waiting for checkpointable state", { error }); - - if (error instanceof CheckpointReadinessTimeoutError) { - logger.error( - `Failed to become checkpointable in ${CHECKPOINTABLE_TIMEOUT_SECONDS}s for ${reason}`, - { runId: socket.data.runId } - ); - - return { - success: false, - reason: "timeout", - }; - } - - if (error instanceof CheckpointCancelError) { - return { - success: false, - reason: "canceled", - }; - } - - return { - success: false, - reason: typeof error === "string" ? error : "unknown", - }; - } finally { - clearTimeout(timeout); - } - }; - - const updateAttemptFriendlyId = (attemptFriendlyId: string) => { - socket.data.attemptFriendlyId = attemptFriendlyId; - }; - - const updateAttemptNumber = (attemptNumber: string | number) => { - socket.data.attemptNumber = String(attemptNumber); - }; - - this.#platformSocket?.send("LOG", { - metadata: socket.data, - text: "connected", - }); - - socket.on("TEST", (message, callback) => { - logger.log("Handling TEST", { eventName: "TEST", ...getSocketMetadata(), ...message }); - - try { - callback(); - } catch (error) { - logger.error("TEST error", { error }); - } - }); - - // Deprecated: Only workers without support for lazy attempts use this - socket.on("READY_FOR_EXECUTION", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_EXECUTION", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_EXECUTION"); - - try { - const executionAck = await this.#platformSocket?.sendWithAck( - "READY_FOR_EXECUTION", - message - ); - - if (!executionAck) { - log.error("no execution ack"); - - await crashRun({ - name: "ReadyForExecutionError", - message: "No execution ack", - }); - - return; - } - - if (!executionAck.success) { - log.error("failed to get execution payload"); - - await crashRun({ - name: "ReadyForExecutionError", - message: "Failed to get execution payload", - }); - - return; - } - - socket.emit("EXECUTE_TASK_RUN", { - version: "v1", - executionPayload: executionAck.payload, - }); - - updateAttemptFriendlyId(executionAck.payload.execution.attempt.id); - updateAttemptNumber(executionAck.payload.execution.attempt.number); - } catch (error) { - log.error("READY_FOR_EXECUTION error", { error }); - - await crashRun({ - name: "ReadyForExecutionError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: LAZY ATTEMPT - socket.on("READY_FOR_LAZY_ATTEMPT", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_LAZY_ATTEMPT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_LAZY_ATTEMPT"); - - try { - const lazyAttempt = await this.#platformSocket?.sendWithAck("READY_FOR_LAZY_ATTEMPT", { - ...message, - envId: socket.data.envId, - }); - - if (!lazyAttempt) { - log.error("no lazy attempt ack"); - - await crashRun({ - name: "ReadyForLazyAttemptError", - message: "No lazy attempt ack", - }); - - return; - } - - if (!lazyAttempt.success) { - log.error("failed to get lazy attempt payload", { reason: lazyAttempt.reason }); - - await crashRun({ - name: "ReadyForLazyAttemptError", - message: "Failed to get lazy attempt payload", - }); - - return; - } - - await chaosMonkey.call(); - - const lazyPayload = { - ...lazyAttempt.lazyPayload, - metrics: [ - ...(message.startTime - ? [ - { - name: "start", - event: "lazy_payload", - timestamp: message.startTime, - duration: Date.now() - message.startTime, - }, - ] - : []), - ], - }; - - socket.emit("EXECUTE_TASK_RUN_LAZY_ATTEMPT", { - version: "v1", - lazyPayload, - }); - } catch (error) { - if (error instanceof ChaosMonkey.Error) { - log.error("ChaosMonkey error, won't crash run"); - return; - } - - log.error("READY_FOR_LAZY_ATTEMPT error", { error }); - - // await crashRun({ - // name: "ReadyForLazyAttemptError", - // message: - // error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - // }); - - return; - } - }); - - // MARK: RESUME READY - socket.on("READY_FOR_RESUME", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_RESUME", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_RESUME"); - - try { - updateAttemptFriendlyId(message.attemptFriendlyId); - - if (message.version === "v2") { - updateAttemptNumber(message.attemptNumber); - } - - this.#platformSocket?.send("READY_FOR_RESUME", { ...message, version: "v1" }); - } catch (error) { - log.error("READY_FOR_RESUME error", { error }); - - await crashRun({ - name: "ReadyForResumeError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: RUN COMPLETED - socket.on("TASK_RUN_COMPLETED", async (message, callback) => { - const log = logger.child({ - eventName: "TASK_RUN_COMPLETED", - ...getSocketMetadata(), - ...omit(message, "completion", "execution"), - completion: { - id: message.completion.id, - ok: message.completion.ok, - }, - }); - - log.log("Handling TASK_RUN_COMPLETED"); - - try { - const { completion, execution } = message; - - // Cancel all in-progress checkpoints (if any) - this.#cancelCheckpoint(socket.data.runId, { - event: "TASK_RUN_COMPLETED", - attemptNumber: execution.attempt.number, - }); - - await chaosMonkey.call({ throwErrors: false }); - - const sendCompletionWithAck = async (): Promise => { - try { - const response = await this.#platformSocket?.sendWithAck( - "TASK_RUN_COMPLETED_WITH_ACK", - { - version: "v2", - execution, - completion, - }, - TASK_RUN_COMPLETED_WITH_ACK_TIMEOUT_MS - ); - - if (!response) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: no response"); - return false; - } - - if (!response.success) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: error response", { - error: response.error, - }); - return false; - } - - log.log("TASK_RUN_COMPLETED_WITH_ACK: successful response"); - return true; - } catch (error) { - log.error("TASK_RUN_COMPLETED_WITH_ACK: threw error", { error }); - return false; - } - }; - - const completeWithoutCheckpoint = async (shouldExit: boolean) => { - const supportsRetryCheckpoints = message.version === "v1"; - - callback({ willCheckpointAndRestore: false, shouldExit }); - - if (supportsRetryCheckpoints) { - // This is only here for backwards compat - this.#platformSocket?.send("TASK_RUN_COMPLETED", { - version: "v1", - execution, - completion, - }); - } else { - // 99.99% of runs should end up here - - const completedWithAckBackoff = new ExponentialBackoff("FullJitter").maxRetries( - TASK_RUN_COMPLETED_WITH_ACK_MAX_RETRIES - ); - - const result = await completedWithAckBackoff.execute( - async ({ retry, delay, elapsedMs }) => { - logger.log("TASK_RUN_COMPLETED_WITH_ACK: sending with backoff", { - retry, - delay, - elapsedMs, - }); - - const success = await sendCompletionWithAck(); - - if (!success) { - throw new Error("Failed to send completion with ack"); - } - } - ); - - if (!result.success) { - logger.error("TASK_RUN_COMPLETED_WITH_ACK: failed to send with backoff", result); - return; - } - - logger.log("TASK_RUN_COMPLETED_WITH_ACK: sent with backoff", result); - } - }; - - if (completion.ok) { - await completeWithoutCheckpoint(true); - return; - } - - if ( - completion.error.type === "INTERNAL_ERROR" && - completion.error.code === "TASK_RUN_CANCELLED" - ) { - await completeWithoutCheckpoint(true); - return; - } - - if (completion.retry === undefined) { - await completeWithoutCheckpoint(true); - return; - } - - if (completion.retry.delay < this.#delayThresholdInMs) { - await completeWithoutCheckpoint(false); - - // Prevents runs that fail fast from never sending a heartbeat - this.#sendRunHeartbeat(socket.data.runId); - - return; - } - - if (message.version === "v2") { - await completeWithoutCheckpoint(true); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - if (!willCheckpointAndRestore) { - await completeWithoutCheckpoint(false); - return; - } - - // The worker will then put itself in a checkpointable state - callback({ willCheckpointAndRestore: true, shouldExit: false }); - - const ready = await readyToCheckpoint("RETRY"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - - return; - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: socket.data.runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - shouldHeartbeat: true, - }); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - await completeWithoutCheckpoint(false); - return; - } - - log.addFields({ checkpoint }); - - this.#platformSocket?.send("TASK_RUN_COMPLETED", { - version: "v1", - execution, - completion, - checkpoint, - }); - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("TASK_RUN_COMPLETED error", { error }); - - await crashRun({ - name: "TaskRunCompletedError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: TASK FAILED - socket.on("TASK_RUN_FAILED_TO_RUN", async ({ completion }) => { - const log = logger.child({ - eventName: "TASK_RUN_FAILED_TO_RUN", - ...getSocketMetadata(), - completion: { - id: completion.id, - ok: completion.ok, - }, - }); - - log.log("Handling TASK_RUN_FAILED_TO_RUN"); - - try { - // Cancel all in-progress checkpoints (if any) - this.#cancelCheckpoint(socket.data.runId, { - event: "TASK_RUN_FAILED_TO_RUN", - errorType: completion.error.type, - }); - - this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", { - version: "v1", - completion, - }); - - exitRun(); - } catch (error) { - log.error("TASK_RUN_FAILED_TO_RUN error", { error }); - - return; - } - }); - - // MARK: CHECKPOINT - socket.on("READY_FOR_CHECKPOINT", async (message) => { - const log = logger.child({ - eventName: "READY_FOR_CHECKPOINT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling READY_FOR_CHECKPOINT"); - - try { - const checkpointable = this.#checkpointableTasks.get(socket.data.runId); - - if (!checkpointable) { - log.error("No checkpoint scheduled"); - return; - } - - checkpointable.resolve(); - } catch (error) { - log.error("READY_FOR_CHECKPOINT error", { error }); - - return; - } - }); - - // MARK: CXX CHECKPOINT - socket.on("CANCEL_CHECKPOINT", async (message, callback) => { - const log = logger.child({ - eventName: "CANCEL_CHECKPOINT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling CANCEL_CHECKPOINT"); - - try { - if (message.version === "v1") { - this.#cancelCheckpoint(socket.data.runId, { event: "CANCEL_CHECKPOINT", ...message }); - // v1 has no callback - return; - } - - const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId, { - event: "CANCEL_CHECKPOINT", - ...message, - }); - - callback({ version: "v2", checkpointCanceled }); - } catch (error) { - log.error("CANCEL_CHECKPOINT error", { error }); - } - }); - - // MARK: DURATION WAIT - socket.on("WAIT_FOR_DURATION", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_DURATION", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_DURATION"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - const ready = await readyToCheckpoint("WAIT_FOR_DURATION"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }); - - if (!checkpoint) { - // The task container will keep running until the wait duration has elapsed - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_DURATION", - ms: message.ms, - now: message.now, - }, - }); - - if (ack?.keepRunAlive) { - log.log("keeping run alive after duration checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after duration checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_DURATION error", { error }); - - await crashRun({ - name: "WaitForDurationError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: TASK WAIT - socket.on("WAIT_FOR_TASK", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_TASK", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_TASK"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits - if (message.version === "v2") { - const ready = await readyToCheckpoint("WAIT_FOR_TASK"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush( - { - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }, - WAIT_FOR_TASK_CHECKPOINT_DELAY_MS - ); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - log.log("WAIT_FOR_TASK checkpoint created"); - - //setting this means we can only resume from a checkpoint - socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`; - log.log("WAIT_FOR_TASK set requiresCheckpointResumeWithMessage"); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_TASK", - friendlyId: message.friendlyId, - }, - }); - - if (ack?.keepRunAlive) { - socket.data.requiresCheckpointResumeWithMessage = undefined; - log.log("keeping run alive after task checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after duration checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_TASK error", { error }); - - await crashRun({ - name: "WaitForTaskError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: BATCH WAIT - socket.on("WAIT_FOR_BATCH", async (message, callback) => { - const log = logger.child({ - eventName: "WAIT_FOR_BATCH", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling WAIT_FOR_BATCH", message); - - try { - await chaosMonkey.call({ throwErrors: false }); - - if (checkpointInProgress()) { - log.error("Checkpoint already in progress"); - callback({ willCheckpointAndRestore: false }); - return; - } - - const { canCheckpoint, willSimulate } = await this.#checkpointer.init(); - - const willCheckpointAndRestore = canCheckpoint || willSimulate; - - callback({ willCheckpointAndRestore }); - - if (!willCheckpointAndRestore) { - return; - } - - // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits - if (message.version === "v2") { - const ready = await readyToCheckpoint("WAIT_FOR_BATCH"); - - if (!ready.success) { - log.error("Failed to become checkpointable", { reason: ready.reason }); - return; - } - } - - const runId = socket.data.runId; - const attemptNumber = getAttemptNumber(); - - const checkpoint = await this.#checkpointer.checkpointAndPush( - { - runId, - projectRef: socket.data.projectRef, - deploymentVersion: socket.data.deploymentVersion, - attemptNumber, - }, - WAIT_FOR_BATCH_CHECKPOINT_DELAY_MS - ); - - if (!checkpoint) { - log.error("Failed to checkpoint"); - return; - } - - log.addFields({ checkpoint }); - - log.log("WAIT_FOR_BATCH checkpoint created"); - - //setting this means we can only resume from a checkpoint - socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`; - log.log("WAIT_FOR_BATCH set checkpoint"); - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId: socket.data.runId, - attemptFriendlyId: message.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_BATCH", - batchFriendlyId: message.batchFriendlyId, - runFriendlyIds: message.runFriendlyIds, - }, - }); - - if (ack?.keepRunAlive) { - socket.data.requiresCheckpointResumeWithMessage = undefined; - log.log("keeping run alive after batch checkpoint"); - - if (checkpoint.docker && willSimulate) { - // The container is still paused so we need to unpause it - log.log("unpausing container after batch checkpoint"); - this.#checkpointer.unpause(runId, attemptNumber); - } - - return; - } - - if (!checkpoint.docker || !willSimulate) { - exitRun(); - } - } catch (error) { - log.error("WAIT_FOR_BATCH error", { error }); - - await crashRun({ - name: "WaitForBatchError", - message: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - - return; - } - }); - - // MARK: INDEX - socket.on("INDEX_TASKS", async (message, callback) => { - const log = logger.child({ - eventName: "INDEX_TASKS", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling INDEX_TASKS"); - - try { - const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", { - version: "v2", - projectRef: socket.data.projectRef, - envId: socket.data.envId, - deploymentId: message.deploymentId, - metadata: { - contentHash: socket.data.contentHash, - packageVersion: message.packageVersion, - tasks: message.tasks, - }, - supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts, - }); - - if (!workerAck) { - log.debug("no worker ack while indexing"); - } - - callback({ success: !!workerAck?.success }); - } catch (error) { - log.error("INDEX_TASKS error", { error }); - callback({ success: false }); - } - }); - - // MARK: INDEX FAILED - socket.on("INDEXING_FAILED", async (message) => { - const log = logger.child({ - eventName: "INDEXING_FAILED", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling INDEXING_FAILED"); - - try { - this.#platformSocket?.send("INDEXING_FAILED", { - version: "v1", - deploymentId: message.deploymentId, - error: message.error, - }); - } catch (error) { - log.error("INDEXING_FAILED error", { error }); - } - }); - - // MARK: CREATE ATTEMPT - socket.on("CREATE_TASK_RUN_ATTEMPT", async (message, callback) => { - const log = logger.child({ - eventName: "CREATE_TASK_RUN_ATTEMPT", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling CREATE_TASK_RUN_ATTEMPT"); - - try { - await chaosMonkey.call({ throwErrors: false }); - - const createAttempt = await this.#platformSocket?.sendWithAck( - "CREATE_TASK_RUN_ATTEMPT", - { - runId: message.runId, - envId: socket.data.envId, - } - ); - - if (!createAttempt?.success) { - log.debug("no ack while creating attempt", { reason: createAttempt?.reason }); - callback({ success: false, reason: createAttempt?.reason }); - return; - } - - updateAttemptFriendlyId(createAttempt.executionPayload.execution.attempt.id); - updateAttemptNumber(createAttempt.executionPayload.execution.attempt.number); - - callback({ - success: true, - executionPayload: createAttempt.executionPayload, - }); - } catch (error) { - log.error("CREATE_TASK_RUN_ATTEMPT error", { error }); - callback({ - success: false, - reason: - error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error", - }); - } - }); - - socket.on("UNRECOVERABLE_ERROR", async (message) => { - const log = logger.child({ - eventName: "UNRECOVERABLE_ERROR", - ...getSocketMetadata(), - error: message.error, - }); - - log.log("Handling UNRECOVERABLE_ERROR"); - - try { - await crashRun(message.error); - } catch (error) { - log.error("UNRECOVERABLE_ERROR error", { error }); - } - }); - - socket.on("SET_STATE", async (message) => { - const log = logger.child({ - eventName: "SET_STATE", - ...getSocketMetadata(), - ...message, - }); - - log.log("Handling SET_STATE"); - - try { - if (message.attemptFriendlyId) { - updateAttemptFriendlyId(message.attemptFriendlyId); - } - - if (message.attemptNumber) { - updateAttemptNumber(message.attemptNumber); - } - } catch (error) { - log.error("SET_STATE error", { error }); - } - }); - }, - onDisconnect: async (socket, handler, sender, logger) => { - try { - this.#platformSocket?.send("LOG", { - metadata: socket.data, - text: "disconnect", - }); - } catch (error) { - logger.error("onDisconnect error", { error }); - } - }, - handlers: { - TASK_HEARTBEAT: async (message) => { - this.#platformSocket?.send("TASK_HEARTBEAT", message); - }, - TASK_RUN_HEARTBEAT: async (message) => { - this.#sendRunHeartbeat(message.runId); - }, - }, - }); - - return provider; - } - - #sendRunHeartbeat(runId: string) { - this.#platformSocket?.send("TASK_RUN_HEARTBEAT", { - version: "v1", - runId, - }); - } - - #cancelCheckpoint(runId: string, reason?: any): boolean { - logger.log("cancelCheckpoint: call", { runId, reason }); - - const checkpointWait = this.#checkpointableTasks.get(runId); - - if (checkpointWait) { - // Stop waiting for task to reach checkpointable state - checkpointWait.reject(new CheckpointCancelError()); - } - - // Cancel checkpointing procedure - const checkpointCanceled = this.#checkpointer.cancelAllCheckpointsForRun(runId); - - logger.log("cancelCheckpoint: result", { - runId, - reason, - checkpointCanceled, - hadCheckpointWait: !!checkpointWait, - }); - - return checkpointCanceled; - } - - // MARK: HTTP SERVER - #createHttpServer() { - const httpServer = createServer(async (req, res) => { - logger.log(`[${req.method}]`, { url: req.url }); - - const reply = new HttpReply(res); - - switch (req.url) { - case "/health": { - return reply.text("ok"); - } - case "/metrics": { - return reply.text(await register.metrics(), 200, register.contentType); - } - default: { - return reply.empty(404); - } - } - }); - - httpServer.on("clientError", (err, socket) => { - socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); - }); - - httpServer.on("listening", () => { - logger.log("server listening on port", { port: HTTP_SERVER_PORT }); - }); - - return httpServer; - } - - #createInternalHttpServer() { - const httpServer = createServer(async (req, res) => { - logger.log(`[${req.method}]`, { url: req.url }); - - const reply = new HttpReply(res); - - switch (req.url) { - case "/whoami": { - return reply.text(NODE_NAME); - } - case "/checkpoint/duration": { - try { - const body = await getTextBody(req); - const json = safeJsonParse(body); - - if (typeof json !== "object" || !json) { - return reply.text("Invalid body", 400); - } - - if (!("runId" in json) || typeof json.runId !== "string") { - return reply.text("Missing or invalid: runId", 400); - } - - if (!("now" in json) || typeof json.now !== "number") { - return reply.text("Missing or invalid: now", 400); - } - - if (!("ms" in json) || typeof json.ms !== "number") { - return reply.text("Missing or invalid: ms", 400); - } - - let keepRunAlive = false; - if ("keepRunAlive" in json && typeof json.keepRunAlive === "boolean") { - keepRunAlive = json.keepRunAlive; - } - - let async = false; - if ("async" in json && typeof json.async === "boolean") { - async = json.async; - } - - const { runId, now, ms } = json; - - if (!runId) { - return reply.text("Missing runId", 400); - } - - const runSocket = await this.#getRunSocket(runId); - if (!runSocket) { - return reply.text("Run socket not found", 404); - } - - const { data } = runSocket; - - console.log("Manual duration checkpoint", data); - - if (async) { - reply.text("Creating checkpoint in the background", 202); - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: data.runId, - projectRef: data.projectRef, - deploymentVersion: data.deploymentVersion, - attemptNumber: data.attemptNumber ? parseInt(data.attemptNumber) : undefined, - }); - - if (!checkpoint) { - return reply.text("Failed to checkpoint", 500); - } - - if (!data.attemptFriendlyId) { - return reply.text("Socket data missing attemptFriendlyId", 500); - } - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId, - attemptFriendlyId: data.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "WAIT_FOR_DURATION", - ms, - now, - }, - }); - - if (ack?.keepRunAlive || keepRunAlive) { - return reply.json({ - message: `keeping run ${runId} alive after checkpoint`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } - - runSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - - return reply.json({ - message: `checkpoint created for run ${runId}`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } catch (error) { - return reply.json({ - message: `error`, - error, - }); - } - } - case "/checkpoint/manual": { - try { - const body = await getTextBody(req); - const json = safeJsonParse(body); - - if (typeof json !== "object" || !json) { - return reply.text("Invalid body", 400); - } - - if (!("runId" in json) || typeof json.runId !== "string") { - return reply.text("Missing or invalid: runId", 400); - } - - let restoreAtUnixTimeMs: number | undefined; - if ("restoreAtUnixTimeMs" in json && typeof json.restoreAtUnixTimeMs === "number") { - restoreAtUnixTimeMs = json.restoreAtUnixTimeMs; - } - - let keepRunAlive = false; - if ("keepRunAlive" in json && typeof json.keepRunAlive === "boolean") { - keepRunAlive = json.keepRunAlive; - } - - let async = false; - if ("async" in json && typeof json.async === "boolean") { - async = json.async; - } - - const { runId } = json; - - if (!runId) { - return reply.text("Missing runId", 400); - } - - const runSocket = await this.#getRunSocket(runId); - if (!runSocket) { - return reply.text("Run socket not found", 404); - } - - const { data } = runSocket; - - console.log("Manual checkpoint", data); - - if (async) { - reply.text("Creating checkpoint in the background", 202); - } - - const checkpoint = await this.#checkpointer.checkpointAndPush({ - runId: data.runId, - projectRef: data.projectRef, - deploymentVersion: data.deploymentVersion, - attemptNumber: data.attemptNumber ? parseInt(data.attemptNumber) : undefined, - }); - - if (!checkpoint) { - return reply.text("Failed to checkpoint", 500); - } - - if (!data.attemptFriendlyId) { - return reply.text("Socket data missing attemptFriendlyId", 500); - } - - const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", { - version: "v1", - runId, - attemptFriendlyId: data.attemptFriendlyId, - docker: checkpoint.docker, - location: checkpoint.location, - reason: { - type: "MANUAL", - restoreAtUnixTimeMs, - }, - }); - - if (ack?.keepRunAlive || keepRunAlive) { - return reply.json({ - message: `keeping run ${runId} alive after checkpoint`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } - - runSocket.emit("REQUEST_EXIT", { - version: "v1", - }); - - return reply.json({ - message: `checkpoint created for run ${runId}`, - checkpoint, - requestJson: json, - platformAck: ack, - }); - } catch (error) { - return reply.json({ - message: `error`, - error, - }); - } - } - default: { - return reply.empty(404); - } - } - }); - - httpServer.on("clientError", (err, socket) => { - socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); - }); - - httpServer.on("listening", () => { - logger.log("internal server listening on port", { port: HTTP_SERVER_PORT + 100 }); - }); - - return httpServer; - } - - listen() { - this.#httpServer.listen(this.port, this.host); - this.#internalHttpServer.listen(this.port + 100, "127.0.0.1"); - } -} - -const coordinator = new TaskCoordinator(HTTP_SERVER_PORT); -coordinator.listen(); diff --git a/apps/coordinator/src/util.ts b/apps/coordinator/src/util.ts deleted file mode 100644 index 649eb3000af..00000000000 --- a/apps/coordinator/src/util.ts +++ /dev/null @@ -1,31 +0,0 @@ -export const boolFromEnv = (env: string, defaultValue: boolean): boolean => { - const value = process.env[env]; - - if (!value) { - return defaultValue; - } - - return ["1", "true"].includes(value); -}; - -export const numFromEnv = (env: string, defaultValue: number): number => { - const value = process.env[env]; - - if (!value) { - return defaultValue; - } - - return parseInt(value, 10); -}; - -export function safeJsonParse(json?: string): unknown { - if (!json) { - return; - } - - try { - return JSON.parse(json); - } catch (_e) { - return null; - } -} diff --git a/apps/coordinator/tsconfig.json b/apps/coordinator/tsconfig.json deleted file mode 100644 index e03fd024126..00000000000 --- a/apps/coordinator/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "resolveJsonModule": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/apps/docker-provider/.env.example b/apps/docker-provider/.env.example deleted file mode 100644 index 75c54083d1a..00000000000 --- a/apps/docker-provider/.env.example +++ /dev/null @@ -1,11 +0,0 @@ -HTTP_SERVER_PORT=8050 - -PLATFORM_WS_PORT=3030 -PLATFORM_SECRET=provider-secret -SECURE_CONNECTION=false - -OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel - -# Use this if you are on macOS -# COORDINATOR_HOST="host.docker.internal" -# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/docker-provider/.gitignore b/apps/docker-provider/.gitignore deleted file mode 100644 index 5c84119d635..00000000000 --- a/apps/docker-provider/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/docker-provider/Containerfile b/apps/docker-provider/Containerfile deleted file mode 100644 index 4a86a7734fa..00000000000 --- a/apps/docker-provider/Containerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine - -WORKDIR /app - -FROM node-22-alpine AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-22-alpine AS base - -RUN apk add --no-cache dumb-init docker - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -USER node - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/docker-provider/README.md b/apps/docker-provider/README.md deleted file mode 100644 index 647db280a5b..00000000000 --- a/apps/docker-provider/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Docker provider - -The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions. diff --git a/apps/docker-provider/package.json b/apps/docker-provider/package.json deleted file mode 100644 index aa812c68d4a..00000000000 --- a/apps/docker-provider/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "docker-provider", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t docker-provider", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@trigger.dev/core": "workspace:*", - "execa": "^8.0.1" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts deleted file mode 100644 index 8572b90145a..00000000000 --- a/apps/docker-provider/src/index.ts +++ /dev/null @@ -1,295 +0,0 @@ -import type { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3"; -import type { - TaskOperations, - TaskOperationsCreateOptions, - TaskOperationsIndexOptions, - TaskOperationsRestoreOptions, -} from "@trigger.dev/core/v3/apps"; -import { ProviderShell, SimpleLogger, isExecaChildProcess } from "@trigger.dev/core/v3/apps"; -import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly"; -import { $, type ExecaChildProcess, execa } from "execa"; -import { setTimeout } from "node:timers/promises"; - -const MACHINE_NAME = process.env.MACHINE_NAME || "local"; -const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020; -const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1"; -const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host"; - -const OTEL_EXPORTER_OTLP_ENDPOINT = - process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318"; - -const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes( - process.env.FORCE_CHECKPOINT_SIMULATION ?? "false" -); - -const logger = new SimpleLogger(`[${MACHINE_NAME}]`); - -type TaskOperationsInitReturn = { - canCheckpoint: boolean; - willSimulate: boolean; -}; - -class DockerTaskOperations implements TaskOperations { - #initialized = false; - #canCheckpoint = false; - - constructor(private opts = { forceSimulate: false }) {} - - async init(): Promise { - if (this.#initialized) { - return this.#getInitReturn(this.#canCheckpoint); - } - - logger.log("Initializing task operations"); - - const testCheckpoint = await testDockerCheckpoint(); - - if (testCheckpoint.ok) { - return this.#getInitReturn(true); - } - - logger.error(testCheckpoint.message, testCheckpoint.error); - return this.#getInitReturn(false); - } - - #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn { - this.#canCheckpoint = canCheckpoint; - - if (canCheckpoint) { - if (!this.#initialized) { - logger.log("Full checkpoint support!"); - } - } - - this.#initialized = true; - - const willSimulate = !canCheckpoint || this.opts.forceSimulate; - - if (willSimulate) { - logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", { - forceSimulate: this.opts.forceSimulate, - }); - } - - return { - canCheckpoint, - willSimulate, - }; - } - - async index(opts: TaskOperationsIndexOptions) { - await this.init(); - - const containerName = this.#getIndexContainerName(opts.shortCode); - - logger.log(`Indexing task ${opts.imageRef}`, { - host: COORDINATOR_HOST, - port: COORDINATOR_PORT, - }); - - logger.debug( - await execa("docker", [ - "run", - `--network=${DOCKER_NETWORK}`, - "--rm", - `--env=INDEX_TASKS=true`, - `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`, - `--env=TRIGGER_API_URL=${opts.apiUrl}`, - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=POD_NAME=${containerName}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--name=${containerName}`, - `${opts.imageRef}`, - ]) - ); - } - - async create(opts: TaskOperationsCreateOptions) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber); - - const runArgs = [ - "run", - `--network=${DOCKER_NETWORK}`, - "--detach", - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=TRIGGER_RUN_ID=${opts.runId}`, - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=POD_NAME=${containerName}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, - `--name=${containerName}`, - ]; - - if (process.env.ENFORCE_MACHINE_PRESETS) { - runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`); - } - - if (opts.dequeuedAt) { - runArgs.push(`--env=TRIGGER_RUN_DEQUEUED_AT_MS=${opts.dequeuedAt}`); - } - - runArgs.push(`${opts.image}`); - - try { - logger.debug(await execa("docker", runArgs)); - } catch (error) { - if (!isExecaChildProcess(error)) { - throw error; - } - - logger.error("Create failed:", { - opts, - exitCode: error.exitCode, - escapedCommand: error.escapedCommand, - stdout: error.stdout, - stderr: error.stderr, - }); - } - } - - async restore(opts: TaskOperationsRestoreOptions) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber); - - if (!this.#canCheckpoint || this.opts.forceSimulate) { - logger.log("Simulating restore"); - - const unpause = logger.debug(await $`docker unpause ${containerName}`); - - if (unpause.exitCode !== 0) { - throw new Error("docker unpause command failed"); - } - - await this.#sendPostStart(containerName); - return; - } - - const { exitCode } = logger.debug( - await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}` - ); - - if (exitCode !== 0) { - throw new Error("docker start command failed"); - } - - await this.#sendPostStart(containerName); - } - - async delete(opts: { runId: string }) { - await this.init(); - - const containerName = this.#getRunContainerName(opts.runId); - await this.#sendPreStop(containerName); - - logger.log("noop: delete"); - } - - async get(opts: { runId: string }) { - await this.init(); - - logger.log("noop: get"); - } - - #getIndexContainerName(suffix: string) { - return `task-index-${suffix}`; - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - async #sendPostStart(containerName: string): Promise { - try { - const port = await this.#getHttpServerPort(containerName); - logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore")); - } catch (error) { - logger.error("postStart error", { error }); - throw new Error("postStart command failed"); - } - } - - async #sendPreStop(containerName: string): Promise { - try { - const port = await this.#getHttpServerPort(containerName); - logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate")); - } catch (error) { - logger.error("preStop error", { error }); - throw new Error("preStop command failed"); - } - } - - async #getHttpServerPort(containerName: string): Promise { - // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes - // FIXME: Skip this in prod - const logs = logger.debug(await $`docker logs ${containerName}`); - const matches = logs.stdout.match(/http server listening on port (?[0-9]+)/); - - const port = Number(matches?.groups?.port); - - if (!port) { - throw new Error("failed to extract port from logs"); - } - - return port; - } - - async #runLifecycleCommand( - containerName: string, - port: number, - type: THookType, - cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses, - retryCount = 0 - ): Promise { - try { - return await execa("docker", [ - "exec", - containerName, - "busybox", - "wget", - "-q", - "-O-", - `127.0.0.1:${port}/${type}?cause=${cause}`, - ]); - } catch (error: any) { - if (type === "postStart" && retryCount < 6) { - logger.debug(`retriable ${type} error`, { retryCount, message: error?.message }); - await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50)); - - return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1); - } - - logger.error(`final ${type} error`, { message: error?.message }); - throw new Error(`${type} command failed after ${retryCount - 1} retries`); - } - } -} - -const provider = new ProviderShell({ - tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }), - type: "docker", -}); - -provider.listen(); - -function exponentialBackoff( - retryCount: number, - exponential: number, - minDelay: number, - maxDelay: number, - jitter: number -): number { - // Calculate the delay using the exponential backoff formula - const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay); - - // Calculate the jitter - const jitterValue = Math.random() * jitter; - - // Return the calculated delay with jitter - return delay + jitterValue; -} diff --git a/apps/docker-provider/tsconfig.json b/apps/docker-provider/tsconfig.json deleted file mode 100644 index f87adfc2d7f..00000000000 --- a/apps/docker-provider/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/apps/kubernetes-provider/.env.example b/apps/kubernetes-provider/.env.example deleted file mode 100644 index f21ee29bac9..00000000000 --- a/apps/kubernetes-provider/.env.example +++ /dev/null @@ -1,9 +0,0 @@ -HTTP_SERVER_PORT=8060 - -PLATFORM_WS_PORT=3030 -PLATFORM_SECRET=provider-secret -SECURE_CONNECTION=false - -# Use this if you are on macOS -# COORDINATOR_HOST="host.docker.internal" -# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318" \ No newline at end of file diff --git a/apps/kubernetes-provider/.gitignore b/apps/kubernetes-provider/.gitignore deleted file mode 100644 index 5c84119d635..00000000000 --- a/apps/kubernetes-provider/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -dist/ -node_modules/ -.env \ No newline at end of file diff --git a/apps/kubernetes-provider/Containerfile b/apps/kubernetes-provider/Containerfile deleted file mode 100644 index 3aa34549f73..00000000000 --- a/apps/kubernetes-provider/Containerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine - -WORKDIR /app - -FROM node-22-alpine AS pruner - -COPY --chown=node:node . . -RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker -RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' + - -FROM node-22-alpine AS base - -RUN apk add --no-cache dumb-init - -COPY --chown=node:node .gitignore .gitignore -COPY --from=pruner --chown=node:node /app/out/json/ . -COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml -COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml - -FROM base AS dev-deps -RUN corepack enable -ENV NODE_ENV development - -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile - -FROM base AS builder -RUN corepack enable - -COPY --from=pruner --chown=node:node /app/out/full/ . -COPY --from=dev-deps --chown=node:node /app/ . -COPY --chown=node:node turbo.json turbo.json - -RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter kubernetes-provider build:bundle - -FROM base AS runner - -RUN corepack enable -ENV NODE_ENV production - -COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs - -EXPOSE 8000 - -USER node - -CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ] diff --git a/apps/kubernetes-provider/README.md b/apps/kubernetes-provider/README.md deleted file mode 100644 index 829c8f2154a..00000000000 --- a/apps/kubernetes-provider/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Kubernetes provider - -The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions. diff --git a/apps/kubernetes-provider/package.json b/apps/kubernetes-provider/package.json deleted file mode 100644 index b4a4307abbb..00000000000 --- a/apps/kubernetes-provider/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "kubernetes-provider", - "private": true, - "version": "0.0.1", - "description": "", - "main": "dist/index.cjs", - "scripts": { - "build": "npm run build:bundle", - "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"", - "build:image": "docker build -f Containerfile . -t kubernetes-provider", - "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts", - "start": "tsx src/index.ts", - "typecheck": "tsc --noEmit" - }, - "keywords": [], - "author": "", - "license": "MIT", - "dependencies": { - "@kubernetes/client-node": "^0.20.0", - "@trigger.dev/core": "workspace:*", - "p-queue": "^8.0.1" - }, - "devDependencies": { - "dotenv": "^16.4.2", - "esbuild": "^0.19.11", - "tsx": "^4.7.0" - } -} diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts deleted file mode 100644 index 82c1b026282..00000000000 --- a/apps/kubernetes-provider/src/index.ts +++ /dev/null @@ -1,782 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import type { - EnvironmentType, - MachinePreset, - PostStartCauses, - PreStopCauses, -} from "@trigger.dev/core/v3"; -import type { - TaskOperations, - TaskOperationsCreateOptions, - TaskOperationsIndexOptions, - TaskOperationsPrePullDeploymentOptions, - TaskOperationsRestoreOptions, -} from "@trigger.dev/core/v3/apps"; -import { ProviderShell, SimpleLogger } from "@trigger.dev/core/v3/apps"; -import { PodCleaner } from "./podCleaner"; -import { TaskMonitor } from "./taskMonitor"; -import { UptimeHeartbeat } from "./uptimeHeartbeat"; -import { assertExhaustive } from "@trigger.dev/core"; -import { CustomLabelHelper } from "./labelHelper"; - -const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; -const NODE_NAME = process.env.NODE_NAME || "local"; -const OTEL_EXPORTER_OTLP_ENDPOINT = - process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318"; -const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined; -const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined; -const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default"; - -const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300"); - -const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL; -const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60"); -const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25"); -const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10"); -const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10"); - -const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi"; -const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi"; - -// Image config -const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true"; -const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS; -const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9"; -const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox"; -const DEPLOYMENT_IMAGE_PREFIX = process.env.DEPLOYMENT_IMAGE_PREFIX; -const RESTORE_IMAGE_PREFIX = process.env.RESTORE_IMAGE_PREFIX; -const UTILITY_IMAGE_PREFIX = process.env.UTILITY_IMAGE_PREFIX; - -const logger = new SimpleLogger(`[${NODE_NAME}]`); -logger.log(`running in ${RUNTIME_ENV} mode`); - -type Namespace = { - metadata: { - name: string; - }; -}; - -type ResourceQuantities = { - [K in "cpu" | "memory" | "ephemeral-storage"]?: string; -}; - -class KubernetesTaskOperations implements TaskOperations { - #namespace: Namespace = { - metadata: { - name: "default", - }, - }; - - #k8sApi: { - core: k8s.CoreV1Api; - batch: k8s.BatchV1Api; - apps: k8s.AppsV1Api; - }; - - #labelHelper = new CustomLabelHelper(); - - constructor(opts: { namespace?: string } = {}) { - if (opts.namespace) { - this.#namespace.metadata.name = opts.namespace; - } - - this.#k8sApi = this.#createK8sApi(); - } - - async init() { - // noop - } - - async index(opts: TaskOperationsIndexOptions) { - await this.#createJob( - { - metadata: { - name: this.#getIndexContainerName(opts.shortCode), - namespace: this.#namespace.metadata.name, - }, - spec: { - completions: 1, - backoffLimit: 0, - ttlSecondsAfterFinished: 300, - template: { - metadata: { - labels: { - ...this.#getSharedLabels(opts), - app: "task-index", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "index", - deployment: opts.deploymentId, - }, - }, - spec: { - ...this.#defaultPodSpec, - containers: [ - { - name: this.#getIndexContainerName(opts.shortCode), - image: getImageRef("deployment", opts.imageRef), - ports: [ - { - containerPort: 8000, - }, - ], - resources: { - limits: { - cpu: "1", - memory: "2G", - "ephemeral-storage": "2Gi", - }, - }, - lifecycle: { - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - env: [ - ...this.#getSharedEnv(opts.envId), - { - name: "INDEX_TASKS", - value: "true", - }, - { - name: "TRIGGER_SECRET_KEY", - value: opts.apiKey, - }, - { - name: "TRIGGER_API_URL", - value: opts.apiUrl, - }, - ], - }, - ], - }, - }, - }, - }, - this.#namespace - ); - } - - async create(opts: TaskOperationsCreateOptions) { - const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber); - - await this.#createPod( - { - metadata: { - name: containerName, - namespace: this.#namespace.metadata.name, - labels: { - ...this.#labelHelper.getAdditionalLabels("create"), - ...this.#getSharedLabels(opts), - app: "task-run", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "create", - run: opts.runId, - }, - }, - spec: { - ...this.#defaultPodSpec, - terminationGracePeriodSeconds: 60 * 60, - containers: [ - { - name: containerName, - image: getImageRef("deployment", opts.image), - ports: [ - { - containerPort: 8000, - }, - ], - resources: this.#getResourcesForMachine(opts.machine), - lifecycle: { - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - env: [ - ...this.#getSharedEnv(opts.envId), - { - name: "TRIGGER_RUN_ID", - value: opts.runId, - }, - ...(opts.dequeuedAt - ? [{ name: "TRIGGER_RUN_DEQUEUED_AT_MS", value: String(opts.dequeuedAt) }] - : []), - ], - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - volumes: [ - { - name: "taskinfo", - emptyDir: {}, - }, - ], - }, - }, - this.#namespace - ); - } - - async restore(opts: TaskOperationsRestoreOptions) { - await this.#createPod( - { - metadata: { - name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`, - namespace: this.#namespace.metadata.name, - labels: { - ...this.#labelHelper.getAdditionalLabels("restore"), - ...this.#getSharedLabels(opts), - app: "task-run", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "restore", - run: opts.runId, - checkpoint: opts.checkpointId, - }, - }, - spec: { - ...this.#defaultPodSpec, - initContainers: [ - { - name: "pull-base-image", - image: getImageRef("deployment", opts.imageRef), - command: ["sleep", "0"], - }, - { - name: "populate-taskinfo", - image: getImageRef("utility", BUSYBOX_IMAGE), - imagePullPolicy: "IfNotPresent", - command: ["/bin/sh", "-c"], - args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"], - env: this.#coordinatorEnvVars, - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - containers: [ - { - name: this.#getRunContainerName(opts.runId), - image: getImageRef("restore", opts.checkpointRef), - ports: [ - { - containerPort: 8000, - }, - ], - resources: this.#getResourcesForMachine(opts.machine), - lifecycle: { - postStart: { - exec: { - command: this.#getLifecycleCommand("postStart", "restore"), - }, - }, - preStop: { - exec: { - command: this.#getLifecycleCommand("preStop", "terminate"), - }, - }, - }, - volumeMounts: [ - { - name: "taskinfo", - mountPath: "/etc/taskinfo", - }, - ], - }, - ], - volumes: [ - { - name: "taskinfo", - emptyDir: {}, - }, - ], - }, - }, - this.#namespace - ); - } - - async delete(opts: { runId: string }) { - await this.#deletePod({ - runId: opts.runId, - namespace: this.#namespace, - }); - } - - async get(opts: { runId: string }) { - await this.#getPod(opts.runId, this.#namespace); - } - - async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) { - if (PRE_PULL_DISABLED) { - logger.debug("Pre-pull is disabled, skipping.", { opts }); - return; - } - - const metaName = this.#getPrePullContainerName(opts.shortCode); - - const metaLabels = { - ...this.#getSharedLabels(opts), - app: "task-prepull", - "app.kubernetes.io/part-of": "trigger-worker", - "app.kubernetes.io/component": "prepull", - deployment: opts.deploymentId, - name: metaName, - } satisfies k8s.V1ObjectMeta["labels"]; - - await this.#createDaemonSet( - { - metadata: { - name: metaName, - namespace: this.#namespace.metadata.name, - labels: metaLabels, - }, - spec: { - selector: { - matchLabels: { - name: metaName, - }, - }, - template: { - metadata: { - labels: metaLabels, - }, - spec: { - ...this.#defaultPodSpec, - restartPolicy: "Always", - affinity: { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { - matchExpressions: [ - { - key: "trigger.dev/pre-pull-disabled", - operator: "DoesNotExist", - }, - ], - }, - ], - }, - }, - }, - initContainers: [ - { - name: "prepull", - image: getImageRef("deployment", opts.imageRef), - command: ["/usr/bin/true"], - resources: { - limits: { - cpu: "0.25", - memory: "100Mi", - "ephemeral-storage": "1Gi", - }, - }, - }, - ], - containers: [ - { - name: "pause", - image: getImageRef("utility", PAUSE_IMAGE), - resources: { - limits: { - cpu: "1m", - memory: "12Mi", - }, - }, - }, - ], - }, - }, - }, - }, - this.#namespace - ); - } - - #envTypeToLabelValue(type: EnvironmentType) { - switch (type) { - case "PRODUCTION": - return "prod"; - case "STAGING": - return "stg"; - case "DEVELOPMENT": - return "dev"; - case "PREVIEW": - return "preview"; - } - } - - get #defaultPodSpec(): Omit { - const pullSecrets = ["registry-trigger", "registry-trigger-failover"]; - - if (ADDITIONAL_PULL_SECRETS) { - pullSecrets.push(...ADDITIONAL_PULL_SECRETS.split(",")); - } - - const imagePullSecrets = pullSecrets.map( - (name) => ({ name }) satisfies k8s.V1LocalObjectReference - ); - - return { - restartPolicy: "Never", - automountServiceAccountToken: false, - imagePullSecrets, - nodeSelector: { - nodetype: "worker", - }, - }; - } - - get #defaultResourceRequests(): ResourceQuantities { - return { - "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST, - }; - } - - get #defaultResourceLimits(): ResourceQuantities { - return { - "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT, - }; - } - - get #coordinatorHostEnvVar(): k8s.V1EnvVar { - return COORDINATOR_HOST - ? { - name: "COORDINATOR_HOST", - value: COORDINATOR_HOST, - } - : { - name: "COORDINATOR_HOST", - valueFrom: { - fieldRef: { - fieldPath: "status.hostIP", - }, - }, - }; - } - - get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined { - if (COORDINATOR_PORT) { - return { - name: "COORDINATOR_PORT", - value: COORDINATOR_PORT, - }; - } - } - - get #coordinatorEnvVars(): k8s.V1EnvVar[] { - const envVars = [this.#coordinatorHostEnvVar]; - - if (this.#coordinatorPortEnvVar) { - envVars.push(this.#coordinatorPortEnvVar); - } - - return envVars; - } - - #getSharedEnv(envId: string): k8s.V1EnvVar[] { - return [ - { - name: "TRIGGER_ENV_ID", - value: envId, - }, - { - name: "DEBUG", - value: process.env.DEBUG ? "1" : "0", - }, - { - name: "HTTP_SERVER_PORT", - value: "8000", - }, - { - name: "OTEL_EXPORTER_OTLP_ENDPOINT", - value: OTEL_EXPORTER_OTLP_ENDPOINT, - }, - { - name: "POD_NAME", - valueFrom: { - fieldRef: { - fieldPath: "metadata.name", - }, - }, - }, - { - name: "MACHINE_NAME", - valueFrom: { - fieldRef: { - fieldPath: "spec.nodeName", - }, - }, - }, - { - name: "TRIGGER_POD_SCHEDULED_AT_MS", - value: Date.now().toString(), - }, - ...this.#coordinatorEnvVars, - ]; - } - - #getSharedLabels( - opts: - | TaskOperationsIndexOptions - | TaskOperationsCreateOptions - | TaskOperationsRestoreOptions - | TaskOperationsPrePullDeploymentOptions - ): Record { - return { - env: opts.envId, - envtype: this.#envTypeToLabelValue(opts.envType), - org: opts.orgId, - project: opts.projectId, - }; - } - - #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities { - return { - cpu: `${preset.cpu * 0.75}`, - memory: `${preset.memory}G`, - }; - } - - #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities { - return { - cpu: `${preset.cpu}`, - memory: `${preset.memory}G`, - }; - } - - #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements { - return { - requests: { - ...this.#defaultResourceRequests, - ...this.#getResourceRequestsForMachine(preset), - }, - limits: { - ...this.#defaultResourceLimits, - ...this.#getResourceLimitsForMachine(preset), - }, - }; - } - - #getLifecycleCommand( - type: THookType, - cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses - ) { - const retries = 5; - - // This will retry sending the lifecycle hook up to `retries` times - // The sleep is required as this may start running before the HTTP server is up - const exec = [ - "/bin/sh", - "-c", - `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`, - ]; - - logger.debug("getLifecycleCommand()", { exec }); - - return exec; - } - - #getIndexContainerName(suffix: string) { - return `task-index-${suffix}`; - } - - #getRunContainerName(suffix: string, attemptNumber?: number) { - return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`; - } - - #getPrePullContainerName(suffix: string) { - return `task-prepull-${suffix}`; - } - - #createK8sApi() { - const kubeConfig = new k8s.KubeConfig(); - - if (RUNTIME_ENV === "local") { - kubeConfig.loadFromDefault(); - } else if (RUNTIME_ENV === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - batch: kubeConfig.makeApiClient(k8s.BatchV1Api), - apps: kubeConfig.makeApiClient(k8s.AppsV1Api), - }; - } - - async #createPod(pod: k8s.V1Pod, namespace: Namespace) { - try { - const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #deletePod(opts: { runId: string; namespace: Namespace }) { - try { - const res = await this.#k8sApi.core.deleteNamespacedPod( - opts.runId, - opts.namespace.metadata.name - ); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #getPod(runId: string, namespace: Namespace) { - try { - const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name); - logger.debug(res.body); - return res.body; - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #createJob(job: k8s.V1Job, namespace: Namespace) { - try { - const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) { - try { - const res = await this.#k8sApi.apps.createNamespacedDaemonSet( - namespace.metadata.name, - daemonSet - ); - logger.debug(res.body); - } catch (err: unknown) { - this.#handleK8sError(err); - } - } - - #throwUnlessRecord(candidate: unknown): asserts candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - throw candidate; - } - } - - #handleK8sError(err: unknown) { - this.#throwUnlessRecord(err); - - if ("body" in err && err.body) { - logger.error(err.body); - this.#throwUnlessRecord(err.body); - - if (typeof err.body.message === "string") { - throw new Error(err.body?.message); - } else { - throw err.body; - } - } else { - logger.error(err); - throw err; - } - } -} - -type ImageType = "deployment" | "restore" | "utility"; - -function getImagePrefix(type: ImageType) { - switch (type) { - case "deployment": - return DEPLOYMENT_IMAGE_PREFIX; - case "restore": - return RESTORE_IMAGE_PREFIX; - case "utility": - return UTILITY_IMAGE_PREFIX; - default: - assertExhaustive(type); - } -} - -function getImageRef(type: ImageType, ref: string) { - const prefix = getImagePrefix(type); - return prefix ? `${prefix}/${ref}` : ref; -} - -const provider = new ProviderShell({ - tasks: new KubernetesTaskOperations({ - namespace: KUBERNETES_NAMESPACE, - }), - type: "kubernetes", -}); - -provider.listen(); - -const taskMonitor = new TaskMonitor({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - onIndexFailure: async (deploymentId, details) => { - logger.log("Indexing failed", { deploymentId, details }); - - try { - provider.platformSocket.send("INDEXING_FAILED", { - deploymentId, - error: { - name: `Crashed with exit code ${details.exitCode}`, - message: details.reason, - stack: details.logs, - }, - overrideCompletion: details.overrideCompletion, - }); - } catch (error) { - logger.error(error); - } - }, - onRunFailure: async (runId, details) => { - logger.log("Run failed:", { runId, details }); - - try { - provider.platformSocket.send("WORKER_CRASHED", { runId, ...details }); - } catch (error) { - logger.error(error); - } - }, -}); - -taskMonitor.start(); - -const podCleaner = new PodCleaner({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS, -}); - -podCleaner.start(); - -if (UPTIME_HEARTBEAT_URL) { - const uptimeHeartbeat = new UptimeHeartbeat({ - runtimeEnv: RUNTIME_ENV, - namespace: KUBERNETES_NAMESPACE, - intervalInSeconds: UPTIME_INTERVAL_SECONDS, - pingUrl: UPTIME_HEARTBEAT_URL, - maxPendingRuns: UPTIME_MAX_PENDING_RUNS, - maxPendingIndeces: UPTIME_MAX_PENDING_INDECES, - maxPendingErrors: UPTIME_MAX_PENDING_ERRORS, - }); - - uptimeHeartbeat.start(); -} else { - logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable."); -} diff --git a/apps/kubernetes-provider/src/labelHelper.ts b/apps/kubernetes-provider/src/labelHelper.ts deleted file mode 100644 index 98cd3d68be4..00000000000 --- a/apps/kubernetes-provider/src/labelHelper.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { assertExhaustive } from "@trigger.dev/core"; - -const CREATE_LABEL_ENV_VAR_PREFIX = "DEPLOYMENT_LABEL_"; -const RESTORE_LABEL_ENV_VAR_PREFIX = "RESTORE_LABEL_"; -const LABEL_SAMPLE_RATE_POSTFIX = "_SAMPLE_RATE"; - -type OperationType = "create" | "restore"; - -type CustomLabel = { - key: string; - value: string; - sampleRate: number; -}; - -export class CustomLabelHelper { - // Labels and sample rates are defined in environment variables so only need to be computed once - private createLabels?: CustomLabel[]; - private restoreLabels?: CustomLabel[]; - - private getLabelPrefix(type: OperationType) { - const prefix = type === "create" ? CREATE_LABEL_ENV_VAR_PREFIX : RESTORE_LABEL_ENV_VAR_PREFIX; - return prefix.toLowerCase(); - } - - private getLabelSampleRatePostfix() { - return LABEL_SAMPLE_RATE_POSTFIX.toLowerCase(); - } - - // Can only range from 0 to 1 - private fractionFromPercent(percent: number) { - return Math.min(1, Math.max(0, percent / 100)); - } - - private isLabelSampleRateEnvVar(key: string) { - return key.toLowerCase().endsWith(this.getLabelSampleRatePostfix()); - } - - private isLabelEnvVar(type: OperationType, key: string) { - const prefix = this.getLabelPrefix(type); - return key.toLowerCase().startsWith(prefix) && !this.isLabelSampleRateEnvVar(key); - } - - private getSampleRateEnvVarKey(type: OperationType, envKey: string) { - return `${envKey.toLowerCase()}${this.getLabelSampleRatePostfix()}`; - } - - private getLabelNameFromEnvVarKey(type: OperationType, key: string) { - return key - .slice(this.getLabelPrefix(type).length) - .toLowerCase() - .replace(/___/g, ".") - .replace(/__/g, "/") - .replace(/_/g, "-"); - } - - private getCaseInsensitiveEnvValue(key: string) { - for (const [envKey, value] of Object.entries(process.env)) { - if (envKey.toLowerCase() === key.toLowerCase()) { - return value; - } - } - } - - /** Returns the sample rate for a given label as fraction of 100 */ - private getSampleRateFromEnvVarKey(type: OperationType, envKey: string) { - // Apply default: always sample - const DEFAULT_SAMPLE_RATE_PERCENT = 100; - const defaultSampleRateFraction = this.fractionFromPercent(DEFAULT_SAMPLE_RATE_PERCENT); - - const value = this.getCaseInsensitiveEnvValue(this.getSampleRateEnvVarKey(type, envKey)); - - if (!value) { - return defaultSampleRateFraction; - } - - const sampleRatePercent = parseFloat(value || String(DEFAULT_SAMPLE_RATE_PERCENT)); - - if (isNaN(sampleRatePercent)) { - return defaultSampleRateFraction; - } - - const fractionalSampleRate = this.fractionFromPercent(sampleRatePercent); - - return fractionalSampleRate; - } - - private getCustomLabels(type: OperationType): CustomLabel[] { - switch (type) { - case "create": - if (this.createLabels) { - return this.createLabels; - } - break; - case "restore": - if (this.restoreLabels) { - return this.restoreLabels; - } - break; - default: - assertExhaustive(type); - } - - const customLabels: CustomLabel[] = []; - - for (const [envKey, value] of Object.entries(process.env)) { - const key = envKey.toLowerCase(); - - // Only process env vars that start with the expected prefix - if (!this.isLabelEnvVar(type, key)) { - continue; - } - - // Skip sample rates - deal with them separately - if (this.isLabelSampleRateEnvVar(key)) { - continue; - } - - const labelName = this.getLabelNameFromEnvVarKey(type, key); - const sampleRate = this.getSampleRateFromEnvVarKey(type, key); - - const label = { - key: labelName, - value: value || "", - sampleRate, - } satisfies CustomLabel; - - customLabels.push(label); - } - - return customLabels; - } - - getAdditionalLabels(type: OperationType): Record { - const labels = this.getCustomLabels(type); - - const additionalLabels: Record = {}; - - for (const { key, value, sampleRate } of labels) { - // Always apply label if sample rate is 1 - if (sampleRate === 1) { - additionalLabels[key] = value; - continue; - } - - if (Math.random() <= sampleRate) { - additionalLabels[key] = value; - continue; - } - } - - return additionalLabels; - } -} diff --git a/apps/kubernetes-provider/src/podCleaner.ts b/apps/kubernetes-provider/src/podCleaner.ts deleted file mode 100644 index 909df9b2416..00000000000 --- a/apps/kubernetes-provider/src/podCleaner.ts +++ /dev/null @@ -1,251 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; - -type PodCleanerOptions = { - runtimeEnv: "local" | "kubernetes"; - namespace?: string; - intervalInSeconds?: number; -}; - -export class PodCleaner { - private enabled = false; - private namespace = "default"; - private intervalInSeconds = 300; - - private logger = new SimpleLogger("[PodCleaner]"); - private k8sClient: { - core: k8s.CoreV1Api; - apps: k8s.AppsV1Api; - kubeConfig: k8s.KubeConfig; - }; - - constructor(private opts: PodCleanerOptions) { - if (opts.namespace) { - this.namespace = opts.namespace; - } - - if (opts.intervalInSeconds) { - this.intervalInSeconds = opts.intervalInSeconds; - } - - this.k8sClient = this.#createK8sClient(); - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - apps: kubeConfig.makeApiClient(k8s.AppsV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.logger.debug("K8s API Error", err); - } else { - this.logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - async #deletePods(opts: { - namespace: string; - dryRun?: boolean; - fieldSelector?: string; - labelSelector?: string; - }) { - return await this.k8sClient.core - .deleteCollectionNamespacedPod( - opts.namespace, - undefined, // pretty - undefined, // continue - opts.dryRun ? "All" : undefined, - opts.fieldSelector, - undefined, // gracePeriodSeconds - opts.labelSelector - ) - .catch(this.#handleK8sError.bind(this)); - } - - async #deleteDaemonSets(opts: { - namespace: string; - dryRun?: boolean; - fieldSelector?: string; - labelSelector?: string; - }) { - return await this.k8sClient.apps - .deleteCollectionNamespacedDaemonSet( - opts.namespace, - undefined, // pretty - undefined, // continue - opts.dryRun ? "All" : undefined, - opts.fieldSelector, - undefined, // gracePeriodSeconds - opts.labelSelector - ) - .catch(this.#handleK8sError.bind(this)); - } - - async #deleteCompletedRuns() { - this.logger.log("Deleting completed runs"); - - const start = Date.now(); - - const result = await this.#deletePods({ - namespace: this.namespace, - fieldSelector: "status.phase=Succeeded", - labelSelector: "app=task-run", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting completed runs: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting completed runs: Done", { total, elapsedMs }); - } - - async #deleteFailedRuns() { - this.logger.log("Deleting failed runs"); - - const start = Date.now(); - - const result = await this.#deletePods({ - namespace: this.namespace, - fieldSelector: "status.phase=Failed", - labelSelector: "app=task-run", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting failed runs: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting failed runs: Done", { total, elapsedMs }); - } - - async #deleteCompletedPrePulls() { - this.logger.log("Deleting completed pre-pulls"); - - const start = Date.now(); - - const result = await this.#deleteDaemonSets({ - namespace: this.namespace, - labelSelector: "app=task-prepull", - }); - - const elapsedMs = Date.now() - start; - - if (!result) { - this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs }); - return; - } - - const total = (result.response as any)?.body?.items?.length ?? 0; - - this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs }); - } - - async start() { - this.enabled = true; - this.logger.log("Starting"); - - const completedInterval = setInterval(async () => { - if (!this.enabled) { - clearInterval(completedInterval); - return; - } - - try { - await this.#deleteCompletedRuns(); - } catch (error) { - this.logger.error("Error deleting completed runs", error); - } - }, this.intervalInSeconds * 1000); - - const failedInterval = setInterval( - async () => { - if (!this.enabled) { - clearInterval(failedInterval); - return; - } - - try { - await this.#deleteFailedRuns(); - } catch (error) { - this.logger.error("Error deleting completed runs", error); - } - }, - // Use a longer interval for failed runs. This is only a backup in case the task monitor fails. - 2 * this.intervalInSeconds * 1000 - ); - - const completedPrePullInterval = setInterval( - async () => { - if (!this.enabled) { - clearInterval(completedPrePullInterval); - return; - } - - try { - await this.#deleteCompletedPrePulls(); - } catch (error) { - this.logger.error("Error deleting completed pre-pulls", error); - } - }, - 2 * this.intervalInSeconds * 1000 - ); - - // this.#launchTests(); - } - - async stop() { - if (!this.enabled) { - return; - } - - this.enabled = false; - this.logger.log("Shutting down.."); - } -} diff --git a/apps/kubernetes-provider/src/taskMonitor.ts b/apps/kubernetes-provider/src/taskMonitor.ts deleted file mode 100644 index 156052e37fb..00000000000 --- a/apps/kubernetes-provider/src/taskMonitor.ts +++ /dev/null @@ -1,385 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3"; -import { - EXIT_CODE_ALREADY_HANDLED, - EXIT_CODE_CHILD_NONZERO, - SimpleLogger, -} from "@trigger.dev/core/v3/apps"; -import PQueue from "p-queue"; -import { setTimeout } from "timers/promises"; - -type FailureDetails = Prettify<{ - exitCode: number; - reason: string; - logs: string; - overrideCompletion: boolean; - errorCode: TaskRunInternalError["code"]; -}>; - -type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise; - -type RunFailureHandler = (runId: string, details: FailureDetails) => Promise; - -type TaskMonitorOptions = { - runtimeEnv: "local" | "kubernetes"; - onIndexFailure?: IndexFailureHandler; - onRunFailure?: RunFailureHandler; - namespace?: string; -}; - -export class TaskMonitor { - #enabled = false; - - #logger = new SimpleLogger("[TaskMonitor]"); - #taskInformer: ReturnType>; - #processedPods = new Map(); - #queue = new PQueue({ concurrency: 10 }); - - #k8sClient: { - core: k8s.CoreV1Api; - kubeConfig: k8s.KubeConfig; - }; - - private namespace = "default"; - private fieldSelector = "status.phase=Failed"; - private labelSelector = "app in (task-index, task-run)"; - - constructor(private opts: TaskMonitorOptions) { - if (opts.namespace) { - this.namespace = opts.namespace; - } - - this.#k8sClient = this.#createK8sClient(); - - this.#taskInformer = this.#createTaskInformer(); - this.#taskInformer.on("connect", this.#onInformerConnected.bind(this)); - this.#taskInformer.on("error", this.#onInformerError.bind(this)); - this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this)); - } - - #createTaskInformer() { - const listTasks = () => - this.#k8sClient.core.listNamespacedPod( - this.namespace, - undefined, - undefined, - undefined, - this.fieldSelector, - this.labelSelector - ); - - // Uses watch with local caching - // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes - const informer = k8s.makeInformer( - this.#k8sClient.kubeConfig, - `/api/v1/namespaces/${this.namespace}/pods`, - listTasks, - this.labelSelector, - this.fieldSelector - ); - - return informer; - } - - async #onInformerConnected() { - this.#logger.log("Connected"); - } - - async #onInformerError(error: any) { - this.#logger.error("Error:", error); - - // Automatic reconnect - await setTimeout(2_000); - this.#taskInformer.start(); - } - - #enqueueOnPodUpdated(pod: k8s.V1Pod) { - this.#queue.add(async () => { - try { - // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it - await this.#onPodUpdated(pod); - } catch (error) { - this.#logger.error("Caught onPodUpdated() error:", error); - } - }); - } - - async #onPodUpdated(pod: k8s.V1Pod) { - this.#logger.debug(`Updated: ${pod.metadata?.name}`); - this.#logger.debug("Updated", JSON.stringify(pod, null, 2)); - - // We only care about failures - if (pod.status?.phase !== "Failed") { - return; - } - - const podName = pod.metadata?.name; - - if (!podName) { - this.#logger.error("Pod is nameless", { pod }); - return; - } - - const containerStatus = pod.status.containerStatuses?.[0]; - - if (!containerStatus?.state) { - this.#logger.error("Pod failed, but container status doesn't have state", { - status: pod.status, - }); - return; - } - - if (this.#processedPods.has(podName)) { - this.#logger.debug("Pod update already processed", { - podName, - timestamp: this.#processedPods.get(podName), - }); - return; - } - - this.#processedPods.set(podName, Date.now()); - - const podStatus = this.#getPodStatusSummary(pod.status); - const containerState = this.#getContainerStateSummary(containerStatus.state); - const exitCode = containerState.exitCode ?? -1; - - if (exitCode === EXIT_CODE_ALREADY_HANDLED) { - this.#logger.debug("Ignoring pod failure, already handled by worker", { - podName, - }); - return; - } - - const rawLogs = await this.#getLogTail(podName); - - this.#logger.log(`${podName} failed with:`, { - podStatus, - containerState, - rawLogs, - }); - - const rawReason = podStatus.reason ?? containerState.reason ?? ""; - const message = podStatus.message ?? containerState.message ?? ""; - - let reason = rawReason || "Unknown error"; - let logs = rawLogs || ""; - - /** This will only override existing task errors. It will not crash the run. */ - let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO; - - let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR; - - switch (rawReason) { - case "Error": - reason = "Unknown error."; - errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR; - break; - case "Evicted": - if (message.startsWith("Pod ephemeral local storage usage")) { - reason = "Storage limit exceeded."; - errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED; - } else if (message) { - reason = `Evicted: ${message}`; - errorCode = TaskRunErrorCodes.POD_EVICTED; - } else { - reason = "Evicted for unknown reason."; - errorCode = TaskRunErrorCodes.POD_EVICTED; - } - - if (logs.startsWith("failed to try resolving symlinks")) { - logs = ""; - } - break; - case "OOMKilled": - reason = - "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak."; - errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED; - break; - default: - break; - } - - const failureInfo = { - exitCode, - reason, - logs, - overrideCompletion: onlyOverrideExistingError, - errorCode, - } satisfies FailureDetails; - - const app = pod.metadata?.labels?.app; - - switch (app) { - case "task-index": - const deploymentId = pod.metadata?.labels?.deployment; - - if (!deploymentId) { - this.#logger.error("Index is missing ID", { pod }); - return; - } - - if (this.opts.onIndexFailure) { - await this.opts.onIndexFailure(deploymentId, failureInfo); - } - break; - case "task-run": - const runId = pod.metadata?.labels?.run; - - if (!runId) { - this.#logger.error("Run is missing ID", { pod }); - return; - } - - if (this.opts.onRunFailure) { - await this.opts.onRunFailure(runId, failureInfo); - } - break; - default: - this.#logger.error("Pod has invalid app label", { pod }); - return; - } - - await this.#deletePod(podName); - } - - async #getLogTail(podName: string) { - try { - const logs = await this.#k8sClient.core.readNamespacedPodLog( - podName, - this.namespace, - undefined, - undefined, - undefined, - 1024, // limitBytes - undefined, - undefined, - undefined, - 20 // tailLines - ); - - const responseBody = logs.body ?? ""; - - if (responseBody.startsWith("unable to retrieve container logs")) { - return ""; - } - - // Type is wrong, body may be undefined - return responseBody; - } catch (error) { - this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown"); - return ""; - } - } - - #getPodStatusSummary(status: k8s.V1PodStatus) { - return { - reason: status.reason, - message: status.message, - }; - } - - #getContainerStateSummary(state: k8s.V1ContainerState) { - return { - reason: state.terminated?.reason, - exitCode: state.terminated?.exitCode, - message: state.terminated?.message, - }; - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.#logger.debug("K8s API Error", err); - } else { - this.#logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - #printStats(includeMoreDetails = false) { - this.#logger.log("Stats:", { - cacheSize: this.#taskInformer.list().length, - totalProcessed: this.#processedPods.size, - ...(includeMoreDetails && { - processedPods: this.#processedPods, - }), - }); - } - - async #deletePod(name: string) { - this.#logger.debug("Deleting pod:", name); - - await this.#k8sClient.core - .deleteNamespacedPod(name, this.namespace) - .catch(this.#handleK8sError.bind(this)); - } - - async start() { - this.#enabled = true; - - const interval = setInterval(() => { - if (!this.#enabled) { - clearInterval(interval); - return; - } - - this.#printStats(); - }, 300_000); - - await this.#taskInformer.start(); - - // this.#launchTests(); - } - - async stop() { - if (!this.#enabled) { - return; - } - - this.#enabled = false; - this.#logger.log("Shutting down.."); - - await this.#taskInformer.stop(); - - this.#printStats(true); - } -} diff --git a/apps/kubernetes-provider/src/uptimeHeartbeat.ts b/apps/kubernetes-provider/src/uptimeHeartbeat.ts deleted file mode 100644 index 9ff63032f0b..00000000000 --- a/apps/kubernetes-provider/src/uptimeHeartbeat.ts +++ /dev/null @@ -1,272 +0,0 @@ -import * as k8s from "@kubernetes/client-node"; -import { SimpleLogger } from "@trigger.dev/core/v3/apps"; - -type UptimeHeartbeatOptions = { - runtimeEnv: "local" | "kubernetes"; - pingUrl: string; - namespace?: string; - intervalInSeconds?: number; - maxPendingRuns?: number; - maxPendingIndeces?: number; - maxPendingErrors?: number; - leadingEdge?: boolean; -}; - -export class UptimeHeartbeat { - private enabled = false; - private namespace: string; - - private intervalInSeconds: number; - private maxPendingRuns: number; - private maxPendingIndeces: number; - private maxPendingErrors: number; - - private leadingEdge = true; - - private logger = new SimpleLogger("[UptimeHeartbeat]"); - private k8sClient: { - core: k8s.CoreV1Api; - kubeConfig: k8s.KubeConfig; - }; - - constructor(private opts: UptimeHeartbeatOptions) { - this.namespace = opts.namespace ?? "default"; - - this.intervalInSeconds = opts.intervalInSeconds ?? 60; - this.maxPendingRuns = opts.maxPendingRuns ?? 25; - this.maxPendingIndeces = opts.maxPendingIndeces ?? 10; - this.maxPendingErrors = opts.maxPendingErrors ?? 10; - - this.k8sClient = this.#createK8sClient(); - } - - #createK8sClient() { - const kubeConfig = new k8s.KubeConfig(); - - if (this.opts.runtimeEnv === "local") { - kubeConfig.loadFromDefault(); - } else if (this.opts.runtimeEnv === "kubernetes") { - kubeConfig.loadFromCluster(); - } else { - throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`); - } - - return { - core: kubeConfig.makeApiClient(k8s.CoreV1Api), - kubeConfig: kubeConfig, - }; - } - - #isRecord(candidate: unknown): candidate is Record { - if (typeof candidate !== "object" || candidate === null) { - return false; - } else { - return true; - } - } - - #logK8sError(err: unknown, debugOnly = false) { - if (debugOnly) { - this.logger.debug("K8s API Error", err); - } else { - this.logger.error("K8s API Error", err); - } - } - - #handleK8sError(err: unknown) { - if (!this.#isRecord(err) || !this.#isRecord(err.body)) { - this.#logK8sError(err); - return; - } - - this.#logK8sError(err, true); - - if (typeof err.body.message === "string") { - this.#logK8sError({ message: err.body.message }); - return; - } - - this.#logK8sError({ body: err.body }); - } - - async #getPods(opts: { - namespace: string; - fieldSelector?: string; - labelSelector?: string; - }): Promise | undefined> { - const listReturn = await this.k8sClient.core - .listNamespacedPod( - opts.namespace, - undefined, // pretty - undefined, // allowWatchBookmarks - undefined, // _continue - opts.fieldSelector, - opts.labelSelector, - this.maxPendingRuns * 2, // limit - undefined, // resourceVersion - undefined, // resourceVersionMatch - undefined, // sendInitialEvents - this.intervalInSeconds, // timeoutSeconds, - undefined // watch - ) - .catch(this.#handleK8sError.bind(this)); - - return listReturn?.body.items; - } - - async #getPendingIndeces(): Promise | undefined> { - return await this.#getPods({ - namespace: this.namespace, - fieldSelector: "status.phase=Pending", - labelSelector: "app=task-index", - }); - } - - async #getPendingTasks(): Promise | undefined> { - return await this.#getPods({ - namespace: this.namespace, - fieldSelector: "status.phase=Pending", - labelSelector: "app=task-run", - }); - } - - #countPods(pods: Array): number { - return pods.length; - } - - #filterPendingPods( - pods: Array, - waitingReason: "CreateContainerError" | "RunContainerError" - ): Array { - return pods.filter((pod) => { - const containerStatus = pod.status?.containerStatuses?.[0]; - return containerStatus?.state?.waiting?.reason === waitingReason; - }); - } - - async #sendPing() { - this.logger.log("Sending ping"); - - const start = Date.now(); - const controller = new AbortController(); - - const timeoutMs = (this.intervalInSeconds * 1000) / 2; - - const fetchTimeout = setTimeout(() => { - controller.abort(); - }, timeoutMs); - - try { - const response = await fetch(this.opts.pingUrl, { - signal: controller.signal, - }); - - if (!response.ok) { - this.logger.error("Failed to send ping, response not OK", { - status: response.status, - }); - return; - } - - const elapsedMs = Date.now() - start; - this.logger.log("Ping sent", { elapsedMs }); - } catch (error) { - if (error instanceof DOMException && error.name === "AbortError") { - this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs }); - return; - } - - this.logger.error("Failed to send ping", error); - } finally { - clearTimeout(fetchTimeout); - } - } - - async #heartbeat() { - this.logger.log("Performing heartbeat"); - - const start = Date.now(); - - const pendingTasks = await this.#getPendingTasks(); - - if (!pendingTasks) { - this.logger.error("Failed to get pending tasks"); - return; - } - - const totalPendingTasks = this.#countPods(pendingTasks); - - const pendingIndeces = await this.#getPendingIndeces(); - - if (!pendingIndeces) { - this.logger.error("Failed to get pending indeces"); - return; - } - - const totalPendingIndeces = this.#countPods(pendingIndeces); - - const elapsedMs = Date.now() - start; - - this.logger.log("Finished heartbeat checks", { elapsedMs }); - - if (totalPendingTasks > this.maxPendingRuns) { - this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks }); - return; - } - - if (totalPendingIndeces > this.maxPendingIndeces) { - this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces }); - return; - } - - const totalCreateContainerErrors = this.#countPods( - this.#filterPendingPods(pendingTasks, "CreateContainerError") - ); - const totalRunContainerErrors = this.#countPods( - this.#filterPendingPods(pendingTasks, "RunContainerError") - ); - - if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) { - this.logger.log("Too many pending tasks with errors, skipping heartbeat", { - totalRunContainerErrors, - totalCreateContainerErrors, - }); - return; - } - - await this.#sendPing(); - - this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs }); - } - - async start() { - this.enabled = true; - this.logger.log("Starting"); - - if (this.leadingEdge) { - await this.#heartbeat(); - } - - const heartbeat = setInterval(async () => { - if (!this.enabled) { - clearInterval(heartbeat); - return; - } - - try { - await this.#heartbeat(); - } catch (error) { - this.logger.error("Error while heartbeating", error); - } - }, this.intervalInSeconds * 1000); - } - - async stop() { - if (!this.enabled) { - return; - } - - this.enabled = false; - this.logger.log("Shutting down.."); - } -} diff --git a/apps/kubernetes-provider/tsconfig.json b/apps/kubernetes-provider/tsconfig.json deleted file mode 100644 index 6ec7865b64e..00000000000 --- a/apps/kubernetes-provider/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2020", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "strict": true, - "skipLibCheck": true, - "paths": { - "@trigger.dev/core": ["../../packages/core/src"], - "@trigger.dev/core/*": ["../../packages/core/src/*"], - "@trigger.dev/core/v3": ["../../packages/core/src/v3"], - "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] - } - } -} diff --git a/packages/core/src/v3/apps/http.ts b/packages/core/src/v3/apps/http.ts index 720dccbf1b0..51c6522f13c 100644 --- a/packages/core/src/v3/apps/http.ts +++ b/packages/core/src/v3/apps/http.ts @@ -1,19 +1,5 @@ import type { IncomingMessage, RequestListener } from "node:http"; -export const getTextBody = (req: IncomingMessage) => - new Promise((resolve) => { - let body = ""; - req.on("readable", () => { - const chunk = req.read(); - if (chunk) { - body += chunk; - } - }); - req.on("end", () => { - resolve(body); - }); - }); - export async function getJsonBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { let body = ""; diff --git a/packages/core/src/v3/apps/index.ts b/packages/core/src/v3/apps/index.ts index 80cc2d5a916..d1d20785bf2 100644 --- a/packages/core/src/v3/apps/index.ts +++ b/packages/core/src/v3/apps/index.ts @@ -1,7 +1,3 @@ export * from "./backoff.js"; -export * from "./logger.js"; -export * from "./process.js"; export * from "./http.js"; -export * from "./provider.js"; -export * from "./isExecaChildProcess.js"; export * from "./exec.js"; diff --git a/packages/core/src/v3/apps/isExecaChildProcess.ts b/packages/core/src/v3/apps/isExecaChildProcess.ts deleted file mode 100644 index 559705f5de6..00000000000 --- a/packages/core/src/v3/apps/isExecaChildProcess.ts +++ /dev/null @@ -1,6 +0,0 @@ -// @ts-ignore -import type { ExecaChildProcess } from "execa"; - -export function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited { - return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca; -} diff --git a/packages/core/src/v3/apps/logger.ts b/packages/core/src/v3/apps/logger.ts deleted file mode 100644 index 7e1e99feb50..00000000000 --- a/packages/core/src/v3/apps/logger.ts +++ /dev/null @@ -1,35 +0,0 @@ -export class SimpleLogger { - #debugEnabled = ["1", "true"].includes(process.env.DEBUG ?? ""); - - constructor(private prefix?: string) {} - - log(arg0: TFirstArg, ...argN: any[]) { - console.log(...this.#getPrefixedArgs(arg0, ...argN)); - - return arg0; - } - - debug(arg0: TFirstArg, ...argN: any[]) { - if (!this.#debugEnabled) { - return arg0; - } - - console.debug(...this.#getPrefixedArgs("DEBUG", arg0, ...argN)); - - return arg0; - } - - error(arg0: TFirstArg, ...argN: any[]) { - console.error(...this.#getPrefixedArgs(arg0, ...argN)); - - return arg0; - } - - #getPrefixedArgs(...args: any[]) { - if (!this.prefix) { - return args; - } - - return [this.prefix, ...args]; - } -} diff --git a/packages/core/src/v3/apps/process.ts b/packages/core/src/v3/apps/process.ts deleted file mode 100644 index 1c90eb86eaa..00000000000 --- a/packages/core/src/v3/apps/process.ts +++ /dev/null @@ -1,4 +0,0 @@ -/** This was used by the old build system in case of indexing failures */ -export const EXIT_CODE_ALREADY_HANDLED = 111; -/** This means what it says and is only set once we have completed the attempt */ -export const EXIT_CODE_CHILD_NONZERO = 112; diff --git a/packages/core/src/v3/apps/provider.ts b/packages/core/src/v3/apps/provider.ts deleted file mode 100644 index 8705f63941e..00000000000 --- a/packages/core/src/v3/apps/provider.ts +++ /dev/null @@ -1,382 +0,0 @@ -import { createServer } from "node:http"; -import { getRandomPortNumber, HttpReply, getTextBody } from "./http.js"; -import { SimpleLogger } from "./logger.js"; -import { isExecaChildProcess } from "./isExecaChildProcess.js"; -import { setTimeout } from "node:timers/promises"; -import { EXIT_CODE_ALREADY_HANDLED } from "./process.js"; -import type { EnvironmentType } from "../schemas/schemas.js"; -import type { MachinePreset } from "../schemas/common.js"; -import { - ProviderToPlatformMessages, - PlatformToProviderMessages, - ClientToSharedQueueMessages, - SharedQueueToClientMessages, - clientWebsocketMessages, -} from "../schemas/messages.js"; -import { ZodMessageSender } from "../zodMessageHandler.js"; -import { ZodSocketConnection } from "../zodSocket.js"; - -const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || getRandomPortNumber()); -const MACHINE_NAME = process.env.MACHINE_NAME || "local"; - -const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1"; -const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030; -const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret"; -const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false"); - -const logger = new SimpleLogger(`[${MACHINE_NAME}]`); - -export interface TaskOperationsIndexOptions { - shortCode: string; - imageRef: string; - apiKey: string; - apiUrl: string; - // identifiers - envId: string; - envType: EnvironmentType; - orgId: string; - projectId: string; - deploymentId: string; -} - -export interface TaskOperationsCreateOptions { - image: string; - machine: MachinePreset; - version: string; - nextAttemptNumber?: number; - // identifiers - envId: string; - envType: EnvironmentType; - orgId: string; - projectId: string; - runId: string; - dequeuedAt?: number; -} - -export interface TaskOperationsRestoreOptions { - imageRef: string; - checkpointRef: string; - machine: MachinePreset; - attemptNumber?: number; - // identifiers - envId: string; - envType: EnvironmentType; - orgId: string; - projectId: string; - runId: string; - checkpointId: string; -} - -export interface TaskOperationsPrePullDeploymentOptions { - shortCode: string; - imageRef: string; - // identifiers - envId: string; - envType: EnvironmentType; - orgId: string; - projectId: string; - deploymentId: string; -} - -export interface TaskOperations { - init: () => Promise; - - // CRUD - index: (opts: TaskOperationsIndexOptions) => Promise; - create: (opts: TaskOperationsCreateOptions) => Promise; - restore: (opts: TaskOperationsRestoreOptions) => Promise; - - // unimplemented - delete?: (...args: any[]) => Promise; - get?: (...args: any[]) => Promise; - - prePullDeployment?: (opts: TaskOperationsPrePullDeploymentOptions) => Promise; -} - -type ProviderShellOptions = { - tasks: TaskOperations; - type: "docker" | "kubernetes"; - host?: string; - port?: number; -}; - -interface Provider { - tasks: TaskOperations; -} - -export class ProviderShell implements Provider { - tasks: TaskOperations; - - #httpPort: number; - #httpServer: ReturnType; - platformSocket: ZodSocketConnection< - typeof ProviderToPlatformMessages, - typeof PlatformToProviderMessages - >; - - constructor(private options: ProviderShellOptions) { - this.tasks = options.tasks; - this.#httpPort = options.port ?? HTTP_SERVER_PORT; - this.#httpServer = this.#createHttpServer(); - this.platformSocket = this.#createPlatformSocket(); - this.#createSharedQueueSocket(); - } - - #createSharedQueueSocket() { - const sharedQueueConnection = new ZodSocketConnection({ - namespace: "shared-queue", - host: PLATFORM_HOST, - port: Number(PLATFORM_WS_PORT), - secure: SECURE_CONNECTION, - clientMessages: ClientToSharedQueueMessages, - serverMessages: SharedQueueToClientMessages, - authToken: PLATFORM_SECRET, - handlers: { - SERVER_READY: async (message) => { - // TODO: create new schema without worker requirement - await sender.send("READY_FOR_TASKS", { - backgroundWorkerId: "placeholder", - }); - }, - BACKGROUND_WORKER_MESSAGE: async (message) => { - if (message.data.type === "SCHEDULE_ATTEMPT") { - try { - await this.tasks.create({ - image: message.data.image, - machine: message.data.machine, - version: message.data.version, - nextAttemptNumber: message.data.nextAttemptNumber, - // identifiers - envId: message.data.envId, - envType: message.data.envType, - orgId: message.data.orgId, - projectId: message.data.projectId, - runId: message.data.runId, - dequeuedAt: message.data.dequeuedAt, - }); - } catch (error) { - logger.error("create failed", error); - } - } - }, - }, - }); - - const sender = new ZodMessageSender({ - schema: clientWebsocketMessages, - sender: async (message) => { - return new Promise((resolve, reject) => { - try { - const { type, ...payload } = message; - sharedQueueConnection.socket.emit(type, payload as any); - resolve(); - } catch (err) { - reject(err); - } - }); - }, - }); - - return sharedQueueConnection; - } - - #createPlatformSocket() { - const platformConnection = new ZodSocketConnection({ - namespace: "provider", - host: PLATFORM_HOST, - port: Number(PLATFORM_WS_PORT), - secure: SECURE_CONNECTION, - clientMessages: ProviderToPlatformMessages, - serverMessages: PlatformToProviderMessages, - authToken: PLATFORM_SECRET, - extraHeaders: { - "x-trigger-provider-type": this.options.type, - }, - handlers: { - INDEX: async (message) => { - try { - await this.tasks.index({ - shortCode: message.shortCode, - imageRef: message.imageTag, - apiKey: message.apiKey, - apiUrl: message.apiUrl, - // identifiers - envId: message.envId, - envType: message.envType, - orgId: message.orgId, - projectId: message.projectId, - deploymentId: message.deploymentId, - }); - } catch (error) { - if (isExecaChildProcess(error)) { - logger.error("Index failed", { - socketMessage: message, - exitCode: error.exitCode, - escapedCommand: error.escapedCommand, - stdout: error.stdout, - stderr: error.stderr, - }); - - if (error.exitCode === EXIT_CODE_ALREADY_HANDLED) { - logger.error("Index failure already reported by the worker", { - socketMessage: message, - }); - - // Add a brief delay to avoid messaging race conditions - await setTimeout(2000); - } - - function normalizeStderr(stderr: string) { - return stderr - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .join("\n"); - } - - return { - success: false, - error: { - name: "Index error", - message: `Crashed with exit code ${error.exitCode}`, - stderr: normalizeStderr(error.stderr), - }, - }; - } else { - logger.error("Index failed", error); - } - - if (error instanceof Error) { - return { - success: false, - error: { - name: "Provider error", - message: error.message, - stack: error.stack, - }, - }; - } else { - return { - success: false, - error: { - name: "Provider error", - message: "Unknown error", - }, - }; - } - } - - return { - success: true, - }; - }, - RESTORE: async (message) => { - if (message.type.toLowerCase() !== this.options.type.toLowerCase()) { - logger.error( - `restore failed: ${this.options.type} provider can't restore ${message.type} checkpoints` - ); - return; - } - - try { - await this.tasks.restore({ - checkpointRef: message.location, - machine: message.machine, - imageRef: message.imageRef, - attemptNumber: message.attemptNumber, - // identifiers - envId: message.envId, - envType: message.envType, - orgId: message.orgId, - projectId: message.projectId, - runId: message.runId, - checkpointId: message.checkpointId, - }); - } catch (error) { - logger.error("restore failed", error); - } - }, - PRE_PULL_DEPLOYMENT: async (message) => { - if (!this.tasks.prePullDeployment) { - logger.debug("prePullDeployment not implemented", message); - return; - } - - try { - await this.tasks.prePullDeployment({ - shortCode: message.shortCode, - imageRef: message.imageRef, - // identifiers - envId: message.envId, - envType: message.envType, - orgId: message.orgId, - projectId: message.projectId, - deploymentId: message.deploymentId, - }); - } catch (error) { - logger.error("prePullDeployment failed", error); - } - }, - }, - }); - - return platformConnection; - } - - #createHttpServer() { - const httpServer = createServer(async (req, res) => { - logger.log(`[${req.method}]`, req.url); - - const reply = new HttpReply(res); - - try { - const url = new URL(req.url ?? "", `http://${req.headers.host}`); - - switch (url.pathname) { - case "/health": { - return reply.text("ok"); - } - case "/whoami": { - return reply.text(`${MACHINE_NAME}`); - } - case "/close": { - this.platformSocket.close(); - return reply.text("platform socket closed"); - } - case "/delete": { - const body = await getTextBody(req); - - if (this.tasks.delete) { - await this.tasks.delete({ runId: body }); - return reply.text(`sent delete request: ${body}`); - } else { - return reply.text("delete not implemented", 501); - } - } - default: { - return reply.empty(404); - } - } - } catch (error) { - logger.error("HTTP server error", { error }); - reply.empty(500); - return; - } - }); - - httpServer.on("clientError", (err, socket) => { - socket.end("HTTP/1.1 400 Bad Request\r\n\r\n"); - }); - - httpServer.on("listening", () => { - logger.log("server listening on port", this.#httpPort); - }); - - return httpServer; - } - - async listen() { - this.#httpServer.listen(this.#httpPort, this.options.host ?? "0.0.0.0"); - await this.tasks.init(); - } -} diff --git a/packages/core/src/v3/serverOnly/checkpointTest.ts b/packages/core/src/v3/serverOnly/checkpointTest.ts deleted file mode 100644 index 4778b07bcd0..00000000000 --- a/packages/core/src/v3/serverOnly/checkpointTest.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { isExecaChildProcess } from "../apps/isExecaChildProcess.js"; - -export type CheckpointTestResult = - | { - ok: true; - } - | { - ok: false; - message: string; - error?: unknown; - }; - -export async function testDockerCheckpoint(): Promise { - const { $ } = await import("execa"); - - try { - // Create a dummy container - const container = - await $`docker run -d --rm --name init-dummy-${randomUUID()} docker.io/library/busybox sleep 10`; - - // Checkpoint it - await $`docker checkpoint create ${container} init-check`; - } catch (error) { - if (!isExecaChildProcess(error)) { - return { - ok: false, - message: "No checkpoint support: Unknown error.", - error, - }; - } - - if (error.stderr.includes("criu")) { - if (error.stderr.includes("executable file not found")) { - return { - ok: false, - message: "No checkpoint support: Missing CRIU binary.", - }; - } - - return { - ok: false, - message: "No checkpoint support: Unknown CRIU error.", - error, - }; - } - - if (error.stderr.includes("experimental features enabled")) { - return { - ok: false, - message: "No checkpoint support: Please enable docker experimental features.", - }; - } - - return { - ok: false, - message: "No checkpoint support: Unknown execa error.", - error, - }; - } - - return { - ok: true, - }; -} diff --git a/packages/core/src/v3/serverOnly/index.ts b/packages/core/src/v3/serverOnly/index.ts index 8f56f0e3307..8657ec17613 100644 --- a/packages/core/src/v3/serverOnly/index.ts +++ b/packages/core/src/v3/serverOnly/index.ts @@ -1,5 +1,4 @@ export * from "./checkpointClient.js"; -export * from "./checkpointTest.js"; export * from "./httpServer.js"; export * from "./singleton.js"; export * from "./shutdownManager.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 717ae7008aa..798444abf17 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,75 +146,6 @@ importers: specifier: 4.1.7 version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@22.20.0)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@22.20.0)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@3.12.2)(yaml@2.9.0)) - apps/coordinator: - dependencies: - '@trigger.dev/core': - specifier: workspace:* - version: link:../../packages/core - nanoid: - specifier: ^5.0.6 - version: 5.0.6 - prom-client: - specifier: ^15.1.0 - version: 15.1.0 - socket.io: - specifier: 4.7.4 - version: 4.7.4(bufferutil@4.0.9) - tinyexec: - specifier: ^0.3.0 - version: 0.3.0 - devDependencies: - dotenv: - specifier: ^16.4.2 - version: 16.4.4 - esbuild: - specifier: ^0.19.11 - version: 0.19.11 - tsx: - specifier: ^4.7.0 - version: 4.7.1 - - apps/docker-provider: - dependencies: - '@trigger.dev/core': - specifier: workspace:* - version: link:../../packages/core - execa: - specifier: ^8.0.1 - version: 8.0.1 - devDependencies: - dotenv: - specifier: ^16.4.2 - version: 16.4.4 - esbuild: - specifier: ^0.19.11 - version: 0.19.11 - tsx: - specifier: ^4.7.0 - version: 4.7.1 - - apps/kubernetes-provider: - dependencies: - '@kubernetes/client-node': - specifier: ^0.20.0 - version: 0.20.0(bufferutil@4.0.9) - '@trigger.dev/core': - specifier: workspace:* - version: link:../../packages/core - p-queue: - specifier: ^8.0.1 - version: 8.0.1 - devDependencies: - dotenv: - specifier: ^16.4.2 - version: 16.4.4 - esbuild: - specifier: ^0.19.11 - version: 0.19.11 - tsx: - specifier: ^4.7.0 - version: 4.7.1 - apps/supervisor: dependencies: '@aws-sdk/client-ecr': @@ -3537,12 +3468,6 @@ packages: '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - '@esbuild/aix-ppc64@0.19.11': - resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - '@esbuild/aix-ppc64@0.23.0': resolution: {integrity: sha512-3sG8Zwa5fMcA9bgqB8AfWPQ+HFke6uD3h1s3RIwUNK8EG7a4buxvuFTs3j1IMs2NXAk9F30C/FF4vxRgQCcmoQ==} engines: {node: '>=18'} @@ -3585,12 +3510,6 @@ packages: cpu: [arm64] os: [android] - '@esbuild/android-arm64@0.19.11': - resolution: {integrity: sha512-aiu7K/5JnLj//KOnOfEZ0D90obUkRzDMyqd/wNAUQ34m4YUPVhRZpnqKV9uqDGxT7cToSDnIHsGooyIczu9T+Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - '@esbuild/android-arm64@0.23.0': resolution: {integrity: sha512-EuHFUYkAVfU4qBdyivULuu03FhJO4IJN9PGuABGrFy4vUuzk91P2d+npxHcFdpUnfYKy0PuV+n6bKIpHOB3prQ==} engines: {node: '>=18'} @@ -3639,12 +3558,6 @@ packages: cpu: [arm] os: [android] - '@esbuild/android-arm@0.19.11': - resolution: {integrity: sha512-5OVapq0ClabvKvQ58Bws8+wkLCV+Rxg7tUVbo9xu034Nm536QTII4YzhaFriQ7rMrorfnFKUsArD2lqKbFY4vw==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - '@esbuild/android-arm@0.23.0': resolution: {integrity: sha512-+KuOHTKKyIKgEEqKbGTK8W7mPp+hKinbMBeEnNzjJGyFcWsfrXjSTNluJHCY1RqhxFurdD8uNXQDei7qDlR6+g==} engines: {node: '>=18'} @@ -3687,12 +3600,6 @@ packages: cpu: [x64] os: [android] - '@esbuild/android-x64@0.19.11': - resolution: {integrity: sha512-eccxjlfGw43WYoY9QgB82SgGgDbibcqyDTlk3l3C0jOVHKxrjdc9CTwDUQd0vkvYg5um0OH+GpxYvp39r+IPOg==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - '@esbuild/android-x64@0.23.0': resolution: {integrity: sha512-WRrmKidLoKDl56LsbBMhzTTBxrsVwTKdNbKDalbEZr0tcsBgCLbEtoNthOW6PX942YiYq8HzEnb4yWQMLQuipQ==} engines: {node: '>=18'} @@ -3735,12 +3642,6 @@ packages: cpu: [arm64] os: [darwin] - '@esbuild/darwin-arm64@0.19.11': - resolution: {integrity: sha512-ETp87DRWuSt9KdDVkqSoKoLFHYTrkyz2+65fj9nfXsaV3bMhTCjtQfw3y+um88vGRKRiF7erPrh/ZuIdLUIVxQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - '@esbuild/darwin-arm64@0.23.0': resolution: {integrity: sha512-YLntie/IdS31H54Ogdn+v50NuoWF5BDkEUFpiOChVa9UnKpftgwzZRrI4J132ETIi+D8n6xh9IviFV3eXdxfow==} engines: {node: '>=18'} @@ -3783,12 +3684,6 @@ packages: cpu: [x64] os: [darwin] - '@esbuild/darwin-x64@0.19.11': - resolution: {integrity: sha512-fkFUiS6IUK9WYUO/+22omwetaSNl5/A8giXvQlcinLIjVkxwTLSktbF5f/kJMftM2MJp9+fXqZ5ezS7+SALp4g==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - '@esbuild/darwin-x64@0.23.0': resolution: {integrity: sha512-IMQ6eme4AfznElesHUPDZ+teuGwoRmVuuixu7sv92ZkdQcPbsNHzutd+rAfaBKo8YK3IrBEi9SLLKWJdEvJniQ==} engines: {node: '>=18'} @@ -3831,12 +3726,6 @@ packages: cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-arm64@0.19.11': - resolution: {integrity: sha512-lhoSp5K6bxKRNdXUtHoNc5HhbXVCS8V0iZmDvyWvYq9S5WSfTIHU2UGjcGt7UeS6iEYp9eeymIl5mJBn0yiuxA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - '@esbuild/freebsd-arm64@0.23.0': resolution: {integrity: sha512-0muYWCng5vqaxobq6LB3YNtevDFSAZGlgtLoAc81PjUfiFz36n4KMpwhtAd4he8ToSI3TGyuhyx5xmiWNYZFyw==} engines: {node: '>=18'} @@ -3879,12 +3768,6 @@ packages: cpu: [x64] os: [freebsd] - '@esbuild/freebsd-x64@0.19.11': - resolution: {integrity: sha512-JkUqn44AffGXitVI6/AbQdoYAq0TEullFdqcMY/PCUZ36xJ9ZJRtQabzMA+Vi7r78+25ZIBosLTOKnUXBSi1Kw==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - '@esbuild/freebsd-x64@0.23.0': resolution: {integrity: sha512-XKDVu8IsD0/q3foBzsXGt/KjD/yTKBCIwOHE1XwiXmrRwrX6Hbnd5Eqn/WvDekddK21tfszBSrE/WMaZh+1buQ==} engines: {node: '>=18'} @@ -3927,12 +3810,6 @@ packages: cpu: [arm64] os: [linux] - '@esbuild/linux-arm64@0.19.11': - resolution: {integrity: sha512-LneLg3ypEeveBSMuoa0kwMpCGmpu8XQUh+mL8XXwoYZ6Be2qBnVtcDI5azSvh7vioMDhoJFZzp9GWp9IWpYoUg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - '@esbuild/linux-arm64@0.23.0': resolution: {integrity: sha512-j1t5iG8jE7BhonbsEg5d9qOYcVZv/Rv6tghaXM/Ug9xahM0nX/H2gfu6X6z11QRTMT6+aywOMA8TDkhPo8aCGw==} engines: {node: '>=18'} @@ -3975,12 +3852,6 @@ packages: cpu: [arm] os: [linux] - '@esbuild/linux-arm@0.19.11': - resolution: {integrity: sha512-3CRkr9+vCV2XJbjwgzjPtO8T0SZUmRZla+UL1jw+XqHZPkPgZiyWvbDvl9rqAN8Zl7qJF0O/9ycMtjU67HN9/Q==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - '@esbuild/linux-arm@0.23.0': resolution: {integrity: sha512-SEELSTEtOFu5LPykzA395Mc+54RMg1EUgXP+iw2SJ72+ooMwVsgfuwXo5Fn0wXNgWZsTVHwY2cg4Vi/bOD88qw==} engines: {node: '>=18'} @@ -4023,12 +3894,6 @@ packages: cpu: [ia32] os: [linux] - '@esbuild/linux-ia32@0.19.11': - resolution: {integrity: sha512-caHy++CsD8Bgq2V5CodbJjFPEiDPq8JJmBdeyZ8GWVQMjRD0sU548nNdwPNvKjVpamYYVL40AORekgfIubwHoA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - '@esbuild/linux-ia32@0.23.0': resolution: {integrity: sha512-P7O5Tkh2NbgIm2R6x1zGJJsnacDzTFcRWZyTTMgFdVit6E98LTxO+v8LCCLWRvPrjdzXHx9FEOA8oAZPyApWUA==} engines: {node: '>=18'} @@ -4077,12 +3942,6 @@ packages: cpu: [loong64] os: [linux] - '@esbuild/linux-loong64@0.19.11': - resolution: {integrity: sha512-ppZSSLVpPrwHccvC6nQVZaSHlFsvCQyjnvirnVjbKSHuE5N24Yl8F3UwYUUR1UEPaFObGD2tSvVKbvR+uT1Nrg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - '@esbuild/linux-loong64@0.23.0': resolution: {integrity: sha512-InQwepswq6urikQiIC/kkx412fqUZudBO4SYKu0N+tGhXRWUqAx+Q+341tFV6QdBifpjYgUndV1hhMq3WeJi7A==} engines: {node: '>=18'} @@ -4125,12 +3984,6 @@ packages: cpu: [mips64el] os: [linux] - '@esbuild/linux-mips64el@0.19.11': - resolution: {integrity: sha512-B5x9j0OgjG+v1dF2DkH34lr+7Gmv0kzX6/V0afF41FkPMMqaQ77pH7CrhWeR22aEeHKaeZVtZ6yFwlxOKPVFyg==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - '@esbuild/linux-mips64el@0.23.0': resolution: {integrity: sha512-J9rflLtqdYrxHv2FqXE2i1ELgNjT+JFURt/uDMoPQLcjWQA5wDKgQA4t/dTqGa88ZVECKaD0TctwsUfHbVoi4w==} engines: {node: '>=18'} @@ -4173,12 +4026,6 @@ packages: cpu: [ppc64] os: [linux] - '@esbuild/linux-ppc64@0.19.11': - resolution: {integrity: sha512-MHrZYLeCG8vXblMetWyttkdVRjQlQUb/oMgBNurVEnhj4YWOr4G5lmBfZjHYQHHN0g6yDmCAQRR8MUHldvvRDA==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - '@esbuild/linux-ppc64@0.23.0': resolution: {integrity: sha512-cShCXtEOVc5GxU0fM+dsFD10qZ5UpcQ8AM22bYj0u/yaAykWnqXJDpd77ublcX6vdDsWLuweeuSNZk4yUxZwtw==} engines: {node: '>=18'} @@ -4221,12 +4068,6 @@ packages: cpu: [riscv64] os: [linux] - '@esbuild/linux-riscv64@0.19.11': - resolution: {integrity: sha512-f3DY++t94uVg141dozDu4CCUkYW+09rWtaWfnb3bqe4w5NqmZd6nPVBm+qbz7WaHZCoqXqHz5p6CM6qv3qnSSQ==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - '@esbuild/linux-riscv64@0.23.0': resolution: {integrity: sha512-HEtaN7Y5UB4tZPeQmgz/UhzoEyYftbMXrBCUjINGjh3uil+rB/QzzpMshz3cNUxqXN7Vr93zzVtpIDL99t9aRw==} engines: {node: '>=18'} @@ -4269,12 +4110,6 @@ packages: cpu: [s390x] os: [linux] - '@esbuild/linux-s390x@0.19.11': - resolution: {integrity: sha512-A5xdUoyWJHMMlcSMcPGVLzYzpcY8QP1RtYzX5/bS4dvjBGVxdhuiYyFwp7z74ocV7WDc0n1harxmpq2ePOjI0Q==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - '@esbuild/linux-s390x@0.23.0': resolution: {integrity: sha512-WDi3+NVAuyjg/Wxi+o5KPqRbZY0QhI9TjrEEm+8dmpY9Xir8+HE/HNx2JoLckhKbFopW0RdO2D72w8trZOV+Wg==} engines: {node: '>=18'} @@ -4317,12 +4152,6 @@ packages: cpu: [x64] os: [linux] - '@esbuild/linux-x64@0.19.11': - resolution: {integrity: sha512-grbyMlVCvJSfxFQUndw5mCtWs5LO1gUlwP4CDi4iJBbVpZcqLVT29FxgGuBJGSzyOxotFG4LoO5X+M1350zmPA==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - '@esbuild/linux-x64@0.23.0': resolution: {integrity: sha512-a3pMQhUEJkITgAw6e0bWA+F+vFtCciMjW/LPtoj99MhVt+Mfb6bbL9hu2wmTZgNd994qTAEw+U/r6k3qHWWaOQ==} engines: {node: '>=18'} @@ -4389,12 +4218,6 @@ packages: cpu: [x64] os: [netbsd] - '@esbuild/netbsd-x64@0.19.11': - resolution: {integrity: sha512-13jvrQZJc3P230OhU8xgwUnDeuC/9egsjTkXN49b3GcS5BKvJqZn86aGM8W9pd14Kd+u7HuFBMVtrNGhh6fHEQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - '@esbuild/netbsd-x64@0.23.0': resolution: {integrity: sha512-cRK+YDem7lFTs2Q5nEv/HHc4LnrfBCbH5+JHu6wm2eP+d8OZNoSMYgPZJq78vqQ9g+9+nMuIsAO7skzphRXHyw==} engines: {node: '>=18'} @@ -4467,12 +4290,6 @@ packages: cpu: [x64] os: [openbsd] - '@esbuild/openbsd-x64@0.19.11': - resolution: {integrity: sha512-ysyOGZuTp6SNKPE11INDUeFVVQFrhcNDVUgSQVDzqsqX38DjhPEPATpid04LCoUr2WXhQTEZ8ct/EgJCUDpyNw==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - '@esbuild/openbsd-x64@0.23.0': resolution: {integrity: sha512-6p3nHpby0DM/v15IFKMjAaayFhqnXV52aEmv1whZHX56pdkK+MEaLoQWj+H42ssFarP1PcomVhbsR4pkz09qBg==} engines: {node: '>=18'} @@ -4527,12 +4344,6 @@ packages: cpu: [x64] os: [sunos] - '@esbuild/sunos-x64@0.19.11': - resolution: {integrity: sha512-Hf+Sad9nVwvtxy4DXCZQqLpgmRTQqyFyhT3bZ4F2XlJCjxGmRFF0Shwn9rzhOYRB61w9VMXUkxlBy56dk9JJiQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - '@esbuild/sunos-x64@0.23.0': resolution: {integrity: sha512-BFelBGfrBwk6LVrmFzCq1u1dZbG4zy/Kp93w2+y83Q5UGYF1d8sCzeLI9NXjKyujjBBniQa8R8PzLFAUrSM9OA==} engines: {node: '>=18'} @@ -4575,12 +4386,6 @@ packages: cpu: [arm64] os: [win32] - '@esbuild/win32-arm64@0.19.11': - resolution: {integrity: sha512-0P58Sbi0LctOMOQbpEOvOL44Ne0sqbS0XWHMvvrg6NE5jQ1xguCSSw9jQeUk2lfrXYsKDdOe6K+oZiwKPilYPQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - '@esbuild/win32-arm64@0.23.0': resolution: {integrity: sha512-lY6AC8p4Cnb7xYHuIxQ6iYPe6MfO2CC43XXKo9nBXDb35krYt7KGhQnOkRGar5psxYkircpCqfbNDB4uJbS2jQ==} engines: {node: '>=18'} @@ -4623,12 +4428,6 @@ packages: cpu: [ia32] os: [win32] - '@esbuild/win32-ia32@0.19.11': - resolution: {integrity: sha512-6YOrWS+sDJDmshdBIQU+Uoyh7pQKrdykdefC1avn76ss5c+RN6gut3LZA4E2cH5xUEp5/cA0+YxRaVtRAb0xBg==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - '@esbuild/win32-ia32@0.23.0': resolution: {integrity: sha512-7L1bHlOTcO4ByvI7OXVI5pNN6HSu6pUQq9yodga8izeuB1KcT2UkHaH6118QJwopExPn0rMHIseCTx1CRo/uNA==} engines: {node: '>=18'} @@ -4671,12 +4470,6 @@ packages: cpu: [x64] os: [win32] - '@esbuild/win32-x64@0.19.11': - resolution: {integrity: sha512-vfkhltrjCAb603XaFhqhAF4LGDi2M4OrCRrFusyQ+iTLQ/o60QQXxc9cZC/FFpihBI9N1Grn6SMKVJ4KP7Fuiw==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - '@esbuild/win32-x64@0.23.0': resolution: {integrity: sha512-Arm+WgUFLUATuoxCJcahGuk6Yj9Pzxd6l11Zb/2aAuv5kWWvvfhLFo2fni4uSK5vzlUdCGZ/BdV5tH8klj8p8g==} engines: {node: '>=18'} @@ -4977,9 +4770,6 @@ packages: react: 18.3.1 react-dom: 18.3.1 - '@kubernetes/client-node@0.20.0': - resolution: {integrity: sha512-xxlv5GLX4FVR/dDKEsmi4SPeuB49aRc35stndyxcC73XnUEEwF39vXbROpHOirmDse8WE9vxOjABnSVS+jb7EA==} - '@kubernetes/client-node@1.0.0': resolution: {integrity: sha512-a8NSvFDSHKFZ0sR1hbPSf8IDFNJwctEU5RodSCNiq/moRXWmrdmqhb1RRQzF+l+TSBaDgHw3YsYNxxE92STBzw==} @@ -8548,9 +8338,6 @@ packages: '@types/btoa-lite@1.0.2': resolution: {integrity: sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg==} - '@types/caseless@0.12.5': - resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -8872,9 +8659,6 @@ packages: '@types/regression@2.0.6': resolution: {integrity: sha512-sa+sHOUxh9fywFuAFLCcyupFN0CKX654QUZGW5fAZCmV51I4e5nQy1xL2g/JMUW/PeDoF3Yq2lDXb7MoC3KDNg==} - '@types/request@2.48.12': - resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==} - '@types/resolve@1.20.6': resolution: {integrity: sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ==} @@ -8944,9 +8728,6 @@ packages: '@types/tinycolor2@1.4.3': resolution: {integrity: sha512-Kf1w9NE5HEgGxCRyIcRXR/ZYtDv0V8FVPtYHwLxl0O+maGX0erE77pQlD0gpP+/KByMZ87mOA79SjifhSB3PjQ==} - '@types/tough-cookie@4.0.5': - resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==} - '@types/trouter@3.1.4': resolution: {integrity: sha512-4YIL/2AvvZqKBWenjvEpxpblT2KGO6793ipr5QS7/6DpQ3O3SwZGgNGWezxf3pzeYZc24a2pJIrR/+Jxh/wYNQ==} @@ -8959,9 +8740,6 @@ packages: '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} - '@types/ws@8.5.10': - resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} - '@types/ws@8.5.12': resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==} @@ -9281,9 +9059,6 @@ packages: peerDependencies: ajv: ^8.18.0 - ajv@6.12.6: - resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - ajv@8.18.0: resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} @@ -9396,10 +9171,6 @@ packages: assert-never@1.2.1: resolution: {integrity: sha512-TaTivMB6pYI1kXwrFlEhLeGfOqoDNdTxjCdwRfFFkEA30Eu+k48W34nlok2EYWJfFFzqaEmichdNM7th6M5HNw==} - assert-plus@1.0.0: - resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} - engines: {node: '>=0.8'} - assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -9448,12 +9219,6 @@ packages: avvio@9.1.0: resolution: {integrity: sha512-fYASnYi600CsH/j9EQov7lECAniYiBFiiAtBNuZYLA2leLe9qOvZzqYHFjtIj6gD2VMoMLP14834LFWvr4IfDw==} - aws-sign2@0.7.0: - resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} - - aws4@1.12.0: - resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} - aws4fetch@1.0.18: resolution: {integrity: sha512-3Cf+YaUl07p24MoQ46rFwulAmiyCwH2+1zw1ZyPAX5OtJ34Hh185DwB8y/qRLb6cYYYtSFJ9pthyLc0MD4e8sQ==} @@ -9735,9 +9500,6 @@ packages: resolution: {integrity: sha512-zlOQ80VrQ2Ue+ymH5OuM/DlDq64mEm+B9UTdHULv5osUMD6HalNTblf2b1u/m6QecjsnOkBpqVZ+XPwIVsy7Ng==} engines: {node: '>=12.13'} - caseless@0.12.0: - resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} - ccount@2.0.1: resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} @@ -10053,9 +9815,6 @@ packages: copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} - core-util-is@1.0.2: - resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} @@ -10367,10 +10126,6 @@ packages: dagre-d3-es@7.0.14: resolution: {integrity: sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==} - dashdash@1.14.1: - resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} - engines: {node: '>=0.10'} - data-uri-to-buffer@3.0.1: resolution: {integrity: sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==} engines: {node: '>= 6'} @@ -10629,10 +10384,6 @@ packages: resolution: {integrity: sha512-MVUtAugQMOff5RnBy2d9N31iG0lNwg1qAoAOn7pOK5wf94WIaE3My2p3uwTQuvS2AcqchkcR3bHByjaM0mmi7Q==} engines: {node: '>=20'} - dotenv@16.4.4: - resolution: {integrity: sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==} - engines: {node: '>=12'} - dotenv@16.4.5: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} @@ -10764,9 +10515,6 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - ecc-jsbn@0.1.2: - resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} - ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} @@ -11060,11 +10808,6 @@ packages: engines: {node: '>=12'} hasBin: true - esbuild@0.19.11: - resolution: {integrity: sha512-HJ96Hev2hX/6i5cDVwcqiJBBtuo9+FeIJOtZ9W1kA5M6AMJRHUZlpYZ1/SbEwtO0ioNAW8rUooVpC/WehY2SfA==} - engines: {node: '>=12'} - hasBin: true - esbuild@0.23.0: resolution: {integrity: sha512-1lvV17H2bMYda/WaFb2jLPeHU3zml2k4/yagNMG8Q/YtfMjCwEUZa2eXXMgZTVSL5q1n4H7sQ0X6CdJDqqeCFA==} engines: {node: '>=18'} @@ -11284,10 +11027,6 @@ packages: resolution: {integrity: sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==} engines: {node: '>=4'} - extsprintf@1.3.0: - resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} - engines: {'0': node >=0.6.0} - fast-check@3.23.2: resolution: {integrity: sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==} engines: {node: '>=8.0.0'} @@ -11312,9 +11051,6 @@ packages: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} - fast-json-stable-stringify@2.1.0: - resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} - fast-json-stringify@6.0.1: resolution: {integrity: sha512-s7SJE83QKBZwg54dIbD5rCtzOBVD43V1ReWXXYqBgwCwHLYAAT0RQc/FmrQglXqWPpz6omtryJQOau5jI4Nrvg==} @@ -11461,16 +11197,9 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - forever-agent@0.6.1: - resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} - form-data-encoder@1.7.2: resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} - form-data@2.5.4: - resolution: {integrity: sha512-Y/3MmRiR8Nd+0CUtrbvcKtKzLWiUfpQ7DFVggH8PwmGt/0r7RSy32GuP4hpCJlQNEBusisSx1DLtD8uD386HJQ==} - engines: {node: '>= 0.12'} - form-data@3.0.4: resolution: {integrity: sha512-f0cRzm6dkyVYV3nPoooP8XlccPQukegwhAnpoLcXy+X+A8KfpGOoXwDr9FLZd3wzgLaBGQBE3lY93Zm/i1JvIQ==} engines: {node: '>= 6'} @@ -11625,15 +11354,9 @@ packages: resolution: {integrity: sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==} engines: {node: '>= 0.4'} - get-tsconfig@4.7.2: - resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} - get-tsconfig@4.7.6: resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==} - getpass@0.1.7: - resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} - giget@1.2.3: resolution: {integrity: sha512-8EHPljDvs7qKykr6uw8b+lqLiUc/vUg+KVTI0uND4s63TdsZM2Xus3mflvF0DDG9SiM4RlCkFGL+7aAjRmV7KA==} hasBin: true @@ -11729,14 +11452,6 @@ packages: hachure-fill@0.5.2: resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} - har-schema@2.0.0: - resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} - engines: {node: '>=4'} - - har-validator@5.1.5: - resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} - engines: {node: '>=6'} - hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -11756,9 +11471,6 @@ packages: resolution: {integrity: sha512-CsNUt5x9LUdx6hnk/E2SZLsDyvfqANZSUq4+D3D8RzDJ2M+HDTIkF60ibS1vHaK55vzgiZw1bEPFG9yH7l33wA==} engines: {node: '>=12'} - has-own@1.0.1: - resolution: {integrity: sha512-RDKhzgQTQfMaLvIFhjahU+2gGnRBK6dYOd5Gd9BzkmnBneOCRYjRC003RIMrdAbH52+l+CnMS4bBCXGer8tEhg==} - has-property-descriptors@1.0.2: resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} @@ -11871,10 +11583,6 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} - http-signature@1.2.0: - resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} - engines: {node: '>=0.8', npm: '>=1.3.7'} - https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -12214,9 +11922,6 @@ packages: resolution: {integrity: sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==} engines: {node: '>= 0.4'} - is-typedarray@1.0.0: - resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} - is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -12258,9 +11963,6 @@ packages: peerDependencies: ws: '*' - isstream@0.1.2: - resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} - istanbul-lib-coverage@3.2.2: resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} engines: {node: '>=8'} @@ -12310,9 +12012,6 @@ packages: joi@17.7.0: resolution: {integrity: sha512-1/ugc8djfn93rTE3WRKdCzGGt/EtiYKxITMO4Wiv6q5JL1gl9ePt4kBsl1S499nbosspfctIQTpYIhSmHA3WAg==} - jose@4.15.9: - resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} - jose@5.4.0: resolution: {integrity: sha512-6rpxTHPAQyWMb9A35BroFl1Sp0ST3DpPcm5EVIxZxdH+e0Hv9fwhyB3XLKFUcHNpdSDnETmBfuPPTTlYz5+USw==} @@ -12352,9 +12051,6 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true - jsbn@0.1.1: - resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} - jsbn@1.1.0: resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} @@ -12383,9 +12079,6 @@ packages: json-schema-ref-resolver@2.0.1: resolution: {integrity: sha512-HG0SIB9X4J8bwbxCbnd5FfPEbcXAJYTi1pBJeP/QPON+w8ovSME8iRG+ElHNxZNX2Qh6eYn1GdzJFS4cDFfx0Q==} - json-schema-traverse@0.4.1: - resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} @@ -12399,9 +12092,6 @@ packages: resolution: {integrity: sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg==} engines: {node: '>= 0.4'} - json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -12428,10 +12118,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - jsonpath-plus@7.2.0: - resolution: {integrity: sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==} - engines: {node: '>=12.0.0'} - jsonpointer@5.0.1: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} @@ -12440,10 +12126,6 @@ packages: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} - jsprim@1.4.2: - resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} - engines: {node: '>=0.6.0'} - junk@4.0.1: resolution: {integrity: sha512-Qush0uP+G8ZScpGMZvHUiRfI0YBWuB3gVBYlI0v0vvOJt5FLicco+IkP0a50LqTTQhmts/m6tP5SWE+USyIvcQ==} engines: {node: '>=12.20'} @@ -12722,10 +12404,6 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} - lru-cache@6.0.0: - resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} - engines: {node: '>=10'} - lru-cache@7.18.3: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} @@ -13241,10 +12919,6 @@ packages: resolution: {integrity: sha512-g2Uuh2jEKoht+zvO6vJqXmYpflPqzRBT+Th2h01DKh5z7wbY/AZ2gCQ78cP70YoHPyFdY30YBV5WxgLOEwOykw==} engines: {node: '>=8'} - minipass@4.2.8: - resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} - engines: {node: '>=8'} - minipass@5.0.0: resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} engines: {node: '>=8'} @@ -13368,11 +13042,6 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - nanoid@5.0.6: - resolution: {integrity: sha512-rRq0eMHoGZxlvaFOUdK1Ev83Bd1IgzzR+WJ3IbDJ7QOSdAxYjlurSPqFs9s4lJg29RT6nPwizFtJhQS6V5xgiA==} - engines: {node: ^18 || >=20} - hasBin: true - nanoid@5.1.2: resolution: {integrity: sha512-b+CiXQCNMUGe0Ri64S9SXFcP9hogjAJ2Rd6GdVxhPLRm7mhGaM7VgOvCAJ1ZshfHbqVDI3uqTI5C8/GaKuLI7g==} engines: {node: ^18 || >=20} @@ -13543,9 +13212,6 @@ packages: engines: {node: '>=18'} hasBin: true - oauth-sign@0.9.0: - resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} - oauth4webapi@3.3.0: resolution: {integrity: sha512-ZlozhPlFfobzh3hB72gnBFLjXpugl/dljz1fJSRdqaV2r3D5dmi5lg2QWI0LmUYuazmE+b5exsloEv6toUtw9g==} @@ -13553,10 +13219,6 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} - object-hash@2.2.0: - resolution: {integrity: sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==} - engines: {node: '>= 6'} - object-hash@3.0.0: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} @@ -13589,10 +13251,6 @@ packages: ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} - oidc-token-hash@5.0.3: - resolution: {integrity: sha512-IF4PcGgzAr6XXSff26Sk/+P4KZFJVuHAJZj3wgO3vX2bMdNVp/QXTP3P7CEm9V1IdG8lDLY3HhiqpsE/nOwpPw==} - engines: {node: ^10.13.0 || >=12.0.0} - on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -13650,9 +13308,6 @@ packages: zod: optional: true - openid-client@5.7.1: - resolution: {integrity: sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==} - openid-client@6.3.3: resolution: {integrity: sha512-lTK8AV8SjqCM4qznLX0asVESAwzV39XTVdfMAM185ekuaZCnkWdPzcxMTXNlsm9tsUAMa1Q30MBmKAykdT1LWw==} @@ -13764,10 +13419,6 @@ packages: resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} engines: {node: '>=8'} - p-queue@8.0.1: - resolution: {integrity: sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==} - engines: {node: '>=18'} - p-retry@4.6.2: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} @@ -13784,10 +13435,6 @@ packages: resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} engines: {node: '>=12'} - p-timeout@6.1.2: - resolution: {integrity: sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==} - engines: {node: '>=14.16'} - p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} @@ -13930,9 +13577,6 @@ packages: perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} - performance-now@2.1.0: - resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} - periscopic@3.1.0: resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} @@ -14389,9 +14033,6 @@ packages: pseudomap@1.0.2: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - pump@2.0.1: resolution: {integrity: sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==} @@ -14879,10 +14520,6 @@ packages: remove-accents@0.5.0: resolution: {integrity: sha512-8g3/Otx1eJaVD12e31UbJj1YzdtVvzH85HV7t+9MJYk/u3XmkOUJ5Ys9wQrf9PCPK8+xn4ymzqYCiZl6QWKn+A==} - request@2.88.2: - resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} - engines: {node: '>= 6'} - require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -15345,11 +14982,6 @@ packages: resolution: {integrity: sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==} engines: {node: '>=10.16.0'} - sshpk@1.18.0: - resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} - engines: {node: '>=0.10.0'} - hasBin: true - ssri@10.0.5: resolution: {integrity: sha512-bSf16tAFkGeRlUNDjXu8FzaMQt6g2HZJrun7mtMbIPOddxt3GLMSz5VWUWcqTJUPfLEaDIepGxv+bYQW49596A==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -15637,10 +15269,6 @@ packages: tar-stream@3.1.7: resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} - tar@6.1.13: - resolution: {integrity: sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==} - engines: {node: '>=10'} - tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} @@ -15804,10 +15432,6 @@ packages: toposort@2.0.2: resolution: {integrity: sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==} - tough-cookie@2.5.0: - resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} - engines: {node: '>=0.8'} - tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -15954,11 +15578,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - tsx@4.7.1: - resolution: {integrity: sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==} - engines: {node: '>=18.0.0'} - hasBin: true - tty-table@4.1.6: resolution: {integrity: sha512-kRj5CBzOrakV4VRRY5kUWbNYvo/FpOsz65DzI5op9P+cHov3+IqPbo1JE1ZnQGkHdZgNFDsrEjrfqqy/Ply9fw==} engines: {node: '>=8.0.0'} @@ -16215,9 +15834,6 @@ packages: peerDependencies: browserslist: '>= 4.21.0' - uri-js@4.4.1: - resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} - url-parse-lax@3.0.0: resolution: {integrity: sha512-NjFKA0DidqPa5ciFcSrXnAltTtzz84ogy+NebPvfEgAck0+TNg4UJ4IN+fB7zRZfbgUf0syOo9MDxFkDSMuFaQ==} engines: {node: '>=4'} @@ -16286,10 +15902,6 @@ packages: resolution: {integrity: sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==} hasBin: true - uuid@3.4.0: - resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} - hasBin: true - uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -16339,10 +15951,6 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - verror@1.10.0: - resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} - engines: {'0': node >=0.6.0} - vfile-location@4.0.1: resolution: {integrity: sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==} @@ -16650,18 +16258,6 @@ packages: utf-8-validate: optional: true - ws@8.16.0: - resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.18.0: resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} engines: {node: '>=10.0.0'} @@ -19521,9 +19117,6 @@ snapshots: '@esbuild-kit/core-utils': 3.3.2 get-tsconfig: 4.7.6 - '@esbuild/aix-ppc64@0.19.11': - optional: true - '@esbuild/aix-ppc64@0.23.0': optional: true @@ -19545,9 +19138,6 @@ snapshots: '@esbuild/android-arm64@0.18.20': optional: true - '@esbuild/android-arm64@0.19.11': - optional: true - '@esbuild/android-arm64@0.23.0': optional: true @@ -19572,9 +19162,6 @@ snapshots: '@esbuild/android-arm@0.18.20': optional: true - '@esbuild/android-arm@0.19.11': - optional: true - '@esbuild/android-arm@0.23.0': optional: true @@ -19596,9 +19183,6 @@ snapshots: '@esbuild/android-x64@0.18.20': optional: true - '@esbuild/android-x64@0.19.11': - optional: true - '@esbuild/android-x64@0.23.0': optional: true @@ -19620,9 +19204,6 @@ snapshots: '@esbuild/darwin-arm64@0.18.20': optional: true - '@esbuild/darwin-arm64@0.19.11': - optional: true - '@esbuild/darwin-arm64@0.23.0': optional: true @@ -19644,9 +19225,6 @@ snapshots: '@esbuild/darwin-x64@0.18.20': optional: true - '@esbuild/darwin-x64@0.19.11': - optional: true - '@esbuild/darwin-x64@0.23.0': optional: true @@ -19668,9 +19246,6 @@ snapshots: '@esbuild/freebsd-arm64@0.18.20': optional: true - '@esbuild/freebsd-arm64@0.19.11': - optional: true - '@esbuild/freebsd-arm64@0.23.0': optional: true @@ -19692,9 +19267,6 @@ snapshots: '@esbuild/freebsd-x64@0.18.20': optional: true - '@esbuild/freebsd-x64@0.19.11': - optional: true - '@esbuild/freebsd-x64@0.23.0': optional: true @@ -19716,9 +19288,6 @@ snapshots: '@esbuild/linux-arm64@0.18.20': optional: true - '@esbuild/linux-arm64@0.19.11': - optional: true - '@esbuild/linux-arm64@0.23.0': optional: true @@ -19740,9 +19309,6 @@ snapshots: '@esbuild/linux-arm@0.18.20': optional: true - '@esbuild/linux-arm@0.19.11': - optional: true - '@esbuild/linux-arm@0.23.0': optional: true @@ -19764,9 +19330,6 @@ snapshots: '@esbuild/linux-ia32@0.18.20': optional: true - '@esbuild/linux-ia32@0.19.11': - optional: true - '@esbuild/linux-ia32@0.23.0': optional: true @@ -19791,9 +19354,6 @@ snapshots: '@esbuild/linux-loong64@0.18.20': optional: true - '@esbuild/linux-loong64@0.19.11': - optional: true - '@esbuild/linux-loong64@0.23.0': optional: true @@ -19815,9 +19375,6 @@ snapshots: '@esbuild/linux-mips64el@0.18.20': optional: true - '@esbuild/linux-mips64el@0.19.11': - optional: true - '@esbuild/linux-mips64el@0.23.0': optional: true @@ -19839,9 +19396,6 @@ snapshots: '@esbuild/linux-ppc64@0.18.20': optional: true - '@esbuild/linux-ppc64@0.19.11': - optional: true - '@esbuild/linux-ppc64@0.23.0': optional: true @@ -19863,9 +19417,6 @@ snapshots: '@esbuild/linux-riscv64@0.18.20': optional: true - '@esbuild/linux-riscv64@0.19.11': - optional: true - '@esbuild/linux-riscv64@0.23.0': optional: true @@ -19887,9 +19438,6 @@ snapshots: '@esbuild/linux-s390x@0.18.20': optional: true - '@esbuild/linux-s390x@0.19.11': - optional: true - '@esbuild/linux-s390x@0.23.0': optional: true @@ -19911,9 +19459,6 @@ snapshots: '@esbuild/linux-x64@0.18.20': optional: true - '@esbuild/linux-x64@0.19.11': - optional: true - '@esbuild/linux-x64@0.23.0': optional: true @@ -19947,9 +19492,6 @@ snapshots: '@esbuild/netbsd-x64@0.18.20': optional: true - '@esbuild/netbsd-x64@0.19.11': - optional: true - '@esbuild/netbsd-x64@0.23.0': optional: true @@ -19986,9 +19528,6 @@ snapshots: '@esbuild/openbsd-x64@0.18.20': optional: true - '@esbuild/openbsd-x64@0.19.11': - optional: true - '@esbuild/openbsd-x64@0.23.0': optional: true @@ -20016,9 +19555,6 @@ snapshots: '@esbuild/sunos-x64@0.18.20': optional: true - '@esbuild/sunos-x64@0.19.11': - optional: true - '@esbuild/sunos-x64@0.23.0': optional: true @@ -20040,9 +19576,6 @@ snapshots: '@esbuild/win32-arm64@0.18.20': optional: true - '@esbuild/win32-arm64@0.19.11': - optional: true - '@esbuild/win32-arm64@0.23.0': optional: true @@ -20064,9 +19597,6 @@ snapshots: '@esbuild/win32-ia32@0.18.20': optional: true - '@esbuild/win32-ia32@0.19.11': - optional: true - '@esbuild/win32-ia32@0.23.0': optional: true @@ -20088,9 +19618,6 @@ snapshots: '@esbuild/win32-x64@0.18.20': optional: true - '@esbuild/win32-x64@0.19.11': - optional: true - '@esbuild/win32-x64@0.23.0': optional: true @@ -20445,28 +19972,6 @@ snapshots: react-dom: 18.3.1(react@18.3.1) tldts: 7.0.7 - '@kubernetes/client-node@0.20.0(bufferutil@4.0.9)': - dependencies: - '@types/js-yaml': 4.0.9 - '@types/node': 22.20.0 - '@types/request': 2.48.12 - '@types/ws': 8.5.10 - byline: 5.0.0 - isomorphic-ws: 5.0.0(ws@8.16.0(bufferutil@4.0.9)) - js-yaml: 4.1.1 - jsonpath-plus: 7.2.0 - request: 2.88.2 - rfc4648: 1.5.3 - stream-buffers: 3.0.2 - tar: 6.1.13 - tslib: 2.6.2 - ws: 8.16.0(bufferutil@4.0.9) - optionalDependencies: - openid-client: 5.7.1 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@kubernetes/client-node@1.0.0(patch_hash=ba1a06f46256cdb8d6faf7167246692c0de2e7cd846a9dc0f13be0137e1c3745)(bufferutil@4.0.9)(encoding@0.1.13)': dependencies: '@types/js-yaml': 4.0.9 @@ -25080,8 +24585,6 @@ snapshots: '@types/btoa-lite@1.0.2': {} - '@types/caseless@0.12.5': {} - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -25479,13 +24982,6 @@ snapshots: '@types/regression@2.0.6': {} - '@types/request@2.48.12': - dependencies: - '@types/caseless': 0.12.5 - '@types/node': 22.20.0 - '@types/tough-cookie': 4.0.5 - form-data: 2.5.4 - '@types/resolve@1.20.6': {} '@types/responselike@1.0.0': @@ -25563,8 +25059,6 @@ snapshots: '@types/tinycolor2@1.4.3': {} - '@types/tough-cookie@4.0.5': {} - '@types/trouter@3.1.4': {} '@types/trusted-types@2.0.7': @@ -25574,10 +25068,6 @@ snapshots: '@types/unist@3.0.3': {} - '@types/ws@8.5.10': - dependencies: - '@types/node': 22.20.0 - '@types/ws@8.5.12': dependencies: '@types/node': 22.20.0 @@ -25985,13 +25475,6 @@ snapshots: ajv: 8.20.0 fast-deep-equal: 3.1.3 - ajv@6.12.6: - dependencies: - fast-deep-equal: 3.1.3 - fast-json-stable-stringify: 2.1.0 - json-schema-traverse: 0.4.1 - uri-js: 4.4.1 - ajv@8.18.0: dependencies: fast-deep-equal: 3.1.3 @@ -26119,8 +25602,6 @@ snapshots: assert-never@1.2.1: {} - assert-plus@1.0.0: {} - assertion-error@2.0.1: {} ast-v8-to-istanbul@1.0.2: @@ -26180,10 +25661,6 @@ snapshots: '@fastify/error': 4.2.0 fastq: 1.19.1 - aws-sign2@0.7.0: {} - - aws4@1.12.0: {} - aws4fetch@1.0.18: {} axios@1.16.1: @@ -26529,8 +26006,6 @@ snapshots: case-anything@2.1.13: {} - caseless@0.12.0: {} - ccount@2.0.1: {} chai@6.2.2: {} @@ -26844,8 +26319,6 @@ snapshots: dependencies: toggle-selection: 1.0.6 - core-util-is@1.0.2: {} - core-util-is@1.0.3: {} cors@2.8.5: @@ -27194,10 +26667,6 @@ snapshots: d3: 7.9.0 lodash-es: 4.18.1 - dashdash@1.14.1: - dependencies: - assert-plus: 1.0.0 - data-uri-to-buffer@3.0.1: {} data-view-buffer@1.0.1: @@ -27444,8 +26913,6 @@ snapshots: dependencies: type-fest: 5.6.0 - dotenv@16.4.4: {} - dotenv@16.4.5: {} dotenv@16.4.7: {} @@ -27499,11 +26966,6 @@ snapshots: eastasianwidth@0.2.0: {} - ecc-jsbn@0.1.2: - dependencies: - jsbn: 0.1.1 - safer-buffer: 2.1.2 - ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 @@ -27904,32 +27366,6 @@ snapshots: '@esbuild/win32-ia32': 0.18.20 '@esbuild/win32-x64': 0.18.20 - esbuild@0.19.11: - optionalDependencies: - '@esbuild/aix-ppc64': 0.19.11 - '@esbuild/android-arm': 0.19.11 - '@esbuild/android-arm64': 0.19.11 - '@esbuild/android-x64': 0.19.11 - '@esbuild/darwin-arm64': 0.19.11 - '@esbuild/darwin-x64': 0.19.11 - '@esbuild/freebsd-arm64': 0.19.11 - '@esbuild/freebsd-x64': 0.19.11 - '@esbuild/linux-arm': 0.19.11 - '@esbuild/linux-arm64': 0.19.11 - '@esbuild/linux-ia32': 0.19.11 - '@esbuild/linux-loong64': 0.19.11 - '@esbuild/linux-mips64el': 0.19.11 - '@esbuild/linux-ppc64': 0.19.11 - '@esbuild/linux-riscv64': 0.19.11 - '@esbuild/linux-s390x': 0.19.11 - '@esbuild/linux-x64': 0.19.11 - '@esbuild/netbsd-x64': 0.19.11 - '@esbuild/openbsd-x64': 0.19.11 - '@esbuild/sunos-x64': 0.19.11 - '@esbuild/win32-arm64': 0.19.11 - '@esbuild/win32-ia32': 0.19.11 - '@esbuild/win32-x64': 0.19.11 - esbuild@0.23.0: optionalDependencies: '@esbuild/aix-ppc64': 0.23.0 @@ -28365,8 +27801,6 @@ snapshots: iconv-lite: 0.4.24 tmp: 0.2.7 - extsprintf@1.3.0: {} - fast-check@3.23.2: dependencies: pure-rand: 6.1.0 @@ -28389,8 +27823,6 @@ snapshots: merge2: 1.4.1 micromatch: 4.0.8 - fast-json-stable-stringify@2.1.0: {} - fast-json-stringify@6.0.1: dependencies: '@fastify/merge-json-schemas': 0.2.1 @@ -28568,19 +28000,8 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - forever-agent@0.6.1: {} - form-data-encoder@1.7.2: {} - form-data@2.5.4: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - es-set-tostringtag: 2.1.0 - has-own: 1.0.1 - mime-types: 2.1.35 - safe-buffer: 5.2.1 - form-data@3.0.4: dependencies: asynckit: 0.4.0 @@ -28740,18 +28161,10 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 - get-tsconfig@4.7.2: - dependencies: - resolve-pkg-maps: 1.0.0 - get-tsconfig@4.7.6: dependencies: resolve-pkg-maps: 1.0.0 - getpass@0.1.7: - dependencies: - assert-plus: 1.0.0 - giget@1.2.3: dependencies: citty: 0.1.6 @@ -28904,13 +28317,6 @@ snapshots: hachure-fill@0.5.2: {} - har-schema@2.0.0: {} - - har-validator@5.1.5: - dependencies: - ajv: 6.12.6 - har-schema: 2.0.0 - hard-rejection@2.1.0: {} has-bigints@1.0.2: {} @@ -28921,8 +28327,6 @@ snapshots: has-flag@5.0.1: {} - has-own@1.0.1: {} - has-property-descriptors@1.0.2: dependencies: es-define-property: 1.0.1 @@ -29117,12 +28521,6 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 - http-signature@1.2.0: - dependencies: - assert-plus: 1.0.0 - jsprim: 1.4.2 - sshpk: 1.18.0 - https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 @@ -29417,8 +28815,6 @@ snapshots: dependencies: which-typed-array: 1.1.15 - is-typedarray@1.0.0: {} - is-unicode-supported@0.1.0: {} is-unicode-supported@2.1.0: {} @@ -29443,16 +28839,10 @@ snapshots: isexe@2.0.0: {} - isomorphic-ws@5.0.0(ws@8.16.0(bufferutil@4.0.9)): - dependencies: - ws: 8.16.0(bufferutil@4.0.9) - isomorphic-ws@5.0.0(ws@8.18.0(bufferutil@4.0.9)): dependencies: ws: 8.18.0(bufferutil@4.0.9) - isstream@0.1.2: {} - istanbul-lib-coverage@3.2.2: {} istanbul-lib-report@3.0.1: @@ -29502,9 +28892,6 @@ snapshots: '@sideway/formula': 3.0.1 '@sideway/pinpoint': 2.0.0 - jose@4.15.9: - optional: true - jose@5.4.0: {} jose@6.0.8: {} @@ -29538,8 +28925,6 @@ snapshots: dependencies: argparse: 2.0.1 - jsbn@0.1.1: {} - jsbn@1.1.0: {} jsep@1.4.0: {} @@ -29558,8 +28943,6 @@ snapshots: dependencies: dequal: 2.0.3 - json-schema-traverse@0.4.1: {} - json-schema-traverse@1.0.0: {} json-schema-typed@8.0.2: {} @@ -29574,8 +28957,6 @@ snapshots: jsonify: 0.0.1 object-keys: 1.1.1 - json-stringify-safe@5.0.1: {} - json5@1.0.2: dependencies: minimist: 1.2.7 @@ -29602,8 +28983,6 @@ snapshots: '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) jsep: 1.4.0 - jsonpath-plus@7.2.0: {} - jsonpointer@5.0.1: {} jsonwebtoken@9.0.2: @@ -29619,13 +28998,6 @@ snapshots: ms: 2.1.3 semver: 7.8.1 - jsprim@1.4.2: - dependencies: - assert-plus: 1.0.0 - extsprintf: 1.3.0 - json-schema: 0.4.0 - verror: 1.10.0 - junk@4.0.1: {} jwa@1.4.2: @@ -29857,11 +29229,6 @@ snapshots: dependencies: yallist: 3.1.1 - lru-cache@6.0.0: - dependencies: - yallist: 4.0.0 - optional: true - lru-cache@7.18.3: {} lucide-react@0.229.0(react@18.3.1): @@ -30760,8 +30127,6 @@ snapshots: dependencies: yallist: 4.0.0 - minipass@4.2.8: {} - minipass@5.0.0: {} minipass@7.1.2: {} @@ -30893,8 +30258,6 @@ snapshots: nanoid@3.3.8: {} - nanoid@5.0.6: {} - nanoid@5.1.2: {} napi-build-utils@2.0.0: @@ -31067,15 +30430,10 @@ snapshots: pathe: 2.0.3 tinyexec: 1.2.3 - oauth-sign@0.9.0: {} - oauth4webapi@3.3.0: {} object-assign@4.1.1: {} - object-hash@2.2.0: - optional: true - object-hash@3.0.0: {} object-inspect@1.13.4: {} @@ -31111,9 +30469,6 @@ snapshots: ohash@2.0.11: {} - oidc-token-hash@5.0.3: - optional: true - on-exit-leak-free@2.1.2: {} on-finished@2.3.0: @@ -31183,14 +30538,6 @@ snapshots: transitivePeerDependencies: - encoding - openid-client@5.7.1: - dependencies: - jose: 4.15.9 - lru-cache: 6.0.0 - object-hash: 2.2.0 - oidc-token-hash: 5.0.3 - optional: true - openid-client@6.3.3: dependencies: jose: 6.0.8 @@ -31323,11 +30670,6 @@ snapshots: eventemitter3: 4.0.7 p-timeout: 3.2.0 - p-queue@8.0.1: - dependencies: - eventemitter3: 5.0.1 - p-timeout: 6.1.2 - p-retry@4.6.2: dependencies: '@types/retry': 0.12.0 @@ -31345,8 +30687,6 @@ snapshots: p-timeout@5.1.0: {} - p-timeout@6.1.2: {} - p-try@2.2.0: {} package-json-from-dist@1.0.0: {} @@ -31472,8 +30812,6 @@ snapshots: perfect-debounce@1.0.0: {} - performance-now@2.1.0: {} - periscopic@3.1.0: dependencies: '@types/estree': 1.0.9 @@ -31913,8 +31251,6 @@ snapshots: pseudomap@1.0.2: {} - psl@1.9.0: {} - pump@2.0.1: dependencies: end-of-stream: 1.4.5 @@ -32599,29 +31935,6 @@ snapshots: remove-accents@0.5.0: {} - request@2.88.2: - dependencies: - aws-sign2: 0.7.0 - aws4: 1.12.0 - caseless: 0.12.0 - combined-stream: 1.0.8 - extend: 3.0.2 - forever-agent: 0.6.1 - form-data: 2.5.4 - har-validator: 5.1.5 - http-signature: 1.2.0 - is-typedarray: 1.0.0 - isstream: 0.1.2 - json-stringify-safe: 5.0.1 - mime-types: 2.1.35 - oauth-sign: 0.9.0 - performance-now: 2.1.0 - qs: 6.15.2 - safe-buffer: 5.2.1 - tough-cookie: 2.5.0 - tunnel-agent: 0.6.0 - uuid: 3.4.0 - require-directory@2.1.1: {} require-from-string@2.0.2: {} @@ -33222,18 +32535,6 @@ snapshots: cpu-features: 0.0.10 nan: 2.23.1 - sshpk@1.18.0: - dependencies: - asn1: 0.2.6 - assert-plus: 1.0.0 - bcrypt-pbkdf: 1.0.2 - dashdash: 1.14.1 - ecc-jsbn: 0.1.2 - getpass: 0.1.7 - jsbn: 0.1.1 - safer-buffer: 2.1.2 - tweetnacl: 0.14.5 - ssri@10.0.5: dependencies: minipass: 7.1.3 @@ -33580,15 +32881,6 @@ snapshots: transitivePeerDependencies: - bare-abort-controller - tar@6.1.13: - dependencies: - chownr: 2.0.0 - fs-minipass: 2.1.0 - minipass: 4.2.8 - minizlib: 2.1.2 - mkdirp: 1.0.4 - yallist: 4.0.0 - tar@6.2.1: dependencies: chownr: 2.0.0 @@ -33757,11 +33049,6 @@ snapshots: toposort@2.0.2: {} - tough-cookie@2.5.0: - dependencies: - psl: 1.9.0 - punycode: 2.2.0 - tr46@0.0.3: {} tr46@1.0.1: @@ -33941,13 +33228,6 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - tsx@4.7.1: - dependencies: - esbuild: 0.19.11 - get-tsconfig: 4.7.2 - optionalDependencies: - fsevents: 2.3.3 - tty-table@4.1.6: dependencies: chalk: 4.1.2 @@ -33961,6 +33241,7 @@ snapshots: tunnel-agent@0.6.0: dependencies: safe-buffer: 5.2.1 + optional: true turbo-darwin-64@1.10.3: optional: true @@ -34219,10 +33500,6 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 - uri-js@4.4.1: - dependencies: - punycode: 2.2.0 - url-parse-lax@3.0.0: dependencies: prepend-http: 2.0.0 @@ -34276,8 +33553,6 @@ snapshots: uuid@14.0.0: {} - uuid@3.4.0: {} - uuid@8.3.2: {} uuid@9.0.1: {} @@ -34316,12 +33591,6 @@ snapshots: vary@1.1.2: {} - verror@1.10.0: - dependencies: - assert-plus: 1.0.0 - core-util-is: 1.0.2 - extsprintf: 1.3.0 - vfile-location@4.0.1: dependencies: '@types/unist': 2.0.6 @@ -34710,10 +33979,6 @@ snapshots: optionalDependencies: bufferutil: 4.0.9 - ws@8.16.0(bufferutil@4.0.9): - optionalDependencies: - bufferutil: 4.0.9 - ws@8.18.0(bufferutil@4.0.9): optionalDependencies: bufferutil: 4.0.9