diff --git a/.env.example b/.env.example index ec4488d5f63..37ffa422062 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,6 @@ DIRECT_URL=${DATABASE_URL} # Dedicated run-ops database (@internal/run-ops-database). Only needed to run prisma commands # against it or to enable the run-ops split; start it with `docker compose --profile runops up`. RUN_OPS_DATABASE_URL=postgresql://postgres:postgres@localhost:5434/postgres?schema=public -RUN_OPS_DATABASE_DIRECT_URL=${RUN_OPS_DATABASE_URL} REMIX_APP_PORT=3030 # Dev-only: stream the webapp's logs over a local telnet/TCP socket (nc localhost 6767). Uncomment to enable. # WEBAPP_TELNET_LOGS_PORT=6767 diff --git a/.server-changes/run-ops-auto-migrate.md b/.server-changes/run-ops-auto-migrate.md new file mode 100644 index 00000000000..b62f7c0d9d9 --- /dev/null +++ b/.server-changes/run-ops-auto-migrate.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Automatically migrate the dedicated run-ops database on deploy (entrypoint + `@internal/run-ops-database` deploy/status scripts), and standardize the run-ops DB connection on a single `RUN_OPS_DATABASE_URL` family (dropping the `TASK_RUN_DATABASE_URL` aliases) so migrations always target the DB the app connects to. diff --git a/apps/webapp/app/db.server.ts b/apps/webapp/app/db.server.ts index 8fc033dbf03..0284c192465 100644 --- a/apps/webapp/app/db.server.ts +++ b/apps/webapp/app/db.server.ts @@ -237,16 +237,16 @@ export function selectRunOpsTopology( // nothing new. The builders apply the SAME wrapper pair the control-plane // singletons use (captureInfrastructureErrors(tagDatasource(role, raw))). const runOpsTopology: RunOpsTopology = singleton("runOpsTopology", () => { - const newUrl = env.TASK_RUN_DATABASE_URL; + const newUrl = env.RUN_OPS_DATABASE_URL; // Gate on the opt-in flag too: the distinct-DB sentinel only runs when the flag is on. - const splitEnabled = env.RUN_OPS_SPLIT_ENABLED && !!newUrl && !!env.TASK_RUN_LEGACY_DATABASE_URL; + const splitEnabled = env.RUN_OPS_SPLIT_ENABLED && !!newUrl && !!env.RUN_OPS_LEGACY_DATABASE_URL; return selectRunOpsTopology( { splitEnabled, - legacyUrl: env.TASK_RUN_LEGACY_DATABASE_URL, + legacyUrl: env.RUN_OPS_LEGACY_DATABASE_URL, newUrl, - newReplicaUrl: env.TASK_RUN_DATABASE_READ_REPLICA_URL, + newReplicaUrl: env.RUN_OPS_DATABASE_READ_REPLICA_URL, }, { controlPlane: { writer: prisma, replica: $replica }, @@ -278,8 +278,8 @@ export const runOpsSplitReadEnabled: boolean = computeRunOpsSplitReadEnabled({ newReplica: runOpsNewReplicaClient, controlPlaneWriter: prisma, controlPlaneReplica: $replica, - hasNewUrl: !!env.TASK_RUN_DATABASE_URL, - hasLegacyUrl: !!env.TASK_RUN_LEGACY_DATABASE_URL, + hasNewUrl: !!env.RUN_OPS_DATABASE_URL, + hasLegacyUrl: !!env.RUN_OPS_LEGACY_DATABASE_URL, }); // Boot-time interlock: if the flag is on but the distinct-DB sentinel does not diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 906d3aa225b..dc189720821 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -132,35 +132,24 @@ const EnvironmentSchema = z // Explicit positive opt-in. Split behavior is unreachable unless this is true // AND the distinct-DB sentinel confirms the two URLs are physically distinct DBs. RUN_OPS_SPLIT_ENABLED: BoolEnv.default(false), - // Datasource URL for the dedicated run-ops Prisma schema (migrations/generation). - // The webapp runtime pool is driven by TASK_RUN_DATABASE_URL, not this var. + // Canonical connection URL for the dedicated NEW run-ops DB — drives the runtime pool, the split + // decision, replication, and migrations. Optional so single-DB installs never set it. RUN_OPS_DATABASE_URL: z .string() .refine(isValidDatabaseUrl, "RUN_OPS_DATABASE_URL is invalid") .optional(), - // The NEW dedicated run-ops DB writer. Optional so single-DB installs never set it. - TASK_RUN_DATABASE_URL: z - .string() - .refine(isValidDatabaseUrl, "TASK_RUN_DATABASE_URL is invalid") - .optional(), - // The NEW run-ops DB unpooled/direct endpoint (Prisma migrate/introspection; - // connection poolers break advisory locks). Consumed by the migrations. - TASK_RUN_DATABASE_DIRECT_URL: z - .string() - .refine(isValidDatabaseUrl, "TASK_RUN_DATABASE_DIRECT_URL is invalid") - .optional(), // The LEGACY run-ops DB (the control-plane DB during the transition). When unset, legacy // run-ops reuses the existing DATABASE_URL (legacy run-ops == control-plane DB initially). - TASK_RUN_LEGACY_DATABASE_URL: z + RUN_OPS_LEGACY_DATABASE_URL: z .string() - .refine(isValidDatabaseUrl, "TASK_RUN_LEGACY_DATABASE_URL is invalid") + .refine(isValidDatabaseUrl, "RUN_OPS_LEGACY_DATABASE_URL is invalid") .optional(), // The NEW dedicated run-ops DB read replica. Optional; self-host never sets it. // Refined (unlike the unrefined control-plane DATABASE_READ_REPLICA_URL) so a malformed run-ops // replica URL fails boot loudly rather than silently degrading — do not align it down to the CP shape. - TASK_RUN_DATABASE_READ_REPLICA_URL: z + RUN_OPS_DATABASE_READ_REPLICA_URL: z .string() - .refine(isValidDatabaseUrl, "TASK_RUN_DATABASE_READ_REPLICA_URL is invalid") + .refine(isValidDatabaseUrl, "RUN_OPS_DATABASE_READ_REPLICA_URL is invalid") .optional(), // --- Control-plane datasource repoint. Additive-only. --- // Optional control-plane DB. Unset (self-host/single-DB) -> getClient()/getReplicaClient() fall back to @@ -1724,9 +1713,9 @@ const EnvironmentSchema = z // --- Run-ops DB split — second replication source (the NEW dedicated run-ops DB). --- // Cloud-only; only consulted when isSplitEnabled() is true. Self-host never sets these. - // The NEW source's connection URL is TASK_RUN_DATABASE_URL; these add - // the NEW source's replication slot/publication and an explicit per-source enable so it can be - // brought up independently of the legacy source during the transition. + // The NEW source's connection URL is RUN_OPS_DATABASE_URL; these add the NEW source's replication + // slot/publication and an explicit per-source enable so it can be brought up independently of the + // legacy source during the transition. RUN_REPLICATION_NEW_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v2"), RUN_REPLICATION_NEW_PUBLICATION_NAME: z .string() diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index bfbf38478a2..30a757597fa 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -172,7 +172,7 @@ function initializeRunsReplicationInstance() { const sources = buildReplicationSources({ splitEnabled, legacyUrl: DATABASE_URL, - newUrl: env.RUN_OPS_DATABASE_URL ?? env.TASK_RUN_DATABASE_URL, + newUrl: env.RUN_OPS_DATABASE_URL, newSourceOverride: env.RUN_REPLICATION_NEW_ENABLED === "disabled" ? false : undefined, legacySlotName: env.RUN_REPLICATION_SLOT_NAME, legacyPublicationName: env.RUN_REPLICATION_PUBLICATION_NAME, diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts index ce83c632abf..fb903bc8ea9 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts @@ -449,7 +449,7 @@ export class ControlPlaneResolver { // run-ops topology factory uses); the async isSplitEnabled() distinct-DB sentinel is enforced // at boot elsewhere and is never awaited on a resolver hot path. const SPLIT_ENABLED = - env.RUN_OPS_SPLIT_ENABLED && !!env.TASK_RUN_DATABASE_URL && !!env.TASK_RUN_LEGACY_DATABASE_URL; + env.RUN_OPS_SPLIT_ENABLED && !!env.RUN_OPS_DATABASE_URL && !!env.RUN_OPS_LEGACY_DATABASE_URL; export const controlPlaneResolver = new ControlPlaneResolver({ controlPlanePrimary: prisma, diff --git a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts index 8d0eb807dbf..c1bf592e4a4 100644 --- a/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/splitMode.server.ts @@ -30,7 +30,7 @@ export async function computeSplitEnabled( // Both URLs are required to even consider a split. if (!config.legacyUrl || !config.newUrl) { deps.logger?.warn( - "RUN_OPS_SPLIT_ENABLED is on but TASK_RUN_LEGACY_DATABASE_URL / TASK_RUN_DATABASE_URL are not both set; staying single-DB." + "RUN_OPS_SPLIT_ENABLED is on but RUN_OPS_LEGACY_DATABASE_URL / RUN_OPS_DATABASE_URL are not both set; staying single-DB." ); return false; } @@ -70,8 +70,8 @@ export function isSplitEnabled(): Promise { cached = computeSplitEnabled( { flagEnabled: env.RUN_OPS_SPLIT_ENABLED, - legacyUrl: env.TASK_RUN_LEGACY_DATABASE_URL, - newUrl: env.TASK_RUN_DATABASE_URL, + legacyUrl: env.RUN_OPS_LEGACY_DATABASE_URL, + newUrl: env.RUN_OPS_DATABASE_URL, }, { logger } ); diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 4173fc55eaa..bf1642b5f4e 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -76,7 +76,7 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { // RUN_OPS_SPLIT_ENABLED. Reads must fan out across both DBs so a run that lives on the new // DB stays visible even with the flag off (matches the db.server topology factory). The flag // governs write/mint residency + migration via isSplitEnabled(), not read visibility. -const ROUTING_ENABLED = !!env.TASK_RUN_DATABASE_URL && !!env.TASK_RUN_LEGACY_DATABASE_URL; +const ROUTING_ENABLED = !!env.RUN_OPS_DATABASE_URL && !!env.RUN_OPS_LEGACY_DATABASE_URL; // Resolve the run-ops handles, tolerating contexts where they are absent — tests that mock // ~/db.server minimally omit them, and accessing a missing export under vi.mock throws. A diff --git a/apps/webapp/test/runsReplicationInstance.test.ts b/apps/webapp/test/runsReplicationInstance.test.ts index 39f57497e7f..9dc2dd6a106 100644 --- a/apps/webapp/test/runsReplicationInstance.test.ts +++ b/apps/webapp/test/runsReplicationInstance.test.ts @@ -126,35 +126,19 @@ describe("buildReplicationSources (pure)", () => { expect(sources[0].id).toBe("legacy"); }); - it("RUN_OPS_DATABASE_URL takes precedence: new source pgConnectionUrl === RUN_OPS_DATABASE_URL when both are supplied", () => { + it("new source pgConnectionUrl === the provided RUN_OPS_DATABASE_URL", () => { const runOpsUrl = "postgres://run-ops-dedicated"; - const taskRunUrl = "postgres://task-run-legacy-alias"; const sources = buildReplicationSources({ ...baseArgs, splitEnabled: true, - // Simulates env.RUN_OPS_DATABASE_URL ?? env.TASK_RUN_DATABASE_URL with RUN_OPS set - newUrl: runOpsUrl ?? taskRunUrl, + newUrl: runOpsUrl, }); expect(sources).toHaveLength(2); expect(sources[1]!.id).toBe("new"); expect(sources[1]!.pgConnectionUrl).toBe(runOpsUrl); }); - - it("falls back to TASK_RUN_DATABASE_URL when RUN_OPS_DATABASE_URL is absent", () => { - const taskRunUrl = "postgres://task-run-legacy-alias"; - - const sources = buildReplicationSources({ - ...baseArgs, - splitEnabled: true, - // Simulates env.RUN_OPS_DATABASE_URL ?? env.TASK_RUN_DATABASE_URL with RUN_OPS unset - newUrl: undefined ?? taskRunUrl, - }); - - expect(sources).toHaveLength(2); - expect(sources[1]!.pgConnectionUrl).toBe(taskRunUrl); - }); }); describe("assertReplicationCoversSplit (boot gate-coupling)", () => { diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index bfc035dd170..90589619143 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -13,6 +13,20 @@ else echo "SKIP_POSTGRES_MIGRATIONS=1, skipping Postgres migrations." fi +# Run-ops split: migrate the dedicated NEW run-ops database only when it is configured. Single-DB +# installs never set the URL, so this is a no-op there. +if [ -n "$RUN_OPS_DATABASE_URL" ]; then + if [ "$SKIP_RUN_OPS_MIGRATIONS" != "1" ]; then + echo "Running run-ops migrations" + pnpm --filter @internal/run-ops-database db:migrate:deploy + echo "Run-ops migrations done" + else + echo "SKIP_RUN_OPS_MIGRATIONS=1, skipping run-ops migrations." + fi +else + echo "RUN_OPS_DATABASE_URL not set, skipping run-ops migrations." +fi + if [ "$SKIP_DASHBOARD_AGENT_MIGRATIONS" != "1" ]; then echo "Running dashboard agent migrations" pnpm --filter @internal/dashboard-agent-db db:migrate:deploy diff --git a/internal-packages/run-ops-database/package.json b/internal-packages/run-ops-database/package.json index 6c26398c9c7..cbeb0279b05 100644 --- a/internal-packages/run-ops-database/package.json +++ b/internal-packages/run-ops-database/package.json @@ -27,7 +27,8 @@ "clean": "rimraf dist", "generate": "prisma generate", "db:migrate:dev:create": "prisma migrate dev --create-only", - "db:migrate:deploy": "prisma migrate deploy", + "db:migrate:deploy": "node scripts/migrate.mjs deploy", + "db:migrate:status": "node scripts/migrate.mjs status", "db:push": "prisma db push", "test": "vitest run", "typecheck": "tsc --noEmit", diff --git a/internal-packages/run-ops-database/prisma/schema.prisma b/internal-packages/run-ops-database/prisma/schema.prisma index 49842325c71..8e49fc854e6 100644 --- a/internal-packages/run-ops-database/prisma/schema.prisma +++ b/internal-packages/run-ops-database/prisma/schema.prisma @@ -1,7 +1,6 @@ datasource db { - provider = "postgresql" - url = env("RUN_OPS_DATABASE_URL") - directUrl = env("RUN_OPS_DATABASE_DIRECT_URL") + provider = "postgresql" + url = env("RUN_OPS_DATABASE_URL") } generator client { diff --git a/internal-packages/run-ops-database/scripts/migrate.mjs b/internal-packages/run-ops-database/scripts/migrate.mjs new file mode 100644 index 00000000000..181baa2c476 --- /dev/null +++ b/internal-packages/run-ops-database/scripts/migrate.mjs @@ -0,0 +1,72 @@ +// Run Prisma migrations against the dedicated NEW run-ops database (the second physical DB in the +// split). It owns its own migration history, so it is migrated independently of the control-plane +// DB. Connects via RUN_OPS_DATABASE_URL — the same var the webapp uses — so migrations always +// target the DB the app connects to. +// +// Usage: node scripts/migrate.mjs [deploy|status] (defaults to deploy) +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +// Read from local .env files so dev works without an exported env; deploy environments inject vars directly. +function readFromEnvFiles(key) { + for (const file of [resolve(packageRoot, ".env"), resolve(packageRoot, "../../.env")]) { + let contents; + try { + contents = readFileSync(file, "utf8"); + } catch { + continue; + } + for (const line of contents.split("\n")) { + const match = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*?)\s*$/); + if (!match || match[1] !== key) continue; + let value = match[2]; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + if (value) return value; + } + } + return undefined; +} + +// Expand `${VAR}` refs in env-file values (our manual reader loads them literally, unlike Prisma's +// dotenv-expand), so a `.env` like RUN_OPS_DATABASE_URL=${DATABASE_URL} still resolves. +const expand = (value) => + value?.replace(/\$\{(\w+)\}/g, (_, k) => process.env[k] ?? readFromEnvFiles(k) ?? ""); +const resolveVar = (key) => expand(process.env[key] || readFromEnvFiles(key)); +const redact = (url) => url.replace(/:\/\/[^@]*@/, "://***@"); + +const subcommand = process.argv[2] === "status" ? "status" : "deploy"; + +const databaseUrl = resolveVar("RUN_OPS_DATABASE_URL"); + +if (!databaseUrl) { + // Single-DB installs never set it — safe no-op. A genuinely-expected DB is gated on by the caller. + console.log( + `run-ops migrate ${subcommand}: RUN_OPS_DATABASE_URL is not set (checked env and .env). ` + + "No dedicated run-ops database configured — skipping." + ); + process.exit(0); +} + +console.log( + `Running \`prisma migrate ${subcommand}\` against the run-ops database (${redact(databaseUrl)})` +); + +const result = spawnSync("prisma", ["migrate", subcommand, "--schema", "prisma/schema.prisma"], { + cwd: packageRoot, + stdio: "inherit", + env: { + ...process.env, + RUN_OPS_DATABASE_URL: databaseUrl, + }, +}); + +process.exit(result.status ?? 1); diff --git a/internal-packages/testcontainers/src/utils.ts b/internal-packages/testcontainers/src/utils.ts index b13ab7ea859..385b4113f59 100644 --- a/internal-packages/testcontainers/src/utils.ts +++ b/internal-packages/testcontainers/src/utils.ts @@ -37,8 +37,8 @@ export async function pushDatabaseSchema(databaseUrl: string) { /** * Pushes the DEDICATED run-ops subset schema (@internal/run-ops-database) into the database at - * `databaseUrl`. The run-ops datasource reads RUN_OPS_DATABASE_URL/RUN_OPS_DATABASE_DIRECT_URL, and - * its schema is a subset of the control-plane schema (run-ops tables, no Organization/Project/etc). + * `databaseUrl`. The run-ops datasource reads RUN_OPS_DATABASE_URL, and its schema is a subset of + * the control-plane schema (run-ops tables, no Organization/Project/etc). */ export async function pushRunOpsSchema(databaseUrl: string) { // Resolve the schema (and the package's own prisma binary) through the @internal/run-ops-database @@ -49,7 +49,7 @@ export async function pushRunOpsSchema(databaseUrl: string) { const result = await pushPrismaSchema({ prismaBin: `${runOpsPackagePath}/node_modules/.bin/prisma`, schemaPath, - env: { RUN_OPS_DATABASE_URL: databaseUrl, RUN_OPS_DATABASE_DIRECT_URL: databaseUrl }, + env: { RUN_OPS_DATABASE_URL: databaseUrl }, }); // `db push` derives DDL from the schema datamodel and so cannot create the SQL-only partial unique