diff --git a/.server-changes/agent-detail-metrics-layout.md b/.server-changes/agent-detail-metrics-layout.md
new file mode 100644
index 00000000000..0af763ff3f5
--- /dev/null
+++ b/.server-changes/agent-detail-metrics-layout.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+The agent detail page now matches the shared metrics-page layout: the time filter and pagination sit in a filter row at the top, the activity charts form a tile row beneath it, and the tabs and table flow below in a single page scroll, with the agent config panel as a resizable sidebar on the right.
diff --git a/.server-changes/queue-override-reject-above-limit.md b/.server-changes/queue-override-reject-above-limit.md
new file mode 100644
index 00000000000..63d38690ad9
--- /dev/null
+++ b/.server-changes/queue-override-reject-above-limit.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: breaking
+---
+
+Setting a queue's concurrency limit above the environment limit is now rejected with a clear error instead of being silently capped. Existing overrides are unaffected.
diff --git a/.server-changes/queue-pages-live-and-clarity.md b/.server-changes/queue-pages-live-and-clarity.md
new file mode 100644
index 00000000000..97785debe6e
--- /dev/null
+++ b/.server-changes/queue-pages-live-and-clarity.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Queue pages now refresh their live numbers automatically, and the charts are clearer: better axis labels, hover explanations for the headline stats, and the busiest concurrency key named on the chart.
diff --git a/apps/webapp/app/components/layout/MetricsLayout.tsx b/apps/webapp/app/components/layout/MetricsLayout.tsx
new file mode 100644
index 00000000000..6ae40778aff
--- /dev/null
+++ b/apps/webapp/app/components/layout/MetricsLayout.tsx
@@ -0,0 +1,356 @@
+/**
+ * MetricsLayout — a compound layout for metric / dashboard pages.
+ *
+ * Slots bake all the chrome; there is no `className` on any slot, so pages can't drift apart on
+ * spacing. Need a variant? Add a closed prop (`kind`, `inset`), don't reopen `className`.
+ *
+ * Slots, top to bottom:
+ * - `Filters` — pinned 40px bar under the NavBar. Left/right clusters are child divs.
+ * - `Grid` — tiles; columns derived from tile count unless `columns` is set. `kind="charts"`
+ * bakes the fixed chart-row height.
+ * - `Content` — table / tabs below the tiles. Full-bleed by default; `inset` for a padded column.
+ *
+ * Optional:
+ * - `Sidebar` — a persistent right-hand panel; fixed `width` or `resizable`. Present ⇒ Root
+ * switches to `[main | sidebar]`; absent ⇒ single column.
+ * - `scroll` on Root — `"page"` (default): the whole page scrolls as one. `"regions"`: Root owns
+ * no scroll, the page composes its own scrolling areas.
+ *
+ * Purely presentational. Lives inside a `PageContainer` after the `NavBar`.
+ *
+ * @example
+ * ```tsx
+ *
+ *
+ *
…search + TimeFilter…
+ *
+ *
+ * …stat tiles…
+ * …chart tiles…
+ * …table…
+ *
+ * ```
+ */
+import { Children, isValidElement, type ReactElement, type ReactNode } from "react";
+import { PageBody } from "~/components/layout/AppLayout";
+import {
+ ResizableHandle,
+ ResizablePanel,
+ ResizablePanelGroup,
+ type ResizableSnapshot,
+} from "~/components/primitives/Resizable";
+import { cn } from "~/utils/cn";
+
+type ColumnCount = 1 | 2 | 3 | 4 | 5 | 6;
+
+/**
+ * A responsive column spec. Each key is a breakpoint (mobile-first `base`, then `sm`/`md`/`lg`);
+ * the value is the number of grid columns from that breakpoint up. Pass this to `Grid` when the
+ * tile count shouldn't drive the layout (e.g. a chart grid that is always two-up).
+ */
+export type GridColumns = {
+ base?: ColumnCount;
+ sm?: ColumnCount;
+ md?: ColumnCount;
+ lg?: ColumnCount;
+};
+
+// Static class maps so Tailwind's scanner sees every column class as a literal string.
+const BASE_COLS: Record = {
+ 1: "grid-cols-1",
+ 2: "grid-cols-2",
+ 3: "grid-cols-3",
+ 4: "grid-cols-4",
+ 5: "grid-cols-5",
+ 6: "grid-cols-6",
+};
+const SM_COLS: Record = {
+ 1: "sm:grid-cols-1",
+ 2: "sm:grid-cols-2",
+ 3: "sm:grid-cols-3",
+ 4: "sm:grid-cols-4",
+ 5: "sm:grid-cols-5",
+ 6: "sm:grid-cols-6",
+};
+const MD_COLS: Record = {
+ 1: "md:grid-cols-1",
+ 2: "md:grid-cols-2",
+ 3: "md:grid-cols-3",
+ 4: "md:grid-cols-4",
+ 5: "md:grid-cols-5",
+ 6: "md:grid-cols-6",
+};
+const LG_COLS: Record = {
+ 1: "lg:grid-cols-1",
+ 2: "lg:grid-cols-2",
+ 3: "lg:grid-cols-3",
+ 4: "lg:grid-cols-4",
+ 5: "lg:grid-cols-5",
+ 6: "lg:grid-cols-6",
+};
+
+// When `columns` is omitted the grid figures itself out from the tile count. The breakpoints are
+// chosen so the common metric layouts fall out for free: a trio of stat blocks goes one-up then
+// three-up, a quartet of stat/chart tiles goes two-up then four-up, and anything larger settles
+// into a comfortable two-up.
+function columnsForCount(count: number): GridColumns {
+ switch (count) {
+ case 1:
+ return { base: 1 };
+ case 2:
+ return { base: 1, sm: 2 };
+ case 3:
+ return { base: 1, sm: 3 };
+ case 4:
+ return { base: 2, lg: 4 };
+ default:
+ return { base: 1, sm: 2 };
+ }
+}
+
+/**
+ * Who owns the vertical scroll.
+ * - `"page"` (default): Root owns one `overflow-y-auto` and the column rhythm — the whole page
+ * scrolls as one.
+ * - `"regions"`: Root only bounds the height (a bare `flex` column, no scroll, no rhythm); the
+ * page composes its own scrolling areas inside the slots.
+ */
+export type MetricsScroll = "page" | "regions";
+
+/** A length the resizable panels accept: pixels or percent (the panel library's `Unit`). */
+type PanelLength = `${number}px` | `${number}%`;
+
+type MetricsLayoutSidebarProps = {
+ children: ReactNode;
+ /**
+ * Fixed sidebar width for the non-resizable default (any CSS length, e.g. `"380px"`, `"22rem"`).
+ * Ignored when `resizable` is set. Defaults to `"380px"`.
+ */
+ width?: string;
+ /**
+ * Makes the split draggable. To persist it, pass an `autosaveId` (written to a cookie) plus the
+ * `snapshot` read back in the loader via `getResizableSnapshot(request, autosaveId)`.
+ */
+ resizable?: boolean;
+ /** Resizable only: min width of the sidebar panel. Defaults to `"280px"`. */
+ min?: PanelLength;
+ /** Resizable only: initial width of the sidebar panel. Defaults to `"380px"`. */
+ defaultSize?: PanelLength;
+ /** Resizable only: max width of the sidebar panel. */
+ max?: PanelLength;
+ /** Resizable only: min width of the main panel. Defaults to `"300px"`. */
+ mainMin?: PanelLength;
+ /** Resizable only: cookie name the split is persisted under (also the panel-group id). */
+ autosaveId?: string;
+ /** Resizable only: server-loaded split snapshot to hydrate from (see `getResizableSnapshot`). */
+ snapshot?: ResizableSnapshot;
+};
+
+/**
+ * Marker slot for the persistent side panel. Rendered/positioned entirely by `Root` (this
+ * component is never mounted directly) — Root reads its props to build the `[main | sidebar]`
+ * layout and drops the children into the panel. The panel itself owns its chrome (border, scroll);
+ * pass those as part of the children, not as a class on the slot.
+ */
+function MetricsLayoutSidebar(_props: MetricsLayoutSidebarProps) {
+ return null;
+}
+
+function isSidebarElement(child: ReactNode): child is ReactElement {
+ return isValidElement(child) && child.type === MetricsLayoutSidebar;
+}
+
+function isFiltersElement(child: ReactNode): child is ReactElement {
+ return isValidElement(child) && child.type === MetricsLayoutFilters;
+}
+
+// The main (left) column. Filters is hoisted out of the scroll container so it stays pinned while
+// the rest scrolls. `"page"` scrolls as one with the baked column rhythm; `"regions"` stays bare so
+// the page owns its own scrolling.
+function MetricsLayoutMain({ children, scroll }: { children: ReactNode; scroll: MetricsScroll }) {
+ const arr = Children.toArray(children);
+ const filters = arr.find(isFiltersElement);
+ const rest = filters ? arr.filter((child) => !isFiltersElement(child)) : children;
+
+ return (
+
+ {filters}
+
+ {rest}
+
+
+ );
+}
+
+function MetricsLayoutRoot({
+ children,
+ scroll = "page",
+}: {
+ children: ReactNode;
+ /** Who owns the vertical scroll — see {@link MetricsScroll}. Defaults to `"page"`. */
+ scroll?: MetricsScroll;
+}) {
+ // A single optional Sidebar slot flips Root into a horizontal `[main | sidebar]` layout. When it
+ // is absent the output is the plain single-column markup.
+ const sidebar = Children.toArray(children).find(isSidebarElement);
+ const mainChildren = sidebar
+ ? Children.toArray(children).filter((child) => !isSidebarElement(child))
+ : children;
+
+ const main = {mainChildren};
+
+ if (!sidebar) {
+ return (
+
+ {/* The whole page scrolls as one: filters (pinned) aside, the tiles and content share a
+ single vertical scroll context. */}
+ {main}
+
+ );
+ }
+
+ const {
+ children: sidebarChildren,
+ width = "380px",
+ resizable,
+ min = "280px",
+ defaultSize = "380px",
+ max,
+ mainMin = "300px",
+ autosaveId,
+ snapshot,
+ } = sidebar.props;
+
+ if (resizable) {
+ // Draggable split. `autosaveId`/`snapshot` wire up cookie persistence exactly as the run and
+ // agent pages do (client writes the cookie, the loader hydrates via getResizableSnapshot).
+ return (
+
+
+
+ {main}
+
+
+
+ {sidebarChildren}
+
+
+
+ );
+ }
+
+ // Fixed-width sidebar.
+ return (
+
+
+
{main}
+
+ {sidebarChildren}
+
+
+
+ );
+}
+
+/**
+ * The pinned bar under the NavBar. Baked chrome: a 40px-tall bar with a bottom border and the
+ * standard page insets. Compose left/right clusters as child divs — `justify-between` spreads them
+ * (a single child sits at the start).
+ */
+function MetricsLayoutFilters({ children }: { children: ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+/** Whether a grid holds stat tiles (auto height) or charts (a fixed row height). */
+export type MetricsGridKind = "tiles" | "charts";
+
+/**
+ * A grid of tiles with the baked page gutter and grid gap. Columns are derived from the tile count
+ * unless you pass an explicit `columns` spec. Pass `kind="charts"` for a row of chart cards — it
+ * bakes the fixed chart-row height so the cards fill it (no wrapper needed).
+ */
+function MetricsLayoutGrid({
+ children,
+ columns,
+ kind = "tiles",
+}: {
+ children: ReactNode;
+ /** Explicit responsive columns. Omit to derive the layout from the number of tiles. */
+ columns?: GridColumns;
+ /** `"tiles"` (default) sizes to content; `"charts"` bakes the fixed chart-row height. */
+ kind?: MetricsGridKind;
+}) {
+ const resolved = columns ?? columnsForCount(Children.toArray(children).length);
+ return (
+
+ {children}
+
+ );
+}
+
+/**
+ * The content region below the tiles (tabs / table / list). Full-bleed by default so a list table
+ * spans edge to edge with its own top border; pass `inset` for a padded column (the detail page's
+ * tabs + charts). Either way Content bakes a doubled separation above it, so the tile blocks read
+ * as a distinct band from the content below.
+ */
+function MetricsLayoutContent({
+ children,
+ inset = false,
+}: {
+ children: ReactNode;
+ /** Pad the content into a column (page gutter) instead of letting it span edge to edge. */
+ inset?: boolean;
+}) {
+ return
{children}
;
+}
+
+export const MetricsLayout = {
+ Root: MetricsLayoutRoot,
+ Filters: MetricsLayoutFilters,
+ Grid: MetricsLayoutGrid,
+ Content: MetricsLayoutContent,
+ Sidebar: MetricsLayoutSidebar,
+};
+
+export {
+ MetricsLayoutRoot,
+ MetricsLayoutFilters,
+ MetricsLayoutGrid,
+ MetricsLayoutContent,
+ MetricsLayoutSidebar,
+};
diff --git a/apps/webapp/app/components/metrics/ActivityBarChart.tsx b/apps/webapp/app/components/metrics/ActivityBarChart.tsx
new file mode 100644
index 00000000000..4e361b224b8
--- /dev/null
+++ b/apps/webapp/app/components/metrics/ActivityBarChart.tsx
@@ -0,0 +1,82 @@
+import { type ReactElement, type ReactNode } from "react";
+import { BarChart, ReferenceLine, Tooltip, YAxis } from "recharts";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+
+// Fixed px dims skip ResponsiveContainer's ResizeObserver — otherwise every panel resize
+// re-renders all the charts in a list at once.
+export const ACTIVITY_CHART_WIDTH = 112;
+export const ACTIVITY_CHART_HEIGHT = 24;
+export const ACTIVITY_CHART_PEAK_CLASS =
+ "-mt-1 inline-block min-w-7 text-xxs tabular-nums text-text-dimmed";
+
+type ActivityBarChartProps = {
+ /** Recharts row data; each row is a bucket. The bar `children` read their `dataKey`s off it. */
+ data: ReadonlyArray>;
+ /** Y-axis domain top and the height of the dashed peak line. */
+ max: number;
+ /** The `` element(s) — one stacked series per bar, or a single bar with ``s. */
+ children: ReactNode;
+ /** Recharts `` element for the per-bucket hover card. */
+ tooltip: ReactElement;
+ /** Trailing peak label shown to the right of the chart. */
+ peak: ReactNode;
+ /** Optional tooltip wrapping the peak label. */
+ peakTooltip?: ReactNode;
+ /** Chart width in px. Defaults to the shared ACTIVITY_CHART_WIDTH. */
+ width?: number;
+};
+
+/**
+ * Shared visual frame for the inline activity/backlog mini bar charts (tasks page + queues list).
+ * Owns the fixed dimensions, y-axis, hover tooltip, baseline, dashed peak line, and the trailing
+ * peak label — the single source of truth for how these charts look. Callers supply the bars and
+ * the tooltip content, which is where the two usages differ (stacked per-status vs. single series).
+ */
+export function ActivityBarChart({
+ data,
+ max,
+ children,
+ tooltip,
+ peak,
+ peakTooltip,
+ width = ACTIVITY_CHART_WIDTH,
+}: ActivityBarChartProps) {
+ return (
+
{shouldCompact ? (
diff --git a/apps/webapp/app/components/metrics/MiniLineChart.tsx b/apps/webapp/app/components/metrics/MiniLineChart.tsx
new file mode 100644
index 00000000000..9d561a92182
--- /dev/null
+++ b/apps/webapp/app/components/metrics/MiniLineChart.tsx
@@ -0,0 +1,194 @@
+import { type ReactNode } from "react";
+import { Line, LineChart, ReferenceLine, Tooltip, type TooltipProps, YAxis } from "recharts";
+import { formatDateTime } from "~/components/primitives/DateTime";
+import { Header3 } from "~/components/primitives/Headers";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import TooltipPortal from "~/components/primitives/TooltipPortal";
+import {
+ ACTIVITY_CHART_HEIGHT,
+ ACTIVITY_CHART_PEAK_CLASS,
+ ACTIVITY_CHART_WIDTH,
+} from "./ActivityBarChart";
+
+type UnitLabel = { singular: string; plural: string };
+
+/** Extra px above the plot so the hover activeDot at the peak value isn't clipped by the SVG edge. */
+const DOT_HEADROOM = 3;
+
+type MiniLineChartDatum = {
+ date: Date;
+ count: number;
+ /** Raw per-bucket throttled count (tooltip). */
+ throttledCount: number;
+ /** The queued value again, present only around throttled buckets, so the warning overlay
+ * retraces the same line and reads as one line changing colour. */
+ throttledOverlay: number | null;
+};
+
+export type MiniLineChartProps = {
+ /** Equal-width time buckets, oldest first. */
+ data?: number[];
+ /**
+ * Per-bucket throttled counts aligned 1:1 with `data`. Where throttling occurred, the queued
+ * line itself is retraced in the warning colour — one line that changes colour, with the
+ * throttled magnitude carried by the tooltip.
+ */
+ throttled?: number[];
+ /** Epoch ms of the first bucket's start. When omitted, the last bucket is anchored to now. */
+ bucketStartMs?: number;
+ /** Width of each bucket in ms. Defaults to one hour. */
+ bucketIntervalMs?: number;
+ /** Line colour for the queued series. */
+ color?: string;
+ /** Trailing peak scalar shown after the chart. Defaults to the max of the buckets. */
+ peak?: number;
+ /** Format the trailing peak label. Defaults to `toLocaleString`. */
+ formatPeak?: (peak: number) => string;
+ /** Tooltip content shown on hover of the trailing peak label. */
+ peakTooltip?: ReactNode;
+ /** Unit shown in the per-bucket tooltip (e.g. queued, runs). */
+ unitLabel?: UnitLabel;
+ /** Chart width in px. Defaults to the shared ACTIVITY_CHART_WIDTH. */
+ width?: number;
+ /** Show the trailing peak label to the right of the chart. Defaults to true. */
+ showPeak?: boolean;
+};
+
+/**
+ * Inline fixed-size mini line sparkline for list rows, plus a trailing peak label. Presentational —
+ * the caller supplies zero/carry-forward-filled buckets. Renders an em-dash when there's no data.
+ * The queued series is a thin monotone line (no dots) matching the big Backlog chart; stretches
+ * where the queue was throttled retrace the same line in the warning colour. Shares its fixed
+ * dimensions and trailing peak label with {@link ActivityBarChart}, but plots lines instead of bars.
+ */
+export function MiniLineChart({
+ data,
+ throttled,
+ bucketStartMs,
+ bucketIntervalMs,
+ color = "var(--color-tasks)",
+ peak: peakOverride,
+ formatPeak,
+ peakTooltip,
+ unitLabel = { singular: "value", plural: "values" },
+ width = ACTIVITY_CHART_WIDTH,
+ showPeak = true,
+}: MiniLineChartProps) {
+ const hasPeakOverride = peakOverride !== undefined;
+ if (!data || data.length === 0 || (data.every((v) => v === 0) && !hasPeakOverride)) {
+ return –;
+ }
+
+ // The overlay only draws where throttling happened. Mapping other buckets to null leaves gaps so
+ // a wholly-zero throttled series never paints over the queued line.
+ const hasThrottled = throttled?.some((v) => v > 0) ?? false;
+
+ const max = Math.max(...data);
+ const peak = peakOverride ?? max;
+
+ // Map each bucket to a dated point so the tooltip can show the window it represents. Buckets are
+ // `intervalMs` wide; if the caller didn't pass the first bucket's start, anchor the last bucket to
+ // now (hourly default).
+ const intervalMs = bucketIntervalMs ?? 3600_000;
+ const startMs = bucketStartMs ?? Date.now() - (data.length - 1) * intervalMs;
+ const chartData: MiniLineChartDatum[] = data.map((count, i) => {
+ const t = throttled?.[i] ?? 0;
+ // Extend the mask one bucket forward (a segment needs both endpoints non-null), so even a
+ // single throttled bucket draws a visible warning stretch.
+ const inOverlay = t > 0 || (throttled?.[i - 1] ?? 0) > 0;
+ return {
+ date: new Date(startMs + i * intervalMs),
+ count,
+ throttledCount: t,
+ throttledOverlay: inOverlay ? count : null,
+ };
+ });
+
+ return (
+
+ {/* +DOT_HEADROOM of extra height, spent as top margin, so the hover activeDot at the peak
+ isn't clipped by the SVG edge while the plotted area stays ACTIVITY_CHART_HEIGHT tall. */}
+
+
+ );
+}
diff --git a/apps/webapp/app/components/primitives/Buttons.tsx b/apps/webapp/app/components/primitives/Buttons.tsx
index 323f3743826..5686fe4d333 100644
--- a/apps/webapp/app/components/primitives/Buttons.tsx
+++ b/apps/webapp/app/components/primitives/Buttons.tsx
@@ -22,6 +22,15 @@ const sizes = {
shortcutVariant: "small" as const,
shortcut: "-ml-0.5 -mr-1.5 justify-self-center",
},
+ // Icon-only small button: fixed width so a row of icon buttons (with different icon
+ // aspect ratios) lines up, e.g. the queue block accessories.
+ "small-icon": {
+ button: "h-6 min-w-[34px] px-2 text-xs",
+ icon: "h-3.5 -mx-1",
+ iconSpacing: "gap-x-2.5",
+ shortcutVariant: "small" as const,
+ shortcut: "-ml-0.5 -mr-1.5 justify-self-center",
+ },
medium: {
button: "h-8 px-3 text-sm",
icon: "h-4 -mx-1",
@@ -118,6 +127,7 @@ const variant = {
"primary/large": createVariant("large", "primary"),
"primary/extra-large": createVariant("extra-large", "primary"),
"secondary/small": createVariant("small", "secondary"),
+ "secondary/small-icon": createVariant("small-icon", "secondary"),
"secondary/medium": createVariant("medium", "secondary"),
"secondary/large": createVariant("large", "secondary"),
"secondary/extra-large": createVariant("extra-large", "secondary"),
diff --git a/apps/webapp/app/components/primitives/Table.tsx b/apps/webapp/app/components/primitives/Table.tsx
index 2b740fac0b8..93372f7f8e4 100644
--- a/apps/webapp/app/components/primitives/Table.tsx
+++ b/apps/webapp/app/components/primitives/Table.tsx
@@ -1,3 +1,4 @@
+import { ChevronDownIcon, ChevronUpDownIcon, ChevronUpIcon } from "@heroicons/react/20/solid";
import { ChevronRightIcon } from "@heroicons/react/24/solid";
import { Link } from "@remix-run/react";
import { ClipboardCheckIcon, ClipboardIcon } from "lucide-react";
@@ -181,6 +182,15 @@ type TableHeaderCellProps = TableCellBasicProps & {
hiddenLabel?: boolean;
tooltip?: ReactNode;
disableTooltipHoverableContent?: boolean;
+ /**
+ * When set (together with `onSort`), the header renders a sort indicator and becomes clickable.
+ * `"asc"`/`"desc"` show the active direction; `null` shows the neutral (unsorted) affordance.
+ * This cell is presentational and fully controlled — the parent owns the sort state (see
+ * `useTableSort`).
+ */
+ sortDirection?: "asc" | "desc" | null;
+ /** Invoked when the header is clicked or activated via keyboard. Enables sorting when provided. */
+ onSort?: () => void;
};
export const TableHeaderCell = forwardRef(
@@ -193,6 +203,8 @@ export const TableHeaderCell = forwardRef {
@@ -207,12 +219,48 @@ export const TableHeaderCell = forwardRef{children} : children;
+
+ const tooltipNode = tooltip ? (
+
+ ) : null;
+
+ const sortIndicator = sortable ? (
+
+ {sortDirection === "asc" ? (
+
+ ) : sortDirection === "desc" ? (
+
+ ) : (
+
+ )}
+
+ ) : null;
+
+ const rowClassName = cn("flex items-center gap-1", {
+ "justify-center": alignment === "center",
+ "justify-end": alignment === "right",
+ });
return (
setIsHovered(true)}
- onMouseLeave={() => setIsHovered(false)}
>
- {hiddenLabel ? (
- {children}
+ {sortable ? (
+ // Order is always title → info icon → sort arrows. The info trigger is itself a
);
@@ -259,6 +329,14 @@ type TableCellProps = TableCellBasicProps & {
isSelected?: boolean;
isTabbableCell?: boolean;
children?: ReactNode;
+ /**
+ * Content rendered beside the cell's link/button but OUTSIDE it, so interactive adornments
+ * (tooltip triggers, badges that are themselves buttons) don't nest inside the ``/`
);
}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx
index 6ce526ba7b3..76beaf297f3 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.agents.$agentParam/route.tsx
@@ -6,7 +6,8 @@ import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { z } from "zod";
import { BeakerIcon } from "~/assets/icons/BeakerIcon";
import { CubeSparkleIcon } from "~/assets/icons/CubeSparkleIcon";
-import { PageBody } from "~/components/layout/AppLayout";
+import { PageContainer } from "~/components/layout/AppLayout";
+import { MetricsLayout } from "~/components/layout/MetricsLayout";
import { DirectionSchema, ListPagination } from "~/components/ListPagination";
import { LinkButton } from "~/components/primitives/Buttons";
import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis";
@@ -20,11 +21,6 @@ import { Header2 } from "~/components/primitives/Headers";
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
import * as Property from "~/components/primitives/PropertyTable";
-import {
- ResizableHandle,
- ResizablePanel,
- ResizablePanelGroup,
-} from "~/components/primitives/Resizable";
import { Spinner } from "~/components/primitives/Spinner";
import { TabButton, TabContainer } from "~/components/primitives/Tabs";
import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters";
@@ -45,6 +41,7 @@ import {
import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server";
import { SessionListPresenter } from "~/presenters/v3/SessionListPresenter.server";
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";
+import { getResizableSnapshot } from "~/services/resizablePanel.server";
import { requireUser } from "~/services/session.server";
import {
docsPath,
@@ -174,6 +171,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
})
.catch(() => null);
+ // The [main | config] split is draggable and its width is persisted to a cookie by the
+ // Resizable primitive; hydrate it here so a reload keeps the user's saved split.
+ const sidebarSnapshot = await getResizableSnapshot(request, "agent-detail-sidebar");
+
return typeddefer({
agent,
runActivity,
@@ -182,6 +183,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
llmTokenActivity,
runList,
sessionList,
+ sidebarSnapshot,
});
};
@@ -196,6 +198,7 @@ export default function Page() {
llmTokenActivity,
runList,
sessionList,
+ sidebarSnapshot,
} = useTypedLoaderData();
const organization = useOrganization();
const project = useProject();
@@ -209,7 +212,7 @@ export default function Page() {
const tabLabel = tab === "sessions" ? "Sessions" : "Runs";
return (
- <>
+
-
-
-
-
- {/* Top bar — tabs on the left; TimeFilter + pagination on the right.
- h-10 matches the right-hand sidebar header height. */}
-
-
- {tab === "sessions" ? (
-
-
- {(list) => (list ? : null)}
-
-
- ) : (
-
-
- {(list) => (list ? : null)}
-
-
+
+ {/* Filters — the pinned bar under the NavBar: the TimeFilter and pagination that used to
+ be fused with the tabs now live here, above the charts (Queues list pattern). Left and
+ right clusters are child divs; the slot's baked justify-between spreads them. */}
+
+
- 1 ? `bursts up to ${burstLimit}` : undefined}
- suffixClassName="text-text-dimmed"
- />
-
- 0
- ? `${unlimitedCount} without a limit (can use up to ${envLimit})`
- : "all have limits"
- }
- suffixClassName="text-text-dimmed"
- />
-
-
-
-
- {overAllocated && (
-
- The queue limits add up to more than the environment limit, so queues will compete for
- concurrency when the environment saturates. Reduce limits to guarantee each queue its
- allocation.
-
- )}
-
- {allocation.truncated && (
-
- Showing the first {allocation.queues.length} of {allocation.totalQueues} queues.
- Allocation totals only include the queues shown.
-
- )}
-
-
);
}
-function QueueHealthBadge({
- paused,
- running,
- queued,
- limit,
-}: {
+/** Health as a stock Badge: color carries the state, w-fit keeps it content-width. */
+type QueueHealth = {
paused: boolean;
running: number;
queued: number;
limit: number;
-}) {
- if (paused) {
- return (
-
- Paused
-
- );
- }
- if (running >= limit && queued > 0) {
- return (
-
- At capacity
-
- );
- }
- if (queued > 0) {
- return (
-
- Backlogged
-
- );
- }
- if (running > 0) {
- return (
-
- Active
-
- );
- }
+};
+
+type QueueHealthLabel = "Paused" | "At capacity" | "Backlogged" | "Active" | "Idle";
+
+// Single source of truth for the queue health decision, shared by the badge and the table's
+// health-column sort so the sorted order always matches the labels shown.
+function queueHealthLabel({ paused, running, queued, limit }: QueueHealth): QueueHealthLabel {
+ if (paused) return "Paused";
+ if (running >= limit && queued > 0) return "At capacity";
+ if (queued > 0) return "Backlogged";
+ if (running > 0) return "Active";
+ return "Idle";
+}
+
+const QUEUE_HEALTH_STYLES: Record = {
+ Paused: "text-warning",
+ "At capacity": "text-warning",
+ Backlogged: "text-blue-500",
+ Active: "text-success",
+ Idle: "text-text-dimmed",
+};
+
+function QueueHealthBadge(health: QueueHealth) {
+ const label = queueHealthLabel(health);
return (
-
- Idle
+
+ {label}
);
}
+// The label rendered in the "Limited by" cell, also used to sort that column.
+function queueLimitedByLabel(queue: {
+ concurrency?: { overriddenAt?: Date | string | null } | null;
+ concurrencyLimit?: number | null;
+}): "Override" | "User" | "Environment" {
+ if (queue.concurrency?.overriddenAt) return "Override";
+ if (queue.concurrencyLimit) return "User";
+ return "Environment";
+}
+
+// The `queue_metrics`-prefixed key a queue is stored under (task queues are prefixed `task/`).
+function queueMetricsKey(queue: { type: string; name: string }): string {
+ return `${queue.type === "task" ? "task/" : ""}${queue.name}`;
+}
+
function formatWaitMs(ms: number): string {
if (ms < 1000) return `${Math.round(ms)}ms`;
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
@@ -1549,6 +1548,11 @@ function formatWaitMs(ms: number): string {
return `${(ms / 3_600_000).toFixed(1)}h`;
}
+// Drop a trailing ".00" from whole percentages so "50.00" reads as "50" but "12.50" is preserved.
+function formatOverridePercent(percent: number): string {
+ return Number.isInteger(percent) ? percent.toString() : percent.toFixed(2).replace(/\.?0+$/, "");
+}
+
// Classic Queues page, restored verbatim from before the Queue Metrics feature. Rendered
// when queueMetricsUiEnabled is off so a gated org sees exactly the pre-metrics UI.
function ClassicQueuesView() {
@@ -1570,15 +1574,7 @@ function ClassicQueuesView() {
useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true });
- const limitStatus =
- environment.running === environment.concurrencyLimit * environment.burstFactor
- ? "limit"
- : environment.running > environment.concurrencyLimit
- ? "burst"
- : "within";
-
- const limitClassName =
- limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined;
+ const { limitStatus, limitClassName } = getEnvConcurrencyLimitStatus(environment);
return (
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx
index 560a1760a5d..a2792b4f4d6 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx
@@ -1,10 +1,15 @@
import { type MetaFunction } from "@remix-run/react";
-import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { useMemo } from "react";
+import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { useMemo, type ReactNode } from "react";
+import type { QueueItem } from "@trigger.dev/core/v3/schemas";
import { typedjson, useTypedLoaderData } from "remix-typedjson";
import { z } from "zod";
-import { PageBody, PageContainer } from "~/components/layout/AppLayout";
+import { MainCenteredContainer, PageContainer } from "~/components/layout/AppLayout";
+import { MetricsLayout } from "~/components/layout/MetricsLayout";
+import { BigNumber } from "~/components/metrics/BigNumber";
+import { Header3 } from "~/components/primitives/Headers";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { Spinner } from "~/components/primitives/Spinner";
import { buildActivityTimeAxis } from "~/components/primitives/charts/activityTimeAxis";
import {
Chart,
@@ -12,11 +17,12 @@ import {
type ChartState,
} from "~/components/primitives/charts/ChartCompound";
import { ChartCard } from "~/components/primitives/charts/ChartCard";
+import { ChartSyncProvider } from "~/components/primitives/charts/ChartSyncContext";
+import { useZoomToTimeFilter } from "~/hooks/useZoomToTimeFilter";
import {
QUEUE_METRIC_COLORS as COLORS,
QUEUE_METRICS_DEFAULT_PERIOD,
QueueMetricChartCard as QueueDetailChartCard,
- QueueMetricStat as Stat,
type QueueMetricIds as Ids,
type QueueMetricTimeRange as TimeRangeParams,
clickhouseTimeToMs,
@@ -29,6 +35,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { QueueRetrievePresenter } from "~/presenters/v3/QueueRetrievePresenter.server";
import {
Table,
+ TableBlankRow,
TableBody,
TableCell,
TableHeader,
@@ -36,13 +43,31 @@ import {
TableRow,
} from "~/components/primitives/Table";
import { TabButton, TabContainer } from "~/components/primitives/Tabs";
+import { InfoIconTooltip } from "~/components/primitives/Tooltip";
+import { SearchInput } from "~/components/primitives/SearchInput";
import { engine } from "~/v3/runEngine.server";
import { TimeFilter } from "~/components/runs/v3/SharedFilters";
import { useSearchParams } from "~/hooks/useSearchParam";
+import { useTableSort, type SortColumn } from "~/components/primitives/useTableSort";
import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
import { canAccessQueueMetricsUi } from "~/v3/canAccessQueueMetricsUi.server";
import { requireUserId } from "~/services/session.server";
-import { EnvironmentParamSchema } from "~/utils/pathBuilder";
+import { docsPath, EnvironmentParamSchema, v3RunsPath } from "~/utils/pathBuilder";
+import { formatNumberCompact } from "~/utils/numberFormatter";
+import { cn } from "~/utils/cn";
+import { redirectWithErrorMessage } from "~/models/message.server";
+import { handleQueueMutationAction } from "~/models/queueMutation.server";
+import {
+ QueueOverrideConcurrencyButton,
+ QueuePauseResumeButton,
+} from "~/components/queues/QueueControls";
+import { LinkButton } from "~/components/primitives/Buttons";
+import { RunsIcon } from "~/assets/icons/RunsIcon";
+import { InfoPanel } from "~/components/primitives/InfoPanel";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { InlineCode } from "~/components/code/InlineCode";
+import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon";
+import { BookOpenIcon } from "@heroicons/react/20/solid";
export const meta: MetaFunction = () => [{ title: `Queue metrics | Trigger.dev` }];
@@ -75,17 +100,32 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
const queue = retrieve.queue;
const fullName = queue.type === "task" ? `task/${queue.name}` : queue.name;
- const ckBreakdown = await engine.concurrencyKeyBreakdown(environment, fullName, {
- limit: CK_LIVE_LIMIT,
- });
+ const [ckBreakdown, oldestQueuedAt] = await Promise.all([
+ engine.concurrencyKeyBreakdown(environment, fullName, { limit: CK_LIVE_LIMIT }),
+ // Enqueue time of the oldest run still waiting in the queue right now (any queue, keyed or
+ // not); undefined when the queue is empty. Drives the live "Oldest wait" block.
+ engine.oldestMessageInQueue(environment, fullName),
+ ]);
// Charts + CH-derived stats are fetched client-side per card (see QueueDetailChartCard /
// useQueueMetric) so the drill-down renders instantly. The loader only returns the live
// "now" counts + identifiers the client fetches need.
+ // Link the Queued block to this queue's pending runs (same filter the list uses).
+ const queuedRunsPath = v3RunsPath(
+ { slug: organizationSlug },
+ { slug: projectParam },
+ { slug: envParam },
+ { queues: [fullName], statuses: ["PENDING"], period: "30d", rootOnly: false }
+ );
+
return typedjson({
queue,
fullName,
+ queuedRunsPath,
+ // The override dialog caps at the environment's concurrency limit.
+ environmentConcurrencyLimit: environment.maximumConcurrencyLimit,
ckBreakdown,
+ oldestQueuedAt: oldestQueuedAt ?? null,
loadedAt: Date.now(),
backPath: url.pathname.replace(/\/[^/]+$/, ""),
ids: {
@@ -96,13 +136,80 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
});
};
+export const action = async ({ request, params }: ActionFunctionArgs) => {
+ const userId = await requireUserId(request);
+ const { organizationSlug, projectParam, envParam, queueParam } = ParamsSchema.parse(params);
+
+ const url = new URL(request.url);
+ const redirectPath = `/orgs/${organizationSlug}/projects/${projectParam}/env/${envParam}/queues/${queueParam}${url.search}`;
+
+ if (request.method.toLowerCase() !== "post") {
+ return redirectWithErrorMessage(redirectPath, request, "Wrong method");
+ }
+
+ const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+ if (!project) throw new Response(undefined, { status: 404, statusText: "Project not found" });
+
+ const environment = await findEnvironmentBySlug(project.id, envParam, userId);
+ if (!environment)
+ throw new Response(undefined, { status: 404, statusText: "Environment not found" });
+
+ if (environment.archivedAt) {
+ return redirectWithErrorMessage(redirectPath, request, "This branch is archived");
+ }
+
+ const formData = await request.formData();
+
+ // Pause/resume/override actions are shared with the Queues list route; here we redirect back to
+ // the detail page so the user stays put.
+ const result = await handleQueueMutationAction({
+ request,
+ environment,
+ userId,
+ formData,
+ redirectPath,
+ });
+ if (result) return result;
+
+ return redirectWithErrorMessage(redirectPath, request, "Something went wrong");
+};
+
const CK_LIVE_LIMIT = 50;
+// Whole-queue oldest wait right now: for keyed queues the per-key breakdown carries the oldest
+// enqueue time per key, so the queue's oldest is the max wait across keys; otherwise fall back to
+// the queue's oldest message directly. Returns null when nothing is waiting.
+function wholeQueueOldestWaitMs(
+ breakdown: CkBreakdown,
+ oldestQueuedAt: number | null,
+ now: number
+): number | null {
+ // Only keys with a live backlog (queued > 0) count — a lingering ckIndex entry whose subqueue
+ // has drained would otherwise over-report the oldest wait. Matches the worstKeyNow guard below.
+ const waitingKeys = breakdown.keys.filter((k) => k.queued > 0);
+ if (waitingKeys.length > 0) {
+ return waitingKeys.reduce((max, k) => Math.max(max, now - k.oldestEnqueuedAt), 0);
+ }
+ return oldestQueuedAt !== null ? Math.max(0, now - oldestQueuedAt) : null;
+}
+
export default function Page() {
- const { queue, fullName, ckBreakdown, loadedAt, backPath, ids } =
- useTypedLoaderData();
+ const {
+ queue,
+ fullName,
+ queuedRunsPath,
+ environmentConcurrencyLimit,
+ ckBreakdown,
+ oldestQueuedAt,
+ loadedAt,
+ backPath,
+ ids,
+ } = useTypedLoaderData();
const plan = useCurrentPlan();
- const maxPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number;
+ // Queue metrics are retained for 30 days in ClickHouse, so cap the picker there even for
+ // plans whose query-period limit was raised above it — a longer window would render empty.
+ const planPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number;
+ const maxPeriodDays = Math.min(planPeriodDays ?? 30, 30);
const { value, replace } = useSearchParams();
const timeRange: TimeRangeParams = {
@@ -121,23 +228,27 @@ export default function Page() {
const hasHistory = gateRow
? toNumber(gateRow.peak_keys) > 0 || toNumber(gateRow.peak_wait) > 0
: false;
- const showKeysTab = ckBreakdown.keys.length > 0 || (!gateLoading && hasHistory);
- const view = value("view") === "keys" && showKeysTab ? "keys" : "overview";
+ // Whether this queue has any concurrency-key activity to show (live keys in the ckIndex, or
+ // nonzero CK history in the range). Both tabs always render; when a queue has no keys the
+ // Concurrency keys tab shows an empty state instead of blank charts/table.
+ const hasKeys = ckBreakdown.keys.length > 0 || (!gateLoading && hasHistory);
+ const view = value("view") === "keys" ? "keys" : "overview";
+ const selectedKey = value("key");
return (
-
-
-
-
+
+ {/* Filters — search (concurrency keys) + time filter in one left cluster, above
+ everything, like the Queues list. The time filter scopes the tab charts; search filters
+ the keys table. The bar is pinned by the layout while the page scrolls. */}
+
+
+ {view === "keys" && hasKeys ? (
+
+ ) : null}
+
- {showKeysTab && (
-
- replace({ view: undefined, key: undefined })}
- >
- Overview
-
- replace({ view: "keys" })}
- >
- Concurrency keys
-
-
- )}
+ {/* Live "right now" state of the whole queue — independent of the time filter above.
+ QueueStats renders the stat-tile grid slot (see MetricsLayout.Grid inside it). */}
+
+
+ {/* Tabs + charts share the padded (inset) column. Both tabs always render; the keys tab
+ shows an empty state when the queue has no concurrency keys. */}
+
+
+ replace({ view: undefined, key: undefined })}
+ >
+ Overview
+
+ replace({ view: "keys" })}
+ >
+ Concurrency keys
+
+
{view === "keys" ? (
-
+ hasKeys ? (
+
+ ) : (
+
+ )
) : (
)}
-
-
+
+
+ {/* The per-key table is full-bleed (no inset), matching the Queues list table, so it spans
+ edge to edge. The drill-down that opens under it is charts, so it stays in the padded
+ column. */}
+ {view === "keys" && hasKeys ? (
+ <>
+
+
+
+ {selectedKey ? (
+
+
+
+ ) : null}
+
+ ) : null}
+
);
}
@@ -193,53 +349,87 @@ function OverviewCharts({
timeRange: TimeRangeParams;
queueName: string;
}) {
+ const zoomToTimeFilter = useZoomToTimeFilter();
return (
- <>
-
-
-
-
-
+
+
+
+
+
+
+
+
+
);
}
@@ -253,7 +443,36 @@ type CkBreakdown = {
}>;
};
-function ConcurrencyKeysView({
+// Standard empty state for a queue that has no concurrency keys (no live keys and no CK history).
+// Small, centered panel — same pattern as the runs page's "Create your first task" state.
+function ConcurrencyKeysBlankState() {
+ return (
+
+
+ Concurrency docs
+
+ }
+ >
+
+ This queue doesn't use concurrency keys. Add concurrencyKey to
+ your task to shard the queue per tenant/user.
+
+
+
+ );
+}
+
+function ConcurrencyKeyCharts({
breakdown,
loadedAt,
ids,
@@ -266,52 +485,88 @@ function ConcurrencyKeysView({
timeRange: TimeRangeParams;
queueName: string;
}) {
+ const zoomToTimeFilter = useZoomToTimeFilter();
+ const { value, replace } = useSearchParams();
+ // Same key search as the table: narrows the per-key charts too.
+ const keyFilter = value("query")?.trim().toLowerCase() || undefined;
+
+ // The live most-starved key: among keys with a live backlog, the one whose oldest waiting run
+ // has been waiting longest right now. Names the culprit on the "Worst key wait" card so the chart
+ // (which shows how bad, not who) points at a tenant you can click through to.
+ const worstKeyNow = useMemo(() => {
+ let worst: { key: string; waitMs: number } | null = null;
+ for (const k of breakdown.keys) {
+ if (k.queued <= 0) continue;
+ const waitMs = Math.max(0, loadedAt - k.oldestEnqueuedAt);
+ if (!worst || waitMs > worst.waitMs) worst = { key: k.concurrencyKey, waitMs };
+ }
+ return worst;
+ }, [breakdown, loadedAt]);
+
return (
- <>
-
-
-
-
-
-
+ /* Per-key breakdown: which keys hold the backlog / do the work. */
+
+
+
);
}
+// Live "right now" snapshot of the whole queue: a hybrid feed. The loader (Redis/PG) supplies the
+// first paint so these blocks render instantly; after that a tiny 15s ClickHouse poll keeps them
+// fresh, always reading the newest gauge row and falling back to the loader values until the first
+// poll lands (so we never flash 0). These blocks never change with the filter. Period trends
+// (backlog, throughput, delay over time) live in the charts below.
+// Oldest-wait threshold for the warning tint: the head of the queue sitting unstarted this long
+// signals the queue is stuck, not just busy.
+const OLDEST_WAIT_WARNING_MS = 5 * 60_000;
+
+// How recent the newest ClickHouse gauge bucket must be to drive the live blocks. Above the 10s
+// bucket + pipeline lag; past it we treat the queue as idle and fall back to the loader value.
+const LIVE_GAUGE_FRESH_MS = 90_000;
+
function QueueStats({
queue,
+ environmentConcurrencyLimit,
+ queuedRunsPath,
+ oldestWaitMs,
ids,
timeRange,
queueName,
}: {
- queue: { running: number; queued: number };
+ // Carries the percent override source-of-truth (not part of the shared QueueItem contract) so the
+ // override dialog reopens in percent mode for percent-based overrides.
+ queue: QueueItem & { concurrencyLimitOverridePercent: number | null };
+ environmentConcurrencyLimit: number;
+ queuedRunsPath: string;
+ oldestWaitMs: number | null;
ids: Ids;
timeRange: TimeRangeParams;
queueName: string;
}) {
- // One scalar query feeds the CH-derived stats; the "now" counts come from the loader (live).
- const { rows, showLoading } = useQueueMetric(
- `SELECT max(max_limit) AS lim, max(max_queued) AS peak_queued, deltaSumTimestampMerge(started_delta) AS started,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS worst_p95\nFROM queue_metrics`,
+ // Live "now" is empty for a queue that just drained, so add the window peak/worst as a dimmed
+ // caption (styled like "bursts up to 50" on the Queues list) — a quiet queue still shows how
+ // backed up it got recently. "worst_wait" is the window's worst head-of-line wait (max of the
+ // oldest still-waiting run's age), the same metric as the live "Oldest wait" headline — not the
+ // scheduling-delay percentiles of runs that already started. Only concurrency-keyed queues record
+ // this (max_ck_wait_ms), so it's 0/absent for non-keyed queues.
+ const { rows } = useQueueMetric(
+ `SELECT max(max_queued) AS peak_queued,\n max(max_ck_wait_ms) AS worst_wait\nFROM queue_metrics`,
{ ids, timeRange, queueName }
);
- const row = rows[0];
- const worstP95 = row ? toNumber(row.worst_p95) : 0;
+ const peakQueued = rows[0] ? toNumber(rows[0].peak_queued) : 0;
+ const worstWaitMs = rows[0] ? toNumber(rows[0].worst_wait) : 0;
+
+ // Latest gauges from ClickHouse, polled every 15s so the live blocks keep ticking after first
+ // paint. Read the newest bucket (largest t); until the first poll lands liveRows is empty and the
+ // *Live values stay null, so the blocks show the loader values instead of flashing 0.
+ const { rows: liveRows } = useQueueMetric(
+ `SELECT timeBucket() AS t, max(max_running) AS running, max(max_queued) AS queued, max(max_limit) AS q_limit, max(max_ck_wait_ms) AS ck_wait FROM queue_metrics GROUP BY t ORDER BY t`,
+ {
+ ids,
+ timeRange: { period: "15m", from: null, to: null },
+ defaultPeriod: "15m",
+ queueName,
+ refreshIntervalMs: 15_000,
+ }
+ );
+ // Gauges are only emitted while the queue is active, so a drained queue's newest bucket is a past
+ // one holding its last non-zero reading. Trust the CH gauge only when its newest bucket is recent
+ // (covers the 10s bucket + pipeline lag); once it ages out we fall back to the loader's live
+ // Redis/PG value instead of lingering on a stale count.
+ const latest = liveRows.length > 0 ? liveRows[liveRows.length - 1] : undefined;
+ const latestBucketMs = latest ? clickhouseTimeToMs(latest.t) : NaN;
+ const liveFresh =
+ Number.isFinite(latestBucketMs) && Date.now() - latestBucketMs < LIVE_GAUGE_FRESH_MS;
+ const fresh = latest && liveFresh ? latest : undefined;
+ const runningLive = fresh ? toNumber(fresh.running) : null;
+ const queuedLive = fresh ? toNumber(fresh.queued) : null;
+ const limitLive = fresh ? toNumber(fresh.q_limit) : null;
+ const ckWaitLive = fresh ? toNumber(fresh.ck_wait) : null;
+
+ // Prefer CH once it has landed; loader values before that.
+ const runningDisplay = runningLive ?? queue.running;
+ const queuedDisplay = queuedLive ?? queue.queued;
+ // Limit is queue config, not a live signal: keep the loader's value. Only if the loader had none
+ // do we fall back to the CH gauge for display.
+ const limitDisplay = queue.concurrencyLimit ?? (limitLive || null);
+ // Keyed queues report head-of-line wait via CH (max_ck_wait_ms); use it as the live headline when
+ // present. Non-keyed queues have no CH signal, so they stay on the loader value.
+ const oldestWaitDisplayMs = ckWaitLive !== null && ckWaitLive > 0 ? ckWaitLive : oldestWaitMs;
return (
-
-
-
-
-
+
+
+
+
+ }
/>
- 0 ? `peak ${formatNumberCompact(peakQueued)}` : undefined}
+ suffixClassName="text-text-dimmed"
+ accessory={
+
+ }
/>
- 0 ? formatWaitMs(worstP95) : "–"}
- loading={showLoading}
- className={worstP95 >= 60_000 ? "text-warning" : undefined}
+ 0
+ ? formatWaitMs(oldestWaitDisplayMs)
+ : "0"
+ }
+ valueClassName={cn(
+ "tabular-nums",
+ oldestWaitDisplayMs !== null &&
+ oldestWaitDisplayMs >= OLDEST_WAIT_WARNING_MS &&
+ "text-warning"
+ )}
+ suffix={worstWaitMs > 0 ? `worst ${formatWaitMs(worstWaitMs)}` : undefined}
+ suffixClassName="text-text-dimmed"
/>
+
+ );
+}
+
+/** Live concurrency as a single block: running vs limit with a utilization bar, so "how close to
+ * the ceiling" reads at a glance instead of two separate numbers. Warning-tinted at/over the limit. */
+function ConcurrencyBlock({
+ running,
+ limit,
+ paused = false,
+ loading,
+ accessory,
+}: {
+ running: number;
+ limit: number | null;
+ paused?: boolean;
+ loading?: boolean;
+ accessory?: ReactNode;
+}) {
+ const atLimit = limit !== null && limit > 0 && running >= limit;
+ const pct = limit && limit > 0 ? Math.min(100, Math.round((running / limit) * 100)) : 0;
+ return (
+
+
+
+ Concurrency
+ {paused ? paused : null}
+
+ {accessory ?
{accessory}
: null}
+
+ {loading ? (
+
+ ) : (
+
+
+
+ {running.toLocaleString()}
+
+
+ / {limit !== null ? limit.toLocaleString() : "∞"}
+
+ {limit !== null && limit > 0 && (
+
+ {/* Separator so the limit and the percentage don't read as one number
+ (e.g. "/ 25" + "44%" mashing into "2544%"). */}
+ ·
+ {pct}% of limit
+
+ )}
+
+ {limit !== null && limit > 0 && (
+
+
+
+ )}
+
+ )}
);
}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx
index 2cc8b01110b..5e7082b2411 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx
@@ -7,7 +7,8 @@ import { redirect, typeddefer, useTypedLoaderData } from "remix-typedjson";
import { URL } from "url";
import { UsageBar } from "~/components/billing/UsageBar";
import { getUsageBarBillingLimitDollars } from "~/components/billing/billingAlertsFormat";
-import { PageBody, PageContainer } from "~/components/layout/AppLayout";
+import { PageContainer } from "~/components/layout/AppLayout";
+import { MetricsLayout } from "~/components/layout/MetricsLayout";
import { Card } from "~/components/primitives/charts/Card";
import type { ChartConfig } from "~/components/primitives/charts/Chart";
import { Chart } from "~/components/primitives/charts/ChartCompound";
@@ -126,13 +127,12 @@ export default function Page() {
-
-
-
- Dev environment runs are excluded from the usage data above, since they do
- not have an associated compute cost.
-
-
- );
- }}
-
-
-
-
-
+ ))
+ )}
+
+
+
+ Dev environment runs are excluded from the usage data above, since they do not
+ have an associated compute cost.
+
+
+ );
+ }}
+
+
+
+
);
}
diff --git a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
index 908e8f449a0..9717aa2b942 100644
--- a/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
+++ b/apps/webapp/app/routes/admin.api.v1.environments.$environmentId.ts
@@ -6,6 +6,7 @@ import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { engine } from "~/v3/runEngine.server";
import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
const ParamsSchema = z.object({
environmentId: z.string(),
@@ -46,6 +47,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
await updateEnvConcurrencyLimits(environment);
+ // Percent-based queue overrides follow the environment limit automatically.
+ await concurrencySystem.queues.recalculatePercentLimits(environment);
+
// Org max-concurrency changed too, which is embedded in every env of the org; invalidating
// the org drops the env/authEnv rows for all of them (including this env).
controlPlaneResolver.invalidateOrganization(environment.organizationId);
diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
index c99637a0d10..403df19fe37 100644
--- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
+++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.concurrency.ts
@@ -5,6 +5,7 @@ import { prisma } from "~/db.server";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { updateEnvConcurrencyLimits } from "~/v3/runQueue.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
const ParamsSchema = z.object({
organizationId: z.string(),
@@ -82,6 +83,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
});
await updateEnvConcurrencyLimits({ ...modifiedEnvironment, organization });
+
+ // Percent-based queue overrides follow the environment limit automatically.
+ await concurrencySystem.queues.recalculatePercentLimits({
+ ...modifiedEnvironment,
+ organization,
+ });
}
// Org + every affected env's concurrency changed; one org invalidation covers them all.
diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
index 3223a2a6062..acc723d61cb 100644
--- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
+++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts
@@ -4,11 +4,23 @@ import { z } from "zod";
import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server";
import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
+import {
+ MAX_QUEUE_OVERRIDE_PERCENT,
+ MIN_QUEUE_OVERRIDE_PERCENT,
+} from "~/v3/services/concurrencySystem.server";
-const BodySchema = z.object({
- type: RetrieveQueueType.default("id"),
- concurrencyLimit: z.number().int().min(0).max(100000),
-});
+const BodySchema = z
+ .object({
+ type: RetrieveQueueType.default("id"),
+ // Absolute concurrency limit. Backwards compatible with existing callers.
+ concurrencyLimit: z.number().int().min(0).max(100000).optional(),
+ // Percentage of the environment's maximum concurrency limit (0 < percent <= 100).
+ // Stored as the source of truth; the absolute limit is materialized from it.
+ percent: z.number().gt(MIN_QUEUE_OVERRIDE_PERCENT).max(MAX_QUEUE_OVERRIDE_PERCENT).optional(),
+ })
+ .refine((body) => (body.concurrencyLimit === undefined) !== (body.percent === undefined), {
+ message: "Provide exactly one of `concurrencyLimit` or `percent`",
+ });
export const { action } = createActionApiRoute(
{
@@ -26,8 +38,11 @@ export const { action } = createActionApiRoute(
name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"),
};
+ const override =
+ body.percent !== undefined ? { percent: body.percent } : { limit: body.concurrencyLimit! };
+
return concurrencySystem.queues
- .overrideQueueConcurrencyLimit(authentication.environment, input, body.concurrencyLimit)
+ .overrideQueueConcurrencyLimit(authentication.environment, input, override)
.match(
(queue) => {
return json(
@@ -51,6 +66,10 @@ export const { action } = createActionApiRoute(
case "queue_not_found": {
return json({ error: "Queue not found" }, { status: 404 });
}
+ case "invalid_override":
+ case "concurrency_limit_exceeds_maximum": {
+ return json({ error: error.message }, { status: 400 });
+ }
case "queue_update_failed": {
return json({ error: "Failed to update queue concurrency limit" }, { status: 500 });
}
diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
index da36e2871d4..047a6ab4342 100644
--- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
+++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx
@@ -1218,7 +1218,7 @@ function WaitingInQueueBlock({
title="Backlog"
headline={`${waiting.queued.toLocaleString()} queued`}
query={`SELECT timeBucket() AS t, max(max_queued) AS v\nFROM queue_metrics\nGROUP BY t\nORDER BY t`}
- color="#A78BFA"
+ color="var(--color-tasks)"
ids={waiting.ids}
queueName={queueName}
/>
diff --git a/apps/webapp/app/routes/storybook.charts/route.tsx b/apps/webapp/app/routes/storybook.charts/route.tsx
index 8c7580cb613..040211437ad 100644
--- a/apps/webapp/app/routes/storybook.charts/route.tsx
+++ b/apps/webapp/app/routes/storybook.charts/route.tsx
@@ -13,6 +13,7 @@ import {
useDateRange,
} from "~/components/primitives/charts/DateRangeContext";
import type { ZoomRange } from "~/components/primitives/charts/hooks/useZoomSelection";
+import { MiniLineChart } from "~/components/metrics/MiniLineChart";
import { Paragraph } from "~/components/primitives/Paragraph";
import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton";
import SegmentedControl from "~/components/primitives/SegmentedControl";
@@ -262,6 +263,126 @@ function ChartsDashboard() {
/>
+
+ {/* Line with per-bucket warning overlay (queues redesign) */}
+
+
+
+
+
+ {/* The base line stays the series colour; buckets strictly above `threshold` retrace
+ in the warning colour, so the same line reads blue -> yellow -> blue as it crosses. */}
+
+
+
+
+
+
+ {/* Line with gradient threshold split (queues redesign) */}
+
+
+
+
+ Threshold stroke{" "}
+ (gradient split)
+
+
+
+ {/* Gradient split above the threshold. Best when the threshold sits mid-domain — if it
+ sits far below the data max the split collapses onto the baseline; prefer
+ `warningOverlay` in that case (see the Warning overlay example). */}
+
+
+
+
+
+
+ {/* Line with outside-placed reference line label (queues redesign) */}
+
+
+
+
+ Reference line{" "}
+ (label outside, in the gutter)
+
+
+
+ {/* labelPlacement: "outside" renders the label in the right gutter at the line's y;
+ the chart's right margin is widened automatically so it isn't clipped. */}
+
+
+
+
+
+
+ {/* MiniLineChart backlog sparklines (queues redesign) */}
+
+
+
+
+ Mini line chart{" "}
+ (inline backlog sparkline)
+
+
+
+ {/* Fixed-size inline sparkline sized for a table cell, with a trailing peak label. The
+ second row passes aligned `throttled` buckets so throttled stretches retrace the same
+ line in the warning colour. */}
+
+
+
+
Queue
+
Backlog
+
+
+
+
+
emails
+
+
+
+
+
+
image-processing
+
+
+
+
+
+
+
+
);
@@ -681,6 +802,30 @@ const API_DATA = {
"analyze-document": 5678,
},
],
+ // Single-series queue depth that crosses the 5,000 threshold twice (blue -> yellow -> blue),
+ // used by the warning overlay, threshold stroke and outside reference-line examples.
+ queueDepthData: [
+ { day: "2023-11-01", queued: 1200 },
+ { day: "2023-11-02", queued: 2400 },
+ { day: "2023-11-03", queued: 3800 },
+ { day: "2023-11-04", queued: 4600 },
+ { day: "2023-11-05", queued: 6200 },
+ { day: "2023-11-06", queued: 7400 },
+ { day: "2023-11-07", queued: 6800 },
+ { day: "2023-11-08", queued: 5200 },
+ { day: "2023-11-09", queued: 3600 },
+ { day: "2023-11-10", queued: 2800 },
+ { day: "2023-11-11", queued: 3400 },
+ { day: "2023-11-12", queued: 5600 },
+ { day: "2023-11-13", queued: 7800 },
+ { day: "2023-11-14", queued: 6400 },
+ { day: "2023-11-15", queued: 4200 },
+ ],
+ // Inline sparkline buckets (oldest first). No throttling on this one.
+ miniLineData: [3, 5, 8, 12, 9, 14, 20, 18, 11, 7, 4, 6],
+ // A backlog that was throttled across a stretch in the middle; `throttled` aligns 1:1 with `data`.
+ miniLineThrottledData: [2, 4, 9, 16, 24, 30, 27, 19, 12, 8, 5, 3],
+ miniLineThrottledBuckets: [0, 0, 0, 6, 14, 18, 11, 0, 0, 0, 0, 0],
};
const lineChartConfig = {
@@ -694,6 +839,13 @@ const lineChartConfig = {
},
} satisfies ChartConfig;
+const queueDepthConfig = {
+ queued: {
+ label: "Queue depth",
+ color: "var(--color-tasks)",
+ },
+} satisfies ChartConfig;
+
const barChartBigDatasetConfig = {
"sync-data": {
label: (
diff --git a/apps/webapp/app/routes/storybook.layout/route.tsx b/apps/webapp/app/routes/storybook.layout/route.tsx
new file mode 100644
index 00000000000..08242fe51da
--- /dev/null
+++ b/apps/webapp/app/routes/storybook.layout/route.tsx
@@ -0,0 +1,373 @@
+import { useState } from "react";
+import { PageContainer } from "~/components/layout/AppLayout";
+import { MetricsLayout } from "~/components/layout/MetricsLayout";
+import { Badge } from "~/components/primitives/Badge";
+import { Header3 } from "~/components/primitives/Headers";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import SegmentedControl from "~/components/primitives/SegmentedControl";
+import {
+ Table,
+ TableBody,
+ TableCell,
+ TableHeader,
+ TableHeaderCell,
+ TableRow,
+} from "~/components/primitives/Table";
+import { TabButton, TabContainer } from "~/components/primitives/Tabs";
+import { cn } from "~/utils/cn";
+
+// A placeholder for a search input / TimeFilter / pagination — the real pages drop live controls
+// into the Filters slot; here we only show the row shape.
+function FilterChip({ children }: { children: React.ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+// A stat BigNumber-style tile.
+function StatTile({ label, value }: { label: string; value: string }) {
+ return (
+
+ {label}
+
+ {value}
+
+
+ );
+}
+
+// A chart-card-style tile with a fake bar sparkline so the grid's height + column behaviour is
+// visible.
+function ChartTile({ label, className }: { label: string; className?: string }) {
+ return (
+
+ );
+}
+
+// The config panel dropped into MetricsLayout.Sidebar in the sidebar demos.
+function SidebarPanel({ resizable }: { resizable?: boolean }) {
+ return (
+
+
+ Sidebar slot
+
+
+
+ {resizable
+ ? "This sidebar is resizable — drag the handle on its left edge. Pass an autosaveId + a loader snapshot to persist the split across reloads."
+ : "This sidebar is fixed-width (width prop). The main column fills the rest and owns the page scroll."}
+
+ {resizable ? "resizable" : "fixed width"}
+ {Array.from({ length: 6 }).map((_, i) => (
+ Config option {i + 1}
+ ))}
+
+
+ );
+}
+
+// The overview demo. Every slot carries its baked chrome — no className is passed to Root,
+// Filters, Grid or Content. The pinned Filters bar spreads a left and right cluster; the Grids
+// bake the page gutter and adapt their columns (or take an explicit `columns` / `kind="charts"`);
+// Content toggles between a full-bleed table and an inset panel. The whole page scrolls as one.
+function OverviewDemo() {
+ const [tab, setTab] = useState<"panel" | "table">("panel");
+
+ return (
+
+ {/* Filters slot — the pinned bar directly under the NavBar. Left + right clusters as child
+ divs; the slot bakes the 40px height, border and insets. */}
+
+
+ Search…
+ Period: 7d
+ Filters slot
+
+ {"< 1 / 4 >"}
+
+
+ {/* Grid slot — 4 stat tiles. Columns are derived from the tile count: two-up, four-up
+ from lg. The gutter + gap are baked. */}
+
+
+
+
+
+
+
+
+
+ {/* Content slot — a doubled separation above it is baked in, so the tiles read as their own
+ band. `inset` toggles between a padded column (panel) and full-bleed (edge-to-edge
+ table). */}
+
+
+ setTab("panel")}
+ >
+ Panel (inset)
+
+ setTab("table")}
+ >
+ Table (full-bleed)
+
+
+ {tab === "table" ? (
+
+ ) : (
+
+
+ With inset, Content becomes a padded column: this panel and the tabs
+ above it sit on the standard page gutter. Switch to the table to see Content go
+ full-bleed — the table spans edge to edge with its own top border. Either way the
+ whole page (filters aside) shares one vertical scroll.
+
+
+ )}
+
+
+ );
+}
+
+// Fixed-width sidebar: a child flips Root into a [main | sidebar] layout.
+// The main column keeps its normal top-to-bottom slots and owns the page scroll.
+function SidebarFixedDemo() {
+ return (
+
+
+
+ Search…
+ main column
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+// Resizable sidebar: same [main | sidebar] layout, but the split is draggable via the shared
+// Resizable primitives. autosaveId persists the split to a cookie (in a real page the loader
+// hydrates it back through a snapshot); here it persists live within the session.
+function SidebarResizableDemo() {
+ return (
+
+
+
+ Search…
+ drag the handle →
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+// scroll="regions": Root does NOT create the page scroll. It only bounds the height as a flex
+// column, so the page composes its own independently-scrolling areas — here a fixed toolbar over
+// two side-by-side lists that each scroll on their own.
+function RegionsDemo() {
+ return (
+
+
+ fixed toolbar (does not scroll)
+ Period: 7d
+
+
+
+
+ Left region — scrolls independently
+
+
+
+
+
+ Right region — scrolls independently
+
+
+
+
+
+ );
+}
+
+type Demo = "overview" | "sidebar-fixed" | "sidebar-resizable" | "regions";
+
+const DEMO_OPTIONS: { label: string; value: Demo }[] = [
+ { label: "Overview", value: "overview" },
+ { label: "Sidebar (fixed)", value: "sidebar-fixed" },
+ { label: "Sidebar (resizable)", value: "sidebar-resizable" },
+ { label: "Scroll: regions", value: "regions" },
+];
+
+/**
+ * Storybook for the MetricsLayout compound. A segmented control swaps between demos, each filling
+ * the page:
+ * - Overview — the base Filters / count-adaptive Grid / Content slots, page scroll.
+ * - Sidebar (fixed) — a fixed-width MetricsLayout.Sidebar beside the main column.
+ * - Sidebar (resizable) — the same, but with a draggable split (autosaveId persistence).
+ * - Scroll: regions — scroll="regions" with two independently-scrolling areas.
+ */
+export default function Story() {
+ const [demo, setDemo] = useState("overview");
+
+ return (
+
+
+
+
+ setDemo(value as Demo)}
+ />
+
+
+ {demo === "overview" ? (
+
+ ) : demo === "sidebar-fixed" ? (
+
+ ) : demo === "sidebar-resizable" ? (
+
+ ) : (
+
+ )}
+
+ );
+}
diff --git a/apps/webapp/app/routes/storybook/route.tsx b/apps/webapp/app/routes/storybook/route.tsx
index 012d47827de..8309fee5b2b 100644
--- a/apps/webapp/app/routes/storybook/route.tsx
+++ b/apps/webapp/app/routes/storybook/route.tsx
@@ -71,6 +71,10 @@ const stories: Story[] = [
name: "Inline code",
slug: "inline-code",
},
+ {
+ name: "Layout",
+ slug: "layout",
+ },
{
name: "Loading bar divider",
slug: "loading-bar-divider",
diff --git a/apps/webapp/app/v3/services/allocateConcurrency.server.ts b/apps/webapp/app/v3/services/allocateConcurrency.server.ts
index b9a28daba60..78c8b82915a 100644
--- a/apps/webapp/app/v3/services/allocateConcurrency.server.ts
+++ b/apps/webapp/app/v3/services/allocateConcurrency.server.ts
@@ -3,6 +3,7 @@ import { ManageConcurrencyPresenter } from "~/presenters/v3/ManageConcurrencyPre
import { BaseService } from "./baseService.server";
import { updateEnvConcurrencyLimits } from "../runQueue.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { concurrencySystem } from "./concurrencySystemInstance.server";
type Input = {
userId: string;
@@ -90,6 +91,14 @@ export class AllocateConcurrencyService extends BaseService {
await updateEnvConcurrencyLimits(updatedEnvironment);
}
+ // Percent-based queue overrides follow the environment limit automatically. Note the
+ // deliberate asymmetry with the env-level push above: `updateEnvConcurrencyLimits` is gated
+ // on `!paused`, but we recalculate queue limits even for paused environments. Queue-level
+ // pushes on a paused env are inert (the env-level gate stops dequeueing regardless), and
+ // keeping the queue limits synced means resume needs no extra reconciliation — skipping
+ // them here would instead leave stale engine limits after the env resumes.
+ await concurrencySystem.queues.recalculatePercentLimits(updatedEnvironment);
+
// maximumConcurrencyLimit changed in the control-plane; drop any cached copy.
controlPlaneResolver.invalidateEnvironment(environment.id);
}
diff --git a/apps/webapp/app/v3/services/concurrencySystem.server.ts b/apps/webapp/app/v3/services/concurrencySystem.server.ts
index 488f38ce279..f030cb72e5f 100644
--- a/apps/webapp/app/v3/services/concurrencySystem.server.ts
+++ b/apps/webapp/app/v3/services/concurrencySystem.server.ts
@@ -2,6 +2,7 @@ import type { TaskQueue, User } from "@trigger.dev/database";
import { errAsync, fromPromise, okAsync } from "neverthrow";
import type { PrismaClientOrTransaction } from "~/db.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
import { removeQueueConcurrencyLimits, updateQueueConcurrencyLimits } from "../runQueue.server";
import { engine } from "../runEngine.server";
@@ -12,6 +13,42 @@ export type ConcurrencySystemOptions = {
export type QueueInput = string | { type: "task" | "custom"; name: string };
+/**
+ * The concurrency-limit override to apply to a queue. Either an absolute `limit` or a `percent`
+ * of the environment's maximum concurrency limit. A bare `number` is accepted for backwards
+ * compatibility and is treated as an absolute limit.
+ */
+export type ConcurrencyLimitOverride = number | { limit: number } | { percent: number };
+
+/**
+ * Materializes an absolute concurrency limit from a percentage of the environment limit.
+ * Reused by the recalculation task that runs when an environment limit changes.
+ *
+ * Clamped to `>= 1` so a percent-based override never produces a `0` (pause-like) limit, and
+ * to `<= envLimit` so it can never exceed the environment maximum.
+ */
+/**
+ * Valid range for a percent-based queue concurrency override: greater than 0 and up to 100% of
+ * the environment limit. Shared by every layer that validates the percent (the API zod schema,
+ * the dashboard mutation handler, and the override service) so the bound never drifts apart.
+ */
+export const MIN_QUEUE_OVERRIDE_PERCENT = 0;
+export const MAX_QUEUE_OVERRIDE_PERCENT = 100;
+
+/** Whether `percent` is a valid queue-override percentage (0 < percent <= 100). */
+export function isValidQueueOverridePercent(percent: number): boolean {
+ return (
+ Number.isFinite(percent) &&
+ percent > MIN_QUEUE_OVERRIDE_PERCENT &&
+ percent <= MAX_QUEUE_OVERRIDE_PERCENT
+ );
+}
+
+export function materializePercentLimit(envLimit: number, percent: number): number {
+ const materialized = Math.floor((envLimit * percent) / 100);
+ return Math.min(Math.max(materialized, 1), envLimit);
+}
+
export class ConcurrencySystem {
constructor(private readonly options: ConcurrencySystemOptions) {}
@@ -24,18 +61,12 @@ export class ConcurrencySystem {
overrideQueueConcurrencyLimit: (
environment: AuthenticatedEnvironment,
queue: QueueInput,
- concurrencyLimit: number,
+ override: ConcurrencyLimitOverride,
overriddenBy?: User
) => {
return findQueueFromInput(this.db, environment, queue)
.andThen((queue) =>
- overrideQueueConcurrencyLimit(
- this.db,
- environment,
- queue,
- concurrencyLimit,
- overriddenBy
- )
+ overrideQueueConcurrencyLimit(this.db, environment, queue, override, overriddenBy)
)
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
.andThen((queue) => getQueueStats(environment, queue));
@@ -46,6 +77,59 @@ export class ConcurrencySystem {
.andThen((queue) => syncQueueConcurrencyToEngine(environment, queue))
.andThen((queue) => getQueueStats(environment, queue));
},
+ /**
+ * Recalculates the materialized limit of every percent-based override in the environment
+ * against its CURRENT maximumConcurrencyLimit and syncs changed queues to the run engine.
+ * Call AFTER the environment-limit DB update has committed (engine syncs must not run
+ * inside an open transaction). Idempotent: unchanged queues are skipped. One failing queue
+ * is logged and skipped so the rest still converge.
+ */
+ recalculatePercentLimits: async (environment: AuthenticatedEnvironment) => {
+ const queues = await this.db.taskQueue.findMany({
+ where: {
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimitOverridePercent: { not: null },
+ },
+ });
+
+ let updated = 0;
+ for (const queue of queues) {
+ try {
+ const percent = queue.concurrencyLimitOverridePercent;
+ if (percent === null) continue;
+ const newLimit = materializePercentLimit(
+ environment.maximumConcurrencyLimit,
+ percent.toNumber()
+ );
+
+ // Only write the DB when the materialized value actually changed.
+ if (newLimit !== queue.concurrencyLimit) {
+ await this.db.taskQueue.update({
+ where: { id: queue.id },
+ data: { concurrencyLimit: newLimit },
+ });
+ updated++;
+ }
+
+ // Always attempt the engine push (it's idempotent) for active queues — even when the
+ // DB value was unchanged — so a previously-failed sync self-heals on the next recalc
+ // instead of leaving the DB and engine diverged forever. Paused queues keep their
+ // engine limit at 0 (the pause/resume flow re-syncs from the stored value on resume);
+ // push nothing for them so a percent recalc never effectively un-pauses a queue.
+ if (!queue.paused) {
+ await updateQueueConcurrencyLimits(environment, queue.name, newLimit);
+ }
+ } catch (error) {
+ logger.error("Failed to recalculate percent queue limit", {
+ queueId: queue.id,
+ environmentId: environment.id,
+ error,
+ });
+ }
+ }
+
+ return { total: queues.length, updated };
+ },
};
}
}
@@ -117,13 +201,49 @@ function overrideQueueConcurrencyLimit(
db: PrismaClientOrTransaction,
environment: AuthenticatedEnvironment,
queue: TaskQueue,
- concurrencyLimit: number,
+ override: ConcurrencyLimitOverride,
overriddenBy?: User
) {
- const newConcurrencyLimit = Math.max(
- Math.min(concurrencyLimit, environment.maximumConcurrencyLimit),
- 0
- );
+ const maximum = environment.maximumConcurrencyLimit;
+
+ // Normalize the input into the absolute limit to persist and the percent source-of-truth
+ // (null for absolute overrides).
+ let newConcurrencyLimit: number;
+ let overridePercent: number | null;
+
+ if (typeof override === "object" && "percent" in override) {
+ const percent = override.percent;
+
+ if (!isValidQueueOverridePercent(percent)) {
+ return errAsync({
+ type: "invalid_override" as const,
+ message: `Percent must be greater than ${MIN_QUEUE_OVERRIDE_PERCENT} and less than or equal to ${MAX_QUEUE_OVERRIDE_PERCENT}`,
+ });
+ }
+
+ newConcurrencyLimit = materializePercentLimit(maximum, percent);
+ overridePercent = percent;
+ } else {
+ const limit = typeof override === "number" ? override : override.limit;
+
+ if (!Number.isFinite(limit) || limit < 0) {
+ return errAsync({
+ type: "invalid_override" as const,
+ message: "Concurrency limit must be a non-negative number",
+ });
+ }
+
+ // Cap: an absolute override may not exceed the environment limit. Reject rather than clamp.
+ if (limit > maximum) {
+ return errAsync({
+ type: "concurrency_limit_exceeds_maximum" as const,
+ message: `Concurrency limit (${limit}) cannot exceed the environment limit (${maximum})`,
+ });
+ }
+
+ newConcurrencyLimit = limit;
+ overridePercent = null;
+ }
const concurrencyLimitBase = queue.concurrencyLimitOverriddenAt
? queue.concurrencyLimitBase
@@ -137,6 +257,7 @@ function overrideQueueConcurrencyLimit(
data: {
concurrencyLimit: newConcurrencyLimit,
concurrencyLimitBase: concurrencyLimitBase ?? null,
+ concurrencyLimitOverridePercent: overridePercent,
concurrencyLimitOverriddenAt: new Date(),
concurrencyLimitOverriddenBy: overriddenBy?.id ?? null,
},
@@ -162,6 +283,7 @@ function resetQueueConcurrencyLimit(db: PrismaClientOrTransaction, queue: TaskQu
concurrencyLimitOverriddenAt: null,
concurrencyLimit: newConcurrencyLimit,
concurrencyLimitBase: null,
+ concurrencyLimitOverridePercent: null,
concurrencyLimitOverriddenBy: null,
},
}),
diff --git a/apps/webapp/seed-queue-metrics.mts b/apps/webapp/seed-queue-metrics.mts
index 709ba8f25ed..911ce51d9c6 100644
--- a/apps/webapp/seed-queue-metrics.mts
+++ b/apps/webapp/seed-queue-metrics.mts
@@ -615,6 +615,20 @@ async function stageRedisUsage(scenario: Scenario, ids: Ids, seed: number, clear
const prefix = "engine:runqueue:";
const logicalBase = `{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:queue:`;
const base = `${prefix}${logicalBase}`;
+
+ // Env-level structures the Queues list "Queued"/"Running" blocks read:
+ // lengthOfEnvQueue -> ZCARD(envQueueKey) (ZSET, no proj section)
+ // concurrencyOfEnvQueue -> SCARD(envCurrentDequeuedKey) (SET)
+ // We accumulate the per-queue staged counts below and stage these so the blocks
+ // equal the table's per-queue sums instead of showing 0/0.
+ const envQueueKey = `${prefix}{org:${ids.organization_id}}:env:${ids.environment_id}`;
+ const envCurrentDequeuedKey = `${prefix}{org:${ids.organization_id}}:proj:${ids.project_id}:env:${ids.environment_id}:currentDequeued`;
+ await redis.del(envQueueKey, envCurrentDequeuedKey);
+ // Table Queued = ZCARD(base) + lengthCounter (base zset unstaged -> only CK queues
+ // contribute their lengthCounter). Table Running = SCARD(currentDequeued) per queue.
+ let envQueuedTotal = 0;
+ let envRunningTotal = 0;
+
for (const [q, profile] of scenario.queues.entries()) {
const key = `${base}${profile.name}:currentDequeued`;
await redis.del(key);
@@ -642,6 +656,7 @@ async function stageRedisUsage(scenario: Scenario, ids: Ids, seed: number, clear
if (count > 0) {
await redis.sadd(key, ...Array.from({ length: count }, (_v, i) => `sim_run_${i}`));
}
+ envRunningTotal += count;
if (profile.ck) {
const now = Date.now();
@@ -670,8 +685,28 @@ async function stageRedisUsage(scenario: Scenario, ids: Ids, seed: number, clear
}
// The aggregate "Queued now" reads ZCARD(base) + this counter; keep them coherent.
await redis.set(lengthCounterKey, totalCkQueued, "EX", 24 * 3600);
+ envQueuedTotal += totalCkQueued;
+ }
+ }
+
+ // Stage the env-level structures so the list-page blocks match the table sums.
+ // Members are unique across queues (each per-queue set uses its own key, but the
+ // env set/zset needs distinct members to reach the summed cardinality).
+ if (!clear) {
+ if (envRunningTotal > 0) {
+ await redis.sadd(
+ envCurrentDequeuedKey,
+ ...Array.from({ length: envRunningTotal }, (_v, i) => `sim_env_run_${i}`)
+ );
+ }
+ if (envQueuedTotal > 0) {
+ const now = Date.now();
+ const zargs: Array = [];
+ for (let i = 0; i < envQueuedTotal; i++) zargs.push(now + i, `sim_env_queued_${i}`);
+ await redis.zadd(envQueueKey, ...zargs);
}
}
+
await redis.quit();
console.log(
clear
@@ -733,7 +768,16 @@ async function ensureTaskQueues(
projectId,
type: "NAMED",
},
- update: { concurrencyLimit },
+ // Reset any dashboard override left from manual testing: re-seeding overwrites the
+ // materialized concurrencyLimit, so a surviving override percent/base would contradict it
+ // (e.g. "10 (77%)" with an env limit of 25).
+ update: {
+ concurrencyLimit,
+ concurrencyLimitBase: null,
+ concurrencyLimitOverridePercent: null,
+ concurrencyLimitOverriddenAt: null,
+ concurrencyLimitOverriddenBy: null,
+ },
});
}
diff --git a/apps/webapp/test/concurrencySystemPercentOverride.test.ts b/apps/webapp/test/concurrencySystemPercentOverride.test.ts
new file mode 100644
index 00000000000..f81f1b1d10b
--- /dev/null
+++ b/apps/webapp/test/concurrencySystemPercentOverride.test.ts
@@ -0,0 +1,370 @@
+import { postgresTest } from "@internal/testcontainers";
+import type { PrismaClient } from "@trigger.dev/database";
+import { describe, expect, it, vi } from "vitest";
+import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { ConcurrencySystem, materializePercentLimit } from "~/v3/services/concurrencySystem.server";
+
+// The real run engine opens eager Redis connections and needs the whole engine
+// wired up. These tests exercise the DB-write + recalculation logic against a
+// real Postgres (postgresTest), so we replace the engine singleton with a thin
+// stub that satisfies the sync + stats calls the service makes. The engine sync
+// itself (RunQueue/Redis) is therefore NOT exercised here — see the note in the
+// verification report. Everything the service persists to Postgres IS real.
+// A controllable spy for the engine's queue-limit push so a test can simulate a push failure and
+// prove the next recalc self-heals the divergence.
+// This mocks the ENGINE method (`engine.runQueue.updateQueueConcurrencyLimits`). The service calls
+// the top-level `updateQueueConcurrencyLimits` from `~/v3/runQueue.server`, which forwards straight
+// to this engine method with the same `(environment, queueName, concurrency)` argument order (no
+// transformation) — so mocking the engine method genuinely exercises the recalc's sync path and the
+// `(env, queueName, 10)` assertion below is exact.
+const { updateQueueConcurrencyLimitsMock } = vi.hoisted(() => ({
+ updateQueueConcurrencyLimitsMock: vi.fn(async (..._args: unknown[]) => undefined),
+}));
+
+vi.mock("~/v3/runEngine.server", () => ({
+ engine: {
+ lengthOfQueues: async () => ({}),
+ currentConcurrencyOfQueues: async () => ({}),
+ runQueue: {
+ updateQueueConcurrencyLimits: updateQueueConcurrencyLimitsMock,
+ removeQueueConcurrencyLimits: async () => undefined,
+ updateEnvConcurrencyLimits: async () => undefined,
+ },
+ },
+}));
+
+vi.setConfig({ testTimeout: 30_000 });
+
+describe("materializePercentLimit", () => {
+ it("floors the materialized value", () => {
+ // 10 * 55 / 100 = 5.5 -> floor -> 5
+ expect(materializePercentLimit(10, 55)).toBe(5);
+ // 7 * 50 / 100 = 3.5 -> floor -> 3
+ expect(materializePercentLimit(7, 50)).toBe(3);
+ });
+
+ it("clamps to at least 1 so a percent override never produces a 0 (pause-like) limit", () => {
+ // 10 * 1 / 100 = 0.1 -> floor -> 0 -> clamp up to 1
+ expect(materializePercentLimit(10, 1)).toBe(1);
+ // tiny env, small percent still floors to 0 then clamps to 1
+ expect(materializePercentLimit(1, 1)).toBe(1);
+ expect(materializePercentLimit(3, 10)).toBe(1);
+ });
+
+ it("clamps to at most the environment limit at 100%", () => {
+ expect(materializePercentLimit(10, 100)).toBe(10);
+ expect(materializePercentLimit(1, 100)).toBe(1);
+ expect(materializePercentLimit(250, 100)).toBe(250);
+ });
+
+ it("handles a tiny environment limit at the 100% boundary", () => {
+ expect(materializePercentLimit(1, 50)).toBe(1); // 0.5 -> 0 -> clamp to 1
+ expect(materializePercentLimit(2, 50)).toBe(1); // 1.0 -> 1
+ expect(materializePercentLimit(2, 100)).toBe(2);
+ });
+
+ it("supports fractional percentages", () => {
+ // 100 * 12.5 / 100 = 12.5 -> floor -> 12
+ expect(materializePercentLimit(100, 12.5)).toBe(12);
+ });
+});
+
+async function seedEnvAndQueue(
+ prisma: PrismaClient,
+ opts: { maximumConcurrencyLimit: number; queueConcurrencyLimit?: number | null }
+) {
+ const slug = `s${Math.random().toString(36).slice(2, 10)}`;
+
+ const organization = await prisma.organization.create({
+ data: { title: slug, slug },
+ });
+
+ const project = await prisma.project.create({
+ data: { name: slug, slug, organizationId: organization.id, externalRef: slug },
+ });
+
+ const environment = await prisma.runtimeEnvironment.create({
+ data: {
+ slug,
+ type: "PRODUCTION",
+ projectId: project.id,
+ organizationId: organization.id,
+ apiKey: slug,
+ pkApiKey: slug,
+ shortcode: slug,
+ maximumConcurrencyLimit: opts.maximumConcurrencyLimit,
+ },
+ });
+
+ const queue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_${slug}`,
+ name: `task/${slug}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: opts.queueConcurrencyLimit ?? null,
+ },
+ });
+
+ const authEnv = {
+ id: environment.id,
+ maximumConcurrencyLimit: environment.maximumConcurrencyLimit,
+ } as unknown as AuthenticatedEnvironment;
+
+ return { organization, project, environment, queue, authEnv };
+}
+
+describe("ConcurrencySystem percent overrides", () => {
+ postgresTest(
+ "materializes a percent override and stores the percent as the source of truth",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 50,
+ });
+
+ expect(result.isOk()).toBe(true);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ // floor(10 * 50 / 100) = 5
+ expect(row.concurrencyLimit).toBe(5);
+ expect(row.concurrencyLimitOverridePercent?.toNumber()).toBe(50);
+ // base captures the pre-override absolute limit
+ expect(row.concurrencyLimitBase).toBe(8);
+ expect(row.concurrencyLimitOverriddenAt).not.toBeNull();
+ }
+ );
+
+ postgresTest(
+ "rejects an absolute override that exceeds the environment limit and persists nothing",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const result = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ limit: 9999,
+ });
+
+ expect(result.isErr()).toBe(true);
+ if (result.isErr()) {
+ expect(result.error.type).toBe("concurrency_limit_exceeds_maximum");
+ }
+
+ // Nothing should have been written.
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(8);
+ expect(row.concurrencyLimitOverridePercent).toBeNull();
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ expect(row.concurrencyLimitBase).toBeNull();
+ }
+ );
+
+ postgresTest("rejects an out-of-range percent as invalid_override", async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const tooHigh = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 101,
+ });
+ expect(tooHigh.isErr()).toBe(true);
+ if (tooHigh.isErr()) expect(tooHigh.error.type).toBe("invalid_override");
+
+ const tooLow = await system.queues.overrideQueueConcurrencyLimit(authEnv, queue.friendlyId, {
+ percent: 0,
+ });
+ expect(tooLow.isErr()).toBe(true);
+ if (tooLow.isErr()) expect(tooLow.error.type).toBe("invalid_override");
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ });
+
+ postgresTest("reset clears both the absolute limit and the percent", async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+
+ const reset = await system.queues.resetConcurrencyLimit(authEnv, queue.friendlyId);
+ expect(reset.isOk()).toBe(true);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ // restored to the base captured at override time
+ expect(row.concurrencyLimit).toBe(8);
+ expect(row.concurrencyLimitOverridePercent).toBeNull();
+ expect(row.concurrencyLimitBase).toBeNull();
+ expect(row.concurrencyLimitOverriddenAt).toBeNull();
+ expect(row.concurrencyLimitOverriddenBy).toBeNull();
+ });
+
+ postgresTest(
+ "recalculatePercentLimits recomputes percent queues and leaves absolute overrides untouched",
+ async ({ prisma }) => {
+ const { project, environment, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ // A percent-based queue: 50% of env(10) -> 5
+ const percentQueue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_pct_${Math.random().toString(36).slice(2, 8)}`,
+ name: `task/pct-${Math.random().toString(36).slice(2, 8)}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: 6,
+ },
+ });
+ const pctResult = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ percentQueue.friendlyId,
+ { percent: 50 }
+ );
+ expect(pctResult.isOk()).toBe(true);
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } });
+ expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5)
+ }
+
+ // An absolute-override queue: limit 4, no percent
+ const absQueue = await prisma.taskQueue.create({
+ data: {
+ friendlyId: `queue_abs_${Math.random().toString(36).slice(2, 8)}`,
+ name: `task/abs-${Math.random().toString(36).slice(2, 8)}`,
+ projectId: project.id,
+ runtimeEnvironmentId: environment.id,
+ concurrencyLimit: 6,
+ },
+ });
+ const absResult = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ absQueue.friendlyId,
+ { limit: 4 }
+ );
+ expect(absResult.isOk()).toBe(true);
+
+ // Simulate the environment limit changing from 10 -> 20 and recalculate.
+ const bumpedEnv = {
+ id: environment.id,
+ maximumConcurrencyLimit: 20,
+ } as unknown as AuthenticatedEnvironment;
+
+ const outcome = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(outcome.total).toBe(1); // only the percent queue is considered
+ expect(outcome.updated).toBe(1);
+
+ const pctRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: percentQueue.id } });
+ // floor(20 * 50 / 100) = 10
+ expect(pctRow.concurrencyLimit).toBe(10);
+ expect(pctRow.concurrencyLimitOverridePercent?.toNumber()).toBe(50);
+
+ const absRow = await prisma.taskQueue.findUniqueOrThrow({ where: { id: absQueue.id } });
+ // absolute override must be untouched
+ expect(absRow.concurrencyLimit).toBe(4);
+ expect(absRow.concurrencyLimitOverridePercent).toBeNull();
+ }
+ );
+
+ postgresTest(
+ "recalculatePercentLimits is idempotent when the limit is unchanged",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+
+ // Recalculate against the SAME environment limit -> nothing to update.
+ const outcome = await system.queues.recalculatePercentLimits(authEnv);
+ expect(outcome.total).toBe(1);
+ expect(outcome.updated).toBe(0);
+
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(5);
+ }
+ );
+
+ postgresTest(
+ "recalculatePercentLimits re-syncs the engine on a later recalc after an engine push failed, even when the DB limit is unchanged",
+ async ({ prisma }) => {
+ const { queue, authEnv } = await seedEnvAndQueue(prisma, {
+ maximumConcurrencyLimit: 10,
+ queueConcurrencyLimit: 8,
+ });
+
+ const system = new ConcurrencySystem({ db: prisma, reader: prisma });
+
+ const overridden = await system.queues.overrideQueueConcurrencyLimit(
+ authEnv,
+ queue.friendlyId,
+ { percent: 50 }
+ );
+ expect(overridden.isOk()).toBe(true);
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(5); // floor(10 * 0.5)
+ }
+
+ const bumpedEnv = {
+ id: authEnv.id,
+ maximumConcurrencyLimit: 20,
+ } as unknown as AuthenticatedEnvironment;
+
+ // The env-limit bump recalc writes the new DB limit (10) but the engine push fails and is
+ // swallowed by the per-queue catch: DB and engine are now diverged (DB=10, engine=5).
+ updateQueueConcurrencyLimitsMock.mockClear();
+ updateQueueConcurrencyLimitsMock.mockRejectedValueOnce(new Error("engine push failed"));
+
+ const first = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(first.updated).toBe(1); // DB was written before the push threw
+ {
+ const row = await prisma.taskQueue.findUniqueOrThrow({ where: { id: queue.id } });
+ expect(row.concurrencyLimit).toBe(10);
+ }
+
+ // A later recalc at the SAME env limit makes no DB change, but MUST still push to the engine
+ // so the previously-failed sync self-heals. (Regression guard: the old code `continue`d when
+ // newLimit === concurrencyLimit and left the engine stuck at 5 forever.)
+ updateQueueConcurrencyLimitsMock.mockClear();
+ const second = await system.queues.recalculatePercentLimits(bumpedEnv);
+ expect(second.updated).toBe(0); // no DB write
+ expect(updateQueueConcurrencyLimitsMock).toHaveBeenCalledWith(
+ expect.anything(),
+ queue.name,
+ 10
+ );
+ }
+ );
+});
diff --git a/apps/webapp/test/useTableSort.test.ts b/apps/webapp/test/useTableSort.test.ts
new file mode 100644
index 00000000000..4b6c83666d9
--- /dev/null
+++ b/apps/webapp/test/useTableSort.test.ts
@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+import { compareColumn, sortRows, type SortColumn } from "~/components/primitives/useTableSort";
+
+type Row = { id: number; name: string | null; queued: number | null };
+
+const rows: Row[] = [
+ { id: 0, name: "banana", queued: 3 },
+ { id: 1, name: "Apple", queued: null },
+ { id: 2, name: "cherry", queued: 3 },
+ { id: 3, name: null, queued: 1 },
+];
+
+describe("sortRows", () => {
+ it("sorts numbers ascending with nulls last", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 0, 2, 1]);
+ });
+
+ it("keeps nulls last even when descending", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ // 3 and 0 both have queued=3; stable order (0 before 2) is preserved, null (id 1) stays last.
+ expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([0, 2, 3, 1]);
+ });
+
+ it("sorts alphabetically case-insensitively with empty/null last", () => {
+ const column: SortColumn = { key: "name", type: "alpha", value: (r) => r.name };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([1, 0, 2, 3]);
+ expect(sortRows(rows, column, "desc").map((r) => r.id)).toEqual([2, 0, 1, 3]);
+ });
+
+ it("is stable for equal values (preserves original order)", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: () => 5 };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([0, 1, 2, 3]);
+ });
+
+ it("supports a custom comparator", () => {
+ // Sort by name length.
+ const column: SortColumn = {
+ key: "name",
+ type: "custom",
+ compare: (a, b) => (a.name?.length ?? 0) - (b.name?.length ?? 0),
+ };
+ expect(sortRows(rows, column, "asc").map((r) => r.id)).toEqual([3, 1, 0, 2]);
+ });
+});
+
+describe("compareColumn", () => {
+ it("treats NaN as null (sorts last)", () => {
+ const column: SortColumn = { key: "queued", type: "number", value: (r) => r.queued };
+ const withNaN: Row = { id: 9, name: "x", queued: NaN };
+ const normal: Row = { id: 10, name: "y", queued: 1 };
+ expect(compareColumn(column, withNaN, normal, "asc")).toBe(1);
+ expect(compareColumn(column, withNaN, normal, "desc")).toBe(1);
+ });
+});
diff --git a/internal-packages/clickhouse/src/queueMetrics.ts b/internal-packages/clickhouse/src/queueMetrics.ts
index dce9323ef26..d86645e5153 100644
--- a/internal-packages/clickhouse/src/queueMetrics.ts
+++ b/internal-packages/clickhouse/src/queueMetrics.ts
@@ -51,6 +51,7 @@ const QueueMetricsSummaryRow = z.object({
p95_wait_ms: z.coerce.number(),
peak_queued: z.coerce.number(),
started_count: z.coerce.number(),
+ throttled_count: z.coerce.number(),
});
// Callers align window bounds to the bucket grid so repeated loads share cache entries.
@@ -68,7 +69,8 @@ export function getQueueListMetricsSummary(reader: ClickhouseReader) {
round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50_wait_ms,
round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95_wait_ms,
max(max_queued) AS peak_queued,
- deltaSumTimestampMerge(started_delta) AS started_count
+ deltaSumTimestampMerge(started_delta) AS started_count,
+ sum(throttled_count) AS throttled_count
FROM trigger_dev.queue_metrics_v1
WHERE organization_id = {organizationId: String}
AND project_id = {projectId: String}
@@ -91,16 +93,22 @@ const QueueDepthSparklineRow = z.object({
queue_name: z.string(),
bucket: z.string(),
depth: z.coerce.number(),
+ throttled: z.coerce.number(),
});
-/** Per-queue, per-bucket peak depth for inline sparklines (carry-forward filled by the caller). */
+/**
+ * Per-queue, per-bucket peak depth (carry-forward filled by the caller) plus the throttled
+ * count in each bucket, so the sparkline can tint the exact buckets where throttling occurred.
+ * The extra aggregate rides the same scan as the depth series — no additional round trip.
+ */
export function getQueueDepthSparklines(reader: ClickhouseReader) {
return reader.query({
name: "getQueueDepthSparklines",
query: `SELECT
queue_name,
toStartOfInterval(bucket_start, toIntervalSecond({bucketSeconds: UInt32})) AS bucket,
- max(max_queued) AS depth
+ max(max_queued) AS depth,
+ sum(throttled_count) AS throttled
FROM trigger_dev.queue_metrics_v1
WHERE organization_id = {organizationId: String}
AND project_id = {projectId: String}
diff --git a/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql
new file mode 100644
index 00000000000..cf1aa0ed84e
--- /dev/null
+++ b/internal-packages/database/prisma/migrations/20260716120000_add_task_queue_concurrency_limit_override_percent/migration.sql
@@ -0,0 +1,5 @@
+-- Adds a nullable column to store a queue concurrency-limit override expressed as a percentage
+-- of the environment limit. This percentage is the source of truth; the absolute
+-- "concurrencyLimit" is materialized from it at save time. Additive and nullable, so existing
+-- absolute overrides are unaffected. Decimal(5,2) supports fractional percentages (0.01–100.00).
+ALTER TABLE "TaskQueue" ADD COLUMN IF NOT EXISTS "concurrencyLimitOverridePercent" DECIMAL(5,2);
diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma
index 1b8642e866f..b861b80d83a 100644
--- a/internal-packages/database/prisma/schema.prisma
+++ b/internal-packages/database/prisma/schema.prisma
@@ -1829,14 +1829,18 @@ model TaskQueue {
runtimeEnvironmentId String
/// Represents the current concurrency limit for the queue
- concurrencyLimit Int?
+ concurrencyLimit Int?
/// When the concurrency limit was overridden
- concurrencyLimitOverriddenAt DateTime?
+ concurrencyLimitOverriddenAt DateTime?
/// Who overrode the concurrency limit (will be null if overridden via the API)
- concurrencyLimitOverriddenBy String?
+ concurrencyLimitOverriddenBy String?
/// If concurrencyLimit is overridden, this is the overridden value
- concurrencyLimitBase Int?
- rateLimit Json?
+ concurrencyLimitBase Int?
+ /// If the override was expressed as a percentage of the environment limit, this stores that
+ /// percentage (the source of truth). The absolute concurrencyLimit is materialized from it.
+ /// Decimal(5,2) allows fractional percentages like 12.50% (0.01–100.00).
+ concurrencyLimitOverridePercent Decimal? @db.Decimal(5, 2)
+ rateLimit Json?
paused Boolean @default(false)