Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions packages/commands/src/commands/advisor/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -272,9 +272,7 @@ export default defineCommand({
return result;
});

const analyzeIntentPromise = analyzeIntent(ctx.client, userInput, {
intentDetectBaseUrl: settings.intentDetectBaseUrl,
}).then((result) => {
const analyzeIntentPromise = analyzeIntent(ctx.client, userInput).then((result) => {
intentReady = true;
if (!modelsReady) {
spinner.update("Agent: Intent analyzed, loading model data...");
Expand Down
63 changes: 40 additions & 23 deletions packages/commands/tests/e2e/advisor-recommend.e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: advisor recommend (DashScope)", ()
expect(data.result?.recommendations?.[0]?.highlights?.length).toBeGreaterThan(0);
}, 120_000);

// ---- Model preference: positive cases ----
// ---- Mode coverage: all 4 modes ----

test("scoped preference — intent contains modelPreference.mode=scoped when family is specified", async () => {
test("mode: scoped — family-scoped query sets mode=scoped with targets", async () => {
const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [
"advisor",
"recommend",
Expand All @@ -112,39 +112,62 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: advisor recommend (DashScope)", ()
const data = parseStdoutJson<{
intent?: { modelPreference?: { mode?: string; targets?: string[] } };
}>(stdout);
// Model preference detection depends on LLM interpretation
// Accept either "scoped" or "unconstrained" as valid
const mode = data.intent?.modelPreference?.mode;
expect(mode === "scoped" || mode === "unconstrained" || mode === undefined).toBe(true);
const pref = data.intent?.modelPreference;
expect(pref?.mode).toBe("scoped");
expect(pref?.targets?.length).toBeGreaterThan(0);
// Should contain "deepseek" (case-insensitive substring)
expect(pref?.targets?.some((t) => t.toLowerCase().includes("deepseek"))).toBe(true);
}, 60_000);

test("comparison preference — intent contains modelPreference.mode=comparison when comparing models", async () => {
test("mode: comparison — comparing two models sets mode=comparison with both targets", async () => {
const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [
"advisor",
"recommend",
"--dry-run",
"--message",
"Which is better for code generation, qwen-max or deepseek-v3?",
"Compare qwen-max and deepseek-v3 for legal contract review, high precision required",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{
intent?: { modelPreference?: { mode?: string; targets?: string[] } };
}>(stdout);
// Model preference detection depends on LLM interpretation
// Accept either "comparison" or "unconstrained" as valid
const mode = data.intent?.modelPreference?.mode;
expect(mode === "comparison" || mode === "unconstrained" || mode === undefined).toBe(true);
const pref = data.intent?.modelPreference;
expect(pref?.mode).toBe("comparison");
expect(pref?.targets?.length).toBeGreaterThanOrEqual(2);
const targetsLower = pref?.targets?.map((t) => t.toLowerCase()) ?? [];
expect(targetsLower.some((t) => t.includes("qwen"))).toBe(true);
expect(targetsLower.some((t) => t.includes("deepseek"))).toBe(true);
}, 60_000);

test("mode: alternative — reference model query sets mode=alternative with target", async () => {
const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [
"advisor",
"recommend",
"--dry-run",
"--message",
"Something like qwen-max but cheaper, for text summarization",
"--output",
"json",
]);
expect(exitCode, stderr).toBe(0);
const data = parseStdoutJson<{
intent?: { modelPreference?: { mode?: string; targets?: string[] } };
}>(stdout);
const pref = data.intent?.modelPreference;
expect(pref?.mode).toBe("alternative");
expect(pref?.targets?.length).toBeGreaterThan(0);
expect(pref?.targets?.some((t) => t.toLowerCase().includes("qwen"))).toBe(true);
}, 60_000);

test("excludes preference — intent detects modelPreference when excluding models", async () => {
test("mode: excludes — excluding a family populates excludes array", async () => {
const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [
"advisor",
"recommend",
"--dry-run",
"--message",
"Not qwen, recommend a model suitable for text generation",
"Recommend a model for text generation, but not qwen",
"--output",
"json",
]);
Expand All @@ -157,18 +180,12 @@ describe.skipIf(!isDashScopeE2EReady())("e2e: advisor recommend (DashScope)", ()
};
};
}>(stdout);
// Model preference detection depends on LLM interpretation
// If excludes is detected, verify it contains qwen; otherwise accept as valid
const pref = data.intent?.modelPreference;
if (pref?.excludes && pref.excludes.length > 0) {
expect(pref.excludes.some((e) => e.toLowerCase().includes("qwen"))).toBe(true);
}
// Test passes if exit code is 0, regardless of whether excludes was detected
expect(pref?.excludes?.length).toBeGreaterThan(0);
expect(pref?.excludes?.some((e) => e.toLowerCase().includes("qwen"))).toBe(true);
}, 60_000);

// ---- Model preference: negative cases ----

test("no preference — intent has no modelPreference or mode=unconstrained for generic queries", async () => {
test("mode: unconstrained — generic query has no modelPreference", async () => {
const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [
"advisor",
"recommend",
Expand Down
3 changes: 0 additions & 3 deletions packages/core/src/advisor/constants/index.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
export { DEFAULT_INTENT } from "./defaults.ts";
export {
INTENT_DETECT_MODEL,
INTENT_DETECT_TOOL,
buildIntentDetectSystemPrompt,
INTENT_EXTRACTION_MODEL,
INTENT_SYSTEM_PROMPT,
JSON_RETRY_HINT,
Expand Down
82 changes: 3 additions & 79 deletions packages/core/src/advisor/constants/prompts.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
export const RANKING_MODEL = "qwen-flash";

/**
* Dedicated intent-detection model. Sub-100ms latency, designed for fast
* classification + tool routing. Provides mode/targets/excludes/complexity.
*/
export const INTENT_DETECT_MODEL = "tongyi-intent-detect-v3";

/**
* Rich field extraction model. Runs in parallel with detect-v3 to extract
* taskSummary, modalities, budget, qualityPreference, etc.
* Intent extraction model. Runs a single call to extract taskSummary,
* modalities, budget, qualityPreference, modelPreference, and all other
* structured fields from the user's input.
*/
export const INTENT_EXTRACTION_MODEL = "qwen3.6-flash";

Expand Down Expand Up @@ -193,74 +188,3 @@ The intent's modelPreference.targets is the reference model.

## Output Format
{"type":"single","recommendations":[{"model":"model ID","reason":"alternative analysis","highlights":["differentiators"]}]}`;

/**
* Tool definition for `tongyi-intent-detect-v3`. Serialized to a JSON string
* and embedded in the system prompt (NOT passed via the request body `tools`
* field — this model doesn't use OpenAI function-calling; it has its own
* `<tags>` / `
</think>

` output format driven by the system prompt).
*/
export const INTENT_DETECT_TOOL = {
name: "classify_intent",
description:
"Classify the user's model recommendation intent. Extract the mode and any model/family names mentioned.",
parameters: {
type: "object",
properties: {
mode: {
type: "string",
enum: ["unconstrained", "scoped", "comparison", "alternative"],
description: "The detected intent mode.",
},
targets: {
type: "array",
items: { type: "string" },
description:
"Model or family names the user mentioned or wants to evaluate. Empty for unconstrained.",
},
excludes: {
type: "array",
items: { type: "string" },
description:
"Model or family names the user explicitly wants to exclude. Empty if none mentioned.",
},
complexity: {
type: "string",
enum: ["single", "pipeline"],
description:
"Whether the task needs a single model or a multi-step pipeline. Default to single unless the user clearly describes chained steps.",
},
},
required: ["mode"],
},
} as const;

/**
* Build the system prompt for `tongyi-intent-detect-v3` following the official
* template from the model's documentation. The tools JSON is embedded in the
* prompt text — the model reads it from the system message, not from a separate
* `tools` request field.
*
* Official template:
* "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.
* You may call one or more tools to assist with the user query.
* The tools you can use are as follows:
* {tools_string}
* Response in INTENT_MODE."
*
* `INTENT_MODE` tells the model to emit `<tags>label</tags>` + `
</think>

`
* output. The tag carries the mode classification; the tool_call carries
* structured targets/excludes/complexity extracted from the user's prompt.
*/
export function buildIntentDetectSystemPrompt(): string {
const toolsString = JSON.stringify([INTENT_DETECT_TOOL], null, 2);
return `You are Qwen, created by Alibaba Cloud. You are a helpful assistant. You may call one or more tools to assist with the user query. The tools you can use are as follows:
${toolsString}
Response in INTENT_MODE.`;
}
Loading