266 lines
17 KiB
TypeScript
266 lines
17 KiB
TypeScript
import { beforeEach } from 'node:test';
|
|
import { questionLog } from './question-log.js';
|
|
beforeEach(t => {
|
|
if (!('mock' in t)) throw new Error('Expected test context');
|
|
t.mock.method(questionLog, 'start', async () => 'synthetic-question');
|
|
t.mock.method(questionLog, 'finish', async () => {});
|
|
});
|
|
import assert from "node:assert/strict";
|
|
import test, { type TestContext } from "node:test";
|
|
import { Hono } from "hono";
|
|
import router from "./index.js";
|
|
import { vehicleRepository } from "../vehicles/repository.js";
|
|
import { gapStore } from "./gaps.js";
|
|
import { addNode, getSession } from "./session.js";
|
|
import type { AuthUser } from "../../auth/types.js";
|
|
import type { Vehicle } from "../../types.js";
|
|
import type { AssistantRequest, AssistantResult, GapReason, QueryPlan } from "../../../shared/assets/assistant.js";
|
|
|
|
// Exact questions from the 2026-09-23 user-supplied boss-perspective report.
|
|
// All data below is synthetic; model responses and persistence are mocked.
|
|
export const BOSS_QUESTIONS = [
|
|
"我是公司负责人。请用当前数据给我一份资产经营简报：资产总数、运营和库存各多少、运营率多少；哪个车型库存最多，最值得我优先关注？请给数字依据、数据截至时间和统计口径。",
|
|
"现在总共有多少台车？运营多少、库存多少，整体运营率是多少？",
|
|
"按车型看库存，按库存数量从多到少排序，告诉我库存最多的车型占总库存多少。",
|
|
"这些4.5T库存主要在哪些区域？只看可运营的，按数量降序。",
|
|
"换个问题：最近7天交车、还车分别多少？其中替换交车多少？比前7天是改善还是变差？请写明起止日期和统计口径。",
|
|
"全部车辆，不沿用之前4.5T或库存筛选。查询2026年9月17日至9月23日交车、还车、替换交车数量，按天列出。",
|
|
"当前库存比上个月月底增加了多少？这些闲置车每个月损失多少钱？没有相应数据请直接说明，不要估算。",
|
|
"现在全部公司的待交车有多少？列出待交车清单。",
|
|
"查询待交车数量",
|
|
"我问的是“待交车”，不是“待运营”。请区分这两个业务状态，重新查询待交车数量；如果不支持，请明确说明，不能用待运营数量替代。",
|
|
] as const;
|
|
|
|
const query = (overrides: Partial<QueryPlan> = {}): QueryPlan => ({
|
|
kind: "assets", category: "all", groupBy: "type", view: "table", filters: {}, analysis: "distribution", ...overrides,
|
|
});
|
|
let sequence = 0;
|
|
function harness(t: TestContext, mode: "model" | "local" = "model") {
|
|
const user: AuthUser = { userId: `boss-${++sequence}`, userName: "老板体验验收", loginName: "boss-test", depCode: "D", depName: "合成部门", permissionLevel: "personal", roles: [] };
|
|
const inventory = [
|
|
{ id: "plain-east", type: "4.5T", model: "4.5吨普货", province: "浙江省", city: "嘉兴市", operationStatus: "3" },
|
|
{ id: "cold-east", type: "4.5T", model: "4.5吨冷链车", province: "浙江省", city: "嘉兴市", operationStatus: "3" },
|
|
{ id: "cold-south", type: "4.5T", model: "4.5吨冷链车", province: "广东省", city: "广州市", operationStatus: "4" },
|
|
{ id: "cold-northwest", type: "4.5T", model: "4.5吨冷链车", province: "新疆", city: "乌鲁木齐市", operationStatus: "4" },
|
|
{ id: "18-awaiting", type: "18T", model: "18吨厢货", province: "浙江省", city: "嘉兴市", operationStatus: "4" },
|
|
{ id: "49-available", type: "49T", model: "49吨牵引", province: "广东省", city: "广州市", operationStatus: "3" },
|
|
];
|
|
const vehicles = [
|
|
...inventory.map((v) => ({ ...v, status: "Inventory" })),
|
|
...Array.from({ length: 4 }, (_, i) => ({ id: `operating-${i}`, type: "18T", model: "18吨厢货", province: "浙江省", city: "嘉兴市", operationStatus: "1", status: "Operating" })),
|
|
].map((v) => ({ ...v, plateNumber: `合成-${v.id}`, vin: `SYNTHETIC-${v.id}`, subjectOrg: "合成公司", departmentName: "合成部门", managerId: user.userId }) as Vehicle);
|
|
t.mock.method(vehicleRepository, "getVehicles", async () => vehicles);
|
|
const flowCalls: { start: string; end: string }[] = [];
|
|
t.mock.method(vehicleRepository, "getFlowDetailRows", async (start: string, end: string) => {
|
|
flowCalls.push({ start, end });
|
|
return [
|
|
{ id: `${start}-delivered`, truck_id: "49-available", stat_date: start, type: "delivered", type_label: "交车" },
|
|
{ id: `${start}-replaced`, truck_id: "operating-0", stat_date: start, type: "replaced", type_label: "替换" },
|
|
{ id: `${start}-returned`, truck_id: "cold-south", stat_date: end, type: "returned", type_label: "还车" },
|
|
];
|
|
});
|
|
const gaps: { demand: string; reason: GapReason }[] = [];
|
|
t.mock.method(gapStore, "record", async (_user: string, demand: string, reason: GapReason) => { gaps.push({ demand, reason }); });
|
|
const oldKey = process.env.DEEPSEEK_API_KEY, oldEnabled = process.env.ASSET_AI_EXTERNAL_ENABLED;
|
|
process.env.DEEPSEEK_API_KEY = "synthetic-test-key";
|
|
process.env.ASSET_AI_EXTERNAL_ENABLED = mode === "model" ? "1" : "0";
|
|
t.after(() => {
|
|
if (oldKey === undefined) delete process.env.DEEPSEEK_API_KEY; else process.env.DEEPSEEK_API_KEY = oldKey;
|
|
if (oldEnabled === undefined) delete process.env.ASSET_AI_EXTERNAL_ENABLED; else process.env.ASSET_AI_EXTERNAL_ENABLED = oldEnabled;
|
|
});
|
|
const proposals: Record<string, unknown>[] = [];
|
|
const presentationCalls: string[] = [];
|
|
// Count planning separately and return a minimal valid display for each result.
|
|
const calls: string[] = [];
|
|
t.mock.method(globalThis, "fetch", async (_url: unknown, init?: RequestInit) => {
|
|
assert.equal(mode, "model", "Local acceptance must not call any external endpoint");
|
|
const body = JSON.parse(String(init?.body));
|
|
const context = JSON.parse(body.messages.find((message: { role: string }) => message.role === "user").content);
|
|
if (context.stage === "presentation") {
|
|
presentationCalls.push(String(init?.body));
|
|
const presentation = { layout: "stack", results: context.results.map((result: { id: string }) => ({ id: result.id, blocks: [{ id: "summary", kind: "summary" }], interactions: [] })) };
|
|
return new Response(JSON.stringify({ choices: [{ message: { content: JSON.stringify(presentation) }, finish_reason: "stop" }] }));
|
|
}
|
|
calls.push(String(init?.body));
|
|
assert.ok(proposals.length, "Every model response must be explicitly supplied");
|
|
return new Response(JSON.stringify({ choices: [{ message: { content: JSON.stringify(proposals.shift()) }, finish_reason: "stop" }] }), { headers: { "Content-Type": "application/json" } });
|
|
});
|
|
const app = new Hono<{ Variables: { user: AuthUser } }>();
|
|
app.use("*", async (c, next) => { c.set("user", user); await next(); });
|
|
app.route("/ai", router);
|
|
async function ask(input: Partial<AssistantRequest>) {
|
|
const response = await app.request("/ai/ask", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ subject: null, ...input }) });
|
|
assert.equal(response.status, 200);
|
|
const events = (await response.text()).split("\n").filter((line) => line.startsWith("data:")).map((line) => JSON.parse(line.slice(5)));
|
|
const error = events.find((event) => event.type === "error");
|
|
assert.equal(error, undefined, error?.message);
|
|
const result = events.find((event) => event.type === "result")?.result as AssistantResult;
|
|
assert.ok(result, "Every query must return a result or explicit capability response");
|
|
return result;
|
|
}
|
|
return { user, vehicles, ask, proposals, calls, presentationCalls, gaps, flowCalls };
|
|
}
|
|
const based = (result: AssistantResult) => ({ sessionId: result.sessionId, baseResultId: result.resultId });
|
|
function expectCoreMetrics(result: AssistantResult) {
|
|
assert.equal(result.metrics.find((m) => /全部资产|资产总数|总资产/.test(m.label))?.value, 10);
|
|
assert.equal(result.metrics.find((m) => m.label === "运营车辆")?.value, 4);
|
|
assert.equal(result.metrics.find((m) => /库存/.test(m.label))?.value, 6);
|
|
assert.equal(result.metrics.find((m) => m.label === "运营率")?.value, 40);
|
|
}
|
|
|
|
test("Q01 briefing answers supported facts and inventory concentration using computed data", async (t) => {
|
|
const { ask, proposals } = harness(t);
|
|
proposals.push({ kind: "query", plan: query({ analysis: "briefing" }), reply: "资产766台、运营475台、库存291台。" });
|
|
const result = await ask({ message: BOSS_QUESTIONS[0] });
|
|
expectCoreMetrics(result);
|
|
assert.ok(result.inventoryBreakdown);
|
|
const largest = result.inventoryBreakdown.groups[0];
|
|
assert.equal(largest.name, "4.5T");
|
|
assert.equal(largest.count, 4);
|
|
assert.equal(largest.share, 66.7);
|
|
assert.doesNotMatch(result.summary, /766|475|291/, "Model-provided numbers must not replace real calculations");
|
|
assert.equal(result.scope?.sourceUpdatedAt, null);
|
|
assert.ok(result.scope?.definitions.some((definition) => /库存/.test(definition)));
|
|
assert.ok(result.fetchedAt);
|
|
});
|
|
|
|
test("Q08/Q09 use pending even when the model incorrectly proposes awaiting or unsupported", async (t) => {
|
|
const { ask, proposals, gaps } = harness(t);
|
|
for (const [question, proposal] of [
|
|
[BOSS_QUESTIONS[7], { kind: "query", plan: query({ category: "awaiting", view: "list" }) }],
|
|
[BOSS_QUESTIONS[8], { kind: "clarify", reply: "无法单独查询待交车，目前只能统计待运营。", reason: "unsupported" }],
|
|
] as const) {
|
|
proposals.push(proposal);
|
|
const result = await ask({ message: question });
|
|
assert.equal(result.plan.kind, "assets");
|
|
assert.equal(result.plan.category, "pending");
|
|
assert.equal(result.metrics[0].value, 0);
|
|
assert.equal(result.rows.length, 0);
|
|
assert.equal(result.outcome, "empty");
|
|
assert.equal(result.gap, undefined);
|
|
assert.doesNotMatch(result.summary, /待运营库存共\s*3|不能.*待交|无法.*待交/);
|
|
}
|
|
assert.equal(gaps.length, 0);
|
|
});
|
|
|
|
test("Q10 explicitly retracts an earlier wrong awaiting answer and queries pending", async (t) => {
|
|
const { ask, proposals, user } = harness(t);
|
|
const session = getSession(user, null);
|
|
const wrong = addNode(session, {
|
|
plan: query({ category: "awaiting" }), label: "待运营库存 · 车型", demand: BOSS_QUESTIONS[8], question: BOSS_QUESTIONS[8], answer: "待运营库存共3台。",
|
|
});
|
|
proposals.push({ kind: "clarify", reply: "目前只能统计待运营，不能单独查询待交车。", reason: "unsupported" });
|
|
const result = await ask({ sessionId: session.id, baseResultId: wrong.id, message: BOSS_QUESTIONS[9] });
|
|
assert.equal(result.plan.category, "pending");
|
|
assert.equal(result.metrics[0].value, 0);
|
|
assert.equal(result.correction?.resultId, wrong.id);
|
|
assert.match(result.correction?.message || "", /待运营/);
|
|
assert.match(result.correction?.message || "", /不能|不应|不适用|撤回|纠正/);
|
|
assert.equal(result.gap, undefined);
|
|
});
|
|
|
|
test("correcting a wrong pending plan removes conflicting awaiting filters and shows real deliveries", async (t) => {
|
|
const { ask, proposals, vehicles } = harness(t);
|
|
vehicles.push({ ...vehicles[0], id: "real-pending", status: "Pending", operationStatus: "3", pendingDelivery: { contractNo: "SYNTHETIC-CONTRACT", customerName: "合成客户" } });
|
|
proposals.push({ kind: "query", plan: query({ category: "awaiting", filters: { status: ["待运营"] }, exception: "overdue", view: "list" }) });
|
|
const result = await ask({ message: BOSS_QUESTIONS[7] });
|
|
assert.equal(result.plan.category, "pending");
|
|
assert.equal(result.plan.filters.status, undefined);
|
|
assert.equal(result.plan.exception, undefined);
|
|
assert.equal(result.metrics[0].value, 1);
|
|
assert.deepEqual(result.rows.map((row) => row.id), ["real-pending"]);
|
|
assert.equal(result.outcome, "complete");
|
|
});
|
|
|
|
test("a pending category with a hidden awaiting constraint cannot turn a real delivery into a false zero", async (t) => {
|
|
const { ask, proposals, vehicles } = harness(t);
|
|
vehicles.push({ ...vehicles[0], id: "pending-with-available-operation", status: "Pending", operationStatus: "3" });
|
|
proposals.push({ kind: "query", plan: query({ category: "pending", filters: { status: ["待运营"] }, view: "list" }) });
|
|
const result = await ask({ message: BOSS_QUESTIONS[8] });
|
|
assert.equal(result.plan.filters.status, undefined);
|
|
assert.equal(result.metrics[0].value, 1);
|
|
assert.deepEqual(result.rows.map((row) => row.id), ["pending-with-available-operation"]);
|
|
});
|
|
|
|
test("mixed supported and unsupported requests produce partial answers instead of a blanket refusal", async (t) => {
|
|
const { ask, proposals, gaps } = harness(t);
|
|
proposals.push({ kind: "query", plan: query({ analysis: "briefing" }), unanswered: [{ question: "每月闲置损失金额", reason: "未接入成本与收入数据，无法可靠计算。" }] });
|
|
const result = await ask({ message: "给我资产经营简报，并告诉我每月闲置损失金额；没有数据的部分请说明。" });
|
|
expectCoreMetrics(result);
|
|
assert.equal(result.outcome, "partial");
|
|
assert.ok(result.unanswered?.some((item) => /损失/.test(item.question)));
|
|
assert.ok(result.unanswered?.some((item) => /成本|收入/.test(item.reason)));
|
|
assert.ok(gaps.some((gap) => /闲置损失/.test(gap.demand)));
|
|
});
|
|
|
|
test("the complete ten-question report works through the deterministic local fallback", async (t) => {
|
|
const { ask, calls, flowCalls } = harness(t, "local");
|
|
const answers: AssistantResult[] = [];
|
|
for (const message of BOSS_QUESTIONS) {
|
|
const last = answers.at(-1);
|
|
answers.push(await ask({ ...(last ? based(last) : {}), message }));
|
|
}
|
|
expectCoreMetrics(answers[0]);
|
|
assert.ok(answers[0].inventoryBreakdown);
|
|
expectCoreMetrics(answers[1]);
|
|
assert.equal(answers[2].plan.category, "inventory");
|
|
assert.equal(answers[2].groups[0].name, "4.5T");
|
|
assert.equal(answers[2].groups[0].share, 66.7);
|
|
assert.equal(answers[3].plan.category, "available");
|
|
assert.deepEqual(answers[3].plan.filters.type, ["4.5T"]);
|
|
assert.equal(answers[3].rows.length, 2);
|
|
assert.equal(answers[3].plan.groupBy, "region");
|
|
const recent = answers[4];
|
|
assert.equal(recent.plan.kind, "flow");
|
|
assert.deepEqual(recent.plan.filters, {});
|
|
assert.equal(recent.plan.category, "all");
|
|
assert.equal(Date.parse(recent.plan.end!) - Date.parse(recent.plan.start!), 6 * 86400000);
|
|
assert.equal(recent.plan.comparePrevious, true);
|
|
assert.equal(recent.metrics[0].value, 2);
|
|
assert.equal(recent.metrics[1].value, 1);
|
|
assert.equal(recent.metrics[2].value, 1);
|
|
assert.ok(recent.comparisonPeriod);
|
|
assert.equal(Date.parse(recent.comparisonPeriod.end), Date.parse(recent.plan.start!) - 86400000);
|
|
assert.doesNotMatch(recent.summary, /运营车辆净增\s*1|运营净增\s*1/);
|
|
assert.ok(recent.unanswered?.some((item) => /改善|变差/.test(item.question)) || recent.insights?.some((item) => /改善|变差|业务目标|判断/.test(item)));
|
|
assert.equal(answers[5].plan.start, "2026-09-17");
|
|
assert.equal(answers[5].plan.end, "2026-09-23");
|
|
assert.deepEqual(answers[5].plan.filters, {});
|
|
assert.equal(answers[5].rows.length, 3);
|
|
assert.equal(answers[6].outcome, "unsupported");
|
|
assert.equal(answers[6].rows.length, 0);
|
|
for (const pending of answers.slice(7)) {
|
|
assert.equal(pending.plan.category, "pending");
|
|
assert.equal(pending.metrics[0].value, 0);
|
|
assert.equal(pending.outcome, "empty");
|
|
}
|
|
assert.ok(flowCalls.length >= 3);
|
|
assert.equal(calls.length, 0);
|
|
});
|
|
|
|
test("result scope exposes company, permission, applied filters, grouping and distinct source time", async (t) => {
|
|
const { ask, proposals } = harness(t);
|
|
proposals.push({ kind: "query", plan: query({ category: "available", groupBy: "region", filters: { type: ["4.5T"] } }) });
|
|
const result = await ask({ subject: "合成公司", message: BOSS_QUESTIONS[3] });
|
|
assert.equal(result.scope?.company, "合成公司");
|
|
assert.ok(result.scope?.permission);
|
|
assert.ok(result.scope?.filters.some((filter) => /4\.5T/.test(filter)));
|
|
assert.ok(result.scope?.filters.some((filter) => /可运营/.test(filter)));
|
|
assert.match(result.scope?.grouping || "", /区域/);
|
|
assert.match(result.scope?.period || "", /当前|实时/);
|
|
assert.equal(result.scope?.sourceUpdatedAt, null);
|
|
assert.ok(result.scope?.definitions.some((definition) => /4\.5T/.test(definition) && /普货/.test(definition) && /冷链/.test(definition)));
|
|
});
|
|
|
|
test("4.5T tonnage results can split into ordinary freight and cold-chain inventory", async (t) => {
|
|
const { ask, proposals } = harness(t);
|
|
proposals.push({ kind: "query", plan: query({ category: "inventory", groupBy: "type" }) });
|
|
const total = await ask({ message: BOSS_QUESTIONS[2] });
|
|
proposals.push({ kind: "query", plan: query({ category: "inventory", groupBy: "vehicleClass", filters: { type: ["4.5T"] } }) });
|
|
const split = await ask({ ...based(total), message: "把4.5T进一步拆成普货和冷链" });
|
|
assert.equal(split.rows.length, 4);
|
|
assert.equal(split.groups.find((group) => /普货/.test(group.name))?.count, 1);
|
|
assert.equal(split.groups.find((group) => /冷链/.test(group.name))?.count, 3);
|
|
assert.equal(split.groups.reduce((sum, group) => sum + group.count, 0), total.groups.find((group) => group.name === "4.5T")?.count);
|
|
});
|