shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(mileage): stream workbook export rows

Browse Source
This commit is contained in:
lingniu
2026-07-16 09:08:04 +08:00
parent b78cf1696f
commit 61c1bbc365
8 changed files with 191 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
vehicle-data-platform/apps/web/src/v2/domain/mileageExport.test.ts
View File

@@ -1,5 +1,5 @@
import { afterEach, expect, test, vi } from 'vitest';
import { downloadMileageWorkbook } from './mileageExport';
import { createMileageExportStream, downloadMileageWorkbook } from './mileageExport';
import { createMileageWorkbook } from './mileageWorkbook';
afterEach(() => vi.unstubAllGlobals());
@@ -81,12 +81,35 @@ test('terminates the workbook worker when an active export is cancelled', async
dateTo: '2026-07-14',
dates: ['2026-07-13', '2026-07-14'],
vehicles: [],
mileageRows: [],
mileageRows: [{ vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', startMileageKm: 100, endMileageKm: 188.7, dailyMileageKm: 88.7, source: 'GB32960' }],
sources: []
}, controller.signal);
expect(postMessage).toHaveBeenCalledTimes(1);
expect(postMessage).toHaveBeenCalledTimes(3);
expect(postMessage.mock.calls.map(([message]) => message.type)).toEqual(['start', 'rows', 'finish']);
expect(postMessage.mock.calls[1][0].rows).toEqual([{ vin: 'LTEST000000000001', date: '2026-07-13', dailyMileageKm: 88.7 }]);
controller.abort();
await expect(exportPromise).rejects.toMatchObject({ name: 'AbortError' });
expect(terminate).toHaveBeenCalledTimes(1);
});
test('releases a streaming export before finish when the route aborts', async () => {
const terminate = vi.fn();
const postMessage = vi.fn();
class WorkerMock {
onmessage: ((event: MessageEvent) => void) | null = null;
onerror: ((event: ErrorEvent) => void) | null = null;
postMessage = postMessage;
terminate = terminate;
}
vi.stubGlobal('Worker', WorkerMock);
const controller = new AbortController();
const stream = createMileageExportStream({ dateFrom: '2026-01-01', dateTo: '2026-12-31', dates: [], sources: [] }, controller.signal);
stream.appendRows([{ vin: 'VIN001', date: '2026-01-01', dailyMileageKm: 12.5 }]);
expect(postMessage.mock.calls.map(([message]) => message.type)).toEqual(['start', 'rows']);
controller.abort();
expect(terminate).toHaveBeenCalledTimes(1);
await expect(stream.finish([{ vin: 'VIN001', plate: '粤A00001' }])).rejects.toMatchObject({ name: 'AbortError' });
});

109
vehicle-data-platform/apps/web/src/v2/domain/mileageExport.ts
View File

@@ -3,18 +3,23 @@ import { downloadBlob } from './download';
export type MileageExportVehicle = { vin: string; plate: string };
export type MileageExportSource = { protocol: string; label: string; mileageType: string };
export type MileageWorkbookRow = Pick<DailyMileageRow, 'vin' | 'date' | 'dailyMileageKm'> & Partial<Omit<DailyMileageRow, 'vin' | 'date' | 'dailyMileageKm'>>;
export type MileageExportInput = {
dateFrom: string;
dateTo: string;
dates: string[];
vehicles: MileageExportVehicle[];
mileageRows: DailyMileageRow[];
mileageRows: MileageWorkbookRow[];
sources: MileageExportSource[];
exportedAt?: Date;
};
type MileageWorkerResponse = { ok: true; buffer: ArrayBuffer } | { ok: false; message: string };
type MileageWorkerRequest =
| { type: 'start'; input: Omit<MileageExportInput, 'vehicles' | 'mileageRows'> }
| { type: 'rows'; rows: MileageWorkbookRow[] }
| { type: 'finish'; vehicles: MileageExportVehicle[] };
function abortError() {
return new DOMException('里程导出已取消', 'AbortError');
@@ -24,42 +29,88 @@ function throwIfAborted(signal?: AbortSignal) {
if (signal?.aborted) throw abortError();
}
async function createMileageWorkbookBuffer(input: MileageExportInput, signal?: AbortSignal) {
export type MileageExportStream = {
appendRows: (rows: readonly MileageWorkbookRow[]) => void;
finish: (vehicles: MileageExportVehicle[]) => Promise<void>;
dispose: () => void;
};
export function createMileageExportStream(input: Omit<MileageExportInput, 'vehicles' | 'mileageRows'>, signal?: AbortSignal): MileageExportStream {
throwIfAborted(signal);
if (typeof Worker === 'undefined') {
throw new Error('当前浏览器不支持后台 Excel 导出，请升级到最新版 Chrome、Edge 或 Safari 后重试');
}
return new Promise<ArrayBuffer>((resolve, reject) => {
const worker = new Worker(new URL('./mileageExport.worker.ts', import.meta.url), { type: 'module' });
let settled = false;
const cleanup = () => {
signal?.removeEventListener('abort', cancel);
worker.terminate();
};
const finish = (callback: () => void) => {
if (settled) return;
settled = true;
cleanup();
callback();
};
const cancel = () => finish(() => reject(abortError()));
worker.onmessage = (event: MessageEvent<MileageWorkerResponse>) => {
finish(() => event.data.ok ? resolve(event.data.buffer) : reject(new Error(event.data.message)));
};
worker.onerror = (event) => finish(() => reject(new Error(event.message || 'Excel 工作线程执行失败')));
signal?.addEventListener('abort', cancel, { once: true });
if (signal?.aborted) {
cancel();
const worker = new Worker(new URL('./mileageExport.worker.ts', import.meta.url), { type: 'module' });
let disposed = false;
let finishing = false;
let resolveFinish: (() => void) | undefined;
let rejectFinish: ((reason?: unknown) => void) | undefined;
let earlyError: Error | undefined;
const dispose = () => {
if (disposed) return;
disposed = true;
signal?.removeEventListener('abort', cancel);
worker.terminate();
};
const reject = (error: Error) => {
if (disposed) return;
if (rejectFinish) rejectFinish(error);
else earlyError = error;
dispose();
};
const cancel = () => reject(abortError());
worker.onmessage = (event: MessageEvent<MileageWorkerResponse>) => {
if (disposed) return;
if (!event.data.ok) {
reject(new Error(event.data.message));
return;
}
worker.postMessage(input);
});
const buffer = event.data.buffer;
const blob = new Blob([buffer], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' });
downloadBlob(blob, `车辆里程查询_${input.dateFrom.split('-').join('')}-${input.dateTo.split('-').join('')}_${finishVehicles.length}辆.xlsx`);
resolveFinish?.();
dispose();
};
worker.onerror = (event) => reject(new Error(event.message || 'Excel 工作线程执行失败'));
signal?.addEventListener('abort', cancel, { once: true });
let finishVehicles: MileageExportVehicle[] = [];
const post = (message: MileageWorkerRequest) => worker.postMessage(message);
post({ type: 'start', input });
return {
appendRows(rows) {
if (disposed || finishing) return;
post({ type: 'rows', rows: rows.map((row) => ({ vin: row.vin, date: row.date, dailyMileageKm: row.dailyMileageKm })) });
},
finish(vehicles) {
if (disposed) return Promise.reject(earlyError ?? abortError());
if (finishing) return Promise.reject(new Error('里程导出已经进入生成阶段'));
finishing = true;
finishVehicles = vehicles;
return new Promise<void>((resolve, rejectPromise) => {
resolveFinish = resolve;
rejectFinish = rejectPromise;
if (earlyError) {
rejectPromise(earlyError);
dispose();
return;
}
post({ type: 'finish', vehicles });
});
},
dispose
};
}
export async function downloadMileageWorkbook(input: MileageExportInput, signal?: AbortSignal) {
const buffer = await createMileageWorkbookBuffer(input, signal);
throwIfAborted(signal);
const blob = new Blob([buffer], { type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' });
downloadBlob(blob, `车辆里程查询_${input.dateFrom.split('-').join('')}-${input.dateTo.split('-').join('')}_${input.vehicles.length}辆.xlsx`);
const stream = createMileageExportStream({
dateFrom: input.dateFrom,
dateTo: input.dateTo,
dates: input.dates,
sources: input.sources,
exportedAt: input.exportedAt
}, signal);
stream.appendRows(input.mileageRows);
await stream.finish(input.vehicles);
}

27
vehicle-data-platform/apps/web/src/v2/domain/mileageExport.worker.ts
View File

@@ -1,21 +1,40 @@
import type { MileageExportInput } from './mileageExport';
import type { MileageExportInput, MileageWorkbookRow, MileageExportVehicle } from './mileageExport';
import { createMileageWorkbook } from './mileageWorkbook';
import { transferableWorkbookBuffer } from './transferableBuffer';
type MileageWorkerResponse = { ok: true; buffer: ArrayBuffer } | { ok: false; message: string };
type MileageWorkerRequest =
| { type: 'start'; input: Omit<MileageExportInput, 'vehicles' | 'mileageRows'> }
| { type: 'rows'; rows: MileageWorkbookRow[] }
| { type: 'finish'; vehicles: MileageExportVehicle[] };
type WorkerScope = {
onmessage: ((event: MessageEvent<MileageExportInput>) => void) | null;
onmessage: ((event: MessageEvent<MileageWorkerRequest>) => void) | null;
postMessage: (message: MileageWorkerResponse, transfer?: Transferable[]) => void;
};
const workerScope = self as unknown as WorkerScope;
let baseInput: Omit<MileageExportInput, 'vehicles' | 'mileageRows'> | undefined;
let mileageRows: MileageWorkbookRow[] = [];
workerScope.onmessage = async (event) => {
if (event.data.type === 'start') {
baseInput = event.data.input;
mileageRows = [];
return;
}
if (event.data.type === 'rows') {
mileageRows.push(...event.data.rows);
return;
}
try {
const workbook = await createMileageWorkbook(event.data);
if (!baseInput) throw new Error('Excel 工作线程未初始化');
const workbook = await createMileageWorkbook({ ...baseInput, vehicles: event.data.vehicles, mileageRows });
mileageRows = [];
const output = await workbook.xlsx.writeBuffer();
const buffer = new Uint8Array(output as ArrayBuffer).slice().buffer;
const buffer = transferableWorkbookBuffer(output as ArrayBuffer | Uint8Array);
workerScope.postMessage({ ok: true, buffer }, [buffer]);
} catch (error) {
mileageRows = [];
workerScope.postMessage({
ok: false,
message: error instanceof Error ? error.message : 'Excel 工作簿生成失败'

7
vehicle-data-platform/apps/web/src/v2/domain/mileageWorkbook.ts
View File

@@ -1,5 +1,4 @@
import type { DailyMileageRow } from '../../api/types';
import type { MileageExportInput } from './mileageExport';
import type { MileageExportInput, MileageWorkbookRow } from './mileageExport';
function excelColumn(index: number) {
let value = index;
@@ -87,14 +86,14 @@ export async function createMileageWorkbook(input: MileageExportInput) {
if (isDateHeader) cell.numFmt = 'm/d';
});
const rowsByVin = new Map<string, Map<string, DailyMileageRow>>();
const rowsByVin = new Map<string, Map<string, MileageWorkbookRow>>();
for (const row of input.mileageRows) {
if (!rowsByVin.has(row.vin)) rowsByVin.set(row.vin, new Map());
rowsByVin.get(row.vin)!.set(row.date, row);
}
input.vehicles.forEach((vehicle, index) => {
const rowNumber = firstDataRow + index;
const mileageByDate = rowsByVin.get(vehicle.vin) ?? new Map<string, DailyMileageRow>();
const mileageByDate = rowsByVin.get(vehicle.vin) ?? new Map<string, MileageWorkbookRow>();
const dailyValues = input.dates.map((date) => mileageByDate.get(date)?.dailyMileageKm ?? null);
const total = dailyValues.reduce<number>((sum, value) => sum + (value ?? 0), 0);
const row = sheet.getRow(rowNumber);

16
vehicle-data-platform/apps/web/src/v2/domain/transferableBuffer.test.ts Normal file
View File

@@ -0,0 +1,16 @@
import { expect, test } from 'vitest';
import { transferableWorkbookBuffer } from './transferableBuffer';
test('reuses complete workbook buffers and only copies partial views', () => {
const direct = new ArrayBuffer(8);
expect(transferableWorkbookBuffer(direct)).toBe(direct);
const completeView = new Uint8Array(new ArrayBuffer(8));
expect(transferableWorkbookBuffer(completeView)).toBe(completeView.buffer);
const source = new Uint8Array([1, 2, 3, 4]);
const partial = source.subarray(1, 3);
const transferred = transferableWorkbookBuffer(partial);
expect(transferred).not.toBe(source.buffer);
expect([...new Uint8Array(transferred)]).toEqual([2, 3]);
});

7
vehicle-data-platform/apps/web/src/v2/domain/transferableBuffer.ts Normal file
View File

@@ -0,0 +1,7 @@
export function transferableWorkbookBuffer(output: ArrayBuffer | Uint8Array): ArrayBuffer {
if (output instanceof ArrayBuffer) return output;
if (output.buffer instanceof ArrayBuffer && output.byteOffset === 0 && output.byteLength === output.buffer.byteLength) {
return output.buffer;
}
return output.slice().buffer as ArrayBuffer;
}

30
vehicle-data-platform/apps/web/src/v2/pages/StatisticsPage.test.tsx
View File

@@ -6,11 +6,11 @@ import StatisticsPage from './StatisticsPage';
import { ROUTER_FUTURE } from '../routing/routerConfig';
const mocks = vi.hoisted(() => ({ mileageStatistics: vi.fn(), dailyMileage: vi.fn(), vehicles: vi.fn(), vehicleCoverage: vi.fn() }));
const exportMocks = vi.hoisted(() => ({ downloadMileageWorkbook: vi.fn() }));
const exportMocks = vi.hoisted(() => ({ createMileageExportStream: vi.fn(), appendRows: vi.fn(), finish: vi.fn(), dispose: vi.fn() }));
vi.mock('../../api/client', () => ({ api: mocks }));
vi.mock('../domain/mileageExport', () => exportMocks);
vi.mock('../domain/mileageExport', () => ({ createMileageExportStream: exportMocks.createMileageExportStream }));
afterEach(() => { cleanup(); vi.restoreAllMocks(); window.localStorage.clear(); Object.values(mocks).forEach((mock) => mock.mockReset()); exportMocks.downloadMileageWorkbook.mockReset(); });
afterEach(() => { cleanup(); vi.restoreAllMocks(); window.localStorage.clear(); Object.values(mocks).forEach((mock) => mock.mockReset()); Object.values(exportMocks).forEach((mock) => mock.mockReset()); });
function renderPage(initialEntry = '/statistics') {
const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
@@ -18,6 +18,8 @@ function renderPage(initialEntry = '/statistics') {
}
function prepareData() {
exportMocks.finish.mockResolvedValue(undefined);
exportMocks.createMileageExportStream.mockReturnValue({ appendRows: exportMocks.appendRows, finish: exportMocks.finish, dispose: exportMocks.dispose });
mocks.mileageStatistics.mockResolvedValue({
dateFrom: '2026-07-01', dateTo: '2026-07-14', vehicleCount: 1, recordCount: 2, sourceCount: 2,
periodMileageKm: 193.3, fleetLatestMileageKm: 119925, averageMileagePerVin: 193.3, averageDailyMileageKm: 96.65,
@@ -188,20 +190,30 @@ test('paginates all unique vehicles when no license plate is selected', async ()
test('exports the full fleet mileage with one paginated query instead of VIN batches', async () => {
prepareData();
exportMocks.downloadMileageWorkbook.mockResolvedValue(undefined);
const exportRows = [
{ vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', startMileageKm: 119731.7, endMileageKm: 119820.4, dailyMileageKm: 88.7, source: 'GB32960' },
{ vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-14', startMileageKm: 119820.4, endMileageKm: 119925, dailyMileageKm: 104.6, source: 'GB32960' }
];
mocks.dailyMileage.mockImplementation((params: URLSearchParams) => {
if (params.get('vehicleScope') !== 'bound') return Promise.resolve({ items: exportRows, total: 2, limit: 10000, offset: 0 });
const offset = Number(params.get('offset') ?? 0);
return Promise.resolve({ items: exportRows.slice(offset, offset + 1), total: 2, limit: 1, offset });
});
renderPage('/statistics?dateFrom=2026-07-13&dateTo=2026-07-14');
await waitFor(() => expect(screen.getByRole('button', { name: /导出 Excel/ })).toBeEnabled());
fireEvent.click(screen.getByRole('button', { name: /导出 Excel/ }));
await waitFor(() => expect(exportMocks.downloadMileageWorkbook).toHaveBeenCalledTimes(1));
await waitFor(() => expect(exportMocks.finish).toHaveBeenCalledTimes(1));
const exportMileageCall = mocks.dailyMileage.mock.calls[mocks.dailyMileage.mock.calls.length - 1][0] as URLSearchParams;
expect(exportMileageCall.get('vins')).toBeNull();
expect(exportMileageCall.get('vehicleScope')).toBe('bound');
expect(exportMocks.downloadMileageWorkbook.mock.calls[0][0].vehicles).toHaveLength(1);
expect(exportMocks.appendRows).toHaveBeenCalledTimes(2);
expect(exportMocks.appendRows.mock.calls.map(([rows]) => rows.map((row: { date: string }) => row.date))).toEqual([['2026-07-13'], ['2026-07-14']]);
expect(exportMocks.finish.mock.calls[0][0]).toHaveLength(1);
expect(mocks.vehicleCoverage.mock.calls[mocks.vehicleCoverage.mock.calls.length - 1][1]).toBeInstanceOf(AbortSignal);
expect(mocks.dailyMileage.mock.calls[mocks.dailyMileage.mock.calls.length - 1][1]).toBeInstanceOf(AbortSignal);
expect(exportMocks.downloadMileageWorkbook.mock.calls[0][1]).toBeInstanceOf(AbortSignal);
expect(exportMocks.createMileageExportStream.mock.calls[0][1]).toBeInstanceOf(AbortSignal);
expect(screen.getByText(/已导出 1 辆车/)).toBeInTheDocument();
});
@@ -227,7 +239,7 @@ test('shows export progress and lets the user cancel the active request', async
await waitFor(() => expect(exportSignal?.aborted).toBe(true));
expect(await screen.findByText(/已取消导出/)).toBeInTheDocument();
expect(exportMocks.downloadMileageWorkbook).not.toHaveBeenCalled();
expect(exportMocks.createMileageExportStream).not.toHaveBeenCalled();
});
test('aborts an active export when the mileage route unmounts', async () => {
@@ -248,5 +260,5 @@ test('aborts an active export when the mileage route unmounts', async () => {
view.unmount();
expect(exportSignal?.aborted).toBe(true);
expect(exportMocks.downloadMileageWorkbook).not.toHaveBeenCalled();
expect(exportMocks.createMileageExportStream).not.toHaveBeenCalled();
});

28
vehicle-data-platform/apps/web/src/v2/pages/StatisticsPage.tsx
View File

@@ -4,7 +4,7 @@ import { FormEvent, useEffect, useMemo, useRef, useState } from 'react';
import { useSearchParams } from 'react-router-dom';
import { api } from '../../api/client';
import type { DailyMileageRow, MileageStatistics, Page, VehicleRow } from '../../api/types';
import { downloadMileageWorkbook } from '../domain/mileageExport';
import { createMileageExportStream, type MileageExportStream } from '../domain/mileageExport';
import { formatZhNumber } from '../domain/formatters';
import { InlineError } from '../shared/AsyncState';
import { QUERY_MEMORY, retainPreviousPageWithinScope } from '../queryPolicy';
@@ -347,6 +347,7 @@ export default function StatisticsPage() {
};
setIsExporting(true);
setExportFeedback('');
let exportStream: MileageExportStream | undefined;
try {
let vehicles: VehicleOption[] = criteria.vehicles.map((vehicle) => ({ ...vehicle }));
if (!vehicles.length) {
@@ -362,7 +363,14 @@ export default function StatisticsPage() {
if (offset >= result.total) break;
}
}
const mileageRows: DailyMileageRow[] = [];
const plateByVin = new Map(vehicles.filter((vehicle) => vehicle.plate).map((vehicle) => [vehicle.vin, vehicle.plate]));
exportStream = createMileageExportStream({
dateFrom: criteria.dateFrom,
dateTo: criteria.dateTo,
dates,
sources: criteria.sources.filter((source) => source.enabled),
exportedAt: new Date()
}, controller.signal);
const vehicleBatches = criteria.vehicles.length
? Array.from({ length: Math.ceil(vehicles.length / EXPORT_VIN_BATCH_SIZE) }, (_, index) => vehicles.slice(index * EXPORT_VIN_BATCH_SIZE, (index + 1) * EXPORT_VIN_BATCH_SIZE))
: [[] as VehicleOption[]];
@@ -375,24 +383,17 @@ export default function StatisticsPage() {
while (true) {
const params = mileageParams({ ...criteria, vehicles: vehicleBatch }, offset);
const result = await api.dailyMileage(params, controller.signal);
mileageRows.push(...result.items);
for (const row of result.items) if (row.plate) plateByVin.set(row.vin, row.plate);
exportStream.appendRows(result.items);
offset += result.items.length;
reportProgress({ label: mileageLabel, completed: offset, total: result.total });
if (!result.items.length || offset >= result.total) break;
}
}
const plateByVin = new Map(mileageRows.filter((row) => row.plate).map((row) => [row.vin, row.plate]));
vehicles = vehicles.map((vehicle) => ({ ...vehicle, plate: vehicle.plate || plateByVin.get(vehicle.vin) || '' }));
reportProgress({ label: '正在后台生成 Excel 文件' });
await downloadMileageWorkbook({
dateFrom: criteria.dateFrom,
dateTo: criteria.dateTo,
dates,
vehicles,
mileageRows,
sources: criteria.sources.filter((source) => source.enabled),
exportedAt: new Date()
}, controller.signal);
await exportStream.finish(vehicles);
exportStream = undefined;
if (mountedRef.current) setExportFeedback(`已导出 ${vehicles.length} 辆车`);
} catch (error) {
if (mountedRef.current) {
@@ -401,6 +402,7 @@ export default function StatisticsPage() {
: error instanceof Error ? `导出失败：${error.message}` : '导出失败，请稍后重试');
}
} finally {
exportStream?.dispose();
if (exportControllerRef.current === controller) exportControllerRef.current = null;
if (mountedRef.current) {
setIsExporting(false);