feat(platform): add ops capacity action board
This commit is contained in:
@@ -57,6 +57,16 @@ type CapacityMetricRow = {
|
|||||||
detail: string;
|
detail: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type OpsActionItem = {
|
||||||
|
key: string;
|
||||||
|
title: string;
|
||||||
|
level: 'P0' | 'P1' | 'P2';
|
||||||
|
owner: string;
|
||||||
|
evidence: string;
|
||||||
|
action: string;
|
||||||
|
acceptance: string;
|
||||||
|
};
|
||||||
|
|
||||||
const emptyCapacityMetrics: CapacityMetrics = {
|
const emptyCapacityMetrics: CapacityMetrics = {
|
||||||
activeConnections: 0,
|
activeConnections: 0,
|
||||||
kafkaLag: 0,
|
kafkaLag: 0,
|
||||||
@@ -256,6 +266,124 @@ function opsCapacityHandoffText(health: OpsHealth | null) {
|
|||||||
].join('\n');
|
].join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function opsActionItems(health: OpsHealth | null): OpsActionItem[] {
|
||||||
|
const metrics = getCapacityMetrics(health);
|
||||||
|
const actions: OpsActionItem[] = [];
|
||||||
|
|
||||||
|
if (metrics.bridgeAckPending > 0 || metrics.bridgeConsumerPending > 10000) {
|
||||||
|
actions.push({
|
||||||
|
key: 'nats-kafka-bridge',
|
||||||
|
title: '稳定 NATS 到 Kafka 桥接',
|
||||||
|
level: metrics.bridgeAckPending > 100 ? 'P0' : 'P1',
|
||||||
|
owner: '接入网关 / 消息队列',
|
||||||
|
evidence: `ACK ${formatNumber(metrics.bridgeAckPending)},待消费 ${formatNumber(metrics.bridgeConsumerPending)}`,
|
||||||
|
action: '检查 Kafka broker、topic 写入、NATS bridge 日志和服务器磁盘;必要时临时扩容桥接消费者。',
|
||||||
|
acceptance: 'ACK pending 回到 0,consumer pending 连续下降,Kafka Lag 不再增长。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metrics.fastWriterAckPending > 0 || metrics.fastWriterConsumerPending > 10000) {
|
||||||
|
actions.push({
|
||||||
|
key: 'fast-writer',
|
||||||
|
title: '保护实时写入通道',
|
||||||
|
level: metrics.fastWriterAckPending > 10 ? 'P0' : 'P1',
|
||||||
|
owner: '实时投影 / 存储',
|
||||||
|
evidence: `ACK ${formatNumber(metrics.fastWriterAckPending)},待消费 ${formatNumber(metrics.fastWriterConsumerPending)}`,
|
||||||
|
action: '优先确认 Redis 在线投影和 TDengine 快写是否正常,再看字段扁平化写入耗时。',
|
||||||
|
acceptance: '实时 KV 延迟恢复到秒级内,在线 TTL 正常续期,快速写入 pending 清零。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metrics.historyBatchPending > 0 || metrics.historyRowsPending > 0 || metrics.kafkaLag > 0) {
|
||||||
|
actions.push({
|
||||||
|
key: 'history-consumer',
|
||||||
|
title: '疏通历史与统计消费',
|
||||||
|
level: metrics.kafkaLag > 1000 || metrics.historyRowsPending > 10000 ? 'P0' : 'P1',
|
||||||
|
owner: '历史消费 / 统计',
|
||||||
|
evidence: `Kafka Lag ${formatNumber(metrics.kafkaLag)},历史待写 ${formatNumber(metrics.historyRowsPending)} 行`,
|
||||||
|
action: '查看 Kafka 消费者组、TDengine 批量写入错误和统计任务耗时;恢复后补看轨迹、RAW、里程统计。',
|
||||||
|
acceptance: 'Lag 持续归零,历史待写行归零,按 VIN 的历史查询恢复毫秒到百毫秒级。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metrics.activeConnections > 100000) {
|
||||||
|
actions.push({
|
||||||
|
key: 'gateway-capacity',
|
||||||
|
title: '扩容接入网关容量',
|
||||||
|
level: 'P1',
|
||||||
|
owner: '网关 / ECS',
|
||||||
|
evidence: `活跃连接 ${formatNumber(metrics.activeConnections)}`,
|
||||||
|
action: '检查 FD、CPU、网卡、端口 accept backlog 和协议源分布;准备按协议拆分实例。',
|
||||||
|
acceptance: '连接数增长时 CPU 和内存稳定,登录、注册、位置、实时帧响应无超时。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (health && (!health.tdengineWritable || !health.mysqlWritable)) {
|
||||||
|
actions.push({
|
||||||
|
key: 'storage-writable',
|
||||||
|
title: '恢复核心存储可写',
|
||||||
|
level: 'P0',
|
||||||
|
owner: '数据库 / 应用',
|
||||||
|
evidence: `TDengine ${statusText(health.tdengineWritable)},MySQL ${statusText(health.mysqlWritable)}`,
|
||||||
|
action: '先确认网络和账号,再看连接池、慢写、磁盘和表结构变更;避免实时链路被关系库写入拖慢。',
|
||||||
|
acceptance: '健康检查全部恢复正常,RAW、location、snapshot 和身份表写入成功。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (health && (health.activeConnections ?? 0) > 0 && (health.redisOnlineKeys ?? 0) === 0) {
|
||||||
|
actions.push({
|
||||||
|
key: 'redis-online',
|
||||||
|
title: '校验 Redis 在线投影',
|
||||||
|
level: 'P1',
|
||||||
|
owner: '实时投影',
|
||||||
|
evidence: `活跃连接 ${formatNumber(health.activeConnections)},在线 Key ${formatNumber(health.redisOnlineKeys)}`,
|
||||||
|
action: '确认只有实时数据帧进入 Redis,检查 online TTL 写入、VIN 解析和协议过滤条件。',
|
||||||
|
acceptance: '按协议、VIN 的在线状态可查,离线时间统计有连续依据。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (health?.runtime?.amapWebJsConfigured && !health.runtime.amapSecurityProxyEnabled) {
|
||||||
|
actions.push({
|
||||||
|
key: 'amap-security',
|
||||||
|
title: '收敛地图密钥暴露风险',
|
||||||
|
level: health.runtime.amapSecurityCodeExposed ? 'P1' : 'P2',
|
||||||
|
owner: '前端 / 平台 API',
|
||||||
|
evidence: amapSecurityText(health),
|
||||||
|
action: '启用高德安全代理,前端只保留 Web JS Key,服务端负责逆地理和安全密钥。',
|
||||||
|
acceptance: '轨迹回放和实时地图可用,浏览器端不暴露安全密钥。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!actions.length) {
|
||||||
|
actions.push({
|
||||||
|
key: 'steady-watch',
|
||||||
|
title: '维持生产容量巡检',
|
||||||
|
level: 'P2',
|
||||||
|
owner: '值班 / 平台',
|
||||||
|
evidence: '当前未发现积压或核心写入异常',
|
||||||
|
action: '继续观察 32960、808、MQTT 三类实时数据源、在线状态、地图服务和历史查询耗时。',
|
||||||
|
acceptance: '10W 车辆接入准备项持续无红色风险,实时投影和历史消费无积压。'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return actions;
|
||||||
|
}
|
||||||
|
|
||||||
|
function opsCapacityActionPlanText(health: OpsHealth | null) {
|
||||||
|
const actions = opsActionItems(health);
|
||||||
|
return [
|
||||||
|
'【车辆数据中台容量处置计划】',
|
||||||
|
`生成版本:${health?.runtime?.platformRelease || '-'}`,
|
||||||
|
...actions.map((item, index) => [
|
||||||
|
`${index + 1}. ${item.level} ${item.title}`,
|
||||||
|
`负责人:${item.owner}`,
|
||||||
|
`证据:${item.evidence}`,
|
||||||
|
`动作:${item.action}`,
|
||||||
|
`验收:${item.acceptance}`
|
||||||
|
].join('\n'))
|
||||||
|
].join('\n\n');
|
||||||
|
}
|
||||||
|
|
||||||
async function copyText(value: string, label: string) {
|
async function copyText(value: string, label: string) {
|
||||||
try {
|
try {
|
||||||
await navigator.clipboard.writeText(value);
|
await navigator.clipboard.writeText(value);
|
||||||
@@ -285,6 +413,7 @@ export function OpsQuality() {
|
|||||||
const capacityRisks = (health?.capacityFindings ?? []).map(capacityRisk);
|
const capacityRisks = (health?.capacityFindings ?? []).map(capacityRisk);
|
||||||
const capacityRows = capacityMetricRows(health);
|
const capacityRows = capacityMetricRows(health);
|
||||||
const bridgeRisks = capacityRisks.filter((item) => item.category === 'NATS -> Kafka');
|
const bridgeRisks = capacityRisks.filter((item) => item.category === 'NATS -> Kafka');
|
||||||
|
const actionItems = opsActionItems(health);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="vp-page">
|
<div className="vp-page">
|
||||||
@@ -294,6 +423,7 @@ export function OpsQuality() {
|
|||||||
actions={(
|
actions={(
|
||||||
<Space>
|
<Space>
|
||||||
<Button size="small" icon={<IconCopy />} aria-label="复制容量交接" disabled={!health} onClick={() => copyText(opsCapacityHandoffText(health), '运维容量交接')}>复制容量交接</Button>
|
<Button size="small" icon={<IconCopy />} aria-label="复制容量交接" disabled={!health} onClick={() => copyText(opsCapacityHandoffText(health), '运维容量交接')}>复制容量交接</Button>
|
||||||
|
<Button size="small" icon={<IconCopy />} aria-label="复制处置计划" disabled={!health} onClick={() => copyText(opsCapacityActionPlanText(health), '容量处置计划')}>复制处置计划</Button>
|
||||||
<Button size="small" loading={loading} onClick={load}>刷新</Button>
|
<Button size="small" loading={loading} onClick={load}>刷新</Button>
|
||||||
</Space>
|
</Space>
|
||||||
)}
|
)}
|
||||||
@@ -372,6 +502,23 @@ export function OpsQuality() {
|
|||||||
/>
|
/>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
|
<Card bordered title="容量风险处置工作台" loading={loading} style={{ marginTop: 16 }}>
|
||||||
|
<div className="vp-ops-action-grid">
|
||||||
|
{actionItems.map((item) => (
|
||||||
|
<div key={item.key} className="vp-ops-action-item">
|
||||||
|
<Space wrap>
|
||||||
|
<Tag color={item.level === 'P0' ? 'red' : item.level === 'P1' ? 'orange' : 'green'}>{item.level}</Tag>
|
||||||
|
<Tag color="blue">{item.owner}</Tag>
|
||||||
|
</Space>
|
||||||
|
<Typography.Title heading={6} style={{ margin: 0 }}>{item.title}</Typography.Title>
|
||||||
|
<div className="vp-ops-action-evidence">{item.evidence}</div>
|
||||||
|
<Typography.Text type="secondary">{item.action}</Typography.Text>
|
||||||
|
<div className="vp-ops-action-acceptance">{item.acceptance}</div>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
|
||||||
{capacityRisks.length ? (
|
{capacityRisks.length ? (
|
||||||
<Card bordered title="容量与风险发现" style={{ marginTop: 16 }}>
|
<Card bordered title="容量与风险发现" style={{ marginTop: 16 }}>
|
||||||
<div className="vp-risk-grid">
|
<div className="vp-risk-grid">
|
||||||
|
|||||||
@@ -881,6 +881,39 @@ button.vp-realtime-command-item:focus-visible {
|
|||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.vp-ops-action-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vp-ops-action-item {
|
||||||
|
min-height: 196px;
|
||||||
|
padding: 12px;
|
||||||
|
border: 1px solid var(--vp-border);
|
||||||
|
border-radius: var(--vp-radius);
|
||||||
|
background: #fbfcff;
|
||||||
|
display: grid;
|
||||||
|
gap: 10px;
|
||||||
|
align-content: start;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vp-ops-action-evidence {
|
||||||
|
color: var(--vp-text);
|
||||||
|
font-size: 18px;
|
||||||
|
font-weight: 700;
|
||||||
|
line-height: 26px;
|
||||||
|
word-break: break-word;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vp-ops-action-acceptance {
|
||||||
|
padding-top: 8px;
|
||||||
|
border-top: 1px solid var(--vp-border);
|
||||||
|
color: var(--vp-text-muted);
|
||||||
|
font-size: 13px;
|
||||||
|
line-height: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
.vp-alert-ops-grid {
|
.vp-alert-ops-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(0, 1.4fr) minmax(360px, 0.6fr);
|
grid-template-columns: minmax(0, 1.4fr) minmax(360px, 0.6fr);
|
||||||
@@ -1702,6 +1735,7 @@ button.vp-realtime-command-item:focus-visible {
|
|||||||
.vp-alert-command-grid,
|
.vp-alert-command-grid,
|
||||||
.vp-sla-summary,
|
.vp-sla-summary,
|
||||||
.vp-risk-grid,
|
.vp-risk-grid,
|
||||||
|
.vp-ops-action-grid,
|
||||||
.vp-alert-ops-grid,
|
.vp-alert-ops-grid,
|
||||||
.vp-stat-workspace,
|
.vp-stat-workspace,
|
||||||
.vp-stat-insight-grid,
|
.vp-stat-insight-grid,
|
||||||
|
|||||||
@@ -3645,6 +3645,10 @@ test('renders ops quality as a standalone runtime health page', async () => {
|
|||||||
expect(screen.getByText('历史行待写')).toBeInTheDocument();
|
expect(screen.getByText('历史行待写')).toBeInTheDocument();
|
||||||
expect(screen.getByText('1,482,047')).toBeInTheDocument();
|
expect(screen.getByText('1,482,047')).toBeInTheDocument();
|
||||||
expect(screen.getByText('4,000')).toBeInTheDocument();
|
expect(screen.getByText('4,000')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('容量风险处置工作台')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('稳定 NATS 到 Kafka 桥接')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText('恢复核心存储可写')).toBeInTheDocument();
|
||||||
|
expect(screen.getByText(/历史待写 320 行/)).toBeInTheDocument();
|
||||||
expect(screen.getByText('Kafka lag 42')).toBeInTheDocument();
|
expect(screen.getByText('Kafka lag 42')).toBeInTheDocument();
|
||||||
expect(screen.getByText('NATS 桥接 ACK 未确认')).toBeInTheDocument();
|
expect(screen.getByText('NATS 桥接 ACK 未确认')).toBeInTheDocument();
|
||||||
expect(screen.getAllByText('NATS 桥接待消费').length).toBeGreaterThanOrEqual(1);
|
expect(screen.getAllByText('NATS 桥接待消费').length).toBeGreaterThanOrEqual(1);
|
||||||
@@ -3658,6 +3662,11 @@ test('renders ops quality as a standalone runtime health page', async () => {
|
|||||||
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('NATS桥接:consumer pending 1,482,047 / ack pending 4,000'));
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('NATS桥接:consumer pending 1,482,047 / ack pending 4,000'));
|
||||||
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('存储状态:TDengine 正常 / MySQL 异常'));
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('存储状态:TDengine 正常 / MySQL 异常'));
|
||||||
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('1. 先看 Kafka Lag 与 NATS ACK pending'));
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('1. 先看 Kafka Lag 与 NATS ACK pending'));
|
||||||
|
|
||||||
|
fireEvent.click(screen.getByRole('button', { name: '复制处置计划' }));
|
||||||
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('【车辆数据中台容量处置计划】'));
|
||||||
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('P0 稳定 NATS 到 Kafka 桥接'));
|
||||||
|
expect(writeText).toHaveBeenCalledWith(expect.stringContaining('负责人:数据库 / 应用'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('copies notification text from quality priority queue', async () => {
|
test('copies notification text from quality priority queue', async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user