From c735372cc1ab0f4d85a766f8f81d2981366be5fa Mon Sep 17 00:00:00 2001 From: lingniu Date: Fri, 3 Jul 2026 19:59:10 +0800 Subject: [PATCH] ops(go): schedule capacity health checks --- deploy/systemd/lingniu-go-capacity-check.service | 13 +++++++++++++ deploy/systemd/lingniu-go-capacity-check.timer | 12 ++++++++++++ docs/ops/go-service-observability.md | 11 +++++++++++ docs/ops/vehicle-ingest-runbook.md | 12 ++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 deploy/systemd/lingniu-go-capacity-check.service create mode 100644 deploy/systemd/lingniu-go-capacity-check.timer diff --git a/deploy/systemd/lingniu-go-capacity-check.service b/deploy/systemd/lingniu-go-capacity-check.service new file mode 100644 index 00000000..81e0d2d1 --- /dev/null +++ b/deploy/systemd/lingniu-go-capacity-check.service @@ -0,0 +1,13 @@ +[Unit] +Description=Lingniu Go Capacity Health Check +Documentation=file:/opt/lingniu-go-native/current/docs/ops/go-service-observability.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/opt/lingniu-go-native/current +ExecStart=/opt/lingniu-go-native/current/capacity-check +TimeoutStartSec=10 +StandardOutput=journal +StandardError=journal diff --git a/deploy/systemd/lingniu-go-capacity-check.timer b/deploy/systemd/lingniu-go-capacity-check.timer new file mode 100644 index 00000000..f53a40d1 --- /dev/null +++ b/deploy/systemd/lingniu-go-capacity-check.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run Lingniu Go Capacity Health Check every minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +AccuracySec=5s +Persistent=true +Unit=lingniu-go-capacity-check.service + +[Install] +WantedBy=timers.target diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index de64747b..7badc290 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -37,6 +37,17 @@ curl -fsS http://127.0.0.1:20200/readyz 它会抓取 Gateway、History writer、Stat writer、NATS bridge、Realtime API、NATS fast writer 的本地 `/metrics`，输出 JSON。退出码 `0` 表示当前关键 backlog 和拒绝计数正常，退出码 `2` 表示存在 pending、Kafka lag、连接拒绝或 metrics 抓取失败，适合接入 cron/告警。 +ECS 上通过 systemd timer 每分钟执行一次： + +```bash +systemctl status lingniu-go-capacity-check.timer +systemctl list-timers lingniu-go-capacity-check.timer +journalctl -u lingniu-go-capacity-check.service --since '10 minutes ago' --no-pager +systemctl start lingniu-go-capacity-check.service +``` + +`lingniu-go-capacity-check.service` 是 oneshot 服务。容量健康时退出码为 `0`；不健康时退出码为 `2`，timer 会保留 failed 结果，JSON findings 会写入 journal。 + ## Core Counters | Metric | Meaning | diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index 984a045f..e1b41bd9 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -76,6 +76,8 @@ curl -fsS http://127.0.0.1:20212/metrics | grep vehicle_history_kafka_lag curl -fsS http://127.0.0.1:20212/metrics | grep vehicle_history_batch curl -fsS http://127.0.0.1:20213/metrics | grep vehicle_stat_kafka_lag curl -fsS http://127.0.0.1:20200/metrics | grep vehicle_realtime_kafka_lag + +/opt/lingniu-go-native/current/capacity-check ``` ## 健康基线 @@ -85,10 +87,20 @@ curl -fsS http://127.0.0.1:20200/metrics | grep vehicle_realtime_kafka_lag - 所有 `/readyz` 都返回 `status=ok`。 - `vehicle_bridge_nats_consumer_ack_pending` 为 `0`。 - history、stat、realtime 的 Kafka lag 为 `0` 或短时间小幅波动后归零。 +- `capacity-check` 返回 `status=ok` 且退出码为 `0`。 - gateway 的帧计数持续增长。 - bridge 的 Kafka write 和 NATS ack 计数同时增长。 - writer 的成功计数增长，同时 Kafka lag 不持续扩大。 +定时容量巡检由 systemd timer 触发： + +```bash +systemctl status lingniu-go-capacity-check.timer +journalctl -u lingniu-go-capacity-check.service --since '10 minutes ago' --no-pager +``` + +如果 `lingniu-go-capacity-check.service` 失败，先看 journal JSON 里的 `findings`，再按对应层级处理。 + GB32960 和 JT/T 808 的活跃连接数受上游平台连接方式影响，不能直接等同于车辆数。突然归零或持续异常下降才是信号。 ## 压测入口