diff --git a/.gitignore b/.gitignore index 48302d07..1295ccb8 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,8 @@ hs_err_pid* replay_pid* .worktrees/ + +# Local frontend caches and generated review artifacts +.vite/ +outputs/ +vehicle-data-platform/apps/web/public/app-config.js diff --git a/deploy/systemd/lingniu-go-fields-projector.service b/deploy/systemd/lingniu-go-fields-projector.service new file mode 100644 index 00000000..34a947d7 --- /dev/null +++ b/deploy/systemd/lingniu-go-fields-projector.service @@ -0,0 +1,21 @@ +[Unit] +Description=Lingniu Go RAW Fields Projector +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=/opt/lingniu-go-native/current +EnvironmentFile=/opt/lingniu-go-native/env/base.env +EnvironmentFile=/opt/lingniu-go-native/env/fields-projector.env +Environment=GOMEMLIMIT=384MiB +ExecStart=/opt/lingniu-go-native/current/fields-projector +Restart=always +RestartSec=3 +LimitNOFILE=1048576 +MemoryLimit=512M +KillSignal=SIGTERM +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/docs/architecture/production-data-plane-inventory.md b/docs/architecture/production-data-plane-inventory.md index 7f6b40bd..98fae4f8 100644 --- a/docs/architecture/production-data-plane-inventory.md +++ b/docs/architecture/production-data-plane-inventory.md @@ -10,11 +10,14 @@ | 服务 | systemd 单元 | 端口 | 职责 | | --- | --- | --- | --- | -| Gateway | `lingniu-go-gateway.service` | `0.0.0.0:32960`、`0.0.0.0:808`、`127.0.0.1:20211` | GB32960、JT808、宇通 MQTT 接入和协议解析 | -| NATS Kafka bridge | `lingniu-go-nats-kafka-bridge.service` | `127.0.0.1:20214` | NATS JetStream 到 Kafka 的可靠桥接 | -| History writer | `lingniu-go-history-writer.service` | `127.0.0.1:20212` | Kafka raw 消费，写 TDengine | -| Stat writer | `lingniu-go-stat-writer.service` | `127.0.0.1:20213` | Kafka raw 消费，写每日里程 | -| Realtime API/projector | `lingniu-go-realtime-api.service` | `0.0.0.0:20200` | Kafka raw 消费，写 Redis/MySQL 实时表，并提供 API | +| Gateway | `lingniu-go-gateway.service` | `0.0.0.0:32960`、`0.0.0.0:808`、`127.0.0.1:20211` | GB32960、JT808、宇通 MQTT 接入、协议解析、即时鉴权响应和内存身份解析 | +| NATS fast writer | `lingniu-go-nats-fast-writer.service` | `127.0.0.1:20215` | NATS raw 消费，写 Redis 实时投影；生产默认关闭 TDengine stage | +| NATS Kafka bridge | `lingniu-go-nats-kafka-bridge.service` | `127.0.0.1:20214` | NATS canonical raw 到 Kafka raw/fields 的可靠桥接和字段投影 | +| History writer | `lingniu-go-history-writer.service` | `127.0.0.1:20212` | 多 worker 消费 Kafka raw，写 TDengine | +| Stat writer | `lingniu-go-stat-writer.service` | `127.0.0.1:20213` | 多 worker 消费 Kafka fields，写每日里程 | +| Realtime writer | `lingniu-go-realtime-writer.service` | `127.0.0.1:20216` | 多 worker 消费 Kafka raw，写 MySQL 实时表；生产关闭 Redis projector | +| Identity writer | `lingniu-go-identity-writer.service` | `127.0.0.1:20217` | 多 worker 消费 Kafka JT808 raw，幂等写注册、鉴权和低频在线触达事实 | +| Realtime API | `lingniu-go-realtime-api.service` | `0.0.0.0:20200` | 读取 Redis/MySQL/TDengine 并提供查询 API，不消费 Kafka | ## 总线 @@ -25,24 +28,41 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。 | 项 | 当前值 | | --- | --- | | Stream | `VEHICLE_INGEST` | -| Subjects | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` | +| Subjects | Gateway 只新增三类 `vehicle.raw.go.*`；三类 `vehicle.fields.go.*` 暂留在 Stream 合同中兼容切换前消息，不再由 Gateway 直接写入 | | Durable consumer | `vehicle-kafka-bridge` | -| 语义 | Gateway 先写 NATS raw subject；bridge 写 Kafka 成功后才 ACK NATS | +| 语义 | Gateway 每帧只发布一份 canonical raw；bridge 复用其中已计算的 `parsed_fields` 生成 fields，Kafka raw 与 fields 都成功后才 ACK 同一条 NATS raw | | 保留策略 | `NATS_STREAM_MAX_BYTES=21474836480`，`NATS_STREAM_MAX_AGE_HOURS=24`，`NATS_STREAM_ENSURE_TIMEOUT_SECONDS=60`，NATS 仅做入口缓冲 | +协议 parser 和字段扁平化都只在 Gateway 接入线程执行一次。TDengine、Redis、MySQL、bridge 和统计链路只能读取 envelope 中预计算的 `parsed_fields`；实时帧缺失该字段时 RAW 仍保留，但实时投影会以 `skipped_missing_fields` 跳过，禁止在存储层重新解析或误标在线。 + +Gateway 到 JetStream 的生产接受边界是本地分段 WAL：canonical RAW 先进入 1ms/256 条分组提交并完成 `fsync`，然后才异步发布 JetStream；只有收到 PubAck 才把记录标记完成，提交失败或 ACK 超时会释放给有界重放。进程崩溃后从 WAL 重建待发布记录，使用稳定 `Nats-Msg-Id=kind:subject:event_id` 让崩溃窗口内的重复重放由 JetStream 去重。生产基线为 16MiB/5s 分段、100000 append queue、10000 PubAck inflight；禁止和旧逐条 JSON spool 共用目录。Redis 快路径使用 `FAST_WRITER_WORKERS=16`、`FAST_WRITER_BATCH_SIZE=100`、`FAST_WRITER_FETCH_WAIT_MS=5`，仍在 Redis 成功后手工 ACK NATS。 + ### Kafka 当前 Go 链路正在使用的 topic： | Topic | 分区 | 写入方 | 主要消费方 | | --- | --- | --- | --- | -| `vehicle.raw.go.gb32960.v1` | 12 | NATS Kafka bridge | history、stat、realtime | -| `vehicle.raw.go.jt808.v1` | 12 | NATS Kafka bridge | history、stat、realtime | -| `vehicle.raw.go.yutong-mqtt.v1` | 12 | NATS Kafka bridge | history、stat、realtime | +| `vehicle.raw.go.gb32960.v1` | 12 | NATS Kafka bridge | history、realtime | +| `vehicle.raw.go.jt808.v1` | 12 | NATS Kafka bridge | history、realtime、identity | +| `vehicle.raw.go.yutong-mqtt.v1` | 12 | NATS Kafka bridge | history、realtime | +| `vehicle.fields.go.gb32960.v1` | 12 | NATS Kafka bridge 从 canonical raw 投影 | stat | +| `vehicle.fields.go.jt808.v1` | 12 | NATS Kafka bridge 从 canonical raw 投影 | stat | +| `vehicle.fields.go.yutong-mqtt.v1` | 12 | NATS Kafka bridge 从 canonical raw 投影 | stat | | `vehicle.event.go.unified.v1` | 12 | 显式打开 `PUBLISH_UNIFIED_ENABLED=true` 后才写入 | 兼容旧统一事件消费；当前最小链路不依赖 | 审计时 Kafka 上仍存在旧 Java/Xinda/telemetry topic，例如 `vehicle.raw.gb32960.v1`、`vehicle.raw.jt808.v1`、`vehicle.event.xinda.v1`、`vehicle.raw.telemetry-input.v1`。这些不属于当前 Go 最小链路，后续如果确认没有消费者依赖，应单独做 Kafka topic 下线清单，不在业务代码里继续引用。 +history、stat、realtime-writer、identity-writer 四个消费服务会以 `vehicle_kafka_consumer_info{service,group,topic}` 暴露当前运行时 Kafka 订阅配置；容量巡检将实时 writer 采集为 `realtime`，将身份 writer 采集为 `identity`，将查询服务采集为 `realtime-api`，以此区分“消费线程实际挂载 topic”和“API 可用性”。 + +history、stat、realtime-writer、identity-writer 的 Kafka 批消费采用 failure-closed 语义：存储失败后不抓取下一批；history/realtime/stat 只提交各分区连续成功前缀并保留失败后缀，identity 的 MySQL 批事务整体重试；commit 失败时四个 writer 都只重试 commit，不重复写存储。`vehicle_{history,realtime,stat}_retry_pending_messages` 和 `vehicle_identity_writer_retry_pending_messages` 表示当前卡在失败重试中的消息数，生产健康值必须为 `0`。 + +history、realtime、stat 和 identity writer 默认都在一个进程内启动 3 个同 group Kafka consumer，由 Kafka 分配 topic 分区。bridge 使用车辆键作为 Kafka key，因此同一车辆、同一协议的消息仍在单一分区内有序处理；不同分区分别并行写 TDengine、MySQL 实时投影、MySQL 统计和 JT808 身份事实。`HISTORY_WORKERS`、`REALTIME_WORKERS`、`STATS_WORKERS`、`IDENTITY_WRITER_WORKERS` 只能在 topic 分区数和后端连接容量允许的范围内调整，不能靠并发打乱单车状态、累计里程或注册鉴权顺序。 + +生产环境中 `nats-fast-writer` 只负责 Redis 快速当前态，`FAST_WRITER_TDENGINE_ENABLED=false`；TDengine RAW/位置历史由 `history-writer` 通过 Kafka raw topic 单路写入，避免 NATS fast path 与 Kafka history path 双写同一证据层。 + +生产环境中 Redis 当前态只允许 `nats-fast-writer` 写入。`realtime-writer` 的 `REALTIME_REDIS_PROJECTOR_ENABLED=false`，只保留 MySQL 当前态投影；`realtime-api` 只负责查询。指标 `vehicle_realtime_redis_projector_enabled` 必须为 `0`，否则 `capacity-check` 判为 degraded，避免 Kafka 慢链路覆盖 NATS 快链路的实时结果。 + ## TDengine 数据库：`lingniu_vehicle_ts` @@ -51,7 +71,7 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。 | --- | --- | --- | --- | | `raw_frames` | `ts`、`frame_id`、`event_id`、`message_id`、`event_time`、`received_at`、`raw_size_bytes`、`raw_hex`、`raw_text`、`parsed_json`、`parse_status`、`parse_error`、`source_endpoint` | `protocol`、`vehicle_key`、`vin`、`phone`、`device_id` | RAW 证据层，保存完整解析 JSON | | `raw_frame_payload_chunks` | 分片 payload 列 | 协议和车辆标识 tags | 保存超长 raw/parsed payload | -| `vehicle_locations` | `ts`、`event_id`、`frame_id`、`received_at`、`longitude`、`latitude`、`altitude_m`、`speed_kmh`、`direction_deg`、`alarm_flag`、`status_flag`、`total_mileage_km` | `protocol`、`vin` | 高频历史位置和总里程查询；只写 VIN 非空车辆 | +| `vehicle_locations` | `ts`、`event_id`、`frame_id`、`received_at`、`longitude`、`latitude`、`altitude_m`、`speed_kmh`、`direction_deg`、`alarm_flag`、`status_flag`、`total_mileage_km`、`soc_percent` | `protocol`、`vin` | 高频历史位置、SOC 和总里程查询；只写 VIN 非空车辆。writer 启动时幂等执行 `ADD COLUMN soc_percent`，兼容已有 stable | 不应重新出现： @@ -64,15 +84,31 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。 | 表 | 核心字段 | 写入方 | 职责 | | --- | --- | --- | --- | -| `vehicle_identity_binding` | `vin`、`plate`、`phone`、`oem` | 导入/人工维护，Gateway 只读 | 通过车牌、手机号反查 VIN | -| `jt808_registration` | `phone`、`device_id`、`plate`、`vin`、`manufacturer`、`auth_token`、`source_endpoint`、首次/最新注册鉴权时间 | Gateway | JT808 注册、鉴权和 VIN 匹配状态 | -| `vehicle_realtime_snapshot` | `protocol`、`vin`、`plate`、`event_time`、`received_at`、`event_id` | Realtime API/projector | 每协议每 VIN 最新事件轻量快照 | -| `vehicle_realtime_location` | `protocol`、`vin`、`plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 | -| `vehicle_daily_mileage` | `vin`、`stat_date`、`protocol`、`daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程；无 VIN 数据不进入正式统计表 | +| `vehicle` | `vin`、`plate`、`oem`、`enabled` | 导入/人工维护，Gateway 只读 | 车辆主实体，承载 VIN 到默认车牌/厂商的轻量索引 | +| `vehicle_identifier` | `protocol`、`source_code`、`identifier_type`、`identifier_value`、`vin`、`plate`、`oem` | 映射文件导入/人工维护，Gateway 只读 | 多平台、多标识反查 VIN；当前主要承载 JT808 手机号/车牌映射 | +| `vehicle_identity_binding` | `vin`、`plate`、`phone`、`oem` | 导入/人工维护，Gateway 兼容只读 | 旧版 VIN/车牌/手机号事实表；新映射导入会用它反查 VIN，运行期解析优先使用 `vehicle_identifier` | +| `jt808_registration` | `phone`、`device_id`、`plate`、`vin`、`manufacturer`、`auth_token`、`source_endpoint`、首次/最新注册鉴权时间 | Identity writer | JT808 注册、鉴权和 VIN 匹配状态；可从 Kafka JT808 raw 重放重建 | +| `vehicle_realtime_snapshot` | `protocol`、`vin`、`plate`、`platform_name`、`peer`、`parsed_json`、`event_time`、`received_at`、`event_id`；`access_first_seen_at/access_previous_received_at/access_latest_received_at/access_report_interval_ms/access_sample_count/access_latest_event_id/access_first_seen_source` | Realtime writer | 每协议每 VIN 最新合并扁平字段快照；同一条原子 upsert 还维护与设备事件时间解耦的接收证据。重复 event ID、相同或回退接收时间不推进连续间隔；旧行只做 `snapshot_backfill` 上线基线，不宣称历史首次接入 | +| `vehicle_realtime_location` | `protocol`、`vin`、`plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime writer | 每协议每 VIN 最新位置业务缓存；由 Kafka raw 写入 MySQL，不参与 Redis 当前态 | +| `vehicle_data_source` | `protocol`、`source_ip`、`source_code`、`platform_name`、`trust_priority`、`enabled` | Stat writer / 映射导入 / 人工维护 | 车辆数据来源管理；`source_code` 是机器可用的稳定来源编码，`platform_name` 是展示和人工维护名称 | +| `vehicle_daily_mileage_source` | `vin`、`stat_date`、`protocol`、`source_key`、`source_ip`、首末总里程、样本数、质量状态、是否选中 | Stat writer | 每协议、每来源的每日里程事实层；高频更新，保留多源候选 | +| `vehicle_daily_mileage` | `vin`、`stat_date`、`protocol`、`source_id`、`daily_mileage_km`、`latest_total_mileage_km` | Stat writer | 对外查询的每日里程结果层；由 `vehicle_daily_mileage_source` 按来源优先级/质量选举投影 | -身份表使用业务主键：`vehicle_identity_binding` 以 `vin` 为主键，`jt808_registration` 以 `phone` 为主键；两者不保留代理自增主键和 `created_at`。`vehicle_identity_binding` 是外部维护的事实表，服务运行时不回写。 +身份表使用业务主键：`vehicle` 以 `vin` 为主键，`vehicle_identifier` 以 `(protocol, source_code, identifier_type, identifier_value)` 为主键，`vehicle_identity_binding` 以 `vin` 为主键，`jt808_registration` 以 `phone` 为主键；这些核心身份表不保留代理自增主键和 `created_at`。`vehicle_identity_binding` 是外部维护的兼容事实表，服务运行时不回写。 -日里程表使用 `(vin, stat_date, protocol)` 作为业务主键，不保留自增 `id` 和 `created_at`；首末总里程和样本数保留用于解释差值计算。 +Gateway 默认每 60 秒把 `vehicle_identity_binding`、`vehicle_identifier`、`jt808_registration` 和 `vehicle_data_source` 原子刷新为本机只读身份快照。808 每帧只查内存，注册帧仍会立即更新当前 Gateway 的 phone 会话并返回鉴权码；持久化由 `identity-writer` 消费 Kafka JT808 raw 完成，Gateway 必须设置 `JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false`。注册、鉴权帧不节流，普通位置帧默认每个 phone 每 10 分钟触达一次；MySQL 写失败时不提交 Kafka offset，恢复后继续重放。快照刷新失败继续使用上一版，MySQL 不可用不会阻断 TCP 接入。 + +NATS→Kafka Bridge 只消费 canonical RAW，并从 RAW 中已有的 `parsed_fields` 派生 fields envelope，不重新解析协议。单批次按 Kafka topic 分组并发写入，默认 `BRIDGE_KAFKA_WRITE_CONCURRENCY=6`；同一个 RAW 对应的 RAW/fields topic 全部成功后才 ACK NATS，失败时保留源消息重放。因此并行化只缩短独立 topic 的网络确认等待，不改变 RAW 证据和统计字段的一致性边界。 + +日里程事实表 `vehicle_daily_mileage_source` 使用 `(vin, stat_date, protocol, source_key)` 作为业务主键，保留首末总里程和样本数用于解释差值计算；对外结果表 `vehicle_daily_mileage` 使用 `(vin, stat_date, protocol)` 作为业务主键，不保留自增 `id`、`created_at` 和候选来源明细。 + +Stat writer 每条有效里程样本都会更新 `vehicle_daily_mileage_source`，但 `vehicle_daily_mileage` 的选举投影默认按 `STATS_PROJECT_INTERVAL_SECONDS=15` 节流；新车辆/新协议/新日期/新来源会立即投影。这样保留每日里程事实的实时性，同时避免高频帧对 MySQL 结果表做多 SQL 放大。 + +每日里程只采用同来源累计里程边界差值：`当日最新累计总里程 - 最近历史日最后累计总里程`。优先取前一自然日；前一日没有数据时继续向更早日期查找，历史完全为空才以当天第一条为基线。不同协议、平台和终端来源分别计算候选，再由来源质量和优先级选举最终结果，不混用两个来源的累计总里程。 + +`vehicle_data_source` 以 `(protocol, source_ip)` 唯一识别来源。直连终端可能产生很多来源 IP，这些记录可以保留但默认不要求平台名；转发平台来源通过 `vehicle_identifier.source_code` 和 `jt808_registration.source_endpoint` 推断后补充 `source_code/platform_name`，后续统计优先使用 `source_id/trust_priority/enabled` 做可信源选择。 + +统计分日优先使用协议事件时间；如果事件时间明显晚于接收时间超过 10 分钟，则认为设备时间异常，使用接收时间归属统计日。RAW 历史仍保留原始事件时间，便于追溯。 两张实时当前态表均使用 `(protocol, vin)` 作为业务主键，不保留自增 `id` 和 `created_at`；最新更新时间使用 `updated_at`。 @@ -91,16 +127,22 @@ Redis 使用 DB 50，定位为实时缓存，不作为历史事实来源。 | `vehicle:latest:{vin}` | 跨协议合并后的最新核心字段快照，不重复保存完整 parsed；只写 VIN 非空车辆 | | `vehicle:latest:{vin}:{protocol}` | 单协议最新轻量快照，保留核心 fields 和时间信息，不重复保存完整 parsed | | `vehicle:realtime-raw:{protocol}:{vin}` | 单协议最新完整 parsed 状态，是实时完整协议字段的唯一 Redis 副本 | -| `vehicle:online:{vin}` | 在线状态和 TTL | +| `vehicle:rt-kv:{protocol}:{vin}:values` | 单协议扁平化实时字段值，字段名遵循协议字段映射 | +| `vehicle:rt-kv:{protocol}:{vin}:types` | `values` 中每个字段的值类型 | +| `vehicle:rt-kv:{protocol}:{vin}:times` | `values` 中每个字段最后写入的归一事件时间；快路径用它防止乱序旧帧覆盖新字段 | +| `vehicle:rt-kv:{protocol}:{vin}:meta` | 单协议 KV 投影的最新事件、接收时间和字段映射版本 | +| `vehicle:online:{protocol}:{vin}` | 在线状态和 TTL | +| `vehicle:online-state:{protocol}:{vin}` | 在线状态的 Hash 副本，便于分页查询 | | `vehicle:protocols:{vin}` | 当前车辆最近出现过的协议集合 | | `vehicle:last_seen` | 最近活跃车辆排序集合 | -审计时活跃 key 族主要是 `vehicle:latest:*`、`vehicle:realtime-raw:*`、`vehicle:online:*`、`vehicle:protocols:*`、`vehicle:last_seen`。旧文档中的 `vehicle:realtime:*`、`vehicle:merged:*` 不是当前活跃 key 族。 +审计时活跃 key 族主要是 `vehicle:latest:*`、`vehicle:realtime-raw:*`、`vehicle:rt-kv:*`、`vehicle:online:*`、`vehicle:online-state:*`、`vehicle:protocols:*`、`vehicle:last_seen`。旧文档中的 `vehicle:realtime:*`、`vehicle:merged:*` 不是当前活跃 key 族。 ## 后续优化约束 -1. 接入层先写 NATS，Kafka 是持久回放层，下游消费者不能反向依赖 Gateway 内存状态。 +1. 接入层每帧只解析和扁平化一次并写一份 NATS canonical raw，Kafka 是持久回放层；fields 必须由可重放消费者从 raw 中已有的 `parsed_fields` 投影，不能由 Gateway 双写或由存储层重算。 2. TDengine 只放高写入时序数据：RAW 证据和位置历史。 3. MySQL 只放低基数业务状态：身份、实时轻量快照、每日里程。 4. Redis 只放当前态，所有 key 都必须允许 TTL 过期后从 Kafka/TDengine/MySQL 重建。 -5. 协议新增字段默认进入 `raw_frames.parsed_json`；VIN 非空车辆同步进入 Redis realtime-raw。只有稳定查询需求出现后才提升为 TDengine/MySQL 列。 +5. 协议新增字段默认进入 `raw_frames.parsed_json` 物理列中的扁平 `parsed_fields` JSON，并同步进入 Redis KV；只有稳定查询需求出现后才提升为 TDengine/MySQL 独立列。 +6. Gateway 不直接写业务数据库；JT808 即时会话留在内存，注册鉴权事实由 Kafka identity-writer 单路投影。 diff --git a/docs/architecture/storage-minimal-contract.md b/docs/architecture/storage-minimal-contract.md index 13de8817..fdb1939f 100644 --- a/docs/architecture/storage-minimal-contract.md +++ b/docs/architecture/storage-minimal-contract.md @@ -12,21 +12,22 @@ 4. TDengine 只承担高写入时间序列：raw、位置历史。 5. MySQL 只承担低基数业务状态：身份映射、实时快照、日指标。 6. 同一个事实只落一个主表，其他表只存查询必要的投影。 -7. 新字段先进入 `parsed_json`，只有稳定查询需求出现后才提升为列。 +7. 新字段先由 Gateway 一次扁平化后进入 `parsed_fields`；TDengine 物理列仍名为 `parsed_json`，只有稳定查询需求出现后才提升为列。 ## 目标表边界 | 存储 | 表/Key | 保留内容 | 不保留内容 | | --- | --- | --- | --- | -| TDengine | `raw_frames` | raw hex/text、完整 `parsed_json`、解析状态、协议标签、车辆标识标签 | `fields_json` 这类可从 `parsed_json`/业务表再得到的重复字段 | +| TDengine | `raw_frames` | raw hex/text、扁平 `parsed_fields` JSON（物理列 `parsed_json`）、解析状态、协议标签、车辆标识标签 | 原始嵌套解析树、`fields_json` 等重复字段 | | TDengine | `raw_frame_payload_chunks` | 超长 raw/parsed payload 分片 | 业务查询字段 | | TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息、phone/device_id/vehicle_key 兜底标识 | -| MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON | +| MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件态、合并扁平字段，以及独立的首次/前次/最新接收时间、连续间隔、样本数和证据来源 | phone、device、原始嵌套解析树、raw 报文、无限接收历史 | | MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 | | MySQL | `vehicle_daily_mileage` | 日期、VIN、协议、日里程、首末总里程、样本数 | 临时 vehicle key、泛化 metric key/value、自增 id、created_at、每帧细节、位置点列表 | | MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、oem 映射 | 注册历史、协议状态、device_id、自增 id、created_at | | MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态、来源端点 | GB32960/MQTT 注册信息、位置历史、created_at | | Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议每 VIN 最新完整 parsed 状态 | 无 VIN 临时身份、历史数据、统计结果 | +| Redis | `vehicle:rt-kv:{protocol}:{vin}:values/types/times/meta` | 每协议每 VIN 最新扁平实时字段、类型、字段级事件时间和投影元信息 | 历史字段值、统计结果、无 VIN 临时身份 | ## 当前应收敛的重复点 @@ -60,6 +61,7 @@ 6. MySQL realtime 当前态表不保留代理主键和创建时间。 - 原因：`vehicle_realtime_snapshot`、`vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影，业务主键就是 `(protocol, vin)`。 + - 接入证据按接收时间单调推进，不受设备事件时间乱序影响；重复事件 ID 或回退接收时间不能增加样本数。`access_first_seen_source=snapshot_backfill` 仅表示部署时基线，只有 `live_writer` 新行能证明 writer 上线后的首次观测。 - 状态：schema 使用 `PRIMARY KEY (protocol, vin)`，不保留自增 `id` 和 `created_at`，对外只暴露最新 `updated_at`。 - 索引：不在 `vehicle_realtime_location` 维护经纬度组合索引；实时表只回答当前态，地理范围和轨迹类查询走 TDengine 历史位置。 - 查询：`/api/realtime/snapshots`、`/api/realtime/locations` 默认不执行 `COUNT(*)`，`total` 表示本页返回条数；只有需要精确总数时传 `includeTotal=true`。 @@ -83,9 +85,9 @@ ## 字段提升规则 -协议字段进入系统后分三层： +协议字段只在 Gateway 解析、映射和扁平化一次，进入系统后分三层。`vehicle.fields.*` 事件中的所有字段名必须位于对应协议命名空间：`gb32960.*`、`jt808.*` 或 `yutong_mqtt.*`；统计层不读取内部标准化核心字段作为兼容兜底。 -1. `parsed_json`：默认入口，保存完整结构化解析。 +1. `parsed_fields`：默认入口，保存全量扁平协议字段；TDengine 继续使用兼容物理列名 `parsed_json`。 2. 核心列：只有跨协议稳定查询需要时才提升，例如经纬度、速度、SOC、总里程。 3. 指标表：只有聚合口径稳定、产品需要分页/排序/报表时才持久化。 diff --git a/docs/architecture/tdengine-batch-writer-design.md b/docs/architecture/tdengine-batch-writer-design.md index 1e199a45..41abf31f 100644 --- a/docs/architecture/tdengine-batch-writer-design.md +++ b/docs/architecture/tdengine-batch-writer-design.md @@ -69,7 +69,7 @@ - `history.Writer.AppendAllBatch` 按 raw child table 和 location child table 生成多行 `INSERT ... VALUES (...),(...)`。 - oversized payload chunks 仍写入 `raw_frame_payload_chunks`，同样按 child table 批量写。 -- `history-writer` Kafka 消费默认按 `HISTORY_BATCH_SIZE=200`、`HISTORY_BATCH_WAIT_MS=100` 收集消息。 +- `history-writer` 默认启动 `HISTORY_WORKERS=3` 个同组消费者，每个 worker 按 `HISTORY_BATCH_SIZE=200`、`HISTORY_BATCH_WAIT_MS=20` 收集其已分配分区的消息。 - TDengine batch 成功后才批量提交 Kafka messages；batch 失败不提交 offset，让 Kafka 保留可重放语义。 - 保留 `processHistoryMessage` 和 `AppendAll` 单条路径，便于回退和测试。 diff --git a/docs/ops/100k-capacity-baseline.md b/docs/ops/100k-capacity-baseline.md index ed7af7bb..c332d2f5 100644 --- a/docs/ops/100k-capacity-baseline.md +++ b/docs/ops/100k-capacity-baseline.md @@ -1,6 +1,6 @@ # 100K 车辆接入容量基线 -更新时间：2026-07-03 +更新时间：2026-07-13 ## 目标 @@ -40,6 +40,7 @@ - 已新增可重复的 TCP 连接压测工具，后续需要用它跑 10K/50K/100K 阶段测试并记录结果。 - 仍缺读超时按协议维度计数；Gateway frame duration 已提供 histogram，可用于 parse + enqueue + response 的 p95/p99 估算。 - TDengine history writer 和 NATS fast writer 已使用 batch 写入；后续需要用带帧率压测验证 batch size、flush latency 和存储端承载能力。 +- History、realtime、stat、identity writer 均支持单进程多个同 group consumer，默认分别为 `HISTORY_WORKERS=3`、`REALTIME_WORKERS=3`、`STATS_WORKERS=3`、`IDENTITY_WRITER_WORKERS=3`。四者按 Kafka 分区并行且保持单车分区顺序；后续压测需联合观察 TDengine/MySQL 写延迟、连接数和各 consumer lag，再决定是否增加 worker。 - Kafka topic 当前 12 分区，100K 目标下需要结合实际 FPS 再评估分区数。 ## ECS OS 参数建议 @@ -73,6 +74,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | History writer | `vehicle_history_batch_pending_messages` | 稳态接近 0，burst 后下降 | | History writer | `vehicle_history_batch_pending_rows` | 稳态接近 0，burst 后下降 | | History writer | `vehicle_history_batch_flush_duration_ms{status}` | flush 延迟不持续上升 | +| History writer | `vehicle_history_config{setting="workers"}` | 不低于 `3` | +| History writer | `vehicle_history_worker_active` | 每个配置 worker 均为 `1` | | NATS bridge | `vehicle_bridge_nats_consumer_ack_pending` | 稳态为 0 | | NATS bridge | `vehicle_bridge_nats_consumer_pending` | burst 后下降 | | NATS bridge | `vehicle_bridge_batch_pending_messages` | 稳态接近 0，burst 后下降 | @@ -86,7 +89,12 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | Realtime Redis | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 毫秒级且不持续上升 | | Realtime MySQL | `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 不持续上升 | | Realtime MySQL | async queue depth | 不持续增长 | +| Realtime MySQL | `vehicle_realtime_config{setting="workers"}` | 不低于 `3` | +| Realtime MySQL | `vehicle_realtime_worker_active` | 每个配置 worker 均为 `1` | | Stat writer | `vehicle_stat_write_duration_ms_histogram_bucket` | p99 不持续上升 | +| Stat writer | `vehicle_stat_config{setting="workers"}` | 不低于 `3` | +| Stat writer | `vehicle_stat_worker_active` | 每个配置 worker 均为 `1` | +| Identity writer | `vehicle_identity_writer_worker_active` | 每个配置 worker 均为 `1` | ## Retention Guardrails @@ -96,7 +104,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 | --- | --- | | NATS JetStream `VEHICLE_INGEST` | `NATS_STREAM_MAX_BYTES=21474836480`，`NATS_STREAM_MAX_AGE_HOURS=24`，`NATS_STREAM_ENSURE_TIMEOUT_SECONDS=60` | | Kafka `vehicle.raw.go.*` / `vehicle.fields.go.*` | `retention.ms=21600000`、`segment.ms=600000`、`segment.bytes=268435456` | -| Fast writer | `FAST_WRITER_OPERATION_TIMEOUT_MS=1000`，避免 TDengine 批量写尾延迟导致整批重投 | +| Gateway NATS publisher | `NATS_ASYNC_RAW_WORKERS=128`，吸收平台按秒集中上报形成的瞬时突发；以 `vehicle_async_sink_queue_wait_recent_p99_ms` 校验，不按稳态 queue depth 猜测 | +| Fast writer | `FAST_WRITER_WORKERS=16`、`FAST_WRITER_BATCH_SIZE=100`、`FAST_WRITER_FETCH_WAIT_MS=5`、`FAST_WRITER_OPERATION_TIMEOUT_MS=1000`；继续使用手工 ACK，避免以可靠性换延迟 | | Root disk | 使用率低于 `80%`，超过 `85%` 进入容量告警 | 如果 NATS `consumer_pending` 下降但仍高，说明系统在追历史积压；如果 `ack_pending=0` 且 Kafka lag 为 `0`，不要重启服务，重点观察 NATS data size 和 pending 下降斜率。 @@ -122,6 +131,8 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 - JT808 会变更包头手机号、流水号和 `0x0200` 位置时间，并重新计算转义和校验码。 - GB32960 会变更 VIN 和实时数据时间，并重新计算 BCC。 +- JT808 发送模式默认持续读取服务端通用应答；禁止关闭 `-drain-responses` 后用应答写阻塞产生的延迟评价服务端性能。 +- JT808 使用 `-cleanup-registration` 时会在结束后只清理指定模拟手机号区间且来源为 loopback 的注册记录；RAW 仍保留作为压测证据。每轮发送压测必须使用未在 identity-writer 10 分钟 touch 节流窗口内使用过的新号段，否则注册不会重复写入，`rows_deleted` 可以为 `0`。 - 低 FPS 全链路压测必须使用该模式，避免重复静态帧导致 event id 冲突。 - 对生产端口执行 `-send=true` 会写入合成 raw 数据，只能在明确隔离标识和压测窗口后执行。 @@ -175,8 +186,42 @@ systemd gateway 已配置 `LimitNOFILE=1048576`，需要持续保持。 - `2026-07-03 18:15:36 CST` 5W 压测后 JT808 active 回落到约 `238`，`ss` 约 `239`。 - `2026-07-03 18:16:49 CST` NATS ack pending 回到 `0`，history/stat/realtime Kafka lag 总和均为 `0`。 - `2026-07-03 18:23:02 CST` 10W 总连接压测后 JT808 active 回落到 `236`，GB32960 active 回落到 `2`，NATS ack pending 为 `0`，history/stat/realtime Kafka lag 总和均为 `0`。 -- `20211/20212/20213/20214/20200` readyz 均 OK。 +- `20211/20212/20213/20214/20215/20216/20200` readyz 均 OK。 - 压测窗口未出现 gateway `error|failed|panic|fatal|rejected` 日志。 - `vehicle_gateway_connection_rejections_total` 未输出，表示本轮没有连接拒绝计数。 结论：单 ECS 已通过本机 10W 总连接 hold-only 基线测试。该结果证明当前内核参数、gateway 连接上限和 systemd 文件句柄设置可以承载 10W 空闲长连接；下一阶段必须进入带真实帧率的低 FPS 压测，验证解析、NATS、Kafka、Redis、TDengine、MySQL 全链路吞吐。 + +## 2026-07-13 1000 FPS 全链路压测记录 + +压测参数：JT808 `0x0200`，1000 个连接，每连接 1 FPS，持续 60 秒，手机号区间 `139000000000-139000000999`；开启服务端应答读取和注册自动清理。每轮均打开 1000 个连接、失败 0，发送 59,500 帧、写错误 0，读取应答约 1.19 MB、读错误 0，结束后清理 1000 条模拟注册。 + +未读取 JT808 应答的早期结果不作为延迟基线：客户端接收缓冲区会填满并反向阻塞 Gateway 写应答，测到的是压测器缺陷而非服务端真实链路。 + +| 版本 | 观测点 | Gateway 应答 p99 | Redis p99 | Bridge RAW p99 | TDengine history p99 | MySQL stat p99 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| topic 串行写 Kafka | 压测中段 | 约 32ms | 约 212ms | 约 304ms | 约 466ms | 约 332ms | +| topic 串行写 Kafka | 压测结束 | 约 10ms | 约 162ms | 约 326ms | 约 427ms | 约 313ms | +| topic 并行写 Kafka，concurrency=6 | 压测中段 | 约 24ms | 约 207ms | 约 311ms | 约 409ms | 约 247ms | +| topic 并行写 Kafka，concurrency=6 | 压测结束 | 约 29ms | 约 184ms | 约 288ms | 约 368ms | 约 316ms | + +并行版本在同一 NATS 批次内按 Kafka topic 分组并发写，RAW 与派生 fields 全部成功后才 ACK 原 NATS 消息；任一 topic 失败仍保留对应源消息待重放。低负载 Bridge p99 从约 110ms 降到约 65ms；1000 FPS 下 history p99 下降约 14%，Bridge RAW p99 下降约 12%。压测中段 NATS pending 短时峰值 71，Gateway async queue 短时 285，结束时均归零；Kafka lag 和各 writer retry 均为 0。主机为 4 核，瞬时 load average 约 9.68，但采样时 CPU 仍约 52% idle、无 D 状态进程、可用内存约 6.4 GiB。 + +增加队列等待指标后，使用五个全新 JT808 号段进行了 worker A/B；每轮仍是 59,500 帧、发送/读取错误为 0、清理注册 1000 条。以下均为压测中段最近 512 样本窗口，秒级集中发送会比均匀流量更严格： + +| Gateway RAW worker | Fast writer worker / fetch | Gateway queue wait p99 | Gateway 应答 p99 | Redis p99 | 结论 | +| ---: | --- | ---: | ---: | ---: | --- | +| 32 | `8 / 20ms` | 约 95ms | 约 92ms | 约 234ms | 入口突发排队明显 | +| 64 | `8 / 20ms` | 约 84ms | 约 79ms | 约 254ms | Gateway 有小幅收益，下游未改善 | +| 128 | `16 / 5ms` | 约 42ms | 约 17ms | 约 151ms | 当前最佳组合 | +| 128 | `32 / 5ms` | 约 50ms | 约 32ms | 约 221ms | pull worker 过多产生调度竞争，回退 | + +最终保留 `NATS_ASYNC_RAW_WORKERS=128`、`FAST_WRITER_WORKERS=16`、`FAST_WRITER_FETCH_WAIT_MS=5`。Redis 批写自身 p99 低于 `5ms`，约 151ms 的剩余尾延迟主要位于 JetStream 持久化发布和 consumer delivery；NATS/Kafka pending、Kafka lag 与 writer retry 在每轮结束后均归零。 + +## 2026-07-14 WAL Outbox 验证 + +Gateway 已切换为分段 WAL + 异步 JetStream PubAck。WAL 使用 16MiB/5s 分段、1ms/256 条 group commit、`fsync=true`、10000 最大 inflight；本地 Apple M4 并发基准约 1926 records/s（`519144 ns/op`），没有以关闭 `fsync` 换取吞吐。 + +ECS 生产使用 JT808 `0x0200`、1000 个连接、每连接 1 FPS、持续 60 秒验证：连接成功 1000、失败 0，发送 59500 帧、写错误 0、读错误 0，WAL `submitted=acked=66776`（含同期真实三协议流量），结束后 WAL backlog/inflight、NATS pending、Kafka lag 全部为 0。压测结束 Gateway JT808 响应 recent p99 约 80ms。 + +随后在约 1000 FPS 下对 Gateway 执行真实 `SIGKILL`。systemd 约 5 秒后自动拉起；启动后立即从崩溃前 WAL 重放并确认约 563 条记录，最终 WAL backlog/inflight、NATS pending 和 history/stat/realtime/identity Kafka lag 均回到 0，日志没有 CRC 损坏、publish error 或 panic。故障窗口内压测客户端的写/读错误是 TCP 连接被强制中断的预期结果，不代表已持久接受的服务端数据丢失。 diff --git a/docs/ops/go-service-observability.md b/docs/ops/go-service-observability.md index 7badc290..a1e3f06f 100644 --- a/docs/ops/go-service-observability.md +++ b/docs/ops/go-service-observability.md @@ -14,6 +14,7 @@ The Go services expose local-only health and metrics endpoints. They are intende | History writer | `lingniu-go-history-writer.service` | `127.0.0.1:20212` | `/healthz` | `/readyz` | `/metrics` | | Stat writer | `lingniu-go-stat-writer.service` | `127.0.0.1:20213` | `/healthz` | `/readyz` | `/metrics` | | NATS Kafka bridge | `lingniu-go-nats-kafka-bridge.service` | `127.0.0.1:20214` | `/healthz` | `/readyz` | `/metrics` | +| Identity writer | `lingniu-go-identity-writer.service` | `127.0.0.1:20217` | `/healthz` | `/readyz` | `/metrics` | | Realtime API | `lingniu-go-realtime-api.service` | `0.0.0.0:20200` | `/healthz` | `/readyz` | `/metrics` | ## Quick Checks @@ -26,6 +27,7 @@ curl -fsS http://127.0.0.1:20211/metrics curl -fsS http://127.0.0.1:20212/readyz curl -fsS http://127.0.0.1:20213/readyz curl -fsS http://127.0.0.1:20214/readyz +curl -fsS http://127.0.0.1:20217/readyz curl -fsS http://127.0.0.1:20200/readyz ``` @@ -35,7 +37,19 @@ curl -fsS http://127.0.0.1:20200/readyz /opt/lingniu-go-native/current/capacity-check ``` -它会抓取 Gateway、History writer、Stat writer、NATS bridge、Realtime API、NATS fast writer 的本地 `/metrics`，输出 JSON。退出码 `0` 表示当前关键 backlog 和拒绝计数正常，退出码 `2` 表示存在 pending、Kafka lag、连接拒绝或 metrics 抓取失败，适合接入 cron/告警。 +它会抓取 Gateway、History writer、Stat writer、Identity writer、NATS bridge、Realtime writer/API、NATS fast writer 的本地 `/metrics`，输出 JSON。退出码 `0` 表示当前关键 backlog 和拒绝计数正常，退出码 `2` 表示存在超阈值 pending、Kafka lag、连接拒绝或 metrics 抓取失败，适合接入 cron/告警。 + +Identity writer 必须消费 `vehicle.raw.go.jt808.v1`，且 Gateway 指标 `vehicle_gateway_jt808_registration_gateway_writes_enabled` 必须为 `0`。两者同时写 `jt808_registration` 会被 `capacity-check` 判为重复 writer。 + +Gateway 的 NATS 接受边界由 WAL outbox 提供。`vehicle_durable_outbox_backlog_records` 表示已经 `fsync` 但尚未 PubAck 的记录，`vehicle_durable_outbox_inflight` 表示正在等待 PubAck 的记录；健康时两者最终回到 `0`。`vehicle_durable_outbox_publish_total` 的 `submit_error`、`ack_error`、`remove_error`、`close_timeout` 默认按 outbox 错误门禁处理。稳定 event ID/NATS MsgId 负责崩溃恢复期间的 JetStream 去重。 + +默认还会用 histogram 估算链路 p99：Gateway frame `250ms`、Gateway identity `100ms`、async sink `100ms`、bridge batch `5000ms`、fast-writer stage `100ms`、history flush `1000ms`、realtime store `100ms`、stat write `100ms`；history/stat/realtime/identity 的 Kafka lag 总和超过 `100` 时降级。Gateway async sink 队列深度默认超过 `10000` 降级，可用 `-async-sink-queue-depth-max` 调整；Gateway 身份快照必须 ready 且最近成功刷新时间不能超过 `180s`，分别用 `-gateway-identity-snapshot-required`、`-gateway-identity-snapshot-max-age-seconds` 调整。Gateway 身份解析缓存默认容量阈值是 `300000`，Stat writer 统计缓存默认容量阈值是 `1000000`，Realtime API 的 MySQL 写入节流缓存和 VIN 车牌缓存默认容量阈值分别是 `1000000` 和 `200000`，可分别用 `-gateway-identity-cache-entries-max`、`-stat-cache-entries-max`、`-realtime-throttle-cache-entries-max`、`-realtime-plate-cache-entries-max` 调整，设为 `0` 表示关闭该项容量检查。history、realtime、stat、identity writer 的 worker 数默认都不得低于 `3`，分别由 `-history-workers-min`、`-realtime-workers-min`、`-stat-workers-min`、`-identity-workers-min` 检查。低样本数会跳过 p99 判断，阈值设为 `0` 可临时关闭对应延迟检查。 + +批处理内部 in-flight pending 默认允许 `100` 条窗口：`-fast-writer-batch-pending-max=100`、`-history-batch-pending-max=100`、`-history-rows-pending-max=100`、`-stat-batch-pending-max=100`。这类指标用于发现批写卡住，不再因为高频流量下几十条正在 flush/ack 的瞬时批次直接降级；NATS consumer pending、ack pending 和 Kafka lag 仍按独立阈值判断真实积压。 + +topic 流转检查默认只抓完全断层：history/realtime 必须消费三类 raw topic，stat 必须消费三类 fields topic；bridge 未配置 subject 的 `route_error` 默认超过 `0` 条就降级，可用 `-bridge-route-error-max=-1` 临时关闭；fast-writer `invalid_json` 默认超过 `0` 条就降级，可用 `-fast-writer-invalid-json-max=-1` 临时关闭；history-writer `invalid_json` 默认超过 `0` 条就降级，可用 `-history-invalid-json-max=-1` 临时关闭；realtime projector `invalid_json` 默认超过 `0` 条就降级，可用 `-realtime-invalid-json-max=-1` 临时关闭；stat-writer `invalid_json` 默认超过 `0` 条就降级，可用 `-stat-invalid-json-max=-1` 临时关闭；某协议 gateway raw publish 达到 `100` 条，扣除 `skipped_non_realtime` 后仍然没有 fields publish 时降级；实时 fields 缺失或 publish error 比例超过 `20%` 时也会降级；某协议 gateway raw/fields publish 达到 `1000` 条但 bridge 没有写对应 Kafka topic 时降级；某协议 gateway raw publish 达到 `1000` 条但 fast-writer 没有消费同 raw subject 并成功写 Redis/ack 时降级；某 raw topic bridge 写 Kafka 达到 `1000` 条但 history 未收到/写入或 realtime 未收到/更新时降级；某 fields topic bridge 写 Kafka 达到 `100` 条但 stat-writer 未配置或未收到该 topic 时降级。解析质量检查默认在单协议 gateway frame 样本达到 `100` 条后启用，`OK` 以外的 `PARTIAL/BAD_FRAME/other` 比例超过 `5%` 时降级，用于发现协议解析器或上游报文质量突然恶化。响应质量检查默认在单协议 gateway response 尝试样本达到 `100` 条后启用，`build_error/write_error` 比例超过 `5%` 时降级；`skipped` 表示该消息类型无需响应，不计入分母。身份解析质量检查默认在单协议 identity 样本达到 `100` 条后启用，`resolved` 以外的 `unresolved/error/timeout` 比例超过 `20%` 时降级，用于发现 VIN/车牌/phone 映射突然失效。history-writer 会对实时数据帧检查 raw envelope 是否携带 `parsed_fields`，默认单 topic/protocol 达到 `100` 帧后，缺失比例超过 `5%` 降级，用于发现历史证据层丢失扁平化解析字段。Redis 快路径字段质量检查默认在单 raw subject 写入字段达到 `1000` 个后启用，`skipped_stale/seen` 超过 `20%` 时降级，用于发现大面积旧帧补发或设备时间漂移。stat-writer 还会在单个 fields topic 成功 append 达到 `100` 条但未抽取到任何里程样本时降级，或 VIN 缺失、里程缺失、非正里程、source 缺失这类可行动跳过比例超过 `60%` 时降级，用于发现字段映射或身份/source 关联断裂；单 topic 找到 `100` 条里程样本但没有 source tracking，或 source endpoint 缺失比例超过 `60%` 时也会降级，用于发现 `vehicle_data_source` 来源管理断层；单 topic 已写入 `100` 条里程样本但最终 `vehicle_daily_mileage` 投影写入仍为 `0` 时也会降级，用于发现 source 候选层到最终查询层的断层；`skipped_same_mileage` 是重复总里程去重，不计入可行动跳过比例。已出现过的成功/received `*_last_*_unix_seconds` 如果超过 `300s` 未刷新，也会降级；刚启动尚未出现过该指标时不会因缺失 last activity 降级。`capacity-check` JSON 会额外派生 `*_last_*_age_seconds`，便于直接读取距今秒数。特殊环境可用 `-required-consumer-topics=''` 临时关闭消费 topic 合同检查。 + +`capacity-check` 还会默认请求 Realtime API 的 `/api/stats/daily-metrics/diagnostics/reasons`，把当天每日里程诊断聚合放到 JSON 的 `daily_mileage_diagnostics`。这项默认只展示 `vehicle_total`、`actionable_issue_total` 和原因分布，不会因为存在业务缺口直接退出 `2`；需要把缺口纳入发布/巡检阻断时，显式传 `-daily-mileage-diagnostics-max-actionable=0` 或其他阈值。诊断 API 不可访问会被视为观测能力故障并降级；特殊环境可用 `-daily-mileage-diagnostics-url=''` 关闭。 ECS 上通过 systemd timer 每分钟执行一次： @@ -56,32 +70,106 @@ systemctl start lingniu-go-capacity-check.service | `vehicle_gateway_connection_closes_total` | TCP connection closes by protocol and reason. Labels: `protocol`, `reason`. Reasons include `eof`, `read_timeout`, `read_error`, `extract_error`, `context_cancelled`, `max_connections`. | | `vehicle_gateway_connection_rejections_total` | TCP connection rejections by protocol and reason. Labels: `protocol`, `reason`. | | `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. | +| `vehicle_gateway_last_frame_unix_seconds` | Last observed gateway frame time by protocol and parse status. Labels: `protocol`, `status`. | | `vehicle_gateway_frame_duration_ms` | Last observed Gateway frame handling duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. | -| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. | +| `vehicle_gateway_identity_total` | Gateway identity resolve results. Labels: `protocol`, `status`; status includes `resolved`, `unresolved`, `error`, `timeout`. | +| `vehicle_gateway_identity_duration_ms` | Last observed identity resolve duration and histogram buckets for p95/p99. Labels: `protocol`, `status`. | +| `vehicle_gateway_identity_cache_entries` | Gateway identity resolver cache entry count. Labels: `cache`; values include runtime `lookup`/`registration`/`source_code`/`location_touch` and `snapshot_binding`/`snapshot_identifier`/`snapshot_registration`/`snapshot_source`. | +| `vehicle_gateway_identity_cache_max_entries` | Gateway identity resolver cache entry cap per cache. Controlled by `IDENTITY_LOOKUP_CACHE_MAX_ENTRIES`, default `300000`. | +| `vehicle_gateway_identity_snapshot_ready` | `1` after at least one complete atomic identity snapshot refresh; `0` means frames continue ingesting but unknown JT808 identities cannot be resolved. | +| `vehicle_gateway_identity_snapshot_entries` | Current identity snapshot entry counts. Labels: `kind` = `binding`, `identifier`, `registration`, `source`. | +| `vehicle_gateway_identity_snapshot_refresh_total` | Snapshot refresh attempts. Labels: `status` = `ok` or `error`; an error keeps the last known good snapshot. | +| `vehicle_gateway_identity_snapshot_last_success_unix_seconds` | Last successful complete snapshot publication time. | +| `vehicle_gateway_publish_total` | Gateway publish/delegation result. In NATS mode raw is published and fields uses `status="delegated"`; unified appears only when explicitly enabled. Labels: `protocol`, `kind`, `status`. | +| `vehicle_gateway_last_publish_unix_seconds` | Last gateway publish time by protocol/kind/status, useful for protocol freshness checks. Labels: `protocol`, `kind`, `status`. | +| `vehicle_gateway_fields_total` | Gateway fields eligibility result. Labels: `protocol`, `status`; NATS mode uses `delegated_to_bridge`, while compatibility direct mode may use `published`/`publish_error`. | +| `vehicle_gateway_response_total` | Gateway protocol response build/write result. Labels: `protocol`, `message_id`, `status`; status includes `ok`, `skipped`, `build_error`, `write_error`. | +| `vehicle_gateway_last_response_unix_seconds` | Last gateway protocol response build/write result time. Labels: `protocol`, `message_id`, `status`. | +| `vehicle_gateway_response_e2e_recent_p99_ms` | Bounded recent p99 from frame receipt through successful protocol response. Labels: `protocol`; capacity target defaults to `100ms`. | +| `vehicle_gateway_response_e2e_recent_samples` | Samples in the bounded response-latency window. Labels: `protocol`. | +| `vehicle_gateway_authentication_total` | Protocol authentication decisions. Labels: `protocol`, `mode`, `source`, `status`; `source=configured` means a configured credential matched, `source=device` means the JT808 phone's snapshot token matched, and `source=none` means no credential matched. `observe` records mismatches without rejecting, while `enforce` returns a protocol failure and closes the connection. | +| `vehicle_gateway_authentication_mode` | Active authentication mode by protocol. Labels: `protocol`, `mode`. | +| `vehicle_gateway_authentication_credentials` | Configured account count, never credential values. Labels: `protocol`. | | `vehicle_async_sink_publish_duration_ms_histogram` | Async sink publish duration histogram covering worker publish calls to the delegate sink. Labels: `sink`, `kind`, `status`. | +| `vehicle_async_sink_queue_capacity` | Gateway async sink queue capacity. Labels: `sink`. NATS uses `NATS_ASYNC_QUEUE_SIZE`; Kafka fallback uses `KAFKA_ASYNC_QUEUE_SIZE`. | | `vehicle_bridge_messages_total` | NATS messages fetched by the bridge. Labels: `subject`, `status`. | +| `vehicle_bridge_fields_projection_total` | Fields projection result from canonical raw. Labels: `protocol`, `status`; `published` is healthy, error/skip labels explain why no fields event was emitted. | +| `vehicle_bridge_fields_projection_count` | Latest flattened field count emitted by bridge per protocol/status. | +| `vehicle_bridge_last_message_unix_seconds` | Last NATS message fetched by subject/status. Labels: `subject`, `status`. | | `vehicle_bridge_kafka_writes_total` | Bridge writes to Kafka. Labels: `topic`, `status`. | -| `vehicle_bridge_nats_acks_total` | NATS ack results after Kafka write. Labels: `subject`, `status`. | +| `vehicle_bridge_last_kafka_write_unix_seconds` | Last bridge Kafka write by topic/status. Labels: `topic`, `status`. | +| `vehicle_bridge_nats_acks_total` | NATS ack results after Kafka write or unrouted-subject isolation. Labels: `subject`, `status`; status includes `ok`, `error`, and `dropped_route_error`. | +| `vehicle_bridge_last_ack_unix_seconds` | Last NATS ack by subject/status. Labels: `subject`, `status`. | | `vehicle_bridge_nats_consumer_pending` | JetStream messages pending for the bridge durable consumer. | | `vehicle_bridge_nats_consumer_ack_pending` | JetStream messages delivered to bridge but not yet acked. | | `vehicle_bridge_nats_consumer_waiting` | Pull requests waiting on the bridge durable consumer. | | `vehicle_bridge_batch_pending_messages` | NATS messages fetched by the bridge but not yet written to Kafka and acked. | | `vehicle_bridge_batch_pending_kafka_messages` | Kafka messages prepared for the current bridge batch. | | `vehicle_bridge_batch_duration_ms_histogram` | Bridge batch duration histogram covering Kafka write and NATS ack. Labels: `status`. | +| `vehicle_history_batch_flush_duration_ms_histogram` | History writer TDengine batch flush duration histogram. Labels: `status`. | +| `vehicle_realtime_store_update_duration_ms_histogram` | Realtime MySQL/Redis projection duration histogram. Labels: `store`, `protocol`, `status`. | +| `vehicle_stat_write_duration_ms_histogram` | Stat writer MySQL write duration histogram. Labels: `topic`, `status`. | +| `vehicle_stat_samples_total` | Stat writer mileage sample results. Labels: `topic`, `protocol`, `status`; status includes `found`, `written`, `skipped_missing_fields`, `skipped_missing_vin`, `skipped_missing_mileage`, `skipped_non_positive_mileage`, `skipped_same_mileage`, `skipped_missing_source`, `event_time_future_adjusted`. `skipped_missing_fields` means the fields topic carried an envelope without flattened fields and should be treated as a stream-contract issue. The last status means device event time was more than 10 minutes ahead of received time and stats used received time instead. | | `vehicle_fast_writer_nats_consumer_pending` | JetStream messages pending for the fast writer durable consumer. | | `vehicle_fast_writer_nats_consumer_ack_pending` | JetStream messages delivered to fast writer but not yet acked. | | `vehicle_fast_writer_nats_consumer_waiting` | Pull requests waiting on the fast writer durable consumer. | | `vehicle_fast_writer_batch_pending_messages` | NATS fast writer messages fetched but not yet written to TDengine/Redis and acked. | | `vehicle_fast_writer_batch_pending_envelopes` | Valid parsed envelopes in the current NATS fast writer batch. | +| `vehicle_fast_writer_messages_total` | Fast writer message results by NATS subject. Labels: `subject`, `status`; status includes `ok`, `error`, `invalid_json`, and `ack_error`. `invalid_json` is acked and isolated so it will not block the raw fast path, but it indicates the stream contract is polluted. | | `vehicle_fast_writer_stage_duration_ms_histogram` | Fast writer stage duration histogram for TDengine, Redis, and NATS ack. The Redis stage is one batch pipeline when the repository supports batch updates. Labels: `subject`, `stage`, `status`. | -| `vehicle_history_writes_total` | TDengine history writes. Labels: `topic`, `status`. | +| `vehicle_fast_writer_redis_envelopes_total` | Redis realtime projection envelope result from the NATS fast path. Labels: `subject`, `status`; status includes `seen`, `updated`, `skipped_non_realtime`, `skipped_missing_vin`, `skipped_missing_vehicle_key`, `skipped_missing_fields`. Missing fields is an ingress contract failure: RAW is retained, but Redis online/KV projection is skipped and never re-flattened downstream. | +| `vehicle_fast_writer_redis_fields_total` | Redis realtime KV field write result from the NATS fast path. Labels: `subject`, `status`; status includes `seen`, `written`, `skipped_stale`. `skipped_stale` means an older event-time frame was allowed to add missing fields but was blocked from overwriting newer field values. | +| `vehicle_fast_writer_last_message_unix_seconds` | Last fast-writer message processing result by subject/status. Labels: `subject`, `status`. | +| `vehicle_fast_writer_last_stage_unix_seconds` | Last fast-writer stage completion by subject/stage/status. Labels: `subject`, `stage`, `status`. | +| `vehicle_history_writes_total` | TDengine history writes for valid envelopes only. Labels: `topic`, `status`; invalid JSON messages are committed after `vehicle_history_kafka_messages_total{status="invalid_json"}` and are not counted as writes. | +| `vehicle_history_parsed_fields_total` | Realtime raw envelopes that did or did not carry precomputed `parsed_fields` before TDengine history write. Labels: `topic`, `protocol`, `status`; status includes `present`, `missing`. | +| `vehicle_history_last_message_unix_seconds` | Last history Kafka message by topic/status. Labels: `topic`, `status`. | +| `vehicle_history_last_write_unix_seconds` | Last history TDengine write result by topic/status. Labels: `topic`, `status`. | +| `vehicle_history_last_commit_unix_seconds` | Last history Kafka commit by topic/status. Labels: `topic`, `status`. | | `vehicle_history_batch_pending_messages` | Messages fetched by history writer but not yet flushed and committed. | | `vehicle_history_batch_pending_rows` | Parsed raw envelopes waiting in the current TDengine batch. | +| `vehicle_history_retry_pending_messages` | Fetched messages retained in memory behind a failed TDengine write or Kafka commit. Healthy value is `0`; the writer does not fetch a new batch while this is non-zero. | +| `vehicle_history_batch_retries_total` | Failure-closed history batch retries. Labels: `reason`; `write_error` retries only the uncommitted suffix and `commit_error` retries commit without replaying TDengine writes. | | `vehicle_history_batch_flush_duration_ms` | Last TDengine batch flush duration. Labels: `status`. | +| `vehicle_history_config{setting="workers"}` | Configured in-process history Kafka consumer count; default and capacity minimum are `3`. | +| `vehicle_history_worker_active` | Active history consumer gauge by `worker`; all configured workers should be `1`. | | `vehicle_stat_writes_total` | MySQL metric writes. Labels: `topic`, `status`. | +| `vehicle_stat_kafka_messages_total` | Stat writer Kafka message results by fields topic. Labels: `topic`, `status`; `invalid_json` messages are committed and isolated before MySQL mileage statistics. | +| `vehicle_stat_last_message_unix_seconds` | Last stat Kafka message by topic/status. Labels: `topic`, `status`. | +| `vehicle_stat_last_write_unix_seconds` | Last stat MySQL append result by topic/status. Labels: `topic`, `status`. | +| `vehicle_stat_last_commit_unix_seconds` | Last stat Kafka commit by topic/status. Labels: `topic`, `status`. | | `vehicle_stat_write_duration_ms_histogram` | MySQL stat writer append duration histogram. Labels: `topic`, `status`. | +| `vehicle_stat_config{setting="workers"}` | Configured in-process Kafka consumer count. Default and capacity-check minimum are `3`. | +| `vehicle_stat_worker_active` | Active stat consumer gauge by `worker`; all configured workers should remain `1` while the service is running. | +| `vehicle_stat_batch_pending_messages` | Messages fetched by stat writer but not yet appended to MySQL and committed. Controlled by `STATS_BATCH_SIZE` and `STATS_BATCH_WAIT_MS`; capacity check default threshold is `1000`. | +| `vehicle_stat_retry_pending_messages` | Fetched messages retained in memory behind a failed MySQL write or Kafka commit. Healthy value is `0`. | +| `vehicle_stat_batch_retries_total` | Failure-closed stat batch retries. Labels: `reason`; commit-only retries do not append the mileage sample twice in the same process. | +| `vehicle_stat_cache_entries` | Stat writer in-memory cache entries by cache type. Labels: `cache`; cache includes `last_total_mileage`, `source_seen`, `projection`, and `baseline`. Runtime cap is controlled by `STATS_CACHE_MAX_ENTRIES` and defaults to `1000000`. | +| `vehicle_stat_cache_max_entries` | Stat writer configured per-cache entry cap. `0` means unlimited and should not be used in production without an external memory guard. | +| `vehicle_stat_cache_evictions_total` | Cumulative stat writer cache evictions caused by capacity pressure. Labels: `cache`. | +| `vehicle_stat_samples_total` | Mileage samples extracted and written by stat writer. Use this with `vehicle_stat_writes_total`: write `ok` only means the append call succeeded, while `status="written"` proves a sample reached `vehicle_daily_mileage_source`. | +| `vehicle_stat_sources_total` | Stat writer source tracking results for `vehicle_data_source`. Labels: `topic`, `protocol`, `status`; status includes `attempted`, `written`, `skipped_throttled`, `skipped_missing_endpoint`. This intentionally avoids source IP labels to keep metrics low-cardinality. | +| `vehicle_stat_projections_total` | Final daily mileage projection results from `vehicle_daily_mileage_source` to `vehicle_daily_mileage`. Labels: `topic`, `protocol`, `status`; status includes `attempted`, `written`, `skipped_throttled`. | | `vehicle_realtime_updates_total` | Redis/MySQL realtime projector updates. Labels: `topic`, `status`. | +| `vehicle_realtime_kafka_messages_total` | Realtime projector Kafka message results by raw topic. Labels: `topic`, `status`; `invalid_json` messages are committed and isolated before Redis/MySQL projection. | +| `vehicle_realtime_last_message_unix_seconds` | Last realtime Kafka message by topic/status. Labels: `topic`, `status`. | +| `vehicle_realtime_last_update_unix_seconds` | Last realtime projection result by topic/status. Labels: `topic`, `status`. | +| `vehicle_realtime_last_commit_unix_seconds` | Last realtime Kafka commit by topic/status. Labels: `topic`, `status`. | | `vehicle_realtime_store_update_duration_ms_histogram` | Redis/MySQL store update duration histogram. Labels: `store`, `protocol`, `status`. | +| `vehicle_realtime_config{setting="workers"}` | Configured in-process realtime Kafka consumer count; default and capacity minimum are `3`. | +| `vehicle_realtime_worker_active` | Active realtime consumer gauge by `worker`; all configured workers should be `1`. | +| `vehicle_realtime_async_queue_total` | Realtime API async secondary projection queue events for MySQL snapshot/location. Labels: `store`, `protocol`, `status`; status includes `queued`, `dropped`, `closed`. | +| `vehicle_realtime_async_queue_depth` | Realtime API async MySQL projection queue depth. Labels: `store`, `protocol`. | +| `vehicle_realtime_async_queue_capacity` | Realtime API async MySQL projection queue capacity. Labels: `store`; controlled by `MYSQL_REALTIME_ASYNC_QUEUE_SIZE`. | +| `vehicle_realtime_retry_pending_messages` | Fetched messages retained in memory behind a failed realtime projection or Kafka commit. Healthy value is `0`. | +| `vehicle_realtime_batch_retries_total` | Failure-closed realtime batch retries. Labels: `reason`; only the uncommitted suffix is projected again after a write failure. | +| `vehicle_identity_writer_retry_pending_messages` | JT808 raw messages retained behind a failed registration MySQL transaction or Kafka commit. Healthy value is `0`; identity writer does not fetch newer offsets while non-zero. | +| `vehicle_identity_writer_config{setting="workers"}` | Configured identity Kafka consumer count; default and capacity minimum are `3`. | +| `vehicle_identity_writer_worker_active` | Active identity consumer gauge by `worker`; every configured worker should remain `1`. | +| `vehicle_identity_writer_batch_retries_total` | Failure-closed identity retries. Labels: `reason`; `write_error` retries the transaction and `commit_error` retries only Kafka commit. | +| `vehicle_realtime_plate_cache_entries` | Realtime API VIN-to-plate binding cache entry count. Default runtime cap is controlled by `PLATE_CACHE_MAX_ENTRIES` and defaults to `200000`. | +| `vehicle_realtime_plate_cache_max_entries` | Realtime API configured VIN-to-plate cache entry cap. `0` means unlimited and should not be used in production without an external memory guard. | +| `vehicle_realtime_plate_cache_evictions_total` | Cumulative VIN-to-plate cache evictions caused by TTL expiry cleanup or capacity pressure. | | `vehicle_history_kafka_lag` | Estimated Kafka lag for history writer by topic and partition. | | `vehicle_stat_kafka_lag` | Estimated Kafka lag for stat writer by topic and partition. | | `vehicle_realtime_kafka_lag` | Estimated Kafka lag for realtime projector by topic and partition. | diff --git a/docs/ops/go-vehicle-ingest-memory.md b/docs/ops/go-vehicle-ingest-memory.md index 9c7d63df..94dbf49c 100644 --- a/docs/ops/go-vehicle-ingest-memory.md +++ b/docs/ops/go-vehicle-ingest-memory.md @@ -58,8 +58,8 @@ Go 服务使用 systemd 裸机部署，不使用 Docker。 最近 release： ```text +/opt/lingniu-go-native/releases/production-track-telemetry-20260714082451 /opt/lingniu-go-native/releases/100k-hardening-20260703175730 -/opt/lingniu-go-native/releases/parsed-fields-once-20260703163821 ``` 服务重启： @@ -511,14 +511,28 @@ vehicle_gateway_frame_duration_ms_histogram_sum{protocol,status} 该耗时覆盖单帧从解析、identity resolve、parsed fields 生成、publish enqueue 到协议响应写入的整体入口处理路径。带帧率压测时用它按协议观察 p95/p99，不能只看 `vehicle_gateway_frame_duration_ms` 最后一条 gauge。 +### Gateway identity resolve duration + +2026-07-11 已为 Gateway 身份解析单独增加耗时 histogram 和超时分类： + +```text +vehicle_gateway_identity_total{protocol,status} +vehicle_gateway_identity_duration_ms_histogram_bucket{protocol,status,le} +vehicle_gateway_identity_duration_ms_histogram_count{protocol,status} +vehicle_gateway_identity_duration_ms_histogram_sum{protocol,status} +``` + +生产默认使用周期身份快照，每帧只查本机内存；`status="timeout"` 主要用于兼容关闭 `IDENTITY_SNAPSHOT_ONLY_ENABLED` 后的回源查询模式。快照模式应重点观察 `vehicle_gateway_identity_snapshot_ready` 和最近成功刷新时间；RDS 故障时入口继续接收，上一版快照继续服务，未知 phone 暂时 unresolved。 + ### History writer TDengine 批写 2026-07-03 已实现 history-writer 第一阶段批写： - `history.Writer.AppendAllBatch` 按 TDengine child table 聚合 raw/location 多行 `INSERT`。 -- `cmd/history-writer` 默认 `HISTORY_BATCH_SIZE=200`、`HISTORY_BATCH_WAIT_MS=100`。 +- `cmd/history-writer` 默认 `HISTORY_WORKERS=3`、`HISTORY_BATCH_SIZE=200`、`HISTORY_BATCH_WAIT_MS=20`。 - TDengine batch 成功后才批量提交 Kafka messages；失败不提交 offset。 - 保留单条 `AppendAll` 路径作为回退。 +- `production-track-telemetry-20260714082451` 起，`vehicle_locations` stable 增加 `soc_percent`。history-writer 在建表后执行幂等 `ALTER STABLE ... ADD COLUMN soc_percent DOUBLE`，只忽略 TDengine 明确的重复列错误；其他迁移错误会阻止 ready，禁止带病继续写入。单条和批量位置 INSERT 都必须携带 SOC。 新增指标： diff --git a/docs/ops/vehicle-ingest-runbook.md b/docs/ops/vehicle-ingest-runbook.md index b0f5726d..8cb7b885 100644 --- a/docs/ops/vehicle-ingest-runbook.md +++ b/docs/ops/vehicle-ingest-runbook.md @@ -23,9 +23,12 @@ | --- | --- | --- | --- | | 接入 | Gateway | `lingniu-go-gateway.service` | `127.0.0.1:20211` | | 桥接 | NATS Kafka bridge | `lingniu-go-nats-kafka-bridge.service` | `127.0.0.1:20214` | +| 字段投影 | Kafka RAW fields projector | `lingniu-go-fields-projector.service` | `127.0.0.1:20218` | | 历史 | TDengine writer | `lingniu-go-history-writer.service` | `127.0.0.1:20212` | | 统计 | MySQL stat writer | `lingniu-go-stat-writer.service` | `127.0.0.1:20213` | -| 实时 | Realtime API/projector | `lingniu-go-realtime-api.service` | `127.0.0.1:20200` | +| 实时写入 | Realtime writer | `lingniu-go-realtime-writer.service` | `127.0.0.1:20216` | +| 身份事实 | JT808 identity writer | `lingniu-go-identity-writer.service` | `127.0.0.1:20217` | +| 实时/API | Realtime API | `lingniu-go-realtime-api.service` | `127.0.0.1:20200` | ## Topic 配置基线 @@ -34,20 +37,113 @@ | 服务 | 必要 topic | | --- | --- | | `lingniu-go-history-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` | -| `lingniu-go-stat-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` | -| `lingniu-go-realtime-api.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` | -| `lingniu-go-nats-kafka-bridge.service` | NATS raw subjects 到同名 Kafka topic；`vehicle.event.go.unified.v1` 只在兼容开关打开时使用 | +| `lingniu-go-stat-writer.service` | `vehicle.fields.go.gb32960.v1`、`vehicle.fields.go.jt808.v1`、`vehicle.fields.go.yutong-mqtt.v1` | +| `lingniu-go-realtime-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` | +| `lingniu-go-identity-writer.service` | `vehicle.raw.go.jt808.v1` | +| `lingniu-go-fields-projector.service` | 消费三类 `vehicle.raw.go.*`，把 RAW 中已有的 `parsed_fields` 投影到对应 `vehicle.fields.go.*` | +| `lingniu-go-nats-kafka-bridge.service` | NATS canonical raw 到 Kafka raw，并从 raw 中已有的 `parsed_fields` 投影同协议 Kafka fields；`vehicle.event.go.unified.v1` 只在兼容开关打开时使用 | -如果 stat-writer 少消费某个 raw topic，对应协议的每日里程不会进入 `vehicle_daily_mileage`。修正后可能出现短时间 Kafka lag，这是在追补历史 backlog；只要 `vehicle_stat_writes_total` 持续增长且 lag 下降，就是健康状态。 +Gateway 在 NATS 模式下设置 `FIELDS_DERIVE_FROM_RAW_ENABLED=true`，每帧只发布 canonical raw；bridge 设置 `BRIDGE_DERIVE_FIELDS_FROM_RAW_ENABLED=true`，只复用 raw envelope 中已计算一次的 `parsed_fields`，不会重新解析协议原文。bridge 对同一条 NATS raw 生成 Kafka raw 和可选 Kafka fields，两个输出都成功后才 ACK；任一输出失败会保留 NATS 消息等待重放。登录、注册、鉴权等非实时帧只写 raw，不生成 fields。 + +如果 stat-writer 少消费某个 fields topic，对应协议的每日里程不会进入 `vehicle_daily_mileage`。stat-writer 启动时会拒绝 `vehicle.raw.*` 这类 RAW topic 配置，避免统计链路重新从原始报文解析并破坏 RAW/fields 解耦。修正后可能出现短时间 Kafka lag，这是在追补历史 backlog；`vehicle_stat_writes_total{status="ok"}` 只表示 append 调用成功，真正证明里程样本入库的是 `vehicle_stat_samples_total{status="written"}` 持续增长且 lag 下降。 + +stat-writer 默认 `STATS_WORKERS=3`，每个 worker 是同一 consumer group 的独立 Kafka reader。Kafka 按车辆 key 固定分区，因此单车累计里程仍按分区顺序执行，不同分区并行。调整 worker 后同时核对 `vehicle_stat_config{setting="workers"}`、每个 `vehicle_stat_worker_active{worker}`、MySQL 连接数、write p99 和 stat lag；worker 不应超过 fields topic 的有效分区并行度。 + +history-writer 和 realtime-writer 同样默认分别使用 `HISTORY_WORKERS=3`、`REALTIME_WORKERS=3`。每个 worker 只顺序处理 Kafka 分配给自己的分区，TDengine 表缓存、MySQL 实时节流缓存和车牌缓存均支持并发访问。调整后必须同时核对对应的 `*_config{setting="workers"}`、`*_worker_active`、pending/retry、后端连接数和 Kafka lag。 + +history-writer 启动会对既有 `vehicle_locations` stable 幂等补齐 `soc_percent DOUBLE`。发布后除 `/readyz` 外，必须抽查一台支持 SOC 的车辆在 TDengine 历史位置中 `soc_percent IS NOT NULL`；如果迁移报错且不是明确的重复列错误，服务应保持未就绪，先处理 TDengine DDL/权限，不能跳过迁移后强启。 + +identity-writer 默认使用 `IDENTITY_WRITER_WORKERS=3`。每个 worker 持有独立的位置触达节流器并共享 MySQL 连接池；Kafka phone/vehicle key 保证同一终端在单一分区内有序，分区再平衡最多产生一次幂等触达。调整后核对 `vehicle_identity_writer_config{setting="workers"}`、`vehicle_identity_writer_worker_active`、pending/retry、MySQL 连接数和 identity lag。 + +Gateway 和 NATS bridge 启动时也会检查 raw/fields 合同：Kafka raw topic 必须是 `vehicle.raw.*`，fields topic 必须是 `vehicle.fields.*`；NATS raw subject 和 fields subject 不允许相同。若服务启动失败并提示 `raw and fields ... must be different` 或 `fields kafka topic ... must start with "vehicle.fields."`，应先修正 `/opt/lingniu-go-native/env/*.env` 的 topic/subject 配置。 Realtime API 是当前态投影，默认在没有已提交 offset 时从 latest 开始消费。切换 topic 或新建 consumer group 后，不应让 realtime 追扫历史 raw backlog；需要重建当前态时，应使用明确的回放任务或手动 reset offset。 +history、stat、realtime、identity 四个 Kafka 消费服务都会暴露 `vehicle_kafka_consumer_info{service,group,topic}`。`capacity-check` 会把缺少该指标的消费服务判为 degraded，用来捕获“进程活着但没有挂上 Kafka topic”的配置错误。Identity writer 上线后 Gateway 必须设置 `JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false`，容量巡检会阻止双写 `jt808_registration`。 + 业务端口： - GB32960 TCP：`0.0.0.0:32960` - JT/T 808 TCP：`0.0.0.0:808` +- 实时 writer：`127.0.0.1:20216` +- 身份 writer：`127.0.0.1:20217` - 实时/API：`0.0.0.0:20200` +## 协议鉴权配置 + +Gateway 的鉴权策略分为 `disabled`、`observe`、`enforce`。生产新增或变更账号时先使用 `observe`，确认 `vehicle_gateway_authentication_total` 中只有 `accepted` 后再切换 `enforce`。 + +GB32960 多平台账号使用仅 root 可读的 JSON 文件，不把密码提交到 Git 或写入 systemd unit： + +```json +{ + "platform-a": "current-password", + "platform-b": ["migration-old-password", "current-password"] +} +``` + +`gateway.env` 只配置文件地址和模式： + +```bash +GB32960_PLATFORM_CREDENTIALS_FILE=/opt/lingniu-go-native/secrets/gb32960-platform-credentials.json +GB32960_AUTH_MODE=observe +JT808_AUTH_MODE=observe +JT808_REGISTER_AUTH_CODE=issued-auth-code +IDENTITY_RESOLVE_TIMEOUT_MS=50 +``` + +Gateway 生产必须启用独立 WAL outbox，不与旧 `NATS_SPOOL_DIR` 混用： + +```bash +NATS_DURABLE_OUTBOX_ENABLED=true +NATS_OUTBOX_DIR=/opt/lingniu-go-native/spool/nats-outbox-wal +NATS_OUTBOX_MAX_INFLIGHT=10000 +NATS_OUTBOX_ACK_TIMEOUT_MS=3000 +NATS_OUTBOX_REPLAY_BATCH_SIZE=1000 +NATS_OUTBOX_REPLAY_INTERVAL_MS=1000 +NATS_OUTBOX_FSYNC=true +NATS_OUTBOX_CLOSE_TIMEOUT_MS=5000 +NATS_OUTBOX_WAL_SEGMENT_BYTES=16777216 +NATS_OUTBOX_WAL_SEGMENT_AGE_MS=5000 +NATS_OUTBOX_WAL_APPEND_QUEUE_SIZE=100000 +NATS_OUTBOX_WAL_COMMIT_BATCH_SIZE=256 +NATS_OUTBOX_WAL_COMMIT_INTERVAL_MS=1 +``` + +WAL 在分组 `fsync` 成功后才向协议处理线程返回接受成功，网络发布不阻塞设备连接；PubAck 前记录始终可恢复。正常值为 `vehicle_durable_outbox_backlog_records=0`、`vehicle_durable_outbox_inflight=0`，且 `submitted` 与 `acked` 最终一致。`submit_error`、`ack_error`、`remove_error` 或 `close_timeout` 任一增长都必须排查 NATS、磁盘和进程关闭过程。WAL 的 CRC 损坏会拒绝 Gateway 启动，禁止跳过或删除文件后强行启动；应先保留目录副本再恢复。 + +GB32960 登录密码在鉴权完成后会从 `parsed_fields` 删除，只保留 `password_present`；协议原始帧仍可能包含凭据，因此 RAW 查询接口和 TDengine `raw_hex` 必须按敏感数据控制访问。JT808 的 `0x0102` 会先匹配配置的公共鉴权码，再按规范化手机号匹配 `jt808_registration.auth_token` 的只读内存快照，全程不查询 MySQL；指标 `source=device` 表示命中终端历史鉴权码。强制模式拒绝的鉴权帧不会覆盖该可信鉴权码和最近鉴权成功时间。JT808 的 `enforce` 不会拒绝未发注册/认证、直接上报 `0x0200` 的兼容平台；这类来源继续通过身份缺口指标治理。 + +## 标准发布流程 + +Go 原生服务发布必须使用 Linux amd64 构建产物，不要直接上传本机默认 `go build` 结果。仓库提供统一脚本： + +```bash +cd go/vehicle-gateway + +# 首次上线 identity-writer 前准备 systemd、env 和 Gateway 单写开关。 +scripts/install-identity-writer-service.sh --host root@115.29.187.205 + +# 首次上线 fields-projector 前准备 systemd 和 env；该步骤不会自动关闭 bridge 的兼容字段投影。 +scripts/install-fields-projector-service.sh --host root@115.29.187.205 + +# 只构建并校验 ELF x86-64 产物。 +scripts/deploy-ecs-release.sh --build-only --release verify-$(date +%Y%m%d%H%M%S) + +# 构建、上传并校验 release，但不切换 current、不重启服务。 +scripts/deploy-ecs-release.sh \ + --host root@115.29.187.205 \ + --release staged-$(date +%Y%m%d%H%M%S) \ + --stage-only + +# 构建、上传新 release、切换 current、重启服务并执行 readyz/capacity-check。 +scripts/deploy-ecs-release.sh \ + --host root@115.29.187.205 \ + --release production-$(date +%Y%m%d%H%M%S) +``` + +脚本会构建 `gateway`、`history-writer`、`stat-writer`、`identity-writer`、`realtime-api`、`nats-fast-writer`、`nats-kafka-bridge`、`fields-projector`、`capacity-check`、`load-sim`、`stats-backfill`、`identity-import`，并逐个校验 `file` 输出包含 `ELF 64-bit` 和 `x86-64`。正式发布按 bridge、fields projector、下游 writer、Gateway 的依赖顺序重启，确保 canonical raw 到 fields 的派生能力先于入口切换生效；发布窗口暂停 `lingniu-go-capacity-check.timer`，发布后恢复 timer、检查 9 个 systemd 服务、9 个 `/readyz` 和 `capacity-check`。`--stage-only` 只落盘 release，不改变运行态。生产发布如需把业务统计缺口纳入发布门禁，使用 `--capacity-args "-daily-mileage-diagnostics-max-actionable=50"` 这类参数，和 `lingniu-go-capacity-check.service` 的定时巡检口径保持一致。 + ## 五分钟排查顺序 先回答最核心的问题：数据有没有进来，有没有排队，有没有被消费，最后有没有被查询到。 @@ -56,30 +152,43 @@ Realtime API 是当前态投影，默认在没有已提交 offset 时从 latest 2. 检查 gateway 帧计数和 TCP 活跃连接。 3. 检查 NATS bridge 的 pending 和 ack-pending。 4. 检查 bridge 写 Kafka 与 NATS ack 是否同时增长。 -5. 检查 history、stat、realtime 三类 Kafka consumer lag。 +5. 检查 history、stat、realtime、identity 四类 Kafka consumer lag。 6. 检查各 writer 的写入、提交、更新计数。 7. 最后再查存储和业务查询 API。 ```bash -for port in 20211 20212 20213 20214 20200; do +for port in 20211 20212 20213 20214 20215 20216 20217 20218 20200; do curl -fsS "http://127.0.0.1:${port}/readyz" echo done curl -fsS http://127.0.0.1:20211/metrics \ - | grep -E 'vehicle_gateway_(active_connections|frames_total|publish_total)|vehicle_async_sink' + | grep -E 'vehicle_gateway_(active_connections|frames_total|last_frame|publish_total|last_publish)|vehicle_async_sink' curl -fsS http://127.0.0.1:20214/metrics \ - | grep -E 'vehicle_bridge_(nats_consumer|kafka_writes_total|nats_acks_total)' + | grep -E 'vehicle_bridge_(nats_consumer|fields_projection|kafka_writes_total|last_kafka_write|nats_acks_total|last_ack|last_message)' curl -fsS http://127.0.0.1:20212/metrics | grep vehicle_history_kafka_lag curl -fsS http://127.0.0.1:20212/metrics | grep vehicle_history_batch curl -fsS http://127.0.0.1:20213/metrics | grep vehicle_stat_kafka_lag -curl -fsS http://127.0.0.1:20200/metrics | grep vehicle_realtime_kafka_lag +curl -fsS http://127.0.0.1:20213/metrics | grep vehicle_stat_samples_total +curl -fsS http://127.0.0.1:20213/metrics | grep vehicle_stat_sources_total +curl -fsS http://127.0.0.1:20213/metrics | grep vehicle_stat_last_ +curl -fsS http://127.0.0.1:20216/metrics | grep vehicle_realtime_kafka_lag +curl -fsS http://127.0.0.1:20217/metrics | grep vehicle_identity_writer_kafka_lag +for port in 20212 20213 20216 20217 20200; do + curl -fsS "http://127.0.0.1:${port}/metrics" | grep vehicle_kafka_consumer_info +done /opt/lingniu-go-native/current/capacity-check ``` +`capacity-check` 会解析 histogram 并做链路耗时早期预警，默认 `-gateway-frame-p99-ms=250`、`-gateway-identity-p99-ms=100`、`-async-sink-p99-ms=100`、`-async-sink-queue-wait-recent-p99-ms=100`、`-bridge-batch-p99-ms=5000`、`-fast-writer-stage-p99-ms=100`、`-history-flush-p99-ms=1000`、`-realtime-batch-p99-ms=500`、`-realtime-store-p99-ms=100`、`-stat-write-p99-ms=100`、`-kafka-lag-max=1000`、`-histogram-min-samples=100`。其中 realtime batch 是 Kafka projector 的整批 flush 耗时，主要看吞吐是否吃紧；单车实时投影延迟仍看 fast-writer Redis stage、realtime store update、Kafka lag 和 pending。批处理内部 in-flight pending 默认允许 `-fast-writer-batch-pending-max=1000`、`-history-batch-pending-max=1000`、`-history-rows-pending-max=5000`、`-stat-batch-pending-max=1000`、`-realtime-batch-pending-max=1000`，用于避免高频流量里百条级正在 flush/ack 的批次误报；failure-closed 重试则由 `-consumer-retry-pending-max=0` 单独门禁，history/realtime/stat/identity 任一 `*_retry_pending_messages` 非零都会降级，设置为负数才关闭。bridge/fast-writer 的 unacked ratio 要达到 `-bridge-message-min-received=5000`、`-fast-writer-message-min-received=5000` 后才判断。刚重启样本不足时不判断 p99；少量 Kafka lag 可能只是高频流量里的 in-flight 消息，只有 history/stat/realtime/identity lag 总和超过阈值才降级；压测阶段可以临时调低阈值。 + +topic 流转检查默认只抓完全断层：`-required-consumer-topics` 要求 history/realtime 消费三类 raw topic、stat 消费三类 fields topic；`-gateway-field-min-raw=100` 检查某协议 raw publish 足够多，扣除登录、注册、鉴权等 `skipped_non_realtime` 后仍然没有 fields 直接发布或委托 bridge；`-gateway-field-max-missing-ratio=0.20` 检查实时帧缺少预计算 fields 或发布错误比例是否过高；`-gateway-bridge-min-publishes=1000` 检查某协议 raw/委托 fields 已经由 gateway 接受，但 bridge 没有写对应 Kafka topic；`-gateway-fast-writer-min-raw-publishes=1000` 检查某协议 raw 已经由 gateway 发布，但 fast-writer 没有消费同 raw subject 并成功写 Redis/ack；`-raw-fanout-min-bridge-writes=1000` 检查某 raw topic 已经由 bridge 写入 Kafka，但 history 未收到/写入或 realtime 未收到/更新；`-gateway-parse-min-frames=100` 和 `-gateway-parse-max-bad-ratio=0.05` 检查某协议入口帧解析质量是否突然恶化；`-gateway-response-min-samples=100` 和 `-gateway-response-max-error-ratio=0.05` 检查 32960 登录 ACK、808 注册/通用 ACK 等协议响应是否大量 build/write 失败，`skipped` 表示该消息类型无需响应，不计入比例；`-gateway-identity-snapshot-required=1` 和 `-gateway-identity-snapshot-max-age-seconds=180` 检查内存身份快照是否可用且持续刷新，`-gateway-identity-min-samples=100` 和 `-gateway-identity-max-unresolved-ratio=0.20` 检查某协议 VIN/车牌/phone 映射质量是否突然失效；`-history-parsed-field-min-frames=100` 和 `-history-parsed-field-max-missing-ratio=0.05` 检查 history-writer 收到的实时 raw envelope 是否缺失预计算 `parsed_fields`；`-fast-writer-redis-envelope-min-seen=100` 检查 Redis 快路径看到足够 raw envelope 后是否完全没有更新当前态，`-fast-writer-redis-envelope-max-bad-ratio=0.50` 检查缺 VIN/vehicle_key 的实时帧比例是否异常；`-fast-writer-redis-field-min-seen=1000` 和 `-fast-writer-redis-field-max-stale-ratio=0.20` 检查 Redis 快路径旧帧字段跳过比例是否异常升高；`-stat-topic-min-bridge-writes=1000` 检查某 fields topic 已经由 bridge 写入 Kafka，但 stat-writer 未配置或未收到该 topic。统计链路还会用 `-stat-sample-min-writes=100` 检查“某个 fields topic 已成功 append 但完全抽不到里程样本”的配置/字段映射故障，并用 `-stat-sample-max-actionable-skip-ratio=0.60` 检查 fields 缺失、VIN 缺失、里程缺失、非正里程、source 缺失这类可行动跳过是否占比过高；`-stat-source-min-samples=100` 和 `-stat-source-max-missing-ratio=0.60` 检查已找到里程样本但来源端点缺失或 `vehicle_data_source` tracking 没有工作；`-stat-projection-min-sample-writes=100` 检查某 fields topic 已写入 source 候选里程样本但最终 `vehicle_daily_mileage` 投影完全没有写入；`skipped_same_mileage` 只是重复总里程去重，不计入该比例。`-last-activity-stale-seconds=300` 检查已出现过的成功/received `*_last_*_unix_seconds` 是否超过 5 分钟未刷新。上述阈值设为 `0` 可临时关闭；特殊环境可用 `-required-consumer-topics=''` 关闭消费 topic 合同检查。 + +每日里程业务诊断也会进入 `capacity-check` 的 JSON：默认通过 `-daily-mileage-diagnostics-url=http://127.0.0.1:20200/api/stats/daily-metrics/diagnostics/reasons` 查询当天原因汇总，输出 `daily_mileage_diagnostics.vehicle_total`、`actionable_issue_total` 和 `items`。二进制默认 `-daily-mileage-diagnostics-max-actionable=-1` 表示只展示不阻断；生产定时巡检和发布门禁应显式设置一个阈值，例如当前已知 MQTT 稀疏总里程期间可先用 `50`，待问题清理后收紧到 `0`。诊断接口不可访问会降级，因为这表示 MySQL/API 观测链路不可用；临时环境可用 `-daily-mileage-diagnostics-url=''` 关闭。 + ## 健康基线 当前生产环境的健康特征： @@ -87,10 +196,13 @@ curl -fsS http://127.0.0.1:20200/metrics | grep vehicle_realtime_kafka_lag - 所有 `/readyz` 都返回 `status=ok`。 - `vehicle_bridge_nats_consumer_ack_pending` 为 `0`。 - history、stat、realtime 的 Kafka lag 为 `0` 或短时间小幅波动后归零。 +- history、stat、realtime 都能看到 `vehicle_kafka_consumer_info`。 - `capacity-check` 返回 `status=ok` 且退出码为 `0`。 - gateway 的帧计数持续增长。 - bridge 的 Kafka write 和 NATS ack 计数同时增长。 -- writer 的成功计数增长，同时 Kafka lag 不持续扩大。 +- writer 的成功计数增长，同时 Kafka lag 不持续扩大；统计链路还应看到 `vehicle_stat_samples_total{status="written"}` 随有效里程字段流量增长。 + +生产默认日志级别保持 `LOG_LEVEL=info`。Gateway 的 TCP 连接打开、关闭、空闲超时属于高频事件，只保留在 debug 级别；排查单车连接问题时可以临时设置 `LOG_LEVEL=debug`，排查结束后应恢复 info，常态监控以 `vehicle_gateway_active_connections` 和 `vehicle_gateway_connection_closes_total` 为准。 定时容量巡检由 systemd timer 触发： @@ -103,6 +215,8 @@ journalctl -u lingniu-go-capacity-check.service --since '10 minutes ago' --no-pa GB32960 和 JT/T 808 的活跃连接数受上游平台连接方式影响，不能直接等同于车辆数。突然归零或持续异常下降才是信号。 +计数器只说明服务启动后累计发生过，不代表当前仍有流量。排查断流时同时看 `*_last_*_unix_seconds`：gateway 看 `vehicle_gateway_last_frame_unix_seconds` 和 `vehicle_gateway_last_publish_unix_seconds`，bridge 看 `vehicle_bridge_last_kafka_write_unix_seconds`，fast-writer 看 `vehicle_fast_writer_last_message_unix_seconds` 和 `vehicle_fast_writer_last_stage_unix_seconds`，history/stat/realtime 分别看对应 `last_message`、`last_write/update`、`last_commit`。如果 counter 很高但 last 时间长时间不变，说明对应协议或 topic 已经停止活动。`capacity-check` JSON 会额外派生 `*_last_*_age_seconds`，现场排查优先看 age 是否持续扩大。 + ## 压测入口 Go 版本提供 `cmd/load-sim` 用于阶段性连接和帧写入压测。压测生产入口前必须先确认上游真实数据窗口，避免和业务流量混淆。 @@ -121,6 +235,26 @@ go run ./cmd/load-sim \ -send=false ``` +带帧压测 JT808 时必须读取服务端应答，并使用隔离手机号区间和自动清理： + +```bash +export MYSQL_DSN="$(sed -n 's/^MYSQL_DSN=//p' /opt/lingniu-go-native/env/base.env | tail -n 1)" + +/opt/lingniu-go-native/current/load-sim \ + -protocol jt808 \ + -addr 127.0.0.1:808 \ + -connections 1000 \ + -connect-rate 500 \ + -send-interval 1s \ + -duration 60s \ + -template 0200 \ + -jt808-phone-base 139000012000 \ + -drain-responses=true \ + -cleanup-registration +``` + +输出必须满足 `connections_failed=0`、`write_errors=0`、`response_bytes>0`、`read_errors=0`。每轮使用未在最近 10 分钟压测过的新手机号区间时，结束后的 `rows_deleted` 应等于模拟连接数；重复号段可能被 identity-writer 的 location touch 节流跳过写入，因此清理数为 `0` 也不表示删除失败。`-cleanup-only` 可用于异常中断后的补清理；删除条件固定受手机号区间和 loopback 来源约束，不清理真实注册。Bridge 默认 `BRIDGE_KAFKA_WRITE_CONCURRENCY=6`，同一批次按 topic 并行写 Kafka；RAW 和 fields 均成功后才 ACK NATS 源消息。当前生产压测基线使用 `NATS_ASYNC_RAW_WORKERS=128`、`FAST_WRITER_WORKERS=16`、`FAST_WRITER_FETCH_WAIT_MS=5`。 + ## 告警阈值建议 | 信号 | 建议阈值 | 含义 | @@ -130,28 +264,82 @@ go run ./cmd/load-sim \ | `vehicle_gateway_connection_closes_total{reason="read_error"}` | 连续增长 | 入口 TCP 读失败，优先查网络、客户端断连和内核连接状态。 | | `vehicle_gateway_connection_closes_total{reason="extract_error"}` | 连续增长 | 报文边界或协议提取异常，优先抽查 raw 日志和协议 extractor。 | | `vehicle_gateway_connection_closes_total{reason="read_timeout"}` | 突然高于历史基线 | 车辆长时间无上报或链路空闲超时，需结合在线数和上游平台状态判断。 | -| `vehicle_gateway_frames_total{status!="OK"}` | 连续 2 分钟增长 | 解析器或上游报文质量异常。 | -| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 | -| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落 | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。 | -| `vehicle_async_sink_enqueue_total{status="timeout"}` | 任意增长 | Gateway publish 队列已满或 worker 长时间阻塞，入口可能开始丢实时性。 | +| `vehicle_gateway_frames_total{status!="OK"}` | 单协议样本达到 `capacity-check -gateway-parse-min-frames` 后，异常比例超过 `capacity-check -gateway-parse-max-bad-ratio` | 解析器或上游报文质量异常。优先抽查 raw 日志、协议 extractor、最近部署的解析规则和上游转发内容。 | +| `vehicle_gateway_frame_duration_ms_histogram_bucket` | p99 超过 `capacity-check -gateway-frame-p99-ms` 或连续 5 分钟超过容量目标 | Gateway parse、identity resolve、publish enqueue 或响应链路变慢。 | +| `vehicle_gateway_response_total{status="build_error"}` / `status="write_error"` | 单协议响应尝试样本达到 `capacity-check -gateway-response-min-samples` 后，错误比例超过 `capacity-check -gateway-response-max-error-ratio` | 协议响应构造或 TCP 写回失败。优先查 32960 登录/实时 ACK、808 注册/通用 ACK、连接断开、响应编码和上游是否提前断链。 | +| `vehicle_gateway_response_e2e_recent_p99_ms` | 单协议样本达到 `capacity-check -histogram-min-samples` 后超过 `capacity-check -gateway-response-recent-p99-ms`，默认 `100ms` | 收帧、解析、内存身份映射、RAW 入队或协议写回变慢。结合 identity、async queue 和 NATS 指标定位具体阶段。 | +| `vehicle_gateway_authentication_total{mode="observe",status!="accepted"}` | 任意增长 | 当前登录/认证与配置不一致但仍放行。先核对账号覆盖、上游迁移密码和 JT808 认证码，确认无误后才能切换 `enforce`。 | +| `vehicle_gateway_authentication_total{mode="enforce",status!="accepted"}` | 任意增长 | 网关已返回协议失败并关闭该连接；检查是否为未授权来源、过期账号或配置错误。 | +| `vehicle_gateway_publish_total{kind="fields"}` | 某协议 `raw` 达到 `capacity-check -gateway-field-min-raw` 但 fields 直接发布或 `delegated` 都为 0 | Gateway 收到实时帧但没有可供 bridge 派生的 `parsed_fields`，优先查协议解析和字段映射。 | +| `vehicle_gateway_fields_total{status="skipped_non_realtime"}` | 持续增长但 fields publish 正常 | Gateway 正确跳过登录、注册、鉴权、心跳等非实时帧，不会进入统计 topic；如果实时数据也断流，再查上游是否只在发控制帧。 | +| `vehicle_gateway_fields_total{status="skipped_missing_fields"}` / `publish_error` | 单协议达到 `capacity-check -gateway-field-min-raw` 后，缺失比例超过 `capacity-check -gateway-field-max-missing-ratio` | 实时帧没有生成扁平字段，或兼容直发模式发布失败。NATS canonical raw 模式优先查协议字段映射和 `parsed_fields`。 | +| `vehicle_gateway_publish_total{kind="raw"}` / `vehicle_gateway_publish_total{kind="fields",status="delegated"}` | 某协议某 kind 达到 `capacity-check -gateway-bridge-min-publishes` 但 bridge 无对应 Kafka topic 写入 | Gateway 已接受 raw/fields 派生任务，但 bridge 没有写 Kafka。优先查 bridge projection、route、NATS subject、Kafka write 和 bridge 日志。 | +| `vehicle_gateway_publish_total{kind="raw"}` | 某协议 raw 达到 `capacity-check -gateway-fast-writer-min-raw-publishes` 但 fast-writer 无同 raw subject `ok` | Gateway 已经发布到 NATS，但 Redis 快路径未消费或未成功写入/ack。优先查 `vehicle_fast_writer_messages_total`、`vehicle_fast_writer_stage_duration_ms_histogram_bucket{stage="redis"}`、`stage="ack"`、NATS durable pending 和 Redis 健康。 | +| `vehicle_gateway_last_frame_unix_seconds` / `vehicle_gateway_last_publish_unix_seconds` | 已出现过的成功 activity 超过 `capacity-check -last-activity-stale-seconds` 未刷新 | 对应协议入口或 publish 已停止活动，先对比连接数、上游转发状态和 bridge last 指标。 | +| `vehicle_gateway_identity_total{status!="resolved"}` | 单协议样本达到 `capacity-check -gateway-identity-min-samples` 后，异常比例超过 `capacity-check -gateway-identity-max-unresolved-ratio` | VIN/车牌/phone 映射质量下降。优先查 `vehicle_identifier` 导入、`vehicle_identity_binding` 基础车牌/VIN、`jt808_registration` phone 状态和 resolver 缓存 TTL。 | +| `vehicle_gateway_identity_total{status="timeout"}` | 任意持续增长 | Gateway 身份解析超过 `IDENTITY_RESOLVE_TIMEOUT_MS`，帧会继续 partial/unresolved 下发，但 VIN/车牌映射可能滞后。 | +| `vehicle_gateway_identity_duration_ms_histogram_bucket` | p99 超过 `capacity-check -gateway-identity-p99-ms` 或接近 `IDENTITY_RESOLVE_TIMEOUT_MS` | MySQL/缓存身份解析开始拖慢入口链路，优先查 `vehicle_identifier`、`vehicle_identity_binding`、`jt808_registration` 索引和 RDS 状态。 | +| `vehicle_gateway_identity_cache_entries` | 超过 `capacity-check -gateway-identity-cache-entries-max`，默认 `300000` | Gateway 身份解析缓存超过容量预期。优先确认 `IDENTITY_LOOKUP_CACHE_MAX_ENTRIES`、`IDENTITY_LOOKUP_CACHE_TTL_SECONDS`、`IDENTITY_LOOKUP_CACHE_CLEANUP_INTERVAL_SECONDS`，并检查上游是否出现大量异常 phone、source IP 或历史回放。 | +| `vehicle_gateway_identity_snapshot_ready` / `vehicle_gateway_identity_snapshot_last_success_unix_seconds` | ready 不是 `1`，或最近成功刷新超过 `180s` | Gateway 会继续接入并保留 raw，但新 808 phone 可能暂时无法映射 VIN。检查 RDS 连通性和四张身份来源表；恢复后周期刷新会自动发布新快照，不需要重启。 | +| `vehicle_durable_outbox_backlog_records` / `vehicle_durable_outbox_inflight` | 持续非 0 或超过容量门禁 | Gateway 已持久接受但尚未收到 JetStream PubAck。短时突发可自动回落；持续增长时检查 NATS 可用性、磁盘延迟、ACK timeout 和 inflight 上限。 | +| `vehicle_durable_outbox_publish_total{status=~"submit_error|ack_error|remove_error|close_timeout"}` | 任意增长 | WAL 后的异步发布、PubAck、分段删除或关闭等待失败。记录不会因提交/ACK 失败丢失；修复依赖后由 replay 自动恢复。 | +| `vehicle_async_sink_queue_depth{sink="nats"}` | 持续增长且不回落，或超过 `capacity-check -async-sink-queue-depth-max`，默认 `10000` | Gateway 到 NATS/Kafka 的异步 publish 队列开始积压。结合 `vehicle_async_sink_queue_capacity` 判断是否接近满队列；NATS 入队等待由 `NATS_ASYNC_ENQUEUE_TIMEOUT_MS` 控制，Kafka fallback 由 `KAFKA_ASYNC_ENQUEUE_TIMEOUT_MS` 控制。 | +| `vehicle_async_sink_queue_wait_recent_p99_ms` | 样本达到 `capacity-check -histogram-min-samples` 后超过 `-async-sink-queue-wait-recent-p99-ms`，默认 `100ms` | 秒级突发曾造成排队，即使当前 queue depth 已回到 0 也会被最近 512 样本窗口捕获。结合 `vehicle_async_sink_workers`、publish duration 和 NATS pending 判断是 worker 不足还是持久化确认变慢。 | +| `vehicle_async_sink_enqueue_total{status="timeout"}` | 任意增长 | Gateway publish 队列已满或 worker 长时间阻塞，入口会快速失败并记录 publish error，避免连接处理 goroutine 长时间堆积。优先查 NATS/Kafka 写入延迟、async worker 数、队列容量和 bridge/fast-writer pending。 | | `vehicle_async_sink_publish_total{status="error"}` | 连续增长 | NATS/Kafka publish 失败，需要先查中间件连接和日志。 | -| `vehicle_async_sink_publish_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Gateway async worker 写 NATS/Kafka 变慢，通常会带动 queue depth 增长。 | +| `vehicle_async_sink_publish_duration_ms_histogram_bucket` | p99 超过 `capacity-check -async-sink-p99-ms` 或连续 5 分钟上升 | Gateway async worker 写 NATS/Kafka 变慢，通常会带动 queue depth 增长。 | | `vehicle_history_batch_flush_total{status="error"}` | 任意增长 | TDengine 批写失败；Kafka offset 不会提交，应先查 TDengine 和 SQL 错误。 | -| `vehicle_history_batch_pending_messages` | 持续非 0 或 burst 后不回落 | history-writer 已拉取但未完成写入/提交，可能卡在 TDengine 或 Kafka commit。 | -| `vehicle_history_batch_pending_rows` | 持续非 0 或 burst 后不回落 | TDengine 有批量写入积压，通常早于 Kafka lag 放大。 | -| `vehicle_history_batch_flush_duration_ms{status="ok"}` | 持续上升 | TDengine 写入延迟增加，可能需要降低 batch size 或扩容 TDengine。 | +| `vehicle_history_parsed_fields_total{status="missing"}` | 单 topic/protocol 实时帧样本达到 `capacity-check -history-parsed-field-min-frames` 后，缺失比例超过 `capacity-check -history-parsed-field-max-missing-ratio` | history-writer 收到的实时 raw envelope 没有携带预计算 `parsed_fields`，TDengine RAW 证据层会缺少扁平化解析字段。优先查 gateway `BuildFieldsEnvelope` / raw envelope 构造、协议字段映射和最近部署。 | +| `vehicle_history_batch_pending_messages` | 超过 `capacity-check -history-batch-pending-max` 或 burst 后不回落 | history-writer 已拉取但未完成写入/提交，可能卡在 TDengine 或 Kafka commit。 | +| `vehicle_history_batch_pending_rows` | 超过 `capacity-check -history-rows-pending-max` 或 burst 后不回落 | TDengine 有批量写入积压，通常早于 Kafka lag 放大。 | +| `vehicle_history_batch_flush_duration_ms_histogram_bucket{status="ok"}` | p99 超过 `capacity-check -history-flush-p99-ms` 或持续上升 | TDengine 写入延迟增加，可能需要降低 batch size 或扩容 TDengine。 | +| `vehicle_history_kafka_messages_total{status="invalid_json"}` | 超过 `capacity-check -history-invalid-json-max`，默认 `0` | Kafka raw topic 中出现无法反序列化的 envelope。history-writer 会 commit 并隔离该消息，避免阻塞历史落库；必须检查 NATS bridge 写入 Kafka 的 payload、raw topic 是否混入非 envelope 数据，以及 gateway 发布合同。 | | `vehicle_bridge_nats_consumer_ack_pending` | 连续 2 分钟 `> 0` | 消息已投递给 bridge，但 Kafka 写入后未完成 ack。 | | `vehicle_bridge_nats_consumer_pending` | 持续增长且 `> 10000` | bridge 消费 NATS 的速度跟不上生产速度。 | | `vehicle_bridge_batch_pending_messages` | 持续非 0 或 burst 后不回落 | bridge 已拉取 NATS 消息但尚未完成 Kafka 写入和 NATS ack。 | -| `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 | -| `vehicle_fast_writer_nats_consumer_ack_pending` | 连续 2 分钟 `> 0` | 消息已投递给 fast-writer，但 TDengine/Redis 写入后未完成 ack。 | +| `vehicle_bridge_batch_duration_ms_histogram_bucket` | p99 超过 `capacity-check -bridge-batch-p99-ms` 或连续 5 分钟上升 | Kafka 写入或 NATS ack 开始变慢，通常会先于 ack-pending 扩大。 | +| `vehicle_bridge_fields_projection_total` | `invalid_json`、`invalid_envelope`、`protocol_mismatch`、`skipped_missing_fields` 或 `marshal_error` 增长 | canonical raw 无法派生统计 fields。raw 仍进入历史链路，但每日里程可能缺样本；按协议检查 Gateway `parsed_fields` 和字段合同。 | +| `vehicle_bridge_messages_total{status="route_error"}` / `vehicle_bridge_nats_acks_total{status="dropped_route_error"}` | route_error 超过 `capacity-check -bridge-route-error-max`，默认 `0` | NATS stream 中出现 bridge 未配置的 subject。bridge 会 ACK 并丢弃该消息，避免 poison message 阻塞正常 subject；必须检查 `NATS_STREAM_SUBJECTS`、`NATS_FILTER` 和 raw/fields subject 到 Kafka topic 的路由配置。 | +| `vehicle_bridge_kafka_writes_total{topic="vehicle.fields.go.*"}` | 某 fields topic 达到 `capacity-check -stat-topic-min-bridge-writes` 但 stat-writer 无同 topic consumer 或 received | fields 已进 Kafka，但统计消费配置或消费循环断层。 | +| `vehicle_bridge_kafka_writes_total{topic="vehicle.raw.go.*"}` | 某 raw topic 达到 `capacity-check -raw-fanout-min-bridge-writes` 但 history/realtime 无同 topic received/write/update | raw 已进 Kafka，但历史落库或当前态投影断层。先查对应服务 `vehicle_kafka_consumer_info`，再查 `vehicle_history_kafka_messages_total` / `vehicle_history_writes_total` / `vehicle_realtime_kafka_messages_total` / `vehicle_realtime_updates_total`。 | +| `vehicle_bridge_last_message_unix_seconds` / `vehicle_bridge_last_kafka_write_unix_seconds` / `vehicle_bridge_last_ack_unix_seconds` | 已出现过的成功 activity 超过 `capacity-check -last-activity-stale-seconds` 未刷新 | NATS 到 Kafka 桥接某 subject/topic 停止流动；先查 NATS pending、Kafka 写入和 ack。 | +| `vehicle_fast_writer_nats_consumer_ack_pending` | 超过 `100` 或持续不回落 | 消息已投递给 fast-writer，但 Redis 写入后未完成 ack；短暂个位数/十几条通常只是高频流量里的 NATS ack 飞行窗口，如果显式启用 TDengine stage，也可能卡在 TDengine。 | | `vehicle_fast_writer_nats_consumer_pending` | 持续增长且 `> 10000` | fast-writer 消费 NATS 的速度跟不上入口写入速度。 | -| `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 | +| `vehicle_fast_writer_batch_pending_messages` | 超过 `capacity-check -fast-writer-batch-pending-max` 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 Redis 写入和 ack；如果显式启用 TDengine stage，也可能卡在 TDengine。 | | `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 | -| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢；TDengine 阶段可小步调整 `FAST_WRITER_TDENGINE_MAX_OPEN_CONNS`，Redis 阶段是整批 pipeline 写入耗时，调整后必须观察是否出现选库错误和 ack-pending 增长。 | -| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢，会直接影响 realtime consumer 追平能力。 | -| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢，需结合 async queue depth 和 dropped 计数判断。 | -| `vehicle_stat_write_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | MySQL 每日里程统计写入变慢，可能导致 stat Kafka lag 增长。 | +| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 超过 `capacity-check -fast-writer-stage-p99-ms` 或连续 5 分钟上升 | NATS 快速写链路在 Redis 或 NATS ack 某一阶段变慢；生产默认 `FAST_WRITER_TDENGINE_ENABLED=false`，如果临时启用 TDengine stage，再观察 `stage="tdengine"`。 | +| `vehicle_fast_writer_messages_total{status="invalid_json"}` | 超过 `capacity-check -fast-writer-invalid-json-max`，默认 `0` | NATS raw subject 中出现无法反序列化的 envelope。fast-writer 会 ACK 并隔离该消息，避免阻塞 Redis 当前态；必须检查 gateway 发布 payload、NATS stream 是否混入非 envelope 数据，以及 bridge/fast-writer 的 subject 过滤配置。 | +| `vehicle_fast_writer_redis_envelopes_total{status="skipped_non_realtime"}` | 持续增长但 `updated` 正常 | raw topic 中存在登录、注册、鉴权等非实时帧，Redis 当前态会跳过它们，不代表污染当前态；如果 `updated=0` 且只有 skip 增长，应检查 gateway 是否停止发送实时帧。 | +| `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_vin"}` / `skipped_missing_vehicle_key` | 持续增长 | 实时帧进入 Redis 快路径但身份解析不完整，当前态不会写入。优先查 identity resolver、`vehicle_identity_binding`、`jt808_registration`、MQTT VIN 映射和 GB32960 VIN 字段。 | +| `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_fields"}` | 任意持续增长 | Gateway 发布了实时 RAW，但没有附带一次计算后的 `parsed_fields`。RAW 仍写历史，Redis 在线/KV 和 MySQL snapshot/location 均跳过；检查协议 parser、字段映射和 Gateway envelope 构造，禁止在 writer 中增加重新解析兜底。 | +| `vehicle_fast_writer_redis_fields_total{status="skipped_stale"}` | 某协议持续增长且占 `status="seen"` 比例异常升高 | Redis 当前态收到大量旧事件时间字段，旧帧不会覆盖新字段，但会影响上游时序质量判断。优先按 `subject` 查上游平台补发、设备时间漂移、NATS backlog 和 gateway event-time 归一化记录。 | +| `vehicle_fast_writer_last_message_unix_seconds` / `vehicle_fast_writer_last_stage_unix_seconds` | 已出现过的成功 activity 超过 `capacity-check -last-activity-stale-seconds` 未刷新 | Redis 快路径某 raw subject 或具体 stage 停止活动。优先对比 gateway last publish、NATS pending、`vehicle_fast_writer_messages_total`、Redis 健康和 `stage="redis"` / `stage="ack"` 的最近时间。 | +| `vehicle_fast_writer_tdengine_enabled` | 生产非 `0` | fast-writer 正在写 TDengine，可能与 history-writer 双写 RAW/位置证据层；`capacity-check` 会降级，除压测或故障切换外应保持 `0`。 | +| `vehicle_kafka_consumer_info` | history/stat/realtime 缺失 | 服务进程可能启动了，但 Kafka broker、group 或 topic 配置没有生效。 | +| `vehicle_history_config{setting="workers"}` / `vehicle_history_worker_active` | 配置低于 `-history-workers-min=3` 或任一 worker 不活跃 | 检查 `HISTORY_WORKERS`、Kafka group 分区分配、进程日志和 TDengine 连接承载。 | +| `capacity-check` finding: `required kafka consumer topic missing` | history/realtime/stat 缺少生产合同 topic | 对照 `/opt/lingniu-go-native/env/*.env` 的 `KAFKA_TOPICS` 或 `KAFKA_TOPIC` 配置。history/realtime 应有三类 raw topic，stat 应有三类 fields topic。 | +| `vehicle_realtime_redis_projector_enabled` | 生产非 `0` | realtime-api 正在从 Kafka 重写 Redis，可能与 NATS fast-writer 双写当前态；`capacity-check` 会降级，生产应保持 `0`。 | +| `vehicle_realtime_kafka_messages_total{status="invalid_json"}` | 超过 `capacity-check -realtime-invalid-json-max`，默认 `0` | realtime-api 的 Kafka 当前态投影收到无法反序列化的 raw envelope。服务会 commit 并隔离该消息，避免阻塞 Redis/MySQL 当前态；必须检查 bridge 写入 Kafka 的 payload、raw topic 污染和 gateway 发布合同。 | +| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | 生产出现或 p99 连续 5 分钟上升 | 仅临时启用 realtime Redis projector 时使用；常规 Redis 写入延迟看 `vehicle_fast_writer_stage_duration_ms_histogram_bucket{stage="redis"}`。 | +| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 超过 `capacity-check -realtime-store-p99-ms` 或连续 5 分钟上升 | MySQL 当前态/位置投影变慢，需结合 async queue depth 和 dropped 计数判断。 | +| `vehicle_realtime_config{setting="workers"}` / `vehicle_realtime_worker_active` | 配置低于 `-realtime-workers-min=3` 或任一 worker 不活跃 | 检查 `REALTIME_WORKERS`、Kafka group 分区分配和 MySQL 连接/锁等待。 | +| `vehicle_identity_writer_config{setting="workers"}` / `vehicle_identity_writer_worker_active` | 配置低于 `-identity-workers-min=3` 或任一 worker 不活跃 | 检查 `IDENTITY_WRITER_WORKERS`、JT808 raw 分区分配和 MySQL 连接/锁等待。 | +| `vehicle_realtime_async_queue_total{status="dropped"}` / `status="closed"` | 任意增长 | MySQL 当前态/位置投影的异步二级队列没有接收该帧。`dropped` 表示队列满，优先查 MySQL 写入延迟和 `MYSQL_REALTIME_ASYNC_WORKERS`；`closed` 只应出现在服务退出窗口，退出时会先 drain 已排队数据再关闭 MySQL 连接。 | +| `vehicle_realtime_plate_cache_entries` | 超过 `capacity-check -realtime-plate-cache-entries-max`，默认 `200000` | Realtime API 的 VIN 到车牌缓存超过容量预期。优先确认 `PLATE_CACHE_MAX_ENTRIES`、`PLATE_CACHE_TTL_SECONDS` 和 `vehicle_identity_binding` 的 VIN 质量；若短时间快速增长，通常说明上游出现大量异常 VIN 或历史回放范围过大。 | +| `vehicle_stat_write_duration_ms_histogram_bucket` | p99 超过 `capacity-check -stat-write-p99-ms` 或连续 5 分钟上升 | MySQL 每日里程统计写入变慢，可能导致 stat Kafka lag 增长。 | +| `vehicle_stat_kafka_messages_total{status="invalid_json"}` | 超过 `capacity-check -stat-invalid-json-max`，默认 `0` | Kafka fields topic 中出现无法反序列化的 envelope。stat-writer 会 commit 并隔离该消息，避免阻塞每日里程统计；必须检查 gateway fields 发布、NATS bridge 路由以及是否误把 RAW/非 envelope 数据写入 fields topic。 | +| `vehicle_stat_batch_pending_messages` | 超过 `capacity-check -stat-batch-pending-max`，默认 `1000` | stat-writer 已从 Kafka 拉取一批 fields 消息，但还没完成 MySQL append 和 Kafka commit。短暂非 0 是批处理窗口；持续升高优先查 MySQL 写入延迟、`vehicle_stat_write_duration_ms_histogram_bucket` 和 stat Kafka lag。 | +| `vehicle_stat_cache_entries` / `vehicle_stat_cache_evictions_total` | entries 超过 `capacity-check -stat-cache-entries-max`，默认 `1000000`；evictions 持续增长 | stat-writer 的去重、source touch、投影节流、历史基线缓存达到运行时上限。优先确认 `STATS_CACHE_MAX_ENTRIES`、`STATS_CACHE_RETENTION_HOURS`、`STATS_CACHE_CLEANUP_INTERVAL_SECONDS`，并检查是否有大范围历史回放、异常 VIN/source 或 Kafka 积压恢复。 | +| `vehicle_stat_samples_total{status="found"}` | 单 topic `vehicle_stat_writes_total{status="ok"}` 达到 `capacity-check -stat-sample-min-writes` 但 `found=0` | fields topic 有消费但完全抽不到里程样本，优先检查 VIN 映射、协议字段映射和总里程字段是否进入 fields；该指标带 `protocol` 标签，可直接按 GB32960/JT808/YUTONG_MQTT 聚合。 | +| `vehicle_stat_samples_total{status="skipped_missing_fields"}` | 任意持续增长，或单 topic 可行动跳过数 / `vehicle_stat_writes_total{status="ok"}` 超过 `capacity-check -stat-sample-max-actionable-skip-ratio` | fields topic 收到了 envelope 但没有扁平化 `fields`。优先查 gateway `BuildFieldsEnvelope`、NATS/Kafka bridge topic 映射，以及是否误把 RAW envelope 写进 fields topic。 | +| `vehicle_stat_samples_total{status="skipped_missing_vin"}` / `skipped_missing_mileage` / `skipped_non_positive_mileage` / `skipped_missing_source` | 单 topic 可行动跳过数 / `vehicle_stat_writes_total{status="ok"}` 超过 `capacity-check -stat-sample-max-actionable-skip-ratio` | fields topic 有消费但大量样本无法进入每日里程。按协议和跳过原因分别检查 VIN 映射、总里程字段映射、里程单位/异常值、`vehicle_data_source` 关联。 | +| `vehicle_stat_samples_total{status="event_time_future_adjusted"}` | 连续增长或集中在某协议/source | 设备事件时间比接收时间未来超过 10 分钟，统计已回退到接收时间。优先检查上游平台或终端时钟；RAW 帧仍保留设备原始时间，业务 location、Redis 当前态和每日里程使用归一后的时间。 | +| `vehicle_stat_samples_total{status="written"}` | stat-writer `ok` 增长但 `written` 不增长 | fields topic 有消费但没有有效里程样本入库，按 `skipped_missing_vin`、`skipped_missing_mileage`、`skipped_non_positive_mileage`、`skipped_missing_source`、`skipped_same_mileage` 定位原因；如果主要是 `skipped_same_mileage`，通常表示上游总里程未变化，不等同故障。 | +| `vehicle_stat_sources_total{status="written"}` / `skipped_throttled` | `vehicle_stat_samples_total{status="found"}` 增长但 source tracking 无增长 | 来源管理链路没有维护 `vehicle_data_source`。优先查 source endpoint 是否为空、`vehicle_data_source` 表结构、MySQL 写入错误和 `STATS_SOURCE_TOUCH_INTERVAL_SECONDS`。 | +| `vehicle_stat_sources_total{status="skipped_missing_endpoint"}` | 单 topic 缺 endpoint / found 超过 `capacity-check -stat-source-max-missing-ratio` | fields envelope 缺 `source_endpoint`，无法区分平台来源和维护 source。优先查 Gateway 是否填充 TCP remote/MQTT endpoint，以及 NATS/Kafka bridge 是否保留 envelope 字段。 | +| `vehicle_stat_projections_total{status="written"}` / `skipped_throttled` | source 候选持续写入但最终日里程更新频率低 | `vehicle_daily_mileage_source` 是每样本候选事实层，`vehicle_daily_mileage` 是最终投影层。`skipped_throttled` 增长通常表示受 `STATS_PROJECT_INTERVAL_SECONDS` 节流，不代表样本丢失；如需强一致核验可临时设投影间隔为 `0`。 | +| `vehicle_history_last_*_unix_seconds` / `vehicle_stat_last_*_unix_seconds` / `vehicle_realtime_last_*_unix_seconds` | 已出现过的成功 activity 超过 `capacity-check -last-activity-stale-seconds` 未刷新 | 下游消费、写入或 commit 某 topic 停止活动；结合 Kafka lag 和对应存储健康判断。 | +| `vehicle_stat_project_interval_seconds` | 生产异常为 `0` 或被调得过小 | `vehicle_daily_mileage_source` 仍每样本更新，但 `vehicle_daily_mileage` 选举投影会被节流；过小会放大 MySQL 压力。 | | Kafka lag | 连续 5 分钟增长或 `> 10000` | 下游 consumer 或存储存在瓶颈。 | | Writer 成功计数 | 入口增长但 writer 不增长 | bridge、Kafka、consumer 或存储链路断开。 | @@ -162,7 +350,7 @@ go run ./cmd/load-sim \ 1. 先确认监听端口。 ```bash -ss -lntp | grep -E ':(808|32960|20200|20211|20212|20213|20214) ' +ss -lntp | grep -E ':(808|32960|20200|20211|20212|20213|20214|20215|20216) ' ``` 2. 检查 gateway readiness 和日志。 @@ -218,21 +406,129 @@ df -h / 当前建议值：`NATS_STREAM_MAX_BYTES=21474836480`，即 `20GiB`；`NATS_STREAM_ENSURE_TIMEOUT_SECONDS=60`，避免大 stream 元数据更新时被 NATS 客户端默认 5s 超时误杀。Kafka topic 只作为短期缓冲，当前建议保留 `6h`，不要把 Kafka 或 NATS 当长期历史存储；长期历史和 RAW 查询以 TDengine/MySQL 投影为准。 -fast-writer 的 `FAST_WRITER_OPERATION_TIMEOUT_MS` 建议为 `1000`。实时链路仍以 100ms 级为目标，但 TDengine 批量写存在尾延迟，过小的超时会造成 NATS 消息反复重投和重复写压力。 +fast-writer 的 `FAST_WRITER_OPERATION_TIMEOUT_MS` 建议为 `1000`。生产默认设置 `FAST_WRITER_TDENGINE_ENABLED=false`，让 fast-writer 只写 Redis 实时投影，TDengine RAW/位置由 history-writer 通过 Kafka raw topic 单路写入；如果临时启用 TDengine stage，过小的超时会造成 NATS 消息反复重投和重复写压力。 + +生产 Redis 当前态只由 `nats-fast-writer` 写入；`realtime-writer` 应设置 `REALTIME_ROLE=writer` 和 `REALTIME_REDIS_PROJECTOR_ENABLED=false`，继续从 Kafka raw 写 MySQL snapshot/location；`realtime-api` 应设置 `REALTIME_ROLE=api`，只提供 Redis/MySQL/TDengine 查询 API。若 `capacity-check` 提示 `realtime redis projector enabled`，应先检查 `/opt/lingniu-go-native/env/realtime-writer.env`。 + +stat-writer 默认 `STATS_WORKERS=3`、`STATS_PROJECT_INTERVAL_SECONDS=15`。`vehicle_daily_mileage_source` 每条有效里程样本都会更新，用于保留多源事实和最新总里程；`vehicle_daily_mileage` 是对外查询结果层，会按来源优先级/质量从 source 表投影，投影被节流以降低高频帧下的 MySQL 写放大。需要故障回放或强一致核验时可临时设为 `0` 恢复每样本投影，处理完成后应调回正常值。 + +`vehicle_data_source.latest_seen_at` 表示平台来源最后一次被本系统接收到的时间，stat-writer 优先使用 `received_at_ms` 更新，且 SQL 层禁止该字段被补发、乱序或设备时间异常的帧回退。里程统计日期仍按设备事件时间计算，只有事件时间超出接收时间 `10min` 以上时才回退到接收时间。 + +`vehicle_realtime_snapshot.access_*` 是车辆/协议级接入证据，由 realtime writer 在快照同一条原子 upsert 中维护。`access_latest_received_at` 只按更大的 `received_at_ms` 且不同 event ID 推进，同时把旧值写入 `access_previous_received_at`，计算 `access_report_interval_ms` 并增加 `access_sample_count`；设备事件时间乱序不影响此路径。迁移 `008` 将存量行标为 `snapshot_backfill`，该时间只是上线基线，不得对外描述为历史首次接入。验证上线时至少观察同一 VIN 的样本数递增、间隔非负、首次时间不变，并确认重复/补发帧没有回退最新接收时间。 ### Raw 有数据但实时查不到 -1. 查 realtime Kafka lag。 +1. 查 realtime writer Kafka lag。 2. 查 realtime update 计数。 3. 查 realtime API readiness。 4. 查最新 snapshot/location API。 ```bash +curl -fsS http://127.0.0.1:20216/readyz curl -fsS http://127.0.0.1:20200/readyz curl -fsS 'http://127.0.0.1:20200/api/realtime/locations?limit=1' curl -fsS 'http://127.0.0.1:20200/api/realtime/snapshots?limit=1' ``` +### 更新 JT808 平台映射 + +多平台 808 车牌/手机号映射统一进入 `vehicle_identifier`，不要再直接把不同平台的手机号硬塞进 `vehicle_identity_binding` 的唯一列。导入器会先读取映射文件，再用旧 `vehicle_identity_binding` 的车牌/手机号反查 VIN，只写入可确定 VIN 的记录；冲突和未匹配记录留在 JSON 报告里人工处理。 + +```bash +release=$(basename "$(readlink -f /opt/lingniu-go-native/current)") +input=/opt/lingniu-go-native/imports/${release}/jt808_mapping + +# 先 dry-run，确认 scan.sources、resolved / unresolved / conflicts。 +/opt/lingniu-go-native/current/identity-import \ + -input "${input}" \ + -ensure-schema \ + -unresolved-out "/tmp/${release}-identity-unresolved.csv" \ + -conflicts-out "/tmp/${release}-identity-conflicts.csv" \ + -timeout 5m > /tmp/${release}-identity-dryrun.json + +jq '.scan.sources' /tmp/${release}-identity-dryrun.json +jq '.scan.unsupported_items // []' /tmp/${release}-identity-dryrun.json +jq '.source_results' /tmp/${release}-identity-dryrun.json + +# dry-run 无冲突后再 apply。 +/opt/lingniu-go-native/current/identity-import \ + -input "${input}" \ + -apply \ + -sync-data-sources \ + -timeout 5m > /tmp/${release}-identity-apply.json + +# 只同步来源表时可以不传 input；不带 -apply 只输出候选/跳过数量，不写库。 +/opt/lingniu-go-native/current/identity-import \ + -sync-data-sources \ + -timeout 1m > /tmp/${release}-source-sync-dryrun.json + +/opt/lingniu-go-native/current/identity-import \ + -sync-data-sources \ + -apply \ + -timeout 1m > /tmp/${release}-source-sync-apply.json +``` + +导入后检查： + +- `scan.sources` 是否覆盖全部平台目录；重点看每个 `source_code` 的 `phone_records`、`plate_records`、`skipped`，如果某个平台记录数异常低，先处理源文件表头/列位置，不要直接 apply。 +- `scan.unsupported_items` 必须为空；如果出现 `.xls` 或 `.xlsb`，先转成 `.xlsx` 再导入，避免某个平台文件被跳过后造成 VIN 映射缺口。`identity-import -apply` 会在连接 MySQL/建表前拒绝带 unsupported 文件的目录。 +- `source_results` 是否逐平台呈现合理的 `resolved/unresolved/conflicts`；如果某个平台 `unresolved` 很高，先补 `vehicle_identity_binding` 的 VIN/车牌基础事实；如果 `conflicts` 不为 `0`，先人工确认同一平台下手机号/车牌是否重复指向不同车辆。 +- 同一平台同一手机号/车牌重复出现但不冲突时，导入器会合并非空字段，优先保留可用于 VIN 解析的车牌；如果同一标识对应多个不同车牌，会进入 conflict，不会静默覆盖。 +- `identity-import -apply` 会跳过 unresolved/conflict 项，只写入可确定 VIN 且无冲突的标识；实际写入包在单个 MySQL 事务里，任一写入错误都会回滚本次导入。 +- `vehicle_identifier` 总数和各 `source_code` 分布是否符合文件规模。 +- `vehicle_data_source.source_code/platform_name/source_kind` 会从 `jt808_registration.phone/source_ip` 与 `vehicle_identifier` 推断；同一来源 IP 推断出多个平台时会跳过，且不会覆盖人工维护的平台名。 +- 来源维护 API 支持 `sourceCodeMissing=true` 快速列出未绑定平台编码的来源，例如：`/api/stats/data-sources?protocol=JT808&sourceCodeMissing=true&includeTotal=true`。 +- 来源类型使用 `source_kind` 维护：`PLATFORM` 表示稳定平台源，`DIRECT` 表示车辆直连或动态 IP，`UNKNOWN` 表示未分类。每日里程最终投影在同等质量下按 `PLATFORM -> UNKNOWN -> DIRECT` 选择来源，然后再比较 `trust_priority`、样本数和最新时间。 +- 每日里程 API 会通过 `source_id` 关联 `vehicle_data_source` 返回选中来源证据，例如：`/api/stats/daily-metrics?protocol=JT808&dateFrom=2026-07-12&dateTo=2026-07-12&limit=1`。排查异常里程时同时核对 `source_id`、`source_ip`、`platform_name`、`source_code`、`source_kind` 和 `latest_total_mileage_km`，不要只看 `daily_mileage_km`。 +- 每日里程候选来源 API 用于审计多源选举，例如：`/api/stats/daily-metrics/sources?protocol=JT808&dateFrom=2026-07-12&dateTo=2026-07-12&selected=true&limit=1`。它返回 `source_key`、`phone`、`sample_count`、`first_total_mileage_km`、`latest_total_mileage_km`、`quality_status`、`quality_reason`、`is_selected`，用于确认每个来源独立算出的里程以及最终是否被选中。`first_event_time` 可能是跨日前推得到的历史基线时间，排查时应结合 `quality_reason` 理解。 +- 每日里程诊断 API 用于排查“实时在线但统计缺失”，例如：`/api/stats/daily-metrics/diagnostics?protocol=YUTONG_MQTT&date=2026-07-12&diagnosis=NO_SOURCE_SAMPLE&includeTotal=true`。它会从 `vehicle_realtime_snapshot/location` 找当天活跃车辆，再关联最终表和候选来源表输出 `diagnosis`：`OK` 表示最终日里程已存在；`MISSING_DAILY` 表示候选来源已有样本但最终投影缺失；`NO_SOURCE_SAMPLE` 表示当天确实收到总里程字段但 stat-writer 没抽到候选样本；`NO_TOTAL_MILEAGE` 表示当天实时活跃但没有收到总里程字段。宇通 MQTT 可能稀疏上报，`vehicle_realtime_location.total_mileage_km` 会保留旧值，判断当天是否真实上报要看 `total_mileage_event_time`。 +- `NO_TOTAL_MILEAGE` 的 `reason` 会进一步说明源头问题：`realtime_total_mileage_missing` 表示当前态没有总里程；`realtime_total_mileage_non_positive` 表示源头总里程为 0 或负数；`realtime_total_mileage_time_missing` 表示当前态有总里程旧值但缺少该字段的上报时间证据；`realtime_total_mileage_not_reported_on_stat_date` 表示当前态保留了旧总里程，但指定业务日期没有重新上报总里程字段。 +- `vehicle_realtime_snapshot.parsed_json` 是稀疏帧合并后的当前态，字段存在只证明历史上曾上报，不能证明当前帧或统计日上报。字段诊断返回 `mapped_protocol_field_without_fresh_evidence` 时应按 `raw_frame_query_path` 核对原始帧；只有 `vehicle_realtime_location.total_mileage_event_time` 落在统计日内，才把标准总里程视为新鲜证据。 +- 每日里程诊断汇总 API 用于大屏和告警入口，例如：`/api/stats/daily-metrics/diagnostics/summary?date=2026-07-12`。它按协议返回 `active_count`、`ok_count`、`missing_daily_count`、`no_source_sample_count`、`no_total_mileage_count` 和 `actionable_issue_count`；健康状态下应优先关注 `actionable_issue_count=0`，异常时再下钻到诊断明细接口。 +- 每日里程诊断原因汇总 API 用于告警聚合和运营看板，例如：`/api/stats/daily-metrics/diagnostics/reasons?date=2026-07-12`。它按 `protocol + diagnosis + reason` 返回车辆数；需要只看某类问题时可传 `diagnosis=NO_TOTAL_MILEAGE`，比拉取逐车明细更适合高频轮询。 +- `stats-backfill` 用于从 TDengine RAW 证据层补算 MySQL 每日里程。ECS 上直接执行会默认加载 `/opt/lingniu-go-native/env/base.env` 和 `/opt/lingniu-go-native/env/stat-writer.env`；默认 `BACKFILL_DRY_RUN=true`，不会写库。补算采用 `BACKFILL_METHOD=last_diff`：对每个协议、VIN、来源分别取当日最新累计总里程和同一来源在目标日前最近一次累计总里程，按 `当日最新 - 最近历史总里程` 计算。补算窗口首日会聚合目标日前的历史并取 `LAST(event_time)`；因此前一日缺失时会继续向更早日期寻找，历史完全为空时才使用当天第一条作基线。JT808 历史补算结束后会按 `source_code` 归并平台来源键，保证历史与实时统计使用同一来源。正式写入前先 dry-run，例如： + +```bash +BACKFILL_PROTOCOLS=YUTONG_MQTT \ +BACKFILL_DATE_FROM=2026-07-12 \ +BACKFILL_DATE_TO=2026-07-12 \ +/opt/lingniu-go-native/current/stats-backfill + +BACKFILL_DRY_RUN=false \ +BACKFILL_PROTOCOLS=GB32960,JT808,YUTONG_MQTT \ +BACKFILL_DATE_FROM=2026-07-12 \ +BACKFILL_DATE_TO=2026-07-12 \ +/opt/lingniu-go-native/current/stats-backfill +``` + +生产定时补算使用 systemd timer，默认每天 01:30 补算昨天往前 3 天，覆盖上游晚到或断传后恢复的总里程字段： + +```bash +cd go/vehicle-gateway + +scripts/install-stats-backfill-timer.sh \ + --host root@115.29.187.205 \ + --days-back 1 \ + --window-days 3 \ + --on-calendar '*-*-* 01:30:00' + +systemctl list-timers lingniu-go-stats-backfill.timer --no-pager +systemctl cat lingniu-go-stats-backfill.service lingniu-go-stats-backfill.timer --no-pager +``` + +如果只想验证定时任务配置不写库，可以加 `--dry-run` 安装；正式生产不加 `--dry-run`，服务内会设置 `BACKFILL_DRY_RUN=false`。timer 设置 `Persistent=false`，避免首次安装时立刻执行当天已错过的计划；即使 ECS 短暂停机，下一次 3 天窗口也会补到遗漏日期。 + +- 诊断 API 判断“指定日期活跃”时会同时看 `event_time`、`received_at`、`updated_at`，任一时间落在业务日期内即纳入；这样设备时间漂移或未来时间不会遮蔽服务端当天实际收到的数据。 +- 每日里程质量规则会把 1km 内的小幅负漂移按 0km 处理，`quality_reason=negative_jitter_clamped`。优先用前一自然日同来源末值；前一日缺失时继续向前查找最近历史日末值，并按“2500km × 两个累计里程端点相隔自然日数”放大物理合理性上限，但仍把完整差值记在当前统计日，不平均分摊或伪造中间日期。该上限覆盖车辆以约 100km/h 连续运行 24 小时的极端场景，同时继续拒绝约 4000km/日的异常跳变。超过窗口上限的正向突增或超过 1km 的倒退是 `INVALID_DELTA`，不会进入最终 `vehicle_daily_mileage`；历史完全为空时使用当天第一条。如果设备事件时间比接收时间未来超过 10 分钟，业务层会统一使用接收时间；TDengine RAW 帧仍保留设备原始时间作为证据。 +- 来源诊断 API 支持按来源 IP 汇总注册手机号和 `vehicle_identifier` 匹配情况，例如：`/api/stats/data-sources/diagnostics?protocol=JT808&sourceCodeMissing=true&includeTotal=true`。`reason=no_identifier_match` 表示该来源下手机号还没有进入 `vehicle_identifier`；`reason=ambiguous_source_code` 表示同一来源 IP 匹配到多个平台编码，需要人工确认；`reason=candidate_available` 表示可以同步或确认候选 `source_code`。 +- 来源类型建议 API 是只读辅助，例如：`/api/stats/data-sources/kind-suggestions?protocol=JT808&sourceKind=UNKNOWN&includeTotal=true`。它会输出 `suggested_source_kind`、`suggestion_confidence` 和 `suggestion_reason`，用于人工确认后再 PATCH `source_kind`；不要把建议结果无审核地批量写回。 +- `source_code` 用于程序稳定识别来源，`platform_name` 用于展示和人工修正；如果两者看起来不一致，优先核对该来源 IP 下的 `jt808_registration.phone` 是否来自同一平台，再决定是否人工修正平台名。 +- Gateway 默认启用 `IDENTITY_SOURCE_CODE_LOOKUP_ENABLED=true`。808 VIN 解析会先用 `source_endpoint -> vehicle_data_source.source_code` 做 `vehicle_identifier` 精确匹配，精确未命中时再回退到全局 `vehicle_identifier`、旧 `vehicle_identity_binding` 和 `jt808_registration`，因此多平台同一手机号/车牌映射不同 VIN 时，应优先维护正确的 `vehicle_data_source.source_code`。 +- `jt808_registration` 中 `vin = 'unknown'` 的手机号是否仍能在 `vehicle_identifier` 中匹配；如果能匹配，等待下一帧 808 位置或注册/鉴权帧触发自动更新。 +- Gateway `vehicle_gateway_identity_total{protocol="JT808",status="resolved"}` 应持续增长。 + +Gateway 默认启用 `IDENTITY_SNAPSHOT_ONLY_ENABLED=true`，启动时并每隔 `IDENTITY_SNAPSHOT_REFRESH_INTERVAL_SECONDS=60` 秒把 `vehicle_data_source`、`vehicle_identifier`、旧 `vehicle_identity_binding` 和 `jt808_registration` 原子载入内存。快照包含 JT808 的 VIN、设备、车牌和最近可信鉴权码；每帧身份解析及鉴权均不查询 MySQL。注册/鉴权/位置触达由独立 identity-writer 投影写入 `jt808_registration`，人工改库或新鉴权码最多等待一个刷新周期即可生效。快照刷新失败保留上一版，启动时 RDS 不可用也不阻止接入；未知 phone 会暂时 unresolved，RDS 恢复并完成刷新后自动解析。 + ## 重启顺序 优先重启最小故障层： diff --git a/docs/vehicle-data-platform-analysis.md b/docs/vehicle-data-platform-analysis.md new file mode 100644 index 00000000..72fdbf72 --- /dev/null +++ b/docs/vehicle-data-platform-analysis.md @@ -0,0 +1,231 @@ +# 车辆数据中台一期现状分析与总体方案 + +> 结论日期：2026-07-14 +> 范围：只完成现状分析和方案设计，不进入 V2 功能开发或 ECS 部署。 +> 代码依据：`/Users/lingniu/project/ai-coding/ln-bi`、`vehicle-data-platform`、`go/vehicle-gateway`，以及 `docs/architecture`、`docs/ops` 中的生产说明。 + +## 1. 结论摘要 + +当前数据接入链路已经具备可用的生产底座：GB32960、JT808、YUTONG_MQTT 经 NATS/Kafka 分流后，分别形成 Redis 当前态、TDengine RAW/位置历史、MySQL 身份/实时/日里程投影；现有平台 BFF 也已提供车辆、实时位置、轨迹点、RAW、里程、质量和运维健康等只读接口。 + +但现有前端不是可直接扩展的一期产品：生产入口加载的是约 6000 行的 `PrototypeApp.tsx`，仍混有 mock 数据和“接口缺失时展示能力边界”的原型逻辑；另一个模块化 `App.tsx` 并非当前入口。现有地图每次最多创建 500 个普通 Marker，没有聚合、MassMarks/Canvas 图层和视口查询，无法满足 1 万辆级监控。所谓告警接口实际上是质量问题接口的别名，没有规则、事件、处理记录和通知持久化。 + +建议丢弃现有原型页面的产品实现，但保留经验证的数据适配、领域工具、测试、运行时配置和部署脚手架，建立独立 V2。前端沿用 React + TypeScript + Vite，吸收 `ln-bi` 的视觉语言、壳层、鉴权思想和基础组件，但采用正式路由、查询缓存、虚拟表格和独立地图 SDK 层。后端保留现有接入链路，通过平台 BFF 和新增的告警/导出 worker 补齐业务能力，不让浏览器直接访问 Redis、TDengine 或 MySQL。 + +## 2. 当前前端技术栈与目录 + +### 2.1 `ln-bi` 参考项目 + +| 项目 | 当前实现 | 评价 | +| --- | --- | --- | +| 框架 | React 19、React DOM 19、TypeScript 5.8、Vite 6.2 | 可作为 V2 目标栈参考；不建议仅为对齐版本立刻升级已有平台依赖 | +| 样式 | Tailwind CSS 4、少量 CSS 变量 | 视觉语言清晰，但业务组件 class 较分散，需要提炼 token 和组件配方 | +| 图标/动效 | lucide-react、Motion | 可复用设计语言；监控页面动效应克制并支持 reduced motion | +| 图表 | Recharts 3.8 | 适合常规 BI；百万点、多 Y 轴时序更推荐现平台已有 ECharts 5 | +| 地图 | `@amap/amap-jsapi-loader` + AMap JS API 2.0 | 加载和实例生命周期可借鉴；目前只是热力图业务组件，不是通用地图 SDK | +| API | 原生 `fetch` + JWT 注入 | 可借鉴鉴权流程；错误类型、超时、重试、取消、缓存仍不足 | +| 服务端 | Hono、MySQL/Postgres、JWT、XLSX | 属于 `ln-bi` 自身 BFF，不应直接搬入车辆接入 Go 数据面 | +| 路由 | pathname + hash + `replaceState` 手写 | 适合少量模块，不适合可分享查询、详情层级和权限路由 | +| 状态 | React 本地状态/Context | 未使用全局状态库或服务端查询缓存 | + +主要目录： + +```text +ln-bi/src/ +├── App.tsx # 模块注册、权限门禁、懒加载 +├── auth/ # jumpToken -> JWT、fetch token 注入 +├── components/ +│ ├── Shell.tsx # 桌面侧栏、移动底栏、模块切换、水印 +│ ├── SearchSelect.tsx +│ ├── MultiSearchSelect.tsx +│ └── ui/surface.tsx # PageFrame、SurfaceCard、MetricTile、状态组件 +├── modules/ +│ ├── vehicle-heatmap/ # AMap 热力图和筛选/详情面板 +│ ├── hydrogen-heatmap/ +│ ├── mileage/ # 表格、详情弹窗、XLSX 导出 +│ └── ... +├── server/ # Hono BFF、认证、DB 查询 +├── shared/auth/roles.ts # 角色判断 +└── index.css # Tailwind、主题变量、地图控件样式 +``` + +### 2.2 当前车辆平台前端 + +目录为 `vehicle-data-platform/apps/web`，使用 React 18.3、TypeScript 5.7、Vite 6、Semi UI 2.71、ECharts 5.6。当前入口 `src/main.tsx` 加载 `PrototypeApp`，不是 `src/App.tsx`。 + +```text +vehicle-data-platform/apps/web/src/ +├── main.tsx # 当前生产入口 -> PrototypeApp +├── PrototypeApp.tsx # 大型原型单体，真实接口与 mock 逻辑混合 +├── App.tsx # 未作为入口的模块化旧实现 +├── api/ # 统一响应信封、类型和调用方法 +├── components/ # VehicleMap、状态标签、空状态等 +├── config/ # API/高德运行时配置 +├── domain/ # 路由、导出、车辆查询等纯函数 +├── integrations/amap.ts # AMap loader、坐标校验、逆地理编码 +├── layout/AppShell.tsx +├── pages/ # 旧模块化页面 +├── prototype/ # mock、真实数据适配、view model、字段映射 +└── styles/ # token、全局样式、原型样式 +``` + +V2 的原则是“页面重建、能力甄别复用”：不复制 `PrototypeApp.tsx` 和 `prototype.css`；保留可独立测试的 `api`、`domain`、`integrations`、真实数据归一化逻辑及其测试，再按新接口契约重构。 + +## 3. 可复用能力清单 + +| 能力 | 来源 | 复用决策 | 说明 | +| --- | --- | --- | --- | +| 80px 深色侧栏、顶部面包屑、水印、移动底栏 | `ln-bi/components/Shell.tsx` | 设计复用、代码重构 | V2 桌面优先，应改为正式路由和可折叠侧栏 | +| PageFrame、SurfaceCard、MetricTile、SegmentedNav | `ln-bi/components/ui/surface.tsx` | 可迁移后组件化 | 统一 token、尺寸和可访问性后复用 | +| SearchSelect、MultiSearchSelect | `ln-bi/components` | 交互参考 | 当前仅接收字符串数组，缺少远程搜索、虚拟列表、label/value 和键盘完整性 | +| 加载、空、错误状态 | 两项目 | 合并重构 | 建立统一 `AsyncState`，禁止页面各自拼接 | +| JWT/jumpToken 门禁、角色判断 | `ln-bi/auth`、`shared/auth` | 复用认证协议思想 | token 仍应保存在 session；权限改为路由/操作级声明，后端必须二次校验 | +| AMap loader、实例销毁、运行时密钥 | 两项目 | 合并为通用 SDK | 统一 loader promise、插件注册、安全代理和错误状态 | +| 热力图数据归一化 | `ln-bi/vehicle-heatmap` | 可复用算法 | `log1p/sqrt` 强度变换适合密度层，不等于车辆状态点图层 | +| 表格/详情弹窗/XLSX | `ln-bi/mileage` | 交互与导出格式参考 | 一期大数据导出必须后端异步，不能沿用浏览器全量 XLSX | +| ECharts | 当前车辆平台 | 保留 | 更适合多 Y 轴、dataZoom、断点、阶梯线和大数据采样 | +| `api/client.ts` 响应信封和 traceId | 当前车辆平台 | 扩展复用 | 增加鉴权、AbortSignal、超时、错误码、幂等键和查询缓存 | +| 车辆 lookup、字段归一化、CSV 纯函数及测试 | 当前车辆平台 | 审核后复用 | 去掉 mock fallback，统一到 V2 DTO/领域模型 | +| Semi UI 页面组件 | 当前车辆平台 | 暂不作为 V2 默认 | 与 `ln-bi` Tailwind 视觉体系并存会形成双设计系统；V2 启动前只选一套基础组件策略 | + +不存在可直接复用的“通用图表组件”或“通用高性能表格组件”。`ln-bi` 的 Recharts 与页面耦合，当前平台的 ECharts 也需要建立 `TimeSeriesChart`、轴/单位/空值策略和采样契约；表格需新增服务端分页、列配置、固定列和虚拟滚动封装。 + +## 4. 高德地图当前封装方式 + +### 4.1 `ln-bi` + +`AmapHeatmapCanvas.tsx` 动态导入 `@amap/amap-jsapi-loader`，写入 `_AMapSecurityConfig.securityJsCode`，加载 AMap 2.0 的 HeatMap、ToolBar、Scale 插件；创建白色地图和 HeatMap，点击地图回传经纬度，数据变化时调用 `setDataSet`，焦点变化时计算 bounds，卸载时销毁实例。 + +优点是 loader 官方、生命周期完整、地图与侧面板分屏清楚。限制是车辆和加氢模块各有近似实现，密钥直接下发前端，未封装 Marker/聚合/信息窗/轨迹/播放，也没有实例共享和视口事件节流。 + +### 4.2 当前车辆平台 + +`integrations/amap.ts` 自行加载 `https://webapi.amap.com/loader.js`，按插件集合缓存 Promise，支持 Scale、Geocoder、坐标合法性检查、URI Marker 链接和浏览器逆地理编码；`appConfig.ts` 从 `window.__LINGNIU_APP_CONFIG__` 或 Vite 环境变量读取 Web JS key、安全代码/代理和 API base URL。 + +`VehicleMap.tsx` 在组件内创建 Map、Marker、Polyline，重绘时清除 overlay，最多截取 500 个点，并在未配置地图时提供坐标预览。它适合验证和小规模页面，不适合一期：每点 HTML button、没有聚合、状态图层、InfoWindow 管理、车辆朝向、轨迹抽稀/播放或视口查询。 + +### 4.3 V2 建议封装 + +建立 `features/map-sdk`，分为： + +- `AMapProvider/useAMap`：唯一 loader、插件按需加载、实例注册、resize/destroy。 +- `BaseMap`：纯地图容器，只接收中心、缩放、样式和事件。 +- `VehiclePointLayer`：小规模用 Marker，规模化优先 MarkerCluster/LabelsLayer/MassMarks 或 Canvas/WebGL 能力；详情卡只保留一个 InfoWindow。 +- `TrackLayer`：Polyline、起终点、停车/告警节点和移动标记；播放状态在业务 hook，不绑入地图实例。 +- `HeatmapLayer`、`GeocoderService`、`MapControls`：独立能力。 +- 视口查询使用 `bounds + zoom + filterHash`；移动结束后防抖请求服务器聚合结果，前端不一次拉取全量 1 万点 DOM。 + +高德 Web JS key 可由运行时配置下发，但安全密钥优先使用服务端安全代理；逆地理编码优先走现有 `/api/map/reverse-geocode`，避免泄露 REST key并便于限流/缓存。 + +## 5. 当前车辆相关接口 + +现有平台 BFF 的有效能力如下；详细缺口见 `vehicle-data-platform-api-gap.md`。 + +| 领域 | 接口 | 当前用途 | +| --- | --- | --- | +| 总览 | `GET /api/dashboard/summary` | 在线、今日活跃、帧数、问题数、协议/服务/链路统计 | +| 车辆 | `GET /api/vehicles`、`/api/vehicles/resolve`、`/api/vehicles/coverage`、`/summary` | 车辆列表、VIN/车牌/手机号解析、来源覆盖 | +| 单车 | `GET /api/vehicle-service`、`/summary`、`/overview`；`POST /overviews` | 聚合身份、实时、历史、RAW、里程、质量证据 | +| 实时 | `GET /api/realtime/vehicles`、`/api/realtime/locations` | VIN 聚合实时状态和每协议最新位置 | +| 历史 | `GET /api/history/locations`、`GET/POST /api/history/raw-frames` | TDengine 位置点和 RAW 证据分页 | +| 里程 | `GET /api/mileage/summary`、`/api/mileage/daily` | MySQL 日里程统计 | +| 在线 | `GET /api/statistics/online-summary`、`/online-vehicles` | 当前固定口径的在线统计和车辆状态 | +| 数据质量 | `GET /api/quality/summary`、`/issues`、`/notification-plan` | 由当前数据推导的质量信号和静态通知方案 | +| 告警兼容 | `GET /api/alert-events/summary`、`/alert-events`、`/notification-plan` | 实际是上述质量接口别名，不是业务告警事件 | +| 地图 | `GET /api/map/reverse-geocode` | 服务端高德逆地理编码 | +| 运维 | `GET /api/ops/health`、`/source-readiness` | 数据源可写性、Kafka lag、连接/Redis key、发布版本和接入准备度 | + +同时，Go `realtime-api` 还提供面向底层表的查询接口和 OpenAPI，但 V2 浏览器应统一访问平台 BFF，避免前端绑定多个端口与底层存储模型。 + +## 6. 数据职责概览 + +- TDengine `lingniu_vehicle_ts`：`raw_frames` 是原始接收与扁平解析证据，`raw_frame_payload_chunks` 保存超长 payload，`vehicle_locations` 保存 VIN 级位置、速度、方向、告警标志和总里程历史。它不是车辆档案或告警业务库。 +- Redis DB 50：`vehicle:latest:*`、`vehicle:realtime-raw:*`、`vehicle:rt-kv:*`、`vehicle:online:*`、`vehicle:protocols:*`、`vehicle:last_seen` 提供可过期、可重建的当前态。它不能作为历史、告警或导出任务事实源。 +- MySQL `lingniu_vehicle_data`：保存 `vehicle`、多标识映射、JT808 注册鉴权、每协议实时快照/位置、来源配置和日里程事实/结果。现有 `vehicle` 档案很轻，只含 VIN、车牌、OEM、启用状态，尚不足以支撑车型、公司、车辆类型等产品字段。 + +统一模型及新增模型详见 `vehicle-data-platform-data-model.md`。 + +## 7. 一期缺口结论 + +必须新增或升级的核心能力： + +1. 完整车辆档案、组织/车型/协议/接入厂家字典与首次接入事实。 +2. 地图聚合查询、组合筛选、状态统计和可配置在线口径。 +3. 包含方向、SOC、告警标志、停车点、异常点过滤和抽稀元数据的轨迹接口。 +4. 动态指标目录，以及多车、多指标、聚合/抽稀、分页的通用时序查询。 +5. MySQL 持久化告警规则、告警事件、处理记录、站内通知和告警计算 worker。 +6. MySQL 异步导出任务、对象/本地文件存储、worker、进度与下载授权。 +7. 接入状态字段：首次/最近上报、事件时间与接收时间延迟、最近消息类型、最近错误、动态在线阈值。 +8. 真正的认证、菜单/操作权限审计；现车辆平台 BFF 尚未呈现完整权限体系。 + +## 8. 推荐一期路由与菜单 + +```text +/ +├── /monitor 全局监控（默认首页） +├── /vehicles 车辆查询 +│ └── /vehicles/:vin 单车数字档案 +├── /tracks 轨迹查询与回放 +├── /history 历史数据分析 +│ └── /history/exports 导出任务 +├── /alerts +│ ├── /alerts/events 告警事件 +│ ├── /alerts/rules 告警规则 +│ └── /alerts/inbox 站内通知 +├── /access 车辆接入状态 +└── /operations 运维质量（管理员） +``` + +侧栏一级菜单保持 6 个业务入口：全局监控、车辆中心、轨迹回放、历史分析、告警中心、接入管理；导出任务挂在历史分析二级菜单，运维质量按权限置底。详情、告警、轨迹之间用带 `returnTo` 的上下文跳转，查询条件同步 URL，浏览器返回可恢复筛选、时间和选中车辆。 + +## 9. 推荐前后端整体架构 + +```mermaid +flowchart LR + UI["V2 React Web"] --> BFF["Platform API / BFF"] + BFF --> RT["Redis 当前态"] + BFF --> TS["TDengine 时序与 RAW"] + BFF --> DB["MySQL 业务事实"] + BFF --> MAP["高德 REST / 安全代理"] + ING["Gateway + NATS + Kafka Writers"] --> RT + ING --> TS + ING --> DB + K["Kafka fields/raw"] --> AW["Alert evaluator"] + AW --> DB + BFF --> EQ["Export queue"] + EQ --> EW["Export worker"] + EW --> TS + EW --> DB + EW --> FS["文件/对象存储"] +``` + +前端按 `app/router`、`shared`、`entities/vehicle`、`features`、`pages` 分层。服务端状态使用查询缓存库统一取消、去重、失效和后台刷新；少量 UI 状态用 React Context/局部 store。地图、图表、表格均只接收领域 DTO，不直接处理协议字段名。 + +平台 BFF 负责统一鉴权、参数校验、分页、在线状态计算、跨源聚合、字段目录、限流、traceId 和 DTO；禁止让前端分别拼 Redis/MySQL/TDengine。已有接入 writer 保持不变，新告警 worker 消费可重放 fields/raw 流，异步导出 worker 读取 TDengine/MySQL 并把任务状态写 MySQL。 + +## 10. 运维与最终 ECS 部署边界 + +已查阅 `docs/ops/vehicle-ingest-runbook.md`、`docs/ops/go-vehicle-ingest-memory.md`、`docs/architecture/production-data-plane-inventory.md` 和 `vehicle-data-platform/docs/deployment.md`。当前接入数据面采用 systemd 裸机 release 目录与 `current` 软链，平台使用 `/opt/lingniu-vehicle-platform/{releases,current,env}`，HTTP 端口 20300；部署后应验证 `/api/ops/health`、车辆查询和运行时配置。 + +V2 最终仍部署到当前 ECS，但不应在分析阶段执行。实施完成后的发布顺序应是：数据库向前兼容迁移 -> 新 worker（默认禁用规则）-> BFF -> 静态 Web -> 小流量健康验证 -> 启用告警/导出 worker。每一步保留旧 release 和可回滚软链；密钥只进 ECS 环境文件，不进仓库或构建产物。完整发布验收纳入路线图最后阶段。 + +## 11. 风险与待确认事项 + +| 优先级 | 风险/待确认 | 影响与建议 | +| --- | --- | --- | +| P0 | `vehicle` 档案缺车型、类型、公司、接入厂家等 | 先确定主数据来源和维护责任，不要从实时 JSON 猜档案 | +| P0 | 现有告警接口名与真实语义不符 | V2 使用 `/api/v2/alerts/*` 新契约，旧别名标记 deprecated，避免误把质量信号当已处理事件 | +| P0 | 动态指标没有目录、单位、类型和协议映射 | 先落指标元数据；否则图表、规则和导出会各自硬编码 | +| P0 | 用户/角色来源尚未确认 | 明确复用 `ln-bi` jumpToken/JWT，还是接入统一 SSO；后端权限不可只靠菜单隐藏 | +| P1 | 在线阈值是全局、协议级还是车辆级 | 建议全局默认 + 协议覆盖；一期暂不做单车覆盖 | +| P1 | 轨迹异常过滤可能隐藏原始证据 | API 同时返回 raw/filtered 计数和算法版本，允许关闭过滤 | +| P1 | TDengine `raw_frames.parsed_json` 是动态字段唯一历史来源 | 通用指标查询先从 JSON 提取会有成本；高频稳定指标应按使用量逐步物化，而非一期全列化 | +| P1 | 单 ECS 同时承担接入、查询、告警和导出 | 导出/重查询必须有并发、时间范围、行数和资源配额；压测后再定 worker 并发 | +| P1 | 地图 1 万辆的插件/授权与浏览器性能 | 先完成 1k/10k 基准，确定 MarkerCluster、MassMarks 或 Canvas 方案和高德配额 | +| P1 | 站内通知实时方式 | 一期可 SSE + 轮询降级；若 ECS 反代不适合长连接，先使用增量轮询 | +| P2 | Semi UI 与 Tailwind 双设计系统 | V2 启动前确认唯一基础组件策略；建议 Tailwind token + 无样式/轻量基础组件，ECharts 保留 | +| P2 | 导出文件存储位置与保留期 | 确认 OSS 或 ECS 本地盘；推荐 OSS/兼容对象存储，任务和文件设过期清理策略 | + +## 12. 本阶段完成定义 + +本阶段仅交付本分析、路线图、API 缺口和数据模型四份文档。V2 代码、数据库迁移、告警/导出 worker、ECS 发布均须在确认 P0 项后按路线图进入下一阶段。 diff --git a/docs/vehicle-data-platform-api-gap.md b/docs/vehicle-data-platform-api-gap.md new file mode 100644 index 00000000..e9ecbd85 --- /dev/null +++ b/docs/vehicle-data-platform-api-gap.md @@ -0,0 +1,153 @@ +# 车辆数据中台一期 API 能力与缺口 + +## 1. 判定规则 + +- “已有”表示当前平台 BFF 已有路由和生产 Store 实现。 +- “部分”表示可作为底层数据来源，但 DTO、筛选、性能或业务语义不足。 +- “缺失”表示当前没有可支撑验收的持久化/API 链路。 +- 新接口建议统一放 `/api/v2`；旧 `/api/alert-events*` 是质量接口别名，不得作为新告警契约继续扩展。 + +## 2. 现有接口清单 + +| 方法与路径 | 状态 | 数据源/说明 | +| --- | --- | --- | +| `GET /api/dashboard/summary` | 已有 | MySQL/运维探针聚合；口径偏现有服务健康 | +| `GET /api/vehicles` | 已有 | 车辆身份与实时快照集合，支持关键词/协议等基础过滤 | +| `GET /api/vehicles/resolve` | 已有 | 通过 VIN、车牌、手机号解析车辆 | +| `GET /api/vehicles/coverage`、`/summary` | 已有 | 多协议来源覆盖、绑定和在线来源数 | +| `GET /api/vehicle-service`、`/summary`、`/overview` | 已有 | 单车/车队证据聚合 | +| `POST /api/vehicle-service/overviews` | 已有 | 批量关键词概览 | +| `GET /api/realtime/vehicles` | 已有 | VIN 级聚合实时状态 | +| `GET /api/realtime/locations` | 已有 | MySQL 每协议最新位置 | +| `GET /api/history/locations` | 已有 | TDengine 位置历史；基础分页和时间过滤 | +| `GET /api/history/raw-frames` | 已有 | TDengine RAW；默认 100 条，可选解析字段 | +| `POST /api/history/raw-frames/query` | 已有 | 复杂 RAW 查询 | +| `GET /api/mileage/summary`、`/daily` | 已有 | MySQL 日里程 | +| `GET /api/statistics/online-summary`、`/online-vehicles` | 已有 | 固定口径在线统计 | +| `GET /api/quality/summary`、`/issues`、`/notification-plan` | 已有 | 推导质量信号和静态方案，不持久化 | +| `GET /api/alert-events*` | 语义错误 | 上述质量接口别名，不是告警规则/事件/通知 | +| `GET /api/map/reverse-geocode` | 已有 | 服务端高德逆地理编码 | +| `GET /api/ops/health`、`/source-readiness` | 已有 | 链路、release、lag、连接、可写性和来源准备度 | + +## 3. 按一期功能的缺口矩阵 + +| 功能 | 当前可复用 | 仍缺少 | 优先级 | +| --- | --- | --- | --- | +| 全局统计 | V2 monitor summary 已实现接入/在线/离线/行驶/静止/未知/今日上报，以及活跃业务告警 VIN 与同筛选车辆集合交集后的权威告警车辆数；页面仅在服务端明确可用时展示数值 | 更复杂的车型/企业/接入商组合字典可在车辆规模增长时从接入管理筛选下沉复用 | P0（一期完成） | +| 地图车辆 | V2 monitor map 已实现 bounds/zoom、关键词/协议/状态筛选、低缩放聚合、高缩放轻量点、2,000 点自适应降级；前端独立缓存、防抖请求和 MassMarks，列表上限 200 | 差量 cursor/SSE 作为车辆规模或刷新频率继续增长后的扩展项 | P0（一期完成） | +| 车辆信息卡 | 选中车辆按需并发组合 realtime、vehicle service、daily mileage、服务端逆地理编码和 `status=active` 告警；展示当日里程、地址、全部真实来源、接入厂家、当前告警及三处深链 | 权威档案未提供接入厂家时明确显示“待补充”，不从协议猜测 | P0（一期完成） | +| 单车档案 | vehicle service 已合并 `vehicle_profile`；管理员可单车维护或用 CSV/API 批量同步，外部同步具备来源/版本幂等、人工及异源保护、显式接管、dry-run、身份校验和版本审计 | 仍需为具体车厂/GPS 厂商配置定时连接器；首次接入/运行时长的自动投影仍待网关权威源 | P0（部分完成） | +| 最新遥测 | `GET /api/v2/vehicles/:vin/telemetry/latest` 已服务端选择每个统一指标/源字段最新值，动态分组，保留厂家扩展，并返回中文名、单位、类型、设备/接收时间、帧、协议、端点、freshness/delay 与值级质量原因；页面独立缓存刷新且不再猜测 RAW 元数据 | 更多动态 RAW 指标进入统一目录仍需受控发现和管理端审计，不影响一期最新值取证 | P0（一期完成） | +| 轨迹 | V2 playback 已有覆盖边界、最多 7 天校验、GPS 质量过滤、推断停车、活动分段、关键点保留抽稀、方向/SOC/告警同步字段和统计元数据 | ignition 可信怠速、持久 queryId/游标分段仍缺少 | P0（部分完成） | +| 回放地址 | 暂停点按坐标缓存逆地理编码，播放时暂停解析，避免逐点请求 | 服务端批量地址目录仅在未来需要轨迹点地址表时再建设 | P1（一期完成） | +| 历史列表/曲线 | V2 动态指标目录、多车多指标列表、服务端分页、按单位拆轴的时间序列聚合、空窗/覆盖率/异常证据和 60–600 点受控曲线已上线 | 更多可聚合遥测指标需随统一指标目录逐项开放；RAW 保持离散证据 | P0（一期完成） | +| 导出 | 单 ECS 持久异步 CSV 队列已上线：进度/状态/完成时间/下载、单并发、31 天/5 车/32 指标/100 万行/30 分钟配额、游标流式写入、重启恢复和原子文件发布 | XLSX、取消和多实例对象存储属于扩容项，不阻塞一期 CSV 验收 | P0（一期完成） | +| 告警规则 | V2 MySQL 规则、版本审计、CRUD/启停/校验、统一指标目录动态校验、数值区间内外、布尔状态变化、协议/VIN/OEM/车型/企业范围、持续/恢复/重复抑制已实现 | 软删除、复杂范围组合预估和批量规则导入仍待补齐 | P0（部分完成） | +| 告警事件 | V2 持久事件、事件时间持续候选、活跃指纹去重、恢复/处置时间线、通知、Kafka fields active consumer、规则副作用与 checkpoint 同事务、数据库权威重放抑制、迟到门禁、动态/墙钟所有权拆分、真实规则灰度、锁竞争及连续运行门禁均已上线 | 跨窗口历史重算属于后续分析能力；短信/邮件/企微供应商明确不在一期范围 | P0（一期完成） | +| 站内通知 | V2 列表、未读数、批量已读和页面轮询已实现；外部通道明确为 reserved | SSE 可在通知规模增长时替换轮询；外部供应商不在一期范围 | P1（一期完成） | +| 接入状态 | V2 summary/vehicles/thresholds 已实现动态阈值、事件/接收延迟、状态区分、协议/车辆厂家/车型/接入厂家/首次接入/最新上报筛选、同口径统计和版本审计；网关同一快照 upsert 已维护首次/前次/最新接收、连续间隔、样本数及回填边界；缺 VIN 的 JT808 终端已进入脱敏处置队列 | 仍需运维核对来源并维护权威 phone→VIN 绑定；历史首次接入只能由未来权威台账补齐，不能把上线回填当历史真值 | P0（部分完成） | +| 权限审计 | ECS 强制 Bearer 鉴权；viewer/operator/admin 累积权限、前端会话门禁、后端逐操作校验、规则/阈值/告警动作版本审计及越权拒绝均已验证 | 企业 SSO 与复杂多租户不在一期范围 | P0（一期完成） | + +## 4. 推荐 V2 API + +### 4.1 元数据与车辆 + +| 方法与路径 | 用途 | 关键参数/响应 | +| --- | --- | --- | +| `GET /api/v2/meta/vehicle-filters` | 全局筛选字典 | OEM、车型、公司、协议、接入厂家、状态；带版本/ETag | +| `GET /api/v2/metrics` | 动态指标目录 | `protocol, category, valueType, searchable, chartable, alertable` | +| `GET /api/v2/vehicles/search` | 远程车辆选择 | `q, cursor, limit`；返回 VIN、车牌、车辆编号 | +| `GET /api/v2/vehicles/:vin` | 单车数字档案 | 档案、实时摘要、接入摘要、当前告警摘要 | +| `GET /api/v2/vehicles/:vin/telemetry/latest` | 动态最新遥测 | 按 category 分组，值含时间、单位、质量、来源 | + +### 4.2 全局监控 + +| 方法与路径 | 用途 | 关键参数/响应 | +| --- | --- | --- | +| `POST /api/v2/monitor/summary` | 与筛选完全一致的状态统计 | 组合过滤对象、`asOf`、在线阈值版本 | +| `POST /api/v2/monitor/map` | 视口聚合/车辆点 | `bounds, zoom, filters, cursor`；返回 `mode=clusters|points`、聚合数或轻量点 | +| `GET /api/v2/monitor/vehicles/:vin/card` | 点位详情卡 | 核心实时、当日里程、来源、告警、地址缓存 | +| `GET /api/v2/monitor/changes` | 可选差量刷新 | `since` 或 SSE；返回变更车辆和统计版本 | + +`monitor/map` 必须有最大点数和聚合降级；低缩放级别绝不返回全部明细。轻量点不携带完整 telemetry JSON。 + +### 4.3 轨迹 + +| 方法与路径 | 用途 | 关键参数/响应 | +| --- | --- | --- | +| `POST /api/v2/tracks/query` | 轨迹分段查询 | VIN、时间、协议、filter、targetPoints、cursor；返回点、起终点、过滤/抽稀计数和算法版本 | +| `GET /api/v2/tracks/:queryId/segments` | 大查询后续分段 | segment/cursor；支持取消/过期 | +| `GET /api/v2/tracks/:queryId/stops` | 停车点 | 起止时间、时长、坐标、地址缓存状态 | + +轨迹点至少包含 `eventTime, receivedAt, lng, lat, speedKmh, directionDeg, socPercent, totalMileageKm, alarmFlag, statusFlag, state`。抽稀必须保留起终点、停车边界和告警点，并返回原始/有效/返回点数。 + +### 4.4 历史分析 + +| 方法与路径 | 用途 | 关键参数/响应 | +| --- | --- | --- | +| `POST /api/v2/timeseries/query` | 多车多指标曲线 | VINs、metricKeys、时间、granularity、aggregation、targetPoints、fillPolicy | +| `POST /api/v2/timeseries/rows` | 动态列列表 | 同上 + cursor、limit、sort、filters | +| `POST /api/v2/timeseries/estimate` | 查询/导出预估 | 估算点数、行数、扫描范围并给出建议粒度 | + +服务端只允许指标目录白名单，不能把客户端 metricKey 直接拼入 SQL。响应带指标显示名、单位、类型、来源、采样方式和空值策略。 + +### 4.5 导出任务 + +| 方法与路径 | 用途 | +| --- | --- | +| `POST /api/v2/exports` | 创建任务；携带查询快照、CSV/XLSX、时区；使用幂等键 | +| `GET /api/v2/exports` | 当前用户任务分页 | +| `GET /api/v2/exports/:id` | 状态、进度、行数、完成/过期时间、错误 | +| `POST /api/v2/exports/:id/cancel` | 取消排队/运行任务 | +| `GET /api/v2/exports/:id/download` | 权限检查后短期下载或签名 URL | + +状态：`queued, running, succeeded, failed, canceled, expired`。同步 API 不生成大文件。 + +### 4.6 告警与通知 + +| 方法与路径 | 用途 | +| --- | --- | +| `GET/POST /api/v2/alert-rules` | 规则列表/创建 | +| `GET/PUT/DELETE /api/v2/alert-rules/:id` | 详情/修改/软删除 | +| `POST /api/v2/alert-rules/:id/enable|disable` | 明确启停操作 | +| `POST /api/v2/alert-rules/validate` | 校验指标类型、操作符、阈值和范围 | +| `GET /api/v2/alert-events`、`/summary` | 事件列表和统计 | +| `GET /api/v2/alert-events/:id` | 事件详情与完整处理时间线 | +| `POST /api/v2/alert-events/:id/acknowledge` | 确认/进入处理中 | +| `POST /api/v2/alert-events/:id/close` | 关闭，要求备注 | +| `POST /api/v2/alert-events/:id/ignore` | 忽略，要求原因 | +| `GET /api/v2/notifications`、`/unread-count` | 站内通知/未读数 | +| `POST /api/v2/notifications/read` | 批量已读 | + +所有变更接口记录操作者、时间、前后状态、备注和 traceId；规则修改采用 `version` 乐观锁。 + +实现记录（2026-07-14）：实际落地路由统一在 `/api/v2/alerts/*`，列表/汇总使用 POST 同构筛选体，处置统一为 `POST /events/:id/actions`，规则启停为版本化 `PUT /rules/:id/enabled`。迁移为 `002_alert_center.sql`、`003_alert_rule_advanced.sql` 与 `004_alert_repeat_index.sql`，独立 `alert-evaluator` systemd 服务以当前 MySQL realtime location 作为一期评估输入。旧 `/api/alert-events*` 继续只是质量投影兼容接口，新页面不依赖它们。Kafka 可重放消费、迟到窗口和 traceId 写入通用审计仍是生产退出缺口。 + +指标目录实现记录（2026-07-14）：`GET /api/v2/metrics` 已返回统一 key、中文名、单位、类别、值类型、协议及源字段映射和三类能力标记；生产权威数据已落入 `vehicle_metric_definition/vehicle_metric_protocol_mapping`，`005_metric_catalog.sql` 仅补齐初始缺失项，不覆盖后续配置。告警编辑器只消费其中 `alertable=true` 且类型匹配的指标，规则写接口再次校验白名单、能力、值类型和 evaluator 支持。`GET /api/v2/vehicles/:vin/telemetry/latest` 现消费同一目录并为未入目录的厂家字段保留源字段、动态分类和质量证据；历史页仍保留 `/api/v2/history/metrics` 兼容目录。后续仅剩动态 RAW 指标受控发现与管理端版本审计。 + +### 4.7 接入管理 + +| 方法与路径 | 用途 | +| --- | --- | +| `POST /api/v2/access/summary` | 按同一筛选口径统计协议、厂家、在线率、今日上报、长离线、从未上报、延迟异常 | +| `POST /api/v2/access/vehicles` | 接入状态分页；支持阈值、时间、厂家/车型/协议过滤 | +| `POST /api/v2/access/unresolved-identities` | 无权威 VIN 的真实终端处置队列；只返回哈希 ID、脱敏标识和登记/上报证据 | +| `GET /api/v2/access/thresholds` | 获取全局默认和协议覆盖阈值 | +| `PUT /api/v2/access/thresholds` | 管理员更新阈值并记录版本/审计 | + +接入行应返回 `firstSeenAt, latestEventAt, latestReceivedAt, reportIntervalSec, dataDelaySec, onlineState, thresholdSec, latestMessageType, latestError`，并区分 `online/offline/never_reported/unknown`。 + +实现记录（2026-07-14）：状态/阈值接口、同口径筛选、动态状态、阈值乐观锁和 MySQL 审计已落地；gateway access projection 已维护 first/previous/latest received、latest error、样本数和连续间隔。`platform-v2-20260714080619` 新增身份待绑定接口和紧凑处置队列：生产当前识别 1 个真实 JT808 缺 VIN 终端，响应和复制证据均仅含脱敏标识，20 次顺序查询 p50 2.02ms、p95 2.44ms、最大 4.66ms。该队列用于推动权威绑定，不会把手机号猜成 VIN，也不会让未绑定身份参与车辆告警。 + +## 5. 横切契约 + +- 时间统一使用 RFC 3339/UTC 传输，UI 按 Asia/Shanghai 展示；同时保留事件时间和接收时间。 +- 列表固定 `items + pageInfo`；大时序优先 cursor，管理列表可 offset；总数默认可选，避免高成本 COUNT。 +- 所有筛选/排序字段白名单化；请求体大小、VIN 数、指标数、时间范围、返回点数均设上限。 +- API 返回 `traceId`、数据 `asOf`、口径/算法版本；可取消请求使用 `AbortSignal`。 +- 查询 GET 可 ETag/短缓存；任务/规则 POST 使用幂等键；状态修改使用版本号。 +- 前端只访问平台 BFF。Redis/TDengine/MySQL 连接与高德 REST key 均不暴露。 + +## 6. 兼容与下线 + +旧接口在 V2 联调期保留；新页面不依赖旧 `alert-events` 别名。完成迁移后先增加 deprecation 响应头和调用监控，再按版本窗口下线。现有 `/api/history/locations` 和 RAW 查询可继续作为运维证据接口，不强行替换为业务时序 DTO。 diff --git a/docs/vehicle-data-platform-data-model.md b/docs/vehicle-data-platform-data-model.md new file mode 100644 index 00000000..1c886ab4 --- /dev/null +++ b/docs/vehicle-data-platform-data-model.md @@ -0,0 +1,224 @@ +# 车辆数据中台一期数据模型 + +## 1. 数据职责原则 + +| 存储 | 负责 | 不负责 | +| --- | --- | --- | +| Redis DB 50 | 最新状态、在线 TTL、字段级当前值、热查询缓存 | 历史事实、告警事件、规则、导出任务 | +| TDengine `lingniu_vehicle_ts` | 高频 RAW 证据、位置/稳定时序历史 | 车辆档案、权限、工作流状态 | +| MySQL `lingniu_vehicle_data` | 车辆主数据、身份映射、接入配置、当前业务投影、统计、规则、事件、通知、导出任务 | 每帧完整历史和无限增长的遥测明细 | + +统一业务主键为 VIN。车牌、JT808 手机号、设备号、平台标识只是可变标识，通过映射解析到 VIN；无法解析 VIN 的数据仍留 RAW/接入排障，不进入正式车辆统计。 + +## 2. 当前 TDengine 模型 + +### 2.1 `raw_frames` stable + +列：`ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint`。 + +Tags：`protocol, vehicle_key, vin, phone, device_id`。 + +用途：证明收到什么、何时收到、解析成什么。`parsed_json` 物理列保存 Gateway 已一次扁平化的 `parsed_fields`，是动态协议字段的历史证据。新字段默认先进入这里，不为每个厂家字段立即新增物理列。 + +### 2.2 `raw_frame_payload_chunks` stable + +列：事件/帧、接收时间、payload 类型、分片序号/总数、分片文本；Tags 与 RAW 身份类似。 + +用途：保存超过主表列长度的 raw/parsed payload，不参与常规业务查询。 + +### 2.3 `vehicle_locations` stable + +列：`ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km`。 + +Tags：`protocol, vin`。 + +用途：VIN 非空车辆的高频位置、速度、方向、告警/状态位、总里程历史。当前平台 DTO 尚未完整透出方向、告警和状态位，也没有 SOC 列；轨迹一期需先补 DTO/查询，SOC 可从 RAW 指标取值或按验证后的需求物化。 + +数据库当前配置 `KEEP 7300 DURATION 10 BUFFER 256`。保留期和磁盘容量应纳入运维监控，不在业务页面查询时临时改变。 + +## 3. 当前 Redis 模型 + +| Key | 值/用途 | 约束 | +| --- | --- | --- | +| `vehicle:latest:{vin}` | 跨协议最新核心字段 | 仅 VIN 非空；轻量快照 | +| `vehicle:latest:{vin}:{protocol}` | 单协议最新核心字段 | 可过期/重建 | +| `vehicle:realtime-raw:{protocol}:{vin}` | 单协议最新完整 parsed 状态 | Redis 中完整协议字段唯一副本 | +| `vehicle:rt-kv:{protocol}:{vin}:values` | 扁平字段当前值 | 与类型/时间/meta 配套 | +| `...:types` | 字段值类型 | 支持动态展示和规则类型校验的实时侧依据 | +| `...:times` | 每字段最新归一事件时间 | 防止乱序旧帧覆盖 | +| `...:meta` | 最新事件/接收时间、映射版本 | 用于新鲜度和追踪 | +| `vehicle:online:{protocol}:{vin}` | TTL 在线状态 | 在线事实的快速判断，不应永久保存 | +| `vehicle:online-state:{protocol}:{vin}` | 可分页的 Hash 副本 | 用于在线列表 | +| `vehicle:protocols:{vin}` | 最近出现协议集合 | 车辆来源发现 | +| `vehicle:last_seen` | 最近活跃车辆 ZSET | 分页/排序候选 | + +生产只允许 `nats-fast-writer` 写 Redis 当前态；Kafka 慢链路的 realtime writer 不应覆盖快路径。在线产品状态应由“最新时间 + 配置阈值”计算并返回阈值/依据，不能只信一个固定 Boolean。 + +## 4. 当前 MySQL 模型 + +### 4.1 身份与档案 + +| 表 | 主键/关键字段 | 用途 | +| --- | --- | --- | +| `vehicle` | `vin`；`plate, oem, enabled, updated_at` | 轻量车辆主实体；当前字段不足一期完整档案 | +| `vehicle_identifier` | `(protocol, source_code, identifier_type, identifier_value)`；VIN/车牌/OEM | 多协议、多来源、多标识解析到 VIN | +| `vehicle_identity_binding` | VIN；车牌、phone、OEM | 外部维护的兼容映射事实，运行期只读 | +| `jt808_registration` | `phone`；device、plate、VIN、厂家、鉴权、来源、首次/最近注册鉴权/出现时间 | JT808 注册鉴权与绑定证据 | + +### 4.2 当前态与来源 + +| 表 | 主键/关键字段 | 用途 | +| --- | --- | --- | +| `vehicle_realtime_snapshot` | `(protocol, vin)`；plate、platform、peer、扁平 `parsed_json`、event/received time | 每协议每 VIN 最新合并遥测快照 | +| `vehicle_realtime_location` | `(protocol, vin)`；经纬度、速度、总里程、SOC、海拔、方向、alarm/status、event/received time | 每协议每 VIN 最新位置业务缓存 | +| `vehicle_data_source` | 自增 ID，唯一 `(protocol, source_ip)`；source code/kind、platform、priority、enabled、first/latest seen | 数据来源配置和可信源选择 | + +### 4.3 日里程 + +| 表 | 主键/关键字段 | 用途 | +| --- | --- | --- | +| `vehicle_daily_mileage_source` | `(vin, stat_date, protocol, source_key)`；首末里程、样本、质量、是否选中 | 多来源日里程事实和质量解释 | +| `vehicle_daily_mileage` | `(vin, stat_date, protocol)`；source ID、日里程、最新总里程 | 对外查询结果投影 | + +当前明确不存在/不应恢复的旧模型包括泛化 `vehicle_daily_metric`、TDengine `vehicle_mileage_points` 和 `raw_frames.fields_json` 重复列。 + +## 5. 一期统一领域模型 + +### 5.1 车辆标识 + +```text +VehicleId = VIN（内部规范化大写、去空格） +VehicleIdentifier = protocol + sourceCode + type + value -> VIN +Plate = 可变展示标识，不作为跨表唯一主键 +Source = protocol + sourceCode/sourceId + endpoint +``` + +所有 API 都以 VIN 作为详情路径主键；搜索接口可接受车牌、VIN、车辆编号/手机号并返回解析结果。多协议数据保留 `protocol` 和 `sourceId` 做归因，不能在合并时丢失来源。 + +### 5.2 时间模型 + +- `eventTime`：车端/协议事件时间，是轨迹、曲线和告警判断的主要时间。 +- `receivedAt`：平台接收时间，用于延迟、迟到数据和链路健康。 +- `storedAt/updatedAt`：投影或业务记录更新时间，不冒充车辆上报时间。 +- `dataDelaySec = receivedAt - eventTime`；`silenceSec = now - max(receivedAt/eventTime 的已确认口径)`。 + +### 5.3 在线状态 + +```text +never_reported: 无任何 latestReceivedAt +online: silenceSec <= thresholdSec +offline: silenceSec > thresholdSec +unknown: 时间非法、未来时间过大或来源状态无法判断 +``` + +行驶/静止是独立 motion 状态，建议以速度阈值 + 最近更新时间判断；告警也是独立状态。不要把 `online/running/alarm` 压成一个互斥枚举，UI 可基于多维状态决定主色和徽标。 + +### 5.4 指标值 + +```text +MetricValue { + vehicleId, metricKey, value, valueType, unit, + eventTime, receivedAt, protocol, sourceId, + quality, mappingVersion +} +``` + +`metricKey` 是协议无关的规范键；协议原字段通过映射表关联。厂家扩展字段允许命名空间，但必须有显示名、类型、分类和单位后才可进入图表/告警。 + +## 6. 建议新增 MySQL 模型 + +### 6.1 档案与组织 + +建议优先用关联表而非不断扩宽 `vehicle`： + +- `vehicle_profile(vin PK, vehicle_no, model_id, vehicle_type, company_id, operation_status, access_vendor_id, first_access_at, ...)` +- `vehicle_model(id, oem_id, code, name, vehicle_type, ...)` +- `organization(id, parent_id, type, code, name, enabled, ...)` +- `access_vendor(id, code, name, enabled, ...)` + +外部主数据通过受控同步写入 `source_system/source_version/synced_at`。默认策略保护 `manual` 和其他外部来源；只有管理员显式选择 takeover 才能改变所有权。同一来源版本相同载荷幂等，不同载荷冲突，避免上游版本不可变性被破坏；每次创建/更新都进入 `vehicle_profile_audit`。 + +### 6.2 指标目录 + +- `metric_definition(metric_key PK, display_name, category, value_type, unit, precision, chartable, alertable, enabled, sort_order, version)` +- `metric_protocol_mapping(metric_key, protocol, source_field, transform, mapping_version, enabled)` + +`transform` 不能存任意可执行代码；使用有限转换类型或由版本化 Go 映射实现。目录服务可缓存，变更有版本和审计。 + +### 6.3 告警域 + +- `alert_rule(id, name, description, metric_key, operator, threshold_json, severity, duration_sec, recovery_json, repeat_interval_sec, enabled, version, created_by, updated_by, timestamps)` +- `alert_rule_scope(rule_id, scope_type, scope_value)`：vehicle/model/oem/protocol/company；一期可限制组合复杂度。 +- `alert_event(id, rule_id, rule_version, vin, protocol, source_id, status, trigger_value_json, threshold_json, first_triggered_at, last_triggered_at, recovered_at, closed_at, location_json, dedupe_key, assignee, timestamps)` +- `alert_event_action(id, event_id, action, from_status, to_status, operator_id, note, created_at)` +- `notification(id, user_id, event_id, title, body, read_at, created_at)` + +`alert_event.dedupe_key` 建唯一约束或等价幂等机制；阈值快照和 rule version 必须保存在事件上，避免规则修改后历史无法解释。 + +当前实现使用带业务前缀的实体表，完整 DDL 位于 `vehicle-data-platform/deploy/migrations/002_alert_center.sql`： + +- `vehicle_alert_rule` + `vehicle_alert_rule_audit`：当前规则与每版本不可变快照。 +- `vehicle_alert_candidate`：`(rule_id, vin, protocol)` 持续命中计时，避免单帧达到阈值就误开事件。 +- `vehicle_alert_stream_checkpoint`：`(consumer_group, topic, partition_id)` 保存数据库权威 `next_offset`、Kafka high watermark、处理/非法/迟到/回放计数、最近事件证据以及最近非法代码/时间。流 worker 先提交该事务，再提交 Kafka group offset；崩溃后重复抓取的 offset 会被数据库 checkpoint 跳过。累计非法数用于审计，健康状态只对五分钟内仍在发生的非法消息告警。 +- `vehicle_alert_event`：保存规则名/版本、触发阈值快照、证据时间、位置和乐观锁版本；`fingerprint + active status` 用于评估器幂等检查。 +- `vehicle_alert_event_action`：触发、恢复和人工处置的不可变时间线。 +- `vehicle_alert_notification`：站内通知真实已读状态；外部通道只有 `reserved`，没有配置供应商时绝不标记 `sent`。 +- `vehicle_alert_rule_state`：为 Boolean `changed` 规则保存每个 rule/VIN/protocol 的最近观测值；`003` 同时增加区间上限和 OEM 范围，`004` 为 fingerprint 重复间隔查询增加索引。 + +一期 evaluator 读取 `vehicle_realtime_location` 的当前快照并用 candidate 表证明持续时长。除 `freshness_sec` 明确按平台墙钟增长外，候选只允许由不同 `source_event_id` 且事件时间不回退的新观测推进；重复快照不能制造持续时长，迟到观测不能回退 candidate 或 Boolean 状态。规则行会在评估事务内加锁，停用规则、清理 candidate/Boolean 状态和写版本审计在同一事务提交，从而避免 evaluator 与管理员停用并发后留下幽灵候选。它仍不是 Kafka event-time 可重放评估器，因此跨当前快照的迟到数据重算、历史窗口规则和消费 offset 仍是阶段 8 的后续能力。 + +`alert-stream-evaluator` 已以 active 模式消费三类 canonical fields topic。它复用 Kafka consumer group 的分区顺序，校验 `event_kind/field_mapping/protocol/VIN/source_event_id/字段命名空间`，用接收时间识别超过可配置窗口的迟到观测。动态规则锁、当前批次范围内的 candidate/Boolean/活跃事件/重复窗口、事件/动作/通知副作用和 checkpoint 在同一个 MySQL 事务内完成，成功后才 commit Kafka；数据库 ahead 时的重复 offset 由 checkpoint 跳过。超过迟到窗口的观测只允许驱动 `data_delay_sec`，不能开关速度/SOC/告警位事件。快照 evaluator 在 active 模式只负责按墙钟增长的 `freshness_sec`，不再读取动态规则。 + +### 6.4 导出域 + +- `export_job(id, user_id, type, format, query_json, status, progress, estimated_rows, exported_rows, file_uri, file_size, checksum, error_code, error_message, started_at, completed_at, expires_at, created_at)` + +索引至少覆盖 `(user_id, created_at)`、`(status, created_at)` 和过期清理。`query_json` 保存规范化查询快照，不保存数据库密码或签名下载 URL。 + +### 6.5 在线阈值与审计 + +- `online_threshold(scope_type, scope_value, threshold_sec, version, updated_by, updated_at)`，一期建议 scope 仅 global/protocol。 +- `audit_log(id, actor, action, resource_type, resource_id, before_json, after_json, trace_id, created_at)`。 + +当前 V2 接入管理已先落地等价的窄表： + +- `vehicle_access_threshold_config(id=1, version, default_threshold_sec, delay_threshold_sec, long_offline_sec, protocol_overrides_json, updated_by, updated_at)`。 +- `vehicle_access_threshold_audit(id, version, actor, summary, config_json, changed_at)`。 + +表仅向前新增，不改变 gateway 的 `vehicle_realtime_snapshot(protocol, vin)` 主键与写入语义。后续统一审计域上线时可迁移到通用 `audit_log`，但必须保留版本历史。 + +## 7. 查询与物化策略 + +1. 实时页面优先 Redis，MySQL realtime 作为业务查询/降级投影；返回值统一由 BFF 归一化。 +2. 轨迹只查 TDengine `vehicle_locations`，必要的 SOC/告警补充应避免对每个点逐条回查 RAW；先验证是否需要将 SOC 提升为位置列或建立专用指标 stable。 +3. 通用历史指标首版可从 `raw_frames.parsed_json` 按白名单提取，但必须限制车辆数、指标数、时间范围并做基准；高频稳定指标按实际查询热度物化为专用时序模型。 +4. 档案和字典在 BFF/Redis 做版本化短缓存，避免每次实时请求重复查不变数据。 +5. 地图聚合可从 MySQL realtime/Redis 快照构建短周期服务端缓存；不要把地理历史查询压到实时表，也不要把全量点放浏览器聚合。 +6. 告警 worker 使用 Kafka 可重放流和 eventTime，MySQL 保存工作流事实；Redis 可保存短期去重/规则缓存但不是唯一事实。 + +## 8. 数据质量与治理要求 + +- 数值同时记录单位和精度；未知单位不得参与跨协议比较或规则判断。 +- 原始值、规范值和转换版本可追踪；异常值标记不直接篡改 RAW。 +- 未来时间、经纬度越界、速度突变、里程回退、重复事件均定义质量码。 +- 轨迹过滤返回算法版本、原始/过滤/抽稀数量，可选择查看未过滤证据。 +- 协议新增字段先进入扁平 RAW + Redis KV，经目录登记后才进入 UI；形成稳定高频查询后再物化。 +- 所有新增表、字段、索引、保留期和清理任务随 migration 与数据字典发布。 + +## 9. 容量和保留 + +- 约 1,000 到 10,000 车辆、5～30 秒上报意味着时序写入持续增长；TDengine 查询必须带 VIN/时间，禁止开放无界扫描。 +- RAW 保留期、位置保留期、导出文件保留期和告警/审计保留期分别配置，不能共用一个默认值。 +- MySQL 告警事件和审计按时间索引并规划归档；导出任务定时过期，文件删除和 DB 状态更新需幂等。 +- 单 ECS 上告警和导出 worker 设置独立并发、内存、CPU 和 systemd 限制，避免影响接入写链路。 + +## 10. 仍需确认的数据问题 + +1. `vehicle_identity_binding` 的实际 DDL、权威维护方和未来是否继续兼容。 +2. 车型、车辆类型、运营公司、车辆编号、运营状态、接入厂家的权威来源与更新频率。 +3. 各协议 SOC、方向、启动、告警等级、电机/燃料电池等规范字段映射和单位。 +4. “累计运行时长”是已有车端字段、由状态积分计算，还是一期可不提供。 +5. 在线时间基准使用 eventTime 还是 receivedAt，以及不同协议的默认阈值。 +6. 轨迹停车阈值、漂移判定、迟到点容忍和跨协议主轨迹选择规则。 +7. 导出存储与保留期、单用户并发/最大行数。 +8. 告警规则适用范围组合、迟到窗口、重复/恢复语义和事件归属人规则。 diff --git a/docs/vehicle-data-platform-roadmap.md b/docs/vehicle-data-platform-roadmap.md new file mode 100644 index 00000000..796555a2 --- /dev/null +++ b/docs/vehicle-data-platform-roadmap.md @@ -0,0 +1,162 @@ +# 车辆数据中台一期实施路线图 + +> 本路线图从分析完成后的下一阶段开始。任何阶段都以“契约、实现、测试、运行检查、文档”全部完成为退出条件，不并行铺开所有页面。 + +## 1. 阶段总览 + +| 阶段 | 目标 | 主要交付 | 退出条件 | +| --- | --- | --- | --- | +| 0 分析与决策 | 冻结一期边界和关键决策 | 本次四份文档、P0 决策记录 | 档案来源、认证、指标元数据、在线阈值、文件存储有负责人和结论 | +| 1 数据与 API 基座 | 建立统一模型和可演进契约 | MySQL 向前迁移、指标目录、V2 OpenAPI、鉴权/审计、统一错误/分页 | 契约测试通过；旧链路不受影响；迁移可回滚 | +| 2 V2 前端基础 | 建立独立、无 mock 的前端壳 | 路由/菜单、权限、token、API client、AsyncState、地图/图表/表格基础组件 | 构建和组件测试通过；1440×900 基础布局可用；无 mock fallback | +| 3 全局监控 | 先打通实时主路径 | 地图聚合、组合筛选、统计、车辆信息卡、详情跳转 | 1k/10k 数据基准达标；视口移动不卡顿；状态口径一致 | +| 4 单车数字档案 | 形成统一车辆入口 | 档案、实时状态、地图、动态遥测分组、来源证据 | 车牌/VIN/车辆编号均可解析；缺失字段不渲染空大表 | +| 5 轨迹回放 | 完成历史位置产品能力 | 分段/抽稀查询、过滤、停车点、播放控制、列表联动 | 长时段不会全量阻塞；抽稀/过滤可解释；播放状态正确 | +| 6 历史分析与导出 | 建立通用指标分析 | 指标目录、多车多指标列表/曲线、异步 CSV/XLSX | 百万点查询经服务端聚合；导出有任务进度和受控下载 | +| 7 接入管理 | 产品化接入质量 | 动态在线阈值、延迟、最近消息/错误、协议/厂家统计 | 从未上报与离线可区分；事件/接收时间延迟口径明确 | +| 8 告警中心 | 建立可持久业务告警 | 规则 CRUD、评估 worker、事件、处理记录、站内通知 | 数值/Boolean 规则可验证；去重、恢复、确认、关闭链路完整 | +| 9 联调与生产发布 | 达到一期验收并部署当前 ECS | 性能/异常/安全测试、运维文档、release、监控和回滚 | 验收清单通过，ECS 健康、无 lag、旧 release 可回滚 | + +进度注记（2026-07-14）：阶段 7 的 V2 接入汇总、车辆状态、动态阈值、协议覆盖、版本审计及桌面/移动端页面已实现。网关 realtime snapshot 同一条原子 upsert 现维护独立于设备事件时间的首次/前次/最新接收时间、连续间隔、样本数和 event ID 去重；平台读取该投影并合并车型/企业，明确区分 `live_writer` 真首次观测和 `snapshot_backfill` 上线基线。`platform-v2-20260714080619` 又将缺少权威 VIN 的真实 JT808 终端纳入“身份待绑定”队列，只暴露哈希 ID、脱敏终端号和登记/上报证据；当前生产 1 条，页面与复制内容均未泄漏原始手机号，查询 p95 2.44ms。实际 phone→VIN 绑定仍必须由运维核验权威来源，不能猜测。 + +阶段 8 进度注记（2026-07-14）：durable 规则/审计、持续候选、事件/动作、站内通知、独立 evaluator、V2 API 与告警中心三个工作区已上线；确认、规则创建、通知已读和权限通过真实接口联调。`platform-v2-20260714070507` 将持续时间改为由不同 source event 的单调事件时间推进，重复快照和迟到观测不再制造持续异常，`freshness_sec` 作为明确的墙钟规则单独处理；后续 Kafka fields active evaluator 又补齐分区 checkpoint、迟到窗口和数据库权威重放抑制。真实规则灰度、规则编辑锁竞争及 31 分钟连续运行门禁均通过，阶段 8 一期退出项已关闭。 + +Kafka 告警流进度注记（2026-07-14）：`platform-v2-20260714080619` 当前生产仍运行 active event-time evaluator。动态规则锁、批次范围 candidate/Boolean/活跃事件/重复窗口、事件/动作/通知和 `(group,topic,partition)` checkpoint 在同一个 MySQL 事务内完成，随后才 commit Kafka；失败事务不会越过 offset。快照 evaluator 在 active 模式只保留 `freshness_sec`，空动态规则扫描由约 13ms 降至 0.5–1ms。迁移 `010` 已用真实 RAW 样本修正国标告警位和宇通速度/SOC 映射。受限 JT808 `speed>=0`、持续 20 秒、重复 1 小时灰度只打开 1 条事件；关闭后又有 2 批 candidate 推进但未重开，随后规则停用并清理状态。914 个带活跃规则的生产批次 p50 7.80ms、p95 12.07ms、最大 18.68ms，候选推进 11、重复观测 9、迟到观测 1、批次失败 0。迁移 `011` 进一步持久化最近非法原因/时间；当前已定位为单个未绑定 VIN 的 JT808 终端，保留 `missing_vin_jt808` 运维告警并在接入页生成脱敏处置队列。当前 9 分区 lag 0，跨窗口历史重算和代表性峰值长稳仍待阶段 9 门禁。 + +规则编辑锁竞争门禁（2026-07-14）：生产创建了不可能命中业务车辆的启用规则，在 active stream evaluator 持续消费时完成 60 次顺序版本化编辑，HTTP 200 为 60/60，冲突和 5xx 均为 0；编辑延迟 p50 3.991ms、p95 7.220ms、p99 8.584ms、最大 10.875ms。并发窗口 41 个活跃规则批次处理 303 条消息，失败、重放、误开和恢复均为 0；规则随后停用且事件总数为 0。锁竞争退出项已关闭，仍保留代表性峰值长稳与 RDS 峰值观测。 + +阶段 5 进度注记（2026-07-14）：生产真实轨迹基线显示活跃车辆总点数约 9,500–71,361，旧接口仅加工最新 5,000 点但摘要未充分表达覆盖边界。`platform-v2-20260714060112` 已在当前 ECS 上线默认今天、最长 7 天校验、完整/最新切片证据、同协议无效坐标/重复/疑似漂移过滤、多源主来源选择、GPS 推断停车、行驶/停车/数据间隔分段、质量摘要，以及保留事件和分段边界的抽稀映射。8 辆生产 canary 今天窗口为 5.7–34.22ms；正式切流抽样 2,254 个源点完整读取、1,200 个地图点、33 段和 5 次停车为 32.83ms。并行协议不会被拼成一条路线，停车也不冒充 ignition 状态；服务端持久 queryId/游标分段和长周期观测仍待后续数据合同。 + +轨迹与接入补齐注记（2026-07-14）：`platform-v2-20260714083640` 已在当前 ECS 上线。`vehicle_locations` 向前增加 SOC，轨迹返回 SOC 可用性、方向和告警并使用绝对时间戳；页面以 `Asia/Shanghai` 展示，同步卡片只为暂停点做一小时坐标缓存的地址解析。生产宇通 5 分钟 300 点全部有 SOC，JT808 5 分钟 10 点全部有方向/告警；修复了 TDengine 时间窗二次偏移及亚秒采样被截断成零秒后误判 gap 的问题，5,000 点页面分段从 3,203 降到 14，仅保留 1 个真实 28 分钟间隔，无横向溢出或控制台错误。接入页新增车型、接入厂家、首次接入和最新上报范围；生产 `G7s + latestSeenFrom` 深链筛选当前页 50/50 匹配。小窗位置导出 120/120 行完成，Kafka lag 0，MySQL/TDengine 可写。 + +全局监控性能闭环（2026-07-14）：`platform-v2-20260714090040` 已在当前 ECS 上线。此前页面虽有后端聚合接口，实际仍把 1,000 行普通车辆列表直接送入地图；现改为 200 行服务端筛选列表与独立地图查询，`moveend/zoomend` 300ms 防抖并由 React Query 去重缓存。低缩放返回聚合，高缩放按 bounds 返回轻量 MassMarks，超过 2,000 个点时自适应扩大网格，10,000 合成车辆不会退化成 10,000 个点或聚合标记。10k 纯聚合基准 30 次为 0.976ms/op；当前 ECS 727 辆真实数据 30 次低缩放聚合为 29 组，API p50 57.31ms、p95 63.44ms、最大 70.65ms，城市视口 168 点 p50 94.89ms、p95 108.56ms、最大 114.54ms。生产浏览器首屏为 15 个聚合/200 行，无横向溢出；行驶筛选所有加载行均匹配，唯一车牌查询自动从聚合切到 1 个点并聚焦到 2km 比例尺。 + +全局监控产品闭环（2026-07-14）：`platform-v2-20260714093533` 将活跃告警 VIN 与当前关键词/协议/状态筛选后的车辆集合在服务端求交，`alertDataAvailable=true` 后页面展示权威告警车辆数，不再使用占位。选中车辆卡片按需并发读取档案/日里程、当前业务告警和地址，并展示真实来源、接入供应商、今日里程、解析位置及当前告警；无权威供应商时明确“待补充”。生产 727 车显示告警车辆 0，抽样车辆今日里程 59.4 km、三协议来源、嘉兴解析地址和 0 条当前告警；1280×720 无页面溢出。 + +单车最新遥测闭环（2026-07-14）：`platform-v2-20260714092531` 已在当前 ECS 上线 `GET /api/v2/vehicles/:vin/telemetry/latest`。服务端复用权威指标目录统一 key/中文名/单位/类型/分类，同时保留真实源字段、厂家扩展、协议、端点、帧、设备/接收时间、freshness/delay 和 `good|stale|warning` 原因；前端不再从 RAW key 猜测标签或质量。身份只解析一次，三个协议表固定并行各取最多 5 帧且跳过分页计数，真实三源车辆扫描 10 帧返回 155 值/7 分组；生产 30 次 p50 174ms、p95 179ms、最大 179ms，本地 100 帧×50 字段纯归并基准 0.389ms/op。页面使用独立 10 秒缓存、20 秒刷新，一次遍历建立分组与来源索引；1280 桌面和 390×844 移动端均无横向溢出，行内时间为紧凑时分秒，控制台无错误或警告。 + +连续运行门禁（2026-07-14）：alert stream evaluator 自 08:31:50 起连续 active 运行超过 31 分钟且未重启；该窗口 16,226 批、216,118 条消息全部完成处理，valid 216,072、历史未绑定身份导致 invalid 46、replay skipped 0、失败日志 0，批次延迟 p50 6.511ms、p95 10.481ms、p99 12.806ms、最大 29.697ms。结合此前 929 个带活跃规则批次和 60 次并发版本编辑均无失败/误开，已覆盖正常流量长稳及规则锁竞争。窗口结束时 checkpoint 累计 processed 563,598、9 分区 lag 0、MySQL/TDengine 可写；最近非法身份时间已超过 20 分钟且脱敏处置队列仍保留。阶段 9 的代表性连续运行门禁关闭，云数据库 Performance Insights 仍作为容量扩容观察项，不再阻塞单 ECS 一期上线。 + +阶段 6 进度注记（2026-07-14）：`platform-v2-20260714063725` 已在当前 ECS 上线历史时间序列聚合与百万行受控导出。位置历史按车辆与协议在 TDengine 使用 `PARTITION BY protocol INTERVAL(...)` 服务端聚合，速度使用桶均值、总里程使用桶末值并保留 min/max/样本数；最长 31 天、每序列目标 60–600 点、空窗不 `FILL`，前端按单位拆成速度/总里程小多图并明确粒度、覆盖率、缺失桶、原始点数和查询耗时。真实 canary 单车双协议 1,392 个原始点聚合为 127 桶、254 浏览器点，20.67ms；5 台车 8,197 个原始点聚合为 376 桶、752 浏览器点，86.53ms。导出采用单并发队列、5,000 行 TDengine 复合游标、30 分钟超时、100 万行双重配额、流式 CSV、`.part` + `fsync` + 原子发布，并持久化总行数、已处理行数、文件大小和完成时间。预发布门禁发现并修复 8 小时时区偏移和移动端面板压缩；最终桌面与 390×844 验收无横向溢出。RAW 保持离散证据而不伪造连续趋势；日里程趋势、多实例数据库队列/对象存储与更多可聚合指标仍待后续数据合同。 + +上线进度注记（2026-07-14）：`platform-v2-20260714033929` 已发布到当前 ECS `115.29.187.205:20300`。API 与告警 evaluator 均为 active，数据模式强制 `production`，MySQL/TDengine/Redis/容量检查已接入，Kafka lag 为 0；viewer/operator/admin 鉴权、越权拒绝、核心真实查询、根页面与 hashed 静态资源均已冒烟。告警 evaluator 当前扫描 1159 条车辆来源快照约 8–9ms，规则为 0，未产生生产告警；10k 车辆 × 20 规则的本地合成判定与批次规划约 20ms，候选写入已改为每 500 条一次批处理、活跃事件不重复写、超过 5 分钟的动态遥测不继续累积。首轮发布发现 Web 归档遗漏并在 API 冒烟后修复，runbook 已增加根页和主资源门禁。仍需补齐 ECS 10k 真实数据库写入基准、长时间稳定性、真实规则灰度及导出文件生产存储后，阶段 8/9 才可完全退出。 + +一期最终发布注记（2026-07-14）：当前 release 为 `platform-v2-20260714093533`，前一 `platform-v2-20260714092531` 保留可回滚。全部 11 项迁移 journal 校验通过；平台 API、快照 evaluator、流 evaluator 均 active，MySQL/TDengine 可写，9 个 Kafka 分区 lag 0、replay skipped 0，三单元最近发布窗口 error 日志均为 0。未鉴权会话返回 401，管理员会话、monitor、单车、active 告警、最新遥测、根页面与 hashed 主资源均通过真实生产冒烟；Web 22 个测试文件 272 项、API `go test ./...` 和 `go vet ./...` 全部通过。阶段 9 的单 ECS 一期退出项关闭；云数据库峰值容量和多实例对象存储作为后续扩容观察项。 + +导出进度注记（2026-07-14）：CSV 异步导出位于 `/opt/lingniu-vehicle-platform/data/exports` 持久目录，任务索引使用原子 `jobs.json`；完成文件和任务在 API 重启后仍可列出、下载，中断任务会显式失败而非永久 running。`platform-v2-20260714063725` 将旧的 1,000 行内存聚合/OFFSET/2 分钟实现升级为 100 万行流式游标任务。本地真实写入 1,000,000 行为 0.45 秒；ECS 候选位置 1,451 行与 RAW 3,538 行在单并发队列中 0.41 秒完成。正式生产 1,457 行、162,533 字节文件在 API 重启前后 SHA-256 完全一致，桌面和 390×844 任务卡无溢出、控制台无错误。当前满足单 ECS 一期受控导出；多实例扩容前仍需迁移到数据库队列与 OSS/兼容对象存储。 + +高级告警与指标目录进度注记（2026-07-14）：数值区间/区间外、布尔状态变化、OEM 范围、重复间隔、状态表和索引迁移已上线；ECS 以不存在厂家范围的两条规则完成启用灰度，evaluator 扫描 1159 条快照约 9ms、未误开事件，随后停用并保留版本审计。规则编辑器已通过生产浏览器检查。统一 `/api/v2/metrics` 目录现提供协议字段映射及可检索/绘图/告警能力，编辑器和后端写入均按目录校验。事件 INSERT 占位符、状态批写和重复边界已有自动化契约测试；布尔真实翻转、重复抑制与活跃规则长稳仍需独立生产灰度。 + +事件时间告警灰度注记（2026-07-14）：当前 ECS 对单车 `JT808` 建立 `speed_kmh >= 0`、持续 20 秒、重复间隔 1 小时的受限规则。真实新观测形成 candidate 后只打开 1 条事件，事件证据时间 `06:58:57.000`、接收时间 `06:58:57.934`、平台触发时间 `06:59:06.871`；事件随后关闭、通知设为已读、规则停用并保留审计。再次启用时 evaluator 记录 `candidates_advanced` 与 `duplicate_observations`，重复窗口内 `opened=0`。该门禁同时发现 MySQL `UNIX_TIMESTAMP(DATETIME(3))` 返回小数字符串导致候选及最近触发时间整数扫描失败，最终改为直接扫描 `DATETIME(3)` 并增加毫秒精度 SQL mock 契约。发布期间两次失败校验均由软链回滚陷阱恢复，最终 release 为 `platform-v2-20260714070507`。 + +指标配置持久化进度注记（2026-07-14）：`005_metric_catalog.sql` 已建立指标定义与协议源字段映射权威表；生产目录读取和告警规则校验共享该数据源，关闭或修改能力标记会立即约束后续规则写入。种子数据使用 `INSERT IGNORE`，发布不会回写覆盖运维配置；尚缺管理端 CRUD、版本审计以及把动态 RAW 指标纳入统一目录的受控发现流程。 + +告警写入性能注记（2026-07-14）：`alert-benchmark` 默认使用连接级 MySQL 临时表，同时提供必须显式确认且带 advisory lock 的独立物理表持久模式，两者都复制候选表主键与持续时间索引且绝不写业务候选/事件。当前 ECS 到生产 RDS 的持久单事务基准：10,000 行/500 批次为 195ms（约 51,234 行/秒），100,000 行为 1,971ms（约 50,713 行/秒）；两档行数和物理表清理均独立校验。10 万行窗口观察到实例全局 redo 增量约 31MB，但全局计数包含并发业务流量，不能当作本事务精确归因。当前结果证明真实提交、网络和索引路径具备明显余量，最终容量退出仍需 evaluator 活跃规则长稳、锁竞争和 RDS Performance Insights 峰值窗口证据。 + +车辆主数据进度注记（2026-07-14）：采用“网关权威身份 + 平台补充主档”的混合边界。`vehicle_profile` 不覆盖 VIN/车牌/厂家，专门保存车型、车辆类型、所属企业、运营状态、接入服务商、首次接入和累计运行时长；`vehicle_profile_audit`、乐观版本和 admin-only 写入保证可追溯。单车详情已合并完整度与缺失字段，管理员可在档案卡原位维护，只读角色仅查看。外部主数据同步 API 与 CSV 管理入口已实现：单批 500 辆、dry-run、来源/版本幂等、同版本变更拒绝、人工及异源默认保护、显式接管、网关身份校验和逐 VIN 结果；写入持久化 `source_system/source_version/synced_at` 并生成不可变同步审计。具体车厂/GPS 厂商的定时连接器仍需按其认证和字段合同配置。 + +主数据同步上线注记（2026-07-14）：`platform-v2-20260714051915` 已部署当前 ECS，API 与 evaluator 均为 active。管理员 dry-run、operator 403、页面分包和外网根页均已验证；生产可查询身份集合的 410 个唯一 VIN 在同一事务预演中耗时 175.44ms，重复预演 170.08ms，缺失身份为 0，抽样档案前后完全一致。上线验证未向生产写入虚构车型/企业数据；正式写入需由权威车厂/GPS 数据文件或连接器触发。 + +主数据告警范围进度注记（2026-07-14）：规则新增 `scopeModels/scopeCompanies`，评估器以 VIN 主键左联 `vehicle_profile`，未维护对应维度的车辆不会误匹配；协议、VIN、厂家、车型、企业范围均执行去重和大小上限，避免配置膨胀拖慢 10 秒评估周期。规则编辑器、版本快照和数据库持久化共享同一字段合同。 + +## 2. 阶段 0：必须先确认的决策 + +1. 车辆档案采用两者结合：网关身份表维护 VIN/车牌/厂家，平台 `vehicle_profile` 承载补充业务属性并预留外部同步来源元数据。 +2. 认证是否复用 `ln-bi` 的 jumpToken/JWT 交换流程；角色和操作权限的权威来源是什么。 +3. 动态指标目录的首批范围、中文名、单位、类型、协议字段映射及负责人。 +4. 在线阈值是否采用“全局默认 + 协议覆盖”，建议默认 5 分钟并同时展示实际延迟。 +5. 导出文件使用 OSS/兼容对象存储还是 ECS 本地盘，保留天数和单任务配额。 +6. 告警时间基准使用事件时间，迟到数据容忍窗口和重复告警间隔的产品定义。 + +决策记录写 ADR，不把未确认事项固化进页面常量。 + +## 3. 阶段 1：数据与 API 基座 + +### 3.1 MySQL 迁移 + +- 扩充或关联车辆档案：车型、车辆类型、运营公司、运营状态、接入厂家、首次接入时间。 +- 新建 `metric_definition`、`metric_protocol_mapping`。 +- 新建告警域：`alert_rule`、`alert_rule_scope`、`alert_event`、`alert_event_action`、`notification`。 +- 新建导出域：`export_job`，文件本体放对象/文件存储。 +- 为所有业务表增加必要的状态、时间和组合索引；迁移只向前新增，不改写接入 writer 的现有主键语义。 + +### 3.2 BFF 契约 + +- 建立 `/api/v2`，保留旧只读接口作为过渡。 +- 统一游标/offset 分页规则、排序白名单、时间格式、错误码、traceId 和最大查询范围。 +- 生成 OpenAPI 与前端类型，避免 Go/TypeScript 手工重复漂移。 +- 接入鉴权、操作权限、审计日志、限流、请求超时和幂等键。 + +### 3.3 验证 + +- 在生产结构副本/测试库运行 migration up/down 或等价回滚演练。 +- 契约测试覆盖空值、未知协议、时区、分页边界和非法指标。 +- 回归现有 Gateway、writers、realtime API 的写入与查询。 + +## 4. 阶段 2：V2 前端基础 + +建议新建独立入口/目录，而不是在 `PrototypeApp.tsx` 内渐进堆叠： + +```text +src-v2/ +├── app/ # router、providers、权限和菜单 +├── shared/ # ui、api、map、chart、table、utils +├── entities/vehicle/ # 车辆领域类型、状态、展示组件 +├── features/ # 筛选、车辆选择、时间范围、指标选择 +├── pages/ # 路由页面 +└── widgets/ # 地图工作区、详情面板、事件时间线 +``` + +基础能力包括：正式路由、URL 查询恢复、认证门禁、统一 API client、查询缓存、错误边界、加载/空/错误状态、车辆状态 token、远程搜索选择器、服务端表格、ECharts 时序封装、AMap SDK。V2 默认禁止导入 `prototype/mockData.ts`。 + +## 5. 阶段 3 至 8：业务模块次序 + +每个模块遵循相同步骤： + +1. 冻结页面信息架构和 DTO。 +2. 先实现/验证后端接口和样本响应。 +3. 实现页面及深链跳转。 +4. 增加单元、契约和交互测试。 +5. 用真实接口实际运行，检查加载、空、错误、权限和不同屏幕。 +6. 更新已完成/未完成、接口和运维文档后再进入下一模块。 + +告警中心虽然菜单重要，但排在数据查询能力之后：规则依赖统一指标目录、车辆范围和时间语义。可在阶段 1 先建表/契约，阶段 8 再启用评估与通知。 + +## 6. 性能验收门槛 + +以下是建议基线，开发前应在目标 ECS/浏览器上校准： + +| 场景 | 门槛 | +| --- | --- | +| 全局监控首屏 | 1,000 车 P95 API < 1s；10,000 车聚合结果而非 10,000 DOM Marker；主要交互保持流畅 | +| 地图拖动/缩放 | 只在 moveend/zoomend 防抖查询；旧请求可取消；无请求风暴 | +| 实时刷新 | 差量或低频批量刷新，不按车轮询；页面隐藏时降频/暂停 | +| 轨迹 | 单次返回受控；长时间段自动抽稀/分段；保留起终点、停车和告警关键点 | +| 历史曲线 | 浏览器展示点数设上限；TDengine 服务端聚合/抽稀；缩放后按窗口补查 | +| 表格 | 全部服务端分页/排序/筛选；长列表虚拟化 | +| 导出 | 异步任务；并发、行数、时间范围和文件大小有限额；不占用同步 API worker | +| 告警 | 消费失败不丢 offset；事件按规则+车辆+窗口幂等；可观测积压和评估延迟 | + +## 7. 阶段 9：当前 ECS 发布计划 + +现有运维说明表明接入链路和平台都使用 systemd 裸机 release + `current` 软链。最终发布应继续该模式，但平台、告警 worker、导出 worker使用独立 unit 和健康检查。 + +发布顺序： + +1. 备份并执行 MySQL 向前兼容迁移，确认现有服务继续健康。 +2. 上传新 release，先启动告警/导出 worker 的 disabled 或 idle 模式。 +3. 启动/切换平台 BFF，验证 `/readyz`、`/api/ops/health`、核心查询和运行时配置。 +4. 切换 V2 静态 Web，进行全局监控、单车、轨迹、历史、接入、告警 smoke。 +5. 开启 worker，小范围规则验证后再开放全部规则。 +6. 检查 Kafka lag、Redis 在线 key、TDengine/MySQL 可写性、CPU/内存/磁盘和错误日志。 +7. 保留前一 release 与数据库兼容窗口；失败时先回切软链并停新 worker，不做破坏性数据库回滚。 + +密钥放 `/opt/lingniu-vehicle-platform/env` 或对应 worker 环境文件，禁止写入 git、静态 JS 或部署日志。具体命令以实施时更新后的 runbook 为准，不能直接复制过时 release 名。 + +## 8. 一期完成清单 + +- 六大业务模块和导出任务满足目标文件的功能验收。 +- 页面在 1440×900 正常，常用操作三次点击内，真实数据无 mock 残留。 +- 1k/10k 地图、百万点历史、异步导出、告警积压均有测试记录。 +- OpenAPI、数据字典、环境变量、启动、部署、回滚、监控与未完成事项完整。 +- ECS 上全部新旧服务健康，Kafka lag 回零，运行版本可从健康接口确认。 diff --git a/go/vehicle-gateway/cmd/capacity-check/main.go b/go/vehicle-gateway/cmd/capacity-check/main.go index 84b37b77..b9f9ec3f 100644 --- a/go/vehicle-gateway/cmd/capacity-check/main.go +++ b/go/vehicle-gateway/cmd/capacity-check/main.go @@ -9,13 +9,30 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/capacity" ) -const defaultEndpoints = "gateway=http://127.0.0.1:20211,history=http://127.0.0.1:20212,stat=http://127.0.0.1:20213,bridge=http://127.0.0.1:20214,realtime=http://127.0.0.1:20200,fast-writer=http://127.0.0.1:20215" +const defaultEndpoints = "gateway=http://127.0.0.1:20211,history=http://127.0.0.1:20212,stat=http://127.0.0.1:20213,bridge=http://127.0.0.1:20214,fields-projector=http://127.0.0.1:20218,realtime=http://127.0.0.1:20216,identity=http://127.0.0.1:20217,realtime-api=http://127.0.0.1:20200,fast-writer=http://127.0.0.1:20215" +const defaultRequiredServices = "gateway,history,stat,bridge,fields-projector,realtime,identity,realtime-api,fast-writer" +const defaultDailyMileageDiagnosticsURL = "http://127.0.0.1:20200/api/stats/daily-metrics/diagnostics/reasons" +const defaultDailyMileageDiagnosticsPath = "/api/stats/daily-metrics/diagnostics/reasons" +const defaultDailyMileageFieldStatusPath = "/api/stats/daily-metrics/diagnostics/field-status" +const defaultDailyMileageSourceQualityURL = "http://127.0.0.1:20200/api/stats/daily-metrics/sources/quality" +const defaultDailyMileageSourceQualityPath = "/api/stats/daily-metrics/sources/quality" +const defaultDailyMileageSourceSelectionURL = "http://127.0.0.1:20200/api/stats/daily-metrics/sources/selection" +const defaultDailyMileageSourceSelectionPath = "/api/stats/daily-metrics/sources/selection" +const defaultDailyMileageSourcesPath = "/api/stats/daily-metrics/sources" +const defaultDataSourceDiagnosticsURL = "http://127.0.0.1:20200/api/stats/data-sources/diagnostics" +const defaultDataSourceDiagnosticsPath = "/api/stats/data-sources/diagnostics" +const defaultDataSourceKindSuggestionsURL = "http://127.0.0.1:20200/api/stats/data-sources/kind-suggestions" +const defaultDataSourceKindSuggestionsPath = "/api/stats/data-sources/kind-suggestions" +const defaultDataSourceKindSuggestionProtocols = "GB32960,YUTONG_MQTT,JT808" +const defaultJT808IdentityGapsURL = "http://127.0.0.1:20200/api/stats/data-sources/jt808-identity-gaps" +const defaultJT808IdentityGapsPath = "/api/stats/data-sources/jt808-identity-gaps" type endpoint struct { Name string @@ -24,17 +41,353 @@ type endpoint struct { func main() { var rawEndpoints string + var rawRequiredServices string var timeout time.Duration + thresholds := capacity.DefaultThresholds() + var requiredConsumerTopics string + var dailyMileageDiagnosticsURL string + var dailyMileageDiagnosticsDate string + var dailyMileageDiagnosticsMaxActionable int64 + var dailyMileageDiagnosticsMaxPipeline int64 + var dailyMileageDiagnosticsMaxSourceData int64 + var dailyMileageDiagnosticsSampleLimit int + var dailyMileageSourceQualityURL string + var dailyMileageSourceQualityDate string + var dailyMileageSourceQualityLimit int + var dailyMileageSourceQualityMaxInvalidDelta int64 + var dailyMileageSourceQualityMaxNoBaseline int64 + var dailyMileageSourceQualityMaxFallbackAfterInvalidBaseline int64 + var dailyMileageSourceQualityMaxRealtimeLocationFallback int64 + var dailyMileageSourceSelectionURL string + var dailyMileageSourceSelectionDate string + var dailyMileageSourceSelectionLimit int + var dailyMileageSourceSelectionSampleLimit int + var dailyMileageSourceSelectionMinSelected int64 + var dailyMileageSourceSelectionRequiredProtocols string + var dailyMileageSourceSelectionMinSelectedPerProtocol int64 + var dailyMileageSourceSelectionMaxUnmanaged int64 + var dailyMileageSourceSelectionMaxUnknownKind int64 + var dailyMileageSourceSelectionMaxNotSelectedMileageKM float64 + var dataSourceDiagnosticsURL string + var dataSourceDiagnosticsSampleLimit int + var dataSourceDiagnosticsReasonLimit int + var dataSourceDiagnosticsRecentAgeSeconds int64 + var dataSourceDiagnosticsMaxMissingSourceCode int64 + var dataSourceDiagnosticsMaxRecentMissingSourceCode int64 + var dataSourceDiagnosticsMaxActionableCandidates int64 + var dataSourceDiagnosticsMaxMappingIssues int64 + var dataSourceDiagnosticsMaxRecentMappingIssues int64 + var dataSourceKindSuggestionsURL string + var dataSourceKindSuggestionsProtocols string + var dataSourceKindSuggestionsSampleLimit int + var dataSourceKindSuggestionsReasonLimit int + var dataSourceKindSuggestionsRecentAgeSeconds int64 + var dataSourceKindSuggestionsMaxPlatformCandidates int64 + var dataSourceKindSuggestionsMaxRecentPlatformCandidates int64 + var jt808IdentityGapsURL string + var jt808IdentityGapsRecentSeconds int64 + var jt808IdentityGapsSampleLimit int + var jt808IdentityGapsMax int64 flag.StringVar(&rawEndpoints, "endpoints", defaultEndpoints, "comma separated name=url endpoints; /metrics is appended when no path is provided") + flag.StringVar(&rawRequiredServices, "required-services", defaultRequiredServices, "comma separated service names that must be present and successfully scraped; empty disables") flag.DurationVar(&timeout, "timeout", 2*time.Second, "per endpoint timeout") + flag.Float64Var(&thresholds.GatewayFrameP99MS, "gateway-frame-p99-ms", thresholds.GatewayFrameP99MS, "degrade when gateway frame p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityP99MS, "gateway-identity-p99-ms", thresholds.GatewayIdentityP99MS, "degrade when gateway identity resolve p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.GatewayResponseRecentP99MS, "gateway-response-recent-p99-ms", thresholds.GatewayResponseRecentP99MS, "degrade when recent frame receipt to successful protocol response p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.AsyncSinkP99MS, "async-sink-p99-ms", thresholds.AsyncSinkP99MS, "degrade when gateway async publish p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.AsyncSinkQueueWaitRecentP99MS, "async-sink-queue-wait-recent-p99-ms", thresholds.AsyncSinkQueueWaitRecentP99MS, "degrade when recent gateway async queue wait p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeBatchP99MS, "bridge-batch-p99-ms", thresholds.BridgeBatchP99MS, "degrade when NATS to Kafka bridge batch p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeKafkaWriteP99MS, "bridge-kafka-write-p99-ms", thresholds.BridgeKafkaWriteP99MS, "degrade when NATS to Kafka bridge per-topic Kafka write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeKafkaE2EP99MS, "bridge-kafka-e2e-p99-ms", thresholds.BridgeKafkaE2EP99MS, "degrade when gateway received_at to Kafka bridge write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeKafkaRecentE2EP99MS, "bridge-kafka-recent-e2e-p99-ms", thresholds.BridgeKafkaRecentE2EP99MS, "degrade when recent gateway received_at to Kafka bridge write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeFetchWaitMaxMS, "bridge-fetch-wait-max-ms", thresholds.BridgeFetchWaitMaxMS, "degrade when bridge runtime fetch wait exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.BridgeWorkersMin, "bridge-workers-min", thresholds.BridgeWorkersMin, "degrade when bridge runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.BridgeRouteErrorMax, "bridge-route-error-max", thresholds.BridgeRouteErrorMax, "degrade when bridge unrouted NATS subject count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.BridgeKafkaWriteErrorMax, "bridge-kafka-write-error-max", thresholds.BridgeKafkaWriteErrorMax, "degrade when bridge Kafka write errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.BridgeMessageMinReceived, "bridge-message-min-received", thresholds.BridgeMessageMinReceived, "minimum bridge received messages per subject before unacked ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.BridgeMessageMaxUnackedRatio, "bridge-message-max-unacked-ratio", thresholds.BridgeMessageMaxUnackedRatio, "degrade when bridge unacked messages divided by received messages exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.FastWriterStageP99MS, "fast-writer-stage-p99-ms", thresholds.FastWriterStageP99MS, "degrade when NATS fast-writer stage p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisE2EP99MS, "fast-writer-redis-e2e-p99-ms", thresholds.FastWriterRedisE2EP99MS, "degrade when gateway received_at to fast-writer Redis update p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisRecentE2EP99MS, "fast-writer-redis-recent-e2e-p99-ms", thresholds.FastWriterRedisRecentE2EP99MS, "degrade when recent gateway received_at to fast-writer Redis update p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.FastWriterFetchWaitMaxMS, "fast-writer-fetch-wait-max-ms", thresholds.FastWriterFetchWaitMaxMS, "degrade when fast-writer runtime fetch wait exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.FastWriterWorkersMin, "fast-writer-workers-min", thresholds.FastWriterWorkersMin, "degrade when fast-writer runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.FastWriterInvalidJSONMax, "fast-writer-invalid-json-max", thresholds.FastWriterInvalidJSONMax, "degrade when fast-writer invalid JSON count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.FastWriterFallbackErrorMax, "fast-writer-fallback-error-max", thresholds.FastWriterFallbackErrorMax, "degrade when fast-writer batch fallback single writes still fail; <0 disables") + flag.Float64Var(&thresholds.FastWriterDecoupledUpdateErrorMax, "fast-writer-decoupled-update-error-max", thresholds.FastWriterDecoupledUpdateErrorMax, "degrade when fast-writer Redis catch-up after a decoupled storage failure also fails; <0 disables") + flag.Float64Var(&thresholds.FastWriterTDengineEnabledMax, "fast-writer-tdengine-enabled-max", thresholds.FastWriterTDengineEnabledMax, "degrade when fast-writer TDengine stage enabled gauge exceeds this value; <0 disables") + flag.Float64Var(&thresholds.HistoryFlushP99MS, "history-flush-p99-ms", thresholds.HistoryFlushP99MS, "degrade when history TDengine batch flush p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.HistoryInvalidJSONMax, "history-invalid-json-max", thresholds.HistoryInvalidJSONMax, "degrade when history-writer invalid JSON count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.HistoryWriteE2EP99MS, "history-write-e2e-p99-ms", thresholds.HistoryWriteE2EP99MS, "degrade when gateway received_at to history TDengine write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.HistoryWriteRecentE2EP99MS, "history-write-recent-e2e-p99-ms", thresholds.HistoryWriteRecentE2EP99MS, "degrade when recent gateway received_at to history TDengine write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.HistoryWorkersMin, "history-workers-min", thresholds.HistoryWorkersMin, "degrade when history-writer runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.HistoryLocationErrorMax, "history-location-error-max", thresholds.HistoryLocationErrorMax, "degrade when history-writer location derived write errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.HistoryLocationAccountingMaxRatio, "history-location-accounting-max-mismatch-ratio", thresholds.HistoryLocationAccountingMaxRatio, "degrade when location derived status counters do not account for successful history raw writes by more than this ratio; <=0 disables") + flag.Float64Var(&thresholds.HistoryFallbackErrorMax, "history-fallback-error-max", thresholds.HistoryFallbackErrorMax, "degrade when history-writer batch fallback single writes still fail; <0 disables") + flag.Float64Var(&thresholds.HistoryRetryMax, "history-retry-max", thresholds.HistoryRetryMax, "degrade when history-writer transient write retries exceed this value; <0 disables") + flag.Float64Var(&thresholds.HistoryRetryExhaustedMax, "history-retry-exhausted-max", thresholds.HistoryRetryExhaustedMax, "degrade when history-writer exhausted transient write retries exceed this value; <0 disables") + flag.Float64Var(&thresholds.RealtimeBatchP99MS, "realtime-batch-p99-ms", thresholds.RealtimeBatchP99MS, "degrade when realtime projector whole-batch flush p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.RealtimeStoreP99MS, "realtime-store-p99-ms", thresholds.RealtimeStoreP99MS, "degrade when realtime store update p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.RealtimeWorkersMin, "realtime-workers-min", thresholds.RealtimeWorkersMin, "degrade when realtime-writer runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimeStoreErrorMax, "realtime-store-error-max", thresholds.RealtimeStoreErrorMax, "degrade when realtime Redis/MySQL store update errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.RealtimeInvalidJSONMax, "realtime-invalid-json-max", thresholds.RealtimeInvalidJSONMax, "degrade when realtime projector invalid JSON count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.StatWriteP99MS, "stat-write-p99-ms", thresholds.StatWriteP99MS, "degrade when stat write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.StatWriteE2EP99MS, "stat-write-e2e-p99-ms", thresholds.StatWriteE2EP99MS, "degrade when gateway received_at to stat MySQL write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.StatWriteRecentE2EP99MS, "stat-write-recent-e2e-p99-ms", thresholds.StatWriteRecentE2EP99MS, "degrade when recent gateway received_at to stat MySQL write p99 exceeds this many milliseconds; <=0 disables") + flag.Float64Var(&thresholds.StatWorkersMin, "stat-workers-min", thresholds.StatWorkersMin, "degrade when stat-writer runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.IdentityWorkersMin, "identity-workers-min", thresholds.IdentityWorkersMin, "degrade when identity-writer runtime worker count is below this value; <=0 disables") + flag.Float64Var(&thresholds.StatInvalidJSONMax, "stat-invalid-json-max", thresholds.StatInvalidJSONMax, "degrade when stat-writer invalid JSON count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.StatProtocolTopicMismatchMax, "stat-protocol-topic-mismatch-max", thresholds.StatProtocolTopicMismatchMax, "degrade when stat-writer protocol/topic mismatch count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.MessageContractMismatchMax, "message-contract-mismatch-max", thresholds.MessageContractMismatchMax, "degrade when consumer event kind or raw protocol/topic contract mismatch count exceeds this value; <0 disables") + flag.Float64Var(&thresholds.StatRetryMax, "stat-retry-max", thresholds.StatRetryMax, "degrade when stat-writer transient write retries exceed this value; <0 disables") + flag.Float64Var(&thresholds.StatRetryExhaustedMax, "stat-retry-exhausted-max", thresholds.StatRetryExhaustedMax, "degrade when stat-writer exhausted transient write retries exceed this value; <0 disables") + flag.Float64Var(&thresholds.ProcessUptimeMinSec, "process-uptime-min-seconds", thresholds.ProcessUptimeMinSec, "degrade when a service process has been up for fewer seconds than this value; <=0 disables") + flag.Float64Var(&thresholds.ProcessHeapAllocMaxBytes, "process-heap-alloc-max-bytes", thresholds.ProcessHeapAllocMaxBytes, "degrade when a service heap allocation exceeds this many bytes; <=0 disables") + flag.Float64Var(&thresholds.ProcessHeapSysMaxBytes, "process-heap-sys-max-bytes", thresholds.ProcessHeapSysMaxBytes, "degrade when a service heap reservation exceeds this many bytes; <=0 disables") + flag.Float64Var(&thresholds.ProcessGoroutinesMax, "process-goroutines-max", thresholds.ProcessGoroutinesMax, "degrade when a service goroutine count exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.KafkaLagMax, "kafka-lag-max", thresholds.KafkaLagMax, "degrade when summed Kafka lag across history/stat/realtime exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.KafkaConsumerCommitErrorMax, "kafka-consumer-commit-error-max", thresholds.KafkaConsumerCommitErrorMax, "degrade when Kafka consumer commit errors per topic exceed this value; <0 disables") + flag.Float64Var(&thresholds.KafkaConsumerMessageMinReceived, "kafka-consumer-message-min-received", thresholds.KafkaConsumerMessageMinReceived, "minimum Kafka consumer received messages per topic before uncommitted ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.KafkaConsumerMaxUncommittedRatio, "kafka-consumer-max-uncommitted-ratio", thresholds.KafkaConsumerMaxUncommittedRatio, "degrade when Kafka consumer uncommitted messages divided by received messages exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.DurableSpoolBacklogMax, "durable-spool-backlog-max", thresholds.DurableSpoolBacklogMax, "degrade when durable spool backlog files exceed this value; <=0 disables") + flag.Float64Var(&thresholds.DurableSpoolOldestAgeMaxSec, "durable-spool-oldest-age-max-seconds", thresholds.DurableSpoolOldestAgeMaxSec, "degrade when the oldest durable spool file is older than this many seconds; <=0 disables") + flag.Float64Var(&thresholds.DurableSpoolErrorMax, "durable-spool-error-max", thresholds.DurableSpoolErrorMax, "degrade when durable spool record write errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.DurableSpoolReplayErrorMax, "durable-spool-replay-error-max", thresholds.DurableSpoolReplayErrorMax, "degrade when durable spool replay errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.DurableOutboxInflightMax, "durable-outbox-inflight-max", thresholds.DurableOutboxInflightMax, "degrade when durable outbox asynchronous publishes in flight exceed this value; <=0 disables") + flag.Float64Var(&thresholds.DurableOutboxErrorMax, "durable-outbox-error-max", thresholds.DurableOutboxErrorMax, "degrade when durable outbox validation, submit, acknowledgement, removal, close, or quarantine errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.AsyncSinkEnqueueErrorMax, "async-sink-enqueue-error-max", thresholds.AsyncSinkEnqueueErrorMax, "degrade when gateway async sink enqueue timeout or closed events exceed this value; <0 disables") + flag.Float64Var(&thresholds.AsyncSinkPublishErrorMax, "async-sink-publish-error-max", thresholds.AsyncSinkPublishErrorMax, "degrade when gateway async sink background publish errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.AsyncSinkQueueDepthMax, "async-sink-queue-depth-max", thresholds.AsyncSinkQueueDepthMax, "degrade when gateway async sink queue depth exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.AsyncSinkQueueMaxUsageRatio, "async-sink-queue-max-usage-ratio", thresholds.AsyncSinkQueueMaxUsageRatio, "degrade when async sink queue depth divided by capacity exceeds this ratio; <=0 disables") + flag.Float64Var(&thresholds.FastWriterBatchPendingMax, "fast-writer-batch-pending-max", thresholds.FastWriterBatchPendingMax, "degrade when fast-writer in-flight batch messages exceed this value; <=0 disables") + flag.Float64Var(&thresholds.HistoryBatchPendingMax, "history-batch-pending-max", thresholds.HistoryBatchPendingMax, "degrade when history in-flight batch messages exceed this value; <=0 disables") + flag.Float64Var(&thresholds.HistoryRowsPendingMax, "history-rows-pending-max", thresholds.HistoryRowsPendingMax, "degrade when history in-flight TDengine rows exceed this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimeBatchPendingMax, "realtime-batch-pending-max", thresholds.RealtimeBatchPendingMax, "degrade when realtime in-flight batch messages exceed this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimeAsyncQueueDepthMax, "realtime-async-queue-depth-max", thresholds.RealtimeAsyncQueueDepthMax, "degrade when realtime async MySQL queue depth exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimeAsyncQueueMaxUsageRatio, "realtime-async-queue-max-usage-ratio", thresholds.RealtimeAsyncQueueMaxUsageRatio, "degrade when realtime async MySQL queue depth divided by capacity exceeds this ratio; <=0 disables") + flag.Float64Var(&thresholds.RealtimeAsyncQueueDroppedMax, "realtime-async-queue-dropped-max", thresholds.RealtimeAsyncQueueDroppedMax, "degrade when realtime async MySQL queue dropped or closed events exceed this value; <0 disables") + flag.Float64Var(&thresholds.HistoryParsedFieldMinFrames, "history-parsed-field-min-frames", thresholds.HistoryParsedFieldMinFrames, "minimum realtime history frames per topic/protocol before parsed field missing ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.HistoryParsedFieldMaxMissingRatio, "history-parsed-field-max-missing-ratio", thresholds.HistoryParsedFieldMaxMissingRatio, "degrade when history realtime frames missing parsed_fields divided by checked frames exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.MinHistogramSamples, "histogram-min-samples", thresholds.MinHistogramSamples, "minimum histogram sample count before latency thresholds are enforced") + flag.Float64Var(&thresholds.StatSampleMinWrites, "stat-sample-min-writes", thresholds.StatSampleMinWrites, "minimum successful stat append count per topic before requiring extracted mileage samples; <=0 disables") + flag.Float64Var(&thresholds.StatSampleMaxActionableSkipRatio, "stat-sample-max-actionable-skip-ratio", thresholds.StatSampleMaxActionableSkipRatio, "degrade when actionable stat sample skips divided by successful stat appends exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.StatSampleFutureAdjustmentMax, "stat-sample-future-adjustment-max", thresholds.StatSampleFutureAdjustmentMax, "degrade when stat-writer future event-time corrections exceed this value; <0 disables") + flag.Float64Var(&thresholds.StatSourceMinSamples, "stat-source-min-samples", thresholds.StatSourceMinSamples, "minimum stat samples found per topic before source tracking checks are enforced; <=0 disables") + flag.Float64Var(&thresholds.StatSourceMaxMissingRatio, "stat-source-max-missing-ratio", thresholds.StatSourceMaxMissingRatio, "degrade when source missing endpoint count divided by samples found exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.StatProjectionMinSampleWrites, "stat-projection-min-sample-writes", thresholds.StatProjectionMinSampleWrites, "minimum written stat samples per topic before requiring at least one final daily mileage projection; <=0 disables") + flag.Float64Var(&thresholds.StatBatchPendingMax, "stat-batch-pending-max", thresholds.StatBatchPendingMax, "degrade when stat-writer in-flight batch messages exceed this value; <=0 disables") + flag.Float64Var(&thresholds.ConsumerRetryPendingMax, "consumer-retry-pending-max", thresholds.ConsumerRetryPendingMax, "degrade when history/realtime/stat/identity messages waiting for failure-closed retry exceed this value; <0 disables") + flag.Float64Var(&thresholds.StatCacheEntriesMax, "stat-cache-entries-max", thresholds.StatCacheEntriesMax, "degrade when any stat-writer in-memory cache entry gauge exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimeThrottleMinSamples, "realtime-throttle-min-samples", thresholds.RealtimeThrottleMinSamples, "minimum realtime throttle events per store/protocol before requiring some passed MySQL projection writes; <=0 disables") + flag.Float64Var(&thresholds.RealtimeThrottleCacheEntriesMax, "realtime-throttle-cache-entries-max", thresholds.RealtimeThrottleCacheEntriesMax, "degrade when realtime MySQL throttle cache entry gauge exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.RealtimePlateCacheEntriesMax, "realtime-plate-cache-entries-max", thresholds.RealtimePlateCacheEntriesMax, "degrade when realtime VIN-to-plate cache entry gauge exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisEnvelopeMinSeen, "fast-writer-redis-envelope-min-seen", thresholds.FastWriterRedisEnvelopeMinSeen, "minimum Redis realtime envelopes seen per raw subject before update/skip checks are enforced; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisEnvelopeMaxBadRatio, "fast-writer-redis-envelope-max-bad-ratio", thresholds.FastWriterRedisEnvelopeMaxBadRatio, "degrade when missing-vin or missing-vehicle-key Redis envelopes divided by seen envelopes exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisFieldMinSeen, "fast-writer-redis-field-min-seen", thresholds.FastWriterRedisFieldMinSeen, "minimum Redis realtime fields seen per raw subject before stale field ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.FastWriterRedisFieldMaxStaleRatio, "fast-writer-redis-field-max-stale-ratio", thresholds.FastWriterRedisFieldMaxStaleRatio, "degrade when Redis stale/skipped realtime fields divided by seen fields exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.FastWriterMessageMinReceived, "fast-writer-message-min-received", thresholds.FastWriterMessageMinReceived, "minimum fast-writer received messages per raw subject before unacked ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.FastWriterMessageMaxUnackedRatio, "fast-writer-message-max-unacked-ratio", thresholds.FastWriterMessageMaxUnackedRatio, "degrade when fast-writer unacked messages divided by received messages exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.GatewayFieldMinRaw, "gateway-field-min-raw", thresholds.GatewayFieldMinRaw, "minimum gateway raw publishes per protocol before requiring at least one fields publish; <=0 disables") + flag.Float64Var(&thresholds.GatewayFieldMaxMissingRatio, "gateway-field-max-missing-ratio", thresholds.GatewayFieldMaxMissingRatio, "degrade when gateway fields missing/publish-error events divided by realtime candidate raw publishes exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.GatewayFieldMinCount, "gateway-field-min-count", thresholds.GatewayFieldMinCount, "degrade when a protocol has enough published fields events but the latest fields payload has fewer flattened fields than this; <=0 disables") + flag.Float64Var(&thresholds.GatewayBridgeMinPublish, "gateway-bridge-min-publishes", thresholds.GatewayBridgeMinPublish, "minimum gateway publishes per protocol/kind before requiring bridge Kafka writes to the expected topic; <=0 disables") + flag.Float64Var(&thresholds.GatewayFastWriterMinRawPublish, "gateway-fast-writer-min-raw-publishes", thresholds.GatewayFastWriterMinRawPublish, "minimum gateway raw publishes per protocol before requiring fast-writer messages on the expected raw subject; <=0 disables") + flag.Float64Var(&thresholds.GatewayParseMinFrames, "gateway-parse-min-frames", thresholds.GatewayParseMinFrames, "minimum gateway frames per protocol before non-OK parse ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.GatewayParseMaxBadRatio, "gateway-parse-max-bad-ratio", thresholds.GatewayParseMaxBadRatio, "degrade when gateway non-OK parse ratio exceeds this value after enough frames; <=0 disables") + flag.Float64Var(&thresholds.GatewayResponseMinSamples, "gateway-response-min-samples", thresholds.GatewayResponseMinSamples, "minimum gateway response attempts per protocol before response error ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.GatewayResponseMaxErrorRatio, "gateway-response-max-error-ratio", thresholds.GatewayResponseMaxErrorRatio, "degrade when gateway protocol response build/write error ratio exceeds this value after enough samples; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityMinSamples, "gateway-identity-min-samples", thresholds.GatewayIdentityMinSamples, "minimum gateway identity samples per protocol before unresolved ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityMaxUnresolvedRatio, "gateway-identity-max-unresolved-ratio", thresholds.GatewayIdentityMaxUnresolvedRatio, "degrade when gateway non-resolved identity ratio exceeds this value after enough samples; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityCacheEntriesMax, "gateway-identity-cache-entries-max", thresholds.GatewayIdentityCacheEntriesMax, "degrade when any gateway identity cache entry gauge exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentitySnapshotRequired, "gateway-identity-snapshot-required", thresholds.GatewayIdentitySnapshotRequired, "degrade when the gateway identity snapshot is not ready; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentitySnapshotMaxAgeSec, "gateway-identity-snapshot-max-age-seconds", thresholds.GatewayIdentitySnapshotMaxAgeSec, "degrade when the last successful identity snapshot refresh is older than this many seconds; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityLocationTouchFailureMax, "gateway-identity-location-touch-failure-max", thresholds.GatewayIdentityLocationTouchFailureMax, "degrade when jt808 registration location-touch failure backoff cache exceeds this value; <0 disables") + flag.Float64Var(&thresholds.GatewayIdentityRegistrationWriteQueueDepthMax, "gateway-identity-registration-write-queue-depth-max", thresholds.GatewayIdentityRegistrationWriteQueueDepthMax, "degrade when async JT808 registration write queue depth exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio, "gateway-identity-registration-write-queue-max-usage-ratio", thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio, "degrade when async JT808 registration write queue depth divided by capacity exceeds this ratio; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityRegistrationWriteErrorMax, "gateway-identity-registration-write-error-max", thresholds.GatewayIdentityRegistrationWriteErrorMax, "degrade when async/sync JT808 registration write errors exceed this value; <0 disables") + flag.Float64Var(&thresholds.GatewayIdentityStaleMinSamples, "gateway-identity-stale-min-samples", thresholds.GatewayIdentityStaleMinSamples, "minimum gateway identity samples per protocol before stale cache ratio is enforced; <=0 disables") + flag.Float64Var(&thresholds.GatewayIdentityMaxStaleCacheRatio, "gateway-identity-max-stale-cache-ratio", thresholds.GatewayIdentityMaxStaleCacheRatio, "degrade when stale identity cache resolutions divided by identity samples exceeds this value; <=0 disables") + flag.Float64Var(&thresholds.RawFanoutMinBridge, "raw-fanout-min-bridge-writes", thresholds.RawFanoutMinBridge, "minimum bridge writes per raw topic before requiring history writes and realtime updates; <=0 disables") + flag.Float64Var(&thresholds.StatTopicMinBridge, "stat-topic-min-bridge-writes", thresholds.StatTopicMinBridge, "minimum bridge writes per fields topic before requiring stat consumer topic info and received messages; <=0 disables") + flag.Float64Var(&thresholds.LastActivityStaleSec, "last-activity-stale-seconds", thresholds.LastActivityStaleSec, "degrade when an observed successful protocol/topic activity timestamp is older than this many seconds; <=0 disables") + flag.StringVar(&requiredConsumerTopics, "required-consumer-topics", formatRequiredConsumerTopics(capacity.DefaultRequiredConsumerTopics()), "semicolon separated service=topic|topic contract; empty disables required topic checks") + flag.StringVar(&dailyMileageDiagnosticsURL, "daily-mileage-diagnostics-url", defaultDailyMileageDiagnosticsURL, "daily mileage diagnosis reason summary URL; empty disables this business-health probe") + flag.StringVar(&dailyMileageDiagnosticsDate, "daily-mileage-diagnostics-date", "", "YYYY-MM-DD stat date for daily mileage diagnostics; empty lets the API use local current date") + flag.Int64Var(&dailyMileageDiagnosticsMaxActionable, "daily-mileage-diagnostics-max-actionable", -1, "degrade when daily mileage actionable issues exceed this value; <0 reports only") + flag.Int64Var(&dailyMileageDiagnosticsMaxPipeline, "daily-mileage-diagnostics-max-pipeline", -1, "degrade when daily mileage pipeline issues exceed this value; <0 reports only") + flag.Int64Var(&dailyMileageDiagnosticsMaxSourceData, "daily-mileage-diagnostics-max-source-data", -1, "degrade when daily mileage source-data issues exceed this value; <0 reports only") + flag.IntVar(&dailyMileageDiagnosticsSampleLimit, "daily-mileage-diagnostics-sample-limit", 5, "maximum daily mileage diagnosis detail samples to attach to the report; <=0 disables samples") + flag.StringVar(&dailyMileageSourceQualityURL, "daily-mileage-source-quality-url", defaultDailyMileageSourceQualityURL, "daily mileage source quality summary URL; empty disables this business-health probe") + flag.StringVar(&dailyMileageSourceQualityDate, "daily-mileage-source-quality-date", "", "YYYY-MM-DD stat date for daily mileage source quality; empty uses Asia/Shanghai current date") + flag.IntVar(&dailyMileageSourceQualityLimit, "daily-mileage-source-quality-limit", 1000, "maximum daily mileage source quality summary rows to fetch; <=0 fetches 1 row") + flag.Int64Var(&dailyMileageSourceQualityMaxInvalidDelta, "daily-mileage-source-quality-max-invalid-delta", -1, "degrade when INVALID_DELTA source count exceeds this value; <0 reports only") + flag.Int64Var(&dailyMileageSourceQualityMaxNoBaseline, "daily-mileage-source-quality-max-no-baseline", -1, "degrade when NO_PREVIOUS_BASELINE source count exceeds this value; <0 reports only") + flag.Int64Var(&dailyMileageSourceQualityMaxFallbackAfterInvalidBaseline, "daily-mileage-source-quality-max-fallback-after-invalid-baseline", -1, "degrade when OK sources using current-day fallback after invalid baseline exceed this value; <0 reports only") + flag.Int64Var(&dailyMileageSourceQualityMaxRealtimeLocationFallback, "daily-mileage-source-quality-max-realtime-location-fallback", -1, "degrade when OK sources using realtime-location historical fallback exceed this value; <0 reports only") + flag.StringVar(&dailyMileageSourceSelectionURL, "daily-mileage-source-selection-url", defaultDailyMileageSourceSelectionURL, "daily mileage source selection summary URL; empty disables this business-health probe") + flag.StringVar(&dailyMileageSourceSelectionDate, "daily-mileage-source-selection-date", "", "YYYY-MM-DD stat date for daily mileage source selection; empty uses Asia/Shanghai current date") + flag.IntVar(&dailyMileageSourceSelectionLimit, "daily-mileage-source-selection-limit", 1000, "maximum daily mileage source selection summary rows to fetch; <=0 fetches 1 row") + flag.IntVar(&dailyMileageSourceSelectionSampleLimit, "daily-mileage-source-selection-sample-limit", 5, "maximum problematic daily mileage source selection detail samples to attach; <=0 disables samples") + flag.Int64Var(&dailyMileageSourceSelectionMinSelected, "daily-mileage-source-selection-min-selected", -1, "degrade when selected source count is below this value; <0 reports only") + flag.StringVar(&dailyMileageSourceSelectionRequiredProtocols, "daily-mileage-source-selection-required-protocols", defaultDataSourceKindSuggestionProtocols, "comma separated protocols that must each have selected mileage sources when per-protocol selection gating is enabled") + flag.Int64Var(&dailyMileageSourceSelectionMinSelectedPerProtocol, "daily-mileage-source-selection-min-selected-per-protocol", -1, "degrade when any required protocol has fewer selected source rows than this value; <0 reports only") + flag.Int64Var(&dailyMileageSourceSelectionMaxUnmanaged, "daily-mileage-source-selection-max-unmanaged", -1, "degrade when unmanaged selected-candidate source count exceeds this value; <0 reports only") + flag.Int64Var(&dailyMileageSourceSelectionMaxUnknownKind, "daily-mileage-source-selection-max-unknown-kind", -1, "degrade when UNKNOWN source_kind selected-candidate source count exceeds this value; <0 reports only") + flag.Float64Var(&dailyMileageSourceSelectionMaxNotSelectedMileageKM, "daily-mileage-source-selection-max-not-selected-mileage-km", -1, "degrade when not-selected source mileage sum exceeds this value; <0 reports only") + flag.StringVar(&dataSourceDiagnosticsURL, "data-source-diagnostics-url", defaultDataSourceDiagnosticsURL, "data source diagnostics URL for missing source_code; empty disables this business-health probe") + flag.IntVar(&dataSourceDiagnosticsSampleLimit, "data-source-diagnostics-sample-limit", 5, "maximum data source diagnosis samples to attach to the report; <=0 keeps total only") + flag.IntVar(&dataSourceDiagnosticsReasonLimit, "data-source-diagnostics-reason-limit", 1000, "maximum data source diagnosis rows to fetch for reason/actionable summary; <=0 disables reason summary") + flag.Int64Var(&dataSourceDiagnosticsRecentAgeSeconds, "data-source-diagnostics-recent-age-seconds", 300, "classify missing source_code data sources as recent when latest_seen_age_seconds is at or below this value; <=0 disables recent summary") + flag.Int64Var(&dataSourceDiagnosticsMaxMissingSourceCode, "data-source-diagnostics-max-missing-source-code", -1, "degrade when missing source_code data sources exceed this value; <0 reports only") + flag.Int64Var(&dataSourceDiagnosticsMaxRecentMissingSourceCode, "data-source-diagnostics-max-recent-missing-source-code", -1, "degrade when recent missing source_code data sources exceed this value; <0 reports only") + flag.Int64Var(&dataSourceDiagnosticsMaxActionableCandidates, "data-source-diagnostics-max-actionable-candidates", -1, "degrade when actionable data source candidates exceed this value; <0 reports only") + flag.Int64Var(&dataSourceDiagnosticsMaxMappingIssues, "data-source-diagnostics-max-mapping-issues", -1, "degrade when configured data source mapping issues exceed this value; <0 reports only") + flag.Int64Var(&dataSourceDiagnosticsMaxRecentMappingIssues, "data-source-diagnostics-max-recent-mapping-issues", -1, "degrade when recent configured data source mapping issues exceed this value; <0 reports only") + flag.StringVar(&dataSourceKindSuggestionsURL, "data-source-kind-suggestions-url", defaultDataSourceKindSuggestionsURL, "data source kind suggestion URL; empty disables this business-health probe") + flag.StringVar(&dataSourceKindSuggestionsProtocols, "data-source-kind-suggestions-protocols", defaultDataSourceKindSuggestionProtocols, "comma separated protocols to check for UNKNOWN source_kind suggestions") + flag.IntVar(&dataSourceKindSuggestionsSampleLimit, "data-source-kind-suggestions-sample-limit", 5, "maximum source_kind diagnosis samples to attach to the report; <=0 keeps total only") + flag.IntVar(&dataSourceKindSuggestionsReasonLimit, "data-source-kind-suggestions-reason-limit", 1000, "maximum source_kind diagnosis rows to fetch for suggestion summary; <=0 disables reason summary") + flag.Int64Var(&dataSourceKindSuggestionsRecentAgeSeconds, "data-source-kind-suggestions-recent-age-seconds", 300, "classify UNKNOWN source_kind rows as recent when latest_seen_age_seconds is at or below this value; <=0 disables recent summary") + flag.Int64Var(&dataSourceKindSuggestionsMaxPlatformCandidates, "data-source-kind-suggestions-max-platform-candidates", -1, "degrade when UNKNOWN source_kind rows that should be PLATFORM exceed this value; <0 reports only") + flag.Int64Var(&dataSourceKindSuggestionsMaxRecentPlatformCandidates, "data-source-kind-suggestions-max-recent-platform-candidates", 0, "degrade when recent UNKNOWN source_kind rows that should be PLATFORM exceed this value; <0 reports only") + flag.StringVar(&jt808IdentityGapsURL, "jt808-identity-gaps-url", defaultJT808IdentityGapsURL, "JT808 identity gap diagnostics URL; empty disables this business-health probe") + flag.Int64Var(&jt808IdentityGapsRecentSeconds, "jt808-identity-gaps-recent-seconds", 86400, "only count JT808 unresolved identity registrations seen within this many seconds; <=0 checks all rows") + flag.IntVar(&jt808IdentityGapsSampleLimit, "jt808-identity-gaps-sample-limit", 5, "maximum JT808 identity gap samples to attach to the report; <=0 keeps total only") + flag.Int64Var(&jt808IdentityGapsMax, "jt808-identity-gaps-max", -1, "degrade when JT808 unresolved identity gaps exceed this value; <0 reports only") flag.Parse() + thresholds.RequiredConsumerTopics = parseRequiredConsumerTopics(requiredConsumerTopics) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - metricsByService, findings := collectMetrics(ctx, parseEndpoints(rawEndpoints), timeout) - report := capacity.Evaluate(metricsByService) + requiredServices := parseRequiredServices(rawRequiredServices) + endpoints := parseEndpoints(rawEndpoints) + metricsCtx, metricsCancel := context.WithTimeout(context.Background(), timeout) + metricsByService, findings := collectMetrics(metricsCtx, endpoints, timeout) + metricsCancel() + findings = append(findings, requiredServiceFindings(endpoints, metricsByService, requiredServices)...) + report := capacity.EvaluateWithThresholds(metricsByService, thresholds) report.Findings = append(report.Findings, findings...) + if strings.TrimSpace(dailyMileageDiagnosticsURL) != "" { + diagnosticsCtx, diagnosticsCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectDailyMileageDiagnostics(diagnosticsCtx, dailyMileageDiagnosticsURL, dailyMileageDiagnosticsDate, timeout, dailyMileageDiagnosticsSampleLimit) + diagnosticsCancel() + if diagnostics != nil { + report.DailyMileageDiagnostics = diagnostics + report.Findings = append(report.Findings, dailyMileageDiagnosticFindings( + diagnostics, + dailyMileageDiagnosticsMaxActionable, + dailyMileageDiagnosticsMaxPipeline, + dailyMileageDiagnosticsMaxSourceData, + )...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } + if strings.TrimSpace(dailyMileageSourceQualityURL) != "" { + sourceQualityCtx, sourceQualityCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectDailyMileageSourceQuality( + sourceQualityCtx, + dailyMileageSourceQualityURL, + dailyMileageSourceQualityDate, + timeout, + dailyMileageSourceQualityLimit, + ) + sourceQualityCancel() + if diagnostics != nil { + report.DailyMileageSourceQuality = diagnostics + report.Findings = append(report.Findings, dailyMileageSourceQualityFindings( + diagnostics, + dailyMileageSourceQualityMaxInvalidDelta, + dailyMileageSourceQualityMaxNoBaseline, + dailyMileageSourceQualityMaxFallbackAfterInvalidBaseline, + dailyMileageSourceQualityMaxRealtimeLocationFallback, + )...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } + if strings.TrimSpace(dailyMileageSourceSelectionURL) != "" { + sourceSelectionCtx, sourceSelectionCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectDailyMileageSourceSelection( + sourceSelectionCtx, + dailyMileageSourceSelectionURL, + dailyMileageSourceSelectionDate, + timeout, + dailyMileageSourceSelectionLimit, + dailyMileageSourceSelectionSampleLimit, + ) + sourceSelectionCancel() + if diagnostics != nil { + report.DailyMileageSourceSelection = diagnostics + report.Findings = append(report.Findings, dailyMileageSourceSelectionFindings( + diagnostics, + dailyMileageSourceSelectionMinSelected, + parseDataSourceKindSuggestionProtocols(dailyMileageSourceSelectionRequiredProtocols), + dailyMileageSourceSelectionMinSelectedPerProtocol, + dailyMileageSourceSelectionMaxUnmanaged, + dailyMileageSourceSelectionMaxUnknownKind, + dailyMileageSourceSelectionMaxNotSelectedMileageKM, + )...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } + if strings.TrimSpace(dataSourceDiagnosticsURL) != "" { + dataSourceCtx, dataSourceCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectDataSourceDiagnostics( + dataSourceCtx, + dataSourceDiagnosticsURL, + timeout, + dataSourceDiagnosticsSampleLimit, + dataSourceDiagnosticsReasonLimit, + dataSourceDiagnosticsRecentAgeSeconds, + ) + dataSourceCancel() + if diagnostics != nil { + report.DataSourceDiagnostics = diagnostics + report.Findings = append(report.Findings, dataSourceDiagnosticFindings( + diagnostics, + dataSourceDiagnosticsMaxMissingSourceCode, + dataSourceDiagnosticsMaxRecentMissingSourceCode, + dataSourceDiagnosticsMaxActionableCandidates, + dataSourceDiagnosticsMaxMappingIssues, + dataSourceDiagnosticsMaxRecentMappingIssues, + )...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } + if strings.TrimSpace(dataSourceKindSuggestionsURL) != "" { + dataSourceKindCtx, dataSourceKindCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectDataSourceKindSuggestions( + dataSourceKindCtx, + dataSourceKindSuggestionsURL, + timeout, + parseDataSourceKindSuggestionProtocols(dataSourceKindSuggestionsProtocols), + dataSourceKindSuggestionsSampleLimit, + dataSourceKindSuggestionsReasonLimit, + dataSourceKindSuggestionsRecentAgeSeconds, + ) + dataSourceKindCancel() + if diagnostics != nil { + report.DataSourceKindDiagnostics = diagnostics + report.Findings = append(report.Findings, dataSourceKindDiagnosticFindings( + diagnostics, + dataSourceKindSuggestionsMaxPlatformCandidates, + dataSourceKindSuggestionsMaxRecentPlatformCandidates, + )...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } + if strings.TrimSpace(jt808IdentityGapsURL) != "" { + identityGapCtx, identityGapCancel := context.WithTimeout(context.Background(), timeout) + diagnostics, diagnosticsFindings := collectJT808IdentityGaps( + identityGapCtx, + jt808IdentityGapsURL, + timeout, + jt808IdentityGapsRecentSeconds, + jt808IdentityGapsSampleLimit, + ) + identityGapCancel() + if diagnostics != nil { + report.JT808IdentityGaps = diagnostics + report.Findings = append(report.Findings, jt808IdentityGapFindings(diagnostics, jt808IdentityGapsMax)...) + } + report.Findings = append(report.Findings, diagnosticsFindings...) + } if len(report.Findings) > 0 { report.Status = capacity.StatusDegraded } @@ -46,6 +399,351 @@ func main() { } } +func dailyMileageDiagnosticFindings(diagnostics *capacity.DailyMileageDiagnostics, maxActionable, maxPipeline, maxSourceData int64) []string { + if diagnostics == nil { + return nil + } + var findings []string + if maxActionable >= 0 && diagnostics.ActionableIssueTotal > maxActionable { + findings = append(findings, fmt.Sprintf( + "daily mileage actionable issues %d exceeds %d (pipeline=%d, source_data=%d)", + diagnostics.ActionableIssueTotal, + maxActionable, + diagnostics.PipelineIssueTotal, + diagnostics.SourceDataIssueTotal, + )) + } + if maxPipeline >= 0 && diagnostics.PipelineIssueTotal > maxPipeline { + findings = append(findings, fmt.Sprintf( + "daily mileage pipeline issues %d exceeds %d (source_data=%d)", + diagnostics.PipelineIssueTotal, + maxPipeline, + diagnostics.SourceDataIssueTotal, + )) + } + if maxSourceData >= 0 && diagnostics.SourceDataIssueTotal > maxSourceData { + findings = append(findings, fmt.Sprintf( + "daily mileage source-data issues %d exceeds %d (pipeline=%d)", + diagnostics.SourceDataIssueTotal, + maxSourceData, + diagnostics.PipelineIssueTotal, + )) + } + return findings +} + +func dailyMileageSourceQualityFindings(diagnostics *capacity.DailyMileageSourceQualityDiagnostics, maxInvalidDelta, maxNoBaseline, maxFallbackAfterInvalidBaseline, maxRealtimeLocationFallback int64) []string { + if diagnostics == nil { + return nil + } + var findings []string + if maxInvalidDelta >= 0 && diagnostics.InvalidDeltaSourceCount > maxInvalidDelta { + findings = append(findings, fmt.Sprintf( + "daily mileage source INVALID_DELTA count %d exceeds %d", + diagnostics.InvalidDeltaSourceCount, + maxInvalidDelta, + )) + } + if maxNoBaseline >= 0 && diagnostics.NoBaselineSourceCount > maxNoBaseline { + findings = append(findings, fmt.Sprintf( + "daily mileage source NO_PREVIOUS_BASELINE count %d exceeds %d", + diagnostics.NoBaselineSourceCount, + maxNoBaseline, + )) + } + if maxFallbackAfterInvalidBaseline >= 0 && diagnostics.FallbackAfterInvalidBaselineSourceCount > maxFallbackAfterInvalidBaseline { + findings = append(findings, fmt.Sprintf( + "daily mileage source current-day fallback after invalid baseline count %d exceeds %d", + diagnostics.FallbackAfterInvalidBaselineSourceCount, + maxFallbackAfterInvalidBaseline, + )) + } + if maxRealtimeLocationFallback >= 0 && diagnostics.RealtimeLocationFallbackSourceCount > maxRealtimeLocationFallback { + findings = append(findings, fmt.Sprintf( + "daily mileage source realtime-location fallback count %d exceeds %d", + diagnostics.RealtimeLocationFallbackSourceCount, + maxRealtimeLocationFallback, + )) + } + return findings +} + +func dailyMileageSourceSelectionFindings(diagnostics *capacity.DailyMileageSourceSelectionDiagnostics, minSelected int64, requiredProtocols []string, minSelectedPerProtocol int64, maxUnmanaged, maxUnknownKind int64, maxNotSelectedMileageKM float64) []string { + if diagnostics == nil { + return nil + } + var findings []string + if minSelected >= 0 && diagnostics.SelectedSourceCount < minSelected { + findings = append(findings, fmt.Sprintf( + "daily mileage selected source count %d is below %d", + diagnostics.SelectedSourceCount, + minSelected, + )) + } + if minSelectedPerProtocol >= 0 { + byProtocol := dailyMileageSourceSelectionProtocolsByName(diagnostics.Protocols) + for _, protocol := range requiredProtocols { + protocol = strings.ToUpper(strings.TrimSpace(protocol)) + if protocol == "" { + continue + } + selected := byProtocol[protocol].SelectedSourceCount + if selected >= minSelectedPerProtocol { + continue + } + findings = append(findings, fmt.Sprintf( + "daily mileage selected source count for protocol %s is %d below %d", + protocol, + selected, + minSelectedPerProtocol, + )) + } + } + if maxUnmanaged >= 0 && diagnostics.UnmanagedSourceCount > maxUnmanaged { + findings = append(findings, fmt.Sprintf( + "daily mileage unmanaged source count %d exceeds %d", + diagnostics.UnmanagedSourceCount, + maxUnmanaged, + )) + } + if maxUnknownKind >= 0 && diagnostics.UnknownKindSourceCount > maxUnknownKind { + findings = append(findings, fmt.Sprintf( + "daily mileage UNKNOWN source_kind count %d exceeds %d", + diagnostics.UnknownKindSourceCount, + maxUnknownKind, + )) + } + if maxNotSelectedMileageKM >= 0 && diagnostics.NotSelectedMileageKM > maxNotSelectedMileageKM { + findings = append(findings, fmt.Sprintf( + "daily mileage not-selected source mileage %.1fkm exceeds %.1fkm", + diagnostics.NotSelectedMileageKM, + maxNotSelectedMileageKM, + )) + } + return findings +} + +func dailyMileageSourceSelectionProtocolsByName(items []capacity.DailyMileageSourceSelectionProtocol) map[string]capacity.DailyMileageSourceSelectionProtocol { + out := make(map[string]capacity.DailyMileageSourceSelectionProtocol, len(items)) + for _, item := range items { + protocol := strings.ToUpper(strings.TrimSpace(item.Protocol)) + if protocol == "" { + continue + } + out[protocol] = item + } + return out +} + +func dataSourceDiagnosticFindings(diagnostics *capacity.DataSourceDiagnostics, maxMissingSourceCode, maxRecentMissingSourceCode, maxActionableCandidates, maxMappingIssues, maxRecentMappingIssues int64) []string { + if diagnostics == nil { + return nil + } + var findings []string + if maxMissingSourceCode >= 0 && diagnostics.MissingSourceCodeTotal > maxMissingSourceCode { + findings = append(findings, fmt.Sprintf( + "data source missing source_code %d exceeds %d", + diagnostics.MissingSourceCodeTotal, + maxMissingSourceCode, + )) + } + if maxRecentMissingSourceCode >= 0 && diagnostics.RecentMissingSourceCodeTotal > maxRecentMissingSourceCode { + findings = append(findings, fmt.Sprintf( + "data source recent missing source_code %d exceeds %d (recent_age_seconds=%d)", + diagnostics.RecentMissingSourceCodeTotal, + maxRecentMissingSourceCode, + diagnostics.RecentAgeThresholdSeconds, + )) + } + if maxActionableCandidates >= 0 && diagnostics.ActionableCandidateTotal > maxActionableCandidates { + findings = append(findings, fmt.Sprintf( + "data source actionable candidates %d exceeds %d", + diagnostics.ActionableCandidateTotal, + maxActionableCandidates, + )) + } + if maxMappingIssues >= 0 && diagnostics.MappingIssueTotal > maxMappingIssues { + findings = append(findings, fmt.Sprintf( + "data source mapping issues %d exceeds %d", + diagnostics.MappingIssueTotal, + maxMappingIssues, + )) + } + if maxRecentMappingIssues >= 0 && diagnostics.RecentMappingIssueTotal > maxRecentMappingIssues { + findings = append(findings, fmt.Sprintf( + "data source recent mapping issues %d exceeds %d (recent_age_seconds=%d)", + diagnostics.RecentMappingIssueTotal, + maxRecentMappingIssues, + diagnostics.RecentAgeThresholdSeconds, + )) + } + return findings +} + +func dataSourceKindDiagnosticFindings(diagnostics *capacity.DataSourceDiagnostics, maxPlatformCandidates, maxRecentPlatformCandidates int64) []string { + if diagnostics == nil { + return nil + } + var findings []string + if maxPlatformCandidates >= 0 && diagnostics.PlatformKindCandidateTotal > maxPlatformCandidates { + findings = append(findings, fmt.Sprintf( + "data source UNKNOWN source_kind platform candidates %d exceeds %d", + diagnostics.PlatformKindCandidateTotal, + maxPlatformCandidates, + )) + } + if maxRecentPlatformCandidates >= 0 && diagnostics.RecentPlatformKindCandidateTotal > maxRecentPlatformCandidates { + findings = append(findings, fmt.Sprintf( + "data source recent UNKNOWN source_kind platform candidates %d exceeds %d (recent_age_seconds=%d)", + diagnostics.RecentPlatformKindCandidateTotal, + maxRecentPlatformCandidates, + diagnostics.RecentAgeThresholdSeconds, + )) + } + return findings +} + +func jt808IdentityGapFindings(diagnostics *capacity.JT808IdentityGapDiagnostics, maxGaps int64) []string { + if diagnostics == nil { + return nil + } + if maxGaps < 0 || diagnostics.Total <= maxGaps { + return nil + } + return []string{fmt.Sprintf( + "JT808 unresolved identity gaps %d exceeds %d (recent_seconds=%d)", + diagnostics.Total, + maxGaps, + diagnostics.RecentSeconds, + )} +} + +func formatRequiredConsumerTopics(values map[string][]string) string { + if len(values) == 0 { + return "" + } + var services []string + for service := range values { + service = strings.TrimSpace(service) + if service == "" { + continue + } + services = append(services, service) + } + sort.Strings(services) + parts := make([]string, 0, len(services)) + for _, service := range services { + var topics []string + for _, topic := range values[service] { + topic = strings.TrimSpace(topic) + if topic == "" { + continue + } + topics = append(topics, topic) + } + if len(topics) == 0 { + continue + } + sort.Strings(topics) + parts = append(parts, service+"="+strings.Join(topics, "|")) + } + return strings.Join(parts, ";") +} + +func parseRequiredConsumerTopics(value string) map[string][]string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + out := map[string][]string{} + for _, serviceSpec := range strings.Split(value, ";") { + serviceSpec = strings.TrimSpace(serviceSpec) + if serviceSpec == "" { + continue + } + service, rawTopics, ok := strings.Cut(serviceSpec, "=") + if !ok { + continue + } + service = strings.TrimSpace(service) + if service == "" { + continue + } + seen := map[string]struct{}{} + for _, topic := range strings.Split(rawTopics, "|") { + topic = strings.TrimSpace(topic) + if topic == "" { + continue + } + if _, exists := seen[topic]; exists { + continue + } + seen[topic] = struct{}{} + out[service] = append(out[service], topic) + } + } + if len(out) == 0 { + return nil + } + return out +} + +func parseRequiredServices(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + seen := map[string]struct{}{} + var services []string + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if _, exists := seen[item]; exists { + continue + } + seen[item] = struct{}{} + services = append(services, item) + } + return services +} + +func parseDataSourceKindSuggestionProtocols(value string) []string { + protocols := parseRequiredServices(value) + if len(protocols) == 0 { + protocols = parseRequiredServices(defaultDataSourceKindSuggestionProtocols) + } + for i, protocol := range protocols { + protocols[i] = strings.ToUpper(strings.TrimSpace(protocol)) + } + return protocols +} + +func requiredServiceFindings(endpoints []endpoint, metricsByService map[string]string, requiredServices []string) []string { + if len(requiredServices) == 0 { + return nil + } + configured := map[string]struct{}{} + for _, ep := range endpoints { + if ep.Name == "" { + continue + } + configured[ep.Name] = struct{}{} + } + var findings []string + for _, service := range requiredServices { + if _, ok := configured[service]; !ok { + findings = append(findings, fmt.Sprintf("required service endpoint missing: %s", service)) + continue + } + if _, ok := metricsByService[service]; !ok { + findings = append(findings, fmt.Sprintf("required service metrics missing: %s", service)) + } + } + return findings +} + func parseEndpoints(value string) []endpoint { var endpoints []endpoint for _, item := range strings.Split(value, ",") { @@ -107,3 +805,1055 @@ func collectMetrics(ctx context.Context, endpoints []endpoint, timeout time.Dura } return metricsByService, findings } + +func collectDailyMileageDiagnostics(ctx context.Context, rawURL string, date string, timeout time.Duration, sampleLimit int) (*capacity.DailyMileageDiagnostics, []string) { + requestURL, err := dailyMileageDiagnosticsRequestURL(rawURL, date) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage diagnostics request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage diagnostics request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage diagnostics fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage diagnostics read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("daily mileage diagnostics status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.DailyMileageDiagnosisReasonItem `json:"items"` + VehicleTotal int64 `json:"vehicle_total"` + ActionableIssueTotal int64 `json:"actionable_issue_total"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("daily mileage diagnostics decode failed: %v", err)} + } + items, pipelineIssues, sourceDataIssues := capacity.AnnotateDailyMileageDiagnostics(payload.Items) + samples, sampleFindings := collectDailyMileageDiagnosisSamples(ctx, rawURL, date, timeout, items, sampleLimit) + fieldStatus, fieldStatusFindings := collectDailyMileageFieldStatus(ctx, rawURL, date, timeout) + findings := append(sampleFindings, fieldStatusFindings...) + return &capacity.DailyMileageDiagnostics{ + URL: requestURL, + VehicleTotal: payload.VehicleTotal, + ActionableIssueTotal: payload.ActionableIssueTotal, + PipelineIssueTotal: pipelineIssues, + SourceDataIssueTotal: sourceDataIssues, + Items: items, + Samples: samples, + FieldStatus: fieldStatus, + }, findings +} + +func collectDailyMileageFieldStatus(ctx context.Context, rawURL string, date string, timeout time.Duration) (*capacity.DailyMileageFieldStatusDiagnostics, []string) { + requestURL, err := dailyMileageFieldStatusRequestURL(rawURL, date) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage field status request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage field status request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage field status fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage field status read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("daily mileage field status status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.DailyMileageFieldStatusSummaryItem `json:"items"` + Total int64 `json:"total"` + VehicleTotal int64 `json:"vehicle_total"` + ActionableIssueTotal int64 `json:"actionable_issue_total"` + PipelineIssueTotal int64 `json:"pipeline_issue_total"` + SourceDataIssueTotal int64 `json:"source_data_issue_total"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("daily mileage field status decode failed: %v", err)} + } + actionable, pipeline, sourceData := summarizeDailyMileageFieldStatus(payload.Items) + if payload.ActionableIssueTotal == 0 && actionable > 0 { + payload.ActionableIssueTotal = actionable + } + if payload.PipelineIssueTotal == 0 && pipeline > 0 { + payload.PipelineIssueTotal = pipeline + } + if payload.SourceDataIssueTotal == 0 && sourceData > 0 { + payload.SourceDataIssueTotal = sourceData + } + if payload.VehicleTotal == 0 { + for _, item := range payload.Items { + payload.VehicleTotal += item.Count + } + } + return &capacity.DailyMileageFieldStatusDiagnostics{ + URL: requestURL, + SummaryTotal: payload.Total, + VehicleTotal: payload.VehicleTotal, + ActionableIssueTotal: payload.ActionableIssueTotal, + PipelineIssueTotal: payload.PipelineIssueTotal, + SourceDataIssueTotal: payload.SourceDataIssueTotal, + Items: payload.Items, + }, nil +} + +func summarizeDailyMileageFieldStatus(items []capacity.DailyMileageFieldStatusSummaryItem) (actionable, pipeline, sourceData int64) { + for _, item := range items { + severity := strings.ToLower(strings.TrimSpace(item.Severity)) + if severity == "" { + severity = strings.ToLower(strings.TrimSpace(item.FieldStatusSeverity)) + } + switch severity { + case capacity.DailyMileageSeverityPipeline: + actionable += item.Count + pipeline += item.Count + case capacity.DailyMileageSeveritySourceData: + actionable += item.Count + sourceData += item.Count + default: + if !strings.EqualFold(strings.TrimSpace(item.Diagnosis), "OK") { + actionable += item.Count + } + } + } + return actionable, pipeline, sourceData +} + +func collectDailyMileageSourceQuality(ctx context.Context, rawURL string, date string, timeout time.Duration, limit int) (*capacity.DailyMileageSourceQualityDiagnostics, []string) { + requestURL, err := dailyMileageSourceQualityRequestURL(rawURL, date, limit) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source quality request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source quality request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source quality fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source quality read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("daily mileage source quality status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.DailyMileageSourceQualityItem `json:"items"` + Total int64 `json:"total"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("daily mileage source quality decode failed: %v", err)} + } + diagnostics := &capacity.DailyMileageSourceQualityDiagnostics{ + URL: requestURL, + SummaryTotal: payload.Total, + Items: payload.Items, + ItemsTruncated: payload.Total > int64(len(payload.Items)), + } + for _, item := range payload.Items { + diagnostics.SourceCount += item.SourceCount + diagnostics.VehicleCount += item.VehicleCount + diagnostics.SampleCount += item.SampleCount + if strings.EqualFold(strings.TrimSpace(item.QualityReason), "current_day_fallback_after_invalid_baseline") { + diagnostics.FallbackAfterInvalidBaselineSourceCount += item.SourceCount + } + if strings.EqualFold(strings.TrimSpace(item.QualityReason), "realtime_location_fallback_historical_baseline") { + diagnostics.RealtimeLocationFallbackSourceCount += item.SourceCount + } + switch strings.ToUpper(strings.TrimSpace(item.QualityStatus)) { + case "OK": + diagnostics.OKSourceCount += item.SourceCount + case "INVALID_DELTA": + diagnostics.InvalidDeltaSourceCount += item.SourceCount + case "NO_PREVIOUS_BASELINE": + diagnostics.NoBaselineSourceCount += item.SourceCount + } + } + if limit <= 0 || len(diagnostics.Items) == 0 { + diagnostics.Items = nil + } + return diagnostics, nil +} + +func collectDailyMileageSourceSelection(ctx context.Context, rawURL string, date string, timeout time.Duration, limit int, sampleLimit int) (*capacity.DailyMileageSourceSelectionDiagnostics, []string) { + requestURL, err := dailyMileageSourceSelectionRequestURL(rawURL, date, limit) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("daily mileage source selection status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.DailyMileageSourceSelectionItem `json:"items"` + Total int64 `json:"total"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection decode failed: %v", err)} + } + diagnostics := &capacity.DailyMileageSourceSelectionDiagnostics{ + URL: requestURL, + SummaryTotal: payload.Total, + Items: payload.Items, + ItemsTruncated: payload.Total > int64(len(payload.Items)), + } + protocols := map[string]capacity.DailyMileageSourceSelectionProtocol{} + for _, item := range payload.Items { + diagnostics.SourceCount += item.SourceCount + diagnostics.VehicleCount += item.VehicleCount + diagnostics.SampleCount += item.SampleCount + protocolName := strings.ToUpper(strings.TrimSpace(item.Protocol)) + protocolSummary := protocols[protocolName] + protocolSummary.Protocol = protocolName + protocolSummary.SourceCount += item.SourceCount + protocolSummary.VehicleCount += item.VehicleCount + protocolSummary.SampleCount += item.SampleCount + switch strings.ToLower(strings.TrimSpace(item.SelectionStatus)) { + case "selected": + diagnostics.SelectedSourceCount += item.SourceCount + protocolSummary.SelectedSourceCount += item.SourceCount + case "excluded": + diagnostics.ExcludedSourceCount += item.SourceCount + protocolSummary.ExcludedSourceCount += item.SourceCount + default: + diagnostics.NotSelectedSourceCount += item.SourceCount + diagnostics.NotSelectedMileageKM += item.DailyMileageKM + protocolSummary.NotSelectedSourceCount += item.SourceCount + protocolSummary.NotSelectedMileageKM += item.DailyMileageKM + } + reason := strings.ToLower(strings.TrimSpace(item.SelectionReason)) + if strings.Contains(reason, "unmanaged") { + diagnostics.UnmanagedSourceCount += item.SourceCount + protocolSummary.UnmanagedSourceCount += item.SourceCount + } + if strings.Contains(reason, "unknown_source_kind") { + diagnostics.UnknownKindSourceCount += item.SourceCount + protocolSummary.UnknownKindSourceCount += item.SourceCount + } + if protocolName != "" { + protocols[protocolName] = protocolSummary + } + } + diagnostics.Protocols = dailyMileageSourceSelectionProtocolSummaries(protocols) + if limit <= 0 || len(diagnostics.Items) == 0 { + diagnostics.Items = nil + } + var findings []string + if sampleLimit > 0 && (diagnostics.UnmanagedSourceCount > 0 || diagnostics.UnknownKindSourceCount > 0 || diagnostics.NotSelectedMileageKM > 0) { + samples, sampleFindings := collectDailyMileageSourceSelectionSamples(ctx, rawURL, date, timeout, sampleLimit) + diagnostics.Samples = samples + findings = append(findings, sampleFindings...) + } + return diagnostics, findings +} + +func dailyMileageSourceSelectionProtocolSummaries(values map[string]capacity.DailyMileageSourceSelectionProtocol) []capacity.DailyMileageSourceSelectionProtocol { + if len(values) == 0 { + return nil + } + protocols := make([]string, 0, len(values)) + for protocol := range values { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + out := make([]capacity.DailyMileageSourceSelectionProtocol, 0, len(protocols)) + for _, protocol := range protocols { + out = append(out, values[protocol]) + } + return out +} + +func collectDailyMileageSourceSelectionSamples(ctx context.Context, rawURL string, date string, timeout time.Duration, sampleLimit int) ([]capacity.DailyMileageSourceSelectionSample, []string) { + fetchLimit := dailyMileageSourceSelectionSampleFetchLimit(sampleLimit) + requestURL, err := dailyMileageSourceSelectionSampleRequestURL(rawURL, date, fetchLimit) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection sample request invalid: %v", err)} + } + samples, err := fetchDailyMileageSourceSelectionSamples(ctx, requestURL, timeout) + if err != nil { + return nil, []string{fmt.Sprintf("daily mileage source selection sample fetch failed: %v", err)} + } + out := make([]capacity.DailyMileageSourceSelectionSample, 0, sampleLimit) + for _, sample := range samples { + if !isProblematicDailyMileageSourceSelectionSample(sample) { + continue + } + out = append(out, sample) + if len(out) >= sampleLimit { + break + } + } + findings := enrichDailyMileageSourceSelectionSamples(ctx, rawURL, date, timeout, out) + return out, findings +} + +func enrichDailyMileageSourceSelectionSamples(ctx context.Context, rawURL string, date string, timeout time.Duration, samples []capacity.DailyMileageSourceSelectionSample) []string { + var findings []string + for index := range samples { + requestURL, err := dailyMileageSelectedSourceSampleRequestURL(rawURL, date, samples[index], 1) + if err != nil { + findings = append(findings, fmt.Sprintf("daily mileage selected source sample request invalid for vin %s: %v", samples[index].VIN, err)) + continue + } + samples[index].SelectedSourceQueryPath = requestPathWithQuery(requestURL) + selected, err := fetchDailyMileageSourceSelectionSamples(ctx, requestURL, timeout) + if err != nil { + findings = append(findings, fmt.Sprintf("daily mileage selected source sample fetch failed for vin %s: %v", samples[index].VIN, err)) + continue + } + if len(selected) == 0 { + continue + } + samples[index].SelectedSource = &selected[0] + } + return findings +} + +func fetchDailyMileageSourceSelectionSamples(ctx context.Context, requestURL string, timeout time.Duration) ([]capacity.DailyMileageSourceSelectionSample, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, err + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, fmt.Errorf("status %d: %s", response.StatusCode, strings.TrimSpace(string(body))) + } + var payload struct { + Items []capacity.DailyMileageSourceSelectionSample `json:"items"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, err + } + return payload.Items, nil +} + +func isProblematicDailyMileageSourceSelectionSample(sample capacity.DailyMileageSourceSelectionSample) bool { + reason := strings.ToLower(strings.TrimSpace(sample.SelectionReason)) + status := strings.ToLower(strings.TrimSpace(sample.SelectionStatus)) + return strings.Contains(reason, "unmanaged") || + strings.Contains(reason, "unknown_source_kind") || + (status == "not_selected" && sample.DailyMileageKM > 0) +} + +func dailyMileageSourceSelectionSampleFetchLimit(sampleLimit int) int { + if sampleLimit <= 0 { + return 0 + } + fetchLimit := sampleLimit * 20 + if fetchLimit < sampleLimit { + fetchLimit = sampleLimit + } + if fetchLimit > 1000 { + return 1000 + } + return fetchLimit +} + +func collectDataSourceDiagnostics(ctx context.Context, rawURL string, timeout time.Duration, sampleLimit int, reasonLimit int, recentAgeSeconds int64) (*capacity.DataSourceDiagnostics, []string) { + fetchLimit := dataSourceDiagnosticsFetchLimit(sampleLimit, reasonLimit) + requestURL, err := dataSourceDiagnosticsRequestURL(rawURL, fetchLimit) + if err != nil { + return nil, []string{fmt.Sprintf("data source diagnostics request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("data source diagnostics request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("data source diagnostics fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("data source diagnostics read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("data source diagnostics status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.DataSourceDiagnosticSample `json:"items"` + Total int64 `json:"total"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("data source diagnostics decode failed: %v", err)} + } + reasonCounts, actionableCandidates := summarizeDataSourceDiagnosticReasons(payload.Items) + recentMissing := countRecentDataSourceDiagnostics(payload.Items, recentAgeSeconds) + samples := payload.Items + if sampleLimit <= 0 || len(samples) == 0 { + samples = nil + } else if len(samples) > sampleLimit { + samples = samples[:sampleLimit] + } + if reasonLimit <= 0 { + reasonCounts = nil + actionableCandidates = 0 + } + diagnostics := &capacity.DataSourceDiagnostics{ + URL: requestURL, + MissingSourceCodeTotal: payload.Total, + RecentMissingSourceCodeTotal: recentMissing, + RecentAgeThresholdSeconds: normalizedRecentDataSourceAgeThreshold(recentAgeSeconds), + ActionableCandidateTotal: actionableCandidates, + ReasonCounts: reasonCounts, + ReasonCountsTruncated: reasonLimit > 0 && payload.Total > int64(len(payload.Items)), + Samples: samples, + } + + mappingDiagnostics, mappingFindings := collectDataSourceMappingIssueDiagnostics(ctx, rawURL, timeout, sampleLimit, reasonLimit, recentAgeSeconds) + if mappingDiagnostics != nil { + diagnostics.MappingIssueTotal = mappingDiagnostics.MappingIssueTotal + diagnostics.RecentMappingIssueTotal = mappingDiagnostics.RecentMappingIssueTotal + diagnostics.MappingIssueReasonCounts = mappingDiagnostics.MappingIssueReasonCounts + diagnostics.MappingIssueReasonCountsTruncated = mappingDiagnostics.MappingIssueReasonCountsTruncated + diagnostics.MappingIssueSamples = mappingDiagnostics.MappingIssueSamples + } + return diagnostics, mappingFindings +} + +func collectDataSourceMappingIssueDiagnostics(ctx context.Context, rawURL string, timeout time.Duration, sampleLimit int, reasonLimit int, recentAgeSeconds int64) (*capacity.DataSourceDiagnostics, []string) { + fetchLimit := dataSourceDiagnosticsFetchLimit(sampleLimit, reasonLimit) + requestURL, err := dataSourceMappingIssuesRequestURL(rawURL, fetchLimit) + if err != nil { + return nil, []string{fmt.Sprintf("data source mapping diagnostics request invalid: %v", err)} + } + payload, findings := fetchDataSourceDiagnosticPage(ctx, requestURL, timeout, "data source mapping diagnostics") + if len(findings) > 0 { + return nil, findings + } + reasonCounts, _ := summarizeDataSourceDiagnosticReasons(payload.Items) + recentMappingIssues := countRecentDataSourceDiagnostics(payload.Items, recentAgeSeconds) + samples := payload.Items + if sampleLimit <= 0 || len(samples) == 0 { + samples = nil + } else if len(samples) > sampleLimit { + samples = samples[:sampleLimit] + } + if reasonLimit <= 0 { + reasonCounts = nil + } + return &capacity.DataSourceDiagnostics{ + MappingIssueTotal: payload.Total, + RecentMappingIssueTotal: recentMappingIssues, + RecentAgeThresholdSeconds: normalizedRecentDataSourceAgeThreshold(recentAgeSeconds), + MappingIssueReasonCounts: reasonCounts, + MappingIssueReasonCountsTruncated: reasonLimit > 0 && payload.Total > int64(len(payload.Items)), + MappingIssueSamples: samples, + }, nil +} + +func collectDataSourceKindSuggestions(ctx context.Context, rawURL string, timeout time.Duration, protocols []string, sampleLimit int, reasonLimit int, recentAgeSeconds int64) (*capacity.DataSourceDiagnostics, []string) { + fetchLimit := dataSourceDiagnosticsFetchLimit(sampleLimit, reasonLimit) + if len(protocols) == 0 { + protocols = parseDataSourceKindSuggestionProtocols(defaultDataSourceKindSuggestionProtocols) + } + var findings []string + var samples []capacity.DataSourceDiagnosticSample + var unknownTotal int64 + var recentUnknown int64 + var platformCandidates int64 + var recentPlatformCandidates int64 + reasonCounts := map[string]int64{} + reasonSummaryEnabled := reasonLimit > 0 + reasonCountsTruncated := false + var firstURL string + for _, protocol := range protocols { + requestURL, err := dataSourceKindSuggestionsRequestURL(rawURL, protocol, fetchLimit) + if err != nil { + findings = append(findings, fmt.Sprintf("data source kind suggestions request invalid for %s: %v", protocol, err)) + continue + } + if firstURL == "" { + firstURL = requestURL + } + payload, fetchFindings := fetchDataSourceDiagnosticPage(ctx, requestURL, timeout, "data source kind suggestions") + if len(fetchFindings) > 0 { + findings = append(findings, fetchFindings...) + continue + } + unknownTotal += payload.Total + protocolRecentUnknown := countRecentDataSourceDiagnostics(payload.Items, recentAgeSeconds) + recentUnknown += protocolRecentUnknown + protocolReasonCounts, protocolPlatformCandidates, protocolRecentPlatformCandidates := summarizeDataSourceKindSuggestionReasons(payload.Items, recentAgeSeconds) + platformCandidates += protocolPlatformCandidates + recentPlatformCandidates += protocolRecentPlatformCandidates + if reasonSummaryEnabled { + mergeInt64Counts(reasonCounts, protocolReasonCounts) + reasonCountsTruncated = reasonCountsTruncated || payload.Total > int64(len(payload.Items)) + } + if sampleLimit > 0 && len(samples) < sampleLimit { + for _, item := range payload.Items { + if !isPlatformKindCandidate(item) { + continue + } + samples = append(samples, item) + if len(samples) >= sampleLimit { + break + } + } + } + } + if !reasonSummaryEnabled || len(reasonCounts) == 0 { + reasonCounts = nil + } + if sampleLimit <= 0 || len(samples) == 0 { + samples = nil + } + return &capacity.DataSourceDiagnostics{ + URL: firstURL, + UnknownSourceKindTotal: unknownTotal, + RecentUnknownSourceKindTotal: recentUnknown, + RecentAgeThresholdSeconds: normalizedRecentDataSourceAgeThreshold(recentAgeSeconds), + PlatformKindCandidateTotal: platformCandidates, + RecentPlatformKindCandidateTotal: recentPlatformCandidates, + ReasonCounts: reasonCounts, + ReasonCountsTruncated: reasonCountsTruncated, + Samples: samples, + }, findings +} + +func collectJT808IdentityGaps(ctx context.Context, rawURL string, timeout time.Duration, recentSeconds int64, sampleLimit int) (*capacity.JT808IdentityGapDiagnostics, []string) { + requestURL, err := jt808IdentityGapsRequestURL(rawURL, recentSeconds, sampleLimit) + if err != nil { + return nil, []string{fmt.Sprintf("JT808 identity gaps request invalid: %v", err)} + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("JT808 identity gaps request invalid: %v", err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, []string{fmt.Sprintf("JT808 identity gaps fetch failed: %v", err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, []string{fmt.Sprintf("JT808 identity gaps read failed: %v", err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, []string{fmt.Sprintf("JT808 identity gaps status %d: %s", response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload struct { + Items []capacity.JT808IdentityGapSample `json:"items"` + Total int64 `json:"total"` + RecentSeconds int64 `json:"recentSeconds"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, []string{fmt.Sprintf("JT808 identity gaps decode failed: %v", err)} + } + samples := payload.Items + if sampleLimit <= 0 || len(samples) == 0 { + samples = nil + } else if len(samples) > sampleLimit { + samples = samples[:sampleLimit] + } + return &capacity.JT808IdentityGapDiagnostics{ + URL: requestURL, + Total: payload.Total, + RecentSeconds: payload.RecentSeconds, + Samples: samples, + }, nil +} + +type dataSourceDiagnosticPayload struct { + Items []capacity.DataSourceDiagnosticSample `json:"items"` + Total int64 `json:"total"` +} + +func fetchDataSourceDiagnosticPage(ctx context.Context, requestURL string, timeout time.Duration, label string) (dataSourceDiagnosticPayload, []string) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return dataSourceDiagnosticPayload{}, []string{fmt.Sprintf("%s request invalid: %v", label, err)} + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return dataSourceDiagnosticPayload{}, []string{fmt.Sprintf("%s fetch failed: %v", label, err)} + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return dataSourceDiagnosticPayload{}, []string{fmt.Sprintf("%s read failed: %v", label, err)} + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return dataSourceDiagnosticPayload{}, []string{fmt.Sprintf("%s status %d: %s", label, response.StatusCode, strings.TrimSpace(string(body)))} + } + var payload dataSourceDiagnosticPayload + if err := json.Unmarshal(body, &payload); err != nil { + return dataSourceDiagnosticPayload{}, []string{fmt.Sprintf("%s decode failed: %v", label, err)} + } + return payload, nil +} + +func dataSourceDiagnosticsFetchLimit(sampleLimit int, reasonLimit int) int { + limit := sampleLimit + if reasonLimit > limit { + limit = reasonLimit + } + if limit < 0 { + return 0 + } + return limit +} + +func summarizeDataSourceDiagnosticReasons(items []capacity.DataSourceDiagnosticSample) (map[string]int64, int64) { + if len(items) == 0 { + return nil, 0 + } + counts := make(map[string]int64) + var actionable int64 + for _, item := range items { + reason := strings.TrimSpace(item.Reason) + if reason == "" { + reason = "unknown" + } + counts[reason]++ + if reason == "candidate_available" { + actionable++ + } + } + return counts, actionable +} + +func countRecentDataSourceDiagnostics(items []capacity.DataSourceDiagnosticSample, recentAgeSeconds int64) int64 { + if recentAgeSeconds <= 0 { + return 0 + } + var count int64 + for _, item := range items { + if item.LatestSeenAgeSeconds <= recentAgeSeconds { + count++ + } + } + return count +} + +func summarizeDataSourceKindSuggestionReasons(items []capacity.DataSourceDiagnosticSample, recentAgeSeconds int64) (map[string]int64, int64, int64) { + if len(items) == 0 { + return nil, 0, 0 + } + counts := make(map[string]int64) + var platformCandidates int64 + var recentPlatformCandidates int64 + for _, item := range items { + reason := strings.TrimSpace(item.SuggestionReason) + if reason == "" { + reason = strings.TrimSpace(item.Reason) + } + if reason == "" { + reason = "unknown" + } + counts[reason]++ + if !isPlatformKindCandidate(item) { + continue + } + platformCandidates++ + if recentAgeSeconds > 0 && item.LatestSeenAgeSeconds <= recentAgeSeconds { + recentPlatformCandidates++ + } + } + return counts, platformCandidates, recentPlatformCandidates +} + +func isPlatformKindCandidate(item capacity.DataSourceDiagnosticSample) bool { + if !strings.EqualFold(strings.TrimSpace(item.SuggestedSourceKind), "PLATFORM") { + return false + } + return !strings.EqualFold(strings.TrimSpace(item.SuggestionConfidence), "LOW") +} + +func mergeInt64Counts(dst map[string]int64, src map[string]int64) { + for key, value := range src { + if key == "" || value == 0 { + continue + } + dst[key] += value + } +} + +func normalizedRecentDataSourceAgeThreshold(recentAgeSeconds int64) int64 { + if recentAgeSeconds <= 0 { + return 0 + } + return recentAgeSeconds +} + +func collectDailyMileageDiagnosisSamples(ctx context.Context, rawURL string, date string, timeout time.Duration, items []capacity.DailyMileageDiagnosisReasonItem, sampleLimit int) ([]capacity.DailyMileageDiagnosisSample, []string) { + if sampleLimit <= 0 { + return nil, nil + } + var samples []capacity.DailyMileageDiagnosisSample + var findings []string + seen := make(map[string]struct{}) + for _, item := range items { + if len(samples) >= sampleLimit { + break + } + if capacity.DailyMileageIssueSeverity(item) == capacity.DailyMileageSeverityOK || item.Count <= 0 { + continue + } + key := strings.ToUpper(strings.TrimSpace(item.Protocol)) + "|" + strings.ToUpper(strings.TrimSpace(item.Diagnosis)) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + requestURL, err := dailyMileageDiagnosticsSampleRequestURL(rawURL, date, item, sampleLimit-len(samples)) + if err != nil { + findings = append(findings, fmt.Sprintf("daily mileage diagnostics sample request invalid: %v", err)) + continue + } + fetched, err := fetchDailyMileageDiagnosisSamples(ctx, requestURL, timeout) + if err != nil { + findings = append(findings, fmt.Sprintf("daily mileage diagnostics sample fetch failed: %v", err)) + continue + } + for _, sample := range fetched { + if len(samples) >= sampleLimit { + break + } + sample.Severity = capacity.DailyMileageIssueSeverity(capacity.DailyMileageDiagnosisReasonItem{ + Diagnosis: sample.Diagnosis, + Reason: sample.Reason, + Count: 1, + }) + if strings.TrimSpace(sample.RecommendedOperatorAction) == "" { + sample.RecommendedOperatorAction = capacity.DailyMileageRecommendedOperatorAction(sample.Diagnosis, sample.Reason) + } + samples = append(samples, sample) + } + } + return samples, findings +} + +func fetchDailyMileageDiagnosisSamples(ctx context.Context, requestURL string, timeout time.Duration) ([]capacity.DailyMileageDiagnosisSample, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) + if err != nil { + return nil, err + } + response, err := (&http.Client{Timeout: timeout}).Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, fmt.Errorf("status %d: %s", response.StatusCode, strings.TrimSpace(string(body))) + } + var payload struct { + Items []capacity.DailyMileageDiagnosisSample `json:"items"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, err + } + return payload.Items, nil +} + +func dailyMileageDiagnosticsRequestURL(rawURL string, date string) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDailyMileageDiagnosticsPath + } + query := parsed.Query() + if date = strings.TrimSpace(date); date != "" { + query.Set("date", date) + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageFieldStatusRequestURL(rawURL string, date string) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + switch { + case parsed.Path == "" || parsed.Path == "/": + parsed.Path = defaultDailyMileageFieldStatusPath + case strings.HasSuffix(parsed.Path, "/reasons"): + parsed.Path = strings.TrimSuffix(parsed.Path, "/reasons") + "/field-status" + case strings.HasSuffix(parsed.Path, "/summary"): + parsed.Path = strings.TrimSuffix(parsed.Path, "/summary") + "/field-status" + case strings.HasSuffix(parsed.Path, "/diagnostics"): + parsed.Path = parsed.Path + "/field-status" + case strings.HasSuffix(parsed.Path, "/field-status"): + default: + parsed.Path = defaultDailyMileageFieldStatusPath + } + query := parsed.Query() + if date = strings.TrimSpace(date); date != "" { + query.Set("date", date) + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageSourceQualityRequestURL(rawURL string, date string, limit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDailyMileageSourceQualityPath + } + limit = normalizedPositiveLimit(limit) + statDate := dailyMileageSourceQualityDate(date) + query := parsed.Query() + query.Set("dateFrom", statDate) + query.Set("dateTo", statDate) + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageSourceSelectionRequestURL(rawURL string, date string, limit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDailyMileageSourceSelectionPath + } + limit = normalizedPositiveLimit(limit) + statDate := dailyMileageSourceQualityDate(date) + query := parsed.Query() + query.Set("dateFrom", statDate) + query.Set("dateTo", statDate) + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageSourceSelectionSampleRequestURL(rawURL string, date string, limit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + path := strings.TrimRight(parsed.Path, "/") + switch { + case path == "": + parsed.Path = defaultDailyMileageSourcesPath + case strings.HasSuffix(path, "/selection"): + parsed.Path = strings.TrimSuffix(path, "/selection") + case strings.HasSuffix(path, "/quality"): + parsed.Path = strings.TrimSuffix(path, "/quality") + case path == defaultDailyMileageSourcesPath: + parsed.Path = defaultDailyMileageSourcesPath + default: + parsed.Path = defaultDailyMileageSourcesPath + } + limit = normalizedPositiveLimit(limit) + statDate := dailyMileageSourceQualityDate(date) + query := parsed.Query() + query.Set("dateFrom", statDate) + query.Set("dateTo", statDate) + query.Set("selected", "false") + query.Set("includeTotal", "false") + query.Set("orderBy", "dailyMileageDesc") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageSelectedSourceSampleRequestURL(rawURL string, date string, sample capacity.DailyMileageSourceSelectionSample, limit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + parsed.Path = defaultDailyMileageSourcesPath + limit = normalizedPositiveLimit(limit) + statDate := dailyMileageSourceQualityDate(date) + query := parsed.Query() + query.Set("dateFrom", statDate) + query.Set("dateTo", statDate) + query.Set("selected", "true") + query.Set("includeTotal", "false") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + if vin := strings.TrimSpace(sample.VIN); vin != "" { + query.Set("vin", vin) + } + if protocol := strings.TrimSpace(sample.Protocol); protocol != "" { + query.Set("protocol", protocol) + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func requestPathWithQuery(requestURL string) string { + parsed, err := url.Parse(requestURL) + if err != nil { + return requestURL + } + if parsed.RawQuery == "" { + return parsed.EscapedPath() + } + return parsed.EscapedPath() + "?" + parsed.RawQuery +} + +func dailyMileageSourceQualityDate(date string) string { + if date = strings.TrimSpace(date); date != "" { + return date + } + return time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600)).Format("2006-01-02") +} + +func normalizedPositiveLimit(limit int) int { + if limit <= 0 { + return 1 + } + return limit +} + +func dataSourceDiagnosticsRequestURL(rawURL string, sampleLimit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDataSourceDiagnosticsPath + } + limit := normalizedPositiveLimit(sampleLimit) + query := parsed.Query() + query.Set("sourceCodeMissing", "true") + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dataSourceMappingIssuesRequestURL(rawURL string, sampleLimit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDataSourceDiagnosticsPath + } + limit := normalizedPositiveLimit(sampleLimit) + query := parsed.Query() + query.Set("sourceCodeMissing", "false") + query.Set("mappingIssueOnly", "true") + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dataSourceKindSuggestionsRequestURL(rawURL string, protocol string, sampleLimit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultDataSourceKindSuggestionsPath + } + limit := normalizedPositiveLimit(sampleLimit) + query := parsed.Query() + query.Set("protocol", strings.ToUpper(strings.TrimSpace(protocol))) + query.Set("sourceKind", "UNKNOWN") + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func jt808IdentityGapsRequestURL(rawURL string, recentSeconds int64, sampleLimit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + if parsed.Path == "" || parsed.Path == "/" { + parsed.Path = defaultJT808IdentityGapsPath + } + limit := normalizedPositiveLimit(sampleLimit) + query := parsed.Query() + query.Set("includeTotal", "true") + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + if recentSeconds >= 0 { + query.Set("recentSeconds", fmt.Sprintf("%d", recentSeconds)) + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func dailyMileageDiagnosticsSampleRequestURL(rawURL string, date string, item capacity.DailyMileageDiagnosisReasonItem, limit int) (string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("invalid URL %q", rawURL) + } + switch { + case parsed.Path == "" || parsed.Path == "/": + parsed.Path = "/api/stats/daily-metrics/diagnostics" + case strings.HasSuffix(parsed.Path, "/reasons"): + parsed.Path = strings.TrimSuffix(parsed.Path, "/reasons") + case strings.HasSuffix(parsed.Path, "/summary"): + parsed.Path = strings.TrimSuffix(parsed.Path, "/summary") + } + if !strings.HasSuffix(parsed.Path, "/diagnostics") { + parsed.Path = "/api/stats/daily-metrics/diagnostics" + } + query := parsed.Query() + query.Del("reason") + if date == "" { + date = item.StatDate + } + if date != "" { + query.Set("date", date) + } + query.Set("protocol", strings.ToUpper(strings.TrimSpace(item.Protocol))) + query.Set("diagnosis", strings.ToUpper(strings.TrimSpace(item.Diagnosis))) + query.Set("limit", fmt.Sprintf("%d", limit)) + query.Set("offset", "0") + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} diff --git a/go/vehicle-gateway/cmd/capacity-check/main_test.go b/go/vehicle-gateway/cmd/capacity-check/main_test.go index 34d35363..08ee0b08 100644 --- a/go/vehicle-gateway/cmd/capacity-check/main_test.go +++ b/go/vehicle-gateway/cmd/capacity-check/main_test.go @@ -1,8 +1,14 @@ package main import ( + "context" + "net/http" + "net/http/httptest" "strings" "testing" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/capacity" ) func TestParseEndpoints(t *testing.T) { @@ -29,3 +35,1116 @@ func TestParseEndpointsUsesDefaultMetricsURLWhenOnlyBaseURLIsProvided(t *testing t.Fatalf("url = %q, want /metrics suffix", endpoints[0].URL) } } + +func TestDefaultEndpointsSplitRealtimeAPIAndWriter(t *testing.T) { + endpoints := parseEndpoints(defaultEndpoints) + byName := map[string]string{} + for _, endpoint := range endpoints { + byName[endpoint.Name] = endpoint.URL + } + if got, want := byName["realtime"], "http://127.0.0.1:20216/metrics"; got != want { + t.Fatalf("realtime endpoint = %q, want %q", got, want) + } + if got, want := byName["realtime-api"], "http://127.0.0.1:20200/metrics"; got != want { + t.Fatalf("realtime-api endpoint = %q, want %q", got, want) + } + if got, want := byName["identity"], "http://127.0.0.1:20217/metrics"; got != want { + t.Fatalf("identity endpoint = %q, want %q", got, want) + } + if got, want := byName["fields-projector"], "http://127.0.0.1:20218/metrics"; got != want { + t.Fatalf("fields-projector endpoint = %q, want %q", got, want) + } + services := strings.Join(parseRequiredServices(defaultRequiredServices), ",") + for _, want := range []string{"realtime", "identity", "realtime-api", "fields-projector"} { + if !strings.Contains(services, want) { + t.Fatalf("required services %q missing %q", services, want) + } + } +} + +func TestParseRequiredConsumerTopics(t *testing.T) { + parsed := parseRequiredConsumerTopics("history=a|b|a; stat = c | d ; ignored") + + if got := strings.Join(parsed["history"], ","); got != "a,b" { + t.Fatalf("history topics = %q, want a,b", got) + } + if got := strings.Join(parsed["stat"], ","); got != "c,d" { + t.Fatalf("stat topics = %q, want c,d", got) + } + if _, exists := parsed["ignored"]; exists { + t.Fatalf("unexpected ignored service in %#v", parsed) + } +} + +func TestParseRequiredConsumerTopicsReturnsNilForEmptyValue(t *testing.T) { + if parsed := parseRequiredConsumerTopics(" "); parsed != nil { + t.Fatalf("parsed = %#v, want nil", parsed) + } +} + +func TestParseRequiredServicesDeduplicatesAndPreservesOrder(t *testing.T) { + services := parseRequiredServices("gateway, history, gateway, stat") + + if got := strings.Join(services, ","); got != "gateway,history,stat" { + t.Fatalf("services = %q, want gateway,history,stat", got) + } +} + +func TestParseRequiredServicesReturnsNilForEmptyValue(t *testing.T) { + if services := parseRequiredServices(" "); services != nil { + t.Fatalf("services = %#v, want nil", services) + } +} + +func TestRequiredServiceFindingsReportsMissingEndpoint(t *testing.T) { + findings := requiredServiceFindings( + []endpoint{{Name: "gateway", URL: "http://127.0.0.1:20211/metrics"}}, + map[string]string{"gateway": "vehicle_service_info 1"}, + []string{"gateway", "history"}, + ) + + if len(findings) != 1 || findings[0] != "required service endpoint missing: history" { + t.Fatalf("findings = %#v", findings) + } +} + +func TestRequiredServiceFindingsReportsConfiguredButUnscrapedService(t *testing.T) { + findings := requiredServiceFindings( + []endpoint{ + {Name: "gateway", URL: "http://127.0.0.1:20211/metrics"}, + {Name: "history", URL: "http://127.0.0.1:20212/metrics"}, + }, + map[string]string{"gateway": "vehicle_service_info 1"}, + []string{"gateway", "history"}, + ) + + if len(findings) != 1 || findings[0] != "required service metrics missing: history" { + t.Fatalf("findings = %#v", findings) + } +} + +func TestRequiredServiceFindingsCanBeDisabled(t *testing.T) { + findings := requiredServiceFindings(nil, nil, nil) + + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } +} + +func TestFormatRequiredConsumerTopicsIsStable(t *testing.T) { + formatted := formatRequiredConsumerTopics(map[string][]string{ + "stat": {"vehicle.fields.go.jt808.v1", "vehicle.fields.go.gb32960.v1"}, + "history": {"vehicle.raw.go.jt808.v1"}, + }) + + want := "history=vehicle.raw.go.jt808.v1;stat=vehicle.fields.go.gb32960.v1|vehicle.fields.go.jt808.v1" + if formatted != want { + t.Fatalf("formatted = %q, want %q", formatted, want) + } +} + +func TestDailyMileageDiagnosticsRequestURLUsesDefaultPathAndDate(t *testing.T) { + requestURL, err := dailyMileageDiagnosticsRequestURL("http://127.0.0.1:20200", "2026-07-12") + if err != nil { + t.Fatalf("request URL error: %v", err) + } + if !strings.Contains(requestURL, defaultDailyMileageDiagnosticsPath) { + t.Fatalf("url = %q, want default diagnostics path", requestURL) + } + if !strings.Contains(requestURL, "date=2026-07-12") { + t.Fatalf("url = %q, want date query", requestURL) + } +} + +func TestDailyMileageFieldStatusRequestURLDerivesFromReasonsPath(t *testing.T) { + requestURL, err := dailyMileageFieldStatusRequestURL("http://127.0.0.1:20200/api/stats/daily-metrics/diagnostics/reasons?severity=source_data", "2026-07-12") + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDailyMileageFieldStatusPath, + "date=2026-07-12", + "severity=source_data", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } + if strings.Contains(requestURL, "/reasons") { + t.Fatalf("url = %q should not keep /reasons", requestURL) + } +} + +func TestCollectDailyMileageDiagnostics(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("date"); got != "2026-07-12" { + t.Fatalf("date = %q, want 2026-07-12", got) + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case defaultDailyMileageDiagnosticsPath: + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"YUTONG_MQTT","diagnosis":"NO_SOURCE_SAMPLE","reason":"realtime_location_has_total_mileage_but_no_stat_sample","count":7}, + {"stat_date":"2026-07-12","protocol":"JT808","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_non_positive","count":2} + ], + "vehicle_total":349, + "actionable_issue_total":9 + }`)) + case defaultDailyMileageFieldStatusPath: + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"JT808","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_non_positive","realtime_mileage_field_status":"standard_field_non_positive","count":2,"severity":"source_data","field_status_severity":"source_data","field_status_action":"源头总里程小于等于 0"} + ], + "total":1, + "vehicle_total":349, + "actionable_issue_total":9, + "pipeline_issue_total":7, + "source_data_issue_total":2 + }`)) + default: + t.Fatalf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + diagnostics, findings := collectDailyMileageDiagnostics(context.Background(), server.URL, "2026-07-12", time.Second, 0) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.VehicleTotal != 349 || diagnostics.ActionableIssueTotal != 9 { + t.Fatalf("diagnostics = %#v", diagnostics) + } + if diagnostics.PipelineIssueTotal != 7 || diagnostics.SourceDataIssueTotal != 2 { + t.Fatalf("issue split = pipeline:%d source:%d", diagnostics.PipelineIssueTotal, diagnostics.SourceDataIssueTotal) + } + if len(diagnostics.Items) != 2 || diagnostics.Items[0].Severity != "pipeline" || diagnostics.Items[1].Severity != "source_data" { + t.Fatalf("items = %#v", diagnostics.Items) + } + if diagnostics.FieldStatus == nil || diagnostics.FieldStatus.SummaryTotal != 1 || diagnostics.FieldStatus.Items[0].RealtimeMileageFieldStatus != "standard_field_non_positive" { + t.Fatalf("field status = %#v", diagnostics.FieldStatus) + } +} + +func TestCollectDailyMileageDiagnosticsIncludesIssueSamples(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case defaultDailyMileageDiagnosticsPath: + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"JT808","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_non_positive","count":2}, + {"stat_date":"2026-07-12","protocol":"YUTONG_MQTT","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_missing","count":1} + ], + "vehicle_total":3, + "actionable_issue_total":3 + }`)) + case "/api/stats/daily-metrics/diagnostics": + if got := r.URL.Query().Get("date"); got != "2026-07-12" { + t.Fatalf("date = %q, want 2026-07-12", got) + } + if got := r.URL.Query().Get("limit"); got == "" { + t.Fatal("sample detail request missing limit") + } + switch r.URL.Query().Get("protocol") { + case "JT808": + _, _ = w.Write([]byte(`{"items":[{"vin":"LKLG7C4E1NA774752","stat_date":"2026-07-12","protocol":"JT808","plate":"沪A03086F","peer":"222.66.200.68:12962","realtime_total_mileage_km":0,"realtime_field_count":23,"realtime_sample_fields":["jt808.location.latitude","jt808.location.speed_kmh"],"realtime_mileage_field_status":"standard_field_non_positive","realtime_mileage_evidence":"标准里程字段为 0","raw_frame_query_path":"/api/history/raw-frames?protocol=JT808&vin=LKLG7C4E1NA774752","source_sample_query_path":"/api/stats/daily-metrics/sources?protocol=JT808&vin=LKLG7C4E1NA774752","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_non_positive"}]}`)) + case "YUTONG_MQTT": + _, _ = w.Write([]byte(`{"items":[{"vin":"LMRKH9AC9R1004121","stat_date":"2026-07-12","protocol":"YUTONG_MQTT","peer":"mqtt://yutong/ytforward/shln/3","realtime_field_count":27,"realtime_sample_fields":["yutong_mqtt.data.latitude","yutong_mqtt.data.meter_speed"],"realtime_mileage_field_status":"no_candidate_mileage_field","realtime_mileage_evidence":"未发现 mileage 字段","raw_frame_query_path":"/api/history/raw-frames?protocol=YUTONG_MQTT&vin=LMRKH9AC9R1004121","source_sample_query_path":"/api/stats/daily-metrics/sources?protocol=YUTONG_MQTT&vin=LMRKH9AC9R1004121","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_missing"}]}`)) + default: + t.Fatalf("unexpected protocol %q", r.URL.Query().Get("protocol")) + } + case defaultDailyMileageFieldStatusPath: + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"JT808","diagnosis":"NO_TOTAL_MILEAGE","reason":"realtime_total_mileage_non_positive","realtime_mileage_field_status":"standard_field_non_positive","count":2,"field_status_severity":"source_data"} + ], + "total":1, + "vehicle_total":3, + "actionable_issue_total":3, + "pipeline_issue_total":0, + "source_data_issue_total":3 + }`)) + default: + t.Fatalf("unexpected path %q", r.URL.Path) + } + })) + defer server.Close() + + diagnostics, findings := collectDailyMileageDiagnostics(context.Background(), server.URL, "2026-07-12", time.Second, 2) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if len(diagnostics.Samples) != 2 { + t.Fatalf("samples = %#v, want two", diagnostics.Samples) + } + if diagnostics.Samples[0].VIN != "LKLG7C4E1NA774752" || diagnostics.Samples[0].Severity != capacity.DailyMileageSeveritySourceData { + t.Fatalf("first sample = %#v", diagnostics.Samples[0]) + } + if !strings.Contains(diagnostics.Samples[0].RecommendedOperatorAction, "总里程小于等于 0") { + t.Fatalf("first sample recommended action = %q", diagnostics.Samples[0].RecommendedOperatorAction) + } + if diagnostics.Samples[0].RealtimeFieldCount != 23 || diagnostics.Samples[0].RealtimeMileageFieldStatus != "standard_field_non_positive" { + t.Fatalf("first sample field diagnostics lost: %#v", diagnostics.Samples[0]) + } + if diagnostics.Samples[0].RawFrameQueryPath == "" || diagnostics.Samples[0].SourceSampleQueryPath == "" { + t.Fatalf("first sample trace links lost: %#v", diagnostics.Samples[0]) + } + if diagnostics.Samples[1].VIN != "LMRKH9AC9R1004121" || diagnostics.Samples[1].Severity != capacity.DailyMileageSeveritySourceData { + t.Fatalf("second sample = %#v", diagnostics.Samples[1]) + } + if !strings.Contains(diagnostics.Samples[1].RecommendedOperatorAction, "缺少总里程字段") { + t.Fatalf("second sample recommended action = %q", diagnostics.Samples[1].RecommendedOperatorAction) + } + if len(diagnostics.Samples[1].RealtimeSampleFields) != 2 || diagnostics.Samples[1].RealtimeMileageEvidence == "" { + t.Fatalf("second sample realtime evidence lost: %#v", diagnostics.Samples[1]) + } + if diagnostics.Samples[1].RawFrameQueryPath == "" || diagnostics.Samples[1].SourceSampleQueryPath == "" { + t.Fatalf("second sample trace links lost: %#v", diagnostics.Samples[1]) + } +} + +func TestDailyMileageDiagnosticsSampleRequestURLDerivesDetailPath(t *testing.T) { + requestURL, err := dailyMileageDiagnosticsSampleRequestURL( + "http://127.0.0.1:20200/api/stats/daily-metrics/diagnostics/reasons?date=2026-07-11", + "2026-07-12", + capacity.DailyMileageDiagnosisReasonItem{Protocol: "jt808", Diagnosis: "no_total_mileage"}, + 3, + ) + if err != nil { + t.Fatalf("sample request URL error: %v", err) + } + for _, want := range []string{ + "/api/stats/daily-metrics/diagnostics?", + "date=2026-07-12", + "protocol=JT808", + "diagnosis=NO_TOTAL_MILEAGE", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestDailyMileageDiagnosticFindingsCanGateOnlyPipelineIssues(t *testing.T) { + diagnostics := &capacity.DailyMileageDiagnostics{ + ActionableIssueTotal: 9, + PipelineIssueTotal: 7, + SourceDataIssueTotal: 2, + } + + findings := dailyMileageDiagnosticFindings(diagnostics, -1, 0, -1) + + if len(findings) != 1 { + t.Fatalf("findings = %#v, want one pipeline finding", findings) + } + if !strings.Contains(findings[0], "daily mileage pipeline issues 7 exceeds 0") { + t.Fatalf("finding = %q, want pipeline threshold", findings[0]) + } + if strings.Contains(findings[0], "actionable issues") { + t.Fatalf("pipeline-only gate should not use actionable threshold: %q", findings[0]) + } +} + +func TestDailyMileageDiagnosticFindingsReportsOnlyWhenPipelineIsCleanAndSourceDataExists(t *testing.T) { + diagnostics := &capacity.DailyMileageDiagnostics{ + ActionableIssueTotal: 3, + PipelineIssueTotal: 0, + SourceDataIssueTotal: 3, + } + + findings := dailyMileageDiagnosticFindings(diagnostics, -1, 0, -1) + + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none because source-data issues should not block pipeline gate", findings) + } +} + +func TestSummarizeDailyMileageFieldStatusPrefersDiagnosisSeverity(t *testing.T) { + actionable, pipeline, sourceData := summarizeDailyMileageFieldStatus([]capacity.DailyMileageFieldStatusSummaryItem{ + { + Diagnosis: "NO_TOTAL_MILEAGE", + RealtimeMileageFieldStatus: "candidate_field_unmapped", + FieldStatusSeverity: capacity.DailyMileageSeverityPipeline, + Severity: capacity.DailyMileageSeveritySourceData, + Count: 4, + }, + { + Diagnosis: "NO_TOTAL_MILEAGE", + RealtimeMileageFieldStatus: "no_candidate_mileage_field", + FieldStatusSeverity: capacity.DailyMileageSeveritySourceData, + Count: 3, + }, + { + Diagnosis: "OK", + RealtimeMileageFieldStatus: "daily_metric_exists", + FieldStatusSeverity: capacity.DailyMileageSeverityOK, + Count: 10, + }, + }) + + if actionable != 7 || pipeline != 0 || sourceData != 7 { + t.Fatalf("summary actionable=%d pipeline=%d source=%d", actionable, pipeline, sourceData) + } +} + +func TestCollectDailyMileageDiagnosticsReportsHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "mysql unavailable", http.StatusInternalServerError) + })) + defer server.Close() + + diagnostics, findings := collectDailyMileageDiagnostics(context.Background(), server.URL, "", time.Second, 0) + if diagnostics != nil { + t.Fatalf("diagnostics = %#v, want nil", diagnostics) + } + if len(findings) != 1 || !strings.Contains(findings[0], "status 500") { + t.Fatalf("findings = %#v, want status 500", findings) + } +} + +func TestDailyMileageSourceQualityRequestURLUsesDefaultPathAndDateRange(t *testing.T) { + requestURL, err := dailyMileageSourceQualityRequestURL("http://127.0.0.1:20200", "2026-07-12", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDailyMileageSourceQualityPath, + "dateFrom=2026-07-12", + "dateTo=2026-07-12", + "includeTotal=true", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestCollectDailyMileageSourceQuality(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != defaultDailyMileageSourceQualityPath { + t.Fatalf("path = %q, want %q", r.URL.Path, defaultDailyMileageSourceQualityPath) + } + if got := r.URL.Query().Get("dateFrom"); got != "2026-07-12" { + t.Fatalf("dateFrom = %q, want 2026-07-12", got) + } + if got := r.URL.Query().Get("dateTo"); got != "2026-07-12" { + t.Fatalf("dateTo = %q, want 2026-07-12", got) + } + if got := r.URL.Query().Get("includeTotal"); got != "true" { + t.Fatalf("includeTotal = %q, want true", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"JT808","quality_status":"OK","quality_reason":"historical_source_baseline","source_count":10,"vehicle_count":10,"selected_count":10,"sample_count":200,"daily_mileage_km":800.5}, + {"stat_date":"2026-07-12","protocol":"GB32960","quality_status":"OK","quality_reason":"current_day_fallback_after_invalid_baseline","source_count":4,"vehicle_count":4,"selected_count":4,"sample_count":40,"daily_mileage_km":12.3}, + {"stat_date":"2026-07-12","protocol":"JT808","quality_status":"INVALID_DELTA","quality_reason":"outside_daily_range","source_count":3,"vehicle_count":3,"selected_count":0,"sample_count":12,"daily_mileage_km":4000}, + {"stat_date":"2026-07-12","protocol":"YUTONG_MQTT","quality_status":"NO_PREVIOUS_BASELINE","quality_reason":"current_day_first_sample","source_count":2,"vehicle_count":2,"selected_count":0,"sample_count":2,"daily_mileage_km":0}, + {"stat_date":"2026-07-12","protocol":"YUTONG_MQTT","quality_status":"OK","quality_reason":"realtime_location_fallback_historical_baseline","source_count":5,"vehicle_count":5,"selected_count":5,"sample_count":5,"daily_mileage_km":0} + ], + "total":7, + "limit":5, + "offset":0 + }`)) + })) + defer server.Close() + + diagnostics, findings := collectDailyMileageSourceQuality(context.Background(), server.URL, "2026-07-12", time.Second, 5) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.SummaryTotal != 7 || !diagnostics.ItemsTruncated { + t.Fatalf("summary total/truncated = %d/%v, want 7/true", diagnostics.SummaryTotal, diagnostics.ItemsTruncated) + } + if diagnostics.SourceCount != 24 || diagnostics.OKSourceCount != 19 || diagnostics.InvalidDeltaSourceCount != 3 || diagnostics.NoBaselineSourceCount != 2 || diagnostics.FallbackAfterInvalidBaselineSourceCount != 4 || diagnostics.RealtimeLocationFallbackSourceCount != 5 { + t.Fatalf("source quality totals = %#v", diagnostics) + } + if diagnostics.VehicleCount != 24 || diagnostics.SampleCount != 259 { + t.Fatalf("vehicle/sample totals = %d/%d", diagnostics.VehicleCount, diagnostics.SampleCount) + } + if len(diagnostics.Items) != 5 { + t.Fatalf("items = %#v, want five", diagnostics.Items) + } +} + +func TestDailyMileageSourceQualityFindingsCanGateInvalidDelta(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceQualityDiagnostics{ + InvalidDeltaSourceCount: 3, + NoBaselineSourceCount: 2, + } + + findings := dailyMileageSourceQualityFindings(diagnostics, 0, -1, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "daily mileage source INVALID_DELTA count 3 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceQualityFindingsCanGateNoBaseline(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceQualityDiagnostics{ + InvalidDeltaSourceCount: 3, + NoBaselineSourceCount: 2, + } + + findings := dailyMileageSourceQualityFindings(diagnostics, -1, 0, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "daily mileage source NO_PREVIOUS_BASELINE count 2 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceQualityFindingsCanGateFallbackAfterInvalidBaseline(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceQualityDiagnostics{ + FallbackAfterInvalidBaselineSourceCount: 4, + } + + findings := dailyMileageSourceQualityFindings(diagnostics, -1, -1, 0, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "daily mileage source current-day fallback after invalid baseline count 4 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceQualityFindingsCanGateRealtimeLocationFallback(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceQualityDiagnostics{ + RealtimeLocationFallbackSourceCount: 5, + } + + findings := dailyMileageSourceQualityFindings(diagnostics, -1, -1, -1, 0) + + if len(findings) != 1 || !strings.Contains(findings[0], "daily mileage source realtime-location fallback count 5 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceSelectionRequestURLUsesDefaultPathAndDateRange(t *testing.T) { + requestURL, err := dailyMileageSourceSelectionRequestURL("http://127.0.0.1:20200", "2026-07-12", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDailyMileageSourceSelectionPath, + "dateFrom=2026-07-12", + "dateTo=2026-07-12", + "includeTotal=true", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestDailyMileageSourceSelectionSampleRequestURLUsesSourceDetailPath(t *testing.T) { + requestURL, err := dailyMileageSourceSelectionSampleRequestURL("http://127.0.0.1:20200/api/stats/daily-metrics/sources/selection", "2026-07-12", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDailyMileageSourcesPath, + "dateFrom=2026-07-12", + "dateTo=2026-07-12", + "includeTotal=false", + "selected=false", + "orderBy=dailyMileageDesc", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } + if strings.Contains(requestURL, "/selection?") { + t.Fatalf("url = %q, should use source detail path", requestURL) + } +} + +func TestCollectDailyMileageSourceSelection(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("dateFrom"); got != "2026-07-12" { + t.Fatalf("dateFrom = %q, want 2026-07-12", got) + } + if got := r.URL.Query().Get("dateTo"); got != "2026-07-12" { + t.Fatalf("dateTo = %q, want 2026-07-12", got) + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case defaultDailyMileageSourceSelectionPath: + if got := r.URL.Query().Get("includeTotal"); got != "true" { + t.Fatalf("includeTotal = %q, want true", got) + } + _, _ = w.Write([]byte(`{ + "items":[ + {"stat_date":"2026-07-12","protocol":"JT808","selection_status":"selected","selection_reason":"selected_current_projection","source_count":10,"vehicle_count":10,"selected_count":10,"sample_count":200,"daily_mileage_km":800.5}, + {"stat_date":"2026-07-12","protocol":"JT808","selection_status":"not_selected","selection_reason":"unmanaged_source_lower_priority","source_count":3,"vehicle_count":3,"selected_count":0,"sample_count":12,"daily_mileage_km":40}, + {"stat_date":"2026-07-12","protocol":"JT808","selection_status":"not_selected","selection_reason":"unknown_source_kind_lower_priority","source_count":2,"vehicle_count":2,"selected_count":0,"sample_count":8,"daily_mileage_km":20}, + {"stat_date":"2026-07-12","protocol":"JT808","selection_status":"excluded","selection_reason":"invalid_delta","source_count":1,"vehicle_count":1,"selected_count":0,"sample_count":2,"daily_mileage_km":0} + ], + "total":5, + "limit":4, + "offset":0 + }`)) + case defaultDailyMileageSourcesPath: + selected := r.URL.Query().Get("selected") + if got := r.URL.Query().Get("includeTotal"); got != "false" { + t.Fatalf("includeTotal = %q, want false", got) + } + if selected == "true" { + if got := r.URL.Query().Get("vin"); got == "" { + t.Fatalf("selected source query should include vin") + } + if got := r.URL.Query().Get("protocol"); got != "JT808" { + t.Fatalf("selected source protocol = %q, want JT808", got) + } + _, _ = w.Write([]byte(`{ + "items":[ + {"vin":"` + r.URL.Query().Get("vin") + `","stat_date":"2026-07-12","protocol":"JT808","source_key":"JT808|source|selected","source_ip":"10.0.0.9","source_endpoint":"10.0.0.9:43625","phone":"13300000009","platform_name":"可信平台","source_code":"trusted","source_kind":"PLATFORM","daily_mileage_km":12,"sample_count":20,"quality_status":"OK","selection_status":"selected","selection_reason":"selected_current_projection","selection_action":"selected","updated_at":"2026-07-12T10:01:00+08:00"} + ], + "total":1, + "limit":1, + "offset":0 + }`)) + return + } + if selected != "false" { + t.Fatalf("selected = %q, want false", selected) + } + if got := r.URL.Query().Get("orderBy"); got != "dailyMileageDesc" { + t.Fatalf("orderBy = %q, want dailyMileageDesc", got) + } + _, _ = w.Write([]byte(`{ + "items":[ + {"vin":"VIN-UNKNOWN","stat_date":"2026-07-12","protocol":"JT808","source_key":"JT808|source|unknown","source_ip":"10.0.0.3","source_endpoint":"10.0.0.3:43625","phone":"13300000003","platform_name":"未知类型平台","source_code":"unknown","source_kind":"UNKNOWN","daily_mileage_km":30,"sample_count":4,"quality_status":"OK","selection_status":"not_selected","selection_reason":"unknown_source_kind_lower_priority","selection_action":"set_source_kind","updated_at":"2026-07-12T10:00:00+08:00"}, + {"vin":"VIN-UNMANAGED","stat_date":"2026-07-12","protocol":"JT808","source_key":"JT808|source|unmanaged","source_ip":"10.0.0.2","source_endpoint":"10.0.0.2:43625","phone":"13300000002","platform_name":"未维护平台","source_kind":"PLATFORM","daily_mileage_km":20,"sample_count":3,"quality_status":"OK","selection_status":"not_selected","selection_reason":"unmanaged_source_lower_priority","selection_action":"create_vehicle_data_source","updated_at":"2026-07-12T10:00:00+08:00"}, + {"vin":"VIN-LOW","stat_date":"2026-07-12","protocol":"JT808","source_key":"JT808|source|low","source_ip":"10.0.0.1","source_endpoint":"10.0.0.1:43625","phone":"13300000001","platform_name":"低优先级平台","source_code":"G7s","source_kind":"PLATFORM","daily_mileage_km":10,"sample_count":2,"quality_status":"OK","selection_status":"not_selected","selection_reason":"lower_trust_priority_or_sample_count","selection_action":"none","updated_at":"2026-07-12T10:00:00+08:00"} + ], + "total":3, + "limit":40, + "offset":0 + }`)) + default: + t.Fatalf("path = %q, want source selection or source detail path", r.URL.Path) + } + })) + defer server.Close() + + diagnostics, findings := collectDailyMileageSourceSelection(context.Background(), server.URL, "2026-07-12", time.Second, 4, 2) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.SummaryTotal != 5 || !diagnostics.ItemsTruncated { + t.Fatalf("summary total/truncated = %d/%v, want 5/true", diagnostics.SummaryTotal, diagnostics.ItemsTruncated) + } + if diagnostics.SourceCount != 16 || diagnostics.SelectedSourceCount != 10 || diagnostics.NotSelectedSourceCount != 5 || diagnostics.ExcludedSourceCount != 1 { + t.Fatalf("selection totals = %#v", diagnostics) + } + if diagnostics.NotSelectedMileageKM != 60 { + t.Fatalf("not selected mileage = %v, want 60", diagnostics.NotSelectedMileageKM) + } + if diagnostics.UnmanagedSourceCount != 3 || diagnostics.UnknownKindSourceCount != 2 { + t.Fatalf("source management totals unmanaged=%d unknown=%d", diagnostics.UnmanagedSourceCount, diagnostics.UnknownKindSourceCount) + } + if diagnostics.VehicleCount != 16 || diagnostics.SampleCount != 222 { + t.Fatalf("vehicle/sample totals = %d/%d", diagnostics.VehicleCount, diagnostics.SampleCount) + } + if len(diagnostics.Protocols) != 1 { + t.Fatalf("protocols = %#v, want one JT808 summary", diagnostics.Protocols) + } + if got := diagnostics.Protocols[0]; got.Protocol != "JT808" || got.SourceCount != 16 || got.SelectedSourceCount != 10 || got.NotSelectedSourceCount != 5 || got.NotSelectedMileageKM != 60 || got.ExcludedSourceCount != 1 || got.UnmanagedSourceCount != 3 || got.UnknownKindSourceCount != 2 { + t.Fatalf("JT808 protocol summary = %#v", got) + } + if len(diagnostics.Items) != 4 { + t.Fatalf("items = %#v, want four", diagnostics.Items) + } + if len(diagnostics.Samples) != 2 { + t.Fatalf("samples = %#v, want two source management samples", diagnostics.Samples) + } + if diagnostics.Samples[0].VIN != "VIN-UNKNOWN" || diagnostics.Samples[0].SelectionAction != "set_source_kind" { + t.Fatalf("first sample = %#v", diagnostics.Samples[0]) + } + if diagnostics.Samples[0].SelectedSource == nil || diagnostics.Samples[0].SelectedSource.PlatformName != "可信平台" || diagnostics.Samples[0].SelectedSource.DailyMileageKM != 12 { + t.Fatalf("first selected source = %#v", diagnostics.Samples[0].SelectedSource) + } + if !strings.Contains(diagnostics.Samples[0].SelectedSourceQueryPath, "selected=true") || !strings.Contains(diagnostics.Samples[0].SelectedSourceQueryPath, "vin=VIN-UNKNOWN") { + t.Fatalf("first selected source query path = %q", diagnostics.Samples[0].SelectedSourceQueryPath) + } + if diagnostics.Samples[1].VIN != "VIN-UNMANAGED" || diagnostics.Samples[1].SelectionAction != "create_vehicle_data_source" { + t.Fatalf("second sample = %#v", diagnostics.Samples[1]) + } + if diagnostics.Samples[1].SelectedSource == nil || diagnostics.Samples[1].SelectedSource.SourceIP != "10.0.0.9" { + t.Fatalf("second selected source = %#v", diagnostics.Samples[1].SelectedSource) + } +} + +func TestDailyMileageSourceSelectionFindingsCanGateSelectedSourceFloor(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceSelectionDiagnostics{SelectedSourceCount: 0} + + findings := dailyMileageSourceSelectionFindings(diagnostics, 1, nil, -1, -1, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "selected source count 0 is below 1") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceSelectionFindingsCanGateSelectedSourceByProtocol(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceSelectionDiagnostics{ + Protocols: []capacity.DailyMileageSourceSelectionProtocol{ + {Protocol: "GB32960", SelectedSourceCount: 12}, + {Protocol: "JT808", SelectedSourceCount: 0}, + }, + } + + findings := dailyMileageSourceSelectionFindings(diagnostics, -1, []string{"GB32960", "JT808", "YUTONG_MQTT"}, 1, -1, -1, -1) + + if len(findings) != 2 { + t.Fatalf("findings = %#v, want JT808 and YUTONG_MQTT", findings) + } + if !strings.Contains(findings[0], "protocol JT808 is 0 below 1") { + t.Fatalf("first finding = %q", findings[0]) + } + if !strings.Contains(findings[1], "protocol YUTONG_MQTT is 0 below 1") { + t.Fatalf("second finding = %q", findings[1]) + } +} + +func TestDailyMileageSourceSelectionFindingsCanGateUnmanagedAndUnknownKind(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceSelectionDiagnostics{ + UnmanagedSourceCount: 3, + UnknownKindSourceCount: 2, + } + + findings := dailyMileageSourceSelectionFindings(diagnostics, -1, nil, -1, 0, 0, -1) + + if len(findings) != 2 { + t.Fatalf("findings = %#v, want two", findings) + } + if !strings.Contains(findings[0], "unmanaged source count 3 exceeds 0") { + t.Fatalf("first finding = %q", findings[0]) + } + if !strings.Contains(findings[1], "UNKNOWN source_kind count 2 exceeds 0") { + t.Fatalf("second finding = %q", findings[1]) + } +} + +func TestDailyMileageSourceSelectionFindingsCanGateNotSelectedMileage(t *testing.T) { + diagnostics := &capacity.DailyMileageSourceSelectionDiagnostics{ + NotSelectedMileageKM: 16736.6, + } + + findings := dailyMileageSourceSelectionFindings(diagnostics, -1, nil, -1, -1, -1, 1000) + + if len(findings) != 1 || !strings.Contains(findings[0], "not-selected source mileage 16736.6km exceeds 1000.0km") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDailyMileageSourceSelectionSampleIncludesHighMileageNotSelectedSource(t *testing.T) { + sample := capacity.DailyMileageSourceSelectionSample{ + SelectionStatus: "not_selected", + SelectionReason: "lower_trust_priority_or_sample_count", + DailyMileageKM: 1234.5, + } + + if !isProblematicDailyMileageSourceSelectionSample(sample) { + t.Fatalf("sample should be collected for not-selected mileage evidence") + } +} + +func TestDataSourceDiagnosticsRequestURLUsesDefaultPathAndTotal(t *testing.T) { + requestURL, err := dataSourceDiagnosticsRequestURL("http://127.0.0.1:20200", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDataSourceDiagnosticsPath, + "sourceCodeMissing=true", + "includeTotal=true", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestDataSourceMappingIssuesRequestURLUsesDiagnosticsPath(t *testing.T) { + requestURL, err := dataSourceMappingIssuesRequestURL("http://127.0.0.1:20200", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDataSourceDiagnosticsPath, + "sourceCodeMissing=false", + "mappingIssueOnly=true", + "includeTotal=true", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestCollectDataSourceDiagnostics(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != defaultDataSourceDiagnosticsPath { + t.Fatalf("path = %q, want %q", r.URL.Path, defaultDataSourceDiagnosticsPath) + } + if got := r.URL.Query().Get("includeTotal"); got != "true" { + t.Fatalf("includeTotal = %q, want true", got) + } + if got := r.URL.Query().Get("limit"); got != "3" { + t.Fatalf("limit = %q, want 3", got) + } + if got := r.URL.Query().Get("mappingIssueOnly"); got == "true" { + if missing := r.URL.Query().Get("sourceCodeMissing"); missing != "false" { + t.Fatalf("mapping issue sourceCodeMissing = %q, want false", missing) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "items":[ + {"id":7,"protocol":"JT808","source_ip":"115.159.85.149","platform_name":"G7易流","source_code":"dongfang_beidou","source_kind":"PLATFORM","latest_seen_age_seconds":60,"registration_rows":388,"phone_count":388,"identifier_matched_phones":8,"unmapped_phone_count":380,"identifier_match_ratio":0.0206,"configured_source_code_matched_phones":8,"matched_source_code_count":1,"reason":"low_identifier_coverage","recommended_operator_action":"补充该来源手机号映射"} + ], + "total":1, + "limit":3, + "offset":0 + }`)) + return + } + if got := r.URL.Query().Get("sourceCodeMissing"); got != "true" { + t.Fatalf("sourceCodeMissing = %q, want true", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "items":[ + {"id":253218,"protocol":"JT808","source_ip":"117.132.196.43","latest_source_endpoint":"117.132.196.43:38256","source_kind":"UNKNOWN","latest_seen_age_seconds":120,"registration_rows":1,"phone_count":1,"candidate_source_code":"guangan_beidou","candidate_platform_name":"广安北斗","reason":"candidate_available","recommended_operator_action":"确认来源平台","suggested_source_kind":"PLATFORM","suggestion_confidence":"HIGH","suggestion_reason":"single_source_code_with_many_phones_or_long_activity"}, + {"id":253219,"protocol":"JT808","source_ip":"117.132.196.44","source_kind":"UNKNOWN","latest_seen_age_seconds":600,"registration_rows":1,"phone_count":1,"candidate_source_code":"guangan_beidou","candidate_platform_name":"广安北斗","reason":"candidate_available","recommended_operator_action":"确认来源平台","suggested_source_kind":"PLATFORM","suggestion_confidence":"HIGH","suggestion_reason":"single_source_code_with_many_phones_or_long_activity"}, + {"id":253220,"protocol":"JT808","source_ip":"117.132.196.45","source_kind":"UNKNOWN","latest_seen_age_seconds":20,"reason":"no_registration","recommended_operator_action":"等待注册证据","suggested_source_kind":"UNKNOWN","suggestion_confidence":"LOW","suggestion_reason":"insufficient_evidence"} + ], + "total":400, + "limit":3, + "offset":0 + }`)) + })) + defer server.Close() + + diagnostics, findings := collectDataSourceDiagnostics(context.Background(), server.URL, time.Second, 1, 3, 300) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.MissingSourceCodeTotal != 400 { + t.Fatalf("missing source code total = %d, want 400", diagnostics.MissingSourceCodeTotal) + } + if diagnostics.ActionableCandidateTotal != 2 { + t.Fatalf("actionable candidates = %d, want 2", diagnostics.ActionableCandidateTotal) + } + if diagnostics.RecentMissingSourceCodeTotal != 2 || diagnostics.RecentAgeThresholdSeconds != 300 { + t.Fatalf("recent summary = %d threshold=%d, want 2/300", diagnostics.RecentMissingSourceCodeTotal, diagnostics.RecentAgeThresholdSeconds) + } + if diagnostics.ReasonCounts["candidate_available"] != 2 || diagnostics.ReasonCounts["no_registration"] != 1 { + t.Fatalf("reason counts = %#v", diagnostics.ReasonCounts) + } + if !diagnostics.ReasonCountsTruncated { + t.Fatalf("reason counts should be marked truncated when total exceeds fetched rows") + } + if len(diagnostics.Samples) != 1 || diagnostics.Samples[0].CandidateSourceCode != "guangan_beidou" { + t.Fatalf("samples = %#v", diagnostics.Samples) + } + if diagnostics.MappingIssueTotal != 1 || diagnostics.RecentMappingIssueTotal != 1 { + t.Fatalf("mapping issue summary = total %d recent %d", diagnostics.MappingIssueTotal, diagnostics.RecentMappingIssueTotal) + } + if diagnostics.MappingIssueReasonCounts["low_identifier_coverage"] != 1 { + t.Fatalf("mapping reason counts = %#v", diagnostics.MappingIssueReasonCounts) + } + if len(diagnostics.MappingIssueSamples) != 1 || diagnostics.MappingIssueSamples[0].UnmappedPhoneCount != 380 { + t.Fatalf("mapping samples = %#v", diagnostics.MappingIssueSamples) + } +} + +func TestDataSourceKindSuggestionsRequestURLUsesDefaultPathAndUnknownKind(t *testing.T) { + requestURL, err := dataSourceKindSuggestionsRequestURL("http://127.0.0.1:20200", "gb32960", 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultDataSourceKindSuggestionsPath, + "protocol=GB32960", + "sourceKind=UNKNOWN", + "includeTotal=true", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestCollectDataSourceKindSuggestionsAggregatesPlatformCandidates(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != defaultDataSourceKindSuggestionsPath { + t.Fatalf("path = %q, want %q", r.URL.Path, defaultDataSourceKindSuggestionsPath) + } + if got := r.URL.Query().Get("sourceKind"); got != "UNKNOWN" { + t.Fatalf("sourceKind = %q, want UNKNOWN", got) + } + if got := r.URL.Query().Get("includeTotal"); got != "true" { + t.Fatalf("includeTotal = %q, want true", got) + } + if got := r.URL.Query().Get("limit"); got != "3" { + t.Fatalf("limit = %q, want 3", got) + } + w.Header().Set("Content-Type", "application/json") + switch r.URL.Query().Get("protocol") { + case "GB32960": + _, _ = w.Write([]byte(`{ + "items":[ + {"id":7,"protocol":"GB32960","source_ip":"8.134.95.166","latest_source_endpoint":"8.134.95.166:32960","platform_name":"现代 HTWO","source_code":"Hyundai","source_kind":"UNKNOWN","latest_seen_age_seconds":90,"reason":"source_configured","suggested_source_kind":"PLATFORM","suggestion_confidence":"MEDIUM","suggestion_reason":"non_jt808_configured_source"} + ], + "total":1, + "limit":3, + "offset":0 + }`)) + case "JT808": + _, _ = w.Write([]byte(`{ + "items":[ + {"id":253218,"protocol":"JT808","source_ip":"117.132.196.43","source_kind":"UNKNOWN","latest_seen_age_seconds":600,"registration_rows":1,"phone_count":12,"candidate_source_code":"guangan_beidou","candidate_platform_name":"广安北斗","reason":"candidate_available","suggested_source_kind":"PLATFORM","suggestion_confidence":"HIGH","suggestion_reason":"single_source_code_with_many_phones_or_long_activity"}, + {"id":253219,"protocol":"JT808","source_ip":"117.132.196.44","source_kind":"UNKNOWN","latest_seen_age_seconds":60,"reason":"no_registration","suggested_source_kind":"UNKNOWN","suggestion_confidence":"LOW","suggestion_reason":"insufficient_evidence"} + ], + "total":2, + "limit":3, + "offset":0 + }`)) + case "YUTONG_MQTT": + _, _ = w.Write([]byte(`{"items":[],"total":0,"limit":3,"offset":0}`)) + default: + t.Fatalf("unexpected protocol query %q", r.URL.Query().Get("protocol")) + } + })) + defer server.Close() + + diagnostics, findings := collectDataSourceKindSuggestions( + context.Background(), + server.URL, + time.Second, + []string{"GB32960", "JT808", "YUTONG_MQTT"}, + 1, + 3, + 300, + ) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.UnknownSourceKindTotal != 3 { + t.Fatalf("unknown source kind total = %d, want 3", diagnostics.UnknownSourceKindTotal) + } + if diagnostics.RecentUnknownSourceKindTotal != 2 || diagnostics.RecentAgeThresholdSeconds != 300 { + t.Fatalf("recent summary = %d threshold=%d, want 2/300", diagnostics.RecentUnknownSourceKindTotal, diagnostics.RecentAgeThresholdSeconds) + } + if diagnostics.PlatformKindCandidateTotal != 2 { + t.Fatalf("platform kind candidates = %d, want 2", diagnostics.PlatformKindCandidateTotal) + } + if diagnostics.RecentPlatformKindCandidateTotal != 1 { + t.Fatalf("recent platform kind candidates = %d, want 1", diagnostics.RecentPlatformKindCandidateTotal) + } + if diagnostics.ReasonCounts["non_jt808_configured_source"] != 1 || diagnostics.ReasonCounts["single_source_code_with_many_phones_or_long_activity"] != 1 { + t.Fatalf("reason counts = %#v", diagnostics.ReasonCounts) + } + if len(diagnostics.Samples) != 1 || diagnostics.Samples[0].Protocol != "GB32960" { + t.Fatalf("samples = %#v", diagnostics.Samples) + } +} + +func TestJT808IdentityGapsRequestURLUsesDefaultPathAndRecentSeconds(t *testing.T) { + requestURL, err := jt808IdentityGapsRequestURL("http://127.0.0.1:20200", 86400, 3) + if err != nil { + t.Fatalf("request URL error: %v", err) + } + for _, want := range []string{ + defaultJT808IdentityGapsPath, + "includeTotal=true", + "recentSeconds=86400", + "limit=3", + "offset=0", + } { + if !strings.Contains(requestURL, want) { + t.Fatalf("url = %q, want %q", requestURL, want) + } + } +} + +func TestCollectJT808IdentityGaps(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != defaultJT808IdentityGapsPath { + t.Fatalf("path = %q, want %q", r.URL.Path, defaultJT808IdentityGapsPath) + } + if got := r.URL.Query().Get("includeTotal"); got != "true" { + t.Fatalf("includeTotal = %q, want true", got) + } + if got := r.URL.Query().Get("recentSeconds"); got != "86400" { + t.Fatalf("recentSeconds = %q, want 86400", got) + } + if got := r.URL.Query().Get("limit"); got != "2" { + t.Fatalf("limit = %q, want 2", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "items":[ + {"phone":"41025503543","vin":"unknown","source_ip":"122.152.221.156","source_endpoint":"122.152.221.156:23185","source_code":"dongfang_beidou","platform_name":"广安车联","source_kind":"PLATFORM","latest_seen_at":"2026-07-12 14:01:02","latest_seen_age_seconds":30,"reason":"missing_phone_binding","recommended_operator_action":"将该 phone 维护到 vehicle_identifier","raw_frame_query_path":"/api/history/raw-frames?protocol=JT808&phone=41025503543","data_source_query_path":"/api/stats/data-sources?protocol=JT808&sourceIP=122.152.221.156"} + ], + "total":1, + "limit":2, + "offset":0, + "recentSeconds":86400 + }`)) + })) + defer server.Close() + + diagnostics, findings := collectJT808IdentityGaps(context.Background(), server.URL, time.Second, 86400, 2) + if len(findings) != 0 { + t.Fatalf("findings = %#v, want none", findings) + } + if diagnostics == nil { + t.Fatal("diagnostics = nil") + } + if diagnostics.Total != 1 || diagnostics.RecentSeconds != 86400 { + t.Fatalf("diagnostics summary = total %d recent %d, want 1/86400", diagnostics.Total, diagnostics.RecentSeconds) + } + if len(diagnostics.Samples) != 1 || diagnostics.Samples[0].Phone != "41025503543" { + t.Fatalf("samples = %#v", diagnostics.Samples) + } + if diagnostics.Samples[0].RawFrameQueryPath == "" || diagnostics.Samples[0].DataSourceQueryPath == "" { + t.Fatalf("diagnostic links should be preserved: %#v", diagnostics.Samples[0]) + } +} + +func TestDataSourceDiagnosticFindingsCanGateMissingSourceCode(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{MissingSourceCodeTotal: 400} + + findings := dataSourceDiagnosticFindings(diagnostics, 100, -1, -1, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "data source missing source_code 400 exceeds 100") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDataSourceKindDiagnosticFindingsCanGateRecentPlatformCandidates(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{ + PlatformKindCandidateTotal: 3, + RecentPlatformKindCandidateTotal: 1, + RecentAgeThresholdSeconds: 300, + } + + findings := dataSourceKindDiagnosticFindings(diagnostics, -1, 0) + + if len(findings) != 1 || !strings.Contains(findings[0], "recent UNKNOWN source_kind platform candidates 1 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } + if !strings.Contains(findings[0], "recent_age_seconds=300") { + t.Fatalf("finding should include recent age threshold: %q", findings[0]) + } +} + +func TestDataSourceKindDiagnosticFindingsCanGatePlatformCandidates(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{ + PlatformKindCandidateTotal: 3, + } + + findings := dataSourceKindDiagnosticFindings(diagnostics, 0, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "UNKNOWN source_kind platform candidates 3 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestJT808IdentityGapFindingsCanGate(t *testing.T) { + diagnostics := &capacity.JT808IdentityGapDiagnostics{ + Total: 2, + RecentSeconds: 86400, + } + + findings := jt808IdentityGapFindings(diagnostics, 1) + + if len(findings) != 1 || !strings.Contains(findings[0], "JT808 unresolved identity gaps 2 exceeds 1") { + t.Fatalf("findings = %#v", findings) + } + if !strings.Contains(findings[0], "recent_seconds=86400") { + t.Fatalf("finding should include recent seconds: %q", findings[0]) + } +} + +func TestDataSourceDiagnosticFindingsCanGateActionableCandidates(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{ + MissingSourceCodeTotal: 400, + ActionableCandidateTotal: 3, + } + + findings := dataSourceDiagnosticFindings(diagnostics, -1, -1, 0, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "data source actionable candidates 3 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } +} + +func TestDataSourceDiagnosticFindingsCanGateRecentMissingSourceCode(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{ + MissingSourceCodeTotal: 400, + RecentMissingSourceCodeTotal: 2, + RecentAgeThresholdSeconds: 300, + } + + findings := dataSourceDiagnosticFindings(diagnostics, -1, 0, -1, -1, -1) + + if len(findings) != 1 || !strings.Contains(findings[0], "data source recent missing source_code 2 exceeds 0") { + t.Fatalf("findings = %#v", findings) + } + if !strings.Contains(findings[0], "recent_age_seconds=300") { + t.Fatalf("finding should include recent age threshold: %q", findings[0]) + } +} + +func TestDataSourceDiagnosticFindingsCanGateMappingIssues(t *testing.T) { + diagnostics := &capacity.DataSourceDiagnostics{ + MappingIssueTotal: 3, + RecentMappingIssueTotal: 2, + RecentAgeThresholdSeconds: 300, + } + + findings := dataSourceDiagnosticFindings(diagnostics, -1, -1, -1, 0, 1) + + if len(findings) != 2 { + t.Fatalf("findings = %#v, want 2", findings) + } + if !strings.Contains(findings[0], "data source mapping issues 3 exceeds 0") { + t.Fatalf("mapping issue finding missing: %#v", findings) + } + if !strings.Contains(findings[1], "data source recent mapping issues 2 exceeds 1") { + t.Fatalf("recent mapping issue finding missing: %#v", findings) + } +} diff --git a/go/vehicle-gateway/cmd/fields-projector/main.go b/go/vehicle-gateway/cmd/fields-projector/main.go new file mode 100644 index 00000000..e48201fe --- /dev/null +++ b/go/vehicle-gateway/cmd/fields-projector/main.go @@ -0,0 +1,490 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" +) + +func main() { + logger := observability.NewLogger("vehicle-fields-projector") + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + cfg := loadConfig() + if err := cfg.Validate(); err != nil { + logger.Error("invalid fields projector config", "error", err) + os.Exit(1) + } + if err := pingKafka(ctx, cfg.KafkaBrokers); err != nil { + logger.Error("kafka connectivity check failed", "error", err) + os.Exit(1) + } + registry := metrics.NewRegistry() + recordConfigMetrics(registry, cfg) + for _, route := range cfg.Routes { + metrics.RegisterKafkaConsumerInfo(registry, "vehicle-fields-projector", route.GroupID, []string{route.RawTopic}) + } + health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-fields-projector", nil, registry)) + + logger.Info("fields projector started", + "kafka_brokers", strings.Join(cfg.KafkaBrokers, ","), + "group_prefix", cfg.GroupPrefix, + "workers_per_protocol", cfg.WorkersPerProtocol, + "batch_size", cfg.BatchSize, + "batch_wait_ms", cfg.BatchWait.Milliseconds(), + "operation_timeout_ms", cfg.OperationTimeout.Milliseconds(), + "retry_delay_ms", cfg.RetryDelay.Milliseconds(), + "start_offset", cfg.StartOffsetName) + + var workers sync.WaitGroup + for _, route := range cfg.Routes { + for workerID := 1; workerID <= cfg.WorkersPerProtocol; workerID++ { + workers.Add(1) + go func(route projectionRoute, workerID int) { + defer workers.Done() + runProjector(ctx, logger.With("protocol", route.Protocol, "worker", workerID), registry, cfg, route, workerID) + }(route, workerID) + } + } + workers.Wait() +} + +type config struct { + KafkaBrokers []string + GroupPrefix string + Routes []projectionRoute + WorkersPerProtocol int + BatchSize int + BatchWait time.Duration + OperationTimeout time.Duration + RetryDelay time.Duration + StartOffset int64 + StartOffsetName string +} + +type projectionRoute struct { + Protocol envelope.Protocol + RawTopic string + FieldsTopic string + GroupID string +} + +func loadConfig() config { + groupPrefix := env("FIELDS_PROJECTOR_GROUP_PREFIX", "vehicle-fields-projector-v1") + startOffsetName := strings.ToLower(env("KAFKA_START_OFFSET", "last")) + startOffset := int64(kafka.LastOffset) + if startOffsetName == "first" { + startOffset = kafka.FirstOffset + } else { + startOffsetName = "last" + } + routes := []projectionRoute{ + {Protocol: envelope.ProtocolGB32960, RawTopic: env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960), FieldsTopic: env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)}, + {Protocol: envelope.ProtocolJT808, RawTopic: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808), FieldsTopic: env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)}, + {Protocol: envelope.ProtocolYutongMQTT, RawTopic: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT), FieldsTopic: env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)}, + } + for index := range routes { + routes[index].GroupID = groupPrefix + "-" + strings.ToLower(strings.ReplaceAll(string(routes[index].Protocol), "_", "-")) + } + return config{ + KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), + GroupPrefix: groupPrefix, + Routes: routes, + WorkersPerProtocol: envInt("FIELDS_PROJECTOR_WORKERS_PER_PROTOCOL", 1), + BatchSize: envInt("FIELDS_PROJECTOR_BATCH_SIZE", 500), + BatchWait: time.Duration(envInt("FIELDS_PROJECTOR_BATCH_WAIT_MS", 20)) * time.Millisecond, + OperationTimeout: time.Duration(envInt("FIELDS_PROJECTOR_OPERATION_TIMEOUT_MS", 30000)) * time.Millisecond, + RetryDelay: time.Duration(envInt("FIELDS_PROJECTOR_RETRY_DELAY_MS", 500)) * time.Millisecond, + StartOffset: startOffset, + StartOffsetName: startOffsetName, + } +} + +func (c config) Validate() error { + if len(c.KafkaBrokers) == 0 { + return errors.New("kafka brokers are required") + } + if strings.TrimSpace(c.GroupPrefix) == "" { + return errors.New("fields projector group prefix is required") + } + if c.WorkersPerProtocol < 1 || c.BatchSize < 1 || c.BatchWait <= 0 || c.OperationTimeout <= 0 || c.RetryDelay <= 0 { + return errors.New("fields projector worker, batch and timeout settings must be positive") + } + raw := make(map[string]string, len(c.Routes)) + fields := make(map[string]string, len(c.Routes)) + groups := map[string]struct{}{} + for _, route := range c.Routes { + protocol := string(route.Protocol) + raw[protocol] = route.RawTopic + fields[protocol] = route.FieldsTopic + if strings.TrimSpace(route.GroupID) == "" { + return fmt.Errorf("consumer group is required for protocol %s", route.Protocol) + } + if _, exists := groups[route.GroupID]; exists { + return fmt.Errorf("duplicate consumer group %q", route.GroupID) + } + groups[route.GroupID] = struct{}{} + } + return topics.ValidateKafkaRawFields(raw, fields) +} + +type kafkaBatchWriter interface { + WriteMessages(context.Context, ...kafka.Message) error +} + +type kafkaMessageFetcher interface { + FetchMessage(context.Context) (kafka.Message, error) +} + +type kafkaMessageCommitter interface { + CommitMessages(context.Context, ...kafka.Message) error +} + +func runProjector(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, cfg config, route projectionRoute, workerID int) { + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: cfg.KafkaBrokers, + GroupID: route.GroupID, + GroupTopics: []string{route.RawTopic}, + StartOffset: cfg.StartOffset, + MinBytes: 1, + MaxBytes: 10e6, + }) + defer reader.Close() + writer := &kafka.Writer{ + Addr: kafka.TCP(cfg.KafkaBrokers...), + Balancer: &kafka.Hash{}, + RequiredAcks: kafka.RequireAll, + AllowAutoTopicCreation: false, + BatchTimeout: cfg.BatchWait, + Async: false, + } + defer writer.Close() + + labels := metrics.Labels{"protocol": string(route.Protocol), "worker": strconv.Itoa(workerID)} + registry.SetGauge("vehicle_fields_projector_worker_active", labels, 1) + defer registry.SetGauge("vehicle_fields_projector_worker_active", labels, 0) + for { + first, err := reader.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + logger.Error("kafka fetch failed", "error", err) + continue + } + batch := collectBatch(ctx, reader, first, cfg.BatchSize, cfg.BatchWait) + processBatchReliably(ctx, logger, registry, writer, reader, route, batch, cfg.OperationTimeout, cfg.RetryDelay) + } +} + +func collectBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { + if maxSize <= 1 { + return []kafka.Message{first} + } + batch := []kafka.Message{first} + deadline := time.Now().Add(maxWait) + for len(batch) < maxSize { + remaining := time.Until(deadline) + if remaining <= 0 { + break + } + fetchCtx, cancel := context.WithTimeout(ctx, remaining) + message, err := fetcher.FetchMessage(fetchCtx) + cancel() + if err != nil { + break + } + batch = append(batch, message) + } + return batch +} + +func processBatchReliably(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, writer kafkaBatchWriter, committer kafkaMessageCommitter, route projectionRoute, messages []kafka.Message, operationTimeout time.Duration, retryDelay time.Duration) { + labels := metrics.Labels{"protocol": string(route.Protocol)} + defer registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, 0) + for len(messages) > 0 { + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout) + outputs, err := projectBatch(operationCtx, logger, registry, writer, route, messages) + cancel() + if err != nil { + registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages))) + registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "write_error"}) + if !waitForRetry(ctx, retryDelay) { + return + } + continue + } + commitCtx, commitCancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout) + err = committer.CommitMessages(commitCtx, messages...) + commitCancel() + if err == nil { + for _, message := range messages { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok") + } + _ = outputs + return + } + for _, message := range messages { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error") + } + registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages))) + registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "commit_error"}) + logger.Error("kafka source offset commit failed", "topic", route.RawTopic, "messages", len(messages), "error", err) + if !retryCommit(ctx, logger, registry, committer, messages, route.Protocol, operationTimeout, retryDelay) { + return + } + return + } +} + +func projectBatch(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, writer kafkaBatchWriter, route projectionRoute, messages []kafka.Message) (int, error) { + if len(messages) == 0 { + return 0, nil + } + outputs := make([]kafka.Message, 0, len(messages)) + setBatchPending(registry, route.Protocol, len(messages), 0) + defer setBatchPending(registry, route.Protocol, -len(messages), -len(outputs)) + for _, message := range messages { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "received") + recordKafkaLag(registry, message) + var raw envelope.FrameEnvelope + if err := json.Unmarshal(message.Value, &raw); err != nil { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "invalid_json") + logger.Warn("skip invalid raw envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + continue + } + if status, err := topics.ValidateRawEnvelope(route.RawTopic, raw); err != nil { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, status) + logger.Warn("skip mismatched raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", raw.Protocol, "event_id", raw.StableEventID(), "error", err) + continue + } + fields, ok := realtime.BuildFieldsEnvelope(raw) + if !ok { + status := "skipped_missing_fields" + if !envelope.IsRealtimeTelemetryFrame(raw) { + status = "skipped_non_realtime" + } + recordProjectionMetric(registry, route.Protocol, status, 0) + continue + } + if status, err := topics.ValidateFieldsEnvelope(route.FieldsTopic, fields); err != nil { + recordProjectionMetric(registry, route.Protocol, status, len(fields.Fields)) + logger.Warn("skip invalid projected fields envelope", "topic", route.FieldsTopic, "protocol", fields.Protocol, "event_id", fields.StableEventID(), "error", err) + continue + } + payload, err := fields.MarshalJSONBytes() + if err != nil { + recordProjectionMetric(registry, route.Protocol, "marshal_error", len(fields.Fields)) + logger.Warn("skip fields envelope marshal error", "topic", route.FieldsTopic, "event_id", fields.StableEventID(), "error", err) + continue + } + outputs = append(outputs, kafka.Message{ + Topic: route.FieldsTopic, + Key: fields.KafkaKey(), + Value: payload, + Time: message.Time, + }) + recordProjectionMetric(registry, route.Protocol, "projected", len(fields.Fields)) + } + setBatchPending(registry, route.Protocol, 0, len(outputs)) + defer setBatchPending(registry, route.Protocol, 0, -len(outputs)) + if len(outputs) == 0 { + return 0, nil + } + started := time.Now() + err := writer.WriteMessages(ctx, outputs...) + status := "ok" + if err != nil { + status = "error" + } + recordWriteDuration(registry, route.FieldsTopic, status, time.Since(started)) + for range outputs { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_writes_total", route.FieldsTopic, status) + } + if err != nil { + logger.Error("fields kafka write failed", "topic", route.FieldsTopic, "messages", len(outputs), "error", err) + return len(outputs), err + } + return len(outputs), nil +} + +func retryCommit(ctx context.Context, logger interface { + Error(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, protocol envelope.Protocol, operationTimeout, retryDelay time.Duration) bool { + for { + if !waitForRetry(ctx, retryDelay) { + return false + } + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout) + err := committer.CommitMessages(operationCtx, messages...) + cancel() + if err == nil { + for _, message := range messages { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok") + } + return true + } + for _, message := range messages { + recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error") + } + registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(protocol), "reason": "commit_error"}) + logger.Error("kafka source offset commit retry failed", "messages", len(messages), "error", err) + } +} + +func waitForRetry(ctx context.Context, delay time.Duration) bool { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func pingKafka(ctx context.Context, brokers []string) error { + if len(brokers) == 0 { + return errors.New("kafka broker is required") + } + checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + conn, err := kafka.DialContext(checkCtx, "tcp", brokers[0]) + if err != nil { + return err + } + return conn.Close() +} + +func recordConfigMetrics(registry *metrics.Registry, cfg config) { + registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "workers_per_protocol"}, float64(cfg.WorkersPerProtocol)) + registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize)) + registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_wait_ms"}, float64(cfg.BatchWait.Milliseconds())) + registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationTimeout.Milliseconds())) +} + +func recordMessageMetric(registry *metrics.Registry, name, topic, status string) { + if registry == nil { + return + } + labels := metrics.Labels{"topic": topic, "status": status} + registry.IncCounter(name, labels) + switch name { + case "vehicle_fields_projector_kafka_messages_total": + metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_message_unix_seconds", labels) + case "vehicle_fields_projector_kafka_writes_total": + metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_write_unix_seconds", labels) + case "vehicle_fields_projector_kafka_commits_total": + metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_commit_unix_seconds", labels) + } +} + +func recordProjectionMetric(registry *metrics.Registry, protocol envelope.Protocol, status string, fieldCount int) { + if registry == nil { + return + } + labels := metrics.Labels{"protocol": string(protocol), "status": status} + registry.IncCounter("vehicle_fields_projector_projections_total", labels) + if fieldCount > 0 { + registry.SetGauge("vehicle_fields_projector_field_count", labels, float64(fieldCount)) + } +} + +func recordKafkaLag(registry *metrics.Registry, message kafka.Message) { + if registry == nil { + return + } + lag := message.HighWaterMark - message.Offset - 1 + if lag < 0 { + lag = 0 + } + registry.SetGauge("vehicle_fields_projector_kafka_lag", metrics.Labels{ + "topic": message.Topic, "partition": strconv.Itoa(message.Partition), + }, float64(lag)) +} + +var projectorPendingMu sync.Mutex +var projectorPendingMessages = map[envelope.Protocol]int{} +var projectorPendingFields = map[envelope.Protocol]int{} + +func setBatchPending(registry *metrics.Registry, protocol envelope.Protocol, messagesDelta, fieldsDelta int) { + if registry == nil { + return + } + projectorPendingMu.Lock() + projectorPendingMessages[protocol] += messagesDelta + projectorPendingFields[protocol] += fieldsDelta + messages := projectorPendingMessages[protocol] + fields := projectorPendingFields[protocol] + projectorPendingMu.Unlock() + labels := metrics.Labels{"protocol": string(protocol)} + registry.SetGauge("vehicle_fields_projector_batch_pending_messages", labels, float64(messages)) + registry.SetGauge("vehicle_fields_projector_batch_pending_fields", labels, float64(fields)) +} + +var projectorWriteBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + +func recordWriteDuration(registry *metrics.Registry, topic, status string, elapsed time.Duration) { + if registry == nil { + return + } + registry.ObserveHistogram("vehicle_fields_projector_write_duration_ms_histogram", metrics.Labels{ + "topic": topic, "status": status, + }, projectorWriteBucketsMS, float64(elapsed.Milliseconds())) +} + +func env(key, fallback string) string { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + return value +} + +func envInt(key string, fallback int) int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + +func splitCSV(value string) []string { + var out []string + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item != "" { + out = append(out, item) + } + } + return out +} diff --git a/go/vehicle-gateway/cmd/fields-projector/main_test.go b/go/vehicle-gateway/cmd/fields-projector/main_test.go new file mode 100644 index 00000000..11886a2c --- /dev/null +++ b/go/vehicle-gateway/cmd/fields-projector/main_test.go @@ -0,0 +1,242 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" +) + +func TestLoadConfigCreatesProtocolIsolatedConsumerGroups(t *testing.T) { + t.Setenv("FIELDS_PROJECTOR_GROUP_PREFIX", "projector-test") + t.Setenv("KAFKA_START_OFFSET", "first") + cfg := loadConfig() + if cfg.StartOffset != kafka.FirstOffset || cfg.StartOffsetName != "first" { + t.Fatalf("start offset = %d/%q", cfg.StartOffset, cfg.StartOffsetName) + } + wantGroups := map[envelope.Protocol]string{ + envelope.ProtocolGB32960: "projector-test-gb32960", + envelope.ProtocolJT808: "projector-test-jt808", + envelope.ProtocolYutongMQTT: "projector-test-yutong-mqtt", + } + for _, route := range cfg.Routes { + if route.GroupID != wantGroups[route.Protocol] { + t.Fatalf("group for %s = %q", route.Protocol, route.GroupID) + } + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } +} + +func TestProjectBatchUsesPrecomputedFieldsAndPreservesSourceMetadata(t *testing.T) { + raw := projectorRawEnvelope() + payload, err := raw.MarshalJSONBytes() + if err != nil { + t.Fatalf("marshal raw: %v", err) + } + writer := &recordingProjectorWriter{} + registry := metrics.NewRegistry() + route := jt808ProjectionRoute() + count, err := projectBatch(context.Background(), discardProjectorLogger{}, registry, writer, route, []kafka.Message{{ + Topic: route.RawTopic, Key: raw.KafkaKey(), Value: payload, Partition: 2, Offset: 10, HighWaterMark: 11, + }}) + if err != nil { + t.Fatalf("projectBatch() error = %v", err) + } + if count != 1 || writer.callCount() != 1 || len(writer.messages) != 1 { + t.Fatalf("projected=%d calls=%d messages=%d", count, writer.callCount(), len(writer.messages)) + } + var fields envelope.FrameEnvelope + if err := json.Unmarshal(writer.messages[0].Value, &fields); err != nil { + t.Fatalf("decode fields: %v", err) + } + if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID || fields.EventID != raw.EventID+":fields" { + t.Fatalf("fields identity = %#v", fields) + } + if fields.SourceCode != raw.SourceCode || fields.SourceKind != raw.SourceKind || fields.SourceEndpoint != raw.SourceEndpoint { + t.Fatalf("source metadata not preserved: %#v", fields) + } + if got := fields.Fields["jt808.location.total_mileage_km"]; got != 1234.5 { + t.Fatalf("total mileage = %#v", got) + } + if len(fields.Parsed) != 0 || len(fields.ParsedFields) != 0 { + t.Fatalf("fields projection must not duplicate raw payload: parsed=%v parsed_fields=%v", fields.Parsed, fields.ParsedFields) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fields_projector_projections_total{protocol="JT808",status="projected"} 1`, + `vehicle_fields_projector_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_fields_projector_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metric missing %q:\n%s", want, text) + } + } +} + +func TestProcessBatchReliablySkipsNonRealtimeAndInvalidWithoutWriting(t *testing.T) { + route := jt808ProjectionRoute() + nonRealtime := projectorRawEnvelope() + nonRealtime.MessageID = "0x0002" + nonRealtime.ParsedFields = map[string]any{"jt808.header.message_id": "0x0002"} + payload, _ := nonRealtime.MarshalJSONBytes() + messages := []kafka.Message{ + {Topic: route.RawTopic, Value: []byte("{bad"), Partition: 0, Offset: 1, HighWaterMark: 3}, + {Topic: route.RawTopic, Value: payload, Partition: 0, Offset: 2, HighWaterMark: 3}, + } + writer := &recordingProjectorWriter{} + committer := &recordingProjectorCommitter{} + registry := metrics.NewRegistry() + processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond) + if writer.callCount() != 0 { + t.Fatalf("writer calls = %d, want 0", writer.callCount()) + } + if committer.callCount() != 1 || committer.messageCount != 2 { + t.Fatalf("commit calls/messages = %d/%d", committer.callCount(), committer.messageCount) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_fields_projector_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`) || + !strings.Contains(text, `vehicle_fields_projector_projections_total{protocol="JT808",status="skipped_non_realtime"} 1`) { + t.Fatalf("skip metrics missing:\n%s", text) + } +} + +func TestProcessBatchReliablyRetriesWriteBeforeCommitting(t *testing.T) { + route := jt808ProjectionRoute() + payload, _ := projectorRawEnvelope().MarshalJSONBytes() + messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}} + writer := &recordingProjectorWriter{errors: []error{errors.New("kafka unavailable"), nil}} + committer := &recordingProjectorCommitter{} + registry := metrics.NewRegistry() + processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond) + if writer.callCount() != 2 { + t.Fatalf("writer calls = %d, want 2", writer.callCount()) + } + if committer.callCount() != 1 { + t.Fatalf("commit calls = %d, want 1 after successful write", committer.callCount()) + } + if !strings.Contains(registry.Render(), `vehicle_fields_projector_batch_retries_total{protocol="JT808",reason="write_error"} 1`) { + t.Fatalf("write retry metric missing:\n%s", registry.Render()) + } +} + +func TestProcessBatchReliablyRetriesOnlyCommitAfterSuccessfulWrite(t *testing.T) { + route := jt808ProjectionRoute() + payload, _ := projectorRawEnvelope().MarshalJSONBytes() + messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}} + writer := &recordingProjectorWriter{} + committer := &recordingProjectorCommitter{errors: []error{errors.New("commit timeout"), nil}} + processBatchReliably(context.Background(), discardProjectorLogger{}, metrics.NewRegistry(), writer, committer, route, messages, time.Second, time.Millisecond) + if writer.callCount() != 1 { + t.Fatalf("writer calls = %d, want 1", writer.callCount()) + } + if committer.callCount() != 2 { + t.Fatalf("commit calls = %d, want 2", committer.callCount()) + } +} + +func TestProjectBatchWriteFailureDoesNotCommitByItself(t *testing.T) { + route := jt808ProjectionRoute() + payload, _ := projectorRawEnvelope().MarshalJSONBytes() + writer := &recordingProjectorWriter{errors: []error{errors.New("write failed")}} + count, err := projectBatch(context.Background(), discardProjectorLogger{}, nil, writer, route, []kafka.Message{{Topic: route.RawTopic, Value: payload}}) + if err == nil || count != 1 { + t.Fatalf("projectBatch() count/error = %d/%v", count, err) + } +} + +func projectorRawEnvelope() envelope.FrameEnvelope { + return envelope.FrameEnvelope{ + EventID: "raw-event-1", + EventKind: envelope.EventKindRaw, + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + Phone: "13307795425", + SourceEndpoint: "115.231.168.135:43625", + SourceCode: "g7s", + SourceKind: "PLATFORM", + PlatformName: "G7s", + EventTimeMS: 1783960000000, + ReceivedAtMS: 1783960000010, + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.1, + "jt808.location.longitude": 121.2, + "jt808.location.total_mileage_km": 1234.5, + }, + ParseStatus: envelope.ParseOK, + } +} + +func jt808ProjectionRoute() projectionRoute { + return projectionRoute{ + Protocol: envelope.ProtocolJT808, RawTopic: topics.RawJT808, FieldsTopic: topics.FieldsJT808, GroupID: "projector-jt808", + } +} + +type recordingProjectorWriter struct { + mu sync.Mutex + errors []error + calls int + messages []kafka.Message +} + +func (w *recordingProjectorWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error { + w.mu.Lock() + defer w.mu.Unlock() + w.calls++ + w.messages = append(w.messages, messages...) + if len(w.errors) == 0 { + return nil + } + err := w.errors[0] + w.errors = w.errors[1:] + return err +} + +func (w *recordingProjectorWriter) callCount() int { + w.mu.Lock() + defer w.mu.Unlock() + return w.calls +} + +type recordingProjectorCommitter struct { + mu sync.Mutex + errors []error + calls int + messageCount int +} + +func (c *recordingProjectorCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error { + c.mu.Lock() + defer c.mu.Unlock() + c.calls++ + c.messageCount += len(messages) + if len(c.errors) == 0 { + return nil + } + err := c.errors[0] + c.errors = c.errors[1:] + return err +} + +func (c *recordingProjectorCommitter) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.calls +} + +type discardProjectorLogger struct{} + +func (discardProjectorLogger) Error(string, ...any) {} +func (discardProjectorLogger) Warn(string, ...any) {} diff --git a/go/vehicle-gateway/cmd/gateway/main.go b/go/vehicle-gateway/cmd/gateway/main.go index 47bc429a..0880d102 100644 --- a/go/vehicle-gateway/cmd/gateway/main.go +++ b/go/vehicle-gateway/cmd/gateway/main.go @@ -3,6 +3,8 @@ package main import ( "context" "database/sql" + "encoding/json" + "fmt" "log/slog" "os" "os/signal" @@ -13,6 +15,7 @@ import ( _ "github.com/go-sql-driver/mysql" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway" @@ -37,7 +40,7 @@ func main() { os.Exit(1) } defer sink.Close() - resolver, closeResolver, err := buildIdentityResolver(ctx, logger) + resolver, jt808DeviceTokens, closeResolver, err := buildIdentityResolver(ctx, logger, registry) if err != nil { logger.Error("build identity resolver failed", "error", err) os.Exit(1) @@ -45,21 +48,38 @@ func main() { defer closeResolver() health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-gateway", nil, registry)) publishUnified := envBool("PUBLISH_UNIFIED_ENABLED", false) + delegateFields := envBool("FIELDS_DERIVE_FROM_RAW_ENABLED", strings.TrimSpace(os.Getenv("NATS_URL")) != "") + gb32960Authenticator, gb32960AuthMode, gb32960CredentialCount, err := buildGB32960Authenticator() + if err != nil { + logger.Error("build gb32960 authenticator failed", "error", err) + os.Exit(1) + } + jt808AuthCode := env("JT808_REGISTER_AUTH_CODE", "g7gps") + jt808Authenticator, jt808AuthMode, err := buildJT808Authenticator(jt808AuthCode, jt808DeviceTokens) + if err != nil { + logger.Error("build jt808 authenticator failed", "error", err) + os.Exit(1) + } + recordAuthenticationConfig(registry, envelope.ProtocolGB32960, gb32960AuthMode, gb32960CredentialCount) + recordAuthenticationConfig(registry, envelope.ProtocolJT808, jt808AuthMode, int(boolMetric(strings.TrimSpace(jt808AuthCode) != ""))) + logger.Info("protocol authentication configured", "gb32960_mode", gb32960AuthMode, "gb32960_accounts", gb32960CredentialCount, "jt808_mode", jt808AuthMode) protocols := []gateway.TCPProtocol{ { - Protocol: envelope.ProtocolGB32960, - Addr: env("GB32960_TCP_ADDR", ":32960"), - Extract: gb32960.ExtractFrames, - Parse: gb32960.ParseFrame, - Respond: gb32960.AutoResponse, + Protocol: envelope.ProtocolGB32960, + Addr: env("GB32960_TCP_ADDR", ":32960"), + Extract: gb32960.ExtractFrames, + Parse: gb32960.ParseFrame, + Authenticate: gb32960Authenticator, + Respond: gb32960.AutoResponse, }, { - Protocol: envelope.ProtocolJT808, - Addr: env("JT808_TCP_ADDR", ":808"), - Extract: jt808.ExtractFrames, - Parse: jt808.ParseFrame, - Respond: jt808.NewAutoResponder(env("JT808_REGISTER_AUTH_CODE", "g7gps")).Respond, + Protocol: envelope.ProtocolJT808, + Addr: env("JT808_TCP_ADDR", ":808"), + Extract: jt808.ExtractFrames, + Parse: jt808.ParseFrame, + Authenticate: jt808Authenticator, + Respond: jt808.NewAutoResponder(jt808AuthCode).Respond, }, } @@ -76,6 +96,7 @@ func main() { IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second, MaxConnections: envInt("TCP_MAX_CONNECTIONS", 120_000), PublishUnified: publishUnified, + DelegateFields: delegateFields, }) if err != nil { logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err) @@ -110,6 +131,7 @@ func main() { Logger: logger, Metrics: registry, PublishUnified: publishUnified, + DelegateFields: delegateFields, }) if err != nil { logger.Error("build yutong mqtt client failed", "error", err) @@ -122,7 +144,7 @@ func main() { logger.Info("yutong mqtt client started") } - logger.Info("vehicle gateway started") + logger.Info("vehicle gateway started", "fields_derive_from_raw_enabled", delegateFields) select { case <-ctx.Done(): case err := <-errs: @@ -132,63 +154,433 @@ func main() { wg.Wait() } -func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.Resolver, func(), error) { +func buildGB32960Authenticator() (authentication.Authenticator, authentication.Mode, int, error) { + mode, err := authentication.ParseMode(os.Getenv("GB32960_AUTH_MODE"), authentication.ModeObserve) + if err != nil { + return nil, "", 0, err + } + credentials, err := loadGB32960Credentials() + if err != nil { + return nil, "", 0, err + } + if mode == authentication.ModeEnforce && len(credentials) == 0 { + return nil, "", 0, fmt.Errorf("GB32960_AUTH_MODE=enforce requires configured platform credentials") + } + return authentication.NewGB32960PlatformAuthenticator(mode, credentials), mode, len(credentials), nil +} + +func buildJT808Authenticator(authCode string, deviceTokens authentication.JT808DeviceTokenProvider) (authentication.Authenticator, authentication.Mode, error) { + mode, err := authentication.ParseMode(os.Getenv("JT808_AUTH_MODE"), authentication.ModeObserve) + if err != nil { + return nil, "", err + } + if mode == authentication.ModeEnforce && strings.TrimSpace(authCode) == "" && deviceTokens == nil { + return nil, "", fmt.Errorf("JT808_AUTH_MODE=enforce requires a configured or device token provider") + } + return authentication.NewJT808Authenticator(mode, authCode, deviceTokens), mode, nil +} + +func loadGB32960Credentials() (map[string][]string, error) { + payload := strings.TrimSpace(os.Getenv("GB32960_PLATFORM_CREDENTIALS_JSON")) + if path := strings.TrimSpace(os.Getenv("GB32960_PLATFORM_CREDENTIALS_FILE")); path != "" { + contents, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read gb32960 credentials file: %w", err) + } + payload = strings.TrimSpace(string(contents)) + } + if payload == "" { + return map[string][]string{}, nil + } + rawCredentials := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(payload), &rawCredentials); err != nil { + return nil, fmt.Errorf("parse gb32960 platform credentials: %w", err) + } + credentials := make(map[string][]string, len(rawCredentials)) + for username, rawPassword := range rawCredentials { + trimmed := strings.TrimSpace(username) + if trimmed == "" { + continue + } + var passwords []string + var single string + if err := json.Unmarshal(rawPassword, &single); err == nil { + passwords = []string{single} + } else if err := json.Unmarshal(rawPassword, &passwords); err != nil { + return nil, fmt.Errorf("parse gb32960 credentials for account %q: expected string or string array", trimmed) + } + for _, password := range passwords { + if password != "" { + credentials[trimmed] = append(credentials[trimmed], password) + } + } + if len(credentials[trimmed]) == 0 { + delete(credentials, trimmed) + } + } + return credentials, nil +} + +func recordAuthenticationConfig(registry *metrics.Registry, protocol envelope.Protocol, mode authentication.Mode, credentialCount int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_gateway_authentication_mode", metrics.Labels{ + "protocol": string(protocol), + "mode": string(mode), + }, 1) + registry.SetGauge("vehicle_gateway_authentication_credentials", metrics.Labels{ + "protocol": string(protocol), + }, float64(credentialCount)) +} + +func buildIdentityResolver(ctx context.Context, logger *slog.Logger, registry *metrics.Registry) (identity.Resolver, authentication.JT808DeviceTokenProvider, func(), error) { dsn := env("IDENTITY_MYSQL_DSN", strings.TrimSpace(os.Getenv("MYSQL_DSN"))) if strings.TrimSpace(dsn) == "" { logger.Warn("identity mysql dsn is empty; using noop identity resolver") - return identity.NoopResolver{}, func() {}, nil + return identity.NoopResolver{}, nil, func() {}, nil } db, err := sql.Open("mysql", dsn) if err != nil { - return nil, nil, err - } - if err := db.PingContext(ctx); err != nil { - _ = db.Close() - return nil, nil, err + logger.Error("identity mysql configuration failed; continuing with unresolved identities", "error", err) + return identity.NoopResolver{}, nil, func() {}, nil } + db.SetMaxOpenConns(envInt("IDENTITY_MYSQL_MAX_OPEN_CONNS", 16)) + db.SetMaxIdleConns(envInt("IDENTITY_MYSQL_MAX_IDLE_CONNS", 8)) + db.SetConnMaxLifetime(time.Duration(envInt("IDENTITY_MYSQL_CONN_MAX_LIFETIME_SECONDS", 300)) * time.Second) + db.SetConnMaxIdleTime(time.Duration(envInt("IDENTITY_MYSQL_CONN_MAX_IDLE_SECONDS", 60)) * time.Second) table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding") + cacheMaxEntries := envInt("IDENTITY_LOOKUP_CACHE_MAX_ENTRIES", 300000) + cacheCleanupInterval := time.Duration(envInt("IDENTITY_LOOKUP_CACHE_CLEANUP_INTERVAL_SECONDS", 60)) * time.Second + staleLookupTTLSeconds := envInt("IDENTITY_STALE_LOOKUP_TTL_SECONDS", 3600) + registrationGatewayWritesEnabled := envBool("JT808_REGISTRATION_GATEWAY_WRITES_ENABLED", false) resolver := identity.NewMySQLResolverWithOptions(db, table, identity.MySQLResolverOptions{ - LocationTouchInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS", 600)) * time.Second, - LookupCacheTTL: time.Duration(envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600)) * time.Second, + SnapshotOnlyLookups: envBool("IDENTITY_SNAPSHOT_ONLY_ENABLED", true), + LocationTouchInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS", 600)) * time.Second, + LocationTouchRetryInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_RETRY_INTERVAL_SECONDS", 5)) * time.Second, + RegistrationWriteAttempts: envInt("JT808_REGISTRATION_WRITE_RETRY_ATTEMPTS", 2), + RegistrationWriteRetryDelay: time.Duration(envInt("JT808_REGISTRATION_WRITE_RETRY_DELAY_MS", 20)) * time.Millisecond, + LookupCacheTTL: time.Duration(envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600)) * time.Second, + StaleLookupTTL: time.Duration(staleLookupTTLSeconds) * time.Second, + CacheCleanupInterval: cacheCleanupInterval, + MaxCacheEntries: cacheMaxEntries, + SourceCodeLookup: envBool("IDENTITY_SOURCE_CODE_LOOKUP_ENABLED", true), + AsyncRegistrationWrites: envBool("JT808_REGISTRATION_ASYNC_WRITE_ENABLED", true), + RegistrationWriteQueueSize: envInt("JT808_REGISTRATION_WRITE_QUEUE_SIZE", 100000), + RegistrationWriteWorkers: envInt("JT808_REGISTRATION_WRITE_WORKERS", 4), + RegistrationWriteTimeout: time.Duration(envInt("JT808_REGISTRATION_WRITE_TIMEOUT_MS", 5000)) * time.Millisecond, + RegistrationEnqueueTimeout: time.Duration(envInt("JT808_REGISTRATION_WRITE_ENQUEUE_TIMEOUT_MS", 50)) * time.Millisecond, + DisableRegistrationWrites: !registrationGatewayWritesEnabled, + OnRegistrationWriteResult: func(result identity.RegistrationWriteResult) { + recordJT808RegistrationWriteResult(registry, result) + }, + OnRegistrationWriteError: func(err error) { + logger.Warn("jt808 registration async write failed", "error", err) + }, }) - if envBool("IDENTITY_MYSQL_ENSURE_SCHEMA", true) { - if err := resolver.EnsureSchema(ctx); err != nil { - _ = db.Close() - return nil, nil, err - } + startIdentityDatabaseMaintenance( + ctx, + logger, + db, + resolver, + envBool("IDENTITY_MYSQL_ENSURE_SCHEMA", false), + time.Duration(envInt("IDENTITY_MYSQL_PING_TIMEOUT_MS", 3000))*time.Millisecond, + time.Duration(envInt("IDENTITY_MYSQL_SCHEMA_TIMEOUT_SECONDS", 10))*time.Second, + ) + startIdentitySnapshotRefresh( + ctx, + logger, + registry, + resolver, + time.Duration(envInt("IDENTITY_SNAPSHOT_REFRESH_INTERVAL_SECONDS", 60))*time.Second, + time.Duration(envInt("IDENTITY_SNAPSHOT_REFRESH_TIMEOUT_SECONDS", 10))*time.Second, + ) + startIdentityCacheMetrics(ctx, registry, resolver, time.Duration(envInt("IDENTITY_CACHE_METRICS_INTERVAL_SECONDS", 30))*time.Second) + resolveTimeout := time.Duration(envInt("IDENTITY_RESOLVE_TIMEOUT_MS", 50)) * time.Millisecond + registry.SetGauge("vehicle_gateway_jt808_registration_gateway_writes_enabled", nil, boolMetric(registrationGatewayWritesEnabled)) + logger.Info("identity mysql resolver enabled", "table", table, "snapshot_only_enabled", envBool("IDENTITY_SNAPSHOT_ONLY_ENABLED", true), "snapshot_refresh_interval_seconds", envInt("IDENTITY_SNAPSHOT_REFRESH_INTERVAL_SECONDS", 60), "lookup_cache_ttl_seconds", envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600), "stale_lookup_ttl_seconds", staleLookupTTLSeconds, "lookup_cache_max_entries", cacheMaxEntries, "lookup_cache_cleanup_interval_seconds", cacheCleanupInterval.Seconds(), "source_code_lookup_enabled", envBool("IDENTITY_SOURCE_CODE_LOOKUP_ENABLED", true), "resolve_timeout_ms", resolveTimeout.Milliseconds(), "jt808_registration_gateway_writes_enabled", registrationGatewayWritesEnabled, "jt808_registration_location_touch_retry_interval_seconds", envInt("JT808_REGISTRATION_LOCATION_TOUCH_RETRY_INTERVAL_SECONDS", 5), "jt808_registration_write_retry_attempts", envInt("JT808_REGISTRATION_WRITE_RETRY_ATTEMPTS", 2), "jt808_registration_write_retry_delay_ms", envInt("JT808_REGISTRATION_WRITE_RETRY_DELAY_MS", 20), "jt808_registration_async_write_enabled", envBool("JT808_REGISTRATION_ASYNC_WRITE_ENABLED", true), "jt808_registration_write_queue_size", envInt("JT808_REGISTRATION_WRITE_QUEUE_SIZE", 100000), "jt808_registration_write_workers", envInt("JT808_REGISTRATION_WRITE_WORKERS", 4), "jt808_registration_write_timeout_ms", envInt("JT808_REGISTRATION_WRITE_TIMEOUT_MS", 5000), "jt808_registration_write_enqueue_timeout_ms", envInt("JT808_REGISTRATION_WRITE_ENQUEUE_TIMEOUT_MS", 50)) + return identity.TimeoutResolver{Delegate: resolver, Timeout: resolveTimeout}, resolver, func() { + _ = resolver.Close() + _ = db.Close() + }, nil +} + +type identitySnapshotRefresher interface { + RefreshSnapshot(context.Context) (identity.SnapshotRefreshResult, error) +} + +type identityDatabasePinger interface { + PingContext(context.Context) error +} + +type identitySchemaEnsurer interface { + EnsureSchema(context.Context) error +} + +func startIdentityDatabaseMaintenance(ctx context.Context, logger *slog.Logger, db identityDatabasePinger, schema identitySchemaEnsurer, ensureSchema bool, pingTimeout time.Duration, schemaTimeout time.Duration) { + if db == nil { + return } - logger.Info("identity mysql resolver enabled", "table", table, "lookup_cache_ttl_seconds", envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600)) - return resolver, func() { _ = db.Close() }, nil + if pingTimeout <= 0 { + pingTimeout = 3 * time.Second + } + if schemaTimeout <= 0 { + schemaTimeout = 10 * time.Second + } + go func() { + pingCtx, cancelPing := context.WithTimeout(ctx, pingTimeout) + err := db.PingContext(pingCtx) + cancelPing() + if err != nil { + logger.Warn("identity mysql unavailable; ingress remains enabled and snapshot refresh will retry", "error", err) + return + } + if !ensureSchema || schema == nil { + return + } + schemaCtx, cancelSchema := context.WithTimeout(ctx, schemaTimeout) + err = schema.EnsureSchema(schemaCtx) + cancelSchema() + if err != nil { + logger.Warn("identity mysql schema check failed; ingress remains enabled", "error", err) + } + }() +} + +func startIdentitySnapshotRefresh(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, refresher identitySnapshotRefresher, interval time.Duration, timeout time.Duration) { + if refresher == nil { + return + } + if interval <= 0 { + interval = time.Minute + } + if timeout <= 0 { + timeout = 10 * time.Second + } + refresh := func() { + refreshCtx, cancel := context.WithTimeout(ctx, timeout) + result, err := refresher.RefreshSnapshot(refreshCtx) + cancel() + if err != nil { + recordIdentitySnapshotRefresh(registry, identity.SnapshotRefreshResult{}, "error") + logger.Warn("identity snapshot refresh failed; keeping last known good snapshot", "error", err) + return + } + recordIdentitySnapshotRefresh(registry, result, "ok") + logger.Info("identity snapshot refreshed", "binding_entries", result.BindingEntries, "identifier_entries", result.IdentifierEntries, "registration_entries", result.RegistrationEntries, "source_entries", result.SourceEntries) + } + go func() { + refresh() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + refresh() + } + } + }() +} + +func recordIdentitySnapshotRefresh(registry *metrics.Registry, result identity.SnapshotRefreshResult, status string) { + if registry == nil { + return + } + registry.IncCounter("vehicle_gateway_identity_snapshot_refresh_total", metrics.Labels{"status": status}) + if status != "ok" { + return + } + for _, item := range []struct { + kind string + value int + }{ + {kind: "binding", value: result.BindingEntries}, + {kind: "identifier", value: result.IdentifierEntries}, + {kind: "registration", value: result.RegistrationEntries}, + {kind: "source", value: result.SourceEntries}, + } { + registry.SetGauge("vehicle_gateway_identity_snapshot_entries", metrics.Labels{"kind": item.kind}, float64(item.value)) + } + registry.SetGauge("vehicle_gateway_identity_snapshot_last_success_unix_seconds", nil, float64(result.RefreshedAt.Unix())) +} + +type identityCacheStatsReporter interface { + CacheStats() identity.CacheStats +} + +func startIdentityCacheMetrics(ctx context.Context, registry *metrics.Registry, reporter identityCacheStatsReporter, interval time.Duration) { + if registry == nil || reporter == nil { + return + } + if interval <= 0 { + interval = 30 * time.Second + } + recordIdentityCacheStats(registry, reporter.CacheStats()) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + recordIdentityCacheStats(registry, reporter.CacheStats()) + } + } + }() +} + +func recordIdentityCacheStats(registry *metrics.Registry, stats identity.CacheStats) { + if registry == nil { + return + } + for _, item := range []struct { + name string + value int + max int + }{ + {name: "lookup", value: stats.LookupEntries, max: stats.MaxEntries}, + {name: "registration", value: stats.RegistrationEntries, max: stats.MaxEntries}, + {name: "source_code", value: stats.SourceCodeEntries, max: stats.MaxEntries}, + {name: "location_touch", value: stats.LocationTouchEntries, max: stats.MaxEntries}, + {name: "location_touch_failure", value: stats.LocationTouchFailureEntries, max: stats.MaxEntries}, + {name: "registration_write_queue", value: stats.RegistrationWriteQueueDepth, max: stats.RegistrationWriteQueueCap}, + {name: "snapshot_binding", value: stats.SnapshotBindingEntries, max: stats.MaxEntries}, + {name: "snapshot_identifier", value: stats.SnapshotIdentifierEntries, max: stats.MaxEntries}, + {name: "snapshot_registration", value: stats.SnapshotRegistrationEntries, max: stats.MaxEntries}, + {name: "snapshot_source", value: stats.SnapshotSourceEntries, max: stats.MaxEntries}, + } { + registry.SetGauge("vehicle_gateway_identity_cache_entries", metrics.Labels{"cache": item.name}, float64(item.value)) + registry.SetGauge("vehicle_gateway_identity_cache_max_entries", metrics.Labels{"cache": item.name}, float64(item.max)) + } + ready := 0.0 + if stats.SnapshotReady { + ready = 1 + } + registry.SetGauge("vehicle_gateway_identity_snapshot_ready", nil, ready) + if !stats.SnapshotRefreshedAt.IsZero() { + registry.SetGauge("vehicle_gateway_identity_snapshot_last_success_unix_seconds", nil, float64(stats.SnapshotRefreshedAt.Unix())) + } +} + +func recordJT808RegistrationWriteResult(registry *metrics.Registry, result identity.RegistrationWriteResult) { + if registry == nil { + return + } + mode := strings.TrimSpace(result.Mode) + if mode == "" { + mode = "unknown" + } + status := strings.TrimSpace(result.Status) + if status == "" { + status = "unknown" + } + registry.IncCounter("vehicle_gateway_jt808_registration_write_total", metrics.Labels{ + "mode": mode, + "status": status, + }) + metrics.RecordLastActivity(registry, "vehicle_gateway_last_jt808_registration_write_unix_seconds", metrics.Labels{ + "mode": mode, + "status": status, + }) } func buildSink(ctx context.Context, logger *slog.Logger, registry *metrics.Registry) (eventbus.Sink, error) { if strings.TrimSpace(os.Getenv("NATS_URL")) != "" { - sink, err := eventbus.NewNATSSink(natsSinkConfigFromEnv()) + natsConfig := natsSinkConfigFromEnv() + sink, err := eventbus.NewNATSSink(natsConfig) if err != nil { return nil, err } + outboxRuntime := natsOutboxConfigFromEnv(registry, func(err error) { + logger.Warn("nats durable outbox publish failed", "error", err) + }) + if outboxRuntime.Enabled { + outbox, err := eventbus.NewDurableOutboxSink(sink, outboxRuntime.Config) + if err != nil { + _ = sink.Close() + return nil, err + } + go outbox.ReplayLoop(ctx, outboxRuntime.ReplayInterval) + logger.Info("nats durable outbox enabled", + "dir", outboxRuntime.Config.Directory, + "fsync", outboxRuntime.Config.SyncWrites, + "replay_interval_ms", outboxRuntime.ReplayInterval.Milliseconds(), + "replay_batch_size", outboxRuntime.Config.ReplayBatchSize, + "close_timeout_ms", outboxRuntime.Config.CloseTimeout.Milliseconds(), + "wal_segment_bytes", outboxRuntime.Config.WALSegmentBytes, + "wal_segment_age_ms", outboxRuntime.Config.WALSegmentAge.Milliseconds(), + "wal_append_queue_size", outboxRuntime.Config.WALAppendQueue, + "wal_commit_batch_size", outboxRuntime.Config.WALCommitBatch, + "wal_commit_interval_ms", outboxRuntime.Config.WALCommitWait.Milliseconds(), + "max_inflight", natsConfig.AsyncMaxPending, + "ack_timeout_ms", natsConfig.AsyncAckTimeout.Milliseconds(), + ) + return outbox, nil + } var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{ Attempts: envInt("NATS_PUBLISH_ATTEMPTS", 3), Backoff: time.Duration(envInt("NATS_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond, AttemptTimeout: time.Duration(envInt("NATS_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond, }) + spoolDir := strings.TrimSpace(os.Getenv("NATS_SPOOL_DIR")) + if spoolDir != "" { + replayBatchSize := envInt("NATS_SPOOL_REPLAY_BATCH_SIZE", 200) + durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{ + Directory: spoolDir, + ReplayBatchSize: replayBatchSize, + Metrics: registry, + Name: "nats", + }) + interval := time.Duration(envInt("NATS_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond + go durable.ReplayLoop(ctx, interval, func(err error) { + logger.Warn("nats spool replay failed", "error", err) + }) + logger.Info("nats durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds(), "replay_batch_size", replayBatchSize) + out = durable + } if envBool("NATS_ASYNC_ENABLED", true) { queueSize := envInt("NATS_ASYNC_QUEUE_SIZE", envInt("KAFKA_ASYNC_QUEUE_SIZE", 100000)) workers := envInt("NATS_ASYNC_WORKERS", envInt("KAFKA_ASYNC_WORKERS", 8)) + enqueueTimeout := time.Duration(envInt("NATS_ASYNC_ENQUEUE_TIMEOUT_MS", envInt("KAFKA_ASYNC_ENQUEUE_TIMEOUT_MS", 1000))) * time.Millisecond timeout := time.Duration(envInt("NATS_ASYNC_PUBLISH_TIMEOUT_MS", envInt("KAFKA_ASYNC_PUBLISH_TIMEOUT_MS", 30000))) * time.Millisecond - out = eventbus.NewAsyncSink(out, eventbus.AsyncConfig{ - QueueSize: queueSize, - Workers: workers, - OperationTimeout: timeout, - Metrics: registry, - Name: "nats", - OnError: func(err error) { - logger.Warn("nats async publish failed", "error", err) - }, - }) - logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds()) + if envBool("NATS_PARTITIONED_ASYNC_ENABLED", true) { + rawQueueSize, derivedQueueSize, rawWorkers, derivedWorkers := partitionedAsyncConfigFromEnv("NATS", queueSize, workers) + rawEnqueueTimeout, derivedEnqueueTimeout := partitionedAsyncEnqueueTimeoutsFromEnv("NATS", enqueueTimeout) + out = eventbus.NewPartitionedAsyncSink(out, eventbus.PartitionedAsyncConfig{ + RawQueueSize: rawQueueSize, + DerivedQueueSize: derivedQueueSize, + RawWorkers: rawWorkers, + DerivedWorkers: derivedWorkers, + RawEnqueueTimeout: rawEnqueueTimeout, + DerivedEnqueueTimeout: derivedEnqueueTimeout, + EnqueueTimeout: enqueueTimeout, + OperationTimeout: timeout, + Metrics: registry, + Name: "nats", + OnError: func(err error) { + logger.Warn("nats partitioned async publish failed", "error", err) + }, + }) + logger.Info("nats partitioned async publish enabled", "raw_queue_size", rawQueueSize, "derived_queue_size", derivedQueueSize, "raw_workers", rawWorkers, "derived_workers", derivedWorkers, "raw_enqueue_timeout_ms", rawEnqueueTimeout.Milliseconds(), "derived_enqueue_timeout_ms", derivedEnqueueTimeout.Milliseconds(), "publish_timeout_ms", timeout.Milliseconds()) + } else { + out = eventbus.NewAsyncSink(out, eventbus.AsyncConfig{ + QueueSize: queueSize, + Workers: workers, + EnqueueTimeout: enqueueTimeout, + OperationTimeout: timeout, + Metrics: registry, + Name: "nats", + OnError: func(err error) { + logger.Warn("nats async publish failed", "error", err) + }, + }) + logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "enqueue_timeout_ms", enqueueTimeout.Milliseconds(), "publish_timeout_ms", timeout.Milliseconds()) + } } - logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false), "publish_fields_enabled", true) + logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false), "publish_fields_enabled", !envBool("FIELDS_DERIVE_FROM_RAW_ENABLED", true)) return out, nil } brokers := splitCSV(os.Getenv("KAFKA_BROKERS")) @@ -221,7 +613,12 @@ func buildSink(ctx context.Context, logger *slog.Logger, registry *metrics.Regis spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR")) if spoolDir != "" { replayBatchSize := envInt("KAFKA_SPOOL_REPLAY_BATCH_SIZE", 200) - durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir, ReplayBatchSize: replayBatchSize}) + durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{ + Directory: spoolDir, + ReplayBatchSize: replayBatchSize, + Metrics: registry, + Name: "kafka", + }) interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond go durable.ReplayLoop(ctx, interval, func(err error) { logger.Warn("kafka spool replay failed", "error", err) @@ -232,26 +629,73 @@ func buildSink(ctx context.Context, logger *slog.Logger, registry *metrics.Regis if envBool("KAFKA_ASYNC_ENABLED", true) { queueSize := envInt("KAFKA_ASYNC_QUEUE_SIZE", 100000) workers := envInt("KAFKA_ASYNC_WORKERS", 8) + enqueueTimeout := time.Duration(envInt("KAFKA_ASYNC_ENQUEUE_TIMEOUT_MS", 1000)) * time.Millisecond timeout := time.Duration(envInt("KAFKA_ASYNC_PUBLISH_TIMEOUT_MS", 30000)) * time.Millisecond - out = eventbus.NewAsyncSink(out, eventbus.AsyncConfig{ - QueueSize: queueSize, - Workers: workers, - OperationTimeout: timeout, - Metrics: registry, - Name: "kafka", - OnError: func(err error) { - logger.Warn("kafka async publish failed", "error", err) - }, - }) - logger.Info("kafka async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds()) + if envBool("KAFKA_PARTITIONED_ASYNC_ENABLED", true) { + rawQueueSize, derivedQueueSize, rawWorkers, derivedWorkers := partitionedAsyncConfigFromEnv("KAFKA", queueSize, workers) + rawEnqueueTimeout, derivedEnqueueTimeout := partitionedAsyncEnqueueTimeoutsFromEnv("KAFKA", enqueueTimeout) + out = eventbus.NewPartitionedAsyncSink(out, eventbus.PartitionedAsyncConfig{ + RawQueueSize: rawQueueSize, + DerivedQueueSize: derivedQueueSize, + RawWorkers: rawWorkers, + DerivedWorkers: derivedWorkers, + RawEnqueueTimeout: rawEnqueueTimeout, + DerivedEnqueueTimeout: derivedEnqueueTimeout, + EnqueueTimeout: enqueueTimeout, + OperationTimeout: timeout, + Metrics: registry, + Name: "kafka", + OnError: func(err error) { + logger.Warn("kafka partitioned async publish failed", "error", err) + }, + }) + logger.Info("kafka partitioned async publish enabled", "raw_queue_size", rawQueueSize, "derived_queue_size", derivedQueueSize, "raw_workers", rawWorkers, "derived_workers", derivedWorkers, "raw_enqueue_timeout_ms", rawEnqueueTimeout.Milliseconds(), "derived_enqueue_timeout_ms", derivedEnqueueTimeout.Milliseconds(), "publish_timeout_ms", timeout.Milliseconds()) + } else { + out = eventbus.NewAsyncSink(out, eventbus.AsyncConfig{ + QueueSize: queueSize, + Workers: workers, + EnqueueTimeout: enqueueTimeout, + OperationTimeout: timeout, + Metrics: registry, + Name: "kafka", + OnError: func(err error) { + logger.Warn("kafka async publish failed", "error", err) + }, + }) + logger.Info("kafka async publish enabled", "queue_size", queueSize, "workers", workers, "enqueue_timeout_ms", enqueueTimeout.Milliseconds(), "publish_timeout_ms", timeout.Milliseconds()) + } } return out, nil } +func partitionedAsyncConfigFromEnv(prefix string, queueSize int, workers int) (rawQueueSize int, derivedQueueSize int, rawWorkers int, derivedWorkers int) { + derivedQueueDefault := queueSize / 2 + if derivedQueueDefault <= 0 { + derivedQueueDefault = 1 + } + derivedWorkersDefault := workers / 2 + if derivedWorkersDefault <= 0 { + derivedWorkersDefault = 1 + } + rawQueueSize = envInt(prefix+"_ASYNC_RAW_QUEUE_SIZE", queueSize) + derivedQueueSize = envInt(prefix+"_ASYNC_DERIVED_QUEUE_SIZE", derivedQueueDefault) + rawWorkers = envInt(prefix+"_ASYNC_RAW_WORKERS", workers) + derivedWorkers = envInt(prefix+"_ASYNC_DERIVED_WORKERS", derivedWorkersDefault) + return rawQueueSize, derivedQueueSize, rawWorkers, derivedWorkers +} + +func partitionedAsyncEnqueueTimeoutsFromEnv(prefix string, enqueueTimeout time.Duration) (rawEnqueueTimeout time.Duration, derivedEnqueueTimeout time.Duration) { + rawEnqueueTimeout = time.Duration(envInt(prefix+"_ASYNC_RAW_ENQUEUE_TIMEOUT_MS", int(enqueueTimeout/time.Millisecond))) * time.Millisecond + derivedEnqueueTimeout = time.Duration(envInt(prefix+"_ASYNC_DERIVED_ENQUEUE_TIMEOUT_MS", 50)) * time.Millisecond + return rawEnqueueTimeout, derivedEnqueueTimeout +} + func natsSinkConfigFromEnv() eventbus.NATSConfig { return eventbus.NATSConfig{ - URL: env("NATS_URL", ""), - Name: env("NATS_CLIENT_NAME", "lingniu-vehicle-gateway"), + URL: env("NATS_URL", ""), + Name: env("NATS_CLIENT_NAME", "lingniu-vehicle-gateway"), + AsyncMaxPending: envInt("NATS_OUTBOX_MAX_INFLIGHT", 10_000), + AsyncAckTimeout: time.Duration(envInt("NATS_OUTBOX_ACK_TIMEOUT_MS", 3000)) * time.Millisecond, RawSubjects: map[envelope.Protocol]string{ envelope.ProtocolGB32960: env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)), envelope.ProtocolJT808: env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)), @@ -266,6 +710,34 @@ func natsSinkConfigFromEnv() eventbus.NATSConfig { } } +type natsOutboxRuntimeConfig struct { + Enabled bool + Config eventbus.DurableOutboxConfig + ReplayInterval time.Duration +} + +func natsOutboxConfigFromEnv(registry *metrics.Registry, onError func(error)) natsOutboxRuntimeConfig { + directory := strings.TrimSpace(os.Getenv("NATS_OUTBOX_DIR")) + return natsOutboxRuntimeConfig{ + Enabled: envBool("NATS_DURABLE_OUTBOX_ENABLED", false), + Config: eventbus.DurableOutboxConfig{ + Directory: directory, + ReplayBatchSize: envInt("NATS_OUTBOX_REPLAY_BATCH_SIZE", 1000), + SyncWrites: envBool("NATS_OUTBOX_FSYNC", true), + CloseTimeout: time.Duration(envInt("NATS_OUTBOX_CLOSE_TIMEOUT_MS", 5000)) * time.Millisecond, + WALSegmentBytes: int64(envInt("NATS_OUTBOX_WAL_SEGMENT_BYTES", 16<<20)), + WALSegmentAge: time.Duration(envInt("NATS_OUTBOX_WAL_SEGMENT_AGE_MS", 5000)) * time.Millisecond, + WALAppendQueue: envInt("NATS_OUTBOX_WAL_APPEND_QUEUE_SIZE", 100_000), + WALCommitBatch: envInt("NATS_OUTBOX_WAL_COMMIT_BATCH_SIZE", 256), + WALCommitWait: time.Duration(envInt("NATS_OUTBOX_WAL_COMMIT_INTERVAL_MS", 1)) * time.Millisecond, + Metrics: registry, + Name: "nats-outbox", + OnError: onError, + }, + ReplayInterval: time.Duration(envInt("NATS_OUTBOX_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond, + } +} + func env(key string, fallback string) string { value := strings.TrimSpace(os.Getenv(key)) if value == "" { @@ -300,6 +772,13 @@ func envBool(key string, fallback bool) bool { return value == "1" || value == "true" || value == "yes" || value == "on" } +func boolMetric(value bool) float64 { + if value { + return 1 + } + return 0 +} + func splitCSV(value string) []string { var out []string for _, item := range strings.Split(value, ",") { diff --git a/go/vehicle-gateway/cmd/gateway/main_test.go b/go/vehicle-gateway/cmd/gateway/main_test.go index d9cbf985..f8f31477 100644 --- a/go/vehicle-gateway/cmd/gateway/main_test.go +++ b/go/vehicle-gateway/cmd/gateway/main_test.go @@ -1,11 +1,19 @@ package main import ( + "context" + "errors" + "io" + "log/slog" "os" "strings" "testing" + "time" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) func TestNATSSinkConfigFromEnvUsesExplicitSubjects(t *testing.T) { @@ -13,7 +21,12 @@ func TestNATSSinkConfigFromEnvUsesExplicitSubjects(t *testing.T) { t.Setenv("NATS_SUBJECT_GB32960_RAW", "custom.raw.gb32960") t.Setenv("NATS_SUBJECT_JT808_RAW", "custom.raw.jt808") t.Setenv("NATS_SUBJECT_YUTONG_MQTT_RAW", "custom.raw.yutong") + t.Setenv("NATS_SUBJECT_GB32960_FIELDS", "custom.fields.gb32960") + t.Setenv("NATS_SUBJECT_JT808_FIELDS", "custom.fields.jt808") + t.Setenv("NATS_SUBJECT_YUTONG_MQTT_FIELDS", "custom.fields.yutong") t.Setenv("NATS_SUBJECT_UNIFIED", "custom.unified") + t.Setenv("NATS_OUTBOX_MAX_INFLIGHT", "12000") + t.Setenv("NATS_OUTBOX_ACK_TIMEOUT_MS", "4500") cfg := natsSinkConfigFromEnv() @@ -29,9 +42,131 @@ func TestNATSSinkConfigFromEnvUsesExplicitSubjects(t *testing.T) { if got, want := cfg.RawSubjects[envelope.ProtocolYutongMQTT], "custom.raw.yutong"; got != want { t.Fatalf("yutong subject = %q, want %q", got, want) } + if got, want := cfg.FieldsSubjects[envelope.ProtocolGB32960], "custom.fields.gb32960"; got != want { + t.Fatalf("gb32960 fields subject = %q, want %q", got, want) + } + if got, want := cfg.FieldsSubjects[envelope.ProtocolJT808], "custom.fields.jt808"; got != want { + t.Fatalf("jt808 fields subject = %q, want %q", got, want) + } + if got, want := cfg.FieldsSubjects[envelope.ProtocolYutongMQTT], "custom.fields.yutong"; got != want { + t.Fatalf("yutong fields subject = %q, want %q", got, want) + } if got, want := cfg.UnifiedSubject, "custom.unified"; got != want { t.Fatalf("unified subject = %q, want %q", got, want) } + if got, want := cfg.AsyncMaxPending, 12000; got != want { + t.Fatalf("async max pending = %d, want %d", got, want) + } + if got, want := cfg.AsyncAckTimeout, 4500*time.Millisecond; got != want { + t.Fatalf("async ack timeout = %s, want %s", got, want) + } +} + +func TestNATSOutboxConfigFromEnvIsExplicitAndDefaultsToSafeFsync(t *testing.T) { + t.Setenv("NATS_DURABLE_OUTBOX_ENABLED", "true") + t.Setenv("NATS_OUTBOX_DIR", "/var/lib/lingniu-go/nats-outbox") + t.Setenv("NATS_OUTBOX_REPLAY_BATCH_SIZE", "750") + t.Setenv("NATS_OUTBOX_REPLAY_INTERVAL_MS", "250") + t.Setenv("NATS_OUTBOX_CLOSE_TIMEOUT_MS", "7000") + t.Setenv("NATS_OUTBOX_WAL_SEGMENT_BYTES", "8388608") + t.Setenv("NATS_OUTBOX_WAL_SEGMENT_AGE_MS", "4000") + t.Setenv("NATS_OUTBOX_WAL_APPEND_QUEUE_SIZE", "50000") + t.Setenv("NATS_OUTBOX_WAL_COMMIT_BATCH_SIZE", "128") + t.Setenv("NATS_OUTBOX_WAL_COMMIT_INTERVAL_MS", "2") + t.Setenv("NATS_OUTBOX_FSYNC", "") + registry := metrics.NewRegistry() + onError := func(error) {} + + runtime := natsOutboxConfigFromEnv(registry, onError) + + if !runtime.Enabled { + t.Fatal("outbox should be enabled") + } + if got, want := runtime.Config.Directory, "/var/lib/lingniu-go/nats-outbox"; got != want { + t.Fatalf("directory = %q, want %q", got, want) + } + if got, want := runtime.Config.ReplayBatchSize, 750; got != want { + t.Fatalf("replay batch = %d, want %d", got, want) + } + if !runtime.Config.SyncWrites { + t.Fatal("outbox fsync should default to true") + } + if got, want := runtime.Config.CloseTimeout, 7*time.Second; got != want { + t.Fatalf("close timeout = %s, want %s", got, want) + } + if got, want := runtime.Config.WALSegmentBytes, int64(8<<20); got != want { + t.Fatalf("WAL segment bytes = %d, want %d", got, want) + } + if got, want := runtime.Config.WALSegmentAge, 4*time.Second; got != want { + t.Fatalf("WAL segment age = %s, want %s", got, want) + } + if got, want := runtime.Config.WALAppendQueue, 50_000; got != want { + t.Fatalf("WAL append queue = %d, want %d", got, want) + } + if got, want := runtime.Config.WALCommitBatch, 128; got != want { + t.Fatalf("WAL commit batch = %d, want %d", got, want) + } + if got, want := runtime.Config.WALCommitWait, 2*time.Millisecond; got != want { + t.Fatalf("WAL commit wait = %s, want %s", got, want) + } + if got, want := runtime.ReplayInterval, 250*time.Millisecond; got != want { + t.Fatalf("replay interval = %s, want %s", got, want) + } + if runtime.Config.Metrics != registry || runtime.Config.OnError == nil { + t.Fatal("metrics and error callback should be wired") + } +} + +func TestNATSOutboxConfigDoesNotReuseLegacySpoolDirectory(t *testing.T) { + t.Setenv("NATS_DURABLE_OUTBOX_ENABLED", "") + t.Setenv("NATS_OUTBOX_DIR", "") + t.Setenv("NATS_SPOOL_DIR", "/var/lib/lingniu-go/nats-spool") + t.Setenv("NATS_OUTBOX_FSYNC", "false") + + runtime := natsOutboxConfigFromEnv(nil, nil) + + if runtime.Enabled { + t.Fatal("outbox must remain opt-in") + } + if runtime.Config.Directory != "" { + t.Fatalf("WAL directory must be explicit, got %q", runtime.Config.Directory) + } + if runtime.Config.SyncWrites { + t.Fatal("explicit false should disable fsync") + } +} + +func TestBuildGB32960AuthenticatorUsesConfiguredCredentials(t *testing.T) { + t.Setenv("GB32960_AUTH_MODE", "enforce") + t.Setenv("GB32960_PLATFORM_CREDENTIALS_FILE", "") + t.Setenv("GB32960_PLATFORM_CREDENTIALS_JSON", `{"platform-a":"secret-a","platform-b":["secret-b-old","secret-b"]}`) + + authenticator, mode, count, err := buildGB32960Authenticator() + if err != nil { + t.Fatalf("buildGB32960Authenticator() error = %v", err) + } + if mode != authentication.ModeEnforce || count != 2 { + t.Fatalf("mode=%q count=%d", mode, count) + } + result := authenticator.Authenticate(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x05", + Parsed: map[string]any{ + "platform_login": map[string]any{"username": "platform-b", "password": "secret-b"}, + }, + }) + if !result.Allowed || result.Status != authentication.StatusAccepted { + t.Fatalf("authentication result = %#v", result) + } +} + +func TestBuildGB32960AuthenticatorRejectsEnforceWithoutCredentials(t *testing.T) { + t.Setenv("GB32960_AUTH_MODE", "enforce") + t.Setenv("GB32960_PLATFORM_CREDENTIALS_FILE", "") + t.Setenv("GB32960_PLATFORM_CREDENTIALS_JSON", "") + if _, _, _, err := buildGB32960Authenticator(); err == nil { + t.Fatal("expected enforce mode configuration error") + } } func TestGatewayConfiguresIdentityLookupCacheTTL(t *testing.T) { @@ -39,13 +174,158 @@ func TestGatewayConfiguresIdentityLookupCacheTTL(t *testing.T) { if err != nil { t.Fatalf("read main.go: %v", err) } - for _, want := range []string{"LookupCacheTTL", "IDENTITY_LOOKUP_CACHE_TTL_SECONDS"} { + for _, want := range []string{"LookupCacheTTL", "StaleLookupTTL", "IDENTITY_LOOKUP_CACHE_TTL_SECONDS", "IDENTITY_STALE_LOOKUP_TTL_SECONDS", "IDENTITY_LOOKUP_CACHE_MAX_ENTRIES", "IDENTITY_LOOKUP_CACHE_CLEANUP_INTERVAL_SECONDS", "vehicle_gateway_identity_cache_entries", "vehicle_gateway_jt808_registration_write_total", "OnRegistrationWriteResult", "IDENTITY_SOURCE_CODE_LOOKUP_ENABLED", "IDENTITY_RESOLVE_TIMEOUT_MS", "IDENTITY_SNAPSHOT_ONLY_ENABLED", "IDENTITY_SNAPSHOT_REFRESH_INTERVAL_SECONDS", "startIdentitySnapshotRefresh", "JT808_REGISTRATION_LOCATION_TOUCH_RETRY_INTERVAL_SECONDS", "JT808_REGISTRATION_WRITE_RETRY_ATTEMPTS", "JT808_REGISTRATION_WRITE_RETRY_DELAY_MS", "JT808_REGISTRATION_ASYNC_WRITE_ENABLED", "JT808_REGISTRATION_WRITE_QUEUE_SIZE", "JT808_REGISTRATION_WRITE_WORKERS", "JT808_REGISTRATION_WRITE_ENQUEUE_TIMEOUT_MS", "TimeoutResolver"} { if !strings.Contains(string(source), want) { t.Fatalf("gateway should expose identity lookup cache ttl, missing %s", want) } } } +func TestRecordIdentityCacheStatsIncludesLocationTouchFailures(t *testing.T) { + registry := metrics.NewRegistry() + recordIdentityCacheStats(registry, identity.CacheStats{ + LookupEntries: 1, + RegistrationEntries: 2, + SourceCodeEntries: 3, + LocationTouchEntries: 4, + LocationTouchFailureEntries: 5, + SnapshotBindingEntries: 6, + SnapshotIdentifierEntries: 7, + SnapshotRegistrationEntries: 8, + SnapshotSourceEntries: 9, + SnapshotReady: true, + SnapshotRefreshedAt: time.Unix(1234, 0), + MaxEntries: 9, + }) + + rendered := registry.Render() + for _, want := range []string{ + `vehicle_gateway_identity_cache_entries{cache="location_touch_failure"} 5`, + `vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 0`, + `vehicle_gateway_identity_cache_entries{cache="snapshot_binding"} 6`, + `vehicle_gateway_identity_cache_entries{cache="snapshot_identifier"} 7`, + `vehicle_gateway_identity_cache_entries{cache="snapshot_registration"} 8`, + `vehicle_gateway_identity_cache_entries{cache="snapshot_source"} 9`, + `vehicle_gateway_identity_cache_max_entries{cache="location_touch_failure"} 9`, + `vehicle_gateway_identity_snapshot_ready 1`, + `vehicle_gateway_identity_snapshot_last_success_unix_seconds 1234`, + } { + if !strings.Contains(rendered, want) { + t.Fatalf("identity cache metrics missing %s in:\n%s", want, rendered) + } + } +} + +func TestRecordIdentitySnapshotRefresh(t *testing.T) { + registry := metrics.NewRegistry() + recordIdentitySnapshotRefresh(registry, identity.SnapshotRefreshResult{ + BindingEntries: 10, + IdentifierEntries: 20, + RegistrationEntries: 30, + SourceEntries: 40, + RefreshedAt: time.Unix(5678, 0), + }, "ok") + + rendered := registry.Render() + for _, want := range []string{ + `vehicle_gateway_identity_snapshot_refresh_total{status="ok"} 1`, + `vehicle_gateway_identity_snapshot_entries{kind="binding"} 10`, + `vehicle_gateway_identity_snapshot_entries{kind="identifier"} 20`, + `vehicle_gateway_identity_snapshot_entries{kind="registration"} 30`, + `vehicle_gateway_identity_snapshot_entries{kind="source"} 40`, + `vehicle_gateway_identity_snapshot_last_success_unix_seconds 5678`, + } { + if !strings.Contains(rendered, want) { + t.Fatalf("identity snapshot metrics missing %s in:\n%s", want, rendered) + } + } +} + +func TestIdentityDatabaseMaintenanceDoesNotBlockIngressStartup(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db := &blockingIdentityDatabase{ + started: make(chan struct{}), + release: make(chan struct{}), + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + startedAt := time.Now() + startIdentityDatabaseMaintenance(ctx, logger, db, nil, false, time.Second, time.Second) + if elapsed := time.Since(startedAt); elapsed > 100*time.Millisecond { + t.Fatalf("database maintenance blocked gateway startup for %s", elapsed) + } + + select { + case <-db.started: + case <-time.After(time.Second): + t.Fatal("database maintenance did not start in background") + } + close(db.release) +} + +func TestIdentitySnapshotRefreshDoesNotBlockIngressStartup(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + refresher := &blockingIdentitySnapshotRefresher{started: make(chan struct{})} + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + startedAt := time.Now() + startIdentitySnapshotRefresh(ctx, logger, metrics.NewRegistry(), refresher, time.Hour, time.Second) + if elapsed := time.Since(startedAt); elapsed > 100*time.Millisecond { + t.Fatalf("snapshot refresh blocked gateway startup for %s", elapsed) + } + + select { + case <-refresher.started: + case <-time.After(time.Second): + t.Fatal("snapshot refresh did not start in background") + } +} + +func TestRecordJT808RegistrationWriteResult(t *testing.T) { + registry := metrics.NewRegistry() + recordJT808RegistrationWriteResult(registry, identity.RegistrationWriteResult{ + Mode: "async_background", + Status: "error", + }) + + rendered := registry.Render() + for _, want := range []string{ + `vehicle_gateway_jt808_registration_write_total{mode="async_background",status="error"} 1`, + `vehicle_gateway_last_jt808_registration_write_unix_seconds{mode="async_background",status="error"}`, + } { + if !strings.Contains(rendered, want) { + t.Fatalf("registration write metric missing %s in:\n%s", want, rendered) + } + } +} + +type blockingIdentityDatabase struct { + started chan struct{} + release chan struct{} +} + +func (d *blockingIdentityDatabase) PingContext(ctx context.Context) error { + close(d.started) + select { + case <-ctx.Done(): + return ctx.Err() + case <-d.release: + return nil + } +} + +type blockingIdentitySnapshotRefresher struct { + started chan struct{} +} + +func (r *blockingIdentitySnapshotRefresher) RefreshSnapshot(ctx context.Context) (identity.SnapshotRefreshResult, error) { + close(r.started) + <-ctx.Done() + return identity.SnapshotRefreshResult{}, errors.New("test refresh stopped") +} + func TestGatewayDefaultsTo100KConnectionCeiling(t *testing.T) { source, err := os.ReadFile("main.go") if err != nil { @@ -64,6 +344,18 @@ func TestGatewayPassesMetricsRegistryToAsyncSink(t *testing.T) { for _, want := range []string{ "buildSink(ctx, logger, registry)", "Metrics: registry", + "EnqueueTimeout: enqueueTimeout", + "NewPartitionedAsyncSink", + "NATS_PARTITIONED_ASYNC_ENABLED", + "KAFKA_PARTITIONED_ASYNC_ENABLED", + "_ASYNC_RAW_QUEUE_SIZE", + "_ASYNC_DERIVED_QUEUE_SIZE", + "_ASYNC_RAW_ENQUEUE_TIMEOUT_MS", + "_ASYNC_DERIVED_ENQUEUE_TIMEOUT_MS", + "RawEnqueueTimeout", + "DerivedEnqueueTimeout", + "NATS_ASYNC_ENQUEUE_TIMEOUT_MS", + "KAFKA_ASYNC_ENQUEUE_TIMEOUT_MS", `Name: "nats"`, `Name: "kafka"`, } { @@ -73,6 +365,43 @@ func TestGatewayPassesMetricsRegistryToAsyncSink(t *testing.T) { } } +func TestGatewayExposesNATSDurableSpoolConfig(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatalf("read main.go: %v", err) + } + for _, want := range []string{ + "NATS_DURABLE_OUTBOX_ENABLED", + "NATS_OUTBOX_DIR", + "NATS_OUTBOX_MAX_INFLIGHT", + "NATS_OUTBOX_ACK_TIMEOUT_MS", + "NATS_OUTBOX_FSYNC", + "NATS_OUTBOX_CLOSE_TIMEOUT_MS", + "NATS_OUTBOX_REPLAY_BATCH_SIZE", + "NATS_OUTBOX_REPLAY_INTERVAL_MS", + "NATS_OUTBOX_WAL_SEGMENT_BYTES", + "NATS_OUTBOX_WAL_SEGMENT_AGE_MS", + "NATS_OUTBOX_WAL_APPEND_QUEUE_SIZE", + "NATS_OUTBOX_WAL_COMMIT_BATCH_SIZE", + "NATS_OUTBOX_WAL_COMMIT_INTERVAL_MS", + "eventbus.NewDurableOutboxSink", + "nats durable outbox enabled", + "NATS_SPOOL_DIR", + "NATS_SPOOL_REPLAY_BATCH_SIZE", + "NATS_SPOOL_REPLAY_INTERVAL_MS", + "nats durable spool enabled", + "nats spool replay failed", + "eventbus.NewDurableSink", + "Metrics: registry", + `Name: "nats"`, + `Name: "kafka"`, + } { + if !strings.Contains(string(source), want) { + t.Fatalf("gateway nats spool wiring missing %s", want) + } + } +} + func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) { t.Setenv("NATS_URL", "nats://172.17.111.56:4222") @@ -87,6 +416,15 @@ func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) { if got, want := cfg.RawSubjects[envelope.ProtocolYutongMQTT], "vehicle.raw.go.yutong-mqtt.v1"; got != want { t.Fatalf("yutong subject = %q, want %q", got, want) } + if got, want := cfg.FieldsSubjects[envelope.ProtocolGB32960], "vehicle.fields.go.gb32960.v1"; got != want { + t.Fatalf("gb32960 fields subject = %q, want %q", got, want) + } + if got, want := cfg.FieldsSubjects[envelope.ProtocolJT808], "vehicle.fields.go.jt808.v1"; got != want { + t.Fatalf("jt808 fields subject = %q, want %q", got, want) + } + if got, want := cfg.FieldsSubjects[envelope.ProtocolYutongMQTT], "vehicle.fields.go.yutong-mqtt.v1"; got != want { + t.Fatalf("yutong fields subject = %q, want %q", got, want) + } if got, want := cfg.UnifiedSubject, "vehicle.event.go.unified.v1"; got != want { t.Fatalf("unified subject = %q, want %q", got, want) } diff --git a/go/vehicle-gateway/cmd/history-writer/main.go b/go/vehicle-gateway/cmd/history-writer/main.go index c77696a5..039bd2ba 100644 --- a/go/vehicle-gateway/cmd/history-writer/main.go +++ b/go/vehicle-gateway/cmd/history-writer/main.go @@ -4,10 +4,14 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "os" "os/signal" + "sort" "strconv" "strings" + "sync" "syscall" "time" @@ -15,6 +19,7 @@ import ( _ "github.com/taosdata/driver-go/v3/taosWS" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/history" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" @@ -28,6 +33,10 @@ func main() { defer stop() cfg := loadConfig() + if err := cfg.Validate(); err != nil { + logger.Error("invalid history writer config", "error", err) + os.Exit(1) + } db, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN) if err != nil { logger.Error("tdengine open failed", "error", err) @@ -39,18 +48,51 @@ func main() { os.Exit(1) } registry := metrics.NewRegistry() + metrics.RegisterKafkaConsumerInfo(registry, "vehicle-history-writer", cfg.KafkaGroup, cfg.KafkaTopics) + registry.SetGauge("vehicle_history_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers)) health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-history-writer", []health.Check{ {Name: "tdengine", Check: db.PingContext}, }, registry)) - writer := history.NewWriter(db) + writer := history.NewWriterWithDatabase(db, cfg.TDengineDatabase) if cfg.EnsureSchema { if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil { logger.Error("tdengine schema bootstrap failed", "error", err) os.Exit(1) } } + var appender historyAppender = retryHistoryAppender{ + delegate: writer, + attempts: cfg.RetryAttempts, + delay: cfg.RetryDelay, + registry: registry, + } + logger.Info("history writer started", + "driver", cfg.TDengineDriver, + "group", cfg.KafkaGroup, + "topics", strings.Join(cfg.KafkaTopics, ","), + "workers", cfg.Workers, + "batch_size", cfg.BatchSize, + "batch_wait_ms", cfg.BatchWait, + "retry_attempts", cfg.RetryAttempts, + "retry_delay_ms", cfg.RetryDelay.Milliseconds()) + + var workers sync.WaitGroup + for workerID := 1; workerID <= cfg.Workers; workerID++ { + workers.Add(1) + go func(id int) { + defer workers.Done() + runHistoryConsumer(ctx, logger, registry, appender, cfg, id) + }(workerID) + } + workers.Wait() +} + +func runHistoryConsumer(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender historyAppender, cfg config, workerID int) { reader := kafka.NewReader(kafka.ReaderConfig{ Brokers: cfg.KafkaBrokers, GroupID: cfg.KafkaGroup, @@ -60,12 +102,9 @@ func main() { }) defer reader.Close() - logger.Info("history writer started", - "driver", cfg.TDengineDriver, - "group", cfg.KafkaGroup, - "topics", strings.Join(cfg.KafkaTopics, ","), - "batch_size", cfg.BatchSize, - "batch_wait_ms", cfg.BatchWait) + workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)} + registry.SetGauge("vehicle_history_worker_active", workerLabels, 1) + defer registry.SetGauge("vehicle_history_worker_active", workerLabels, 0) for { message, err := reader.FetchMessage(ctx) @@ -77,7 +116,7 @@ func main() { continue } batch := collectHistoryBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond) - processHistoryBatch(ctx, logger, registry, writer, reader, batch) + processHistoryBatchReliablyForWorker(ctx, logger, registry, appender, reader, batch, cfg.RetryDelay, workerLabels) } } @@ -88,6 +127,11 @@ type historyAppender interface { AppendAllBatch(context.Context, []envelope.FrameEnvelope) error } +type historyResultAppender interface { + AppendAllWithResult(context.Context, envelope.FrameEnvelope) (history.AppendResult, error) + AppendAllBatchWithResult(context.Context, []envelope.FrameEnvelope) (history.AppendResult, error) +} + type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } @@ -96,6 +140,17 @@ type kafkaMessageFetcher interface { FetchMessage(context.Context) (kafka.Message, error) } +type historyBatchItem struct { + message kafka.Message + valid bool + processed bool +} + +type historyBatchOutcome struct { + commitMessages []kafka.Message + retryMessages []kafka.Message +} + func collectHistoryBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { if maxSize <= 1 { return []kafka.Message{first} @@ -134,15 +189,38 @@ func processHistoryMessage(ctx context.Context, logger interface { if err := json.Unmarshal(message.Value, &env); err != nil { addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, "invalid_json") logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) - _ = committer.CommitMessages(messageCtx, message) + if err := committer.CommitMessages(messageCtx, message); err != nil { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error") + logger.Error("kafka commit invalid envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + return + } + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok") return } - if err := appender.AppendAll(messageCtx, env); err != nil { - addWriterMetric(registry, "vehicle_history_writes_total", message, "error") - logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) + if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { + addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, status) + logger.Warn("skip mismatched history raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) + if err := committer.CommitMessages(messageCtx, message); err != nil { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error") + logger.Error("kafka commit mismatched raw envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err) + return + } + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok") return } + recordHistoryParsedFieldMetrics(registry, message, env) + result, err := appendHistoryEnvelope(messageCtx, appender, env) + if err != nil { + addWriterMetric(registry, "vehicle_history_writes_total", message, "error") + logger.Error("tdengine raw append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) + return + } + recordHistoryDerivedMetrics(registry, []kafka.Message{message}, []envelope.FrameEnvelope{env}, result) + if result.LocationError != nil { + logger.Warn("tdengine location append failed after raw append", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", result.LocationError) + } addWriterMetric(registry, "vehicle_history_writes_total", message, "ok") + recordHistoryWriteE2EDuration(registry, message, env) if err := committer.CommitMessages(messageCtx, message); err != nil { addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error") logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) @@ -154,30 +232,52 @@ func processHistoryMessage(ctx context.Context, logger interface { func processHistoryBatch(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) -}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message) { +}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message) historyBatchOutcome { + return processHistoryBatchForWorker(ctx, logger, registry, appender, committer, messages, nil) +} + +func processHistoryBatchForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) historyBatchOutcome { if len(messages) == 0 { - return + return historyBatchOutcome{} } messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) defer cancel() envelopes := make([]envelope.FrameEnvelope, 0, len(messages)) + validMessages := make([]kafka.Message, 0, len(messages)) + validItemIndexes := make([]int, 0, len(messages)) + items := make([]historyBatchItem, 0, len(messages)) for _, message := range messages { + itemIndex := len(items) addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, "received") addWriterLagMetric(registry, message) var env envelope.FrameEnvelope if err := json.Unmarshal(message.Value, &env); err != nil { addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, "invalid_json") logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + items = append(items, historyBatchItem{message: message, processed: true}) continue } + if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { + addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, status) + logger.Warn("skip mismatched history raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) + items = append(items, historyBatchItem{message: message, processed: true}) + continue + } + recordHistoryParsedFieldMetrics(registry, message, env) envelopes = append(envelopes, env) + validMessages = append(validMessages, message) + validItemIndexes = append(validItemIndexes, itemIndex) + items = append(items, historyBatchItem{message: message, valid: true}) } if len(envelopes) > 0 { - setBatchPending(registry, len(messages), len(envelopes)) - defer setBatchPending(registry, 0, 0) + setBatchPendingForWorker(registry, workerLabels, len(messages), len(envelopes)) + defer setBatchPendingForWorker(registry, workerLabels, 0, 0) started := time.Now() - err := appender.AppendAllBatch(messageCtx, envelopes) + result, err := appendHistoryBatch(messageCtx, appender, envelopes) elapsed := time.Since(started) status := "ok" if err != nil { @@ -187,15 +287,33 @@ func processHistoryBatch(ctx context.Context, logger interface { addBatchMetric(registry, "vehicle_history_batch_rows_total", status, float64(len(envelopes))) setBatchDuration(registry, status, elapsed) if err != nil { - for _, message := range messages { - addWriterMetric(registry, "vehicle_history_writes_total", message, "error") - } first := messages[0] - logger.Error("tdengine batch append failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "rows", len(envelopes), "error", err) - return + logger.Error("tdengine raw batch append failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "rows", len(envelopes), "error", err) + if isTransientTDengineHistoryError(err) { + for _, message := range validMessages { + addWriterMetric(registry, "vehicle_history_writes_total", message, "error") + } + addBatchMetric(registry, "vehicle_history_batch_fallback_total", "skipped_transient", 1) + committed, commitErr := commitHistoryProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) + return historyFailureOutcome(messages, committed, commitErr) + } + if fallbackErr := fallbackHistoryBatchToSingles(messageCtx, logger, registry, appender, validMessages, envelopes, validItemIndexes, items); fallbackErr != nil { + addBatchMetric(registry, "vehicle_history_batch_fallback_total", "error", 1) + committed, commitErr := commitHistoryProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) + return historyFailureOutcome(messages, committed, commitErr) + } + addBatchMetric(registry, "vehicle_history_batch_fallback_total", "ok", 1) + committed, commitErr := commitHistoryProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) + return historyFailureOutcome(messages, committed, commitErr) } - for _, message := range messages { + recordHistoryDerivedMetrics(registry, validMessages, envelopes, result) + if result.LocationError != nil { + first := messages[0] + logger.Warn("tdengine location batch append failed after raw append", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "rows", len(envelopes), "location_rows", result.LocationRows, "error", result.LocationError) + } + for index, message := range validMessages { addWriterMetric(registry, "vehicle_history_writes_total", message, "ok") + recordHistoryWriteE2EDuration(registry, message, envelopes[index]) } } if err := committer.CommitMessages(messageCtx, messages...); err != nil { @@ -204,18 +322,337 @@ func processHistoryBatch(ctx context.Context, logger interface { } first := messages[0] logger.Error("kafka batch commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) - return + return historyBatchOutcome{commitMessages: messages} } for _, message := range messages { addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok") } + return historyBatchOutcome{} +} + +func historyFailureOutcome(messages []kafka.Message, committed []kafka.Message, commitErr error) historyBatchOutcome { + outcome := historyBatchOutcome{ + retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed), + } + if commitErr != nil { + outcome.commitMessages = committed + } + return outcome +} + +func processHistoryBatchReliably(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) { + processHistoryBatchReliablyForWorker(ctx, logger, registry, appender, committer, messages, retryDelay, nil) +} + +func processHistoryBatchReliablyForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) { + defer registry.SetGauge("vehicle_history_retry_pending_messages", workerLabels, 0) + pending := messages + for len(pending) > 0 { + outcome := processHistoryBatchForWorker(ctx, logger, registry, appender, committer, pending, workerLabels) + if len(outcome.commitMessages) > 0 { + registry.IncCounter("vehicle_history_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !retryHistoryCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) { + return + } + } + pending = outcome.retryMessages + registry.SetGauge("vehicle_history_retry_pending_messages", workerLabels, float64(len(pending))) + if len(pending) == 0 { + return + } + registry.IncCounter("vehicle_history_batch_retries_total", metrics.Labels{"reason": "write_error"}) + if !waitForHistoryRetry(ctx, retryDelay) { + return + } + } +} + +func retryHistoryCommit(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool { + for len(messages) > 0 { + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) + err := committer.CommitMessages(operationCtx, messages...) + cancel() + if err == nil { + for _, message := range messages { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok") + } + return true + } + for _, message := range messages { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error") + } + first := messages[0] + logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) + registry.IncCounter("vehicle_history_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !waitForHistoryRetry(ctx, retryDelay) { + return false + } + } + return true +} + +func waitForHistoryRetry(ctx context.Context, retryDelay time.Duration) bool { + if retryDelay <= 0 { + retryDelay = 100 * time.Millisecond + } + timer := time.NewTimer(retryDelay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func fallbackHistoryBatchToSingles(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender historyAppender, messages []kafka.Message, envelopes []envelope.FrameEnvelope, itemIndexes []int, items []historyBatchItem) error { + for index, env := range envelopes { + message := messages[index] + started := time.Now() + result, err := appendHistoryEnvelope(ctx, appender, env) + setFallbackDuration(registry, statusFromError(err), time.Since(started)) + if err != nil { + addWriterMetric(registry, "vehicle_history_writes_total", message, "error") + logger.Error("tdengine raw single append failed after batch fallback", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) + return err + } + recordHistoryDerivedMetrics(registry, []kafka.Message{message}, []envelope.FrameEnvelope{env}, result) + if result.LocationError != nil { + logger.Warn("tdengine location append failed after raw single fallback", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", result.LocationError) + } + addWriterMetric(registry, "vehicle_history_writes_total", message, "ok") + recordHistoryWriteE2EDuration(registry, message, env) + items[itemIndexes[index]].processed = true + } + return nil +} + +func commitHistoryProcessedPrefixAfterFailure(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, items []historyBatchItem) ([]kafka.Message, error) { + committable := processedHistoryPrefixMessagesByPartition(items) + if len(committable) == 0 { + return nil, nil + } + if err := committer.CommitMessages(ctx, committable...); err != nil { + for _, message := range committable { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error") + } + first := committable[0] + logger.Error("kafka processed-prefix commit failed after raw batch append failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err) + return committable, err + } + for _, message := range committable { + addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok") + } + return committable, nil +} + +func processedHistoryPrefixMessagesByPartition(items []historyBatchItem) []kafka.Message { + type partitionKey struct { + topic string + partition int + } + groups := map[partitionKey][]historyBatchItem{} + for _, item := range items { + key := partitionKey{topic: item.message.Topic, partition: item.message.Partition} + groups[key] = append(groups[key], item) + } + var keys []partitionKey + for key := range groups { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].topic != keys[j].topic { + return keys[i].topic < keys[j].topic + } + return keys[i].partition < keys[j].partition + }) + var out []kafka.Message + for _, key := range keys { + group := groups[key] + sort.Slice(group, func(i, j int) bool { + return group[i].message.Offset < group[j].message.Offset + }) + var previousOffset int64 + for index, item := range group { + if index > 0 && item.message.Offset != previousOffset+1 { + break + } + if !item.processed { + break + } + out = append(out, item.message) + previousOffset = item.message.Offset + } + } + return out +} + +func appendHistoryEnvelope(ctx context.Context, appender historyAppender, env envelope.FrameEnvelope) (history.AppendResult, error) { + if resultAppender, ok := appender.(historyResultAppender); ok { + return resultAppender.AppendAllWithResult(ctx, env) + } + if err := appender.AppendAll(ctx, env); err != nil { + return history.AppendResult{}, err + } + return history.AppendResult{RawRows: 1}, nil +} + +func appendHistoryBatch(ctx context.Context, appender historyAppender, envelopes []envelope.FrameEnvelope) (history.AppendResult, error) { + if resultAppender, ok := appender.(historyResultAppender); ok { + return resultAppender.AppendAllBatchWithResult(ctx, envelopes) + } + if err := appender.AppendAllBatch(ctx, envelopes); err != nil { + return history.AppendResult{}, err + } + return history.AppendResult{RawRows: len(envelopes)}, nil +} + +type retryHistoryAppender struct { + delegate historyAppender + attempts int + delay time.Duration + registry *metrics.Registry +} + +func (a retryHistoryAppender) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error { + _, err := a.AppendAllWithResult(ctx, env) + return err +} + +func (a retryHistoryAppender) AppendAllWithResult(ctx context.Context, env envelope.FrameEnvelope) (history.AppendResult, error) { + if a.delegate == nil { + return history.AppendResult{}, nil + } + attempts := a.attempts + if attempts <= 0 { + attempts = 1 + } + var result history.AppendResult + var err error + for attempt := 1; attempt <= attempts; attempt++ { + result, err = appendHistoryEnvelope(ctx, a.delegate, env) + if err == nil || !isTransientTDengineHistoryError(err) { + return result, err + } + if attempt == attempts { + a.recordRetry("single", "exhausted") + return result, err + } + a.recordRetry("single", "retry") + if waitErr := sleepBeforeRetry(ctx, a.delay); waitErr != nil { + return result, waitErr + } + } + return result, err +} + +func (a retryHistoryAppender) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error { + _, err := a.AppendAllBatchWithResult(ctx, envelopes) + return err +} + +func (a retryHistoryAppender) AppendAllBatchWithResult(ctx context.Context, envelopes []envelope.FrameEnvelope) (history.AppendResult, error) { + if a.delegate == nil { + return history.AppendResult{}, nil + } + attempts := a.attempts + if attempts <= 0 { + attempts = 1 + } + var result history.AppendResult + var err error + for attempt := 1; attempt <= attempts; attempt++ { + result, err = appendHistoryBatch(ctx, a.delegate, envelopes) + if err == nil || !isTransientTDengineHistoryError(err) { + return result, err + } + if attempt == attempts { + a.recordRetry("batch", "exhausted") + return result, err + } + a.recordRetry("batch", "retry") + if waitErr := sleepBeforeRetry(ctx, a.delay); waitErr != nil { + return result, waitErr + } + } + return result, err +} + +func (a retryHistoryAppender) recordRetry(operation string, status string) { + if a.registry == nil { + return + } + labels := metrics.Labels{"operation": operation, "status": status} + a.registry.IncCounter("vehicle_history_write_retries_total", labels) + metrics.RecordLastActivity(a.registry, "vehicle_history_last_write_retry_unix_seconds", labels) +} + +func sleepBeforeRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func isTransientTDengineHistoryError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "timeout") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "connection refused") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "bad connection") || + strings.Contains(text, "i/o timeout") || + text == "eof" || + strings.Contains(text, "unexpected eof") || + strings.Contains(text, "server is down") || + strings.Contains(text, "network is unreachable") || + strings.Contains(text, "no route to host") } func addWriterMetric(registry *metrics.Registry, name string, message kafka.Message, status string) { if registry == nil { return } - registry.IncCounter(name, metrics.Labels{"topic": message.Topic, "status": status}) + labels := metrics.Labels{"topic": message.Topic, "status": status} + registry.IncCounter(name, labels) + switch name { + case "vehicle_history_kafka_messages_total": + metrics.RecordLastActivity(registry, "vehicle_history_last_message_unix_seconds", labels) + case "vehicle_history_writes_total": + metrics.RecordLastActivity(registry, "vehicle_history_last_write_unix_seconds", labels) + case "vehicle_history_kafka_commits_total": + metrics.RecordLastActivity(registry, "vehicle_history_last_commit_unix_seconds", labels) + case "vehicle_history_location_writes_total": + metrics.RecordLastActivity(registry, "vehicle_history_last_location_write_unix_seconds", labels) + } } func addBatchMetric(registry *metrics.Registry, name string, status string, value float64) { @@ -225,19 +662,113 @@ func addBatchMetric(registry *metrics.Registry, name string, status string, valu registry.AddCounter(name, metrics.Labels{"status": status}, value) } +func recordHistoryParsedFieldMetrics(registry *metrics.Registry, message kafka.Message, env envelope.FrameEnvelope) { + if registry == nil || !envelope.IsRealtimeTelemetryFrame(env) { + return + } + status := "present" + if len(env.ParsedFields) == 0 { + status = "missing" + } + registry.IncCounter("vehicle_history_parsed_fields_total", metrics.Labels{ + "topic": message.Topic, + "protocol": historyProtocolLabel(env.Protocol), + "status": status, + }) +} + +func recordHistoryDerivedMetrics(registry *metrics.Registry, messages []kafka.Message, envelopes []envelope.FrameEnvelope, result history.AppendResult) { + if registry == nil || len(messages) == 0 { + return + } + batchStatus := "skipped" + if result.LocationError != nil { + batchStatus = "error" + } else if result.LocationRows > 0 { + batchStatus = "ok" + } + for index, message := range messages { + status := batchStatus + if index < len(envelopes) { + status = history.LocationStatus(envelopes[index]) + if status == history.LocationStatusOK && result.LocationError != nil { + status = "error" + } + } + addWriterMetric(registry, "vehicle_history_location_writes_total", message, status) + } + addBatchMetric(registry, "vehicle_history_location_batch_flush_total", batchStatus, 1) + addBatchMetric(registry, "vehicle_history_location_rows_total", batchStatus, float64(result.LocationRows)) +} + +func historyProtocolLabel(protocol envelope.Protocol) string { + protocolLabel := strings.TrimSpace(string(protocol)) + if protocolLabel == "" { + return "UNKNOWN" + } + return protocolLabel +} + func setBatchDuration(registry *metrics.Registry, status string, elapsed time.Duration) { if registry == nil { return } - registry.SetGauge("vehicle_history_batch_flush_duration_ms", metrics.Labels{"status": status}, float64(elapsed.Milliseconds())) + elapsedMS := float64(elapsed.Milliseconds()) + labels := metrics.Labels{"status": status} + registry.SetGauge("vehicle_history_batch_flush_duration_ms", labels, elapsedMS) + registry.ObserveHistogram("vehicle_history_batch_flush_duration_ms_histogram", labels, historyBatchFlushDurationBucketsMS, elapsedMS) } -func setBatchPending(registry *metrics.Registry, messages int, rows int) { +var historyBatchFlushDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + +func setFallbackDuration(registry *metrics.Registry, status string, elapsed time.Duration) { if registry == nil { return } - registry.SetGauge("vehicle_history_batch_pending_messages", nil, float64(messages)) - registry.SetGauge("vehicle_history_batch_pending_rows", nil, float64(rows)) + elapsedMS := float64(elapsed.Milliseconds()) + labels := metrics.Labels{"status": status} + registry.SetGauge("vehicle_history_batch_fallback_duration_ms", labels, elapsedMS) + registry.ObserveHistogram("vehicle_history_batch_fallback_duration_ms_histogram", labels, historyBatchFallbackDurationBucketsMS, elapsedMS) +} + +var historyBatchFallbackDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + +var historyWriteE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000} +var historyWriteE2ERecent = metrics.NewRecentLatencyByKey(512) + +func recordHistoryWriteE2EDuration(registry *metrics.Registry, message kafka.Message, env envelope.FrameEnvelope) { + if registry == nil || env.ReceivedAtMS <= 0 { + return + } + elapsed := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds() + if elapsed < 0 { + elapsed = 0 + } + labels := metrics.Labels{"topic": message.Topic} + registry.ObserveHistogram("vehicle_history_write_e2e_duration_ms_histogram", labels, historyWriteE2EDurationBucketsMS, float64(elapsed)) + p99, samples := historyWriteE2ERecent.Observe(message.Topic, float64(elapsed)) + registry.SetGauge("vehicle_history_write_e2e_recent_p99_ms", labels, p99) + registry.SetGauge("vehicle_history_write_e2e_recent_samples", labels, float64(samples)) + metrics.RecordLastActivity(registry, "vehicle_history_last_write_e2e_unix_seconds", labels) +} + +func statusFromError(err error) string { + if err != nil { + return "error" + } + return "ok" +} + +func setBatchPending(registry *metrics.Registry, messages int, rows int) { + setBatchPendingForWorker(registry, nil, messages, rows) +} + +func setBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int, rows int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_history_batch_pending_messages", workerLabels, float64(messages)) + registry.SetGauge("vehicle_history_batch_pending_rows", workerLabels, float64(rows)) } func addWriterLagMetric(registry *metrics.Registry, message kafka.Message) { @@ -255,8 +786,26 @@ type config struct { TDengineDSN string TDengineDatabase string EnsureSchema bool + Workers int BatchSize int BatchWait int + RetryAttempts int + RetryDelay time.Duration +} + +func (c config) Validate() error { + if len(c.KafkaTopics) == 0 { + return errors.New("KAFKA_TOPICS must include raw topics") + } + for _, topic := range c.KafkaTopics { + if _, ok := topics.ProtocolForKnownRawTopic(topic); !ok { + return fmt.Errorf("history-writer consumes known raw topics only, got %q", topic) + } + } + if c.Workers <= 0 { + return errors.New("HISTORY_WORKERS must be greater than zero") + } + return nil } func loadConfig() config { @@ -268,8 +817,11 @@ func loadConfig() config { TDengineDSN: env("TDENGINE_DSN", ""), TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase), EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false", + Workers: envInt("HISTORY_WORKERS", 3), BatchSize: envInt("HISTORY_BATCH_SIZE", 200), - BatchWait: envInt("HISTORY_BATCH_WAIT_MS", 100), + BatchWait: envInt("HISTORY_BATCH_WAIT_MS", 20), + RetryAttempts: envInt("HISTORY_RETRY_ATTEMPTS", 3), + RetryDelay: time.Duration(envInt("HISTORY_RETRY_DELAY_MS", 20)) * time.Millisecond, } } diff --git a/go/vehicle-gateway/cmd/history-writer/main_test.go b/go/vehicle-gateway/cmd/history-writer/main_test.go index ab80de64..2c7d4616 100644 --- a/go/vehicle-gateway/cmd/history-writer/main_test.go +++ b/go/vehicle-gateway/cmd/history-writer/main_test.go @@ -6,10 +6,12 @@ import ( "errors" "strings" "testing" + "time" "github.com/segmentio/kafka-go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/history" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) @@ -46,7 +48,16 @@ func TestProcessHistoryMessageUsesUncancelledContextForAppendAndCommit(t *testin } func TestProcessHistoryMessageRecordsMetrics(t *testing.T) { - env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x01"} + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "VIN001", + MessageID: "0x02", + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "gb32960.vehicle.speed_kmh": "42", + }, + } payload, err := json.Marshal(env) if err != nil { t.Fatal(err) @@ -65,9 +76,17 @@ func TestProcessHistoryMessageRecordsMetrics(t *testing.T) { text := registry.Render() for _, want := range []string{ `vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.gb32960.v1"} 1`, + `vehicle_history_last_message_unix_seconds{status="received",topic="vehicle.raw.gb32960.v1"} `, `vehicle_history_writes_total{status="ok",topic="vehicle.raw.gb32960.v1"} 1`, + `vehicle_history_last_write_unix_seconds{status="ok",topic="vehicle.raw.gb32960.v1"} `, + `vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.gb32960.v1"} 1`, + `vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.gb32960.v1"} `, + `vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.gb32960.v1"} `, + `vehicle_history_last_write_e2e_unix_seconds{topic="vehicle.raw.gb32960.v1"} `, `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.gb32960.v1"} 1`, + `vehicle_history_last_commit_unix_seconds{status="ok",topic="vehicle.raw.gb32960.v1"} `, `vehicle_history_kafka_lag{partition="1",topic="vehicle.raw.gb32960.v1"} 2`, + `vehicle_history_parsed_fields_total{protocol="GB32960",status="present",topic="vehicle.raw.gb32960.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("metrics missing %s:\n%s", want, text) @@ -75,9 +94,127 @@ func TestProcessHistoryMessageRecordsMetrics(t *testing.T) { } } +func TestProcessHistoryMessageSkipsExplicitNonRawEventKind(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + MessageID: "0x0200", + VIN: "VIN001", + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &contextCheckingHistoryAppender{} + committer := &contextCheckingHistoryCommitter{} + + processHistoryMessage( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 20, HighWaterMark: 21, Value: payload}, + ) + + if appender.count != 0 || appender.batchCount != 0 { + t.Fatalf("appender count=%d batch=%d, want no TDengine writes", appender.count, appender.batchCount) + } + if committer.count != 1 || committer.messageCount != 1 { + t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("mismatch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryMessageRecordsMissingParsedFieldsForRealtimeFrame(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "13307795425", + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}}, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + + processHistoryMessage( + context.Background(), + discardHistoryLogger{}, + registry, + &contextCheckingHistoryAppender{}, + &contextCheckingHistoryCommitter{}, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Value: payload}, + ) + + text := registry.Render() + if !strings.Contains(text, `vehicle_history_parsed_fields_total{protocol="JT808",status="missing",topic="vehicle.raw.go.jt808.v1"} 1`) { + t.Fatalf("missing parsed field metric not recorded:\n%s", text) + } +} + +func TestRecordHistoryWriteE2EDurationSkipsMissingReceivedAt(t *testing.T) { + registry := metrics.NewRegistry() + + recordHistoryWriteE2EDuration(registry, kafka.Message{Topic: "vehicle.raw.go.jt808.v1"}, envelope.FrameEnvelope{}) + + text := registry.Render() + if strings.Contains(text, "vehicle_history_write_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_history_write_e2e_recent") { + t.Fatalf("unexpected e2e metric without received_at:\n%s", text) + } +} + +func TestProcessHistoryMessageSkipsParsedFieldMetricForNonRealtimeFrame(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0102", + Phone: "13307795425", + ParseStatus: envelope.ParseOK, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + + processHistoryMessage( + context.Background(), + discardHistoryLogger{}, + registry, + &contextCheckingHistoryAppender{}, + &contextCheckingHistoryCommitter{}, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Value: payload}, + ) + + text := registry.Render() + if strings.Contains(text, "vehicle_history_parsed_fields_total") { + t.Fatalf("non-realtime frame should not record parsed field quality metric:\n%s", text) + } +} + func TestProcessHistoryBatchAppendsAllBeforeCommit(t *testing.T) { - first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} - second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + first := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "VIN001", + MessageID: "0x02", + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "gb32960.vehicle.speed_kmh": "42", + }, + } + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), ParseStatus: envelope.ParseOK, Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}}} firstPayload, err := json.Marshal(first) if err != nil { t.Fatal(err) @@ -118,6 +255,15 @@ func TestProcessHistoryBatchAppendsAllBeforeCommit(t *testing.T) { for _, want := range []string{ `vehicle_history_batch_rows_total{status="ok"} 2`, `vehicle_history_batch_flush_total{status="ok"} 1`, + `vehicle_history_batch_flush_duration_ms_histogram_bucket{le="+Inf",status="ok"} 1`, + `vehicle_history_batch_flush_duration_ms_histogram_count{status="ok"} 1`, + `vehicle_history_batch_flush_duration_ms_histogram_sum{status="ok"}`, + `vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.gb32960.v1"} `, + `vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} `, + `vehicle_history_parsed_fields_total{protocol="GB32960",status="present",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_parsed_fields_total{protocol="JT808",status="missing",topic="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("batch metric missing %s:\n%s", want, text) @@ -125,6 +271,124 @@ func TestProcessHistoryBatchAppendsAllBeforeCommit(t *testing.T) { } } +func TestProcessHistoryBatchRecordsDerivedLocationStatusesPerEnvelope(t *testing.T) { + nonRealtime := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + MessageID: "0x0100", + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "jt808.location.longitude": 121.1, + "jt808.location.latitude": 30.2, + }, + } + realtimeLocation := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN002", + MessageID: "0x0200", + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "jt808.location.longitude": 121.2, + "jt808.location.latitude": 30.3, + }, + } + nonRealtimePayload, err := json.Marshal(nonRealtime) + if err != nil { + t.Fatal(err) + } + realtimePayload, err := json.Marshal(realtimeLocation) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{ + result: history.AppendResult{RawRows: 2, LocationRows: 1}, + } + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 12, Value: nonRealtimePayload}, + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 11, HighWaterMark: 12, Value: realtimePayload}, + }, + ) + + text := registry.Render() + for _, want := range []string{ + `vehicle_history_location_writes_total{status="skipped_non_realtime",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_location_rows_total{status="ok"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("derived location status metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchSkipsExplicitNonRawEventKindAndWritesValidRows(t *testing.T) { + mismatched := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + MessageID: "0x0200", + VIN: "VIN_BAD", + } + valid := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindRaw, + MessageID: "0x0200", + VIN: "VIN001", + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + } + mismatchedPayload, err := mismatched.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + validPayload, err := valid.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{} + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 12, Value: mismatchedPayload}, + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 11, HighWaterMark: 12, Value: validPayload}, + }, + ) + + if appender.batchCount != 1 || len(appender.batch) != 1 { + t.Fatalf("batch appends=%d rows=%d, want only valid row", appender.batchCount, len(appender.batch)) + } + if committer.count != 1 || committer.messageCount != 2 { + t.Fatalf("commit calls=%d messages=%d, want 1/2", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch mismatch metric missing %s:\n%s", want, text) + } + } +} + func TestProcessHistoryBatchExposesPendingBatchDepthDuringAppend(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} @@ -174,6 +438,358 @@ func TestProcessHistoryBatchExposesPendingBatchDepthDuringAppend(t *testing.T) { } } +func TestProcessHistoryBatchKeepsPendingGaugePerWorker(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + + processHistoryBatchForWorker( + context.Background(), + discardHistoryLogger{}, + registry, + &contextCheckingHistoryAppender{}, + &contextCheckingHistoryCommitter{}, + []kafka.Message{{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}}, + metrics.Labels{"worker": "2"}, + ) + + text := registry.Render() + for _, want := range []string{ + `vehicle_history_batch_pending_messages{worker="2"} 0`, + `vehicle_history_batch_pending_rows{worker="2"} 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("worker pending gauge missing or not reset, want %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchSkipsInvalidJSONWithoutCountingItAsWriteOK(t *testing.T) { + valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ParseStatus: envelope.ParseOK} + validPayload, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{} + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 13, Value: []byte(`{bad json`)}, + {Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 11, HighWaterMark: 13, Value: validPayload}, + }, + ) + + if appender.batchCount != 1 { + t.Fatalf("batch appends = %d, want 1", appender.batchCount) + } + if got := len(appender.batch); got != 1 { + t.Fatalf("batch size = %d, want only valid envelope", got) + } + if committer.messageCount != 2 { + t.Fatalf("committed messages = %d, want valid and invalid committed", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 2`, + `vehicle_history_batch_rows_total{status="ok"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("invalid json batch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchDoesNotCountInvalidJSONAsWriteError(t *testing.T) { + valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + payload, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{err: errTestHistoryAppend} + registry := metrics.NewRegistry() + committer := &contextCheckingHistoryCommitter{} + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 12, Value: []byte(`{bad json`)}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, HighWaterMark: 12, Value: payload}, + }, + ) + + if committer.messageCount != 1 { + t.Fatalf("committed messages = %d, want invalid prefix only", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_writes_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_batch_rows_total{status="error"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("invalid json write error metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchDoesNotCommitInvalidJSONAfterFailedValidOffset(t *testing.T) { + valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + payload, err := json.Marshal(valid) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{err: errTestHistoryAppend} + committer := &contextCheckingHistoryCommitter{} + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + metrics.NewRegistry(), + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 12, Value: payload}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, HighWaterMark: 12, Value: []byte(`{bad json`)}, + }, + ) + + if committer.count != 0 { + t.Fatalf("commit calls = %d, want 0 because valid offset failed before invalid json", committer.count) + } +} + +func TestProcessedHistoryPrefixMessagesByPartitionStopsAtOffsetGap(t *testing.T) { + items := []historyBatchItem{ + {message: kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10}, processed: true}, + {message: kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 12}, processed: true}, + } + + got := processedHistoryPrefixMessagesByPartition(items) + if len(got) != 1 || got[0].Offset != 10 { + t.Fatalf("processed prefix = %#v, want only offset 10", got) + } +} + +func TestProcessHistoryBatchFallsBackToSinglesAfterNonTransientBatchFailure(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + firstPayload, err := json.Marshal(first) + if err != nil { + t.Fatal(err) + } + secondPayload, err := json.Marshal(second) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{ + batchErr: errors.New("syntax error in batch insert"), + result: history.AppendResult{RawRows: 1, LocationRows: 1}, + } + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, Value: []byte(`{bad json`)}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, Value: firstPayload}, + {Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 20, Value: secondPayload}, + }, + ) + + if appender.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1", appender.batchCount) + } + if appender.count != 2 { + t.Fatalf("single fallback attempts = %d, want 2", appender.count) + } + if committer.messageCount != 3 { + t.Fatalf("committed messages = %d, want invalid and fallback-success messages", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_batch_flush_total{status="error"} 1`, + `vehicle_history_batch_fallback_total{status="ok"} 1`, + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 2`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_history_batch_fallback_duration_ms_histogram_count{status="ok"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("fallback metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchFallbackCommitsOnlyProcessedPrefixWhenSingleFails(t *testing.T) { + success := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + failed := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", MessageID: "0x02"} + successPayload, err := json.Marshal(success) + if err != nil { + t.Fatal(err) + } + failedPayload, err := json.Marshal(failed) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{ + batchErr: errors.New("syntax error in batch insert"), + singleErr: errTestHistoryAppend, + failSingleOnCount: 2, + } + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + outcome := processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, Value: []byte(`{bad json`)}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, Value: successPayload}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 12, Value: failedPayload}, + }, + ) + + if appender.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1", appender.batchCount) + } + if appender.count != 2 { + t.Fatalf("single fallback attempts = %d, want stop after failed second single", appender.count) + } + if committer.messageCount != 2 { + t.Fatalf("committed messages = %d, want invalid and first successful single only", committer.messageCount) + } + if got := committedOffsets(outcome.retryMessages); len(got) != 1 || got[0] != 12 { + t.Fatalf("retry offsets = %#v, want [12]", got) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_batch_fallback_total{status="error"} 1`, + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_writes_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("fallback failure metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessHistoryBatchReliablyRetriesFailedSuffix(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", MessageID: "0x02"} + third := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN003", MessageID: "0x02"} + firstPayload, err := json.Marshal(first) + if err != nil { + t.Fatal(err) + } + secondPayload, err := json.Marshal(second) + if err != nil { + t.Fatal(err) + } + thirdPayload, err := json.Marshal(third) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{ + batchErr: errors.New("syntax error in batch insert"), + singleErr: errTestHistoryAppend, + failSingleOnCount: 2, + } + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + messages := []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, Value: firstPayload}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, Value: secondPayload}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 12, Value: thirdPayload}, + } + + processHistoryBatchReliably( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + messages, + time.Nanosecond, + ) + + if appender.batchCount != 2 { + t.Fatalf("batch attempts = %d, want initial batch and failed suffix retry", appender.batchCount) + } + if appender.count != 4 { + t.Fatalf("single fallback attempts = %d, want 4", appender.count) + } + if got := committedOffsets(committer.messages); len(got) != 3 || got[0] != 10 || got[1] != 11 || got[2] != 12 { + t.Fatalf("committed offsets = %#v, want [10 11 12]", got) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_history_batch_retries_total{reason="write_error"} 1`) { + t.Fatalf("write retry metric missing:\n%s", text) + } +} + +func TestProcessHistoryBatchReliablyRetriesCommitWithoutReappending(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingHistoryAppender{} + committer := &contextCheckingHistoryCommitter{err: errors.New("commit failed"), failOnCount: 1} + registry := metrics.NewRegistry() + messages := []kafka.Message{ + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, Value: payload}, + {Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 11, Value: payload}, + } + + processHistoryBatchReliably( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + messages, + time.Nanosecond, + ) + + if appender.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1 without replay after commit failure", appender.batchCount) + } + if committer.count != 2 { + t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_history_batch_retries_total{reason="commit_error"} 1`) { + t.Fatalf("commit retry metric missing:\n%s", text) + } +} + func TestProcessHistoryBatchDoesNotCommitWhenAppendFails(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} payload, err := json.Marshal(env) @@ -207,6 +823,166 @@ func TestProcessHistoryBatchDoesNotCommitWhenAppendFails(t *testing.T) { } } +func TestProcessHistoryBatchCommitsWhenLocationDerivedWriteFails(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "VIN001", + MessageID: "0x02", + ParsedFields: map[string]any{ + "gb32960.position.longitude": 121.1, + "gb32960.position.latitude": 30.2, + }, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + locationErr := errors.New("location child table failed") + appender := &contextCheckingHistoryAppender{ + result: history.AppendResult{ + RawRows: 1, + LocationError: locationErr, + }, + } + committer := &contextCheckingHistoryCommitter{} + registry := metrics.NewRegistry() + + processHistoryBatch( + context.Background(), + discardHistoryLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}}, + ) + + if committer.messageCount != 1 { + t.Fatalf("committed messages = %d, want raw-success message committed", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_history_location_batch_flush_total{status="error"} 1`, + `vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("location derived failure metric missing %s:\n%s", want, text) + } + } +} + +func TestRetryHistoryAppenderRetriesTransientTDengineError(t *testing.T) { + transientErr := errors.New("taosws i/o timeout") + registry := metrics.NewRegistry() + appender := &contextCheckingHistoryAppender{ + result: history.AppendResult{RawRows: 1, LocationRows: 1}, + } + appender.onAppendBatch = func() { + if appender.batchCount == 1 { + appender.err = transientErr + return + } + appender.err = nil + } + retryAppender := retryHistoryAppender{ + delegate: appender, + attempts: 3, + registry: registry, + } + + result, err := retryAppender.AppendAllBatchWithResult(context.Background(), []envelope.FrameEnvelope{ + {Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}, + }) + if err != nil { + t.Fatalf("AppendAllBatchWithResult returned error: %v", err) + } + if appender.batchCount != 2 { + t.Fatalf("batch attempts = %d, want 2", appender.batchCount) + } + if result.RawRows != 1 || result.LocationRows != 1 { + t.Fatalf("result = %+v, want raw and location rows preserved", result) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_history_write_retries_total{operation="batch",status="retry"} 1`) { + t.Fatalf("retry metric missing:\n%s", text) + } +} + +func TestRetryHistoryAppenderRecordsExhaustedTransientError(t *testing.T) { + registry := metrics.NewRegistry() + appender := &contextCheckingHistoryAppender{err: errors.New("connection reset by peer")} + retryAppender := retryHistoryAppender{ + delegate: appender, + attempts: 2, + registry: registry, + } + + _, err := retryAppender.AppendAllBatchWithResult(context.Background(), []envelope.FrameEnvelope{ + {Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}, + }) + if err == nil { + t.Fatal("AppendAllBatchWithResult error = nil, want exhausted transient error") + } + if appender.batchCount != 2 { + t.Fatalf("batch attempts = %d, want 2", appender.batchCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_history_write_retries_total{operation="batch",status="retry"} 1`, + `vehicle_history_write_retries_total{operation="batch",status="exhausted"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("retry metric missing %s:\n%s", want, text) + } + } +} + +func TestRetryHistoryAppenderDoesNotRetryNonTransientError(t *testing.T) { + appender := &contextCheckingHistoryAppender{err: errors.New("syntax error near table name")} + retryAppender := retryHistoryAppender{ + delegate: appender, + attempts: 3, + } + + _, err := retryAppender.AppendAllBatchWithResult(context.Background(), []envelope.FrameEnvelope{ + {Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}, + }) + if err == nil { + t.Fatal("AppendAllBatchWithResult error = nil, want non-transient error") + } + if appender.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1", appender.batchCount) + } +} + +func TestRetryHistoryAppenderDoesNotRetryLocationDerivedError(t *testing.T) { + locationErr := errors.New("location child table failed") + appender := &contextCheckingHistoryAppender{ + result: history.AppendResult{ + RawRows: 1, + LocationError: locationErr, + }, + } + retryAppender := retryHistoryAppender{ + delegate: appender, + attempts: 3, + } + + result, err := retryAppender.AppendAllBatchWithResult(context.Background(), []envelope.FrameEnvelope{ + {Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}, + }) + if err != nil { + t.Fatalf("AppendAllBatchWithResult returned error: %v", err) + } + if appender.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1", appender.batchCount) + } + if !errors.Is(result.LocationError, locationErr) { + t.Fatalf("LocationError = %v, want %v", result.LocationError, locationErr) + } +} + func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) { cfg := loadConfig() @@ -218,32 +994,77 @@ func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) { if cfg.BatchSize != 200 { t.Fatalf("BatchSize = %d, want 200", cfg.BatchSize) } - if cfg.BatchWait != 100 { - t.Fatalf("BatchWait = %d, want 100", cfg.BatchWait) + if cfg.Workers != 3 { + t.Fatalf("Workers = %d, want 3", cfg.Workers) + } + if cfg.BatchWait != 20 { + t.Fatalf("BatchWait = %d, want 20", cfg.BatchWait) + } + if cfg.RetryAttempts != 3 { + t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts) + } + if cfg.RetryDelay != 20*time.Millisecond { + t.Fatalf("RetryDelay = %s, want 20ms", cfg.RetryDelay) } } func TestLoadConfigReadsBatchSettings(t *testing.T) { + t.Setenv("HISTORY_WORKERS", "5") t.Setenv("HISTORY_BATCH_SIZE", "500") t.Setenv("HISTORY_BATCH_WAIT_MS", "250") + t.Setenv("HISTORY_RETRY_ATTEMPTS", "5") + t.Setenv("HISTORY_RETRY_DELAY_MS", "33") cfg := loadConfig() + if cfg.Workers != 5 { + t.Fatalf("Workers = %d, want 5", cfg.Workers) + } if cfg.BatchSize != 500 { t.Fatalf("BatchSize = %d, want 500", cfg.BatchSize) } if cfg.BatchWait != 250 { t.Fatalf("BatchWait = %d, want 250", cfg.BatchWait) } + if cfg.RetryAttempts != 5 { + t.Fatalf("RetryAttempts = %d, want 5", cfg.RetryAttempts) + } + if cfg.RetryDelay != 33*time.Millisecond { + t.Fatalf("RetryDelay = %s, want 33ms", cfg.RetryDelay) + } +} + +func TestConfigValidateRejectsFieldsTopics(t *testing.T) { + cfg := config{KafkaTopics: []string{"vehicle.fields.go.jt808.v1"}, Workers: 1} + + err := cfg.Validate() + + if err == nil || !strings.Contains(err.Error(), "raw topics only") { + t.Fatalf("Validate() error = %v, want raw topic rejection", err) + } +} + +func TestConfigValidateRequiresPositiveWorkers(t *testing.T) { + cfg := config{KafkaTopics: []string{"vehicle.raw.go.jt808.v1"}, Workers: 0} + + err := cfg.Validate() + + if err == nil || !strings.Contains(err.Error(), "HISTORY_WORKERS") { + t.Fatalf("Validate() error = %v, want HISTORY_WORKERS hint", err) + } } type contextCheckingHistoryAppender struct { - ctxErr error - count int - batchCount int - batch []envelope.FrameEnvelope - err error - onAppendBatch func() + ctxErr error + count int + batchCount int + batch []envelope.FrameEnvelope + err error + batchErr error + singleErr error + failSingleOnCount int + result history.AppendResult + onAppendBatch func() } func (a *contextCheckingHistoryAppender) AppendAll(ctx context.Context, _ envelope.FrameEnvelope) error { @@ -252,7 +1073,23 @@ func (a *contextCheckingHistoryAppender) AppendAll(ctx context.Context, _ envelo if a.ctxErr != nil { return a.ctxErr } - return a.err + return a.currentSingleErr() +} + +func (a *contextCheckingHistoryAppender) AppendAllWithResult(ctx context.Context, _ envelope.FrameEnvelope) (history.AppendResult, error) { + a.ctxErr = ctx.Err() + a.count++ + if a.ctxErr != nil { + return history.AppendResult{}, a.ctxErr + } + if err := a.currentSingleErr(); err != nil { + return history.AppendResult{}, err + } + result := a.result + if result.RawRows == 0 { + result.RawRows = 1 + } + return result, nil } func (a *contextCheckingHistoryAppender) AppendAllBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { @@ -265,6 +1102,49 @@ func (a *contextCheckingHistoryAppender) AppendAllBatch(ctx context.Context, env if a.ctxErr != nil { return a.ctxErr } + return a.currentBatchErr() +} + +func (a *contextCheckingHistoryAppender) AppendAllBatchWithResult(ctx context.Context, envs []envelope.FrameEnvelope) (history.AppendResult, error) { + a.ctxErr = ctx.Err() + a.batchCount++ + a.batch = append([]envelope.FrameEnvelope(nil), envs...) + if a.onAppendBatch != nil { + a.onAppendBatch() + } + if a.ctxErr != nil { + return history.AppendResult{}, a.ctxErr + } + if err := a.currentBatchErr(); err != nil { + return history.AppendResult{}, err + } + result := a.result + if result.RawRows == 0 { + result.RawRows = len(envs) + } + return result, nil +} + +func (a *contextCheckingHistoryAppender) currentSingleErr() error { + if a.failSingleOnCount > 0 && a.count == a.failSingleOnCount { + if a.singleErr != nil { + return a.singleErr + } + return a.err + } + if a.failSingleOnCount > 0 { + return nil + } + if a.singleErr != nil { + return a.singleErr + } + return a.err +} + +func (a *contextCheckingHistoryAppender) currentBatchErr() error { + if a.batchErr != nil { + return a.batchErr + } return a.err } @@ -272,13 +1152,23 @@ type contextCheckingHistoryCommitter struct { ctxErr error count int messageCount int + messages []kafka.Message + err error + failOnCount int } func (c *contextCheckingHistoryCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error { c.ctxErr = ctx.Err() c.count++ c.messageCount += len(messages) - return c.ctxErr + c.messages = append(c.messages, messages...) + if c.ctxErr != nil { + return c.ctxErr + } + if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) { + return c.err + } + return nil } type discardHistoryLogger struct{} @@ -287,3 +1177,11 @@ func (discardHistoryLogger) Error(string, ...any) {} func (discardHistoryLogger) Warn(string, ...any) {} var errTestHistoryAppend = errors.New("test history append failed") + +func committedOffsets(messages []kafka.Message) []int64 { + offsets := make([]int64, len(messages)) + for index, message := range messages { + offsets[index] = message.Offset + } + return offsets +} diff --git a/go/vehicle-gateway/cmd/identity-import/main.go b/go/vehicle-gateway/cmd/identity-import/main.go new file mode 100644 index 00000000..e0637b2a --- /dev/null +++ b/go/vehicle-gateway/cmd/identity-import/main.go @@ -0,0 +1,695 @@ +package main + +import ( + "context" + "database/sql" + "encoding/csv" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + _ "github.com/go-sql-driver/mysql" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" +) + +func main() { + var input string + var dsn string + var legacyTable string + var apply bool + var ensureSchema bool + var reportLimit int + var unresolvedOut string + var conflictsOut string + var syncDataSources bool + var pruneUnmanagedDataSources bool + var pruneDataSourceMinAge time.Duration + var retireStaleUnmanagedDataSources bool + var retireDataSourceMinAge time.Duration + var timeout time.Duration + flag.StringVar(&input, "input", env("IDENTITY_MAPPING_INPUT", ""), "directory that contains 808 plate/phone mapping workbooks") + flag.StringVar(&dsn, "mysql-dsn", env("IDENTITY_MYSQL_DSN", env("MYSQL_DSN", "")), "MySQL DSN; empty means scan only") + flag.StringVar(&legacyTable, "legacy-table", env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding"), "legacy VIN/plate binding table used to resolve VIN") + flag.BoolVar(&apply, "apply", false, "write resolved mappings to vehicle and vehicle_identifier") + flag.BoolVar(&ensureSchema, "ensure-schema", false, "create vehicle and vehicle_identifier tables before import") + flag.IntVar(&reportLimit, "report-limit", 50, "max unresolved/conflict items embedded in JSON; use -1 for all") + flag.StringVar(&unresolvedOut, "unresolved-out", "", "optional CSV path for all unresolved mappings") + flag.StringVar(&conflictsOut, "conflicts-out", "", "optional CSV path for all conflict mappings") + flag.BoolVar(&syncDataSources, "sync-data-sources", false, "infer JT808 vehicle_data_source platform names from jt808_registration and vehicle_identifier") + flag.BoolVar(&pruneUnmanagedDataSources, "prune-unmanaged-data-sources", false, "delete stale unclassified vehicle_data_source rows that have no registration evidence and are not referenced by final mileage") + flag.DurationVar(&pruneDataSourceMinAge, "prune-data-source-min-age", time.Hour, "minimum latest_seen/updated age before -prune-unmanaged-data-sources can delete rows") + flag.BoolVar(&retireStaleUnmanagedDataSources, "retire-stale-unmanaged-data-sources", false, "disable stale unclassified vehicle_data_source rows that have no registration evidence but may be referenced by historical mileage") + flag.DurationVar(&retireDataSourceMinAge, "retire-data-source-min-age", 24*time.Hour, "minimum latest_seen/updated age before -retire-stale-unmanaged-data-sources can disable rows") + flag.DurationVar(&timeout, "timeout", 2*time.Minute, "import timeout") + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + input = strings.TrimSpace(input) + output := map[string]any{} + var records []identity.MappingRecord + var scan identity.MappingScanReport + if input != "" { + var err error + records, scan, err = identity.ReadMappingDirectory(input) + if err != nil { + fail(err) + } + output["input"] = input + output["scan"] = scan + } else if !syncDataSources && !pruneUnmanagedDataSources && !retireStaleUnmanagedDataSources { + fail(errors.New("-input is required unless -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set")) + } + if err := validateMappingApplyInput(apply, scan); err != nil { + fail(err) + } + if strings.TrimSpace(dsn) == "" { + if syncDataSources || pruneUnmanagedDataSources || retireStaleUnmanagedDataSources { + fail(errors.New("-mysql-dsn is required when -sync-data-sources, -prune-unmanaged-data-sources or -retire-stale-unmanaged-data-sources is set")) + } + output["mode"] = "scan_only" + writeJSON(output) + return + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + fail(err) + } + defer func() { _ = db.Close() }() + if err := db.PingContext(ctx); err != nil { + fail(err) + } + if input != "" && (ensureSchema || apply) { + if err := identity.EnsureVehicleIdentifierSchema(ctx, db); err != nil { + fail(err) + } + } + if input != "" { + report, err := identity.ImportMappingRecords(ctx, db, records, scan, identity.MappingImportOptions{ + Apply: apply, + LegacyTable: legacyTable, + ReportItemLimit: reportItemLimit(reportLimit, unresolvedOut, conflictsOut), + }) + if err != nil { + fail(err) + } + if strings.TrimSpace(unresolvedOut) != "" { + if err := writeUnresolvedCSV(unresolvedOut, report.UnresolvedItems); err != nil { + fail(err) + } + } + if strings.TrimSpace(conflictsOut) != "" { + if err := writeConflictsCSV(conflictsOut, report.ConflictItems); err != nil { + fail(err) + } + } + output["report"] = report + output["mode"] = map[bool]string{true: "apply", false: "dry_run"}[apply] + } + if syncDataSources { + syncReport, err := syncJT808DataSourcesFromIdentifiers(ctx, db, apply) + if err != nil { + fail(err) + } + output["data_source_sync"] = syncReport + } + if pruneUnmanagedDataSources { + pruneReport, err := pruneUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, pruneDataSourceMinAge) + if err != nil { + fail(err) + } + output["data_source_prune"] = pruneReport + } + if retireStaleUnmanagedDataSources { + retireReport, err := retireStaleUnmanagedDataSourcesWithoutEvidence(ctx, db, apply, retireDataSourceMinAge) + if err != nil { + fail(err) + } + output["data_source_retire"] = retireReport + } + if input == "" { + output["mode"] = maintenanceMode(apply, syncDataSources, pruneUnmanagedDataSources, retireStaleUnmanagedDataSources) + } + writeJSON(output) +} + +func maintenanceMode(apply bool, syncDataSources bool, pruneUnmanagedDataSources bool, retireStaleUnmanagedDataSources bool) string { + var parts []string + if syncDataSources { + parts = append(parts, "sync") + } + if pruneUnmanagedDataSources { + parts = append(parts, "prune") + } + if retireStaleUnmanagedDataSources { + parts = append(parts, "retire") + } + if len(parts) == 0 { + return map[bool]string{true: "apply", false: "dry_run"}[apply] + } + parts = append(parts, map[bool]string{true: "apply", false: "dry_run"}[apply]) + return strings.Join(parts, "_") +} + +func reportItemLimit(limit int, unresolvedOut string, conflictsOut string) int { + if strings.TrimSpace(unresolvedOut) != "" || strings.TrimSpace(conflictsOut) != "" { + return -1 + } + return limit +} + +func validateMappingApplyInput(apply bool, scan identity.MappingScanReport) error { + if !apply { + return nil + } + return unsupportedMappingFilesError(scan) +} + +func unsupportedMappingFilesError(scan identity.MappingScanReport) error { + if scan.UnsupportedFiles <= 0 { + return nil + } + items := make([]string, 0, len(scan.UnsupportedItems)) + for _, item := range scan.UnsupportedItems { + path := strings.TrimSpace(item.File) + if path == "" { + path = strings.TrimSpace(item.Ext) + } + if path != "" { + items = append(items, path) + } + if len(items) >= 3 { + break + } + } + suffix := "" + if len(items) > 0 { + suffix = ": " + strings.Join(items, ", ") + } + return fmt.Errorf("identity mapping import has %d unsupported workbook(s); convert .xls/.xlsb to .xlsx before -apply%s", scan.UnsupportedFiles, suffix) +} + +func env(key string, fallback string) string { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + return value +} + +func writeJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + fail(err) + } +} + +func writeUnresolvedCSV(path string, items []identity.MappingRecord) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + writer := csv.NewWriter(file) + defer writer.Flush() + if err := writer.Write([]string{ + "file", "sheet", "row", "source_code", "source_name", "protocol", + "identifier_type", "identifier_value", "raw_value", "plate", "oem", "reason", + }); err != nil { + return err + } + for _, item := range items { + if err := writer.Write([]string{ + item.File, + item.Sheet, + fmt.Sprint(item.Row), + item.SourceCode, + item.SourceName, + item.Protocol, + item.IdentifierType, + item.IdentifierValue, + item.RawValue, + item.Plate, + item.OEM, + "vin_not_found_in_legacy_binding", + }); err != nil { + return err + } + } + return writer.Error() +} + +func writeConflictsCSV(path string, items []identity.MappingConflict) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + writer := csv.NewWriter(file) + defer writer.Flush() + if err := writer.Write([]string{ + "file", "sheet", "row", "source_code", "source_name", "protocol", + "identifier_type", "identifier_value", "raw_value", "plate", "oem", + "existing_vin", "new_vin", "reason", + }); err != nil { + return err + } + for _, item := range items { + record := item.Record + if err := writer.Write([]string{ + record.File, + record.Sheet, + fmt.Sprint(record.Row), + record.SourceCode, + record.SourceName, + record.Protocol, + record.IdentifierType, + record.IdentifierValue, + record.RawValue, + record.Plate, + record.OEM, + item.ExistingVIN, + item.NewVIN, + item.Reason, + }); err != nil { + return err + } + } + return writer.Error() +} + +type dataSourceSyncReport struct { + Apply bool `json:"apply"` + CandidateSources int64 `json:"candidate_sources"` + SkippedSources int64 `json:"skipped_sources"` + ConflictingSources int64 `json:"conflicting_sources"` + PlatformKindCandidates int64 `json:"platform_kind_candidates"` + PlatformKindClassified int64 `json:"platform_kind_classified,omitempty"` + ReprojectDailyMileageTargets int64 `json:"reproject_daily_mileage_targets,omitempty"` + ReprojectedDailyMileageTargets int64 `json:"reprojected_daily_mileage_targets,omitempty"` + Synced int64 `json:"synced,omitempty"` +} + +type dailyMileageProjectionTarget struct { + VIN string + StatDate string + Protocol envelope.Protocol +} + +type dataSourcePruneReport struct { + Apply bool `json:"apply"` + MinAgeSeconds int64 `json:"min_age_seconds"` + CandidateSources int64 `json:"candidate_sources"` + Pruned int64 `json:"pruned,omitempty"` +} + +type dataSourceRetireReport struct { + Apply bool `json:"apply"` + MinAgeSeconds int64 `json:"min_age_seconds"` + CandidateSources int64 `json:"candidate_sources"` + Retired int64 `json:"retired,omitempty"` +} + +func syncJT808DataSourcesFromIdentifiers(ctx context.Context, db *sql.DB, apply bool) (dataSourceSyncReport, error) { + report := dataSourceSyncReport{Apply: apply} + var err error + report.CandidateSources, err = queryInt64(ctx, db, syncJT808DataSourcesCandidateCountSQL) + if err != nil { + return report, err + } + report.SkippedSources, err = queryInt64(ctx, db, syncJT808DataSourcesSkippedCountSQL) + if err != nil { + return report, err + } + report.ConflictingSources, err = queryInt64(ctx, db, syncJT808DataSourcesConflictCountSQL) + if err != nil { + return report, err + } + if apply { + result, err := db.ExecContext(ctx, syncJT808DataSourcesSQL) + if err != nil { + return report, err + } + rowsAffected, err := result.RowsAffected() + if err == nil { + report.Synced = rowsAffected + } + } + report.PlatformKindCandidates, err = queryInt64(ctx, db, classifyConfiguredDataSourcesCountSQL) + if err != nil { + return report, err + } + if apply { + result, err := db.ExecContext(ctx, classifyConfiguredDataSourcesSQL) + if err != nil { + return report, err + } + rowsAffected, err := result.RowsAffected() + if err == nil { + report.PlatformKindClassified = rowsAffected + } + } + targets, err := queryDailyMileageProjectionTargets(ctx, db) + if err != nil { + return report, err + } + report.ReprojectDailyMileageTargets = int64(len(targets)) + if apply { + for _, target := range targets { + if err := stats.ProjectDailyMileage(ctx, db, target.VIN, target.StatDate, target.Protocol); err != nil { + return report, err + } + report.ReprojectedDailyMileageTargets++ + } + } + return report, nil +} + +func queryDailyMileageProjectionTargets(ctx context.Context, db *sql.DB) ([]dailyMileageProjectionTarget, error) { + rows, err := db.QueryContext(ctx, reprojectSelectableDailyMileageSourcesSQL) + if err != nil { + return nil, err + } + defer rows.Close() + var targets []dailyMileageProjectionTarget + for rows.Next() { + var target dailyMileageProjectionTarget + var protocol string + if err := rows.Scan(&target.VIN, &target.StatDate, &protocol); err != nil { + return nil, err + } + target.Protocol = envelope.Protocol(strings.TrimSpace(protocol)) + if strings.TrimSpace(target.VIN) != "" && strings.TrimSpace(target.StatDate) != "" && strings.TrimSpace(string(target.Protocol)) != "" { + targets = append(targets, target) + } + } + return targets, rows.Err() +} + +func pruneUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourcePruneReport, error) { + if minAge < 0 { + minAge = 0 + } + minAgeSeconds := int64(minAge.Seconds()) + report := dataSourcePruneReport{Apply: apply, MinAgeSeconds: minAgeSeconds} + var err error + report.CandidateSources, err = queryInt64WithArgs(ctx, db, pruneUnmanagedDataSourcesCountSQL, minAgeSeconds) + if err != nil { + return report, err + } + if !apply { + return report, nil + } + result, err := db.ExecContext(ctx, pruneUnmanagedDataSourcesSQL, minAgeSeconds) + if err != nil { + return report, err + } + rowsAffected, err := result.RowsAffected() + if err == nil { + report.Pruned = rowsAffected + } + return report, nil +} + +func retireStaleUnmanagedDataSourcesWithoutEvidence(ctx context.Context, db *sql.DB, apply bool, minAge time.Duration) (dataSourceRetireReport, error) { + if minAge < 0 { + minAge = 0 + } + minAgeSeconds := int64(minAge.Seconds()) + report := dataSourceRetireReport{Apply: apply, MinAgeSeconds: minAgeSeconds} + var err error + report.CandidateSources, err = queryInt64WithArgs(ctx, db, retireStaleUnmanagedDataSourcesCountSQL, minAgeSeconds) + if err != nil { + return report, err + } + if !apply { + return report, nil + } + result, err := db.ExecContext(ctx, retireStaleUnmanagedDataSourcesSQL, minAgeSeconds) + if err != nil { + return report, err + } + rowsAffected, err := result.RowsAffected() + if err == nil { + report.Retired = rowsAffected + } + return report, nil +} + +func queryInt64(ctx context.Context, db *sql.DB, query string) (int64, error) { + return queryInt64WithArgs(ctx, db, query) +} + +func queryInt64WithArgs(ctx context.Context, db *sql.DB, query string, args ...any) (int64, error) { + var count int64 + err := db.QueryRowContext(ctx, query, args...).Scan(&count) + return count, err +} + +const jt808DataSourceInferenceSQL = ` + SELECT + 'JT808' AS protocol, + TRIM(r.source_ip) AS source_ip, + MAX(TRIM(r.source_endpoint)) AS latest_source_endpoint, + MIN(COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_name, + MIN(NULLIF(TRIM(vi.source_code), '')) AS source_code, + MIN(COALESCE(r.first_registered_at, r.latest_registered_at, r.latest_authenticated_at, r.latest_seen_at, r.updated_at, CURRENT_TIMESTAMP)) AS first_seen_at, + MAX(COALESCE(r.latest_seen_at, r.latest_authenticated_at, r.latest_registered_at, r.updated_at, CURRENT_TIMESTAMP)) AS latest_seen_at, + COUNT(DISTINCT COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS platform_count, + COUNT(DISTINCT NULLIF(TRIM(vi.source_code), '')) AS source_code_count + FROM jt808_registration r + JOIN vehicle_identifier vi + ON vi.protocol = 'JT808' + AND vi.identifier_type = 'JT808_PHONE' + AND vi.identifier_value = r.phone + AND vi.enabled = 1 + WHERE r.source_endpoint IS NOT NULL + AND TRIM(r.source_endpoint) <> '' + AND r.source_ip IS NOT NULL + AND TRIM(r.source_ip) <> '' + GROUP BY TRIM(r.source_ip) +` + +const jt808DataSourceCandidateWhereSQL = `inferred.source_ip IS NOT NULL + AND inferred.source_ip <> '' + AND inferred.platform_name IS NOT NULL + AND inferred.platform_name <> '' + AND inferred.platform_count = 1 + AND inferred.source_code_count = 1` + +const jt808DataSourceNeedsSyncWhereSQL = `(ds.id IS NULL + OR ds.platform_name IS NULL OR TRIM(ds.platform_name) = '' + OR ds.source_code IS NULL OR TRIM(ds.source_code) = '' + OR ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN' + OR (ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%' OR ds.remark = 'auto-reenabled: source evidence restored')))` + +const syncJT808DataSourcesSQL = ` +INSERT INTO vehicle_data_source + (protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, first_seen_at, latest_seen_at) +SELECT + inferred.protocol, + inferred.source_ip, + inferred.latest_source_endpoint, + inferred.platform_name, + inferred.source_code, + 'PLATFORM', + inferred.first_seen_at, + inferred.latest_seen_at +FROM (` + jt808DataSourceInferenceSQL + `) inferred +LEFT JOIN vehicle_data_source ds + ON ds.protocol = inferred.protocol + AND ds.source_ip = inferred.source_ip +WHERE ` + jt808DataSourceCandidateWhereSQL + ` + AND ` + jt808DataSourceNeedsSyncWhereSQL + ` +ON DUPLICATE KEY UPDATE + latest_source_endpoint = VALUES(latest_source_endpoint), + platform_name = CASE + WHEN vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = '' + THEN VALUES(platform_name) + ELSE vehicle_data_source.platform_name + END, + source_code = CASE + WHEN vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = '' + THEN VALUES(source_code) + ELSE vehicle_data_source.source_code + END, + source_kind = CASE + WHEN vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN' + THEN VALUES(source_kind) + ELSE vehicle_data_source.source_kind + END, + enabled = CASE + WHEN vehicle_data_source.enabled = 0 + AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored') + AND ( + (VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '') + OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '') + OR VALUES(source_kind) <> 'UNKNOWN' + ) + THEN 1 + ELSE vehicle_data_source.enabled + END, + remark = CASE + WHEN vehicle_data_source.enabled = 0 + AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored') + AND ( + (VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '') + OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '') + OR VALUES(source_kind) <> 'UNKNOWN' + ) + THEN 'auto-reenabled: source evidence restored' + ELSE vehicle_data_source.remark + END, + first_seen_at = COALESCE(vehicle_data_source.first_seen_at, VALUES(first_seen_at)), + latest_seen_at = GREATEST(COALESCE(vehicle_data_source.latest_seen_at, VALUES(latest_seen_at)), VALUES(latest_seen_at)), + updated_at = CURRENT_TIMESTAMP +` + +const reprojectSelectableDailyMileageSourcesSQL = ` +SELECT DISTINCT s.vin, s.stat_date, s.protocol +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds + ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip +LEFT JOIN vehicle_daily_mileage m + ON m.vin = s.vin AND m.stat_date = s.stat_date AND m.protocol = s.protocol +WHERE s.protocol = 'JT808' + AND s.quality_status = 'OK' + AND (ds.id IS NULL OR ds.enabled = 1 OR ( + COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN' + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + )) + AND ( + m.vin IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM vehicle_daily_mileage_source selected + WHERE selected.vin = s.vin + AND selected.stat_date = s.stat_date + AND selected.protocol = s.protocol + AND selected.is_selected = 1 + ) + ) +ORDER BY s.stat_date DESC, s.vin ASC +` + +const syncJT808DataSourcesCandidateCountSQL = ` +SELECT COUNT(*) +FROM (` + jt808DataSourceInferenceSQL + `) inferred +LEFT JOIN vehicle_data_source ds + ON ds.protocol = inferred.protocol + AND ds.source_ip = inferred.source_ip +WHERE ` + jt808DataSourceCandidateWhereSQL + ` + AND ` + jt808DataSourceNeedsSyncWhereSQL + +const syncJT808DataSourcesSkippedCountSQL = ` +SELECT COUNT(*) +FROM (` + jt808DataSourceInferenceSQL + `) inferred +WHERE inferred.source_ip IS NOT NULL + AND inferred.source_ip <> '' + AND inferred.platform_name IS NOT NULL + AND inferred.platform_name <> '' + AND NOT (` + jt808DataSourceCandidateWhereSQL + `)` + +const syncJT808DataSourcesConflictCountSQL = ` +SELECT COUNT(*) +FROM (` + jt808DataSourceInferenceSQL + `) inferred +JOIN vehicle_data_source ds + ON ds.protocol = inferred.protocol + AND ds.source_ip = inferred.source_ip +WHERE ` + jt808DataSourceCandidateWhereSQL + ` + AND ds.source_code IS NOT NULL + AND TRIM(ds.source_code) <> '' + AND ds.source_code <> inferred.source_code` + +const classifyConfiguredDataSourcesWhereSQL = ` + (source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN') + AND ( + (source_code IS NOT NULL AND TRIM(source_code) <> '') + OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '') + )` + +const classifyConfiguredDataSourcesCountSQL = ` +SELECT COUNT(*) +FROM vehicle_data_source +WHERE ` + classifyConfiguredDataSourcesWhereSQL + +const classifyConfiguredDataSourcesSQL = ` +UPDATE vehicle_data_source +SET source_kind = 'PLATFORM', + updated_at = CURRENT_TIMESTAMP +WHERE ` + classifyConfiguredDataSourcesWhereSQL + +const pruneUnmanagedDataSourcesWhereSQL = ` + (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN') + AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ? + AND NOT EXISTS ( + SELECT 1 + FROM vehicle_daily_mileage m + WHERE m.source_id = ds.id + LIMIT 1 + ) + AND NOT EXISTS ( + SELECT 1 + FROM jt808_registration r + WHERE ds.protocol = 'JT808' + AND r.source_ip = ds.source_ip + LIMIT 1 + )` + +const pruneUnmanagedDataSourcesCountSQL = ` +SELECT COUNT(*) +FROM vehicle_data_source ds +WHERE ` + pruneUnmanagedDataSourcesWhereSQL + +const pruneUnmanagedDataSourcesSQL = ` +DELETE ds +FROM vehicle_data_source ds +WHERE ` + pruneUnmanagedDataSourcesWhereSQL + +const retireStaleUnmanagedDataSourcesWhereSQL = ` + ds.enabled = 1 + AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN') + AND TIMESTAMPDIFF(SECOND, COALESCE(ds.latest_seen_at, ds.updated_at, ds.created_at), CURRENT_TIMESTAMP) >= ? + AND NOT EXISTS ( + SELECT 1 + FROM jt808_registration r + WHERE ds.protocol = 'JT808' + AND r.source_ip = ds.source_ip + LIMIT 1 + )` + +const retireStaleUnmanagedDataSourcesCountSQL = ` +SELECT COUNT(*) +FROM vehicle_data_source ds +WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL + +const retireStaleUnmanagedDataSourcesSQL = ` +UPDATE vehicle_data_source ds +SET enabled = 0, + remark = CASE + WHEN remark IS NULL OR TRIM(remark) = '' + THEN 'auto-retired: stale unmanaged source without registration evidence' + ELSE remark + END, + updated_at = CURRENT_TIMESTAMP +WHERE ` + retireStaleUnmanagedDataSourcesWhereSQL + +func fail(err error) { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/go/vehicle-gateway/cmd/identity-import/main_test.go b/go/vehicle-gateway/cmd/identity-import/main_test.go new file mode 100644 index 00000000..d24ae1af --- /dev/null +++ b/go/vehicle-gateway/cmd/identity-import/main_test.go @@ -0,0 +1,426 @@ +package main + +import ( + "context" + "database/sql/driver" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" +) + +func TestReportItemLimitExpandsWhenCSVOutputIsRequested(t *testing.T) { + if got := reportItemLimit(50, "", ""); got != 50 { + t.Fatalf("reportItemLimit without csv = %d, want 50", got) + } + if got := reportItemLimit(50, "unresolved.csv", ""); got != -1 { + t.Fatalf("reportItemLimit with unresolved csv = %d, want -1", got) + } + if got := reportItemLimit(50, "", "conflicts.csv"); got != -1 { + t.Fatalf("reportItemLimit with conflicts csv = %d, want -1", got) + } +} + +func TestUnsupportedMappingFilesError(t *testing.T) { + err := unsupportedMappingFilesError(identity.MappingScanReport{}) + if err != nil { + t.Fatalf("unsupportedMappingFilesError(empty) = %v", err) + } + err = unsupportedMappingFilesError(identity.MappingScanReport{ + UnsupportedFiles: 2, + UnsupportedItems: []identity.MappingUnsupportedFileReport{ + {File: "G7s/legacy.xls", Ext: ".xls"}, + {File: "信达/legacy.xlsb", Ext: ".xlsb"}, + }, + }) + if err == nil { + t.Fatal("unsupportedMappingFilesError() nil, want error") + } + text := err.Error() + for _, want := range []string{"2 unsupported workbook", "G7s/legacy.xls", "信达/legacy.xlsb", "before -apply"} { + if !strings.Contains(text, want) { + t.Fatalf("error missing %q: %s", want, text) + } + } +} + +func TestValidateMappingApplyInputAllowsDryRunButBlocksApply(t *testing.T) { + scan := identity.MappingScanReport{ + UnsupportedFiles: 1, + UnsupportedItems: []identity.MappingUnsupportedFileReport{ + {File: "G7s/legacy.xls", Ext: ".xls"}, + }, + } + if err := validateMappingApplyInput(false, scan); err != nil { + t.Fatalf("validateMappingApplyInput(dry-run) = %v", err) + } + if err := validateMappingApplyInput(true, scan); err == nil { + t.Fatal("validateMappingApplyInput(apply) nil, want error") + } +} + +func TestWriteUnresolvedCSV(t *testing.T) { + path := filepath.Join(t.TempDir(), "unresolved.csv") + err := writeUnresolvedCSV(path, []identity.MappingRecord{ + { + File: "G7s/example.xlsx", + Sheet: "Sheet1", + Row: 2, + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: identity.IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + RawValue: "013307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + }) + if err != nil { + t.Fatalf("writeUnresolvedCSV() error = %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + text := string(data) + for _, want := range []string{"identifier_type", "JT808_PHONE", "13307795425", "vin_not_found_in_legacy_binding"} { + if !strings.Contains(text, want) { + t.Fatalf("csv missing %q:\n%s", want, text) + } + } +} + +func TestWriteConflictsCSV(t *testing.T) { + path := filepath.Join(t.TempDir(), "conflicts.csv") + err := writeConflictsCSV(path, []identity.MappingConflict{ + { + Record: identity.MappingRecord{ + File: "source.xlsx", + Sheet: "Sheet1", + Row: 3, + SourceCode: "xinda", + Protocol: "JT808", + IdentifierType: identity.IdentifierTypePlate, + IdentifierValue: "粤AG18312", + Plate: "粤AG18312", + }, + ExistingVIN: "VIN001", + NewVIN: "VIN002", + Reason: "identifier already points to another vin", + }, + }) + if err != nil { + t.Fatalf("writeConflictsCSV() error = %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + text := string(data) + for _, want := range []string{"existing_vin", "VIN001", "VIN002", "identifier already points to another vin"} { + if !strings.Contains(text, want) { + t.Fatalf("csv missing %q:\n%s", want, text) + } + } +} + +func TestSyncJT808DataSourcesFromIdentifiersPreservesManualPlatformNames(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(3)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2)) + mock.ExpectExec("INSERT INTO vehicle_data_source"). + WillReturnResult(driver.RowsAffected(3)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(5)) + mock.ExpectExec("UPDATE vehicle_data_source"). + WillReturnResult(driver.RowsAffected(5)) + mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}). + AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808")) + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO vehicle_daily_mileage"). + WillReturnResult(driver.RowsAffected(1)) + mock.ExpectExec("UPDATE vehicle_daily_mileage_source s"). + WillReturnResult(driver.RowsAffected(1)) + mock.ExpectExec("DELETE FROM vehicle_daily_mileage"). + WillReturnResult(driver.RowsAffected(0)) + mock.ExpectCommit() + + report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, true) + if err != nil { + t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err) + } + if !report.Apply || report.CandidateSources != 3 || report.SkippedSources != 1 || report.ConflictingSources != 2 || report.Synced != 3 || report.PlatformKindCandidates != 5 || report.PlatformKindClassified != 5 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 1 { + t.Fatalf("report = %#v", report) + } + for _, want := range []string{ + "COUNT(DISTINCT", + "vehicle_identifier vi", + "TRIM(r.source_ip) AS source_ip", + "GROUP BY TRIM(r.source_ip)", + "inferred.source_code", + "'PLATFORM'", + "inferred.source_code_count = 1", + "vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''", + "ELSE vehicle_data_source.source_code", + "vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'", + "ELSE vehicle_data_source.source_kind", + "vehicle_data_source.enabled = 0", + "vehicle_data_source.remark LIKE 'auto-retired:%'", + "vehicle_data_source.remark = 'auto-reenabled: source evidence restored'", + "THEN 'auto-reenabled: source evidence restored'", + "THEN 1", + "LEFT JOIN vehicle_data_source ds", + "ds.id IS NULL", + } { + if !strings.Contains(syncJT808DataSourcesSQL, want) { + t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL) + } + } + if strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") < 0 || + strings.Index(syncJT808DataSourcesSQL, "remark = CASE") < 0 || + strings.Index(syncJT808DataSourcesSQL, "enabled = CASE") > strings.Index(syncJT808DataSourcesSQL, "remark = CASE") { + t.Fatalf("sync sql must restore enabled before updating remark because MySQL evaluates assignments in order:\n%s", syncJT808DataSourcesSQL) + } + for _, want := range []string{ + "LEFT JOIN vehicle_data_source ds", + "ds.id IS NULL", + "ds.source_code IS NULL OR TRIM(ds.source_code) = ''", + "ds.enabled = 0 AND (ds.remark LIKE 'auto-retired:%'", + "ds.remark = 'auto-reenabled: source evidence restored'", + } { + if !strings.Contains(syncJT808DataSourcesCandidateCountSQL, want) { + t.Fatalf("candidate count sql missing %q:\n%s", want, syncJT808DataSourcesCandidateCountSQL) + } + } + for _, want := range []string{ + "JOIN vehicle_data_source ds", + "ds.source_code <> inferred.source_code", + } { + if !strings.Contains(syncJT808DataSourcesConflictCountSQL, want) { + t.Fatalf("conflict count sql missing %q:\n%s", want, syncJT808DataSourcesConflictCountSQL) + } + } + for _, want := range []string{ + "vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''", + "ELSE vehicle_data_source.platform_name", + } { + if !strings.Contains(syncJT808DataSourcesSQL, want) { + t.Fatalf("sync sql missing %q:\n%s", want, syncJT808DataSourcesSQL) + } + } + for _, want := range []string{ + "source_kind IS NULL OR TRIM(source_kind) = '' OR source_kind = 'UNKNOWN'", + "source_code IS NOT NULL AND TRIM(source_code) <> ''", + "platform_name IS NOT NULL AND TRIM(platform_name) <> ''", + "SET source_kind = 'PLATFORM'", + } { + if !strings.Contains(classifyConfiguredDataSourcesSQL, want) && !strings.Contains(classifyConfiguredDataSourcesCountSQL, want) { + t.Fatalf("configured-source classification sql missing %q:\n%s\n%s", want, classifyConfiguredDataSourcesSQL, classifyConfiguredDataSourcesCountSQL) + } + } + for _, want := range []string{ + "SELECT DISTINCT s.vin, s.stat_date, s.protocol", + "vehicle_daily_mileage_source s", + "LEFT JOIN vehicle_data_source ds", + "LEFT JOIN vehicle_daily_mileage m", + "s.protocol = 'JT808'", + "s.quality_status = 'OK'", + "ds.enabled = 1", + "m.vin IS NULL", + "selected.is_selected = 1", + } { + if !strings.Contains(reprojectSelectableDailyMileageSourcesSQL, want) { + t.Fatalf("reproject sql missing %q:\n%s", want, reprojectSelectableDailyMileageSourcesSQL) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestSyncJT808DataSourcesDryRunDoesNotWrite(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(4)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(6)) + mock.ExpectQuery("SELECT DISTINCT s.vin, s.stat_date, s.protocol"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "stat_date", "protocol"}). + AddRow("LNXNEGRRXSR319449", "2026-07-13", "JT808")) + + report, err := syncJT808DataSourcesFromIdentifiers(context.Background(), db, false) + if err != nil { + t.Fatalf("syncJT808DataSourcesFromIdentifiers() error = %v", err) + } + if report.Apply || report.CandidateSources != 4 || report.SkippedSources != 2 || report.ConflictingSources != 1 || report.PlatformKindCandidates != 6 || report.PlatformKindClassified != 0 || report.Synced != 0 || report.ReprojectDailyMileageTargets != 1 || report.ReprojectedDailyMileageTargets != 0 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMaintenanceModeNamesAllSourceMaintenanceSteps(t *testing.T) { + tests := []struct { + name string + apply bool + sync bool + prune bool + retire bool + want string + }{ + {name: "dry run", want: "dry_run"}, + {name: "apply", apply: true, want: "apply"}, + {name: "sync dry run", sync: true, want: "sync_dry_run"}, + {name: "sync prune retire apply", apply: true, sync: true, prune: true, retire: true, want: "sync_prune_retire_apply"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := maintenanceMode(tt.apply, tt.sync, tt.prune, tt.retire); got != tt.want { + t.Fatalf("maintenanceMode() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestPruneUnmanagedDataSourcesDryRunDoesNotDelete(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs(int64(3600)). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(339)) + + report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, time.Hour) + if err != nil { + t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err) + } + if report.Apply || report.MinAgeSeconds != 3600 || report.CandidateSources != 339 || report.Pruned != 0 { + t.Fatalf("report = %#v", report) + } + for _, want := range []string{ + "vehicle_data_source ds", + "vehicle_daily_mileage m", + "m.source_id = ds.id", + "jt808_registration r", + "r.source_ip = ds.source_ip", + "ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'", + "TIMESTAMPDIFF(SECOND", + } { + if !strings.Contains(pruneUnmanagedDataSourcesCountSQL, want) { + t.Fatalf("prune count sql missing %q:\n%s", want, pruneUnmanagedDataSourcesCountSQL) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestRetireStaleUnmanagedDataSourcesDryRunDoesNotDisable(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs(int64(86400)). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(83)) + + report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, false, 24*time.Hour) + if err != nil { + t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err) + } + if report.Apply || report.MinAgeSeconds != 86400 || report.CandidateSources != 83 || report.Retired != 0 { + t.Fatalf("report = %#v", report) + } + for _, want := range []string{ + "vehicle_data_source ds", + "ds.enabled = 1", + "jt808_registration r", + "r.source_ip = ds.source_ip", + "ds.source_kind IS NULL OR TRIM(ds.source_kind) = '' OR ds.source_kind = 'UNKNOWN'", + "TIMESTAMPDIFF(SECOND", + } { + if !strings.Contains(retireStaleUnmanagedDataSourcesCountSQL, want) { + t.Fatalf("retire count sql missing %q:\n%s", want, retireStaleUnmanagedDataSourcesCountSQL) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestRetireStaleUnmanagedDataSourcesApplyDisablesOnlyCandidates(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs(int64(7200)). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(7)) + mock.ExpectExec("UPDATE vehicle_data_source ds"). + WithArgs(int64(7200)). + WillReturnResult(driver.RowsAffected(7)) + + report, err := retireStaleUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 2*time.Hour) + if err != nil { + t.Fatalf("retireStaleUnmanagedDataSourcesWithoutEvidence() error = %v", err) + } + if !report.Apply || report.MinAgeSeconds != 7200 || report.CandidateSources != 7 || report.Retired != 7 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestPruneUnmanagedDataSourcesApplyDeletesOnlyCandidates(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\)"). + WithArgs(int64(1800)). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(12)) + mock.ExpectExec("DELETE ds\\s+FROM vehicle_data_source ds"). + WithArgs(int64(1800)). + WillReturnResult(driver.RowsAffected(12)) + + report, err := pruneUnmanagedDataSourcesWithoutEvidence(context.Background(), db, true, 30*time.Minute) + if err != nil { + t.Fatalf("pruneUnmanagedDataSourcesWithoutEvidence() error = %v", err) + } + if !report.Apply || report.MinAgeSeconds != 1800 || report.CandidateSources != 12 || report.Pruned != 12 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} diff --git a/go/vehicle-gateway/cmd/identity-writer/main.go b/go/vehicle-gateway/cmd/identity-writer/main.go new file mode 100644 index 00000000..2c33fb61 --- /dev/null +++ b/go/vehicle-gateway/cmd/identity-writer/main.go @@ -0,0 +1,512 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" +) + +const identityBatchOperationTimeout = 30 * time.Second + +func main() { + logger := observability.NewLogger("vehicle-identity-writer") + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + cfg := loadConfig() + if err := cfg.Validate(); err != nil { + logger.Error("invalid identity writer config", "error", err) + os.Exit(1) + } + db, err := sql.Open("mysql", cfg.MySQLDSN) + if err != nil { + logger.Error("mysql open failed", "error", err) + os.Exit(1) + } + defer db.Close() + db.SetMaxOpenConns(cfg.MySQLMaxOpenConns) + db.SetMaxIdleConns(cfg.MySQLMaxIdleConns) + db.SetConnMaxLifetime(cfg.MySQLConnMaxLifetime) + if err := db.PingContext(ctx); err != nil { + logger.Error("mysql ping failed", "error", err) + os.Exit(1) + } + if cfg.EnsureSchema { + if err := identity.EnsureJT808RegistrationSchema(ctx, db); err != nil { + logger.Error("jt808 registration schema bootstrap failed", "error", err) + os.Exit(1) + } + } + + registry := metrics.NewRegistry() + metrics.RegisterKafkaConsumerInfo(registry, "vehicle-identity-writer", cfg.KafkaGroup, []string{cfg.KafkaTopic}) + registry.SetGauge("vehicle_identity_writer_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize)) + registry.SetGauge("vehicle_identity_writer_config", metrics.Labels{"setting": "batch_wait_ms"}, float64(cfg.BatchWait.Milliseconds())) + registry.SetGauge("vehicle_identity_writer_config", metrics.Labels{"setting": "location_touch_interval_seconds"}, cfg.LocationTouchInterval.Seconds()) + registry.SetGauge("vehicle_identity_writer_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers)) + health.Start(ctx, logger, health.NewServer(cfg.HealthAddr, "vehicle-identity-writer", []health.Check{ + {Name: "mysql", Check: db.PingContext}, + }, registry)) + + store := identity.NewJT808RegistrationStore(db) + + logger.Info("identity writer started", + "group", cfg.KafkaGroup, + "topic", cfg.KafkaTopic, + "workers", cfg.Workers, + "batch_size", cfg.BatchSize, + "batch_wait_ms", cfg.BatchWait.Milliseconds(), + "location_touch_interval_seconds", cfg.LocationTouchInterval.Seconds()) + var workers sync.WaitGroup + for workerID := 1; workerID <= cfg.Workers; workerID++ { + workers.Add(1) + go func(id int) { + defer workers.Done() + runIdentityConsumer(ctx, logger, registry, store, cfg, id) + }(workerID) + } + workers.Wait() +} + +func runIdentityConsumer( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + store registrationBatchStore, + cfg config, + workerID int, +) { + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: cfg.KafkaBrokers, + GroupID: cfg.KafkaGroup, + GroupTopics: []string{cfg.KafkaTopic}, + MinBytes: 1, + MaxBytes: 10e6, + StartOffset: cfg.StartOffset, + }) + defer reader.Close() + + // Kafka keeps one phone on one partition. Per-worker throttling avoids a + // global hot lock; a rebalance can only cause one harmless idempotent touch. + projector := identity.NewJT808RegistrationProjector(cfg.Location, cfg.LocationTouchInterval) + workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)} + registry.SetGauge("vehicle_identity_writer_worker_active", workerLabels, 1) + defer registry.SetGauge("vehicle_identity_writer_worker_active", workerLabels, 0) + + logger.Info("identity kafka consumer started", "worker", workerID, "topic", cfg.KafkaTopic) + for { + first, err := reader.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + return + } + logger.Error("kafka fetch failed", "error", err) + if !waitForRetry(ctx, cfg.RetryDelay) { + return + } + continue + } + batch := collectIdentityBatch(ctx, reader, first, cfg.BatchSize, cfg.BatchWait) + if !processIdentityBatchReliablyForWorker(ctx, logger, registry, projector, store, reader, batch, cfg.RetryDelay, workerLabels) { + return + } + } +} + +type registrationBatchStore interface { + UpsertBatch(context.Context, []identity.JT808RegistrationFact) error +} + +type registrationProjector interface { + ProjectBatch([]envelope.FrameEnvelope) []identity.JT808RegistrationFact + MarkPersisted([]identity.JT808RegistrationFact) +} + +type kafkaMessageFetcher interface { + FetchMessage(context.Context) (kafka.Message, error) +} + +type kafkaMessageCommitter interface { + CommitMessages(context.Context, ...kafka.Message) error +} + +const ( + identityFailureWrite = "write_error" + identityFailureCommit = "commit_error" +) + +type identityBatchFailure struct { + reason string + err error +} + +func (e *identityBatchFailure) Error() string { return e.err.Error() } +func (e *identityBatchFailure) Unwrap() error { return e.err } + +func collectIdentityBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { + if maxSize <= 1 { + return []kafka.Message{first} + } + if maxWait <= 0 { + maxWait = 20 * time.Millisecond + } + batch := []kafka.Message{first} + deadline := time.Now().Add(maxWait) + for len(batch) < maxSize { + remaining := time.Until(deadline) + if remaining <= 0 { + break + } + fetchCtx, cancel := context.WithTimeout(ctx, remaining) + message, err := fetcher.FetchMessage(fetchCtx) + cancel() + if err != nil { + break + } + batch = append(batch, message) + } + return batch +} + +func processIdentityBatch( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + projector registrationProjector, + store registrationBatchStore, + committer kafkaMessageCommitter, + messages []kafka.Message, +) error { + return processIdentityBatchAttemptForWorker(ctx, logger, registry, projector, store, committer, messages, true, nil) +} + +func processIdentityBatchAttempt( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + projector registrationProjector, + store registrationBatchStore, + committer kafkaMessageCommitter, + messages []kafka.Message, + recordReceived bool, +) error { + return processIdentityBatchAttemptForWorker(ctx, logger, registry, projector, store, committer, messages, recordReceived, nil) +} + +func processIdentityBatchAttemptForWorker( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + projector registrationProjector, + store registrationBatchStore, + committer kafkaMessageCommitter, + messages []kafka.Message, + recordReceived bool, + workerLabels metrics.Labels, +) error { + if len(messages) == 0 { + return nil + } + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), identityBatchOperationTimeout) + defer cancel() + + registry.SetGauge("vehicle_identity_writer_batch_pending_messages", workerLabels, float64(len(messages))) + defer registry.SetGauge("vehicle_identity_writer_batch_pending_messages", workerLabels, 0) + envelopes := make([]envelope.FrameEnvelope, 0, len(messages)) + for _, message := range messages { + if recordReceived { + recordIdentityMessage(registry, message, "received") + registry.SetKafkaLag("vehicle_identity_writer_kafka_lag", message.Topic, message.Partition, message.Offset, message.HighWaterMark) + } + var env envelope.FrameEnvelope + if err := json.Unmarshal(message.Value, &env); err != nil { + if recordReceived { + recordIdentityMessage(registry, message, "invalid_json") + logger.Warn("skip invalid identity raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + } + continue + } + if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { + if recordReceived { + recordIdentityMessage(registry, message, status) + logger.Warn("skip mismatched identity raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "error", err) + } + continue + } + envelopes = append(envelopes, env) + } + facts := projector.ProjectBatch(envelopes) + registry.SetGauge("vehicle_identity_writer_batch_pending_facts", workerLabels, float64(len(facts))) + defer registry.SetGauge("vehicle_identity_writer_batch_pending_facts", workerLabels, 0) + if len(facts) > 0 { + started := time.Now() + if err := store.UpsertBatch(operationCtx, facts); err != nil { + recordIdentityWrite(registry, "error", len(facts), time.Since(started)) + return &identityBatchFailure{reason: identityFailureWrite, err: fmt.Errorf("upsert jt808 registration facts: %w", err)} + } + projector.MarkPersisted(facts) + recordIdentityWrite(registry, "ok", len(facts), time.Since(started)) + } + if err := committer.CommitMessages(operationCtx, messages...); err != nil { + for _, message := range messages { + recordIdentityCommit(registry, message, "error") + } + return &identityBatchFailure{reason: identityFailureCommit, err: fmt.Errorf("commit identity kafka batch: %w", err)} + } + for _, message := range messages { + recordIdentityCommit(registry, message, "ok") + } + return nil +} + +func processIdentityBatchReliably( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + projector registrationProjector, + store registrationBatchStore, + committer kafkaMessageCommitter, + messages []kafka.Message, + retryDelay time.Duration, +) bool { + return processIdentityBatchReliablyForWorker(ctx, logger, registry, projector, store, committer, messages, retryDelay, nil) +} + +func processIdentityBatchReliablyForWorker( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + projector registrationProjector, + store registrationBatchStore, + committer kafkaMessageCommitter, + messages []kafka.Message, + retryDelay time.Duration, + workerLabels metrics.Labels, +) bool { + defer registry.SetGauge("vehicle_identity_writer_retry_pending_messages", workerLabels, 0) + recordReceived := true + for { + err := processIdentityBatchAttemptForWorker(ctx, logger, registry, projector, store, committer, messages, recordReceived, workerLabels) + if err == nil { + return true + } + if ctx.Err() != nil { + return false + } + reason := identityFailureReason(err) + registry.SetGauge("vehicle_identity_writer_retry_pending_messages", workerLabels, float64(len(messages))) + registry.IncCounter("vehicle_identity_writer_batch_retries_total", metrics.Labels{"reason": reason}) + logger.Error("identity batch failed; retrying without fetching newer offsets", "messages", len(messages), "reason", reason, "error", err) + if reason == identityFailureCommit { + return retryIdentityCommit(ctx, logger, registry, committer, messages, retryDelay) + } + if !waitForRetry(ctx, retryDelay) { + return false + } + recordReceived = false + } +} + +func retryIdentityCommit( + ctx context.Context, + logger *slog.Logger, + registry *metrics.Registry, + committer kafkaMessageCommitter, + messages []kafka.Message, + retryDelay time.Duration, +) bool { + for { + if !waitForRetry(ctx, retryDelay) { + return false + } + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), identityBatchOperationTimeout) + err := committer.CommitMessages(operationCtx, messages...) + cancel() + if err == nil { + for _, message := range messages { + recordIdentityCommit(registry, message, "ok") + } + return true + } + for _, message := range messages { + recordIdentityCommit(registry, message, "error") + } + registry.IncCounter("vehicle_identity_writer_batch_retries_total", metrics.Labels{"reason": identityFailureCommit}) + logger.Error("identity kafka commit retry failed", "messages", len(messages), "error", err) + } +} + +func identityFailureReason(err error) string { + var failure *identityBatchFailure + if errors.As(err, &failure) && failure.reason != "" { + return failure.reason + } + return identityFailureWrite +} + +var identityWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + +func recordIdentityMessage(registry *metrics.Registry, message kafka.Message, status string) { + labels := metrics.Labels{"topic": message.Topic, "status": status} + registry.IncCounter("vehicle_identity_writer_kafka_messages_total", labels) + metrics.RecordLastActivity(registry, "vehicle_identity_writer_last_message_unix_seconds", labels) +} + +func recordIdentityWrite(registry *metrics.Registry, status string, facts int, elapsed time.Duration) { + labels := metrics.Labels{"status": status} + registry.IncCounter("vehicle_identity_writer_batches_total", labels) + registry.AddCounter("vehicle_identity_writer_facts_total", labels, float64(facts)) + registry.ObserveHistogram("vehicle_identity_writer_write_duration_ms_histogram", labels, identityWriteDurationBucketsMS, float64(elapsed.Milliseconds())) + metrics.RecordLastActivity(registry, "vehicle_identity_writer_last_write_unix_seconds", labels) +} + +func recordIdentityCommit(registry *metrics.Registry, message kafka.Message, status string) { + labels := metrics.Labels{"topic": message.Topic, "status": status} + registry.IncCounter("vehicle_identity_writer_kafka_commits_total", labels) + metrics.RecordLastActivity(registry, "vehicle_identity_writer_last_commit_unix_seconds", labels) +} + +func waitForRetry(ctx context.Context, delay time.Duration) bool { + if delay <= 0 { + delay = time.Second + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +type config struct { + KafkaBrokers []string + KafkaTopic string + KafkaGroup string + MySQLDSN string + MySQLMaxOpenConns int + MySQLMaxIdleConns int + MySQLConnMaxLifetime time.Duration + EnsureSchema bool + HealthAddr string + Location *time.Location + LocationTouchInterval time.Duration + BatchSize int + BatchWait time.Duration + RetryDelay time.Duration + StartOffset int64 + Workers int +} + +func loadConfig() config { + location, err := time.LoadLocation(env("LOCAL_TZ", "Asia/Shanghai")) + if err != nil { + location = time.FixedZone("Asia/Shanghai", 8*60*60) + } + startOffset := kafka.LastOffset + if strings.EqualFold(env("KAFKA_START_OFFSET", "last"), "first") { + startOffset = kafka.FirstOffset + } + return config{ + KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), + KafkaTopic: env("KAFKA_TOPIC", topics.RawJT808), + KafkaGroup: env("KAFKA_GROUP", "go-identity-writer"), + MySQLDSN: env("MYSQL_DSN", ""), + MySQLMaxOpenConns: envInt("MYSQL_MAX_OPEN_CONNS", 8), + MySQLMaxIdleConns: envInt("MYSQL_MAX_IDLE_CONNS", 4), + MySQLConnMaxLifetime: time.Duration(envInt("MYSQL_CONN_MAX_LIFETIME_SECONDS", 300)) * time.Second, + EnsureSchema: envBool("MYSQL_ENSURE_SCHEMA", true), + HealthAddr: env("HEALTH_ADDR", "127.0.0.1:20217"), + Location: location, + LocationTouchInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS", 600)) * time.Second, + BatchSize: envInt("IDENTITY_WRITER_BATCH_SIZE", 500), + BatchWait: time.Duration(envInt("IDENTITY_WRITER_BATCH_WAIT_MS", 20)) * time.Millisecond, + RetryDelay: time.Duration(envInt("IDENTITY_WRITER_RETRY_DELAY_MS", 1000)) * time.Millisecond, + StartOffset: startOffset, + Workers: envInt("IDENTITY_WRITER_WORKERS", 3), + } +} + +func (c config) Validate() error { + if len(c.KafkaBrokers) == 0 { + return fmt.Errorf("KAFKA_BROKERS is required") + } + if strings.TrimSpace(c.MySQLDSN) == "" { + return fmt.Errorf("MYSQL_DSN is required") + } + protocol, ok := topics.ProtocolForKnownRawTopic(c.KafkaTopic) + if !ok || protocol != string(envelope.ProtocolJT808) { + return fmt.Errorf("identity writer consumes JT808 raw topic only, got %q", c.KafkaTopic) + } + if c.BatchSize <= 0 { + return fmt.Errorf("IDENTITY_WRITER_BATCH_SIZE must be positive") + } + if c.Workers <= 0 { + return fmt.Errorf("IDENTITY_WRITER_WORKERS must be positive") + } + return nil +} + +func env(key string, fallback string) string { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + return value +} + +func envInt(key string, fallback int) int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + +func envBool(key string, fallback bool) bool { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fallback + } + return parsed +} + +func splitCSV(value string) []string { + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, item := range parts { + if trimmed := strings.TrimSpace(item); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} diff --git a/go/vehicle-gateway/cmd/identity-writer/main_test.go b/go/vehicle-gateway/cmd/identity-writer/main_test.go new file mode 100644 index 00000000..ad5d63ba --- /dev/null +++ b/go/vehicle-gateway/cmd/identity-writer/main_test.go @@ -0,0 +1,314 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "strings" + "testing" + "time" + + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" +) + +type fakeRegistrationProjector struct { + facts []identity.JT808RegistrationFact + envelopes []envelope.FrameEnvelope + markedFacts []identity.JT808RegistrationFact +} + +func (p *fakeRegistrationProjector) ProjectBatch(envs []envelope.FrameEnvelope) []identity.JT808RegistrationFact { + p.envelopes = append(p.envelopes, envs...) + return p.facts +} + +func (p *fakeRegistrationProjector) MarkPersisted(facts []identity.JT808RegistrationFact) { + p.markedFacts = append(p.markedFacts, facts...) +} + +type fakeRegistrationStore struct { + facts []identity.JT808RegistrationFact + err error + count int + failOnCount int +} + +func (s *fakeRegistrationStore) UpsertBatch(_ context.Context, facts []identity.JT808RegistrationFact) error { + s.count++ + s.facts = append(s.facts, facts...) + if s.err != nil && (s.failOnCount == 0 || s.count == s.failOnCount) { + return s.err + } + return nil +} + +type fakeCommitter struct { + messages []kafka.Message + err error + count int + failOnCount int +} + +func (c *fakeCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error { + c.count++ + c.messages = append(c.messages, messages...) + if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) { + return c.err + } + return nil +} + +func TestProcessIdentityBatchDoesNotCommitOrThrottleOnStoreFailure(t *testing.T) { + wantErr := errors.New("mysql unavailable") + projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{{ + Phone: "13307795425", + SeenAt: time.Now(), + }}} + store := &fakeRegistrationStore{err: wantErr} + committer := &fakeCommitter{} + + err := processIdentityBatch( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + metrics.NewRegistry(), + projector, + store, + committer, + []kafka.Message{validIdentityMessage(t, 1)}, + ) + if !errors.Is(err, wantErr) { + t.Fatalf("processIdentityBatch() error = %v, want %v", err, wantErr) + } + if len(committer.messages) != 0 { + t.Fatalf("committed messages = %d, want 0", len(committer.messages)) + } + if len(projector.markedFacts) != 0 { + t.Fatalf("marked facts = %d, want 0", len(projector.markedFacts)) + } +} + +func TestProcessIdentityBatchMarksOnlyAfterStoreAndCommits(t *testing.T) { + fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} + projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} + store := &fakeRegistrationStore{} + committer := &fakeCommitter{} + messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)} + + err := processIdentityBatch( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + metrics.NewRegistry(), + projector, + store, + committer, + messages, + ) + if err != nil { + t.Fatalf("processIdentityBatch() error = %v", err) + } + if len(store.facts) != 1 || len(projector.markedFacts) != 1 { + t.Fatalf("store facts = %d marked facts = %d, want 1/1", len(store.facts), len(projector.markedFacts)) + } + if len(committer.messages) != len(messages) { + t.Fatalf("committed messages = %d, want %d", len(committer.messages), len(messages)) + } +} + +func TestProcessIdentityBatchCommitsPoisonEnvelopeWithoutProjection(t *testing.T) { + projector := &fakeRegistrationProjector{} + store := &fakeRegistrationStore{} + committer := &fakeCommitter{} + message := kafka.Message{Topic: topics.RawJT808, Partition: 1, Offset: 7, Value: []byte("not-json")} + + err := processIdentityBatch( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + metrics.NewRegistry(), + projector, + store, + committer, + []kafka.Message{message}, + ) + if err != nil { + t.Fatalf("processIdentityBatch() error = %v", err) + } + if len(store.facts) != 0 || len(projector.envelopes) != 0 { + t.Fatalf("poison message reached projector/store: envs=%d facts=%d", len(projector.envelopes), len(store.facts)) + } + if len(committer.messages) != 1 { + t.Fatalf("committed messages = %d, want 1", len(committer.messages)) + } +} + +func TestProcessIdentityBatchReliablyRetriesCommitWithoutRewritingMySQL(t *testing.T) { + fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} + projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} + store := &fakeRegistrationStore{} + committer := &fakeCommitter{err: errors.New("commit failed"), failOnCount: 1} + registry := metrics.NewRegistry() + messages := []kafka.Message{validIdentityMessage(t, 1), validIdentityMessage(t, 2)} + + ok := processIdentityBatchReliably( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + registry, + projector, + store, + committer, + messages, + time.Nanosecond, + ) + + if !ok { + t.Fatal("reliable identity batch returned false") + } + if store.count != 1 { + t.Fatalf("mysql writes = %d, want 1 after commit-only retry", store.count) + } + if committer.count != 2 { + t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_identity_writer_batch_retries_total{reason="commit_error"} 1`, + `vehicle_identity_writer_retry_pending_messages 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("missing metric %s:\n%s", want, text) + } + } +} + +func TestProcessIdentityBatchReliablyRetriesStoreBeforeCommit(t *testing.T) { + fact := identity.JT808RegistrationFact{Phone: "13307795425", SeenAt: time.Now()} + projector := &fakeRegistrationProjector{facts: []identity.JT808RegistrationFact{fact}} + store := &fakeRegistrationStore{err: errors.New("mysql unavailable"), failOnCount: 1} + committer := &fakeCommitter{} + registry := metrics.NewRegistry() + messages := []kafka.Message{validIdentityMessage(t, 1)} + + ok := processIdentityBatchReliably( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + registry, + projector, + store, + committer, + messages, + time.Nanosecond, + ) + + if !ok { + t.Fatal("reliable identity batch returned false") + } + if store.count != 2 || committer.count != 1 { + t.Fatalf("mysql writes=%d commits=%d, want 2/1", store.count, committer.count) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_identity_writer_batch_retries_total{reason="write_error"} 1`, + `vehicle_identity_writer_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("missing metric %s:\n%s", want, text) + } + } +} + +func TestConfigRejectsNonJT808Topic(t *testing.T) { + cfg := config{ + KafkaBrokers: []string{"127.0.0.1:9092"}, + KafkaTopic: topics.RawGB32960, + MySQLDSN: "user:pass@tcp(localhost:3306)/db", + BatchSize: 100, + Workers: 3, + } + if err := cfg.Validate(); err == nil { + t.Fatal("Validate() error = nil, want non-JT808 topic error") + } + cfg.KafkaTopic = topics.RawJT808 + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } +} + +func TestLoadConfigDefaultsAndOverridesWorkers(t *testing.T) { + t.Setenv("IDENTITY_WRITER_WORKERS", "") + if got := loadConfig().Workers; got != 3 { + t.Fatalf("default workers = %d, want 3", got) + } + t.Setenv("IDENTITY_WRITER_WORKERS", "5") + if got := loadConfig().Workers; got != 5 { + t.Fatalf("configured workers = %d, want 5", got) + } +} + +func TestConfigRejectsNonPositiveWorkers(t *testing.T) { + cfg := config{ + KafkaBrokers: []string{"127.0.0.1:9092"}, + KafkaTopic: topics.RawJT808, + MySQLDSN: "user:pass@tcp(localhost:3306)/db", + BatchSize: 100, + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDENTITY_WRITER_WORKERS") { + t.Fatalf("Validate() error = %v, want workers error", err) + } +} + +func TestProcessIdentityBatchForWorkerLabelsPendingMetrics(t *testing.T) { + registry := metrics.NewRegistry() + workerLabels := metrics.Labels{"worker": "2"} + err := processIdentityBatchAttemptForWorker( + context.Background(), + slog.New(slog.NewTextHandler(io.Discard, nil)), + registry, + &fakeRegistrationProjector{}, + &fakeRegistrationStore{}, + &fakeCommitter{}, + []kafka.Message{validIdentityMessage(t, 1)}, + true, + workerLabels, + ) + if err != nil { + t.Fatalf("processIdentityBatchAttemptForWorker() error = %v", err) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_identity_writer_batch_pending_messages{worker="2"} 0`, + `vehicle_identity_writer_batch_pending_facts{worker="2"} 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("missing metric %s:\n%s", want, text) + } + } +} + +func validIdentityMessage(t *testing.T, offset int64) kafka.Message { + t.Helper() + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: identity.JT808LocationMessageID, + EventKind: envelope.EventKindRaw, + Phone: "13307795425", + VIN: "LTESTVIN000000001", + ReceivedAtMS: time.Now().UnixMilli(), + ParseStatus: envelope.ParseOK, + } + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + return kafka.Message{ + Topic: topics.RawJT808, + Partition: 1, + Offset: offset, + HighWaterMark: offset + 1, + Value: payload, + } +} diff --git a/go/vehicle-gateway/cmd/load-sim/main.go b/go/vehicle-gateway/cmd/load-sim/main.go index 5d2c90c7..561024a1 100644 --- a/go/vehicle-gateway/cmd/load-sim/main.go +++ b/go/vehicle-gateway/cmd/load-sim/main.go @@ -2,19 +2,26 @@ package main import ( "context" + "database/sql" "flag" "fmt" "log" "os" "os/signal" + "strings" "syscall" + _ "github.com/go-sql-driver/mysql" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/loadsim" ) func main() { log.SetFlags(log.LstdFlags | log.Lmicroseconds) flagCfg := loadsim.RegisterFlags(flag.CommandLine) + cleanupRegistration := flag.Bool("cleanup-registration", false, "delete this run's loopback JT808 registration rows after the simulation") + cleanupOnly := flag.Bool("cleanup-only", false, "skip the simulation and only clean the configured JT808 phone range") + mysqlDSN := flag.String("mysql-dsn", strings.TrimSpace(os.Getenv("MYSQL_DSN")), "MySQL DSN used only by JT808 registration cleanup") if err := flag.CommandLine.Parse(os.Args[1:]); err != nil { log.Fatal(err) } @@ -25,6 +32,14 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + if *cleanupOnly { + deleted, err := cleanupJT808Registrations(ctx, cfg, *mysqlDSN) + if err != nil { + log.Fatal(err) + } + log.Printf("jt808 synthetic registration cleanup completed rows_deleted=%d", deleted) + return + } log.Printf("load simulation started protocol=%s addr=%s connections=%d connect_rate=%d send_interval=%s duration=%s template=%s", cfg.Protocol, cfg.Addr, cfg.Connections, cfg.ConnectRatePerSecond, cfg.SendInterval, cfg.Duration, cfg.Template) @@ -33,13 +48,57 @@ func main() { log.Fatal(err) } log.Print(formatStats(stats)) + if *cleanupRegistration { + deleted, err := cleanupJT808Registrations(ctx, cfg, *mysqlDSN) + if err != nil { + log.Fatal(err) + } + log.Printf("jt808 synthetic registration cleanup completed rows_deleted=%d", deleted) + } +} + +func cleanupJT808Registrations(ctx context.Context, cfg loadsim.Config, dsn string) (int64, error) { + if cfg.Protocol != loadsim.ProtocolJT808 { + return 0, fmt.Errorf("registration cleanup only supports jt808") + } + if strings.TrimSpace(dsn) == "" { + return 0, fmt.Errorf("mysql-dsn or MYSQL_DSN is required for registration cleanup") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + return 0, fmt.Errorf("open mysql for registration cleanup: %w", err) + } + defer db.Close() + return cleanupJT808RegistrationsWithDB(ctx, db, cfg) +} + +type cleanupExecer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func cleanupJT808RegistrationsWithDB(ctx context.Context, exec cleanupExecer, cfg loadsim.Config) (int64, error) { + firstPhone := fmt.Sprintf("%012d", cfg.JT808PhoneBase) + lastPhone := fmt.Sprintf("%012d", cfg.JT808PhoneBase+int64(cfg.Connections)-1) + result, err := exec.ExecContext(ctx, `DELETE FROM jt808_registration +WHERE phone BETWEEN ? AND ? + AND source_ip IN ('127.0.0.1', '::1')`, firstPhone, lastPhone) + if err != nil { + return 0, fmt.Errorf("delete loopback jt808 registrations: %w", err) + } + deleted, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("read registration cleanup result: %w", err) + } + return deleted, nil } func formatStats(stats loadsim.Stats) string { - return fmt.Sprintf("connections_opened=%d connections_failed=%d frames_written=%d write_errors=%d", + return fmt.Sprintf("connections_opened=%d connections_failed=%d frames_written=%d write_errors=%d response_bytes=%d read_errors=%d", stats.ConnectionsOpened, stats.ConnectionsFailed, stats.FramesWritten, stats.WriteErrors, + stats.ResponseBytes, + stats.ReadErrors, ) } diff --git a/go/vehicle-gateway/cmd/load-sim/main_test.go b/go/vehicle-gateway/cmd/load-sim/main_test.go index b41ead55..52bcfeb8 100644 --- a/go/vehicle-gateway/cmd/load-sim/main_test.go +++ b/go/vehicle-gateway/cmd/load-sim/main_test.go @@ -1,9 +1,12 @@ package main import ( + "context" "strings" "testing" + "github.com/DATA-DOG/go-sqlmock" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/loadsim" ) @@ -13,6 +16,8 @@ func TestFormatStatsIncludesCapacityCounters(t *testing.T) { ConnectionsFailed: 2, FramesWritten: 300, WriteErrors: 1, + ResponseBytes: 2048, + ReadErrors: 0, }) for _, want := range []string{ @@ -20,9 +25,37 @@ func TestFormatStatsIncludesCapacityCounters(t *testing.T) { "connections_failed=2", "frames_written=300", "write_errors=1", + "response_bytes=2048", + "read_errors=0", } { if !strings.Contains(out, want) { t.Fatalf("formatStats() = %q, missing %q", out, want) } } } + +func TestCleanupJT808RegistrationsUsesBoundedLoopbackDelete(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectExec(`DELETE FROM jt808_registration`). + WithArgs("139000000000", "139000000999"). + WillReturnResult(sqlmock.NewResult(0, 1000)) + + deleted, err := cleanupJT808RegistrationsWithDB(context.Background(), db, loadsim.Config{ + Protocol: loadsim.ProtocolJT808, + Connections: 1000, + JT808PhoneBase: loadsim.DefaultJT808PhoneBase, + }) + if err != nil { + t.Fatalf("cleanupJT808RegistrationsWithDB() error = %v", err) + } + if deleted != 1000 { + t.Fatalf("deleted = %d, want 1000", deleted) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main.go b/go/vehicle-gateway/cmd/nats-fast-writer/main.go index eb5afb39..9ca08a5f 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main.go @@ -60,24 +60,32 @@ func main() { os.Exit(1) } - tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN) - if err != nil { - logger.Error("tdengine open failed", "error", err) - os.Exit(1) - } - defer tdDB.Close() - tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns) - tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns) - if err := tdDB.PingContext(ctx); err != nil { - logger.Error("tdengine ping failed", "error", err) - os.Exit(1) - } - historyWriter := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase) - if cfg.TDengineEnsureSchema { - if err := historyWriter.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil { - logger.Error("tdengine schema bootstrap failed", "error", err) + var historyWriter fastAppender + var tdCheck health.Check + if cfg.TDengineEnabled { + tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN) + if err != nil { + logger.Error("tdengine open failed", "error", err) os.Exit(1) } + defer tdDB.Close() + tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns) + tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns) + if err := tdDB.PingContext(ctx); err != nil { + logger.Error("tdengine ping failed", "error", err) + os.Exit(1) + } + writer := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase) + if cfg.TDengineEnsureSchema { + if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil { + logger.Error("tdengine schema bootstrap failed", "error", err) + os.Exit(1) + } + } + historyWriter = writer + tdCheck = health.Check{Name: "tdengine", Check: tdDB.PingContext} + } else { + logger.Info("nats fast writer tdengine stage disabled") } redisClient := redis.NewClient(&redis.Options{ @@ -94,23 +102,30 @@ func main() { realtimeRepo := realtime.NewRepository(redisClient, realtime.Config{OnlineTTL: cfg.OnlineTTL}) registry := metrics.NewRegistry() - health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", []health.Check{ + recordFastWriterConfigMetrics(registry, cfg) + registry.SetGauge("vehicle_fast_writer_tdengine_enabled", nil, boolGauge(cfg.TDengineEnabled)) + healthChecks := []health.Check{ {Name: "nats", Check: func(context.Context) error { if conn.Status() != nats.CONNECTED { return fmt.Errorf("nats status is %s", conn.Status().String()) } return nil }}, - {Name: "tdengine", Check: tdDB.PingContext}, {Name: "redis", Check: func(ctx context.Context) error { return redisClient.Ping(ctx).Err() }}, - }, registry)) + } + if tdCheck.Name != "" { + healthChecks = append(healthChecks, tdCheck) + } + health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", healthChecks, registry)) logger.Info("nats fast writer started", "stream", cfg.NATSStream, "durable", cfg.NATSDurable, "filter", cfg.NATSFilter, "batch_size", cfg.BatchSize, + "fetch_wait_ms", cfg.FetchWait.Milliseconds(), "workers", cfg.Workers, + "tdengine_enabled", cfg.TDengineEnabled, "tdengine_max_open_conns", cfg.TDengineMaxOpenConns, "tdengine_max_idle_conns", cfg.TDengineMaxIdleConns, "operation_timeout_ms", cfg.OperationWait.Milliseconds()) @@ -135,6 +150,7 @@ type config struct { StreamMaxBytes int64 StreamEnsureWait time.Duration Workers int + TDengineEnabled bool TDengineDriver string TDengineDSN string TDengineDatabase string @@ -168,13 +184,14 @@ func loadConfig() config { NATSFilter: env("NATS_FILTER", "vehicle.raw.go.>"), NATSSubjects: subjects, BatchSize: envInt("FAST_WRITER_BATCH_SIZE", 100), - FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 100)) * time.Millisecond, + FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 20)) * time.Millisecond, OperationWait: time.Duration(envInt("FAST_WRITER_OPERATION_TIMEOUT_MS", 1000)) * time.Millisecond, AckWait: time.Duration(envInt("NATS_ACK_WAIT_SECONDS", 30)) * time.Second, StreamMaxAge: time.Duration(envInt("NATS_STREAM_MAX_AGE_HOURS", 24)) * time.Hour, StreamMaxBytes: envInt64("NATS_STREAM_MAX_BYTES", 20*1024*1024*1024), StreamEnsureWait: time.Duration(envInt("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", 60)) * time.Second, Workers: workers, + TDengineEnabled: envBool("FAST_WRITER_TDENGINE_ENABLED", false), TDengineDriver: env("TDENGINE_DRIVER", "taosWS"), TDengineDSN: env("TDENGINE_DSN", ""), TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase), @@ -198,10 +215,18 @@ type fastUpdater interface { FastUpdate(context.Context, envelope.FrameEnvelope) error } +type fastResultUpdater interface { + FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error) +} + type fastBatchUpdater interface { FastUpdateBatch(context.Context, []envelope.FrameEnvelope) error } +type fastBatchResultUpdater interface { + FastUpdateBatchWithResult(context.Context, []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) +} + type fastMessage struct { subject string data []byte @@ -227,9 +252,17 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R } msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait)) if err != nil { + if isFastWorkerShutdownFetchError(ctx, err) { + return + } if errors.Is(err, nats.ErrTimeout) { continue } + if isTransientFastFetchError(err) { + logger.Warn("nats fetch interrupted", "error", err) + time.Sleep(time.Second) + continue + } logger.Error("nats fetch failed", "error", err) time.Sleep(time.Second) continue @@ -253,12 +286,33 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R logger.Error("fast write batch failed", "messages", len(fastMessages), "error", err) continue } - for _, msg := range fastMessages { - addFastMetric(registry, msg.subject, "ok") - } } } +func isFastWorkerShutdownFetchError(ctx context.Context, err error) bool { + if err == nil { + return false + } + if ctx.Err() != nil { + return true + } + return errors.Is(err, nats.ErrConnectionClosed) +} + +func isTransientFastFetchError(err error) bool { + if err == nil { + return false + } + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "disconnected during fetch") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "timeout") +} + type natsPullSubscription interface { Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) } @@ -268,18 +322,44 @@ type natsConsumerInfoReader interface { } var fastWriterStageDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var fastWriterRedisE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000} +var fastWriterRedisE2ERecent = metrics.NewRecentLatencyByKey(512) +var fastBatchPending = metrics.PendingPairGauge{} func processFastBatch(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error { if len(messages) == 0 { return nil } + for _, msg := range messages { + addFastMetric(registry, msg.subject, "received") + } envelopes := make([]envelope.FrameEnvelope, 0, len(messages)) validMessages := make([]*fastMessage, 0, len(messages)) for _, msg := range messages { var env envelope.FrameEnvelope if err := json.Unmarshal(msg.data, &env); err != nil { + addFastMetric(registry, msg.subject, "invalid_json") if msg.ack != nil { - _ = msg.ack() + started := time.Now() + err := msg.ack() + recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) + if err != nil { + addFastMetric(registry, msg.subject, "ack_error") + return fmt.Errorf("nats ack invalid json: %w", err) + } + } + continue + } + if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil { + addFastMetric(registry, msg.subject, status) + if msg.ack != nil { + started := time.Now() + err := msg.ack() + recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) + if err != nil { + addFastMetric(registry, msg.subject, "ack_error") + return fmt.Errorf("nats ack mismatched raw envelope: %w", err) + } } continue } @@ -289,85 +369,358 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender if len(envelopes) == 0 { return nil } - setFastBatchPending(registry, len(messages), len(envelopes)) - defer setFastBatchPending(registry, 0, 0) + addFastBatchPending(registry, len(messages), len(envelopes)) + defer addFastBatchPending(registry, -len(messages), -len(envelopes)) subject := fastBatchSubject(validMessages) - started := time.Now() - err := appender.AppendAllBatch(ctx, envelopes) - recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started)) - if err != nil { - return fmt.Errorf("tdengine batch append: %w", err) - } - if batchUpdater, ok := updater.(fastBatchUpdater); ok { - started = time.Now() - err = batchUpdater.FastUpdateBatch(ctx, envelopes) - recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started)) + if appender != nil { + started := time.Now() + err := appender.AppendAllBatch(ctx, envelopes) + recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started)) if err != nil { - return fmt.Errorf("redis fast batch update: %w", err) + if shouldFallbackFastBatchError(err) { + if fallbackErr := processFastMessagesIndividually(ctx, registry, appender, updater, validMessages); fallbackErr != nil { + addFastBatchFallbackMetric(registry, "tdengine", "error") + return fmt.Errorf("tdengine batch fallback: %w", fallbackErr) + } + addFastBatchFallbackMetric(registry, "tdengine", "ok") + return nil + } + addFastBatchFallbackMetric(registry, "tdengine", "skipped_transient") + if catchupErr := updateFastRedisBatchWithoutAck(ctx, registry, updater, validMessages, envelopes); catchupErr != nil { + addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error") + return fmt.Errorf("tdengine batch append: %w; redis catchup: %v", err, catchupErr) + } + addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok") + return fmt.Errorf("tdengine batch append: %w", err) + } + } + if batchUpdater, ok := updater.(fastBatchResultUpdater); ok { + for _, group := range fastSubjectGroups(validMessages, envelopes) { + started := time.Now() + result, err := batchUpdater.FastUpdateBatchWithResult(ctx, group.envelopes) + recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started)) + if err != nil { + if shouldFallbackFastBatchError(err) { + if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil { + addFastBatchFallbackMetric(registry, "redis", "error") + return fmt.Errorf("redis fast batch fallback: %w", fallbackErr) + } + addFastBatchFallbackMetric(registry, "redis", "ok") + return nil + } + addFastBatchFallbackMetric(registry, "redis", "skipped_transient") + return fmt.Errorf("redis fast batch update: %w", err) + } + recordFastWriterRedisEnvelopeMetrics(registry, group.subject, result) + recordFastWriterRedisFieldMetrics(registry, group.subject, result) + recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject) } for _, msg := range validMessages { if msg.ack != nil { - started = time.Now() - err = msg.ack() + started := time.Now() + err := msg.ack() recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) if err != nil { + addFastMetric(registry, msg.subject, "ack_error") return fmt.Errorf("nats ack: %w", err) } } + addFastMetric(registry, msg.subject, "ok") + } + return nil + } + if batchUpdater, ok := updater.(fastBatchUpdater); ok { + for _, group := range fastSubjectGroups(validMessages, envelopes) { + started := time.Now() + err := batchUpdater.FastUpdateBatch(ctx, group.envelopes) + recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started)) + if err != nil { + if shouldFallbackFastBatchError(err) { + if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil { + addFastBatchFallbackMetric(registry, "redis", "error") + return fmt.Errorf("redis fast batch fallback: %w", fallbackErr) + } + addFastBatchFallbackMetric(registry, "redis", "ok") + return nil + } + addFastBatchFallbackMetric(registry, "redis", "skipped_transient") + return fmt.Errorf("redis fast batch update: %w", err) + } + recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject) + } + for _, msg := range validMessages { + if msg.ack != nil { + started := time.Now() + err := msg.ack() + recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) + if err != nil { + addFastMetric(registry, msg.subject, "ack_error") + return fmt.Errorf("nats ack: %w", err) + } + } + addFastMetric(registry, msg.subject, "ok") } return nil } for i, env := range envelopes { msg := validMessages[i] - started = time.Now() - err = updater.FastUpdate(ctx, env) + started := time.Now() + var result realtime.FastUpdateResult + var err error + if resultUpdater, ok := updater.(fastResultUpdater); ok { + result, err = resultUpdater.FastUpdateWithResult(ctx, env) + } else { + err = updater.FastUpdate(ctx, env) + } recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started)) if err != nil { return fmt.Errorf("redis fast update: %w", err) } + recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result) + recordFastWriterRedisFieldMetrics(registry, msg.subject, result) + recordFastWriterRedisE2EDuration(registry, msg.subject, env) if msg.ack != nil { - started = time.Now() - err = msg.ack() + started := time.Now() + err := msg.ack() recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) if err != nil { + addFastMetric(registry, msg.subject, "ack_error") return fmt.Errorf("nats ack: %w", err) } } + addFastMetric(registry, msg.subject, "ok") + } + return nil +} + +func processFastMessagesIndividually(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error { + for _, msg := range messages { + if err := processFastMessageWithReceived(ctx, registry, appender, updater, msg, false); err != nil { + return err + } } return nil } func processFastMessage(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage) error { + return processFastMessageWithReceived(ctx, registry, appender, updater, msg, true) +} + +func processFastMessageWithReceived(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage, recordReceived bool) error { + if recordReceived { + addFastMetric(registry, msg.subject, "received") + } var env envelope.FrameEnvelope if err := json.Unmarshal(msg.data, &env); err != nil { + addFastMetric(registry, msg.subject, "invalid_json") if msg.ack != nil { - _ = msg.ack() + started := time.Now() + err := msg.ack() + recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) + if err != nil { + addFastMetric(registry, msg.subject, "ack_error") + return fmt.Errorf("nats ack invalid json: %w", err) + } } return nil } - started := time.Now() - err := appender.AppendAll(ctx, env) - recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started)) - if err != nil { - return fmt.Errorf("tdengine append: %w", err) + if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil { + addFastMetric(registry, msg.subject, status) + if msg.ack != nil { + started := time.Now() + err := msg.ack() + recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) + if err != nil { + addFastMetric(registry, msg.subject, "ack_error") + return fmt.Errorf("nats ack mismatched raw envelope: %w", err) + } + } + return nil + } + if appender != nil { + started := time.Now() + err := appender.AppendAll(ctx, env) + recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started)) + if err != nil { + if isTransientFastBatchError(err) { + if catchupErr := updateFastRedisSingleWithoutAck(ctx, registry, updater, msg.subject, env); catchupErr != nil { + addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error") + return fmt.Errorf("tdengine append: %w; redis catchup: %v", err, catchupErr) + } + addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok") + } + return fmt.Errorf("tdengine append: %w", err) + } + } + started := time.Now() + var result realtime.FastUpdateResult + var err error + if resultUpdater, ok := updater.(fastResultUpdater); ok { + result, err = resultUpdater.FastUpdateWithResult(ctx, env) + } else { + err = updater.FastUpdate(ctx, env) } - started = time.Now() - err = updater.FastUpdate(ctx, env) recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started)) if err != nil { return fmt.Errorf("redis fast update: %w", err) } + recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result) + recordFastWriterRedisFieldMetrics(registry, msg.subject, result) + recordFastWriterRedisE2EDuration(registry, msg.subject, env) if msg.ack != nil { started = time.Now() err = msg.ack() recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started)) if err != nil { + addFastMetric(registry, msg.subject, "ack_error") return fmt.Errorf("nats ack: %w", err) } } + addFastMetric(registry, msg.subject, "ok") return nil } +func updateFastRedisBatchWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error { + if len(envelopes) == 0 { + return nil + } + subject := fastBatchSubject(messages) + if batchUpdater, ok := updater.(fastBatchResultUpdater); ok { + started := time.Now() + result, err := batchUpdater.FastUpdateBatchWithResult(ctx, envelopes) + recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started)) + if err == nil { + recordFastWriterRedisEnvelopeMetrics(registry, subject, result) + recordFastWriterRedisFieldMetrics(registry, subject, result) + recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject) + return nil + } + if shouldFallbackFastBatchError(err) { + addFastBatchFallbackMetric(registry, "redis_catchup", "attempted") + return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes) + } + return err + } + if batchUpdater, ok := updater.(fastBatchUpdater); ok { + started := time.Now() + err := batchUpdater.FastUpdateBatch(ctx, envelopes) + recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started)) + if err == nil { + recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject) + return nil + } + if shouldFallbackFastBatchError(err) { + addFastBatchFallbackMetric(registry, "redis_catchup", "attempted") + return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes) + } + return err + } + return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes) +} + +func updateFastRedisSinglesWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error { + for index, env := range envelopes { + subject := fastMessageSubject(messages, index, fastBatchSubject(messages)) + if err := updateFastRedisSingleWithoutAck(ctx, registry, updater, subject, env); err != nil { + return err + } + } + return nil +} + +func updateFastRedisSingleWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, subject string, env envelope.FrameEnvelope) error { + started := time.Now() + var result realtime.FastUpdateResult + var err error + if resultUpdater, ok := updater.(fastResultUpdater); ok { + result, err = resultUpdater.FastUpdateWithResult(ctx, env) + } else { + err = updater.FastUpdate(ctx, env) + } + recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started)) + if err != nil { + return err + } + recordFastWriterRedisEnvelopeMetrics(registry, subject, result) + recordFastWriterRedisFieldMetrics(registry, subject, result) + recordFastWriterRedisE2EDuration(registry, subject, env) + return nil +} + +func shouldFallbackFastBatchError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + return !isTransientFastBatchError(err) +} + +func isTransientFastBatchError(err error) bool { + if err == nil { + return false + } + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "timeout") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "connection refused") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "bad connection") || + strings.Contains(text, "i/o timeout") || + text == "eof" || + strings.Contains(text, "unexpected eof") || + strings.Contains(text, "server is down") || + strings.Contains(text, "network is unreachable") || + strings.Contains(text, "no route to host") +} + +func recordFastWriterConfigMetrics(registry *metrics.Registry, cfg config) { + if registry == nil { + return + } + registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers)) + registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize)) + registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "fetch_wait_ms"}, float64(cfg.FetchWait.Milliseconds())) + registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationWait.Milliseconds())) +} + +func boolGauge(value bool) float64 { + if value { + return 1 + } + return 0 +} + +type fastSubjectGroup struct { + subject string + messages []*fastMessage + envelopes []envelope.FrameEnvelope +} + +func fastSubjectGroups(messages []*fastMessage, envelopes []envelope.FrameEnvelope) []fastSubjectGroup { + if len(envelopes) == 0 { + return nil + } + fallback := fastBatchSubject(messages) + groups := make([]fastSubjectGroup, 0, len(envelopes)) + indexBySubject := make(map[string]int, len(envelopes)) + for index, env := range envelopes { + subject := fastMessageSubject(messages, index, fallback) + groupIndex, exists := indexBySubject[subject] + if !exists { + groupIndex = len(groups) + indexBySubject[subject] = groupIndex + groups = append(groups, fastSubjectGroup{subject: subject}) + } + groups[groupIndex].envelopes = append(groups[groupIndex].envelopes, env) + if index >= 0 && index < len(messages) { + groups[groupIndex].messages = append(groups[groupIndex].messages, messages[index]) + } + } + return groups +} + func fastBatchSubject(messages []*fastMessage) string { if len(messages) == 0 { return "unknown" @@ -384,6 +737,16 @@ func fastBatchSubject(messages []*fastMessage) string { return subject } +func fastMessageSubject(messages []*fastMessage, index int, fallback string) string { + if index >= 0 && index < len(messages) && strings.TrimSpace(messages[index].subject) != "" { + return messages[index].subject + } + if strings.TrimSpace(fallback) != "" { + return fallback + } + return "unknown" +} + func ensureStream(js nats.JetStreamContext, cfg config) error { ctx, cancel := context.WithTimeout(context.Background(), cfg.StreamEnsureWait) defer cancel() @@ -416,15 +779,36 @@ func addFastMetric(registry *metrics.Registry, subject string, status string) { if registry == nil { return } - registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status}) + labels := metrics.Labels{"subject": subject, "status": status} + registry.IncCounter("vehicle_fast_writer_messages_total", labels) + metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_message_unix_seconds", labels) } -func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int) { +func addFastBatchFallbackMetric(registry *metrics.Registry, stage string, status string) { if registry == nil { return } - registry.SetGauge("vehicle_fast_writer_batch_pending_messages", nil, float64(messages)) - registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes)) + registry.IncCounter("vehicle_fast_writer_batch_fallback_total", metrics.Labels{ + "stage": stage, + "status": status, + }) +} + +func addFastDecoupledUpdateMetric(registry *metrics.Registry, reason string, status string) { + if registry == nil { + return + } + registry.IncCounter("vehicle_fast_writer_decoupled_updates_total", metrics.Labels{ + "reason": reason, + "status": status, + }) +} + +func addFastBatchPending(registry *metrics.Registry, messages int, envelopes int) { + if registry == nil { + return + } + fastBatchPending.Add(registry, "vehicle_fast_writer_batch_pending_messages", "vehicle_fast_writer_batch_pending_envelopes", messages, envelopes) } func recordFastNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) { @@ -441,11 +825,78 @@ func recordFastWriterStageDuration(registry *metrics.Registry, subject string, s if registry == nil { return } - registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", metrics.Labels{ + labels := metrics.Labels{ "subject": subject, "stage": stage, "status": status, - }, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds())) + } + registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", labels, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds())) + metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_stage_unix_seconds", labels) +} + +func recordFastWriterRedisE2EDurationMessages(registry *metrics.Registry, messages []*fastMessage, envelopes []envelope.FrameEnvelope, fallback string) { + for index, env := range envelopes { + recordFastWriterRedisE2EDuration(registry, fastMessageSubject(messages, index, fallback), env) + } +} + +func recordFastWriterRedisE2EDuration(registry *metrics.Registry, subject string, env envelope.FrameEnvelope) { + if registry == nil || env.ReceivedAtMS <= 0 { + return + } + elapsedMS := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds() + if elapsedMS < 0 { + elapsedMS = 0 + } + labels := metrics.Labels{ + "subject": subject, + } + registry.ObserveHistogram("vehicle_fast_writer_redis_e2e_duration_ms_histogram", labels, fastWriterRedisE2EDurationBucketsMS, float64(elapsedMS)) + p99, samples := fastWriterRedisE2ERecent.Observe(subject, float64(elapsedMS)) + registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_p99_ms", labels, p99) + registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_samples", labels, float64(samples)) + metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_redis_e2e_unix_seconds", labels) +} + +func recordFastWriterRedisFieldMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) { + if registry == nil || result.FieldsSeen == 0 { + return + } + addFastWriterRedisFieldMetric(registry, subject, "seen", result.FieldsSeen) + addFastWriterRedisFieldMetric(registry, subject, "written", result.FieldsWritten) + addFastWriterRedisFieldMetric(registry, subject, "skipped_stale", result.FieldsSkippedStale) +} + +func recordFastWriterRedisEnvelopeMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) { + if registry == nil || result.EnvelopesSeen == 0 { + return + } + addFastWriterRedisEnvelopeMetric(registry, subject, "seen", result.EnvelopesSeen) + addFastWriterRedisEnvelopeMetric(registry, subject, "updated", result.EnvelopesUpdated) + addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_non_realtime", result.EnvelopesSkippedNonRealtime) + addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vin", result.EnvelopesSkippedMissingVIN) + addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vehicle_key", result.EnvelopesSkippedMissingVehicleKey) + addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_fields", result.EnvelopesSkippedMissingFields) +} + +func addFastWriterRedisEnvelopeMetric(registry *metrics.Registry, subject string, status string, value int) { + if value <= 0 { + return + } + registry.AddCounter("vehicle_fast_writer_redis_envelopes_total", metrics.Labels{ + "subject": subject, + "status": status, + }, float64(value)) +} + +func addFastWriterRedisFieldMetric(registry *metrics.Registry, subject string, status string, value int) { + if value <= 0 { + return + } + registry.AddCounter("vehicle_fast_writer_redis_fields_total", metrics.Labels{ + "subject": subject, + "status": status, + }, float64(value)) } func statusFromError(err error) string { @@ -487,6 +938,20 @@ func envInt64(key string, fallback int64) int64 { return parsed } +func envBool(key string, fallback bool) bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(key))) + switch value { + case "": + return fallback + case "1", "true", "yes", "y", "on": + return true + case "0", "false", "no", "n", "off": + return false + default: + return fallback + } +} + func splitCSV(value string) []string { var out []string for _, item := range strings.Split(value, ",") { diff --git a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go index cdf15c65..1c0e7021 100644 --- a/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go +++ b/go/vehicle-gateway/cmd/nats-fast-writer/main_test.go @@ -12,6 +12,7 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" ) func TestProcessFastMessageWritesTDengineAndRedisBeforeAck(t *testing.T) { @@ -58,8 +59,45 @@ func TestProcessFastMessageDoesNotAckWhenRedisFails(t *testing.T) { } } +func TestProcessFastMessageUpdatesRedisButDoesNotAckOnTransientTDengineFailure(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-td-transient-single"} + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{err: errors.New("connection reset by peer")} + updater := &recordingFastSingleUpdater{} + ackCount := 0 + msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { + ackCount++ + return nil + }} + + if err := processFastMessage(context.Background(), registry, appender, updater, msg); err == nil { + t.Fatal("processFastMessage() error = nil, want tdengine transient failure") + } + if updater.count != 1 || ackCount != 0 { + t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`, + `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("tdengine transient redis catchup metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { + t.Fatalf("message should not be counted ok because it was not acked:\n%s", text) + } +} + func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) { - env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3"} + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3", ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli()} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) @@ -77,6 +115,14 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) { `vehicle_fast_writer_stage_duration_ms_histogram_count{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_last_stage_unix_seconds{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_last_stage_unix_seconds{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_last_redis_e2e_unix_seconds{subject="vehicle.raw.go.jt808.v1"} `, } { if !strings.Contains(text, want) { t.Fatalf("fast writer stage metric missing %s:\n%s", want, text) @@ -84,6 +130,151 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) { } } +func TestRecordFastWriterRedisE2EDurationSkipsMissingReceiveTime(t *testing.T) { + registry := metrics.NewRegistry() + + recordFastWriterRedisE2EDuration(registry, "vehicle.raw.go.jt808.v1", envelope.FrameEnvelope{}) + + if text := registry.Render(); strings.Contains(text, "vehicle_fast_writer_redis_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_fast_writer_redis_e2e_recent") { + t.Fatalf("e2e metric should be skipped when received_at_ms is missing:\n%s", text) + } +} + +func TestProcessFastMessageAcksInvalidJSONAndRecordsMetric(t *testing.T) { + registry := metrics.NewRegistry() + ackCount := 0 + msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { + ackCount++ + return nil + }} + updater := &recordingFastUpdater{} + + if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil { + t.Fatalf("processFastMessage() error = %v", err) + } + if ackCount != 1 { + t.Fatalf("ack count = %d, want 1", ackCount) + } + if updater.count != 0 || updater.batchCount != 0 { + t.Fatalf("updater counts = single %d batch %d, want 0", updater.count, updater.batchCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("invalid json metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { + t.Fatalf("invalid json should not be counted as ok:\n%s", text) + } +} + +func TestProcessFastMessageAcksExplicitNonRawEventKindWithoutWriting(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + MessageID: "0x0200", + VIN: "VIN001", + } + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{} + updater := &recordingFastUpdater{} + ackCount := 0 + msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { + ackCount++ + return nil + }} + + if err := processFastMessage(context.Background(), registry, appender, updater, msg); err != nil { + t.Fatalf("processFastMessage() error = %v", err) + } + if appender.count != 0 || updater.count != 0 || updater.batchCount != 0 { + t.Fatalf("appends=%d updates=%d batch_updates=%d, want no writes", appender.count, updater.count, updater.batchCount) + } + if ackCount != 1 { + t.Fatalf("ack count=%d, want 1", ackCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_stage_duration_ms_histogram_count{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("mismatch metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { + t.Fatalf("mismatched event should not be counted ok:\n%s", text) + } +} + +func TestProcessFastMessageRecordsRedisFieldMetrics(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fields"} + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ + EnvelopesSeen: 4, + EnvelopesUpdated: 1, + EnvelopesSkippedNonRealtime: 1, + EnvelopesSkippedMissingVIN: 1, + EnvelopesSkippedMissingVehicleKey: 0, + EnvelopesSkippedMissingFields: 1, + FieldsSeen: 3, + FieldsWritten: 2, + FieldsSkippedStale: 1, + }} + msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }} + + if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil { + t.Fatalf("processFastMessage() error = %v", err) + } + if updater.resultCount != 1 { + t.Fatalf("FastUpdateWithResult count=%d, want 1", updater.resultCount) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 4`, + `vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_vin",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_envelopes_total{status="skipped_non_realtime",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_fields",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 3`, + `vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("redis field metric missing %s:\n%s", want, text) + } + } +} + +func TestAddFastMetricRecordsLastMessage(t *testing.T) { + registry := metrics.NewRegistry() + + addFastMetric(registry, "vehicle.raw.go.gb32960.v1", "ok") + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_fast_writer_last_message_unix_seconds{status="ok",subject="vehicle.raw.go.gb32960.v1"} `, + } { + if !strings.Contains(text, want) { + t.Fatalf("fast writer message metric missing %s:\n%s", want, text) + } + } +} + func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-4"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-5"} @@ -149,6 +340,407 @@ func TestProcessFastBatchUsesRedisBatchUpdaterWhenAvailable(t *testing.T) { } } +func TestProcessFastBatchFallsBackToSinglesAfterNonTransientTDengineBatchFailure(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-td-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{batchErr: errors.New("syntax error in batch insert")} + updater := &recordingFastSingleUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if appender.batchCount != 1 || appender.count != 2 { + t.Fatalf("batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count) + } + if updater.count != 2 || ackCount != 2 { + t.Fatalf("updates=%d acks=%d, want 2/2", updater.count, ackCount) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="ok"} 1`) { + t.Fatalf("tdengine fallback metric missing:\n%s", text) + } +} + +func TestProcessFastBatchUpdatesRedisButDoesNotAckOnTransientTDengineBatchFailure(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-transient"} + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{batchErr: errors.New("i/o timeout")} + updater := &recordingFastSingleUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil { + t.Fatal("processFastBatch() error = nil, want transient tdengine failure") + } + if appender.batchCount != 1 || appender.count != 0 { + t.Fatalf("batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count) + } + if updater.count != 1 || ackCount != 0 { + t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="skipped_transient"} 1`, + `vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`, + `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("tdengine transient catchup metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessFastBatchFallsBackToRedisSinglesAfterNonTransientRedisBatchFailure(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-redis-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-redis-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{} + updater := &recordingFastUpdater{batchErr: errors.New("WRONGTYPE operation against a key holding the wrong kind of value")} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if appender.batchCount != 1 || appender.count != 0 { + t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count) + } + if updater.batchCount != 1 || updater.count != 2 { + t.Fatalf("redis batch updates=%d single updates=%d, want 1/2", updater.batchCount, updater.count) + } + if ackCount != 2 { + t.Fatalf("ack count=%d, want 2", ackCount) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="redis",status="ok"} 1`) { + t.Fatalf("redis fallback metric missing:\n%s", text) + } +} + +func TestProcessFastBatchFallbackLeavesFailedAndRemainingMessagesUnacked(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-prefix-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-prefix-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + appender := &recordingFastAppender{ + batchErr: errors.New("syntax error in batch insert"), + singleErr: errors.New("single insert failed"), + failSingleOnCount: 2, + } + updater := &recordingFastSingleUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil { + t.Fatal("processFastBatch() error = nil, want fallback single write failure") + } + if appender.batchCount != 1 || appender.count != 2 { + t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count) + } + if updater.count != 1 || ackCount != 1 { + t.Fatalf("updates=%d acks=%d, want successful prefix 1/1", updater.count, ackCount) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 1`) { + t.Fatalf("tdengine fallback error metric missing:\n%s", text) + } +} + +func TestProcessFastBatchRecordsRedisFieldMetrics(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-fields-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-fields-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ + EnvelopesSeen: 2, + EnvelopesUpdated: 2, + FieldsSeen: 10, + FieldsWritten: 7, + FieldsSkippedStale: 3, + }} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if updater.batchResultCount != 1 || updater.batchCount != 0 { + t.Fatalf("batch result count=%d legacy batch count=%d, want 1/0", updater.batchResultCount, updater.batchCount) + } + if ackCount != 2 { + t.Fatalf("ack count=%d, want 2", ackCount) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 2`, + `vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 2`, + `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 10`, + `vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 3`, + `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 7`, + } { + if !strings.Contains(text, want) { + t.Fatalf("redis field metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessFastBatchRecordsRedisE2EByOriginalSubject(t *testing.T) { + first := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + EventID: "evt-batch-e2e-1", + ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli(), + } + second := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "VIN002", + EventID: "evt-batch-e2e-2", + ReceivedAtMS: time.Now().Add(-25 * time.Millisecond).UnixMilli(), + } + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ + EnvelopesSeen: 2, + EnvelopesUpdated: 2, + }} + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload}, + {subject: "vehicle.raw.go.gb32960.v1", data: secondPayload}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `, + `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} `, + } { + if !strings.Contains(text, want) { + t.Fatalf("redis e2e metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="mixed"}`) { + t.Fatalf("redis e2e metric should keep original subjects, not mixed:\n%s", text) + } +} + +func TestProcessFastBatchRecordsRedisFieldMetricsByOriginalSubjectForMixedBatch(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-mixed-fields-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", EventID: "evt-batch-mixed-fields-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastDynamicResultUpdater{} + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: firstPayload}, + {subject: "vehicle.raw.go.gb32960.v1", data: secondPayload}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if updater.batchResultCount != 2 { + t.Fatalf("batch result count=%d, want one batch per subject", updater.batchResultCount) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.gb32960.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("redis field metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `subject="mixed"`) { + t.Fatalf("mixed subject should not be used for per-subject Redis metrics:\n%s", text) + } +} + +func TestProcessFastBatchAcksInvalidJSONWithoutMarkingItOK(t *testing.T) { + valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-valid"} + validPayload, err := valid.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if ackCount != 2 { + t.Fatalf("ack count = %d, want 2", ackCount) + } + if updater.batchCount != 1 || updater.batchRows != 1 { + t.Fatalf("redis batch count=%d rows=%d, want one valid row", updater.batchCount, updater.batchRows) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`, + `vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_batch_pending_messages 0`, + `vehicle_fast_writer_batch_pending_envelopes 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch invalid json metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessFastBatchSkipsExplicitNonRawEventKindAndWritesValidRows(t *testing.T) { + mismatched := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindFields, VIN: "VIN_BAD", EventID: "evt-fields-on-raw"} + valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindRaw, VIN: "VIN001", EventID: "evt-valid-raw"} + mismatchedPayload, err := mismatched.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + validPayload, err := valid.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: mismatchedPayload, ack: func() error { ackCount++; return nil }}, + {subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if ackCount != 2 { + t.Fatalf("ack count = %d, want 2", ackCount) + } + if updater.batchCount != 1 || updater.batchRows != 1 { + t.Fatalf("redis batch count=%d rows=%d, want only valid row", updater.batchCount, updater.batchRows) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`, + `vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_fast_writer_batch_pending_messages 0`, + `vehicle_fast_writer_batch_pending_envelopes 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch mismatch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessFastBatchCanSkipTDengineAppender(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-no-td"} + payload, err := env.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &recordingFastUpdater{} + ackCount := 0 + msgs := []*fastMessage{ + {subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }}, + } + + if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { + t.Fatalf("processFastBatch() error = %v", err) + } + if updater.batchCount != 1 || updater.batchRows != 1 { + t.Fatalf("redis batch count=%d rows=%d, want 1/1", updater.batchCount, updater.batchRows) + } + if ackCount != 1 { + t.Fatalf("ack count=%d, want 1", ackCount) + } + text := registry.Render() + if strings.Contains(text, `stage="tdengine"`) { + t.Fatalf("tdengine stage should not be recorded when appender is nil:\n%s", text) + } + if !strings.Contains(text, `stage="redis"`) || !strings.Contains(text, `stage="ack"`) { + t.Fatalf("redis and ack stages should still be recorded:\n%s", text) + } +} + func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"} @@ -208,6 +800,77 @@ func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) { } } +func TestProcessFastBatchPendingAggregatesConcurrentWorkers(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fast-pending-concurrent-1"} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fast-pending-concurrent-2"} + firstPayload, err := first.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + secondPayload, err := second.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + firstStarted := make(chan struct{}) + secondStarted := make(chan struct{}) + release := make(chan struct{}) + appenderFor := func(started chan struct{}) *recordingFastAppender { + return &recordingFastAppender{ + onAppendBatch: func() { + close(started) + <-release + }, + } + } + messages := func(payload []byte, count int) []*fastMessage { + out := make([]*fastMessage, 0, count) + for i := 0; i < count; i++ { + out = append(out, &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }}) + } + return out + } + var wg sync.WaitGroup + wg.Add(2) + var firstErr error + var secondErr error + go func() { + defer wg.Done() + firstErr = processFastBatch(context.Background(), registry, appenderFor(firstStarted), &recordingFastUpdater{}, messages(firstPayload, 2)) + }() + go func() { + defer wg.Done() + secondErr = processFastBatch(context.Background(), registry, appenderFor(secondStarted), &recordingFastUpdater{}, messages(secondPayload, 3)) + }() + <-firstStarted + <-secondStarted + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_batch_pending_messages 5`, + `vehicle_fast_writer_batch_pending_envelopes 5`, + } { + if !strings.Contains(text, want) { + t.Fatalf("aggregate pending metric missing %s during concurrent append:\n%s", want, text) + } + } + + close(release) + wg.Wait() + if firstErr != nil || secondErr != nil { + t.Fatalf("processFastBatch errors = %v / %v", firstErr, secondErr) + } + text = registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_batch_pending_messages 0`, + `vehicle_fast_writer_batch_pending_envelopes 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("aggregate pending metric should reset after concurrent append, missing %s:\n%s", want, text) + } + } +} + func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) { registry := metrics.NewRegistry() cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"} @@ -230,6 +893,41 @@ func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) { } } +func TestIsFastWorkerShutdownFetchError(t *testing.T) { + if isFastWorkerShutdownFetchError(context.Background(), errors.New("temporary nats failure")) { + t.Fatal("temporary failure should not be treated as shutdown") + } + if !isFastWorkerShutdownFetchError(context.Background(), nats.ErrConnectionClosed) { + t.Fatal("connection closed should stop worker without noisy error log") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if !isFastWorkerShutdownFetchError(ctx, errors.New("any fetch error after cancellation")) { + t.Fatal("cancelled context should stop worker without noisy error log") + } +} + +func TestIsTransientFastFetchError(t *testing.T) { + for _, err := range []error{ + errors.New("nats: disconnected during fetch"), + errors.New("nats: connection closed"), + errors.New("read tcp: connection reset by peer"), + errors.New("write tcp: broken pipe"), + errors.New("temporary network unavailable"), + errors.New("i/o timeout"), + } { + if !isTransientFastFetchError(err) { + t.Fatalf("isTransientFastFetchError(%q) = false, want true", err.Error()) + } + } + if isTransientFastFetchError(errors.New("permission denied")) { + t.Fatal("permission denied should remain an unexpected fetch error") + } + if isTransientFastFetchError(nil) { + t.Fatal("nil should not be treated as transient") + } +} + func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) { t.Setenv("FAST_WRITER_WORKERS", "12") t.Setenv("FAST_WRITER_TDENGINE_MAX_OPEN_CONNS", "") @@ -244,6 +942,28 @@ func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) { } } +func TestLoadConfigDefaultsTDengineStageDisabled(t *testing.T) { + cfg := loadConfig() + + if cfg.TDengineEnabled { + t.Fatal("TDengineEnabled = true, want production default false") + } +} + +func TestLoadConfigCanEnableTDengineStageExplicitly(t *testing.T) { + for _, value := range []string{"true", "1", "yes", "on"} { + t.Run(value, func(t *testing.T) { + t.Setenv("FAST_WRITER_TDENGINE_ENABLED", value) + + cfg := loadConfig() + + if !cfg.TDengineEnabled { + t.Fatalf("TDengineEnabled = false for %q, want explicit true", value) + } + }) + } +} + func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) { cfg := loadConfig() @@ -256,12 +976,16 @@ func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) { if got, want := cfg.OperationWait, time.Second; got != want { t.Fatalf("OperationWait = %v, want %v", got, want) } + if got, want := cfg.FetchWait, 20*time.Millisecond; got != want { + t.Fatalf("FetchWait = %v, want %v", got, want) + } } func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) { t.Setenv("NATS_STREAM_MAX_BYTES", "1073741824") t.Setenv("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", "90") t.Setenv("FAST_WRITER_OPERATION_TIMEOUT_MS", "1500") + t.Setenv("FAST_WRITER_FETCH_WAIT_MS", "45") cfg := loadConfig() @@ -274,6 +998,32 @@ func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) { if got, want := cfg.OperationWait, 1500*time.Millisecond; got != want { t.Fatalf("OperationWait = %v, want %v", got, want) } + if got, want := cfg.FetchWait, 45*time.Millisecond; got != want { + t.Fatalf("FetchWait = %v, want %v", got, want) + } +} + +func TestRecordFastWriterConfigMetrics(t *testing.T) { + registry := metrics.NewRegistry() + + recordFastWriterConfigMetrics(registry, config{ + BatchSize: 120, + FetchWait: 20 * time.Millisecond, + OperationWait: 1500 * time.Millisecond, + Workers: 8, + }) + + text := registry.Render() + for _, want := range []string{ + `vehicle_fast_writer_config{setting="batch_size"} 120`, + `vehicle_fast_writer_config{setting="fetch_wait_ms"} 20`, + `vehicle_fast_writer_config{setting="operation_timeout_ms"} 1500`, + `vehicle_fast_writer_config{setting="workers"} 8`, + } { + if !strings.Contains(text, want) { + t.Fatalf("fast writer config metric missing %s:\n%s", want, text) + } + } } func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) { @@ -292,15 +1042,30 @@ func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) { } type recordingFastAppender struct { - count int - batchCount int - batchRows int - err error - onAppendBatch func() + count int + batchCount int + batchRows int + err error + batchErr error + singleErr error + failSingleOnCount int + onAppendBatch func() } func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error { a.count++ + if a.failSingleOnCount > 0 && a.count == a.failSingleOnCount { + if a.singleErr != nil { + return a.singleErr + } + return a.err + } + if a.failSingleOnCount > 0 { + return nil + } + if a.singleErr != nil { + return a.singleErr + } return a.err } @@ -310,24 +1075,45 @@ func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelop if a.onAppendBatch != nil { a.onAppendBatch() } + if a.batchErr != nil { + return a.batchErr + } return a.err } type recordingFastUpdater struct { - count int - batchCount int - batchRows int - err error + count int + batchCount int + batchRows int + err error + batchErr error + singleErr error + failSingleOnCount int } func (u *recordingFastUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { u.count++ + if u.failSingleOnCount > 0 && u.count == u.failSingleOnCount { + if u.singleErr != nil { + return u.singleErr + } + return u.err + } + if u.failSingleOnCount > 0 { + return nil + } + if u.singleErr != nil { + return u.singleErr + } return u.err } func (u *recordingFastUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error { u.batchCount++ u.batchRows += len(envs) + if u.batchErr != nil { + return u.batchErr + } return u.err } @@ -340,3 +1126,73 @@ func (u *recordingFastSingleUpdater) FastUpdate(context.Context, envelope.FrameE u.count++ return u.err } + +type recordingFastResultUpdater struct { + count int + resultCount int + batchCount int + batchResultCount int + batchRows int + result realtime.FastUpdateResult + err error + batchErr error + singleErr error +} + +func (u *recordingFastResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { + u.count++ + if u.singleErr != nil { + return u.singleErr + } + return u.err +} + +func (u *recordingFastResultUpdater) FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { + u.resultCount++ + if u.singleErr != nil { + return u.result, u.singleErr + } + return u.result, u.err +} + +type recordingFastDynamicResultUpdater struct { + batchResultCount int + batchRows int + err error +} + +func (u *recordingFastDynamicResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { + return u.err +} + +func (u *recordingFastDynamicResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { + u.batchResultCount++ + u.batchRows += len(envs) + if u.err != nil { + return realtime.FastUpdateResult{}, u.err + } + return realtime.FastUpdateResult{ + EnvelopesSeen: len(envs), + EnvelopesUpdated: len(envs), + FieldsSeen: len(envs), + FieldsWritten: len(envs), + }, nil +} + +func (u *recordingFastResultUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error { + u.batchCount++ + u.batchRows += len(envs) + if u.batchErr != nil { + return u.batchErr + } + return u.err +} + +func (u *recordingFastResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { + u.batchResultCount++ + u.batchRows += len(envs) + if u.batchErr != nil { + return u.result, u.batchErr + } + return u.result, u.err +} diff --git a/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go b/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go index 1dfdb67c..d2824458 100644 --- a/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go +++ b/go/vehicle-gateway/cmd/nats-kafka-bridge/main.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "os/signal" + "sort" "strconv" "strings" "syscall" @@ -20,6 +21,7 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) @@ -29,6 +31,10 @@ func main() { defer stop() cfg := loadConfig() + if err := cfg.Validate(); err != nil { + logger.Error("invalid bridge config", "error", err) + os.Exit(1) + } conn, err := nats.Connect(cfg.NATSURL, nats.Name(cfg.NATSClientName), nats.Timeout(5*time.Second)) if err != nil { logger.Error("nats connect failed", "error", err) @@ -36,6 +42,7 @@ func main() { } defer conn.Close() registry := metrics.NewRegistry() + recordBridgeConfigMetrics(registry, cfg) health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-kafka-bridge", []health.Check{ {Name: "nats", Check: func(context.Context) error { if conn.Status() != nats.CONNECTED { @@ -66,71 +73,205 @@ func main() { os.Exit(1) } - writer := &kafka.Writer{ - Addr: kafka.TCP(cfg.KafkaBrokers...), - Balancer: &kafka.Hash{}, - AllowAutoTopicCreation: false, - RequiredAcks: kafka.RequireAll, - Async: false, - } - defer writer.Close() - logger.Info("nats kafka bridge started", "nats_url", cfg.NATSURL, "stream", cfg.NATSStream, "durable", cfg.NATSDurable, "filter", cfg.NATSFilter, - "kafka_brokers", strings.Join(cfg.KafkaBrokers, ",")) - runBridge(ctx, logger, registry, js, sub, writer, cfg) + "kafka_brokers", strings.Join(cfg.KafkaBrokers, ","), + "kafka_batch_timeout_ms", cfg.KafkaBatchTimeout.Milliseconds(), + "kafka_write_concurrency", cfg.KafkaWriteConcurrency, + "fetch_wait_ms", cfg.FetchWait.Milliseconds(), + "batch_size", cfg.BatchSize, + "workers", cfg.Workers, + "derive_fields_from_raw_enabled", cfg.DeriveFieldsFromRaw) + for i := 0; i < cfg.Workers; i++ { + writer := newKafkaWriter(cfg) + defer writer.Close() + go runBridge(ctx, logger.With("worker", i), registry, js, sub, writer, cfg) + } + <-ctx.Done() } type config struct { - NATSURL string - NATSClientName string - NATSStream string - NATSDurable string - NATSFilter string - NATSSubjects []string - KafkaBrokers []string - Route map[string]string - BatchSize int - FetchWait time.Duration - OperationWait time.Duration - AckWait time.Duration - StreamMaxAge time.Duration - StreamMaxBytes int64 - StreamEnsureWait time.Duration + NATSURL string + NATSClientName string + NATSStream string + NATSDurable string + NATSFilter string + NATSSubjects []string + KafkaBrokers []string + RawRoutes []subjectRoute + FieldsRoutes []subjectRoute + Route map[string]string + RawFieldRoutes map[string]fieldsProjectionRoute + DeriveFieldsFromRaw bool + KafkaBatchTimeout time.Duration + KafkaWriteConcurrency int + BatchSize int + FetchWait time.Duration + OperationWait time.Duration + AckWait time.Duration + StreamMaxAge time.Duration + StreamMaxBytes int64 + StreamEnsureWait time.Duration + Workers int +} + +type subjectRoute struct { + Protocol envelope.Protocol + Subject string + Topic string +} + +type fieldsProjectionRoute struct { + Protocol envelope.Protocol + Topic string } func loadConfig() config { - route := map[string]string{ - env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960), - env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808), - env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT), - env("NATS_SUBJECT_GB32960_FIELDS", env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)): env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960), - env("NATS_SUBJECT_JT808_FIELDS", env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)): env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808), - env("NATS_SUBJECT_YUTONG_MQTT_FIELDS", env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT), + rawRoutes := []subjectRoute{ + {Protocol: envelope.ProtocolGB32960, Subject: env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)), Topic: env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)}, + {Protocol: envelope.ProtocolJT808, Subject: env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)), Topic: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)}, + {Protocol: envelope.ProtocolYutongMQTT, Subject: env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)), Topic: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)}, } + fieldsRoutes := []subjectRoute{ + {Protocol: envelope.ProtocolGB32960, Subject: env("NATS_SUBJECT_GB32960_FIELDS", env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)), Topic: env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)}, + {Protocol: envelope.ProtocolJT808, Subject: env("NATS_SUBJECT_JT808_FIELDS", env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)), Topic: env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)}, + {Protocol: envelope.ProtocolYutongMQTT, Subject: env("NATS_SUBJECT_YUTONG_MQTT_FIELDS", env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)), Topic: env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)}, + } + route := routeMap(rawRoutes, fieldsRoutes) if unifiedSubject, unifiedTopic, ok := unifiedRouteFromEnv(); ok { route[unifiedSubject] = unifiedTopic } - return config{ - NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"), - NATSClientName: env("NATS_CLIENT_NAME", "lingniu-nats-kafka-bridge"), - NATSStream: env("NATS_STREAM", "VEHICLE_INGEST"), - NATSDurable: env("NATS_DURABLE", "vehicle-kafka-bridge"), - NATSFilter: env("NATS_FILTER", "vehicle.>"), - NATSSubjects: splitCSV(env("NATS_STREAM_SUBJECTS", strings.Join(mapKeys(route), ","))), - KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), - Route: route, - BatchSize: envInt("BRIDGE_BATCH_SIZE", 500), - FetchWait: time.Duration(envInt("BRIDGE_FETCH_WAIT_MS", 1000)) * time.Millisecond, - OperationWait: time.Duration(envInt("BRIDGE_OPERATION_TIMEOUT_MS", 30000)) * time.Millisecond, - AckWait: time.Duration(envInt("NATS_ACK_WAIT_SECONDS", 60)) * time.Second, - StreamMaxAge: time.Duration(envInt("NATS_STREAM_MAX_AGE_HOURS", 24)) * time.Hour, - StreamMaxBytes: envInt64("NATS_STREAM_MAX_BYTES", 20*1024*1024*1024), - StreamEnsureWait: time.Duration(envInt("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", 60)) * time.Second, + workers := envInt("BRIDGE_WORKERS", 4) + if workers < 1 { + workers = 1 } + return config{ + NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"), + NATSClientName: env("NATS_CLIENT_NAME", "lingniu-nats-kafka-bridge"), + NATSStream: env("NATS_STREAM", "VEHICLE_INGEST"), + NATSDurable: env("NATS_DURABLE", "vehicle-kafka-bridge"), + NATSFilter: env("NATS_FILTER", "vehicle.>"), + NATSSubjects: splitCSV(env("NATS_STREAM_SUBJECTS", strings.Join(mapKeys(route), ","))), + KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), + RawRoutes: rawRoutes, + FieldsRoutes: fieldsRoutes, + Route: route, + RawFieldRoutes: rawFieldProjectionRoutes(rawRoutes, fieldsRoutes), + DeriveFieldsFromRaw: envBool("BRIDGE_DERIVE_FIELDS_FROM_RAW_ENABLED", true), + KafkaBatchTimeout: time.Duration(envInt("BRIDGE_KAFKA_BATCH_TIMEOUT_MS", 20)) * time.Millisecond, + KafkaWriteConcurrency: envInt("BRIDGE_KAFKA_WRITE_CONCURRENCY", 6), + BatchSize: envInt("BRIDGE_BATCH_SIZE", 500), + FetchWait: time.Duration(envInt("BRIDGE_FETCH_WAIT_MS", 20)) * time.Millisecond, + OperationWait: time.Duration(envInt("BRIDGE_OPERATION_TIMEOUT_MS", 30000)) * time.Millisecond, + AckWait: time.Duration(envInt("NATS_ACK_WAIT_SECONDS", 60)) * time.Second, + StreamMaxAge: time.Duration(envInt("NATS_STREAM_MAX_AGE_HOURS", 24)) * time.Hour, + StreamMaxBytes: envInt64("NATS_STREAM_MAX_BYTES", 20*1024*1024*1024), + StreamEnsureWait: time.Duration(envInt("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", 60)) * time.Second, + Workers: workers, + } +} + +func recordBridgeConfigMetrics(registry *metrics.Registry, cfg config) { + if registry == nil { + return + } + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers)) + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize)) + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "fetch_wait_ms"}, float64(cfg.FetchWait.Milliseconds())) + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "kafka_batch_timeout_ms"}, float64(cfg.KafkaBatchTimeout.Milliseconds())) + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "kafka_write_concurrency"}, float64(cfg.KafkaWriteConcurrency)) + registry.SetGauge("vehicle_bridge_config", metrics.Labels{"setting": "derive_fields_from_raw_enabled"}, boolMetric(cfg.DeriveFieldsFromRaw)) +} + +func newKafkaWriter(cfg config) *kafka.Writer { + return &kafka.Writer{ + Addr: kafka.TCP(cfg.KafkaBrokers...), + Balancer: &kafka.Hash{}, + AllowAutoTopicCreation: false, + RequiredAcks: kafka.RequireAll, + BatchTimeout: cfg.KafkaBatchTimeout, + Async: false, + } +} + +func routeMap(rawRoutes []subjectRoute, fieldsRoutes []subjectRoute) map[string]string { + route := make(map[string]string, len(rawRoutes)+len(fieldsRoutes)) + for _, item := range rawRoutes { + if item.Subject != "" { + route[item.Subject] = item.Topic + } + } + for _, item := range fieldsRoutes { + if item.Subject != "" { + route[item.Subject] = item.Topic + } + } + return route +} + +func rawFieldProjectionRoutes(rawRoutes []subjectRoute, fieldsRoutes []subjectRoute) map[string]fieldsProjectionRoute { + fieldsByProtocol := make(map[envelope.Protocol]string, len(fieldsRoutes)) + for _, route := range fieldsRoutes { + if route.Protocol != "" && strings.TrimSpace(route.Topic) != "" { + fieldsByProtocol[route.Protocol] = strings.TrimSpace(route.Topic) + } + } + out := make(map[string]fieldsProjectionRoute, len(rawRoutes)) + for _, route := range rawRoutes { + subject := strings.TrimSpace(route.Subject) + topic := fieldsByProtocol[route.Protocol] + if subject == "" || topic == "" { + continue + } + out[subject] = fieldsProjectionRoute{Protocol: route.Protocol, Topic: topic} + } + return out +} + +func (c config) Validate() error { + rawSubjects, rawTopics := routeSubjectsAndTopics(c.RawRoutes) + fieldsSubjects, fieldsTopics := routeSubjectsAndTopics(c.FieldsRoutes) + if err := topics.ValidateKafkaRawFields(rawTopics, fieldsTopics); err != nil { + return err + } + return topics.ValidateRawFieldsDisjoint(rawSubjects, fieldsSubjects, "nats subject") +} + +func routeSubjectsAndTopics(routes []subjectRoute) (map[string]string, map[string]string) { + subjects := make(map[string]string, len(routes)) + kafkaTopics := make(map[string]string, len(routes)) + for _, route := range routes { + name := strings.TrimSpace(route.Subject) + if name == "" { + name = strings.TrimSpace(route.Topic) + } + if name == "" { + name = "unknown" + } + subjects[name] = strings.TrimSpace(route.Subject) + topicName := routeProtocolName(route) + if topicName == "" { + topicName = name + } + kafkaTopics[topicName] = strings.TrimSpace(route.Topic) + } + return subjects, kafkaTopics +} + +func routeProtocolName(route subjectRoute) string { + if protocol := strings.TrimSpace(string(route.Protocol)); protocol != "" { + return protocol + } + if protocol, ok := topics.ProtocolForKnownRawTopic(route.Topic); ok { + return protocol + } + if protocol, ok := topics.ProtocolForKnownFieldsTopic(route.Topic); ok { + return protocol + } + return "" } func unifiedRouteFromEnv() (string, string, bool) { @@ -169,6 +310,18 @@ type bridgeMessage struct { ack func() error } +type routedBridgeMessage struct { + sourceIndex int + kafka kafka.Message + receivedAtMS int64 +} + +type bridgeSourceState struct { + source bridgeMessage + routed bool + failed bool +} + func runBridge(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, infoReader natsConsumerInfoReader, sub natsPullSubscription, writer kafkaBatchWriter, cfg config) { var lastConsumerInfoAt time.Time for { @@ -188,9 +341,17 @@ func runBridge(ctx context.Context, logger *slog.Logger, registry *metrics.Regis } msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait)) if err != nil { + if isBridgeShutdownFetchError(ctx, err) { + return + } if errors.Is(err, nats.ErrTimeout) { continue } + if isTransientBridgeFetchError(err) { + logger.Warn("nats fetch interrupted", "error", err) + time.Sleep(time.Second) + continue + } logger.Error("nats fetch failed", "error", err) time.Sleep(time.Second) continue @@ -207,7 +368,7 @@ func runBridge(ctx context.Context, logger *slog.Logger, registry *metrics.Regis }) } operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cfg.OperationWait) - err = bridgeBatch(operationCtx, registry, writer, bridgeMessages, cfg.Route) + err = bridgeBatchWithProjectionConcurrency(operationCtx, registry, writer, bridgeMessages, cfg.Route, cfg.RawFieldRoutes, cfg.DeriveFieldsFromRaw, cfg.KafkaWriteConcurrency) cancel() if err != nil { logger.Error("bridge batch failed", "count", len(bridgeMessages), "error", err) @@ -216,6 +377,30 @@ func runBridge(ctx context.Context, logger *slog.Logger, registry *metrics.Regis } } +func isBridgeShutdownFetchError(ctx context.Context, err error) bool { + if err == nil { + return false + } + if ctx.Err() != nil { + return true + } + return errors.Is(err, nats.ErrConnectionClosed) +} + +func isTransientBridgeFetchError(err error) bool { + if err == nil { + return false + } + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "disconnected during fetch") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "timeout") +} + func ensureStream(js nats.JetStreamContext, cfg config) error { ctx, cancel := context.WithTimeout(context.Background(), cfg.StreamEnsureWait) defer cancel() @@ -245,6 +430,31 @@ func ensureStream(js nats.JetStreamContext, cfg config) error { } func bridgeBatch(ctx context.Context, registry *metrics.Registry, writer kafkaBatchWriter, messages []bridgeMessage, route map[string]string) error { + return bridgeBatchWithProjectionConcurrency(ctx, registry, writer, messages, route, nil, false, 6) +} + +func bridgeBatchWithProjection( + ctx context.Context, + registry *metrics.Registry, + writer kafkaBatchWriter, + messages []bridgeMessage, + route map[string]string, + rawFieldRoutes map[string]fieldsProjectionRoute, + deriveFieldsFromRaw bool, +) error { + return bridgeBatchWithProjectionConcurrency(ctx, registry, writer, messages, route, rawFieldRoutes, deriveFieldsFromRaw, 6) +} + +func bridgeBatchWithProjectionConcurrency( + ctx context.Context, + registry *metrics.Registry, + writer kafkaBatchWriter, + messages []bridgeMessage, + route map[string]string, + rawFieldRoutes map[string]fieldsProjectionRoute, + deriveFieldsFromRaw bool, + kafkaWriteConcurrency int, +) error { if len(messages) == 0 { return nil } @@ -252,66 +462,240 @@ func bridgeBatch(ctx context.Context, registry *metrics.Registry, writer kafkaBa status := "ok" defer func() { recordBridgeBatchDuration(registry, status, time.Since(started)) - recordBridgeBatchPending(registry, 0, 0) }() - kafkaMessages := make([]kafka.Message, 0, len(messages)) - for _, message := range messages { + routed := make([]routedBridgeMessage, 0, len(messages)*2) + sources := make([]bridgeSourceState, len(messages)) + for sourceIndex, message := range messages { + sources[sourceIndex].source = message addBridgeSubjectMetric(registry, "vehicle_bridge_messages_total", message.subject, "received") topic, ok := route[message.subject] if !ok || topic == "" { addBridgeSubjectMetric(registry, "vehicle_bridge_messages_total", message.subject, "route_error") - status = "error" - return fmt.Errorf("kafka topic not configured for nats subject %q", message.subject) - } - kafkaMessages = append(kafkaMessages, kafkaMessage(topic, message.data)) - } - recordBridgeBatchPending(registry, len(messages), len(kafkaMessages)) - if err := writer.WriteMessages(ctx, kafkaMessages...); err != nil { - for _, message := range kafkaMessages { - addBridgeTopicMetric(registry, "vehicle_bridge_kafka_writes_total", message.Topic, "error") - } - status = "error" - return err - } - for _, message := range kafkaMessages { - addBridgeTopicMetric(registry, "vehicle_bridge_kafka_writes_total", message.Topic, "ok") - } - for _, message := range messages { - if message.ack == nil { + if message.ack != nil { + if err := message.ack(); err != nil { + addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "error") + status = "error" + return fmt.Errorf("ack unrouted nats subject %q: %w", message.subject, err) + } + addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "dropped_route_error") + } continue } - if err := message.ack(); err != nil { - addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "error") + kafkaMessage, receivedAtMS, decoded, decodeErr := kafkaMessageWithEnvelope(topic, message.data) + sources[sourceIndex].routed = true + routed = append(routed, routedBridgeMessage{ + sourceIndex: sourceIndex, + kafka: kafkaMessage, + receivedAtMS: receivedAtMS, + }) + if !deriveFieldsFromRaw { + continue + } + projection, ok := rawFieldRoutes[message.subject] + if !ok { + continue + } + fieldsMessage, fieldsReceivedAtMS, fieldCount, projectionStatus, projected := projectRawFields(message.subject, decoded, decodeErr, projection) + recordBridgeFieldsProjection(registry, projection.Protocol, projectionStatus, fieldCount) + if !projected { + continue + } + routed = append(routed, routedBridgeMessage{ + sourceIndex: sourceIndex, + kafka: fieldsMessage, + receivedAtMS: fieldsReceivedAtMS, + }) + } + if len(routed) == 0 { + addBridgeBatchPending(registry, 0, 0) + return nil + } + addBridgeBatchPending(registry, len(messages), len(routed)) + defer addBridgeBatchPending(registry, -len(messages), -len(routed)) + groups := groupRoutedBridgeMessagesByTopic(routed) + results := writeBridgeKafkaGroups(ctx, writer, groups, kafkaWriteConcurrency) + var firstErr error + for _, result := range results { + group := result.group + err := result.err + recordBridgeKafkaWriteDuration(registry, group[0].kafka.Topic, statusFromError(err), result.elapsed) + if err != nil { + for _, item := range group { + addBridgeTopicMetric(registry, "vehicle_bridge_kafka_writes_total", item.kafka.Topic, "error") + sources[item.sourceIndex].failed = true + } + status = "error" + if firstErr == nil { + firstErr = fmt.Errorf("kafka write topic %s: %w", group[0].kafka.Topic, err) + } + continue + } + for _, item := range group { + addBridgeTopicMetric(registry, "vehicle_bridge_kafka_writes_total", item.kafka.Topic, "ok") + recordBridgeKafkaE2EDuration(registry, item.kafka.Topic, item.receivedAtMS) + } + } + for i := range sources { + source := &sources[i] + if !source.routed || source.failed || source.source.ack == nil { + continue + } + if err := source.source.ack(); err != nil { + addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", source.source.subject, "error") status = "error" return err } - addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", message.subject, "ok") + addBridgeSubjectMetric(registry, "vehicle_bridge_nats_acks_total", source.source.subject, "ok") } - return nil + return firstErr +} + +type bridgeKafkaWriteResult struct { + group []routedBridgeMessage + elapsed time.Duration + err error +} + +func writeBridgeKafkaGroups(ctx context.Context, writer kafkaBatchWriter, groups [][]routedBridgeMessage, concurrency int) []bridgeKafkaWriteResult { + if len(groups) == 0 { + return nil + } + if concurrency < 1 { + concurrency = 1 + } + if concurrency > len(groups) { + concurrency = len(groups) + } + semaphore := make(chan struct{}, concurrency) + results := make(chan bridgeKafkaWriteResult, len(groups)) + for _, group := range groups { + group := group + semaphore <- struct{}{} + go func() { + defer func() { <-semaphore }() + messages := make([]kafka.Message, 0, len(group)) + for _, item := range group { + messages = append(messages, item.kafka) + } + started := time.Now() + err := writer.WriteMessages(ctx, messages...) + results <- bridgeKafkaWriteResult{group: group, elapsed: time.Since(started), err: err} + }() + } + out := make([]bridgeKafkaWriteResult, 0, len(groups)) + for range groups { + out = append(out, <-results) + } + return out +} + +func projectRawFields(subject string, raw envelope.FrameEnvelope, decodeErr error, projection fieldsProjectionRoute) (kafka.Message, int64, int, string, bool) { + if decodeErr != nil { + return kafka.Message{}, 0, 0, "invalid_json", false + } + if _, err := topics.ValidateRawEnvelope(subject, raw); err != nil { + return kafka.Message{}, 0, 0, "invalid_envelope", false + } + if raw.Protocol != projection.Protocol { + return kafka.Message{}, 0, 0, "protocol_mismatch", false + } + if !envelope.IsRealtimeTelemetryFrame(raw) { + return kafka.Message{}, 0, 0, "skipped_non_realtime", false + } + if len(raw.ParsedFields) == 0 { + return kafka.Message{}, 0, 0, "skipped_missing_fields", false + } + fields, ok := realtime.BuildFieldsEnvelope(raw) + if !ok || len(fields.Fields) == 0 { + return kafka.Message{}, 0, 0, "skipped_missing_fields", false + } + if _, err := topics.ValidateFieldsEnvelope(projection.Topic, fields); err != nil { + return kafka.Message{}, 0, 0, "invalid_fields_envelope", false + } + payload, err := fields.MarshalJSONBytes() + if err != nil { + return kafka.Message{}, 0, 0, "marshal_error", false + } + result := kafka.Message{Topic: projection.Topic, Key: fields.KafkaKey(), Value: payload} + return result, fields.ReceivedAtMS, len(fields.Fields), "published", true +} + +func groupRoutedBridgeMessagesByTopic(messages []routedBridgeMessage) [][]routedBridgeMessage { + if len(messages) == 0 { + return nil + } + groupsByTopic := make(map[string][]routedBridgeMessage) + for _, message := range messages { + groupsByTopic[message.kafka.Topic] = append(groupsByTopic[message.kafka.Topic], message) + } + topics := make([]string, 0, len(groupsByTopic)) + for topic := range groupsByTopic { + topics = append(topics, topic) + } + sort.Strings(topics) + groups := make([][]routedBridgeMessage, 0, len(topics)) + for _, topic := range topics { + groups = append(groups, groupsByTopic[topic]) + } + return groups } var bridgeBatchDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var bridgeKafkaWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var bridgeKafkaE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000} +var bridgeKafkaE2ERecent = metrics.NewRecentLatencyByKey(512) +var bridgeBatchPending = metrics.PendingPairGauge{} func addBridgeSubjectMetric(registry *metrics.Registry, name string, subject string, status string) { if registry == nil { return } - registry.IncCounter(name, metrics.Labels{"subject": subject, "status": status}) + labels := metrics.Labels{"subject": subject, "status": status} + registry.IncCounter(name, labels) + if name == "vehicle_bridge_messages_total" { + metrics.RecordLastActivity(registry, "vehicle_bridge_last_message_unix_seconds", labels) + return + } + if name == "vehicle_bridge_nats_acks_total" { + metrics.RecordLastActivity(registry, "vehicle_bridge_last_ack_unix_seconds", labels) + } } func addBridgeTopicMetric(registry *metrics.Registry, name string, topic string, status string) { if registry == nil { return } - registry.IncCounter(name, metrics.Labels{"topic": topic, "status": status}) + labels := metrics.Labels{"topic": topic, "status": status} + registry.IncCounter(name, labels) + if name == "vehicle_bridge_kafka_writes_total" { + metrics.RecordLastActivity(registry, "vehicle_bridge_last_kafka_write_unix_seconds", labels) + } } -func recordBridgeBatchPending(registry *metrics.Registry, messages int, kafkaMessages int) { +func recordBridgeFieldsProjection(registry *metrics.Registry, protocol envelope.Protocol, status string, fieldCount int) { if registry == nil { return } - registry.SetGauge("vehicle_bridge_batch_pending_messages", nil, float64(messages)) - registry.SetGauge("vehicle_bridge_batch_pending_kafka_messages", nil, float64(kafkaMessages)) + protocolLabel := strings.TrimSpace(string(protocol)) + if protocolLabel == "" { + protocolLabel = "unknown" + } + if strings.TrimSpace(status) == "" { + status = "unknown" + } + labels := metrics.Labels{"protocol": protocolLabel, "status": status} + registry.IncCounter("vehicle_bridge_fields_projection_total", labels) + metrics.RecordLastActivity(registry, "vehicle_bridge_last_fields_projection_unix_seconds", labels) + if status == "published" { + registry.SetGauge("vehicle_bridge_fields_projection_count", labels, float64(fieldCount)) + } +} + +func addBridgeBatchPending(registry *metrics.Registry, messages int, kafkaMessages int) { + if registry == nil { + return + } + bridgeBatchPending.Add(registry, "vehicle_bridge_batch_pending_messages", "vehicle_bridge_batch_pending_kafka_messages", messages, kafkaMessages) } func recordBridgeBatchDuration(registry *metrics.Registry, status string, elapsed time.Duration) { @@ -323,6 +707,41 @@ func recordBridgeBatchDuration(registry *metrics.Registry, status string, elapse }, bridgeBatchDurationBucketsMS, float64(elapsed.Milliseconds())) } +func recordBridgeKafkaWriteDuration(registry *metrics.Registry, topic string, status string, elapsed time.Duration) { + if registry == nil { + return + } + registry.ObserveHistogram("vehicle_bridge_kafka_write_duration_ms_histogram", metrics.Labels{ + "topic": topic, + "status": status, + }, bridgeKafkaWriteDurationBucketsMS, float64(elapsed.Milliseconds())) +} + +func recordBridgeKafkaE2EDuration(registry *metrics.Registry, topic string, receivedAtMS int64) { + if registry == nil || receivedAtMS <= 0 { + return + } + elapsedMS := time.Since(time.UnixMilli(receivedAtMS)).Milliseconds() + if elapsedMS < 0 { + elapsedMS = 0 + } + labels := metrics.Labels{ + "topic": topic, + } + registry.ObserveHistogram("vehicle_bridge_kafka_e2e_duration_ms_histogram", labels, bridgeKafkaE2EDurationBucketsMS, float64(elapsedMS)) + p99, samples := bridgeKafkaE2ERecent.Observe(topic, float64(elapsedMS)) + registry.SetGauge("vehicle_bridge_kafka_e2e_recent_p99_ms", labels, p99) + registry.SetGauge("vehicle_bridge_kafka_e2e_recent_samples", labels, float64(samples)) + metrics.RecordLastActivity(registry, "vehicle_bridge_last_kafka_e2e_unix_seconds", labels) +} + +func statusFromError(err error) string { + if err != nil { + return "error" + } + return "ok" +} + func recordNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) { if registry == nil || info == nil { return @@ -334,12 +753,23 @@ func recordNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info } func kafkaMessage(topic string, data []byte) kafka.Message { + message, _ := kafkaMessageWithReceivedAt(topic, data) + return message +} + +func kafkaMessageWithReceivedAt(topic string, data []byte) (kafka.Message, int64) { + message, receivedAtMS, _, _ := kafkaMessageWithEnvelope(topic, data) + return message, receivedAtMS +} + +func kafkaMessageWithEnvelope(topic string, data []byte) (kafka.Message, int64, envelope.FrameEnvelope, error) { var env envelope.FrameEnvelope message := kafka.Message{Topic: topic, Value: data} - if err := json.Unmarshal(data, &env); err == nil { - message.Key = env.KafkaKey() + if err := json.Unmarshal(data, &env); err != nil { + return message, 0, envelope.FrameEnvelope{}, err } - return message + message.Key = env.KafkaKey() + return message, env.ReceivedAtMS, env, nil } func env(key string, fallback string) string { @@ -350,6 +780,25 @@ func env(key string, fallback string) string { return value } +func envBool(key string, fallback bool) bool { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return fallback + } + return parsed +} + +func boolMetric(value bool) float64 { + if value { + return 1 + } + return 0 +} + func envOptional(key string) (string, bool) { value, ok := os.LookupEnv(key) if !ok { diff --git a/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go b/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go index e8ebe393..a840cf33 100644 --- a/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go +++ b/go/vehicle-gateway/cmd/nats-kafka-bridge/main_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -13,6 +15,7 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) func TestBridgeBatchWritesKafkaThenAcks(t *testing.T) { @@ -65,12 +68,303 @@ func TestBridgeBatchDoesNotAckWhenKafkaFails(t *testing.T) { } } -func TestBridgeBatchRecordsMetrics(t *testing.T) { +func TestBridgeBatchProjectsFieldsFromCanonicalRawAndAcksOnce(t *testing.T) { + raw := envelope.FrameEnvelope{ + EventID: "raw-event-1", + EventKind: envelope.EventKindRaw, + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "LTEST000000000001", + EventTimeMS: 1_700_000_000_000, + ReceivedAtMS: 1_700_000_000_100, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "jt808.location.speed_kmh": 52.3, + "jt808.location.total_mileage_km": 12345.6, + }, + } + payload, err := raw.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + acked := 0 + writer := &recordingBridgeWriter{} + registry := metrics.NewRegistry() + + err = bridgeBatchWithProjection( + context.Background(), + registry, + writer, + []bridgeMessage{{subject: topics.RawJT808, data: payload, ack: func() error { acked++; return nil }}}, + map[string]string{topics.RawJT808: topics.RawJT808}, + map[string]fieldsProjectionRoute{topics.RawJT808: {Protocol: envelope.ProtocolJT808, Topic: topics.FieldsJT808}}, + true, + ) + if err != nil { + t.Fatalf("bridgeBatchWithProjection() error = %v", err) + } + if acked != 1 { + t.Fatalf("acks = %d, want exactly one ack for raw plus derived fields", acked) + } + if len(writer.messages) != 2 { + t.Fatalf("kafka messages = %d, want raw and fields", len(writer.messages)) + } + byTopic := map[string]kafka.Message{} + for _, message := range writer.messages { + byTopic[message.Topic] = message + } + if len(byTopic[topics.RawJT808].Value) == 0 || len(byTopic[topics.FieldsJT808].Value) == 0 { + t.Fatalf("projected topics = %#v", byTopic) + } + var fields envelope.FrameEnvelope + if err := json.Unmarshal(byTopic[topics.FieldsJT808].Value, &fields); err != nil { + t.Fatal(err) + } + if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID { + t.Fatalf("fields envelope = %#v", fields) + } + if got := fields.Fields["jt808.location.total_mileage_km"]; got != 12345.6 { + t.Fatalf("projected mileage = %#v", got) + } + metricText := registry.Render() + for _, want := range []string{ + `vehicle_bridge_fields_projection_total{protocol="JT808",status="published"} 1`, + `vehicle_bridge_fields_projection_count{protocol="JT808",status="published"} 2`, + `vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(metricText, want) { + t.Fatalf("projection metric missing %s:\n%s", want, metricText) + } + } +} + +func TestBridgeBatchDoesNotAckRawWhenDerivedFieldsWriteFails(t *testing.T) { + raw := envelope.FrameEnvelope{ + EventKind: envelope.EventKindRaw, + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "LTEST000000000001", + EventTimeMS: 1_700_000_000_000, + ReceivedAtMS: 1_700_000_000_100, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.total_mileage_km": 12345.6}, + } + payload, err := raw.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + acked := 0 + writer := &recordingBridgeWriter{topicErr: map[string]error{topics.FieldsJT808: errors.New("fields unavailable")}} + + err = bridgeBatchWithProjection( + context.Background(), + nil, + writer, + []bridgeMessage{{subject: topics.RawJT808, data: payload, ack: func() error { acked++; return nil }}}, + map[string]string{topics.RawJT808: topics.RawJT808}, + map[string]fieldsProjectionRoute{topics.RawJT808: {Protocol: envelope.ProtocolJT808, Topic: topics.FieldsJT808}}, + true, + ) + if err == nil { + t.Fatal("bridgeBatchWithProjection() error = nil, want fields Kafka error") + } + if acked != 0 { + t.Fatalf("acks = %d, want raw left pending until both Kafka outputs succeed", acked) + } + if len(writer.messages) != 1 || writer.messages[0].Topic != topics.RawJT808 { + t.Fatalf("successful kafka messages = %#v, want only raw before replay", writer.messages) + } +} + +func TestBridgeBatchSkipsFieldsProjectionForNonRealtimeRaw(t *testing.T) { + raw := envelope.FrameEnvelope{ + EventKind: envelope.EventKindRaw, + Protocol: envelope.ProtocolJT808, + MessageID: "0x0100", + Phone: "13307795425", + EventTimeMS: 1_700_000_000_000, + ReceivedAtMS: 1_700_000_000_100, + ParseStatus: envelope.ParseOK, + } + payload, err := raw.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + acked := 0 + writer := &recordingBridgeWriter{} + registry := metrics.NewRegistry() + + err = bridgeBatchWithProjection( + context.Background(), + registry, + writer, + []bridgeMessage{{subject: topics.RawJT808, data: payload, ack: func() error { acked++; return nil }}}, + map[string]string{topics.RawJT808: topics.RawJT808}, + map[string]fieldsProjectionRoute{topics.RawJT808: {Protocol: envelope.ProtocolJT808, Topic: topics.FieldsJT808}}, + true, + ) + if err != nil { + t.Fatalf("bridgeBatchWithProjection() error = %v", err) + } + if acked != 1 || len(writer.messages) != 1 || writer.messages[0].Topic != topics.RawJT808 { + t.Fatalf("acks=%d messages=%#v", acked, writer.messages) + } + if text := registry.Render(); !strings.Contains(text, `vehicle_bridge_fields_projection_total{protocol="JT808",status="skipped_non_realtime"} 1`) { + t.Fatalf("non-realtime projection metric missing:\n%s", text) + } +} + +func TestBridgeBatchAcksSuccessfulTopicWhenAnotherTopicFails(t *testing.T) { + rawEnv := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + fieldsEnv := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + rawPayload, err := json.Marshal(rawEnv) + if err != nil { + t.Fatal(err) + } + fieldsPayload, err := json.Marshal(fieldsEnv) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + writer := &recordingBridgeWriter{ + topicErr: map[string]error{ + "vehicle.fields.go.jt808.v1": errors.New("fields topic unavailable"), + }, + } + acks := map[string]int{} + + err = bridgeBatch(context.Background(), registry, writer, []bridgeMessage{ + {subject: "vehicle.fields.go.jt808.v1", data: fieldsPayload, ack: func() error { + acks["fields"]++ + return nil + }}, + {subject: "vehicle.raw.go.jt808.v1", data: rawPayload, ack: func() error { + acks["raw"]++ + return nil + }}, + }, map[string]string{ + "vehicle.fields.go.jt808.v1": "vehicle.fields.go.jt808.v1", + "vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1", + }) + if err == nil { + t.Fatal("bridgeBatch() error = nil, want failed fields topic") + } + if len(writer.calls) != 2 { + t.Fatalf("writer calls = %d, want one call per topic", len(writer.calls)) + } + if len(writer.messages) != 1 || writer.messages[0].Topic != "vehicle.raw.go.jt808.v1" { + t.Fatalf("successful kafka messages = %#v, want only raw topic", writer.messages) + } + if acks["raw"] != 1 || acks["fields"] != 0 { + t.Fatalf("acks = %#v, want raw acked and fields left unacked", acks) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_kafka_writes_total{status="error",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("partial bridge metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.fields.go.jt808.v1"}`) { + t.Fatalf("failed topic should not be acked:\n%s", text) + } +} + +func TestBridgeBatchAcksAndDropsUnroutedSubject(t *testing.T) { + writer := &recordingBridgeWriter{} + acked := 0 + registry := metrics.NewRegistry() + + err := bridgeBatch(context.Background(), registry, writer, []bridgeMessage{ + {subject: "vehicle.unconfigured.v1", data: []byte(`{"protocol":"JT808"}`), ack: func() error { + acked++ + return nil + }}, + }, map[string]string{"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1"}) + if err != nil { + t.Fatalf("bridgeBatch() error = %v", err) + } + if acked != 1 { + t.Fatalf("acks = %d, want unrouted message acked", acked) + } + if len(writer.messages) != 0 { + t.Fatalf("kafka writes = %d, want 0", len(writer.messages)) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_messages_total{status="route_error",subject="vehicle.unconfigured.v1"} 1`, + `vehicle_bridge_nats_acks_total{status="dropped_route_error",subject="vehicle.unconfigured.v1"} 1`, + `vehicle_bridge_batch_pending_kafka_messages 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metrics missing %s:\n%s", want, text) + } + } +} + +func TestBridgeBatchKeepsValidMessagesWhenUnroutedSubjectIsPresent(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} payload, err := json.Marshal(env) if err != nil { t.Fatal(err) } + writer := &recordingBridgeWriter{} + acked := map[string]int{} + + err = bridgeBatch(context.Background(), nil, writer, []bridgeMessage{ + {subject: "vehicle.unconfigured.v1", data: []byte(`{"protocol":"JT808"}`), ack: func() error { + acked["unknown"]++ + return nil + }}, + {subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { + acked["valid"]++ + return nil + }}, + }, map[string]string{"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1"}) + if err != nil { + t.Fatalf("bridgeBatch() error = %v", err) + } + if len(writer.messages) != 1 { + t.Fatalf("kafka writes = %d, want 1 valid message", len(writer.messages)) + } + if writer.messages[0].Topic != "vehicle.raw.go.jt808.v1" { + t.Fatalf("topic = %q", writer.messages[0].Topic) + } + if acked["unknown"] != 1 || acked["valid"] != 1 { + t.Fatalf("acks = %#v, want both messages acked", acked) + } +} + +func TestBridgeBatchReturnsErrorWhenUnroutedAckFails(t *testing.T) { + writer := &recordingBridgeWriter{} + + err := bridgeBatch(context.Background(), nil, writer, []bridgeMessage{ + {subject: "vehicle.unconfigured.v1", data: []byte(`{"protocol":"JT808"}`), ack: func() error { + return errors.New("nats ack unavailable") + }}, + }, map[string]string{"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1"}) + if err == nil { + t.Fatal("bridgeBatch() error = nil, want ack error") + } + if len(writer.messages) != 0 { + t.Fatalf("kafka writes = %d, want 0", len(writer.messages)) + } +} + +func TestBridgeBatchRecordsMetrics(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "13307795425", + MessageID: "0x0200", + ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli(), + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } registry := metrics.NewRegistry() writer := &recordingBridgeWriter{} @@ -83,8 +377,16 @@ func TestBridgeBatchRecordsMetrics(t *testing.T) { text := registry.Render() for _, want := range []string{ `vehicle_bridge_messages_total{status="received",subject="vehicle.raw.jt808.v1"} 1`, + `vehicle_bridge_last_message_unix_seconds{status="received",subject="vehicle.raw.jt808.v1"} `, `vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.jt808.v1"} 1`, + `vehicle_bridge_last_kafka_write_unix_seconds{status="ok",topic="vehicle.raw.jt808.v1"} `, + `vehicle_bridge_kafka_write_duration_ms_histogram_count{status="ok",topic="vehicle.raw.jt808.v1"} 1`, + `vehicle_bridge_kafka_e2e_duration_ms_histogram_count{topic="vehicle.raw.jt808.v1"} 1`, + `vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.jt808.v1"} `, + `vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.jt808.v1"} `, + `vehicle_bridge_last_kafka_e2e_unix_seconds{topic="vehicle.raw.jt808.v1"} `, `vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.jt808.v1"} 1`, + `vehicle_bridge_last_ack_unix_seconds{status="ok",subject="vehicle.raw.jt808.v1"} `, } { if !strings.Contains(text, want) { t.Fatalf("metrics missing %s:\n%s", want, text) @@ -92,6 +394,16 @@ func TestBridgeBatchRecordsMetrics(t *testing.T) { } } +func TestRecordBridgeKafkaE2EDurationSkipsMissingReceiveTime(t *testing.T) { + registry := metrics.NewRegistry() + + recordBridgeKafkaE2EDuration(registry, "vehicle.raw.go.jt808.v1", 0) + + if text := registry.Render(); strings.Contains(text, "vehicle_bridge_kafka_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_bridge_kafka_e2e_recent") { + t.Fatalf("e2e metric should be skipped when received_at_ms is missing:\n%s", text) + } +} + func TestBridgeBatchExposesPendingAndDurationMetrics(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} payload, err := json.Marshal(env) @@ -135,6 +447,73 @@ func TestBridgeBatchExposesPendingAndDurationMetrics(t *testing.T) { } } +func TestBridgeBatchPendingAggregatesConcurrentWorkers(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + firstStarted := make(chan struct{}) + secondStarted := make(chan struct{}) + release := make(chan struct{}) + writerFor := func(started chan struct{}) *recordingBridgeWriter { + return &recordingBridgeWriter{ + onWrite: func() { + close(started) + <-release + }, + } + } + messages := func(count int) []bridgeMessage { + out := make([]bridgeMessage, 0, count) + for i := 0; i < count; i++ { + out = append(out, bridgeMessage{subject: "vehicle.raw.go.gb32960.v1", data: payload, ack: func() error { return nil }}) + } + return out + } + route := map[string]string{"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1"} + var wg sync.WaitGroup + wg.Add(2) + var firstErr error + var secondErr error + go func() { + defer wg.Done() + firstErr = bridgeBatch(context.Background(), registry, writerFor(firstStarted), messages(2), route) + }() + go func() { + defer wg.Done() + secondErr = bridgeBatch(context.Background(), registry, writerFor(secondStarted), messages(3), route) + }() + <-firstStarted + <-secondStarted + + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_batch_pending_messages 5`, + `vehicle_bridge_batch_pending_kafka_messages 5`, + } { + if !strings.Contains(text, want) { + t.Fatalf("aggregate pending bridge metric missing %s during concurrent writes:\n%s", want, text) + } + } + + close(release) + wg.Wait() + if firstErr != nil || secondErr != nil { + t.Fatalf("bridgeBatch errors = %v / %v", firstErr, secondErr) + } + text = registry.Render() + for _, want := range []string{ + `vehicle_bridge_batch_pending_messages 0`, + `vehicle_bridge_batch_pending_kafka_messages 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("aggregate pending bridge metric should reset after concurrent writes, missing %s:\n%s", want, text) + } + } +} + func TestRecordNATSConsumerInfoMetrics(t *testing.T) { registry := metrics.NewRegistry() cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-kafka-bridge"} @@ -157,8 +536,45 @@ func TestRecordNATSConsumerInfoMetrics(t *testing.T) { } } +func TestIsBridgeShutdownFetchError(t *testing.T) { + if isBridgeShutdownFetchError(context.Background(), errors.New("temporary nats failure")) { + t.Fatal("temporary failure should not be treated as shutdown") + } + if !isBridgeShutdownFetchError(context.Background(), nats.ErrConnectionClosed) { + t.Fatal("connection closed should stop worker without noisy error log") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if !isBridgeShutdownFetchError(ctx, errors.New("any fetch error after cancellation")) { + t.Fatal("cancelled context should stop worker without noisy error log") + } +} + +func TestIsTransientBridgeFetchError(t *testing.T) { + for _, err := range []error{ + errors.New("nats: disconnected during fetch"), + errors.New("nats: connection closed"), + errors.New("read tcp: connection reset by peer"), + errors.New("temporary network unavailable"), + errors.New("i/o timeout"), + } { + if !isTransientBridgeFetchError(err) { + t.Fatalf("isTransientBridgeFetchError(%v) = false, want true", err) + } + } + if isTransientBridgeFetchError(errors.New("permission denied")) { + t.Fatal("non-transient bridge fetch error should stay non-transient") + } + if isTransientBridgeFetchError(nil) { + t.Fatal("nil should not be transient") + } +} + func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) { cfg := loadConfig() + if err := cfg.Validate(); err != nil { + t.Fatalf("default config Validate() error = %v", err) + } want := map[string]string{ "vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1", @@ -176,6 +592,75 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) { t.Fatalf("route[%q] = %q, want %q; route=%#v", subject, got, topic, cfg.Route) } } + if !cfg.DeriveFieldsFromRaw { + t.Fatal("DeriveFieldsFromRaw = false, want canonical raw projection enabled by default") + } + for subject, wantTopic := range map[string]string{ + topics.RawGB32960: topics.FieldsGB32960, + topics.RawJT808: topics.FieldsJT808, + topics.RawYutongMQTT: topics.FieldsYutongMQTT, + } { + if got := cfg.RawFieldRoutes[subject].Topic; got != wantTopic { + t.Fatalf("RawFieldRoutes[%q].Topic = %q, want %q", subject, got, wantTopic) + } + } +} + +func TestConfigValidateRejectsRawFieldsSubjectOverlap(t *testing.T) { + cfg := config{ + RawRoutes: []subjectRoute{ + {Subject: "vehicle.same.jt808", Topic: "vehicle.raw.go.jt808.v1"}, + }, + FieldsRoutes: []subjectRoute{ + {Subject: "vehicle.same.jt808", Topic: "vehicle.fields.go.jt808.v1"}, + }, + } + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want subject overlap rejection") + } + if !strings.Contains(err.Error(), "nats subject") { + t.Fatalf("Validate() error = %q, want nats subject hint", err) + } +} + +func TestConfigValidateRejectsKnownProtocolTopicMismatch(t *testing.T) { + cfg := config{ + RawRoutes: []subjectRoute{ + {Protocol: envelope.ProtocolJT808, Subject: "vehicle.raw.go.jt808.v1", Topic: "vehicle.raw.go.gb32960.v1"}, + }, + } + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want known protocol topic mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("Validate() error = %q, want protocol mismatch hint", err) + } +} + +func TestConfigValidateRejectsFieldsRouteToRawKafkaTopic(t *testing.T) { + cfg := config{ + RawRoutes: []subjectRoute{ + {Subject: "vehicle.raw.go.jt808.v1", Topic: "vehicle.raw.go.jt808.v1"}, + }, + FieldsRoutes: []subjectRoute{ + {Subject: "vehicle.fields.go.jt808.v1", Topic: "vehicle.raw.go.jt808.v1"}, + }, + } + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want fields topic family rejection") + } + if !strings.Contains(err.Error(), "fields kafka topic") { + t.Fatalf("Validate() error = %q, want fields kafka topic hint", err) + } } func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) { @@ -187,11 +672,27 @@ func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) { if got, want := cfg.StreamEnsureWait, 60*time.Second; got != want { t.Fatalf("StreamEnsureWait = %v, want %v", got, want) } + if got, want := cfg.KafkaBatchTimeout, 20*time.Millisecond; got != want { + t.Fatalf("KafkaBatchTimeout = %v, want %v", got, want) + } + if got, want := cfg.KafkaWriteConcurrency, 6; got != want { + t.Fatalf("KafkaWriteConcurrency = %d, want %d", got, want) + } + if got, want := cfg.FetchWait, 20*time.Millisecond; got != want { + t.Fatalf("FetchWait = %v, want %v", got, want) + } + if got, want := cfg.Workers, 4; got != want { + t.Fatalf("Workers = %d, want %d", got, want) + } } func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) { t.Setenv("NATS_STREAM_MAX_BYTES", "1073741824") t.Setenv("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", "90") + t.Setenv("BRIDGE_KAFKA_BATCH_TIMEOUT_MS", "35") + t.Setenv("BRIDGE_KAFKA_WRITE_CONCURRENCY", "4") + t.Setenv("BRIDGE_FETCH_WAIT_MS", "45") + t.Setenv("BRIDGE_WORKERS", "6") cfg := loadConfig() @@ -201,6 +702,57 @@ func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) { if got, want := cfg.StreamEnsureWait, 90*time.Second; got != want { t.Fatalf("StreamEnsureWait = %v, want %v", got, want) } + if got, want := cfg.KafkaBatchTimeout, 35*time.Millisecond; got != want { + t.Fatalf("KafkaBatchTimeout = %v, want %v", got, want) + } + if got, want := cfg.KafkaWriteConcurrency, 4; got != want { + t.Fatalf("KafkaWriteConcurrency = %d, want %d", got, want) + } + if got, want := cfg.FetchWait, 45*time.Millisecond; got != want { + t.Fatalf("FetchWait = %v, want %v", got, want) + } + if got, want := cfg.Workers, 6; got != want { + t.Fatalf("Workers = %d, want %d", got, want) + } +} + +func TestRecordBridgeConfigMetrics(t *testing.T) { + registry := metrics.NewRegistry() + + recordBridgeConfigMetrics(registry, config{ + KafkaBatchTimeout: 35 * time.Millisecond, + KafkaWriteConcurrency: 4, + BatchSize: 600, + FetchWait: 20 * time.Millisecond, + Workers: 6, + DeriveFieldsFromRaw: true, + }) + + text := registry.Render() + for _, want := range []string{ + `vehicle_bridge_config{setting="batch_size"} 600`, + `vehicle_bridge_config{setting="fetch_wait_ms"} 20`, + `vehicle_bridge_config{setting="kafka_batch_timeout_ms"} 35`, + `vehicle_bridge_config{setting="kafka_write_concurrency"} 4`, + `vehicle_bridge_config{setting="workers"} 6`, + `vehicle_bridge_config{setting="derive_fields_from_raw_enabled"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("bridge config metric missing %s:\n%s", want, text) + } + } +} + +func TestNewKafkaWriterUsesConfiguredBatchTimeout(t *testing.T) { + writer := newKafkaWriter(config{ + KafkaBrokers: []string{"127.0.0.1:9092"}, + KafkaBatchTimeout: 17 * time.Millisecond, + }) + defer writer.Close() + + if got, want := writer.BatchTimeout, 17*time.Millisecond; got != want { + t.Fatalf("BatchTimeout = %v, want %v", got, want) + } } func TestLoadConfigIncludesUnifiedOnlyWhenExplicitlyConfigured(t *testing.T) { @@ -215,8 +767,11 @@ func TestLoadConfigIncludesUnifiedOnlyWhenExplicitlyConfigured(t *testing.T) { } type recordingBridgeWriter struct { + mu sync.Mutex messages []kafka.Message + calls [][]kafka.Message err error + topicErr map[string]error onWrite func() } @@ -224,9 +779,72 @@ func (w *recordingBridgeWriter) WriteMessages(_ context.Context, messages ...kaf if w.onWrite != nil { w.onWrite() } + w.mu.Lock() + defer w.mu.Unlock() + call := append([]kafka.Message(nil), messages...) + w.calls = append(w.calls, call) + if len(messages) > 0 && w.topicErr != nil { + if err := w.topicErr[messages[0].Topic]; err != nil { + return err + } + } if w.err != nil { return w.err } w.messages = append(w.messages, messages...) return nil } + +type concurrentBridgeWriter struct { + started chan string + release chan struct{} + active atomic.Int32 + max atomic.Int32 +} + +func (w *concurrentBridgeWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error { + active := w.active.Add(1) + defer w.active.Add(-1) + for { + current := w.max.Load() + if active <= current || w.max.CompareAndSwap(current, active) { + break + } + } + w.started <- messages[0].Topic + <-w.release + return nil +} + +func TestBridgeWritesIndependentKafkaTopicsConcurrently(t *testing.T) { + writer := &concurrentBridgeWriter{ + started: make(chan string, 2), + release: make(chan struct{}), + } + messages := []bridgeMessage{ + {subject: topics.RawJT808, data: []byte(`{"protocol":"JT808"}`), ack: func() error { return nil }}, + {subject: topics.RawGB32960, data: []byte(`{"protocol":"GB32960"}`), ack: func() error { return nil }}, + } + route := map[string]string{ + topics.RawJT808: topics.RawJT808, + topics.RawGB32960: topics.RawGB32960, + } + done := make(chan error, 1) + go func() { + done <- bridgeBatchWithProjectionConcurrency(context.Background(), nil, writer, messages, route, nil, false, 2) + }() + for i := 0; i < 2; i++ { + select { + case <-writer.started: + case <-time.After(time.Second): + t.Fatal("independent Kafka topic writes did not start concurrently") + } + } + if got := writer.max.Load(); got != 2 { + t.Fatalf("max concurrent Kafka writes = %d, want 2", got) + } + close(writer.release) + if err := <-done; err != nil { + t.Fatalf("bridgeBatchWithProjectionConcurrency() error = %v", err) + } +} diff --git a/go/vehicle-gateway/cmd/realtime-api/main.go b/go/vehicle-gateway/cmd/realtime-api/main.go index de619380..e3ac86a8 100644 --- a/go/vehicle-gateway/cmd/realtime-api/main.go +++ b/go/vehicle-gateway/cmd/realtime-api/main.go @@ -4,11 +4,15 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "net/http" "os" "os/signal" + "sort" "strconv" "strings" + "sync" "syscall" "time" @@ -18,6 +22,7 @@ import ( _ "github.com/taosdata/driver-go/v3/taosWS" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/history" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" @@ -27,44 +32,72 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) +const ( + defaultThrottleCacheRetention = 72 * time.Hour + defaultThrottleCacheCleanupInterval = 10 * time.Minute +) + func main() { - logger := observability.NewLogger("vehicle-realtime-api") + role := realtimeRoleFromEnv() + apiEnabled := realtimeRoleAllowsAPI(role) + writerEnabled := realtimeRoleAllowsWriter(role) + redisProjectorEnabled := env("REALTIME_REDIS_PROJECTOR_ENABLED", "true") != "false" + serviceName := realtimeServiceName(role) + logger := observability.NewLogger(serviceName) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - client := redis.NewClient(&redis.Options{ - Addr: env("REDIS_ADDR", "127.0.0.1:6379"), - Username: env("REDIS_USERNAME", ""), - Password: env("REDIS_PASSWORD", ""), - DB: envInt("REDIS_DB", 0), - }) - defer client.Close() - if err := client.Ping(ctx).Err(); err != nil { - logger.Error("redis ping failed", "error", err) - os.Exit(1) - } - - repository := realtime.NewRepository(client, realtime.Config{ - OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", envInt("REALTIME_ONLINE_TTL_SECONDS", 60))) * time.Second, - }) - mux := http.NewServeMux() registry := metrics.NewRegistry() - healthChecks := []health.Check{ - {Name: "redis", Check: func(ctx context.Context) error { return client.Ping(ctx).Err() }}, + metrics.RegisterServiceInfo(registry, serviceName) + registry.SetGauge("vehicle_realtime_role_info", metrics.Labels{"role": role}, 1) + registry.SetGauge("vehicle_realtime_writer_enabled", nil, boolGauge(writerEnabled)) + registry.SetGauge("vehicle_realtime_redis_projector_enabled", nil, boolGauge(redisProjectorEnabled)) + healthChecks := []health.Check{} + var client *redis.Client + var repository *realtime.Repository + if realtimeRoleNeedsRedis(apiEnabled, writerEnabled, redisProjectorEnabled) { + client = redis.NewClient(&redis.Options{ + Addr: env("REDIS_ADDR", "127.0.0.1:6379"), + Username: env("REDIS_USERNAME", ""), + Password: env("REDIS_PASSWORD", ""), + DB: envInt("REDIS_DB", 0), + }) + defer client.Close() + if err := client.Ping(ctx).Err(); err != nil { + logger.Error("redis ping failed", "error", err) + os.Exit(1) + } + repository = realtime.NewRepository(client, realtime.Config{ + OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", envInt("REALTIME_ONLINE_TTL_SECONDS", 60))) * time.Second, + }) + healthChecks = append(healthChecks, health.Check{Name: "redis", Check: func(ctx context.Context) error { return client.Ping(ctx).Err() }}) + } else { + logger.Info("redis dependency disabled by role", "role", role, "redis_projector_enabled", redisProjectorEnabled) } mux.Handle("/metrics", metrics.NewHandler(registry)) - mux.Handle("/openapi.json", realtime.NewOpenAPIHandler()) - mux.Handle("/swagger-ui/", realtime.NewSwaggerUIHandler()) - mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository)) - mux.Handle("/api/realtime/online", realtime.NewOnlineIndexHandler(repository)) - mux.Handle("/api/debug/pipeline", realtime.NewPipelineDebugHandler(repository)) - closeStats := func() {} - var updater realtimeUpdater = storeMetricUpdater{ - store: "redis", - delegate: fastRealtimeUpdater{repository: repository}, - registry: registry, + if apiEnabled { + mux.Handle("/openapi.json", realtime.NewOpenAPIHandler()) + mux.Handle("/swagger-ui/", realtime.NewSwaggerUIHandler()) + mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository)) + mux.Handle("/api/realtime/online", realtime.NewOnlineIndexHandler(repository)) + mux.Handle("/api/debug/pipeline", realtime.NewPipelineDebugHandler(repository)) + } else { + logger.Info("realtime query api disabled by role", "role", role) } + closeStats := func() {} + closeRealtimeUpdaters := func() {} + var redisUpdater realtimeUpdater + if writerEnabled && redisProjectorEnabled { + redisUpdater = storeMetricUpdater{ + store: "redis", + delegate: fastRealtimeUpdater{repository: repository}, + registry: registry, + } + } else { + logger.Info("realtime redis kafka projector disabled; redis query api still enabled") + } + var updater realtimeUpdater = redisUpdater if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" { db, err := sql.Open("mysql", dsn) if err != nil { @@ -79,12 +112,18 @@ func main() { healthChecks = append(healthChecks, health.Check{Name: "mysql", Check: db.PingContext}) closeStats = func() { _ = db.Close() } bindingTable := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding") - plateResolver := realtime.NewCachedPlateResolver( + plateCacheMaxEntries := envInt("PLATE_CACHE_MAX_ENTRIES", 200000) + plateResolver := realtime.NewCachedPlateResolverWithMaxEntries( realtime.NewBindingPlateResolver(db, bindingTable), time.Duration(envInt("PLATE_CACHE_TTL_SECONDS", 600))*time.Second, + plateCacheMaxEntries, ) + plateResolver.SetStatsObserver(func(stats realtime.PlateCacheStats) { + recordPlateCacheStats(registry, stats) + }) + registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(plateCacheMaxEntries)) snapshotWriter := realtime.NewSnapshotWriterWithPlateResolver(db, plateResolver) - if env("MYSQL_REALTIME_SNAPSHOT_ENABLED", "true") != "false" { + if writerEnabled && env("MYSQL_REALTIME_SNAPSHOT_ENABLED", "true") != "false" { if err := snapshotWriter.EnsureSchema(ctx); err != nil { _ = db.Close() logger.Error("realtime snapshot mysql schema bootstrap failed", "error", err) @@ -95,78 +134,149 @@ func main() { attempts: envInt("MYSQL_REALTIME_RETRY_ATTEMPTS", 3), delay: time.Duration(envInt("MYSQL_REALTIME_RETRY_DELAY_MS", 20)) * time.Millisecond, }) - mysqlUpdater := realtimeUpdater(storeMetricUpdater{store: "mysql", delegate: mysqlDelegate, registry: registry}) - if env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false" { - mysqlUpdater = newAsyncSecondaryRealtimeUpdater( + mysqlStoreUpdater := realtimeUpdater(storeMetricUpdater{store: "mysql", delegate: mysqlDelegate, registry: registry}) + mysqlUpdater := mysqlStoreUpdater + minInterval := time.Duration(envInt("MYSQL_REALTIME_MIN_INTERVAL_MS", 10000)) * time.Millisecond + if redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false" { + asyncUpdater := newAsyncSecondaryRealtimeUpdater( nil, - storeMetricUpdater{store: "mysql", delegate: mysqlDelegate, registry: registry}, + mysqlStoreUpdater, envInt("MYSQL_REALTIME_ASYNC_QUEUE_SIZE", 20000), envInt("MYSQL_REALTIME_ASYNC_WORKERS", 4), registry, logger, ) + closeRealtimeUpdaters = asyncUpdater.Close + mysqlUpdater = asyncUpdater } - updater = compositeRealtimeUpdater{ - primary: storeMetricUpdater{store: "redis", delegate: fastRealtimeUpdater{repository: repository}, registry: registry}, - secondary: mysqlUpdater, - } - logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot", "plate_binding_table", bindingTable, "plate_cache_ttl_seconds", envInt("PLATE_CACHE_TTL_SECONDS", 600)) + throttleRetention := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_RETENTION_HOURS", 72)) * time.Hour + throttleCleanupInterval := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second + registry.SetGauge("vehicle_realtime_throttle_cache_retention_seconds", metrics.Labels{"store": "mysql"}, throttleRetention.Seconds()) + registry.SetGauge("vehicle_realtime_throttle_cache_cleanup_interval_seconds", metrics.Labels{"store": "mysql"}, throttleCleanupInterval.Seconds()) + mysqlUpdater = newThrottledRealtimeUpdaterWithCache(mysqlUpdater, minInterval, "mysql", registry, throttleRetention, throttleCleanupInterval) + updater = composeRealtimeUpdater(redisUpdater, mysqlUpdater) + logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot", "plate_binding_table", bindingTable, "plate_cache_ttl_seconds", envInt("PLATE_CACHE_TTL_SECONDS", 600), "plate_cache_max_entries", plateCacheMaxEntries, "async_enabled", redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false", "min_interval_ms", minInterval.Milliseconds(), "throttle_cache_retention_seconds", throttleRetention.Seconds(), "throttle_cache_cleanup_interval_seconds", throttleCleanupInterval.Seconds()) + } + if apiEnabled { + metricHandler := stats.NewMetricHandler(stats.NewMetricRepository(db)) + mux.Handle("/api/stats/daily-metrics", metricHandler) + mux.Handle("/api/stats/daily-metrics/sources", metricHandler) + mux.Handle("/api/stats/daily-metrics/sources/quality", metricHandler) + mux.Handle("/api/stats/daily-metrics/sources/selection", metricHandler) + mux.Handle("/api/stats/daily-metrics/diagnostics", metricHandler) + mux.Handle("/api/stats/daily-metrics/diagnostics/summary", metricHandler) + mux.Handle("/api/stats/daily-metrics/diagnostics/reasons", metricHandler) + mux.Handle("/api/stats/daily-metrics/diagnostics/field-status", metricHandler) + dataSourceHandler := stats.NewDataSourceHandler(stats.NewDataSourceRepository(db)) + mux.Handle("/api/stats/data-sources", dataSourceHandler) + mux.Handle("/api/stats/data-sources/", dataSourceHandler) + mux.Handle("/api/realtime/snapshots", realtime.NewSnapshotQueryHandler(realtime.NewSnapshotQueryRepository(db))) + mux.Handle("/api/realtime/locations", realtime.NewLocationQueryHandler(realtime.NewLocationQueryRepository(db))) + logger.Info("stats mysql query enabled") } - mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db))) - mux.Handle("/api/realtime/snapshots", realtime.NewSnapshotQueryHandler(realtime.NewSnapshotQueryRepository(db))) - mux.Handle("/api/realtime/locations", realtime.NewLocationQueryHandler(realtime.NewLocationQueryRepository(db))) - logger.Info("stats mysql query enabled") } else { mysqlUnavailable := func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusServiceUnavailable) _ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"}) } - mux.HandleFunc("/api/stats/daily-metrics", mysqlUnavailable) - mux.HandleFunc("/api/realtime/snapshots", mysqlUnavailable) - mux.HandleFunc("/api/realtime/locations", mysqlUnavailable) - logger.Warn("MYSQL_DSN is empty; stats query api disabled") + if apiEnabled { + mux.HandleFunc("/api/stats/daily-metrics", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/sources", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/sources/quality", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/sources/selection", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/diagnostics", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/diagnostics/summary", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/diagnostics/reasons", mysqlUnavailable) + mux.HandleFunc("/api/stats/daily-metrics/diagnostics/field-status", mysqlUnavailable) + mux.HandleFunc("/api/stats/data-sources", mysqlUnavailable) + mux.HandleFunc("/api/stats/data-sources/", mysqlUnavailable) + mux.HandleFunc("/api/realtime/snapshots", mysqlUnavailable) + mux.HandleFunc("/api/realtime/locations", mysqlUnavailable) + logger.Warn("MYSQL_DSN is empty; stats query api disabled") + } } defer closeStats() - if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 { - go consumeKafka(ctx, logger, registry, updater, brokers) - } else { + defer closeRealtimeUpdaters() + var realtimeConsumers sync.WaitGroup + if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); writerEnabled && len(brokers) > 0 { + if updater == nil { + logger.Warn("KAFKA_BROKERS is configured but realtime kafka projector has no enabled store") + } else { + kafkaTopics := kafkaTopicsFromEnv() + workerCount := envInt("REALTIME_WORKERS", 3) + if err := validateRealtimeConsumerConfig(kafkaTopics, workerCount); err != nil { + logger.Error("invalid realtime writer config", "error", err) + os.Exit(1) + } + metrics.RegisterKafkaConsumerInfo(registry, "vehicle-realtime-api", env("KAFKA_GROUP", "go-realtime-api"), kafkaTopics) + registry.SetGauge("vehicle_realtime_config", metrics.Labels{"setting": "workers"}, float64(workerCount)) + for workerID := 1; workerID <= workerCount; workerID++ { + realtimeConsumers.Add(1) + go func(id int) { + defer realtimeConsumers.Done() + consumeKafkaWorker( + ctx, + logger, + registry, + updater, + brokers, + kafkaTopics, + envInt("REALTIME_BATCH_SIZE", 100), + time.Duration(envInt("REALTIME_BATCH_WAIT_MS", 20))*time.Millisecond, + time.Duration(envInt("REALTIME_FAILURE_RETRY_DELAY_MS", 100))*time.Millisecond, + id, + ) + }(workerID) + } + } + } else if writerEnabled { logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only") - } - closeHistory := func() {} - if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" { - driver := env("TDENGINE_DRIVER", "taosWS") - db, err := sql.Open(driver, dsn) - if err != nil { - logger.Error("history tdengine open failed", "driver", driver, "error", err) - os.Exit(1) - } - if err := db.PingContext(ctx); err != nil { - _ = db.Close() - logger.Error("history tdengine ping failed", "driver", driver, "error", err) - os.Exit(1) - } - healthChecks = append(healthChecks, health.Check{Name: "tdengine", Check: db.PingContext}) - closeHistory = func() { _ = db.Close() } - database := env("TDENGINE_DATABASE", history.DefaultDatabase) - rawFrameHandler := history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)) - mux.Handle("/api/history/raw-frames", rawFrameHandler) - mux.Handle("/api/history/raw-frames/query", rawFrameHandler) - mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database))) - logger.Info("history query enabled", "driver", driver, "database", database) } else { - historyUnavailable := func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusServiceUnavailable) - _ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"}) + logger.Info("realtime kafka projector disabled by role", "role", role) + } + defer func() { + stop() + realtimeConsumers.Wait() + }() + closeHistory := func() {} + if apiEnabled { + if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" { + driver := env("TDENGINE_DRIVER", "taosWS") + db, err := sql.Open(driver, dsn) + if err != nil { + logger.Error("history tdengine open failed", "driver", driver, "error", err) + os.Exit(1) + } + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + logger.Error("history tdengine ping failed", "driver", driver, "error", err) + os.Exit(1) + } + healthChecks = append(healthChecks, health.Check{Name: "tdengine", Check: db.PingContext}) + closeHistory = func() { _ = db.Close() } + database := env("TDENGINE_DATABASE", history.DefaultDatabase) + rawFrameHandler := history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)) + mux.Handle("/api/history/raw-frames", rawFrameHandler) + mux.Handle("/api/history/raw-frames/query", rawFrameHandler) + mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database))) + logger.Info("history query enabled", "driver", driver, "database", database) + } else { + historyUnavailable := func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"}) + } + mux.HandleFunc("/api/history/raw-frames", historyUnavailable) + mux.HandleFunc("/api/history/raw-frames/query", historyUnavailable) + mux.HandleFunc("/api/history/locations", historyUnavailable) + logger.Warn("TDENGINE_DSN is empty; history query api disabled") } - mux.HandleFunc("/api/history/raw-frames", historyUnavailable) - mux.HandleFunc("/api/history/raw-frames/query", historyUnavailable) - mux.HandleFunc("/api/history/locations", historyUnavailable) - logger.Warn("TDENGINE_DSN is empty; history query api disabled") + } else { + logger.Info("tdengine dependency disabled by role", "role", role) } defer closeHistory() - healthHandler := health.NewHandler("vehicle-realtime-api", healthChecks) + healthHandler := health.NewHandler(serviceName, healthChecks) mux.Handle("/healthz", healthHandler) mux.Handle("/readyz", healthHandler) @@ -182,7 +292,7 @@ func main() { _ = server.Shutdown(shutdownCtx) }() - logger.Info("realtime api started", "addr", server.Addr) + logger.Info("realtime service started", "service", serviceName, "addr", server.Addr, "role", role) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Error("http server failed", "error", err) os.Exit(1) @@ -195,11 +305,13 @@ type realtimeLogger interface { Warn(string, ...any) } -func consumeKafka(ctx context.Context, logger realtimeLogger, registry *metrics.Registry, updater realtimeUpdater, brokers []string) { - kafkaTopics := kafkaTopicsFromEnv() +func consumeKafkaWorker(ctx context.Context, logger realtimeLogger, registry *metrics.Registry, updater realtimeUpdater, brokers []string, kafkaTopics []string, batchSize int, batchWait time.Duration, retryDelay time.Duration, workerID int) { reader := kafka.NewReader(realtimeReaderConfig(brokers, kafkaTopics)) defer reader.Close() - logger.Info("realtime kafka consumer started", "topics", strings.Join(kafkaTopics, ",")) + workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)} + registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 1) + defer registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 0) + logger.Info("realtime kafka consumer started", "worker", workerID, "topics", strings.Join(kafkaTopics, ","), "batch_size", batchSize, "batch_wait_ms", batchWait.Milliseconds()) for { message, err := reader.FetchMessage(ctx) if err != nil { @@ -209,10 +321,26 @@ func consumeKafka(ctx context.Context, logger realtimeLogger, registry *metrics. logger.Error("kafka fetch failed", "error", err) continue } - processRealtimeMessage(ctx, logger, registry, updater, reader, message) + batch := collectRealtimeBatch(ctx, reader, message, batchSize, batchWait) + processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, reader, batch, retryDelay, workerLabels) } } +func validateRealtimeConsumerConfig(kafkaTopics []string, workers int) error { + if len(kafkaTopics) == 0 { + return errors.New("KAFKA_TOPICS must include raw topics") + } + for _, topic := range kafkaTopics { + if _, ok := topics.ProtocolForKnownRawTopic(topic); !ok { + return fmt.Errorf("realtime-writer consumes known raw topics only, got %q", topic) + } + } + if workers <= 0 { + return errors.New("REALTIME_WORKERS must be greater than zero") + } + return nil +} + func realtimeReaderConfig(brokers []string, kafkaTopics []string) kafka.ReaderConfig { return kafka.ReaderConfig{ Brokers: brokers, @@ -235,6 +363,10 @@ type realtimeUpdater interface { Update(context.Context, envelope.FrameEnvelope) error } +type realtimeBatchUpdater interface { + UpdateBatch(context.Context, []envelope.FrameEnvelope) error +} + type fastRealtimeUpdater struct { repository *realtime.Repository } @@ -243,6 +375,13 @@ func (u fastRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvel return u.repository.FastUpdate(ctx, env) } +func (u fastRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + if u.repository == nil || len(envs) == 0 { + return nil + } + return u.repository.FastUpdateBatch(ctx, envs) +} + type storeMetricUpdater struct { store string delegate realtimeUpdater @@ -272,10 +411,55 @@ func (u storeMetricUpdater) Update(ctx context.Context, env envelope.FrameEnvelo u.registry.IncCounter("vehicle_realtime_store_updates_total", labels) u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS) u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS) + metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels) } return err } +func (u storeMetricUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + if u.delegate == nil || len(envs) == 0 { + return nil + } + batchDelegate, ok := u.delegate.(realtimeBatchUpdater) + if !ok { + for _, env := range envs { + if err := u.Update(ctx, env); err != nil { + return err + } + } + return nil + } + started := time.Now() + err := batchDelegate.UpdateBatch(ctx, envs) + elapsed := time.Since(started) + status := statusFromError(err) + if u.registry != nil { + elapsedMS := float64(elapsed.Milliseconds()) + counts := realtimeProtocolCounts(envs) + for protocol, count := range counts { + labels := metrics.Labels{ + "store": u.store, + "protocol": string(protocol), + "status": status, + } + u.registry.AddCounter("vehicle_realtime_store_updates_total", labels, float64(count)) + u.registry.IncCounter("vehicle_realtime_store_batch_updates_total", labels) + u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS) + u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS) + metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels) + } + } + return err +} + +func realtimeProtocolCounts(envs []envelope.FrameEnvelope) map[envelope.Protocol]int { + counts := make(map[envelope.Protocol]int) + for _, env := range envs { + counts[env.Protocol]++ + } + return counts +} + type retryRealtimeUpdater struct { delegate realtimeUpdater attempts int @@ -310,13 +494,207 @@ func (u retryRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnve return err } +func (u retryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + if u.delegate == nil || len(envs) == 0 { + return nil + } + attempts := u.attempts + if attempts <= 0 { + attempts = 1 + } + var err error + for attempt := 1; attempt <= attempts; attempt++ { + err = updateRealtimeBatch(ctx, u.delegate, envs) + if err == nil || !isTransientMySQLWriteError(err) || attempt == attempts { + return err + } + if u.delay <= 0 { + continue + } + timer := time.NewTimer(u.delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + return err +} + +type throttledRealtimeUpdater struct { + delegate realtimeUpdater + interval time.Duration + store string + registry *metrics.Registry + cacheRetention time.Duration + cleanupInterval time.Duration + mu sync.Mutex + lastMS map[string]int64 + lastCleanupMS int64 + lastDeleted int +} + +func newThrottledRealtimeUpdater(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry) realtimeUpdater { + return newThrottledRealtimeUpdaterWithCache(delegate, interval, store, registry, defaultThrottleCacheRetention, defaultThrottleCacheCleanupInterval) +} + +func newThrottledRealtimeUpdaterWithCache(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry, cacheRetention time.Duration, cleanupInterval time.Duration) realtimeUpdater { + if delegate == nil || interval <= 0 { + return delegate + } + if cacheRetention < 0 { + cacheRetention = 0 + } + if cleanupInterval < 0 { + cleanupInterval = 0 + } + return &throttledRealtimeUpdater{ + delegate: delegate, + interval: interval, + store: store, + registry: registry, + cacheRetention: cacheRetention, + cleanupInterval: cleanupInterval, + lastMS: map[string]int64{}, + } +} + +func (u *throttledRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { + if u.delegate == nil { + return nil + } + pass, status := u.shouldPass(env) + u.record(status, env) + u.recordCacheMetrics() + if !pass { + return nil + } + return u.delegate.Update(ctx, env) +} + +func (u *throttledRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + for _, env := range envs { + if err := u.Update(ctx, env); err != nil { + return err + } + } + return nil +} + +func (u *throttledRealtimeUpdater) shouldPass(env envelope.FrameEnvelope) (bool, string) { + if !envelope.IsRealtimeTelemetryFrame(env) { + return false, "skipped_non_realtime" + } + vin := strings.TrimSpace(env.VIN) + if vin == "" { + return false, "skipped_missing_vin" + } + eventMS, ok := envelope.NormalizedEventTimeMS(env) + if !ok || eventMS <= 0 { + eventMS = env.ReceivedAtMS + } + if eventMS <= 0 { + eventMS = time.Now().UnixMilli() + } + key := string(env.Protocol) + "\x00" + vin + intervalMS := u.interval.Milliseconds() + u.mu.Lock() + defer u.mu.Unlock() + if u.lastMS == nil { + u.lastMS = map[string]int64{} + } + u.cleanupLocked(throttleCleanupTimeMS(env, eventMS)) + if last := u.lastMS[key]; last > 0 && eventMS < last+intervalMS { + return false, "skipped_interval" + } + u.lastMS[key] = eventMS + return true, "passed" +} + +func (u *throttledRealtimeUpdater) cleanupLocked(nowMS int64) { + if u.cacheRetention <= 0 || nowMS <= 0 { + return + } + intervalMS := u.cleanupInterval.Milliseconds() + if intervalMS > 0 && u.lastCleanupMS > 0 && nowMS < u.lastCleanupMS+intervalMS { + return + } + u.lastCleanupMS = nowMS + cutoff := nowMS - u.cacheRetention.Milliseconds() + deleted := 0 + for key, last := range u.lastMS { + if last > 0 && last < cutoff { + delete(u.lastMS, key) + deleted++ + } + } + u.lastDeleted = deleted +} + +func throttleCleanupTimeMS(env envelope.FrameEnvelope, fallbackMS int64) int64 { + if env.ReceivedAtMS > 0 { + return env.ReceivedAtMS + } + if fallbackMS > 0 { + return fallbackMS + } + return time.Now().UnixMilli() +} + +func (u *throttledRealtimeUpdater) record(status string, env envelope.FrameEnvelope) { + if u.registry == nil { + return + } + u.registry.IncCounter("vehicle_realtime_throttle_total", metrics.Labels{ + "store": u.store, + "protocol": string(env.Protocol), + "status": status, + }) +} + +func (u *throttledRealtimeUpdater) recordCacheMetrics() { + if u.registry == nil { + return + } + u.mu.Lock() + entries := len(u.lastMS) + lastDeleted := u.lastDeleted + lastCleanupMS := u.lastCleanupMS + u.mu.Unlock() + labels := metrics.Labels{"store": u.store} + u.registry.SetGauge("vehicle_realtime_throttle_cache_entries", labels, float64(entries)) + u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_deleted", labels, float64(lastDeleted)) + if lastCleanupMS > 0 { + u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_unix_seconds", labels, float64(lastCleanupMS)/1000) + } +} + func isTransientMySQLWriteError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } text := strings.ToLower(strings.TrimSpace(err.Error())) return strings.Contains(text, "deadlock") || strings.Contains(text, "error 1213") || strings.Contains(text, "40001") || strings.Contains(text, "lock wait timeout") || - strings.Contains(text, "error 1205") + strings.Contains(text, "error 1205") || + strings.Contains(text, "timeout") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "connection refused") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "bad connection") || + strings.Contains(text, "invalid connection") || + strings.Contains(text, "i/o timeout") || + text == "eof" || + strings.Contains(text, "unexpected eof") || + strings.Contains(text, "server is down") || + strings.Contains(text, "network is unreachable") || + strings.Contains(text, "no route to host") } type compositeRealtimeUpdater struct { @@ -324,9 +702,21 @@ type compositeRealtimeUpdater struct { secondary realtimeUpdater } +func composeRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater) realtimeUpdater { + if primary == nil { + return secondary + } + if secondary == nil { + return primary + } + return compositeRealtimeUpdater{primary: primary, secondary: secondary} +} + func (u compositeRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { - if err := u.primary.Update(ctx, env); err != nil { - return err + if u.primary != nil { + if err := u.primary.Update(ctx, env); err != nil { + return err + } } if u.secondary == nil { return nil @@ -334,13 +724,24 @@ func (u compositeRealtimeUpdater) Update(ctx context.Context, env envelope.Frame return u.secondary.Update(ctx, env) } +func (u compositeRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + if u.primary != nil { + if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil { + return err + } + } + return updateRealtimeBatch(ctx, u.secondary, envs) +} + type asyncSecondaryRealtimeUpdater struct { primary realtimeUpdater secondary realtimeUpdater queue chan envelope.FrameEnvelope - cancel context.CancelFunc registry *metrics.Registry logger realtimeLogger + closeOnce sync.Once + closed chan struct{} + wg sync.WaitGroup } func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater, queueSize int, workers int, registry *metrics.Registry, logger realtimeLogger) *asyncSecondaryRealtimeUpdater { @@ -350,17 +751,18 @@ func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtim if workers <= 0 { workers = 4 } - ctx, cancel := context.WithCancel(context.Background()) updater := &asyncSecondaryRealtimeUpdater{ primary: primary, secondary: secondary, queue: make(chan envelope.FrameEnvelope, queueSize), - cancel: cancel, registry: registry, logger: logger, + closed: make(chan struct{}), } + updater.recordQueueCapacity() + updater.wg.Add(workers) for i := 0; i < workers; i++ { - go updater.run(ctx) + go updater.run() } return updater } @@ -375,10 +777,21 @@ func (u *asyncSecondaryRealtimeUpdater) Update(ctx context.Context, env envelope return nil } select { + case <-u.closed: + u.recordQueueMetric(env, "closed") + u.recordQueueDepth(env) + return nil + default: + } + select { case u.queue <- env: u.recordQueueMetric(env, "queued") u.recordQueueDepth(env) return nil + case <-u.closed: + u.recordQueueMetric(env, "closed") + u.recordQueueDepth(env) + return nil default: u.recordQueueMetric(env, "dropped") u.recordQueueDepth(env) @@ -386,21 +799,74 @@ func (u *asyncSecondaryRealtimeUpdater) Update(ctx context.Context, env envelope } } -func (u *asyncSecondaryRealtimeUpdater) run(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case env := <-u.queue: - u.recordQueueDepth(env) - secondaryCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout) - if err := u.secondary.Update(secondaryCtx, env); err != nil && u.logger != nil { - u.logger.Error("async realtime secondary update failed", "protocol", env.Protocol, "vin", env.VIN, "event_id", env.StableEventID(), "error", err) - } - cancel() - u.recordQueueDepth(env) +func (u *asyncSecondaryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + if u.primary != nil { + if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil { + return err } } + if u.secondary == nil { + return nil + } + for _, env := range envs { + if err := u.enqueueSecondary(env); err != nil { + return err + } + } + return nil +} + +func (u *asyncSecondaryRealtimeUpdater) enqueueSecondary(env envelope.FrameEnvelope) error { + select { + case <-u.closed: + u.recordQueueMetric(env, "closed") + u.recordQueueDepth(env) + return nil + default: + } + select { + case u.queue <- env: + u.recordQueueMetric(env, "queued") + u.recordQueueDepth(env) + return nil + case <-u.closed: + u.recordQueueMetric(env, "closed") + u.recordQueueDepth(env) + return nil + default: + u.recordQueueMetric(env, "dropped") + u.recordQueueDepth(env) + return nil + } +} + +func (u *asyncSecondaryRealtimeUpdater) run() { + defer u.wg.Done() + for { + select { + case env := <-u.queue: + u.updateSecondary(env) + case <-u.closed: + for { + select { + case env := <-u.queue: + u.updateSecondary(env) + default: + return + } + } + } + } +} + +func (u *asyncSecondaryRealtimeUpdater) updateSecondary(env envelope.FrameEnvelope) { + u.recordQueueDepth(env) + secondaryCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout) + if err := u.secondary.Update(secondaryCtx, env); err != nil && u.logger != nil { + u.logger.Error("async realtime secondary update failed", "protocol", env.Protocol, "vin", env.VIN, "event_id", env.StableEventID(), "error", err) + } + cancel() + u.recordQueueDepth(env) } func (u *asyncSecondaryRealtimeUpdater) recordQueueMetric(env envelope.FrameEnvelope, status string) { @@ -424,16 +890,331 @@ func (u *asyncSecondaryRealtimeUpdater) recordQueueDepth(env envelope.FrameEnvel }, float64(len(u.queue))) } -func (u *asyncSecondaryRealtimeUpdater) Close() { - if u.cancel != nil { - u.cancel() +func (u *asyncSecondaryRealtimeUpdater) recordQueueCapacity() { + if u.registry == nil { + return } + u.registry.SetGauge("vehicle_realtime_async_queue_capacity", metrics.Labels{ + "store": "mysql", + }, float64(cap(u.queue))) +} + +func (u *asyncSecondaryRealtimeUpdater) Close() { + u.closeOnce.Do(func() { + close(u.closed) + }) + u.wg.Wait() } type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } +type kafkaMessageFetcher interface { + FetchMessage(context.Context) (kafka.Message, error) +} + +type realtimeBatchItem struct { + message kafka.Message + env envelope.FrameEnvelope + valid bool + processed bool +} + +type realtimeBatchOutcome struct { + commitMessages []kafka.Message + retryMessages []kafka.Message +} + +func collectRealtimeBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { + if maxSize <= 1 { + return []kafka.Message{first} + } + if maxWait <= 0 { + maxWait = 20 * time.Millisecond + } + batch := []kafka.Message{first} + deadline := time.Now().Add(maxWait) + for len(batch) < maxSize { + remaining := time.Until(deadline) + if remaining <= 0 { + break + } + fetchCtx, cancel := context.WithTimeout(ctx, remaining) + message, err := fetcher.FetchMessage(fetchCtx) + cancel() + if err != nil { + break + } + batch = append(batch, message) + } + return batch +} + +func processRealtimeBatch(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message) realtimeBatchOutcome { + return processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, messages, nil) +} + +func processRealtimeBatchForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) realtimeBatchOutcome { + if len(messages) == 0 { + return realtimeBatchOutcome{} + } + messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) + defer cancel() + + setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), 0) + defer setRealtimeBatchPendingForWorker(registry, workerLabels, 0, 0) + items := make([]realtimeBatchItem, 0, len(messages)) + envs := make([]envelope.FrameEnvelope, 0, len(messages)) + for _, message := range messages { + itemIndex := len(items) + items = append(items, realtimeBatchItem{message: message}) + addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "received") + addRealtimeLagMetric(registry, message) + var env envelope.FrameEnvelope + if err := json.Unmarshal(message.Value, &env); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json") + logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + items[itemIndex].processed = true + continue + } + if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status) + logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) + items[itemIndex].processed = true + continue + } + if status := realtimeProjectionSkipStatus(env); status != "" { + addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status) + items[itemIndex].processed = true + continue + } + items[itemIndex].env = env + items[itemIndex].valid = true + envs = append(envs, env) + } + setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), len(envs)) + if len(envs) > 0 { + started := time.Now() + err := updateRealtimeBatch(messageCtx, updater, envs) + recordRealtimeBatchFlush(registry, statusFromError(err), len(messages), len(envs), time.Since(started)) + if err != nil { + logger.Error("realtime batch update failed; falling back to per-message isolation", "messages", len(messages), "valid_messages", len(envs), "error", err) + if !fallbackRealtimeBatch(messageCtx, logger, registry, updater, items) { + committed, commitErr := commitRealtimeProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) + return realtimeFailureOutcome(messages, committed, commitErr) + } + } else { + for index := range items { + if items[index].valid { + items[index].processed = true + addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok") + } + } + } + } + if err := committer.CommitMessages(messageCtx, messages...); err != nil { + for _, message := range messages { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + } + first := messages[0] + logger.Error("kafka batch commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) + return realtimeBatchOutcome{commitMessages: messages} + } + for _, message := range messages { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") + } + return realtimeBatchOutcome{} +} + +func realtimeFailureOutcome(messages []kafka.Message, committed []kafka.Message, commitErr error) realtimeBatchOutcome { + outcome := realtimeBatchOutcome{ + retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed), + } + if commitErr != nil { + outcome.commitMessages = committed + } + return outcome +} + +func processRealtimeBatchReliably(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) { + processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, committer, messages, retryDelay, nil) +} + +func processRealtimeBatchReliablyForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) { + defer registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, 0) + pending := messages + for len(pending) > 0 { + outcome := processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, pending, workerLabels) + if len(outcome.commitMessages) > 0 { + registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !retryRealtimeCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) { + return + } + } + pending = outcome.retryMessages + registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, float64(len(pending))) + if len(pending) == 0 { + return + } + registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "write_error"}) + if !waitForRealtimeRetry(ctx, retryDelay) { + return + } + } +} + +func retryRealtimeCommit(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool { + for len(messages) > 0 { + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) + err := committer.CommitMessages(operationCtx, messages...) + cancel() + if err == nil { + for _, message := range messages { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") + } + return true + } + for _, message := range messages { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + } + first := messages[0] + logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) + registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !waitForRealtimeRetry(ctx, retryDelay) { + return false + } + } + return true +} + +func waitForRealtimeRetry(ctx context.Context, retryDelay time.Duration) bool { + if retryDelay <= 0 { + retryDelay = 100 * time.Millisecond + } + timer := time.NewTimer(retryDelay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func fallbackRealtimeBatch(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, updater realtimeUpdater, items []realtimeBatchItem) bool { + for index := range items { + if !items[index].valid { + continue + } + if err := updater.Update(ctx, items[index].env); err != nil { + message := items[index].message + addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error") + logger.Error("realtime fallback update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", items[index].env.StableEventID(), "error", err) + return false + } + items[index].processed = true + addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok") + } + return true +} + +func commitRealtimeProcessedPrefixAfterFailure(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, items []realtimeBatchItem) ([]kafka.Message, error) { + committable := realtimeProcessedPrefixMessagesByPartition(items) + if len(committable) == 0 { + return nil, nil + } + if err := committer.CommitMessages(ctx, committable...); err != nil { + for _, message := range committable { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + } + first := committable[0] + logger.Error("kafka realtime processed-prefix commit failed after batch update failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err) + return committable, err + } + for _, message := range committable { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") + } + return committable, nil +} + +func realtimeProcessedPrefixMessagesByPartition(items []realtimeBatchItem) []kafka.Message { + type partitionKey struct { + topic string + partition int + } + groups := map[partitionKey][]realtimeBatchItem{} + for _, item := range items { + key := partitionKey{topic: item.message.Topic, partition: item.message.Partition} + groups[key] = append(groups[key], item) + } + var keys []partitionKey + for key := range groups { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].topic != keys[j].topic { + return keys[i].topic < keys[j].topic + } + return keys[i].partition < keys[j].partition + }) + var out []kafka.Message + for _, key := range keys { + group := groups[key] + sort.Slice(group, func(i, j int) bool { + return group[i].message.Offset < group[j].message.Offset + }) + var previousOffset int64 + for index, item := range group { + if index > 0 && item.message.Offset != previousOffset+1 { + break + } + if !item.processed { + break + } + out = append(out, item.message) + previousOffset = item.message.Offset + } + } + return out +} + +func updateRealtimeBatch(ctx context.Context, updater realtimeUpdater, envs []envelope.FrameEnvelope) error { + if updater == nil || len(envs) == 0 { + return nil + } + if batchUpdater, ok := updater.(realtimeBatchUpdater); ok { + return batchUpdater.UpdateBatch(ctx, envs) + } + for _, env := range envs { + if err := updater.Update(ctx, env); err != nil { + return err + } + } + return nil +} + func processRealtimeMessage(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) @@ -447,12 +1228,38 @@ func processRealtimeMessage(ctx context.Context, logger interface { if err := json.Unmarshal(message.Value, &env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json") logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) - _ = committer.CommitMessages(messageCtx, message) + if err := committer.CommitMessages(messageCtx, message); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + logger.Error("kafka commit invalid envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + return + } + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") + return + } + if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status) + logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) + if err := committer.CommitMessages(messageCtx, message); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + logger.Error("kafka commit mismatched realtime raw envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err) + return + } + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") + return + } + if status := realtimeProjectionSkipStatus(env); status != "" { + addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status) + if err := committer.CommitMessages(messageCtx, message); err != nil { + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") + logger.Error("kafka commit skipped realtime envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err) + return + } + addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") return } if err := updater.Update(messageCtx, env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error") - logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) + logger.Error("realtime projector update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "ok") @@ -464,14 +1271,49 @@ func processRealtimeMessage(ctx context.Context, logger interface { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") } +func realtimeProjectionSkipStatus(env envelope.FrameEnvelope) string { + if !envelope.IsRealtimeTelemetryFrame(env) { + return "skipped_non_realtime" + } + if strings.TrimSpace(env.VIN) == "" { + return "skipped_missing_vin" + } + vehicleKey := strings.TrimSpace(env.VehicleKey()) + if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") { + return "skipped_missing_vehicle_key" + } + if len(env.ParsedFields) == 0 { + return "skipped_missing_fields" + } + return "" +} + func addRealtimeMetric(registry *metrics.Registry, name string, message kafka.Message, status string) { if registry == nil { return } - registry.IncCounter(name, metrics.Labels{ + labels := metrics.Labels{ "topic": message.Topic, "status": status, - }) + } + registry.IncCounter(name, labels) + switch name { + case "vehicle_realtime_kafka_messages_total": + metrics.RecordLastActivity(registry, "vehicle_realtime_last_message_unix_seconds", labels) + case "vehicle_realtime_updates_total": + metrics.RecordLastActivity(registry, "vehicle_realtime_last_update_unix_seconds", labels) + case "vehicle_realtime_kafka_commits_total": + metrics.RecordLastActivity(registry, "vehicle_realtime_last_commit_unix_seconds", labels) + } +} + +func recordPlateCacheStats(registry *metrics.Registry, stats realtime.PlateCacheStats) { + if registry == nil { + return + } + registry.SetGauge("vehicle_realtime_plate_cache_entries", nil, float64(stats.Entries)) + registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(stats.MaxEntries)) + registry.SetGauge("vehicle_realtime_plate_cache_evictions_total", nil, float64(stats.Evictions)) } func addRealtimeLagMetric(registry *metrics.Registry, message kafka.Message) { @@ -481,6 +1323,86 @@ func addRealtimeLagMetric(registry *metrics.Registry, message kafka.Message) { registry.SetKafkaLag("vehicle_realtime_kafka_lag", message.Topic, message.Partition, message.Offset, message.HighWaterMark) } +var realtimeBatchFlushDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} + +func recordRealtimeBatchFlush(registry *metrics.Registry, status string, messages int, validMessages int, elapsed time.Duration) { + if registry == nil { + return + } + labels := metrics.Labels{"status": status} + elapsedMS := float64(elapsed.Milliseconds()) + registry.IncCounter("vehicle_realtime_batch_flush_total", labels) + registry.AddCounter("vehicle_realtime_batch_messages_total", labels, float64(messages)) + registry.AddCounter("vehicle_realtime_batch_valid_messages_total", labels, float64(validMessages)) + registry.SetGauge("vehicle_realtime_batch_flush_duration_ms", labels, elapsedMS) + registry.ObserveHistogram("vehicle_realtime_batch_flush_duration_ms_histogram", labels, realtimeBatchFlushDurationBucketsMS, elapsedMS) +} + +func setRealtimeBatchPending(registry *metrics.Registry, messages int, validMessages int) { + setRealtimeBatchPendingForWorker(registry, nil, messages, validMessages) +} + +func setRealtimeBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int, validMessages int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_realtime_batch_pending_messages", workerLabels, float64(messages)) + registry.SetGauge("vehicle_realtime_batch_pending_valid_messages", workerLabels, float64(validMessages)) +} + +func statusFromError(err error) string { + if err != nil { + return "error" + } + return "ok" +} + +func realtimeRoleFromEnv() string { + return normalizeRealtimeRole(env("REALTIME_ROLE", "combined")) +} + +func normalizeRealtimeRole(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "api", "query", "read", "readonly", "read-only": + return "api" + case "writer", "projector", "consumer", "write": + return "writer" + case "combined", "all", "both", "": + return "combined" + default: + return "combined" + } +} + +func realtimeRoleAllowsAPI(role string) bool { + switch normalizeRealtimeRole(role) { + case "writer": + return false + default: + return true + } +} + +func realtimeRoleAllowsWriter(role string) bool { + switch normalizeRealtimeRole(role) { + case "api": + return false + default: + return true + } +} + +func realtimeRoleNeedsRedis(apiEnabled bool, writerEnabled bool, redisProjectorEnabled bool) bool { + return apiEnabled || (writerEnabled && redisProjectorEnabled) +} + +func realtimeServiceName(role string) string { + if normalizeRealtimeRole(role) == "writer" { + return "vehicle-realtime-writer" + } + return "vehicle-realtime-api" +} + func env(key string, fallback string) string { value := strings.TrimSpace(os.Getenv(key)) if value == "" { @@ -501,6 +1423,13 @@ func envInt(key string, fallback int) int { return parsed } +func boolGauge(value bool) float64 { + if value { + return 1 + } + return 0 +} + func splitCSV(value string) []string { var out []string for _, item := range strings.Split(value, ",") { diff --git a/go/vehicle-gateway/cmd/realtime-api/main_test.go b/go/vehicle-gateway/cmd/realtime-api/main_test.go index 516e5f76..199421c4 100644 --- a/go/vehicle-gateway/cmd/realtime-api/main_test.go +++ b/go/vehicle-gateway/cmd/realtime-api/main_test.go @@ -24,9 +24,11 @@ func TestProcessRealtimeMessageUsesUncancelledContextForUpdateAndCommit(t *testi Protocol: envelope.ProtocolJT808, MessageID: "0x0200", Phone: "13307795425", + VIN: "VIN001", EventTimeMS: 1782918600000, ReceivedAtMS: 1782918600000, ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": "10"}, } payload, err := json.Marshal(env) if err != nil { @@ -53,9 +55,11 @@ func TestProcessRealtimeMessageRecordsMetrics(t *testing.T) { Protocol: envelope.ProtocolJT808, MessageID: "0x0200", Phone: "13307795425", + VIN: "VIN001", EventTimeMS: 1782918600000, ReceivedAtMS: 1782918600000, ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": "10"}, } payload, err := json.Marshal(env) if err != nil { @@ -75,8 +79,11 @@ func TestProcessRealtimeMessageRecordsMetrics(t *testing.T) { text := registry.Render() for _, want := range []string{ `vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_last_message_unix_seconds{status="received",topic="vehicle.raw.go.jt808.v1"} `, `vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_last_update_unix_seconds{status="ok",topic="vehicle.raw.go.jt808.v1"} `, `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_last_commit_unix_seconds{status="ok",topic="vehicle.raw.go.jt808.v1"} `, `vehicle_realtime_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 5`, } { if !strings.Contains(text, want) { @@ -85,6 +92,461 @@ func TestProcessRealtimeMessageRecordsMetrics(t *testing.T) { } } +func TestProcessRealtimeMessageCommitsInvalidJSONAndRecordsMetrics(t *testing.T) { + registry := metrics.NewRegistry() + updater := &contextCheckingRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{} + + processRealtimeMessage( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + committer, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 10, HighWaterMark: 12, Value: []byte(`{bad json`)}, + ) + + if updater.count != 0 { + t.Fatalf("updates = %d, want 0 for invalid JSON", updater.count) + } + if committer.count != 1 { + t.Fatalf("commits = %d, want 1", committer.count) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_last_commit_unix_seconds{status="ok",topic="vehicle.raw.go.jt808.v1"} `, + `vehicle_realtime_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("invalid JSON metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeMessageSkipsExplicitNonRawEventKind(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + MessageID: "0x0200", + VIN: "VIN001", + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &contextCheckingRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{} + + processRealtimeMessage( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + committer, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 10, HighWaterMark: 11, Value: payload}, + ) + + if updater.count != 0 { + t.Fatalf("updates = %d, want 0 for mismatched event kind", updater.count) + } + if committer.count != 1 || committer.messageCount != 1 { + t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("mismatch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeMessageRecordsInvalidJSONCommitError(t *testing.T) { + registry := metrics.NewRegistry() + committer := &contextCheckingMessageCommitter{err: errors.New("commit failed")} + logger := &recordingRealtimeLogger{} + + processRealtimeMessage( + context.Background(), + logger, + registry, + &contextCheckingRealtimeUpdater{}, + committer, + kafka.Message{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 8, HighWaterMark: 9, Value: []byte(`{bad json`)}, + ) + + if committer.count != 1 { + t.Fatalf("commits = %d, want 1", committer.count) + } + if !logger.containsError("kafka commit invalid envelope failed") { + t.Fatalf("commit error was not logged: %#v", logger.errors) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_kafka_commits_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`) { + t.Fatalf("commit error metric missing:\n%s", text) + } +} + +func TestProcessRealtimeMessageSkipsNonRealtimeFrameBeforeUpdater(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x01", + VIN: "VIN001", + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918600000, + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"platform_login": map[string]any{"username": "Hyundai"}}, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &contextCheckingRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{} + + processRealtimeMessage( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + committer, + kafka.Message{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}, + ) + + if updater.count != 0 { + t.Fatalf("updates = %d, want 0 for non-realtime frame", updater.count) + } + if committer.messageCount != 1 { + t.Fatalf("committed messages = %d, want 1", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_updates_total{status="skipped_non_realtime",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("non-realtime skip metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeMessageSkipsMissingVINBeforeUpdater(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "13307795425", + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918600000, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": 30}, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &contextCheckingRealtimeUpdater{} + + processRealtimeMessage( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + &contextCheckingMessageCommitter{}, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}, + ) + + if updater.count != 0 { + t.Fatalf("updates = %d, want 0 for missing VIN", updater.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_updates_total{status="skipped_missing_vin",topic="vehicle.raw.go.jt808.v1"} 1`) { + t.Fatalf("missing VIN skip metric missing:\n%s", text) + } +} + +func TestProcessRealtimeMessageSkipsMissingParsedFieldsBeforeUpdater(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918600000, + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}}, + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + registry := metrics.NewRegistry() + updater := &contextCheckingRealtimeUpdater{} + + processRealtimeMessage( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + &contextCheckingMessageCommitter{}, + kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}, + ) + + if updater.count != 0 { + t.Fatalf("updates = %d, want 0 for missing parsed fields", updater.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_updates_total{status="skipped_missing_fields",topic="vehicle.raw.go.jt808.v1"} 1`) { + t.Fatalf("missing parsed fields skip metric missing:\n%s", text) + } +} + +func TestCollectRealtimeBatchUsesBatchLimitAndWait(t *testing.T) { + first := kafka.Message{Offset: 10} + fetcher := &sliceRealtimeFetcher{messages: []kafka.Message{{Offset: 11}, {Offset: 12}, {Offset: 13}}} + + batch := collectRealtimeBatch(context.Background(), fetcher, first, 3, time.Second) + + if len(batch) != 3 { + t.Fatalf("batch len = %d, want 3", len(batch)) + } + if batch[0].Offset != 10 || batch[1].Offset != 11 || batch[2].Offset != 12 { + t.Fatalf("unexpected offsets: %#v", []int64{batch[0].Offset, batch[1].Offset, batch[2].Offset}) + } +} + +func TestProcessRealtimeBatchUsesBatchUpdaterAndCommitsAll(t *testing.T) { + messages := []kafka.Message{ + realtimeTestMessage(t, 10, envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, MessageID: "0x02", VIN: "VIN001", EventTimeMS: 1000, ReceivedAtMS: 1000, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 10}}), + realtimeTestMessage(t, 11, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN002", EventTimeMS: 1100, ReceivedAtMS: 1100, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": 20}}), + } + registry := metrics.NewRegistry() + updater := &recordingBatchRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{} + + processRealtimeBatch(context.Background(), discardRealtimeLogger{}, registry, updater, committer, messages) + + if updater.batchCount != 1 { + t.Fatalf("batch updates = %d, want 1", updater.batchCount) + } + if updater.updateCount != 0 { + t.Fatalf("fallback updates = %d, want 0", updater.updateCount) + } + if len(updater.batchEnvs) != 2 { + t.Fatalf("batch envs = %d, want 2", len(updater.batchEnvs)) + } + if committer.count != 1 || committer.messageCount != 2 { + t.Fatalf("commits=%d committed_messages=%d, want 1/2", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_batch_flush_total{status="ok"} 1`, + `vehicle_realtime_batch_valid_messages_total{status="ok"} 2`, + `vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 1`, + `vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeBatchKeepsPendingGaugePerWorker(t *testing.T) { + message := realtimeTestMessage(t, 10, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 1000, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": 10}, + }) + registry := metrics.NewRegistry() + + processRealtimeBatchForWorker( + context.Background(), + discardRealtimeLogger{}, + registry, + &recordingBatchRealtimeUpdater{}, + &contextCheckingMessageCommitter{}, + []kafka.Message{message}, + metrics.Labels{"worker": "2"}, + ) + + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_batch_pending_messages{worker="2"} 0`, + `vehicle_realtime_batch_pending_valid_messages{worker="2"} 0`, + } { + if !strings.Contains(text, want) { + t.Fatalf("worker pending gauge missing or not reset, want %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeBatchSkipsExplicitNonRawEventKindAndUpdatesValidRows(t *testing.T) { + mismatchedPayload, err := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + MessageID: "0x0200", + VIN: "VIN_BAD", + }.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + valid := realtimeTestMessage(t, 11, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindRaw, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1100, + ReceivedAtMS: 1100, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": "10"}, + }) + messages := []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 0, Offset: 10, HighWaterMark: 12, Value: mismatchedPayload}, + valid, + } + registry := metrics.NewRegistry() + updater := &recordingBatchRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{} + + processRealtimeBatch(context.Background(), discardRealtimeLogger{}, registry, updater, committer, messages) + + if updater.batchCount != 1 || len(updater.batchEnvs) != 1 { + t.Fatalf("batch updates=%d envs=%d, want only valid row", updater.batchCount, len(updater.batchEnvs)) + } + if committer.count != 1 || committer.messageCount != 2 { + t.Fatalf("commits=%d messages=%d, want 1/2", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch mismatch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeBatchFallsBackAndCommitsProcessedPrefix(t *testing.T) { + messages := []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 0, Offset: 10, Value: []byte(`{bad json`)}, + realtimeTestMessage(t, 11, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN001", EventTimeMS: 1100, ReceivedAtMS: 1100, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": 10}}), + realtimeTestMessage(t, 12, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN002", EventTimeMS: 1200, ReceivedAtMS: 1200, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": 20}}), + } + registry := metrics.NewRegistry() + updater := &recordingBatchRealtimeUpdater{batchErr: errTestRealtimeUpdate, failUpdateAt: 2} + committer := &contextCheckingMessageCommitter{} + logger := &recordingRealtimeLogger{} + + outcome := processRealtimeBatch(context.Background(), logger, registry, updater, committer, messages) + + if updater.batchCount != 1 { + t.Fatalf("batch updates = %d, want 1", updater.batchCount) + } + if updater.updateCount != 2 { + t.Fatalf("fallback updates = %d, want 2", updater.updateCount) + } + if committer.count != 1 { + t.Fatalf("commit calls = %d, want 1 processed-prefix commit", committer.count) + } + if len(committer.messages) != 2 || committer.messages[0].Offset != 10 || committer.messages[1].Offset != 11 { + t.Fatalf("committed offsets = %#v, want [10 11]", committedOffsets(committer.messages)) + } + if got := committedOffsets(outcome.retryMessages); len(got) != 1 || got[0] != 12 { + t.Fatalf("retry offsets = %#v, want [12]", got) + } + if !logger.containsError("realtime fallback update failed") { + t.Fatalf("fallback error was not logged: %#v", logger.errors) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_batch_flush_total{status="error"} 1`, + `vehicle_realtime_updates_total{status="error",topic="vehicle.raw.go.jt808.v1"} 1`, + `vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("fallback metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessRealtimeBatchReliablyRetriesFailedSuffix(t *testing.T) { + messages := []kafka.Message{ + {Topic: "vehicle.raw.go.jt808.v1", Partition: 0, Offset: 10, Value: []byte(`{bad json`)}, + realtimeTestMessage(t, 11, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN001", EventTimeMS: 1100, ReceivedAtMS: 1100, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": "10"}}), + realtimeTestMessage(t, 12, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN002", EventTimeMS: 1200, ReceivedAtMS: 1200, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": "20"}}), + } + registry := metrics.NewRegistry() + updater := &recordingBatchRealtimeUpdater{batchErr: errTestRealtimeUpdate, failUpdateAt: 2} + committer := &contextCheckingMessageCommitter{} + + processRealtimeBatchReliably( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + committer, + messages, + time.Nanosecond, + ) + + if updater.batchCount != 2 { + t.Fatalf("batch attempts = %d, want initial batch and failed suffix retry", updater.batchCount) + } + if updater.updateCount != 3 { + t.Fatalf("fallback updates = %d, want 3", updater.updateCount) + } + if got := committedOffsets(committer.messages); len(got) != 3 || got[0] != 10 || got[1] != 11 || got[2] != 12 { + t.Fatalf("committed offsets = %#v, want [10 11 12]", got) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_batch_retries_total{reason="write_error"} 1`) { + t.Fatalf("write retry metric missing:\n%s", text) + } +} + +func TestProcessRealtimeBatchReliablyRetriesCommitWithoutReappending(t *testing.T) { + messages := []kafka.Message{ + realtimeTestMessage(t, 10, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN001", EventTimeMS: 1000, ReceivedAtMS: 1000, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": "10"}}), + realtimeTestMessage(t, 11, envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, MessageID: "0x0200", VIN: "VIN002", EventTimeMS: 1100, ReceivedAtMS: 1100, ParseStatus: envelope.ParseOK, ParsedFields: map[string]any{"jt808.location.speed_kmh": "20"}}), + } + registry := metrics.NewRegistry() + updater := &recordingBatchRealtimeUpdater{} + committer := &contextCheckingMessageCommitter{err: errors.New("commit failed"), failOnCount: 1} + + processRealtimeBatchReliably( + context.Background(), + discardRealtimeLogger{}, + registry, + updater, + committer, + messages, + time.Nanosecond, + ) + + if updater.batchCount != 1 { + t.Fatalf("batch attempts = %d, want 1 without replay after commit failure", updater.batchCount) + } + if committer.count != 2 { + t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_batch_retries_total{reason="commit_error"} 1`) { + t.Fatalf("commit retry metric missing:\n%s", text) + } +} + func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"} redisUpdater := &contextCheckingRealtimeUpdater{} @@ -99,6 +561,32 @@ func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) { } } +func TestComposeRealtimeUpdaterSkipsDisabledRedisProjector(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"} + mysqlUpdater := &contextCheckingRealtimeUpdater{} + updater := composeRealtimeUpdater(nil, mysqlUpdater) + + if err := updater.Update(context.Background(), env); err != nil { + t.Fatalf("Update() error = %v", err) + } + if mysqlUpdater.count != 1 { + t.Fatalf("mysql updates = %d, want 1", mysqlUpdater.count) + } +} + +func TestComposeRealtimeUpdaterReturnsPrimaryWhenMySQLSnapshotDisabled(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"} + redisUpdater := &contextCheckingRealtimeUpdater{} + updater := composeRealtimeUpdater(redisUpdater, nil) + + if err := updater.Update(context.Background(), env); err != nil { + t.Fatalf("Update() error = %v", err) + } + if redisUpdater.count != 1 { + t.Fatalf("redis updates = %d, want 1", redisUpdater.count) + } +} + func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"} primary := &contextCheckingRealtimeUpdater{} @@ -125,6 +613,7 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) { close(secondary.release) text := registry.Render() for _, want := range []string{ + `vehicle_realtime_async_queue_capacity{store="mysql"} 8`, `vehicle_realtime_async_queue_total{protocol="GB32960",status="queued",store="mysql"} 1`, `vehicle_realtime_async_queue_depth{protocol="GB32960",store="mysql"}`, } { @@ -163,6 +652,40 @@ func TestAsyncSecondaryQueueDropDoesNotFailPrimaryUpdate(t *testing.T) { } } +func TestAsyncSecondaryRealtimeUpdaterCloseDrainsQueuedUpdates(t *testing.T) { + secondary := &contextCheckingRealtimeUpdater{} + updater := newAsyncSecondaryRealtimeUpdater(nil, secondary, 8, 1, nil, nil) + + for _, vin := range []string{"VIN001", "VIN002"} { + if err := updater.Update(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: vin}); err != nil { + t.Fatalf("Update(%s) error = %v", vin, err) + } + } + updater.Close() + + if secondary.count != 2 { + t.Fatalf("secondary updates = %d, want 2 after Close drains queue", secondary.count) + } +} + +func TestAsyncSecondaryRealtimeUpdaterDoesNotQueueAfterClose(t *testing.T) { + secondary := &contextCheckingRealtimeUpdater{} + registry := metrics.NewRegistry() + updater := newAsyncSecondaryRealtimeUpdater(nil, secondary, 8, 1, registry, nil) + updater.Close() + + if err := updater.Update(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"}); err != nil { + t.Fatalf("Update() error = %v", err) + } + if secondary.count != 0 { + t.Fatalf("secondary updates = %d, want 0 after Close", secondary.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_async_queue_total{protocol="JT808",status="closed",store="mysql"} 1`) { + t.Fatalf("closed queue metric missing:\n%s", text) + } +} + func TestStoreMetricUpdaterRecordsStoreUpdateResults(t *testing.T) { registry := metrics.NewRegistry() env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"} @@ -182,6 +705,7 @@ func TestStoreMetricUpdaterRecordsStoreUpdateResults(t *testing.T) { `vehicle_realtime_store_update_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="ok",store="redis"} 1`, `vehicle_realtime_store_update_duration_ms_histogram_count{protocol="JT808",status="ok",store="redis"} 1`, `vehicle_realtime_store_update_duration_ms_histogram_sum{protocol="JT808",status="ok",store="redis"}`, + `vehicle_realtime_last_store_update_unix_seconds{protocol="JT808",status="ok",store="redis"} `, } { if !strings.Contains(text, want) { t.Fatalf("store update metric missing %s:\n%s", want, text) @@ -201,6 +725,203 @@ func TestRetryRealtimeUpdaterRetriesTransientMySQLDeadlock(t *testing.T) { } } +func TestRetryRealtimeUpdaterRetriesTransientMySQLConnectionFailure(t *testing.T) { + delegate := &retryOnceRealtimeUpdater{err: errors.New("driver: bad connection: write tcp: broken pipe")} + updater := retryRealtimeUpdater{delegate: delegate, attempts: 3} + + if err := updater.Update(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"}); err != nil { + t.Fatalf("Update() error = %v", err) + } + if delegate.count != 2 { + t.Fatalf("delegate updates = %d, want 2", delegate.count) + } +} + +func TestIsTransientMySQLWriteError(t *testing.T) { + for _, err := range []error{ + errors.New("dial tcp 127.0.0.1:3306: connection refused"), + errors.New("read tcp: connection reset by peer"), + errors.New("write tcp: broken pipe"), + errors.New("driver: bad connection"), + errors.New("invalid connection"), + errors.New("i/o timeout"), + errors.New("EOF"), + errors.New("server is down"), + errors.New("network is unreachable"), + } { + if !isTransientMySQLWriteError(err) { + t.Fatalf("isTransientMySQLWriteError(%q) = false, want true", err.Error()) + } + } + for _, err := range []error{ + context.Canceled, + context.DeadlineExceeded, + errors.New("duplicate key conflict"), + nil, + } { + if isTransientMySQLWriteError(err) { + t.Fatalf("isTransientMySQLWriteError(%v) = true, want false", err) + } + } +} + +func TestThrottledRealtimeUpdaterLimitsMySQLSnapshotFrequencyByVehicle(t *testing.T) { + delegate := &contextCheckingRealtimeUpdater{} + registry := metrics.NewRegistry() + updater := newThrottledRealtimeUpdater(delegate, 10*time.Second, "mysql", registry) + + base := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}}, + } + for _, env := range []envelope.FrameEnvelope{ + base, + func() envelope.FrameEnvelope { + env := base + env.EventTimeMS = 5000 + env.ReceivedAtMS = 5100 + return env + }(), + func() envelope.FrameEnvelope { + env := base + env.EventTimeMS = 11000 + env.ReceivedAtMS = 11100 + return env + }(), + } { + if err := updater.Update(context.Background(), env); err != nil { + t.Fatalf("Update() error = %v", err) + } + } + if delegate.count != 2 { + t.Fatalf("delegate updates = %d, want 2", delegate.count) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_throttle_total{protocol="JT808",status="passed",store="mysql"} 2`, + `vehicle_realtime_throttle_total{protocol="JT808",status="skipped_interval",store="mysql"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("throttle metric missing %s:\n%s", want, text) + } + } +} + +func TestThrottledRealtimeUpdaterCleansExpiredCacheEntries(t *testing.T) { + delegate := &contextCheckingRealtimeUpdater{} + registry := metrics.NewRegistry() + updater := newThrottledRealtimeUpdaterWithCache(delegate, 10*time.Second, "mysql", registry, 48*time.Hour, 0).(*throttledRealtimeUpdater) + loc := time.FixedZone("Asia/Shanghai", 8*3600) + base := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}}, + } + old := base + old.VIN = "VIN-OLD" + old.EventTimeMS = time.Date(2026, 7, 1, 12, 0, 0, 0, loc).UnixMilli() + old.ReceivedAtMS = old.EventTimeMS + current := base + current.VIN = "VIN-CURRENT" + current.EventTimeMS = time.Date(2026, 7, 4, 12, 0, 0, 0, loc).UnixMilli() + current.ReceivedAtMS = current.EventTimeMS + + if err := updater.Update(context.Background(), old); err != nil { + t.Fatalf("old Update() error = %v", err) + } + if err := updater.Update(context.Background(), current); err != nil { + t.Fatalf("current Update() error = %v", err) + } + if delegate.count != 2 { + t.Fatalf("delegate updates = %d, want 2", delegate.count) + } + if len(updater.lastMS) != 1 { + t.Fatalf("throttle cache entries = %d, want 1", len(updater.lastMS)) + } + if _, ok := updater.lastMS[string(envelope.ProtocolJT808)+"\x00VIN-CURRENT"]; !ok { + t.Fatalf("current VIN missing from throttle cache") + } + text := registry.Render() + for _, want := range []string{ + `vehicle_realtime_throttle_cache_entries{store="mysql"} 1`, + `vehicle_realtime_throttle_cache_last_cleanup_deleted{store="mysql"} 1`, + `vehicle_realtime_throttle_cache_last_cleanup_unix_seconds{store="mysql"} 1783137600`, + } { + if !strings.Contains(text, want) { + t.Fatalf("throttle cache metric missing %s:\n%s", want, text) + } + } +} + +func TestThrottledRealtimeUpdaterFallsBackToReceivedTimeForFarFutureEvent(t *testing.T) { + delegate := &contextCheckingRealtimeUpdater{} + updater := newThrottledRealtimeUpdater(delegate, 10*time.Second, "mysql", nil) + received := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC).UnixMilli() + future := time.Date(2026, 7, 13, 10, 0, 0, 0, time.UTC).UnixMilli() + base := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: future, + ReceivedAtMS: received, + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}}, + } + for _, env := range []envelope.FrameEnvelope{ + base, + func() envelope.FrameEnvelope { + env := base + env.EventTimeMS = received + 5_000 + env.ReceivedAtMS = received + 5_000 + return env + }(), + func() envelope.FrameEnvelope { + env := base + env.EventTimeMS = received + 11_000 + env.ReceivedAtMS = received + 11_000 + return env + }(), + } { + if err := updater.Update(context.Background(), env); err != nil { + t.Fatalf("Update() error = %v", err) + } + } + if delegate.count != 2 { + t.Fatalf("delegate updates = %d, want 2", delegate.count) + } +} + +func TestThrottledRealtimeUpdaterSkipsNonRealtimeFramesBeforeDelegate(t *testing.T) { + delegate := &contextCheckingRealtimeUpdater{} + registry := metrics.NewRegistry() + updater := newThrottledRealtimeUpdater(delegate, 10*time.Second, "mysql", registry) + + if err := updater.Update(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x01", + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{"platform_login": map[string]any{"username": "Hyundai"}}, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + if delegate.count != 0 { + t.Fatalf("delegate updates = %d, want 0", delegate.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_realtime_throttle_total{protocol="GB32960",status="skipped_non_realtime",store="mysql"} 1`) { + t.Fatalf("non-realtime throttle metric missing:\n%s", text) + } +} + func TestAsyncSecondaryRealtimeUpdaterLogsSecondaryErrors(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", EventID: "event-1"} logger := &recordingRealtimeLogger{} @@ -229,6 +950,57 @@ func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) { } } +func TestValidateRealtimeConsumerConfigRejectsFieldsTopics(t *testing.T) { + err := validateRealtimeConsumerConfig([]string{"vehicle.fields.go.jt808.v1"}, 1) + + if err == nil || !strings.Contains(err.Error(), "raw topics only") { + t.Fatalf("validateRealtimeConsumerConfig() error = %v, want raw topic rejection", err) + } +} + +func TestValidateRealtimeConsumerConfigRequiresPositiveWorkers(t *testing.T) { + err := validateRealtimeConsumerConfig([]string{"vehicle.raw.go.jt808.v1"}, 0) + + if err == nil || !strings.Contains(err.Error(), "REALTIME_WORKERS") { + t.Fatalf("validateRealtimeConsumerConfig() error = %v, want REALTIME_WORKERS hint", err) + } +} + +func TestThrottledRealtimeUpdaterIsSafeAcrossPartitionWorkers(t *testing.T) { + delegate := &concurrentRealtimeUpdater{} + updater := newThrottledRealtimeUpdater(delegate, time.Minute, "mysql", metrics.NewRegistry()) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "LNBVIN00000000001", + EventTimeMS: 1783947000000, + ReceivedAtMS: 1783947000000, + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{"jt808.location.speed_kmh": 10}, + } + + const workers = 32 + var wait sync.WaitGroup + errorsCh := make(chan error, workers) + for index := 0; index < workers; index++ { + wait.Add(1) + go func() { + defer wait.Done() + if err := updater.Update(context.Background(), env); err != nil { + errorsCh <- err + } + }() + } + wait.Wait() + close(errorsCh) + for err := range errorsCh { + t.Fatal(err) + } + if got := delegate.Count(); got != 1 { + t.Fatalf("delegate updates = %d, want one throttled write for the same VIN", got) + } +} + func TestRealtimeReaderConfigStartsAtLatestOffset(t *testing.T) { cfg := realtimeReaderConfig([]string{"127.0.0.1:9092"}, []string{"vehicle.raw.go.jt808.v1"}) @@ -240,6 +1012,71 @@ func TestRealtimeReaderConfigStartsAtLatestOffset(t *testing.T) { } } +func TestRealtimeRoleControlsQueryAndWriterResponsibilities(t *testing.T) { + tests := []struct { + raw string + wantRole string + wantAPI bool + wantWriter bool + }{ + {raw: "", wantRole: "combined", wantAPI: true, wantWriter: true}, + {raw: "combined", wantRole: "combined", wantAPI: true, wantWriter: true}, + {raw: "api", wantRole: "api", wantAPI: true, wantWriter: false}, + {raw: "query", wantRole: "api", wantAPI: true, wantWriter: false}, + {raw: "writer", wantRole: "writer", wantAPI: false, wantWriter: true}, + {raw: "projector", wantRole: "writer", wantAPI: false, wantWriter: true}, + {raw: "surprise", wantRole: "combined", wantAPI: true, wantWriter: true}, + } + + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + if got := normalizeRealtimeRole(tt.raw); got != tt.wantRole { + t.Fatalf("normalizeRealtimeRole(%q) = %q, want %q", tt.raw, got, tt.wantRole) + } + if got := realtimeRoleAllowsAPI(tt.raw); got != tt.wantAPI { + t.Fatalf("realtimeRoleAllowsAPI(%q) = %v, want %v", tt.raw, got, tt.wantAPI) + } + if got := realtimeRoleAllowsWriter(tt.raw); got != tt.wantWriter { + t.Fatalf("realtimeRoleAllowsWriter(%q) = %v, want %v", tt.raw, got, tt.wantWriter) + } + }) + } +} + +func TestRealtimeRoleFromEnvDefaultsToCombined(t *testing.T) { + t.Setenv("REALTIME_ROLE", "") + if got := realtimeRoleFromEnv(); got != "combined" { + t.Fatalf("role = %q, want combined", got) + } +} + +func TestRealtimeRoleDependencyMatrix(t *testing.T) { + tests := []struct { + name string + role string + redisProjector bool + wantRedis bool + wantServiceName string + }{ + {name: "api queries redis", role: "api", redisProjector: false, wantRedis: true, wantServiceName: "vehicle-realtime-api"}, + {name: "mysql writer skips redis", role: "writer", redisProjector: false, wantRedis: false, wantServiceName: "vehicle-realtime-writer"}, + {name: "redis writer uses redis", role: "writer", redisProjector: true, wantRedis: true, wantServiceName: "vehicle-realtime-writer"}, + {name: "combined queries redis", role: "combined", redisProjector: false, wantRedis: true, wantServiceName: "vehicle-realtime-api"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + apiEnabled := realtimeRoleAllowsAPI(tt.role) + writerEnabled := realtimeRoleAllowsWriter(tt.role) + if got := realtimeRoleNeedsRedis(apiEnabled, writerEnabled, tt.redisProjector); got != tt.wantRedis { + t.Fatalf("realtimeRoleNeedsRedis() = %v, want %v", got, tt.wantRedis) + } + if got := realtimeServiceName(tt.role); got != tt.wantServiceName { + t.Fatalf("realtimeServiceName(%q) = %q, want %q", tt.role, got, tt.wantServiceName) + } + }) + } +} + func TestRealtimeAPIDoesNotExposeDuplicateMileagePointRoute(t *testing.T) { source, err := os.ReadFile("main.go") if err != nil { @@ -255,7 +1092,7 @@ func TestRealtimeAPICachesBindingPlateResolver(t *testing.T) { if err != nil { t.Fatalf("read main.go: %v", err) } - for _, want := range []string{"NewCachedPlateResolver", "PLATE_CACHE_TTL_SECONDS"} { + for _, want := range []string{"NewCachedPlateResolverWithMaxEntries", "PLATE_CACHE_TTL_SECONDS", "PLATE_CACHE_MAX_ENTRIES", "recordPlateCacheStats", "vehicle_realtime_plate_cache_entries"} { if !strings.Contains(string(source), want) { t.Fatalf("realtime api should configure cached plate resolver, missing %s", want) } @@ -284,6 +1121,30 @@ func TestRealtimeAPIExposesOperationalRealtimeRoutes(t *testing.T) { } } +func TestRealtimeAPIExposesDataSourceManagementRoutes(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatalf("read main.go: %v", err) + } + for _, want := range []string{`"/api/stats/data-sources"`, `"/api/stats/data-sources/"`, "NewDataSourceHandler"} { + if !strings.Contains(string(source), want) { + t.Fatalf("realtime api should expose data source management route %s", want) + } + } +} + +func TestRealtimeAPIExposesDailyMetricSourceRoute(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatalf("read main.go: %v", err) + } + for _, want := range []string{`"/api/stats/daily-metrics"`, `"/api/stats/daily-metrics/sources"`, `"/api/stats/daily-metrics/sources/quality"`, `"/api/stats/daily-metrics/sources/selection"`, `"/api/stats/daily-metrics/diagnostics"`, `"/api/stats/daily-metrics/diagnostics/summary"`, `"/api/stats/daily-metrics/diagnostics/reasons"`, `"/api/stats/daily-metrics/diagnostics/field-status"`, "NewMetricHandler"} { + if !strings.Contains(string(source), want) { + t.Fatalf("realtime api should expose metric route %s", want) + } + } +} + func TestRealtimeAPIDoesNotExposeMySQLKVRoute(t *testing.T) { source, err := os.ReadFile("main.go") if err != nil { @@ -294,11 +1155,57 @@ func TestRealtimeAPIDoesNotExposeMySQLKVRoute(t *testing.T) { } } +func TestRealtimeAPIRedisProjectorHasProductionGuardMetric(t *testing.T) { + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatalf("read main.go: %v", err) + } + for _, want := range []string{"REALTIME_ROLE", "vehicle_realtime_role_info", "vehicle_realtime_writer_enabled", "REALTIME_REDIS_PROJECTOR_ENABLED", "vehicle_realtime_redis_projector_enabled"} { + if !strings.Contains(string(source), want) { + t.Fatalf("realtime api should expose redis projector guard %s", want) + } + } +} + +func TestRealtimeAPIClosesAsyncMySQLUpdaterBeforeStatsDB(t *testing.T) { + sourceBytes, err := os.ReadFile("main.go") + if err != nil { + t.Fatalf("read main.go: %v", err) + } + source := string(sourceBytes) + for _, want := range []string{"closeRealtimeUpdaters := func() {}", "closeRealtimeUpdaters = asyncUpdater.Close", "defer closeStats()", "defer closeRealtimeUpdaters()"} { + if !strings.Contains(source, want) { + t.Fatalf("realtime api should close async mysql updater on shutdown, missing %s", want) + } + } + if strings.Index(source, "defer closeStats()") > strings.Index(source, "defer closeRealtimeUpdaters()") { + t.Fatalf("closeRealtimeUpdaters should be deferred after closeStats so it runs first") + } +} + type contextCheckingRealtimeUpdater struct { ctxErr error count int } +type concurrentRealtimeUpdater struct { + mu sync.Mutex + count int +} + +func (u *concurrentRealtimeUpdater) Update(context.Context, envelope.FrameEnvelope) error { + u.mu.Lock() + u.count++ + u.mu.Unlock() + return nil +} + +func (u *concurrentRealtimeUpdater) Count() int { + u.mu.Lock() + defer u.mu.Unlock() + return u.count +} + func (u *contextCheckingRealtimeUpdater) Update(ctx context.Context, _ envelope.FrameEnvelope) error { u.ctxErr = ctx.Err() u.count++ @@ -306,14 +1213,26 @@ func (u *contextCheckingRealtimeUpdater) Update(ctx context.Context, _ envelope. } type contextCheckingMessageCommitter struct { - ctxErr error - count int + ctxErr error + count int + messageCount int + messages []kafka.Message + err error + failOnCount int } -func (c *contextCheckingMessageCommitter) CommitMessages(ctx context.Context, _ ...kafka.Message) error { +func (c *contextCheckingMessageCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error { c.ctxErr = ctx.Err() c.count++ - return c.ctxErr + c.messageCount += len(messages) + c.messages = append(c.messages, messages...) + if c.ctxErr != nil { + return c.ctxErr + } + if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) { + return c.err + } + return nil } type discardRealtimeLogger struct{} @@ -378,3 +1297,72 @@ func (u *retryOnceRealtimeUpdater) Update(context.Context, envelope.FrameEnvelop } return nil } + +type sliceRealtimeFetcher struct { + messages []kafka.Message + index int +} + +func (f *sliceRealtimeFetcher) FetchMessage(ctx context.Context) (kafka.Message, error) { + if f.index >= len(f.messages) { + return kafka.Message{}, ctx.Err() + } + message := f.messages[f.index] + f.index++ + return message, nil +} + +type recordingBatchRealtimeUpdater struct { + batchCount int + updateCount int + batchEnvs []envelope.FrameEnvelope + batchErr error + failUpdateAt int +} + +func (u *recordingBatchRealtimeUpdater) UpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error { + u.batchCount++ + u.batchEnvs = append(u.batchEnvs, envs...) + return u.batchErr +} + +func (u *recordingBatchRealtimeUpdater) Update(context.Context, envelope.FrameEnvelope) error { + u.updateCount++ + if u.failUpdateAt > 0 && u.updateCount == u.failUpdateAt { + return errTestRealtimeUpdate + } + return nil +} + +func realtimeTestMessage(t *testing.T, offset int64, env envelope.FrameEnvelope) kafka.Message { + t.Helper() + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + return kafka.Message{ + Topic: topicForTestProtocol(env.Protocol), + Partition: 0, + Offset: offset, + Value: payload, + } +} + +func topicForTestProtocol(protocol envelope.Protocol) string { + switch protocol { + case envelope.ProtocolGB32960: + return "vehicle.raw.go.gb32960.v1" + case envelope.ProtocolYutongMQTT: + return "vehicle.raw.go.yutong_mqtt.v1" + default: + return "vehicle.raw.go.jt808.v1" + } +} + +func committedOffsets(messages []kafka.Message) []int64 { + offsets := make([]int64, 0, len(messages)) + for _, message := range messages { + offsets = append(offsets, message.Offset) + } + return offsets +} diff --git a/go/vehicle-gateway/cmd/stat-writer/main.go b/go/vehicle-gateway/cmd/stat-writer/main.go index 8bbd57fd..e055835b 100644 --- a/go/vehicle-gateway/cmd/stat-writer/main.go +++ b/go/vehicle-gateway/cmd/stat-writer/main.go @@ -4,9 +4,14 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "os" "os/signal" + "sort" + "strconv" "strings" + "sync" "syscall" "time" @@ -14,6 +19,7 @@ import ( "github.com/segmentio/kafka-go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" @@ -27,6 +33,10 @@ func main() { defer stop() cfg := loadConfig() + if err := cfg.Validate(); err != nil { + logger.Error("invalid stat writer config", "error", err) + os.Exit(1) + } db, err := sql.Open("mysql", cfg.MySQLDSN) if err != nil { logger.Error("mysql open failed", "error", err) @@ -38,18 +48,64 @@ func main() { os.Exit(1) } registry := metrics.NewRegistry() + metrics.RegisterKafkaConsumerInfo(registry, "vehicle-stat-writer", cfg.KafkaGroup, cfg.KafkaTopics) + registry.SetGauge("vehicle_stat_project_interval_seconds", nil, cfg.ProjectInterval.Seconds()) + registry.SetGauge("vehicle_stat_source_touch_interval_seconds", nil, cfg.SourceTouchInterval.Seconds()) + registry.SetGauge("vehicle_stat_cache_retention_seconds", nil, cfg.CacheRetention.Seconds()) + registry.SetGauge("vehicle_stat_cache_cleanup_interval_seconds", nil, cfg.CacheCleanupInterval.Seconds()) + registry.SetGauge("vehicle_stat_baseline_miss_ttl_seconds", nil, cfg.BaselineMissTTL.Seconds()) + registry.SetGauge("vehicle_stat_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers)) health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-stat-writer", []health.Check{ {Name: "mysql", Check: db.PingContext}, }, registry)) writer := stats.NewWriter(db, cfg.Location) + writer.SetProjectionInterval(cfg.ProjectInterval) + writer.SetSourceTouchInterval(cfg.SourceTouchInterval) + writer.SetCacheRetention(cfg.CacheRetention) + writer.SetCacheCleanupInterval(cfg.CacheCleanupInterval) + writer.SetBaselineMissTTL(cfg.BaselineMissTTL) + writer.SetMaxCacheEntries(cfg.CacheMaxEntries) if cfg.EnsureSchema { if err := writer.EnsureSchema(ctx); err != nil { logger.Error("mysql schema bootstrap failed", "error", err) os.Exit(1) } } + if cfg.NormalizePlatformSourcesOnStart { + statDate := time.Now().In(cfg.Location).Format("2006-01-02") + normalizeCtx, cancel := context.WithTimeout(ctx, cfg.NormalizePlatformSourcesTimeout) + normalized, err := stats.NormalizePlatformSourceMileageForDate(normalizeCtx, db, statDate, envelope.ProtocolJT808) + cancel() + if err != nil { + logger.Warn("platform source startup normalization failed", "stat_date", statDate, "protocol", envelope.ProtocolJT808, "normalized", normalized, "error", err) + } else if normalized > 0 { + logger.Info("platform source startup normalization finished", "stat_date", statDate, "protocol", envelope.ProtocolJT808, "normalized", normalized) + } + } + var appender statAppender = retryStatAppender{ + delegate: writer, + attempts: cfg.RetryAttempts, + delay: cfg.RetryDelay, + registry: registry, + } + logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds()) + var workers sync.WaitGroup + for workerID := 1; workerID <= cfg.Workers; workerID++ { + workers.Add(1) + go func(id int) { + defer workers.Done() + runStatConsumer(ctx, logger, registry, appender, cfg, id) + }(workerID) + } + workers.Wait() +} + +func runStatConsumer(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender statAppender, cfg config, workerID int) { reader := kafka.NewReader(kafka.ReaderConfig{ Brokers: cfg.KafkaBrokers, GroupID: cfg.KafkaGroup, @@ -59,7 +115,10 @@ func main() { }) defer reader.Close() - logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ",")) + workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)} + registry.SetGauge("vehicle_stat_worker_active", workerLabels, 1) + defer registry.SetGauge("vehicle_stat_worker_active", workerLabels, 0) + for { message, err := reader.FetchMessage(ctx) if err != nil { @@ -69,7 +128,8 @@ func main() { logger.Error("kafka fetch failed", "error", err) continue } - processStatMessage(ctx, logger, registry, writer, reader, message) + batch := collectStatBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond) + processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, batch, cfg.RetryDelay, workerLabels) } } @@ -79,50 +139,488 @@ type statAppender interface { Append(context.Context, envelope.FrameEnvelope) error } +type statResultAppender interface { + AppendWithResult(context.Context, envelope.FrameEnvelope) (stats.AppendResult, error) +} + +type statCacheReporter interface { + CacheStats() stats.CacheStats +} + type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } +type kafkaMessageFetcher interface { + FetchMessage(context.Context) (kafka.Message, error) +} + +type statBatchItem struct { + message kafka.Message + processed bool +} + +type statBatchOutcome struct { + commitMessages []kafka.Message + retryMessages []kafka.Message +} + var statWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var statWriteE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000} +var statWriteE2ERecent = metrics.NewRecentLatencyByKey(512) + +func collectStatBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { + if maxSize <= 1 { + return []kafka.Message{first} + } + if maxWait <= 0 { + maxWait = 100 * time.Millisecond + } + batch := []kafka.Message{first} + deadline := time.Now().Add(maxWait) + for len(batch) < maxSize { + remaining := time.Until(deadline) + if remaining <= 0 { + break + } + fetchCtx, cancel := context.WithTimeout(ctx, remaining) + message, err := fetcher.FetchMessage(fetchCtx) + cancel() + if err != nil { + break + } + batch = append(batch, message) + } + return batch +} func processStatMessage(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, message kafka.Message) { + processStatBatch(ctx, logger, registry, appender, committer, []kafka.Message{message}) +} + +func processStatBatch(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message) statBatchOutcome { + return processStatBatchForWorker(ctx, logger, registry, appender, committer, messages, nil) +} + +func processStatBatchForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) statBatchOutcome { + if len(messages) == 0 { + return statBatchOutcome{} + } messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) defer cancel() - addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "received") - addStatLagMetric(registry, message) - var env envelope.FrameEnvelope - if err := json.Unmarshal(message.Value, &env); err != nil { - addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "invalid_json") - logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) - _ = committer.CommitMessages(messageCtx, message) + setStatBatchPendingForWorker(registry, workerLabels, len(messages)) + defer setStatBatchPendingForWorker(registry, workerLabels, 0) + defer recordStatCacheMetrics(registry, appender) + items := make([]statBatchItem, len(messages)) + for index, message := range messages { + items[index].message = message + } + for itemIndex, message := range messages { + addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "received") + addStatLagMetric(registry, message) + var env envelope.FrameEnvelope + if err := json.Unmarshal(message.Value, &env); err != nil { + addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "invalid_json") + logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) + items[itemIndex].processed = true + continue + } + if status, err := validateStatFieldsEnvelope(message.Topic, env); err != nil { + addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, status) + logger.Warn("skip mismatched stat fields envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) + items[itemIndex].processed = true + continue + } + started := time.Now() + result, err := appendStatEnvelope(messageCtx, appender, env) + recordStatWriteDuration(registry, message, statusFromError(err), time.Since(started)) + recordStatSampleMetrics(registry, message, env.Protocol, result) + recordStatSourceMetrics(registry, message, env.Protocol, result) + recordStatProjectionMetrics(registry, message, env.Protocol, result) + if err != nil { + addStatMetric(registry, "vehicle_stat_writes_total", message, "error") + logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) + committed, commitErr := commitStatProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) + outcome := statBatchOutcome{retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed)} + if commitErr != nil { + outcome.commitMessages = committed + } + return outcome + } + addStatMetric(registry, "vehicle_stat_writes_total", message, "ok") + recordStatWriteE2EDuration(registry, message, env) + items[itemIndex].processed = true + } + if err := committer.CommitMessages(messageCtx, messages...); err != nil { + for _, message := range messages { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error") + } + first := messages[0] + logger.Error("kafka commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) + return statBatchOutcome{commitMessages: messages} + } + for _, message := range messages { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok") + } + return statBatchOutcome{} +} + +func processStatBatchReliably(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) { + processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, messages, retryDelay, nil) +} + +func processStatBatchReliablyForWorker(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) { + defer registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, 0) + pending := messages + for len(pending) > 0 { + outcome := processStatBatchForWorker(ctx, logger, registry, appender, committer, pending, workerLabels) + if len(outcome.commitMessages) > 0 { + registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !retryStatCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) { + return + } + } + pending = outcome.retryMessages + registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, float64(len(pending))) + if len(pending) == 0 { + return + } + registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "write_error"}) + if !waitForStatRetry(ctx, retryDelay) { + return + } + } +} + +func retryStatCommit(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool { + for len(messages) > 0 { + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) + err := committer.CommitMessages(operationCtx, messages...) + cancel() + if err == nil { + for _, message := range messages { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok") + } + return true + } + for _, message := range messages { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error") + } + first := messages[0] + logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) + registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "commit_error"}) + if !waitForStatRetry(ctx, retryDelay) { + return false + } + } + return true +} + +func waitForStatRetry(ctx context.Context, retryDelay time.Duration) bool { + if retryDelay <= 0 { + retryDelay = 100 * time.Millisecond + } + timer := time.NewTimer(retryDelay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func validateStatFieldsEnvelope(topic string, env envelope.FrameEnvelope) (string, error) { + return topics.ValidateFieldsEnvelope(topic, env) +} + +func commitStatProcessedPrefixAfterFailure(ctx context.Context, logger interface { + Error(string, ...any) + Warn(string, ...any) +}, registry *metrics.Registry, committer kafkaMessageCommitter, items []statBatchItem) ([]kafka.Message, error) { + committable := processedPrefixMessagesByPartition(items) + if len(committable) == 0 { + return nil, nil + } + if err := committer.CommitMessages(ctx, committable...); err != nil { + for _, message := range committable { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error") + } + first := committable[0] + logger.Error("kafka processed-prefix commit failed after mysql append failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err) + return committable, err + } + for _, message := range committable { + addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok") + } + return committable, nil +} + +func processedPrefixMessagesByPartition(items []statBatchItem) []kafka.Message { + type partitionKey struct { + topic string + partition int + } + groups := map[partitionKey][]statBatchItem{} + for _, item := range items { + key := partitionKey{topic: item.message.Topic, partition: item.message.Partition} + groups[key] = append(groups[key], item) + } + var keys []partitionKey + for key := range groups { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].topic != keys[j].topic { + return keys[i].topic < keys[j].topic + } + return keys[i].partition < keys[j].partition + }) + var out []kafka.Message + for _, key := range keys { + group := groups[key] + sort.Slice(group, func(i, j int) bool { + return group[i].message.Offset < group[j].message.Offset + }) + var previousOffset int64 + for index, item := range group { + if index > 0 && item.message.Offset != previousOffset+1 { + break + } + if !item.processed { + break + } + out = append(out, item.message) + previousOffset = item.message.Offset + } + } + return out +} + +func appendStatEnvelope(ctx context.Context, appender statAppender, env envelope.FrameEnvelope) (stats.AppendResult, error) { + if resultAppender, ok := appender.(statResultAppender); ok { + return resultAppender.AppendWithResult(ctx, env) + } + return stats.AppendResult{}, appender.Append(ctx, env) +} + +type retryStatAppender struct { + delegate statAppender + attempts int + delay time.Duration + registry *metrics.Registry +} + +func (a retryStatAppender) Append(ctx context.Context, env envelope.FrameEnvelope) error { + _, err := a.AppendWithResult(ctx, env) + return err +} + +func (a retryStatAppender) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (stats.AppendResult, error) { + if a.delegate == nil { + return stats.AppendResult{}, nil + } + attempts := a.attempts + if attempts <= 0 { + attempts = 1 + } + var result stats.AppendResult + var err error + for attempt := 1; attempt <= attempts; attempt++ { + result, err = appendStatEnvelope(ctx, a.delegate, env) + if err == nil || !isTransientMySQLStatError(err) { + return result, err + } + if attempt == attempts { + a.recordRetry("single", "exhausted") + return result, err + } + a.recordRetry("single", "retry") + if a.delay <= 0 { + continue + } + timer := time.NewTimer(a.delay) + select { + case <-ctx.Done(): + timer.Stop() + return result, ctx.Err() + case <-timer.C: + } + } + return result, err +} + +func (a retryStatAppender) recordRetry(operation string, status string) { + if a.registry == nil { return } - started := time.Now() - err := appender.Append(messageCtx, env) - recordStatWriteDuration(registry, message, statusFromError(err), time.Since(started)) - if err != nil { - addStatMetric(registry, "vehicle_stat_writes_total", message, "error") - logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) - return + labels := metrics.Labels{"operation": operation, "status": status} + a.registry.IncCounter("vehicle_stat_write_retries_total", labels) + metrics.RecordLastActivity(a.registry, "vehicle_stat_last_write_retry_unix_seconds", labels) +} + +func (a retryStatAppender) CacheStats() stats.CacheStats { + if reporter, ok := a.delegate.(statCacheReporter); ok { + return reporter.CacheStats() } - addStatMetric(registry, "vehicle_stat_writes_total", message, "ok") - if err := committer.CommitMessages(messageCtx, message); err != nil { - addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error") - logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) - return + return stats.CacheStats{} +} + +func isTransientMySQLStatError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false } - addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok") + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "deadlock") || + strings.Contains(text, "error 1213") || + strings.Contains(text, "40001") || + strings.Contains(text, "lock wait timeout") || + strings.Contains(text, "error 1205") || + strings.Contains(text, "timeout") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "connection refused") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "bad connection") || + strings.Contains(text, "invalid connection") || + strings.Contains(text, "i/o timeout") || + text == "eof" || + strings.Contains(text, "unexpected eof") || + strings.Contains(text, "server is down") || + strings.Contains(text, "network is unreachable") || + strings.Contains(text, "no route to host") } func addStatMetric(registry *metrics.Registry, name string, message kafka.Message, status string) { if registry == nil { return } - registry.IncCounter(name, metrics.Labels{"topic": message.Topic, "status": status}) + labels := metrics.Labels{"topic": message.Topic, "status": status} + registry.IncCounter(name, labels) + switch name { + case "vehicle_stat_kafka_messages_total": + metrics.RecordLastActivity(registry, "vehicle_stat_last_message_unix_seconds", labels) + case "vehicle_stat_writes_total": + metrics.RecordLastActivity(registry, "vehicle_stat_last_write_unix_seconds", labels) + case "vehicle_stat_kafka_commits_total": + metrics.RecordLastActivity(registry, "vehicle_stat_last_commit_unix_seconds", labels) + } +} + +func recordStatSampleMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) { + addStatSampleMetric(registry, message, protocol, "found", result.SamplesFound) + addStatSampleMetric(registry, message, protocol, "written", result.SamplesWritten) + addStatSampleMetric(registry, message, protocol, "skipped_missing_fields", result.SamplesSkippedMissingFields) + addStatSampleMetric(registry, message, protocol, "skipped_missing_vin", result.SamplesSkippedMissingVIN) + addStatSampleMetric(registry, message, protocol, "skipped_missing_mileage", result.SamplesSkippedMissingMileage) + addStatSampleMetric(registry, message, protocol, "skipped_non_mileage_frame", result.SamplesSkippedNonMileageFrame) + addStatSampleMetric(registry, message, protocol, "skipped_non_positive_mileage", result.SamplesSkippedNonPositiveMileage) + addStatSampleMetric(registry, message, protocol, "skipped_missing_time", result.SamplesSkippedMissingTime) + addStatSampleMetric(registry, message, protocol, "skipped_same_mileage", result.SamplesSkippedSameMileage) + addStatSampleMetric(registry, message, protocol, "skipped_missing_source", result.SamplesSkippedMissingSource) + addStatSampleMetric(registry, message, protocol, "event_time_future_adjusted", result.SamplesAdjustedFutureEventTime) +} + +func addStatSampleMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) { + if registry == nil || count == 0 { + return + } + registry.AddCounter("vehicle_stat_samples_total", metrics.Labels{ + "topic": message.Topic, + "protocol": statProtocolLabel(protocol), + "status": status, + }, float64(count)) +} + +func recordStatSourceMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) { + addStatSourceMetric(registry, message, protocol, "attempted", result.SourceTouchesAttempted) + addStatSourceMetric(registry, message, protocol, "written", result.SourceTouchesWritten) + addStatSourceMetric(registry, message, protocol, "skipped_throttled", result.SourceTouchesSkippedThrottled) + addStatSourceMetric(registry, message, protocol, "skipped_missing_endpoint", result.SourceTouchesSkippedMissing) + addStatSourceMetric(registry, message, protocol, "skipped_unmanaged", result.SourceTouchesSkippedUnmanaged) +} + +func addStatSourceMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) { + if registry == nil || count == 0 { + return + } + registry.AddCounter("vehicle_stat_sources_total", metrics.Labels{ + "topic": message.Topic, + "protocol": statProtocolLabel(protocol), + "status": status, + }, float64(count)) +} + +func recordStatProjectionMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) { + addStatProjectionMetric(registry, message, protocol, "attempted", result.ProjectionsAttempted) + addStatProjectionMetric(registry, message, protocol, "written", result.ProjectionsWritten) + addStatProjectionMetric(registry, message, protocol, "skipped_throttled", result.ProjectionsSkippedThrottled) +} + +func addStatProjectionMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) { + if registry == nil || count == 0 { + return + } + registry.AddCounter("vehicle_stat_projections_total", metrics.Labels{ + "topic": message.Topic, + "protocol": statProtocolLabel(protocol), + "status": status, + }, float64(count)) +} + +func statProtocolLabel(protocol envelope.Protocol) string { + protocolLabel := strings.TrimSpace(string(protocol)) + if protocolLabel == "" { + return "UNKNOWN" + } + return protocolLabel +} + +func recordStatCacheMetrics(registry *metrics.Registry, appender statAppender) { + if registry == nil { + return + } + reporter, ok := appender.(statCacheReporter) + if !ok { + return + } + stats := reporter.CacheStats() + setStatCacheGauge(registry, "last_total_mileage", stats.LastTotalMileageEntries, stats.MaxEntries, stats.LastCleanupTotalMileage, stats.TotalMileageEvictions) + setStatCacheGauge(registry, "source_seen", stats.LastSourceSeenEntries, stats.MaxEntries, stats.LastCleanupSourceSeen, stats.SourceSeenEvictions) + setStatCacheGauge(registry, "projection", stats.LastProjectionEntries, stats.MaxEntries, stats.LastCleanupProjection, stats.ProjectionEvictions) + setStatCacheGauge(registry, "baseline", stats.BaselineEntries, stats.MaxEntries, stats.LastCleanupBaseline, stats.BaselineEvictions) + if !stats.LastCleanupAt.IsZero() { + registry.SetGauge("vehicle_stat_cache_last_cleanup_unix_seconds", nil, float64(stats.LastCleanupAt.Unix())) + } +} + +func setStatCacheGauge(registry *metrics.Registry, cache string, entries int, maxEntries int, lastCleanupDeleted int, evictions int) { + labels := metrics.Labels{"cache": cache} + registry.SetGauge("vehicle_stat_cache_entries", labels, float64(entries)) + registry.SetGauge("vehicle_stat_cache_max_entries", labels, float64(maxEntries)) + registry.SetGauge("vehicle_stat_cache_last_cleanup_deleted", labels, float64(lastCleanupDeleted)) + registry.SetGauge("vehicle_stat_cache_evictions_total", labels, float64(evictions)) } func addStatLagMetric(registry *metrics.Registry, message kafka.Message) { @@ -142,6 +640,33 @@ func recordStatWriteDuration(registry *metrics.Registry, message kafka.Message, }, statWriteDurationBucketsMS, float64(elapsed.Milliseconds())) } +func recordStatWriteE2EDuration(registry *metrics.Registry, message kafka.Message, env envelope.FrameEnvelope) { + if registry == nil || env.ReceivedAtMS <= 0 { + return + } + elapsed := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds() + if elapsed < 0 { + elapsed = 0 + } + labels := metrics.Labels{"topic": message.Topic} + registry.ObserveHistogram("vehicle_stat_write_e2e_duration_ms_histogram", labels, statWriteE2EDurationBucketsMS, float64(elapsed)) + p99, samples := statWriteE2ERecent.Observe(message.Topic, float64(elapsed)) + registry.SetGauge("vehicle_stat_write_e2e_recent_p99_ms", labels, p99) + registry.SetGauge("vehicle_stat_write_e2e_recent_samples", labels, float64(samples)) + metrics.RecordLastActivity(registry, "vehicle_stat_last_write_e2e_unix_seconds", labels) +} + +func setStatBatchPending(registry *metrics.Registry, messages int) { + setStatBatchPendingForWorker(registry, nil, messages) +} + +func setStatBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int) { + if registry == nil { + return + } + registry.SetGauge("vehicle_stat_batch_pending_messages", workerLabels, float64(messages)) +} + func statusFromError(err error) string { if err != nil { return "error" @@ -150,12 +675,40 @@ func statusFromError(err error) string { } type config struct { - KafkaBrokers []string - KafkaTopics []string - KafkaGroup string - MySQLDSN string - EnsureSchema bool - Location *time.Location + KafkaBrokers []string + KafkaTopics []string + KafkaGroup string + MySQLDSN string + EnsureSchema bool + Location *time.Location + ProjectInterval time.Duration + SourceTouchInterval time.Duration + CacheRetention time.Duration + CacheCleanupInterval time.Duration + BaselineMissTTL time.Duration + CacheMaxEntries int + Workers int + BatchSize int + BatchWait int + RetryAttempts int + RetryDelay time.Duration + NormalizePlatformSourcesOnStart bool + NormalizePlatformSourcesTimeout time.Duration +} + +func (c config) Validate() error { + if len(c.KafkaTopics) == 0 { + return fmt.Errorf("KAFKA_TOPICS must include fields topics") + } + for _, topic := range c.KafkaTopics { + if !strings.HasPrefix(topic, "vehicle.fields.") { + return fmt.Errorf("stat-writer consumes fields topics only, got %q", topic) + } + } + if c.Workers <= 0 { + return fmt.Errorf("STATS_WORKERS must be greater than zero") + } + return nil } func loadConfig() config { @@ -164,12 +717,25 @@ func loadConfig() config { loc = time.FixedZone("Asia/Shanghai", 8*3600) } return config{ - KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), - KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT}, ","))), - KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"), - MySQLDSN: env("MYSQL_DSN", ""), - EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false", - Location: loc, + KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")), + KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT}, ","))), + KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"), + MySQLDSN: env("MYSQL_DSN", ""), + EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false", + Location: loc, + ProjectInterval: time.Duration(envInt("STATS_PROJECT_INTERVAL_SECONDS", 15)) * time.Second, + SourceTouchInterval: time.Duration(envInt("STATS_SOURCE_TOUCH_INTERVAL_SECONDS", 60)) * time.Second, + CacheRetention: time.Duration(envInt("STATS_CACHE_RETENTION_HOURS", 72)) * time.Hour, + CacheCleanupInterval: time.Duration(envInt("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second, + BaselineMissTTL: time.Duration(envInt("STATS_BASELINE_MISS_TTL_SECONDS", 60)) * time.Second, + CacheMaxEntries: envInt("STATS_CACHE_MAX_ENTRIES", 1000000), + Workers: envInt("STATS_WORKERS", 3), + BatchSize: envInt("STATS_BATCH_SIZE", 200), + BatchWait: envInt("STATS_BATCH_WAIT_MS", 20), + RetryAttempts: envInt("STATS_RETRY_ATTEMPTS", 3), + RetryDelay: time.Duration(envInt("STATS_RETRY_DELAY_MS", 20)) * time.Millisecond, + NormalizePlatformSourcesOnStart: env("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "true") != "false", + NormalizePlatformSourcesTimeout: time.Duration(envInt("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", 30)) * time.Second, } } @@ -181,6 +747,18 @@ func env(key string, fallback string) string { return value } +func envInt(key string, fallback int) int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + func splitCSV(value string) []string { var out []string for _, item := range strings.Split(value, ",") { diff --git a/go/vehicle-gateway/cmd/stat-writer/main_test.go b/go/vehicle-gateway/cmd/stat-writer/main_test.go index 2a3903f8..73c48719 100644 --- a/go/vehicle-gateway/cmd/stat-writer/main_test.go +++ b/go/vehicle-gateway/cmd/stat-writer/main_test.go @@ -3,15 +3,50 @@ package main import ( "context" "encoding/json" + "errors" "strings" "testing" + "time" "github.com/segmentio/kafka-go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" ) +func statTestFieldsEnvelope(env envelope.FrameEnvelope) envelope.FrameEnvelope { + if env.EventKind == "" { + env.EventKind = envelope.EventKindFields + } + if env.FieldMapping == "" { + env.FieldMapping = "test-field-mapping.v1" + } + if env.ParseStatus == "" { + env.ParseStatus = envelope.ParseOK + } + if len(env.Fields) == 0 { + switch env.Protocol { + case envelope.ProtocolGB32960: + env.Fields = map[string]any{"gb32960.vehicle.total_mileage_km": 4123.9} + case envelope.ProtocolYutongMQTT: + env.Fields = map[string]any{"yutong_mqtt.data.total_mileage": 4123900} + default: + env.Fields = map[string]any{"jt808.location.total_mileage_km": 4123.9} + } + } + return env +} + +func marshalStatTestFieldsEnvelope(t *testing.T, env envelope.FrameEnvelope) []byte { + t.Helper() + payload, err := json.Marshal(statTestFieldsEnvelope(env)) + if err != nil { + t.Fatal(err) + } + return payload +} + func TestProcessStatMessageUsesUncancelledContextForAppendAndCommit(t *testing.T) { parent, cancel := context.WithCancel(context.Background()) cancel() @@ -45,31 +80,94 @@ func TestProcessStatMessageUsesUncancelledContextForAppendAndCommit(t *testing.T } func TestProcessStatMessageRecordsMetrics(t *testing.T) { - env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} - payload, err := json.Marshal(env) - if err != nil { - t.Fatal(err) - } + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()} + payload := marshalStatTestFieldsEnvelope(t, env) registry := metrics.NewRegistry() processStatMessage( context.Background(), discardStatLogger{}, registry, - &contextCheckingStatAppender{}, + &contextCheckingStatAppender{result: stats.AppendResult{ + SamplesFound: 1, + SamplesWritten: 1, + SamplesSkippedMissingFields: 3, + SamplesSkippedNonMileageFrame: 4, + SamplesSkippedSameMileage: 2, + SamplesAdjustedFutureEventTime: 1, + SourceTouchesAttempted: 1, + SourceTouchesWritten: 1, + SourceTouchesSkippedThrottled: 2, + SourceTouchesSkippedMissing: 3, + ProjectionsAttempted: 1, + ProjectionsWritten: 1, + ProjectionsSkippedThrottled: 2, + }, cacheStats: stats.CacheStats{ + LastTotalMileageEntries: 300, + LastSourceSeenEntries: 3, + LastProjectionEntries: 280, + BaselineEntries: 295, + MaxEntries: 1000000, + LastCleanupAt: time.Unix(1782918600, 0), + LastCleanupTotalMileage: 11, + LastCleanupSourceSeen: 1, + LastCleanupProjection: 9, + LastCleanupBaseline: 10, + TotalMileageEvictions: 4, + SourceSeenEvictions: 1, + ProjectionEvictions: 2, + BaselineEvictions: 3, + }}, &contextCheckingStatCommitter{}, - kafka.Message{Topic: "vehicle.raw.jt808.v1", Partition: 4, Offset: 30, HighWaterMark: 41, Value: payload}, + kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 4, Offset: 30, HighWaterMark: 41, Value: payload}, ) text := registry.Render() for _, want := range []string{ - `vehicle_stat_kafka_messages_total{status="received",topic="vehicle.raw.jt808.v1"} 1`, - `vehicle_stat_writes_total{status="ok",topic="vehicle.raw.jt808.v1"} 1`, - `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.raw.jt808.v1"} 1`, - `vehicle_stat_kafka_lag{partition="4",topic="vehicle.raw.jt808.v1"} 10`, - `vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.raw.jt808.v1"} 1`, - `vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.raw.jt808.v1"} 1`, - `vehicle_stat_write_duration_ms_histogram_sum{status="ok",topic="vehicle.raw.jt808.v1"}`, + `vehicle_stat_kafka_messages_total{status="received",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_last_message_unix_seconds{status="received",topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_last_write_unix_seconds{status="ok",topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_last_write_e2e_unix_seconds{topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_fields",topic="vehicle.fields.go.jt808.v1"} 3`, + `vehicle_stat_samples_total{protocol="JT808",status="skipped_non_mileage_frame",topic="vehicle.fields.go.jt808.v1"} 4`, + `vehicle_stat_samples_total{protocol="JT808",status="skipped_same_mileage",topic="vehicle.fields.go.jt808.v1"} 2`, + `vehicle_stat_samples_total{protocol="JT808",status="event_time_future_adjusted",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_sources_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_sources_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_sources_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 2`, + `vehicle_stat_sources_total{protocol="JT808",status="skipped_missing_endpoint",topic="vehicle.fields.go.jt808.v1"} 3`, + `vehicle_stat_projections_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 2`, + `vehicle_stat_cache_entries{cache="last_total_mileage"} 300`, + `vehicle_stat_cache_entries{cache="source_seen"} 3`, + `vehicle_stat_cache_entries{cache="projection"} 280`, + `vehicle_stat_cache_entries{cache="baseline"} 295`, + `vehicle_stat_cache_max_entries{cache="last_total_mileage"} 1000000`, + `vehicle_stat_cache_max_entries{cache="source_seen"} 1000000`, + `vehicle_stat_cache_max_entries{cache="projection"} 1000000`, + `vehicle_stat_cache_max_entries{cache="baseline"} 1000000`, + `vehicle_stat_cache_last_cleanup_deleted{cache="last_total_mileage"} 11`, + `vehicle_stat_cache_last_cleanup_deleted{cache="source_seen"} 1`, + `vehicle_stat_cache_last_cleanup_deleted{cache="projection"} 9`, + `vehicle_stat_cache_last_cleanup_deleted{cache="baseline"} 10`, + `vehicle_stat_cache_evictions_total{cache="last_total_mileage"} 4`, + `vehicle_stat_cache_evictions_total{cache="source_seen"} 1`, + `vehicle_stat_cache_evictions_total{cache="projection"} 2`, + `vehicle_stat_cache_evictions_total{cache="baseline"} 3`, + `vehicle_stat_cache_last_cleanup_unix_seconds 1782918600`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_last_commit_unix_seconds{status="ok",topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_kafka_lag{partition="4",topic="vehicle.fields.go.jt808.v1"} 10`, + `vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_write_duration_ms_histogram_sum{status="ok",topic="vehicle.fields.go.jt808.v1"}`, } { if !strings.Contains(text, want) { t.Fatalf("metrics missing %s:\n%s", want, text) @@ -77,7 +175,712 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) { } } -func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) { +func TestRecordStatWriteE2EDurationSkipsMissingReceivedAt(t *testing.T) { + registry := metrics.NewRegistry() + + recordStatWriteE2EDuration(registry, kafka.Message{Topic: "vehicle.fields.go.jt808.v1"}, envelope.FrameEnvelope{}) + + text := registry.Render() + if strings.Contains(text, "vehicle_stat_write_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_stat_write_e2e_recent") { + t.Fatalf("unexpected e2e metric without received_at:\n%s", text) + } +} + +func TestProcessStatBatchAppendsAllBeforeCommit(t *testing.T) { + first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()} + second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()} + firstPayload := marshalStatTestFieldsEnvelope(t, first) + secondPayload := marshalStatTestFieldsEnvelope(t, second) + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 13, Value: firstPayload}, + {Topic: "vehicle.fields.go.gb32960.v1", Partition: 2, Offset: 20, HighWaterMark: 21, Value: secondPayload}, + }, + ) + + if appender.count != 2 { + t.Fatalf("appends = %d, want 2", appender.count) + } + if committer.count != 1 { + t.Fatalf("commit calls = %d, want 1", committer.count) + } + if committer.messageCount != 2 { + t.Fatalf("committed messages = %d, want 2", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_batch_pending_messages 0`, + `vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`, + `vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.gb32960.v1"} 1`, + `vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} `, + `vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} `, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("batch metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessStatBatchKeepsPendingGaugePerWorker(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload := marshalStatTestFieldsEnvelope(t, env) + registry := metrics.NewRegistry() + + processStatBatchForWorker( + context.Background(), + discardStatLogger{}, + registry, + &contextCheckingStatAppender{}, + &contextCheckingStatCommitter{}, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}}, + metrics.Labels{"worker": "2"}, + ) + + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_batch_pending_messages{worker="2"} 0`) { + t.Fatalf("worker pending gauge missing or not reset:\n%s", text) + } +} + +func TestProcessStatBatchSamplesCacheMetricsOncePerBatch(t *testing.T) { + payload := marshalStatTestFieldsEnvelope(t, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LNBVIN00000000001", + EventTimeMS: 1782918600000, + ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + appender := &contextCheckingStatAppender{ + result: stats.AppendResult{SamplesFound: 1, SamplesWritten: 1}, + cacheStats: stats.CacheStats{ + LastTotalMileageEntries: 12, + MaxEntries: 99, + }, + } + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload}, + }, + ) + + if appender.cacheStatsCalls != 1 { + t.Fatalf("CacheStats calls = %d, want 1 per batch", appender.cacheStatsCalls) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_cache_entries{cache="last_total_mileage"} 12`) { + t.Fatalf("cache metric missing:\n%s", text) + } +} + +func TestProcessStatBatchCommitsInvalidJSONWithoutWriteMetric(t *testing.T) { + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Value: []byte("{bad json")}}, + ) + + if appender.count != 0 { + t.Fatalf("appends = %d, want 0", appender.count) + } + if committer.count != 1 { + t.Fatalf("commit calls = %d, want 1", committer.count) + } + if committer.messageCount != 1 { + t.Fatalf("committed messages = %d, want 1", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("invalid json metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_stat_writes_total`) { + t.Fatalf("invalid json should not record write metric:\n%s", text) + } +} + +func TestProcessStatBatchSkipsKnownFieldsTopicProtocolMismatch(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.gb32960.v1", Partition: 1, Offset: 10, Value: payload}}, + ) + + if appender.count != 0 { + t.Fatalf("appends = %d, want 0", appender.count) + } + if committer.count != 1 || committer.messageCount != 1 { + t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("protocol mismatch metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_stat_writes_total`) { + t.Fatalf("protocol mismatch should not record write metric:\n%s", text) + } +} + +func TestProcessStatBatchSkipsExplicitNonFieldsEventKind(t *testing.T) { + env := envelope.FrameEnvelope{ + EventKind: envelope.EventKindRaw, + Protocol: envelope.ProtocolJT808, + Phone: "13307795425", + MessageID: "0x0200", + } + payload, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}}, + ) + + if appender.count != 0 { + t.Fatalf("appends = %d, want 0", appender.count) + } + if committer.count != 1 || committer.messageCount != 1 { + t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("event kind mismatch metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_stat_writes_total`) { + t.Fatalf("event kind mismatch should not record write metric:\n%s", text) + } +} + +func TestProcessStatBatchSkipsFieldsContractViolation(t *testing.T) { + tests := []struct { + name string + env envelope.FrameEnvelope + status string + }{ + { + name: "missing_field_mapping", + env: envelope.FrameEnvelope{ + EventKind: envelope.EventKindFields, + Protocol: envelope.ProtocolJT808, + Fields: map[string]any{"jt808.location.total_mileage_km": 4123.9}, + }, + status: "missing_field_mapping", + }, + { + name: "missing_fields", + env: envelope.FrameEnvelope{ + EventKind: envelope.EventKindFields, + Protocol: envelope.ProtocolJT808, + FieldMapping: "test-field-mapping.v1", + }, + status: "missing_fields", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + payload, err := json.Marshal(tt.env) + if err != nil { + t.Fatal(err) + } + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}}, + ) + + if appender.count != 0 { + t.Fatalf("appends = %d, want 0", appender.count) + } + if committer.count != 1 || committer.messageCount != 1 { + t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_kafka_messages_total{status="` + tt.status + `",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("contract violation metric missing %s:\n%s", want, text) + } + } + if strings.Contains(text, `vehicle_stat_writes_total`) { + t.Fatalf("contract violation should not record write metric:\n%s", text) + } + }) + } +} + +func TestProcessStatBatchExposesPendingMessagesDuringAppend(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload := marshalStatTestFieldsEnvelope(t, env) + registry := metrics.NewRegistry() + appender := &contextCheckingStatAppender{ + onAppend: func() { + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_batch_pending_messages 2`) { + t.Fatalf("pending batch metric missing while appending:\n%s", text) + } + }, + } + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + &contextCheckingStatCommitter{}, + []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Value: payload}, + {Topic: "vehicle.fields.go.jt808.v1", Value: payload}, + }, + ) + + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_batch_pending_messages 0`) { + t.Fatalf("pending batch metric should reset after append:\n%s", text) + } +} + +func TestProcessStatBatchDoesNotCommitWhenAppendFails(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload := marshalStatTestFieldsEnvelope(t, env) + appender := &contextCheckingStatAppender{err: errTestStatAppend} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Value: payload}}, + ) + + if committer.count != 0 { + t.Fatalf("commit calls = %d, want 0", committer.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_writes_total{status="error",topic="vehicle.fields.go.jt808.v1"} 1`) { + t.Fatalf("write error metric missing:\n%s", text) + } +} + +func TestProcessStatBatchCommitsMissingTimeSample(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + } + payload := marshalStatTestFieldsEnvelope(t, env) + resultAppender := &contextCheckingStatAppender{ + result: stats.AppendResult{ + SamplesSkippedMissingTime: 1, + SourceTouchesAttempted: 1, + SourceTouchesWritten: 1, + }, + } + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + resultAppender, + committer, + []kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}}, + ) + + if committer.messageCount != 1 { + t.Fatalf("committed messages = %d, want 1", committer.messageCount) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_time",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("missing time metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessStatBatchCommitsProcessedPrefixWhenLaterAppendFails(t *testing.T) { + success := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + failed := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + successPayload := marshalStatTestFieldsEnvelope(t, success) + failedPayload := marshalStatTestFieldsEnvelope(t, failed) + appender := &contextCheckingStatAppender{failOnCount: 2, err: errTestStatAppend} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + + outcome := processStatBatch( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: []byte("{bad json")}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: successPayload}, + {Topic: "vehicle.fields.go.gb32960.v1", Partition: 2, Offset: 20, Value: failedPayload}, + }, + ) + + if appender.count != 2 { + t.Fatalf("appends = %d, want 2 valid appends", appender.count) + } + if committer.messageCount != 2 { + t.Fatalf("committed messages = %d, want processed prefix only", committer.messageCount) + } + if got := committedOffsets(outcome.retryMessages); len(got) != 1 || got[0] != 20 { + t.Fatalf("retry offsets = %#v, want [20]", got) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`, + `vehicle_stat_writes_total{status="error",topic="vehicle.fields.go.gb32960.v1"} 1`, + `vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 2`, + } { + if !strings.Contains(text, want) { + t.Fatalf("processed prefix metric missing %s:\n%s", want, text) + } + } +} + +func TestProcessStatBatchReliablyRetriesFailedSuffix(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload := marshalStatTestFieldsEnvelope(t, env) + appender := &contextCheckingStatAppender{failOnCount: 2, err: errTestStatAppend} + committer := &contextCheckingStatCommitter{} + registry := metrics.NewRegistry() + messages := []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12, Value: payload}, + } + + processStatBatchReliably( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + messages, + time.Nanosecond, + ) + + if appender.count != 4 { + t.Fatalf("append attempts = %d, want success + failed + retried suffix", appender.count) + } + if got := committedOffsets(committer.messages); len(got) != 3 || got[0] != 10 || got[1] != 11 || got[2] != 12 { + t.Fatalf("committed offsets = %#v, want [10 11 12]", got) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_batch_retries_total{reason="write_error"} 1`) { + t.Fatalf("write retry metric missing:\n%s", text) + } + if !strings.Contains(text, `vehicle_stat_retry_pending_messages 0`) { + t.Fatalf("retry pending gauge should reset:\n%s", text) + } +} + +func TestProcessStatBatchReliablyRetriesCommitWithoutReappending(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + payload := marshalStatTestFieldsEnvelope(t, env) + appender := &contextCheckingStatAppender{} + committer := &contextCheckingStatCommitter{err: errors.New("commit failed"), failOnCount: 1} + registry := metrics.NewRegistry() + messages := []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload}, + } + + processStatBatchReliably( + context.Background(), + discardStatLogger{}, + registry, + appender, + committer, + messages, + time.Nanosecond, + ) + + if appender.count != 2 { + t.Fatalf("append attempts = %d, want 2 without replay after commit failure", appender.count) + } + if committer.count != 2 { + t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_batch_retries_total{reason="commit_error"} 1`) { + t.Fatalf("commit retry metric missing:\n%s", text) + } +} + +func TestProcessStatBatchDoesNotCommitAcrossOffsetGapAfterFailure(t *testing.T) { + success := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + failed := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"} + successPayload := marshalStatTestFieldsEnvelope(t, success) + failedPayload := marshalStatTestFieldsEnvelope(t, failed) + appender := &contextCheckingStatAppender{failOnCount: 1, err: errTestStatAppend} + committer := &contextCheckingStatCommitter{} + + processStatBatch( + context.Background(), + discardStatLogger{}, + metrics.NewRegistry(), + appender, + committer, + []kafka.Message{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: successPayload}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12, Value: failedPayload}, + }, + ) + + if committer.count != 0 { + t.Fatalf("commit calls = %d, want 0 because failed first processed offset must not be skipped", committer.count) + } +} + +func TestProcessedPrefixMessagesByPartitionStopsAtGap(t *testing.T) { + items := []statBatchItem{ + {message: kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10}, processed: true}, + {message: kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12}, processed: true}, + } + + got := processedPrefixMessagesByPartition(items) + if len(got) != 1 || got[0].Offset != 10 { + t.Fatalf("processed prefix = %#v, want only offset 10", got) + } +} + +func TestRecordStatSampleMetricsLabelsUnknownProtocol(t *testing.T) { + registry := metrics.NewRegistry() + + recordStatSampleMetrics(registry, kafka.Message{Topic: "vehicle.fields.go.unknown.v1"}, "", stats.AppendResult{ + SamplesSkippedMissingVIN: 1, + }) + recordStatProjectionMetrics(registry, kafka.Message{Topic: "vehicle.fields.go.unknown.v1"}, "", stats.AppendResult{ + ProjectionsSkippedThrottled: 1, + }) + + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_samples_total{protocol="UNKNOWN",status="skipped_missing_vin",topic="vehicle.fields.go.unknown.v1"} 1`, + `vehicle_stat_projections_total{protocol="UNKNOWN",status="skipped_throttled",topic="vehicle.fields.go.unknown.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metrics missing %s:\n%s", want, text) + } + } +} + +func TestRetryStatAppenderRetriesTransientMySQLDeadlock(t *testing.T) { + registry := metrics.NewRegistry() + delegate := &contextCheckingStatAppender{ + failOnCount: 1, + err: errors.New("Error 1213: Deadlock found when trying to get lock"), + result: stats.AppendResult{SamplesWritten: 1}, + } + appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry} + + result, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}) + + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if delegate.count != 2 { + t.Fatalf("append attempts = %d, want 2", delegate.count) + } + if result.SamplesWritten != 1 { + t.Fatalf("result = %#v, want successful result", result) + } + text := registry.Render() + if !strings.Contains(text, `vehicle_stat_write_retries_total{operation="single",status="retry"} 1`) { + t.Fatalf("retry metric missing:\n%s", text) + } +} + +func TestRetryStatAppenderRetriesTransientMySQLConnectionFailure(t *testing.T) { + delegate := &contextCheckingStatAppender{ + failOnCount: 1, + err: errors.New("driver: bad connection: read tcp 10.0.0.1:3306: connection reset by peer"), + result: stats.AppendResult{SamplesWritten: 1}, + } + appender := retryStatAppender{delegate: delegate, attempts: 3} + + result, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}) + + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if delegate.count != 2 { + t.Fatalf("append attempts = %d, want 2", delegate.count) + } + if result.SamplesWritten != 1 { + t.Fatalf("result = %#v, want successful retry result", result) + } +} + +func TestRetryStatAppenderRecordsExhaustedTransientError(t *testing.T) { + registry := metrics.NewRegistry() + delegate := &contextCheckingStatAppender{err: errors.New("Error 1205: Lock wait timeout exceeded")} + appender := retryStatAppender{delegate: delegate, attempts: 2, registry: registry} + + _, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}) + + if err == nil { + t.Fatal("AppendWithResult() error = nil, want exhausted transient error") + } + if delegate.count != 2 { + t.Fatalf("append attempts = %d, want 2", delegate.count) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_stat_write_retries_total{operation="single",status="retry"} 1`, + `vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("retry metric missing %s:\n%s", want, text) + } + } +} + +func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) { + wantErr := errors.New("validation failed") + delegate := &contextCheckingStatAppender{err: wantErr} + appender := retryStatAppender{delegate: delegate, attempts: 3} + + _, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}) + + if !errors.Is(err, wantErr) { + t.Fatalf("AppendWithResult() error = %v, want %v", err, wantErr) + } + if delegate.count != 1 { + t.Fatalf("append attempts = %d, want 1", delegate.count) + } +} + +func TestIsTransientMySQLStatError(t *testing.T) { + for _, err := range []error{ + errors.New("dial tcp 127.0.0.1:3306: connection refused"), + errors.New("read tcp: connection reset by peer"), + errors.New("write tcp: broken pipe"), + errors.New("driver: bad connection"), + errors.New("invalid connection"), + errors.New("i/o timeout"), + errors.New("EOF"), + errors.New("server is down"), + errors.New("network is unreachable"), + } { + if !isTransientMySQLStatError(err) { + t.Fatalf("isTransientMySQLStatError(%q) = false, want true", err.Error()) + } + } + for _, err := range []error{ + context.Canceled, + context.DeadlineExceeded, + errors.New("duplicate key conflict"), + nil, + } { + if isTransientMySQLStatError(err) { + t.Fatalf("isTransientMySQLStatError(%v) = true, want false", err) + } + } +} + +func TestRetryStatAppenderForwardsCacheStats(t *testing.T) { + delegate := &contextCheckingStatAppender{cacheStats: stats.CacheStats{BaselineEntries: 12, MaxEntries: 99}} + appender := retryStatAppender{delegate: delegate, attempts: 3} + + got := appender.CacheStats() + + if got.BaselineEntries != 12 || got.MaxEntries != 99 { + t.Fatalf("CacheStats() = %+v, want delegated stats", got) + } +} + +func TestLoadConfigDefaultsToGoFieldsTopics(t *testing.T) { cfg := loadConfig() got := strings.Join(cfg.KafkaTopics, ",") @@ -85,31 +888,213 @@ func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) { if got != want { t.Fatalf("KafkaTopics = %q, want %q", got, want) } + if cfg.BatchSize != 200 { + t.Fatalf("BatchSize = %d, want 200", cfg.BatchSize) + } + if cfg.BatchWait != 20 { + t.Fatalf("BatchWait = %d, want 20", cfg.BatchWait) + } + if cfg.CacheMaxEntries != 1000000 { + t.Fatalf("CacheMaxEntries = %d, want 1000000", cfg.CacheMaxEntries) + } + if cfg.Workers != 3 { + t.Fatalf("Workers = %d, want 3", cfg.Workers) + } + if cfg.BaselineMissTTL != time.Minute { + t.Fatalf("BaselineMissTTL = %s, want 1m", cfg.BaselineMissTTL) + } + if cfg.RetryAttempts != 3 { + t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts) + } + if cfg.RetryDelay != 20*time.Millisecond { + t.Fatalf("RetryDelay = %s, want 20ms", cfg.RetryDelay) + } + if !cfg.NormalizePlatformSourcesOnStart { + t.Fatal("NormalizePlatformSourcesOnStart = false, want default true") + } + if cfg.NormalizePlatformSourcesTimeout != 30*time.Second { + t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 30s", cfg.NormalizePlatformSourcesTimeout) + } +} + +func TestConfigValidateRejectsRawTopics(t *testing.T) { + cfg := config{KafkaTopics: []string{"vehicle.raw.go.jt808.v1"}} + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want raw topic rejection") + } + if !strings.Contains(err.Error(), "fields topics only") { + t.Fatalf("Validate() error = %q, want fields topic hint", err) + } +} + +func TestConfigValidateRequiresTopics(t *testing.T) { + cfg := config{} + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want missing topic rejection") + } + if !strings.Contains(err.Error(), "KAFKA_TOPICS") { + t.Fatalf("Validate() error = %q, want KAFKA_TOPICS hint", err) + } +} + +func TestConfigValidateRequiresPositiveWorkers(t *testing.T) { + cfg := config{ + KafkaTopics: []string{"vehicle.fields.go.jt808.v1"}, + Workers: 0, + } + + err := cfg.Validate() + + if err == nil { + t.Fatal("Validate() error = nil, want worker count rejection") + } + if !strings.Contains(err.Error(), "STATS_WORKERS") { + t.Fatalf("Validate() error = %q, want STATS_WORKERS hint", err) + } +} + +func TestLoadConfigSetsStatWriteIntervals(t *testing.T) { + t.Setenv("STATS_PROJECT_INTERVAL_SECONDS", "7") + t.Setenv("STATS_SOURCE_TOUCH_INTERVAL_SECONDS", "11") + t.Setenv("STATS_CACHE_RETENTION_HOURS", "48") + t.Setenv("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", "300") + t.Setenv("STATS_BASELINE_MISS_TTL_SECONDS", "45") + t.Setenv("STATS_CACHE_MAX_ENTRIES", "12345") + + cfg := loadConfig() + + if cfg.ProjectInterval != 7*time.Second { + t.Fatalf("ProjectInterval = %s, want 7s", cfg.ProjectInterval) + } + if cfg.SourceTouchInterval != 11*time.Second { + t.Fatalf("SourceTouchInterval = %s, want 11s", cfg.SourceTouchInterval) + } + if cfg.CacheRetention != 48*time.Hour { + t.Fatalf("CacheRetention = %s, want 48h", cfg.CacheRetention) + } + if cfg.CacheCleanupInterval != 5*time.Minute { + t.Fatalf("CacheCleanupInterval = %s, want 5m", cfg.CacheCleanupInterval) + } + if cfg.BaselineMissTTL != 45*time.Second { + t.Fatalf("BaselineMissTTL = %s, want 45s", cfg.BaselineMissTTL) + } + if cfg.CacheMaxEntries != 12345 { + t.Fatalf("CacheMaxEntries = %d, want 12345", cfg.CacheMaxEntries) + } +} + +func TestLoadConfigReadsStatBatchSettings(t *testing.T) { + t.Setenv("STATS_WORKERS", "5") + t.Setenv("STATS_BATCH_SIZE", "500") + t.Setenv("STATS_BATCH_WAIT_MS", "250") + t.Setenv("STATS_RETRY_ATTEMPTS", "5") + t.Setenv("STATS_RETRY_DELAY_MS", "33") + t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "false") + t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", "17") + + cfg := loadConfig() + + if cfg.Workers != 5 { + t.Fatalf("Workers = %d, want 5", cfg.Workers) + } + if cfg.BatchSize != 500 { + t.Fatalf("BatchSize = %d, want 500", cfg.BatchSize) + } + if cfg.BatchWait != 250 { + t.Fatalf("BatchWait = %d, want 250", cfg.BatchWait) + } + if cfg.RetryAttempts != 5 { + t.Fatalf("RetryAttempts = %d, want 5", cfg.RetryAttempts) + } + if cfg.RetryDelay != 33*time.Millisecond { + t.Fatalf("RetryDelay = %s, want 33ms", cfg.RetryDelay) + } + if cfg.NormalizePlatformSourcesOnStart { + t.Fatal("NormalizePlatformSourcesOnStart = true, want env override false") + } + if cfg.NormalizePlatformSourcesTimeout != 17*time.Second { + t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 17s", cfg.NormalizePlatformSourcesTimeout) + } } type contextCheckingStatAppender struct { - ctxErr error - count int + ctxErr error + count int + result stats.AppendResult + cacheStats stats.CacheStats + cacheStatsCalls int + err error + failOnCount int + onAppend func() } func (a *contextCheckingStatAppender) Append(ctx context.Context, _ envelope.FrameEnvelope) error { a.ctxErr = ctx.Err() a.count++ - return a.ctxErr + if a.onAppend != nil { + a.onAppend() + } + if a.ctxErr != nil { + return a.ctxErr + } + if a.failOnCount > 0 { + if a.count == a.failOnCount { + return a.err + } + return nil + } + return a.err +} + +func (a *contextCheckingStatAppender) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (stats.AppendResult, error) { + return a.result, a.Append(ctx, env) +} + +func (a *contextCheckingStatAppender) CacheStats() stats.CacheStats { + a.cacheStatsCalls++ + return a.cacheStats } type contextCheckingStatCommitter struct { - ctxErr error - count int + ctxErr error + count int + messageCount int + messages []kafka.Message + err error + failOnCount int } -func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, _ ...kafka.Message) error { +func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error { c.ctxErr = ctx.Err() c.count++ - return c.ctxErr + c.messageCount += len(messages) + c.messages = append(c.messages, messages...) + if c.ctxErr != nil { + return c.ctxErr + } + if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) { + return c.err + } + return nil } type discardStatLogger struct{} func (discardStatLogger) Error(string, ...any) {} func (discardStatLogger) Warn(string, ...any) {} + +var errTestStatAppend = errors.New("test stat append failed") + +func committedOffsets(messages []kafka.Message) []int64 { + offsets := make([]int64, len(messages)) + for index, message := range messages { + offsets[index] = message.Offset + } + return offsets +} diff --git a/go/vehicle-gateway/cmd/stats-backfill/main.go b/go/vehicle-gateway/cmd/stats-backfill/main.go index 1578b219..f7e7167d 100644 --- a/go/vehicle-gateway/cmd/stats-backfill/main.go +++ b/go/vehicle-gateway/cmd/stats-backfill/main.go @@ -22,20 +22,21 @@ import ( ) type config struct { - MySQLDSN string - TDengineDriver string - TDengineDSN string - TDengineDatabase string - DateFrom string - DateTo string - Protocols []envelope.Protocol - Method string - Limit int - DryRun bool - Reset bool - Debug bool - ProgressEvery int64 - Location *time.Location + MySQLDSN string + TDengineDriver string + TDengineDSN string + TDengineDatabase string + DateFrom string + DateTo string + Protocols []envelope.Protocol + Method string + Limit int + DryRun bool + Reset bool + Debug bool + EventTimeFullScan bool + ProgressEvery int64 + Location *time.Location } type rawFrameRow struct { @@ -62,6 +63,9 @@ type metricAgg struct { Phone string DeviceID string SourceEndpoint string + SourceCode string + PlatformName string + SourceKind string FirstEventTime time.Time LatestEventTime time.Time QualityStatus string @@ -74,6 +78,9 @@ type dailySourceLast struct { Phone string DeviceID string SourceEndpoint string + SourceCode string + PlatformName string + SourceKind string FirstTS time.Time TS time.Time FirstTotalKM float64 @@ -81,8 +88,19 @@ type dailySourceLast struct { RawSampleCount int64 } +type sourceHistoryID struct { + VIN string + SourceKey string +} + +var defaultBackfillEnvFiles = []string{ + "/opt/lingniu-go-native/env/base.env", + "/opt/lingniu-go-native/env/stat-writer.env", +} + func main() { - if err := loadEnvFiles(env("BACKFILL_ENV_FILES", os.Getenv("BACKFILL_ENV_FILE"))); err != nil { + envFiles := backfillEnvFiles(os.Getenv("BACKFILL_ENV_FILES"), os.Getenv("BACKFILL_ENV_FILE"), defaultBackfillEnvFiles) + if err := loadEnvFiles(envFiles); err != nil { fail("load env file", err) } cfg, err := loadConfig() @@ -121,18 +139,27 @@ func main() { slog.Info("reset stats rows", "deleted", deleted) } if cfg.Method == "last_diff" { - aggregates, err := buildLastDiffAggregates(ctx, td, cfg) + aggregates, err := buildLastDiffAggregates(ctx, mysqlDB, td, cfg) if err != nil { fail("build last-diff aggregates", err) } + fallbacks, err := addRealtimeLocationFallbackAggregates(ctx, mysqlDB, td, cfg, aggregates) + if err != nil { + fail("build realtime-location fallback aggregates", err) + } var written int64 + var normalized int if !cfg.DryRun { written, err = writeAggregates(ctx, mysqlDB, aggregates, 500) if err != nil { fail("write aggregates", err) } + normalized, err = normalizeHistoricalPlatformSources(ctx, mysqlDB, cfg) + if err != nil { + fail("normalize historical platform sources", err) + } } - slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "written", written) + slog.Info("stats backfill complete", "method", cfg.Method, "dry_run", cfg.DryRun, "aggregates", len(aggregates), "realtimeLocationFallbacks", fallbacks, "written", written, "platformSourcesNormalized", normalized) return } @@ -234,6 +261,7 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample) Phone: sample.Phone, DeviceID: sample.DeviceID, SourceEndpoint: sample.SourceEndpoint, + PlatformName: sample.PlatformName, FirstEventTime: sample.EventTime, LatestEventTime: sample.EventTime, QualityStatus: stats.QualityOK, @@ -255,6 +283,9 @@ func addSamples(aggregates map[string]*metricAgg, samples []stats.MetricSample) if strings.TrimSpace(sample.DeviceID) != "" { agg.DeviceID = sample.DeviceID } + if strings.TrimSpace(sample.PlatformName) != "" { + agg.PlatformName = sample.PlatformName + } } agg.Count++ } @@ -271,6 +302,9 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met Protocol: agg.Protocol, SourceIP: stats.NormalizeSourceIP(agg.SourceEndpoint), SourceEndpoint: agg.SourceEndpoint, + SourceCode: agg.SourceCode, + PlatformName: agg.PlatformName, + SourceKind: agg.SourceKind, } if strings.TrimSpace(identity.SourceIP) == "" { continue @@ -285,7 +319,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met if err := stats.UpsertDataSource(ctx, db, identity, agg.LatestEventTime); err != nil { return written, err } - dailyKM := agg.LatestKM - agg.FirstKM + dailyKM := stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM) candidate := stats.SourceMileageSample{ VIN: agg.VIN, StatDate: agg.Date, @@ -295,6 +329,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met SourceEndpoint: agg.SourceEndpoint, Phone: agg.Phone, DeviceID: agg.DeviceID, + PlatformName: agg.PlatformName, FirstTotalKM: agg.FirstKM, LatestTotalKM: agg.LatestKM, DailyKM: dailyKM, @@ -310,10 +345,7 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met if candidate.QualityReason == "" { candidate.QualityReason = "same_source_previous_day" } - if candidate.QualityStatus == stats.QualityOK && (candidate.DailyKM < 0 || candidate.DailyKM > maxTrustedDailyMileageKM) { - candidate.QualityStatus = stats.QualityInvalidDelta - candidate.QualityReason = "outside_daily_range" - } + stats.ApplyMileageQualityRules(&candidate) if err := stats.UpsertSourceMileage(ctx, db, candidate); err != nil { return written, err } @@ -325,6 +357,27 @@ func writeAggregates(ctx context.Context, db *sql.DB, aggregates map[string]*met return written, nil } +func normalizeHistoricalPlatformSources(ctx context.Context, db *sql.DB, cfg config) (int, error) { + dates, err := dateRange(cfg.DateFrom, cfg.DateTo) + if err != nil { + return 0, err + } + normalized := 0 + for _, protocol := range cfg.Protocols { + if protocol != envelope.ProtocolJT808 { + continue + } + for _, date := range dates { + count, err := stats.NormalizePlatformSourceMileageForDate(ctx, db, date, protocol) + if err != nil { + return normalized, err + } + normalized += count + } + } + return normalized, nil +} + func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, statDate string, protocol envelope.Protocol) error { if db == nil || strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" || strings.TrimSpace(string(protocol)) == "" { return nil @@ -337,47 +390,173 @@ func clearBackfillTargetMileage(ctx context.Context, db *sql.DB, vin string, sta return err } -func buildLastDiffAggregates(ctx context.Context, db *sql.DB, cfg config) (map[string]*metricAgg, error) { +func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config) (map[string]*metricAgg, error) { targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo) if err != nil { return nil, err } aggregates := map[string]*metricAgg{} for _, protocol := range cfg.Protocols { + latestHistory := map[sourceHistoryID]dailySourceLast{} + preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0]) + if err != nil { + return nil, err + } + rememberLatestSourceRows(latestHistory, preWindow) + slog.Info("pre-window baseline loaded", "protocol", protocol, "before_date", targetDates[0], "vehicles", len(preWindow)) for _, date := range targetDates { - current, err := queryDailyLastSourceRows(ctx, db, cfg, protocol, date) + current, err := queryDailyLastSourceRows(ctx, tdDB, cfg, protocol, date) if err != nil { return nil, err } - previous, err := queryPreviousLastSourceRows(ctx, db, cfg, protocol, date) - if err != nil { - return nil, err - } - slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "previousVehicles", len(previous)) + slog.Info("daily last loaded", "protocol", protocol, "date", date, "vehicles", len(current), "historicalSources", len(latestHistory)) for vin, currentSources := range current { - previousBySource := map[string]dailySourceLast{} - for _, previousRow := range previous[vin] { - previousBySource[previousRow.SourceKey] = previousRow - } for _, currentRow := range currentSources { key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey - previousRow, hasPrevious := previousBySource[currentRow.SourceKey] + previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow) + if err != nil { + return nil, err + } aggregates[key] = aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious) } } + // Current-day last values become eligible only for later target dates. + rememberLatestSourceRows(latestHistory, current) } } return aggregates, nil } +func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) { + if aggregates == nil { + return 0, fmt.Errorf("aggregates map is nil") + } + targetDates, err := dateRange(cfg.DateFrom, cfg.DateTo) + if err != nil { + return 0, err + } + var added int + for _, protocol := range cfg.Protocols { + if protocol != envelope.ProtocolYutongMQTT { + continue + } + latestHistory := map[sourceHistoryID]dailySourceLast{} + preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0]) + if err != nil { + return added, err + } + rememberLatestSourceRows(latestHistory, preWindow) + aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol) + for _, date := range targetDates { + current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date) + if err != nil { + return added, err + } + for vin, currentSources := range current { + for _, currentRow := range currentSources { + key := vin + "|" + date + "|" + string(protocol) + "|" + currentRow.SourceKey + if _, exists := aggregates[key]; exists { + continue + } + previousRow, hasPrevious, err := resolvePreviousSourceRow(ctx, mysqlDB, date, protocol, latestHistory, currentRow) + if err != nil { + return added, err + } + agg := aggregateFromDailySource(date, protocol, currentRow, previousRow, hasPrevious) + if hasPrevious { + agg.QualityReason = "realtime_location_fallback_historical_baseline" + } else { + agg.QualityReason = "realtime_location_fallback_current_day_first_baseline" + } + aggregates[key] = agg + added++ + } + } + rememberLatestSourceRows(latestHistory, aggregateRowsByDate[date]) + rememberLatestSourceRows(latestHistory, current) + if len(current) > 0 { + slog.Info("realtime location fallback loaded", "protocol", protocol, "date", date, "vehicles", len(current), "added", added) + } + } + } + return added, nil +} + +func indexAggregateSourceRowsByDate(aggregates map[string]*metricAgg, protocol envelope.Protocol) map[string]map[string][]dailySourceLast { + indexed := map[string]map[string][]dailySourceLast{} + for _, agg := range aggregates { + if agg == nil || agg.Protocol != protocol || strings.TrimSpace(agg.Date) == "" { + continue + } + rows := indexed[agg.Date] + if rows == nil { + rows = map[string][]dailySourceLast{} + indexed[agg.Date] = rows + } + rows[agg.VIN] = append(rows[agg.VIN], dailySourceLast{ + VIN: agg.VIN, + SourceKey: agg.SourceKey, + Phone: agg.Phone, + DeviceID: agg.DeviceID, + SourceEndpoint: agg.SourceEndpoint, + SourceCode: agg.SourceCode, + PlatformName: agg.PlatformName, + SourceKind: agg.SourceKind, + FirstTS: agg.LatestEventTime, + TS: agg.LatestEventTime, + FirstTotalKM: agg.LatestKM, + TotalKM: agg.LatestKM, + RawSampleCount: agg.Count, + }) + } + return indexed +} + +func rememberLatestSourceRows(history map[sourceHistoryID]dailySourceLast, rows map[string][]dailySourceLast) { + for vin, sourceRows := range rows { + for _, row := range sourceRows { + rowVIN := strings.TrimSpace(row.VIN) + if rowVIN == "" { + rowVIN = strings.TrimSpace(vin) + row.VIN = rowVIN + } + id := sourceHistoryID{VIN: rowVIN, SourceKey: strings.TrimSpace(row.SourceKey)} + if id.VIN == "" || id.SourceKey == "" || row.TS.IsZero() { + continue + } + if existing, ok := history[id]; !ok || row.TS.After(existing.TS) { + history[id] = row + } + } + } +} + +func latestHistoricalSourceRow(history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool) { + id := sourceHistoryID{ + VIN: strings.TrimSpace(current.VIN), + SourceKey: strings.TrimSpace(current.SourceKey), + } + previous, ok := history[id] + return previous, ok && previous.TS.Before(current.TS) +} + +func resolvePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, history map[sourceHistoryID]dailySourceLast, current dailySourceLast) (dailySourceLast, bool, error) { + if previous, ok := latestHistoricalSourceRow(history, current); ok { + return previous, true, nil + } + return queryDurablePreviousSourceRow(ctx, db, date, protocol, current) +} + func aggregateFromDailySource(date string, protocol envelope.Protocol, current dailySourceLast, previous dailySourceLast, hasPrevious bool) *metricAgg { firstKM := current.FirstTotalKM firstEventTime := current.FirstTS - qualityReason := "current_day_first_sample" + qualityStatus := stats.QualityOK + qualityReason := stats.QualityReasonCurrentDayFirst if hasPrevious { firstKM = previous.TotalKM firstEventTime = previous.TS - qualityReason = "historical_source_baseline" + qualityStatus = stats.QualityOK + qualityReason = stats.QualityReasonHistorical } if firstEventTime.IsZero() { firstEventTime = current.TS @@ -386,7 +565,7 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d if count <= 0 { count = 1 } - return &metricAgg{ + agg := &metricAgg{ VIN: current.VIN, Date: date, Protocol: protocol, @@ -397,11 +576,43 @@ func aggregateFromDailySource(date string, protocol envelope.Protocol, current d Phone: current.Phone, DeviceID: current.DeviceID, SourceEndpoint: current.SourceEndpoint, + SourceCode: current.SourceCode, + PlatformName: current.PlatformName, + SourceKind: current.SourceKind, FirstEventTime: firstEventTime, LatestEventTime: current.TS, - QualityStatus: stats.QualityOK, + QualityStatus: qualityStatus, QualityReason: qualityReason, } + candidate := stats.SourceMileageSample{ + FirstTotalKM: agg.FirstKM, + LatestTotalKM: agg.LatestKM, + DailyKM: stats.DailyMileageFromDayBoundary(agg.FirstKM, agg.LatestKM), + FirstEventTime: agg.FirstEventTime, + LatestEventTime: agg.LatestEventTime, + QualityStatus: agg.QualityStatus, + QualityReason: agg.QualityReason, + } + stats.ApplyMileageQualityRules(&candidate) + agg.QualityStatus = candidate.QualityStatus + agg.QualityReason = candidate.QualityReason + return agg +} + +func queryDurablePreviousSourceRow(ctx context.Context, db *sql.DB, date string, protocol envelope.Protocol, current dailySourceLast) (dailySourceLast, bool, error) { + if db == nil { + return dailySourceLast{}, false, nil + } + totalKM, eventTime, found, err := stats.LookupLatestSourceBaselineBefore(ctx, db, current.VIN, date, protocol, current.SourceKey) + if err != nil || !found { + return dailySourceLast{}, found, err + } + previous := current + previous.FirstTS = eventTime + previous.TS = eventTime + previous.FirstTotalKM = totalKM + previous.TotalKM = totalKM + return previous, true, nil } type trustedChoice struct { @@ -409,8 +620,6 @@ type trustedChoice struct { previous dailySourceLast } -const maxTrustedDailyMileageKM = 1000 - func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) (trustedChoice, bool) { previousBySource := map[string]dailySourceLast{} for _, row := range previous { @@ -423,8 +632,8 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) if !ok { continue } - delta := currentRow.TotalKM - previousRow.TotalKM - if delta < 0 || delta > maxTrustedDailyMileageKM { + delta, ok, _ := stats.NormalizeDailyMileageDeltaForWindow(stats.DailyMileageFromDayBoundary(previousRow.TotalKM, currentRow.TotalKM), previousRow.TS, currentRow.TS) + if !ok { continue } if chosen.current.SourceKey == "" || delta < chosenDelta { @@ -436,19 +645,18 @@ func chooseTrustedSource(current []dailySourceLast, previous []dailySourceLast) } func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) { - where := []string{ - fmt.Sprintf("ts >= '%s 00:00:00'", quote(date)), - fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(date))), + where := backfillTimePredicates(cfg, date, nextDate(date)) + where = append(where, "parse_status = 'OK'", "vin IS NOT NULL", "vin <> ''", fmt.Sprintf("protocol = '%s'", quote(string(protocol))), realtimeMileageFramePredicate(), - } + ) if predicate := mileageBearingFramePredicate(protocol); predicate != "" { where = append(where, predicate) } - sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(ts), FIRST(parsed_json), LAST(ts), LAST(parsed_json), COUNT(*) + sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*) FROM %s.raw_frames WHERE %s GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")) @@ -456,24 +664,103 @@ GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), s } func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) { - where := []string{ - fmt.Sprintf("ts < '%s 00:00:00'", quote(date)), + // The baseline is the nearest earlier sample, not necessarily yesterday's. + // LAST aggregates the complete pre-window history once per source so empty + // calendar days do not make a backfill fall back to the current day's first + // sample. + where := backfillBeforePredicates(date) + where = append(where, "parse_status = 'OK'", "vin IS NOT NULL", "vin <> ''", fmt.Sprintf("protocol = '%s'", quote(string(protocol))), realtimeMileageFramePredicate(), - } + ) if predicate := mileageBearingFramePredicate(protocol); predicate != "" { where = append(where, predicate) } - sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(ts), LAST(parsed_json), COUNT(*) + sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*) FROM %s.raw_frames WHERE %s GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")) return querySourceRows(ctx, db, cfg, protocol, sqlText, false) } +func backfillBeforePredicates(eventDateExclusive string) []string { + return []string{ + fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)), + } +} + +func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) { + if db == nil { + return nil, nil + } + loc := cfg.Location + if loc == nil { + loc = time.FixedZone("Asia/Shanghai", 8*3600) + } + sqlText := `SELECT l.vin, COALESCE(NULLIF(s.peer, ''), ''), l.total_mileage_event_time, l.total_mileage_km +FROM vehicle_realtime_location l +LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = l.protocol AND s.vin = l.vin +WHERE l.protocol = ? + AND l.vin IS NOT NULL AND l.vin <> '' + AND l.total_mileage_km IS NOT NULL AND l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? + AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)` + rows, err := db.QueryContext(ctx, sqlText, string(protocol), date, date) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string][]dailySourceLast{} + for rows.Next() { + var vin string + var peer sql.NullString + var eventTime time.Time + var totalKM float64 + if err := rows.Scan(&vin, &peer, &eventTime, &totalKM); err != nil { + return nil, err + } + vin = strings.TrimSpace(vin) + if vin == "" || totalKM <= 0 { + continue + } + sourceEndpoint := strings.TrimSpace(peer.String) + if sourceEndpoint == "" && protocol == envelope.ProtocolYutongMQTT { + sourceEndpoint = "mqtt://yutong/realtime-location" + } + deviceID := "" + if protocol == envelope.ProtocolYutongMQTT { + deviceID = vin + } + sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint) + sourceKey := normalizedSourceKeyForSource(string(protocol), "", deviceID, sourceEndpoint, sourceKind, sourceCode) + if sourceKey == "" { + continue + } + row := dailySourceLast{ + VIN: vin, + SourceKey: sourceKey, + DeviceID: deviceID, + SourceEndpoint: sourceEndpoint, + SourceCode: sourceCode, + PlatformName: platformName, + SourceKind: sourceKind, + FirstTS: eventTime.In(loc), + TS: eventTime.In(loc), + FirstTotalKM: totalKM, + TotalKM: totalKM, + RawSampleCount: 1, + } + out[vin] = append(out[vin], row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, sqlText string, includeFirst bool) (map[string][]dailySourceLast, error) { rows, err := db.QueryContext(ctx, sqlText) if err != nil { @@ -512,7 +799,8 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel firstTotalKM = parsedFirst } } - sourceKey := normalizedSourceKey(string(protocol), phone, deviceID, sourceEndpoint) + sourceCode, platformName, sourceKind := knownPlatformSourceMetadata(protocol, sourceEndpoint) + sourceKey := normalizedSourceKeyForSource(string(protocol), phone, deviceID, sourceEndpoint, sourceKind, sourceCode) if sourceKey == "" { continue } @@ -522,8 +810,11 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel Phone: strings.TrimSpace(phone), DeviceID: strings.TrimSpace(deviceID), SourceEndpoint: strings.TrimSpace(sourceEndpoint), + SourceCode: sourceCode, + PlatformName: platformName, + SourceKind: sourceKind, FirstTS: firstTS.In(cfg.Location), - TS: ts, + TS: ts.In(cfg.Location), FirstTotalKM: firstTotalKM, TotalKM: latestTotalKM, RawSampleCount: rawSampleCount, @@ -562,8 +853,19 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string } func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string { + return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "") +} + +func normalizedSourceKeyForSource(protocol string, phone string, deviceID string, endpoint string, sourceKind string, sourceCode string) string { sourceIP := stats.NormalizeSourceIP(endpoint) - return stats.SourceKey(envelope.Protocol(protocol), phone, deviceID, sourceIP) + return stats.SourceKeyForSource(envelope.Protocol(protocol), phone, deviceID, sourceIP, sourceKind, sourceCode) +} + +func knownPlatformSourceMetadata(protocol envelope.Protocol, endpoint string) (sourceCode string, platformName string, sourceKind string) { + if protocol == envelope.ProtocolYutongMQTT && stats.NormalizeSourceIP(endpoint) == "mqtt" { + return "yutong", "宇通", "PLATFORM" + } + return "", "", "" } func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[string]any { @@ -574,7 +876,7 @@ func fieldsForStats(protocol envelope.Protocol, vin string, text string) map[str } parsed := map[string]any{} if err := json.Unmarshal([]byte(text), &parsed); err == nil && len(parsed) > 0 { - if flattened, _, ok := realtime.ParsedFieldsForEnvelope(envelope.FrameEnvelope{ + if flattened, _, ok := realtime.ComputeParsedFieldsForEnvelope(envelope.FrameEnvelope{ Protocol: protocol, VIN: vin, Parsed: parsed, @@ -604,7 +906,12 @@ func mileageKeys(protocol envelope.Protocol) []string { case envelope.ProtocolJT808: return []string{"jt808.location.total_mileage_km"} case envelope.ProtocolYutongMQTT: - return []string{"yutong_mqtt.data.total_mileage", "yutong_mqtt.root.data.total_mileage"} + return []string{ + "yutong_mqtt.data.total_mileage_km", + "yutong_mqtt.root.data.total_mileage_km", + "yutong_mqtt.data.total_mileage", + "yutong_mqtt.root.data.total_mileage", + } default: return nil } @@ -624,38 +931,67 @@ func loadConfig() (config, error) { if err != nil { loc = time.FixedZone("Asia/Shanghai", 8*3600) } - dateTo := env("BACKFILL_DATE_TO", time.Now().In(loc).Format("2006-01-02")) - dateFrom := env("BACKFILL_DATE_FROM", dateTo) + dateFrom, dateTo := resolveBackfillDateRange(time.Now(), loc) protocols, err := parseProtocols(env("BACKFILL_PROTOCOLS", "GB32960,JT808,YUTONG_MQTT")) if err != nil { return config{}, err } return config{ - MySQLDSN: env("MYSQL_DSN", ""), - TDengineDriver: env("TDENGINE_DRIVER", "taosWS"), - TDengineDSN: env("TDENGINE_DSN", ""), - TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase), - DateFrom: dateFrom, - DateTo: dateTo, - Protocols: protocols, - Method: env("BACKFILL_METHOD", "last_diff"), - Limit: envInt("BACKFILL_LIMIT", 0), - DryRun: envBool("BACKFILL_DRY_RUN", true), - Reset: envBool("BACKFILL_RESET", false), - Debug: envBool("BACKFILL_DEBUG", false), - ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)), - Location: loc, + MySQLDSN: env("MYSQL_DSN", ""), + TDengineDriver: env("TDENGINE_DRIVER", "taosWS"), + TDengineDSN: env("TDENGINE_DSN", ""), + TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase), + DateFrom: dateFrom, + DateTo: dateTo, + Protocols: protocols, + Method: env("BACKFILL_METHOD", "last_diff"), + Limit: envInt("BACKFILL_LIMIT", 0), + DryRun: envBool("BACKFILL_DRY_RUN", true), + Reset: envBool("BACKFILL_RESET", false), + Debug: envBool("BACKFILL_DEBUG", false), + EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false), + ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)), + Location: loc, }, nil } +func resolveBackfillDateRange(now time.Time, loc *time.Location) (string, string) { + if loc == nil { + loc = time.FixedZone("Asia/Shanghai", 8*3600) + } + explicitFrom := strings.TrimSpace(os.Getenv("BACKFILL_DATE_FROM")) + explicitTo := strings.TrimSpace(os.Getenv("BACKFILL_DATE_TO")) + if explicitTo != "" || explicitFrom != "" { + dateTo := explicitTo + if dateTo == "" { + dateTo = now.In(loc).Format("2006-01-02") + } + dateFrom := explicitFrom + if dateFrom == "" { + dateFrom = dateTo + } + return dateFrom, dateTo + } + daysBack := envInt("BACKFILL_DAYS_BACK", 0) + if daysBack < 0 { + daysBack = 0 + } + windowDays := envInt("BACKFILL_WINDOW_DAYS", 1) + if windowDays < 1 { + windowDays = 1 + } + dateToTime := now.In(loc).AddDate(0, 0, -daysBack) + dateFromTime := dateToTime.AddDate(0, 0, -(windowDays - 1)) + return dateFromTime.Format("2006-01-02"), dateToTime.Format("2006-01-02") +} + func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, error) { - where := []string{ - fmt.Sprintf("ts >= '%s 00:00:00'", quote(cfg.DateFrom)), - fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(cfg.DateTo))), + where := backfillTimePredicates(cfg, cfg.DateFrom, nextDate(cfg.DateTo)) + where = append(where, "parse_status = 'OK'", "vin IS NOT NULL", "vin <> ''", - } + ) if len(cfg.Protocols) > 0 { quoted := make([]string, 0, len(cfg.Protocols)) for _, protocol := range cfg.Protocols { @@ -667,7 +1003,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json FROM %s.raw_frames WHERE %s -ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")) + ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")) if cfg.Limit > 0 { sqlText += fmt.Sprintf(" LIMIT %d", cfg.Limit) } @@ -675,6 +1011,23 @@ ORDER BY ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND ")) return db.QueryContext(ctx, sqlText) } +func backfillTimePredicates(cfg config, eventDateFrom string, eventDateToExclusive string) []string { + where := make([]string, 0, 4) + if !cfg.EventTimeFullScan { + // ts is the TDengine primary timestamp and may be adjusted slightly to keep + // rows unique. Use it only as a broad index-friendly window; event_time is + // the protocol business boundary used for the final natural-day result. + where = append(where, + fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))), + fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateToExclusive))), + ) + } + return append(where, + fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)), + fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateToExclusive)), + ) +} + func realtimeMileageFramePredicate() string { return `( (protocol = 'GB32960' AND message_id IN (2,3)) @@ -684,17 +1037,39 @@ func realtimeMileageFramePredicate() string { } func mileageBearingFramePredicate(protocol envelope.Protocol) string { - keys := mileageKeys(protocol) - if len(keys) == 0 { + tokens := mileageSearchTokens(protocol) + if len(tokens) == 0 { return "" } - conditions := make([]string, 0, len(keys)) - for _, key := range keys { - conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(key))) + conditions := make([]string, 0, len(tokens)) + for _, token := range tokens { + conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token))) } return "(" + strings.Join(conditions, " OR ") + ")" } +func mileageSearchTokens(protocol envelope.Protocol) []string { + tokens := append([]string{}, mileageKeys(protocol)...) + switch protocol { + case envelope.ProtocolYutongMQTT: + tokens = append(tokens, "TOTAL_MILEAGE", "totalMileage", "total_mileage_km") + } + seen := make(map[string]struct{}, len(tokens)) + out := make([]string, 0, len(tokens)) + for _, token := range tokens { + token = strings.TrimSpace(token) + if token == "" { + continue + } + if _, ok := seen[token]; ok { + continue + } + seen[token] = struct{}{} + out = append(out, token) + } + return out +} + func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) { var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string var messageID int64 @@ -804,6 +1179,26 @@ func loadEnvFiles(paths string) error { return nil } +func backfillEnvFiles(explicitFiles string, explicitFile string, defaults []string) string { + if value := strings.TrimSpace(explicitFiles); value != "" { + return value + } + if value := strings.TrimSpace(explicitFile); value != "" { + return value + } + var existing []string + for _, path := range defaults { + path = strings.TrimSpace(path) + if path == "" { + continue + } + if _, err := os.Stat(path); err == nil { + existing = append(existing, path) + } + } + return strings.Join(existing, ",") +} + func loadEnvFile(path string) error { path = strings.TrimSpace(path) if path == "" { diff --git a/go/vehicle-gateway/cmd/stats-backfill/main_test.go b/go/vehicle-gateway/cmd/stats-backfill/main_test.go index eb64566e..7c45fa79 100644 --- a/go/vehicle-gateway/cmd/stats-backfill/main_test.go +++ b/go/vehicle-gateway/cmd/stats-backfill/main_test.go @@ -2,6 +2,9 @@ package main import ( "context" + "os" + "path/filepath" + "strings" "testing" "time" @@ -47,6 +50,58 @@ func TestChooseTrustedSourceKeepsContinuingSourceAndRejectsNewJump(t *testing.T) } } +func TestChooseTrustedSourceAcceptsSmallNegativeMileageJitter(t *testing.T) { + previous := []dailySourceLast{{ + VIN: "LB9A32A28R0LS1574", + SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53330"), + Phone: "64115156034", + SourceEndpoint: "115.159.85.149:53330", + TotalKM: 15355.4, + }} + current := []dailySourceLast{{ + VIN: "LB9A32A28R0LS1574", + SourceKey: normalizedSourceKey("JT808", "64115156034", "", "115.159.85.149:53338"), + Phone: "64115156034", + SourceEndpoint: "115.159.85.149:53338", + TotalKM: 15355.3, + }} + + chosen, ok := chooseTrustedSource(current, previous) + if !ok { + t.Fatal("chooseTrustedSource() should accept tiny negative mileage jitter") + } + if chosen.current.SourceKey != current[0].SourceKey { + t.Fatalf("chosen source = %q", chosen.current.SourceKey) + } +} + +func TestChooseTrustedSourceAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + sourceKey := normalizedSourceKey("GB32960", "", "", "8.134.95.166:37720") + previous := []dailySourceLast{{ + VIN: "LNXNEGRR1SR319498", + SourceKey: sourceKey, + SourceEndpoint: "8.134.95.166:37720", + TS: time.Date(2026, 7, 4, 11, 7, 58, 0, loc), + TotalKM: 8832.1, + }} + current := []dailySourceLast{{ + VIN: "LNXNEGRR1SR319498", + SourceKey: sourceKey, + SourceEndpoint: "8.134.95.166:37720", + TS: time.Date(2026, 7, 12, 2, 52, 47, 0, loc), + TotalKM: 16665.6, + }} + + chosen, ok := chooseTrustedSource(current, previous) + if !ok { + t.Fatal("chooseTrustedSource() should accept delta within the historical baseline window") + } + if delta := chosen.current.TotalKM - chosen.previous.TotalKM; delta < 7833.4 || delta > 7833.6 { + t.Fatalf("delta = %v, want historical gap delta", delta) + } +} + func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) { sourceA := dailySourceLast{ VIN: "LA9GG64L7PBAF4001", @@ -96,7 +151,7 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) { if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS { t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime) } - if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "historical_source_baseline" { + if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical { t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason) } if agg.Count != 15 { @@ -104,7 +159,40 @@ func TestAggregateFromDailySourceUsesOlderHistoricalBaseline(t *testing.T) { } } -func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing.T) { +func TestAggregateFromDailySourceRejectsHistoricalBaselineJump(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + current := dailySourceLast{ + VIN: "LNXNEGRR6SR319464", + SourceKey: normalizedSourceKey("GB32960", "", "", "8.134.95.166:49206"), + SourceEndpoint: "8.134.95.166:49206", + FirstTS: time.Date(2026, 7, 12, 8, 5, 42, 0, loc), + TS: time.Date(2026, 7, 12, 10, 44, 56, 0, loc), + FirstTotalKM: 28004.2, + TotalKM: 40009.7, + RawSampleCount: 1938, + } + previous := dailySourceLast{ + VIN: current.VIN, + SourceKey: current.SourceKey, + SourceEndpoint: current.SourceEndpoint, + TS: time.Date(2026, 7, 3, 19, 0, 39, 0, loc), + TotalKM: 10009.7, + } + + agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true) + + if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM { + t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM) + } + if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) { + t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime) + } + if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason) + } +} + +func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testing.T) { current := dailySourceLast{ VIN: "LMRKH9AC2R1004087", SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"), @@ -125,11 +213,264 @@ func TestAggregateFromDailySourceUsesCurrentFirstSampleWithoutHistory(t *testing if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS { t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime) } - if agg.QualityStatus != stats.QualityOK || agg.QualityReason != "current_day_first_sample" { + if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst { t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason) } } +func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.T) { + tdDB, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer tdDB.Close() + mock.MatchExpectationsInOrder(true) + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + vin := "LMRKH9AC2R1004087" + endpoint := "mqtt://yutong/ytforward/shln/3" + dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc) + dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc) + currentRows := func() *sqlmock.Rows { + return sqlmock.NewRows([]string{ + "vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", + }) + } + previousRows := func() *sqlmock.Rows { + return sqlmock.NewRows([]string{ + "vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", + }) + } + + mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(previousRows()) + mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4))) + mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(currentRows()) + mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6))) + + aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{ + TDengineDatabase: "lingniu_vehicle_ts", + DateFrom: "2026-07-09", + DateTo: "2026-07-11", + Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT}, + Location: loc, + }) + if err != nil { + t.Fatalf("buildLastDiffAggregates() error = %v", err) + } + sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong") + agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey] + if agg == nil { + t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates)) + } + if agg.FirstKM != 100 || agg.LatestKM != 120 { + t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM) + } + if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical { + t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func aggregateKeys(aggregates map[string]*metricAgg) []string { + keys := make([]string, 0, len(aggregates)) + for key := range aggregates { + keys = append(keys, key) + } + return keys +} + +func TestQueryRealtimeLocationLastRowsBuildsYutongSourceFromPeer(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + eventTime := time.Date(2026, 7, 12, 5, 54, 50, 0, loc) + mock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*LEFT JOIN vehicle_realtime_snapshot s"). + WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}). + AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", eventTime, 11578.0)) + + rows, err := queryRealtimeLocationLastRows(context.Background(), db, config{Location: loc}, envelope.ProtocolYutongMQTT, "2026-07-12") + if err != nil { + t.Fatalf("queryRealtimeLocationLastRows() error = %v", err) + } + sourceRows := rows["LMRKH9AC0R1004086"] + if len(sourceRows) != 1 { + t.Fatalf("rows = %d, want 1", len(sourceRows)) + } + row := sourceRows[0] + wantSourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") + if row.SourceKey != wantSourceKey { + t.Fatalf("source key = %q", row.SourceKey) + } + if row.SourceCode != "yutong" || row.PlatformName != "宇通" || row.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", row.SourceCode, row.PlatformName, row.SourceKind) + } + if row.TotalKM != 11578 || row.DeviceID != "LMRKH9AC0R1004086" || row.RawSampleCount != 1 { + t.Fatalf("unexpected realtime fallback row: %#v", row) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.T) { + mysqlDB, mysqlMock, err := sqlmock.New() + if err != nil { + t.Fatalf("mysql sqlmock.New() error = %v", err) + } + defer mysqlDB.Close() + tdDB, tdMock, err := sqlmock.New() + if err != nil { + t.Fatalf("td sqlmock.New() error = %v", err) + } + defer tdDB.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + currentTS := time.Date(2026, 7, 12, 5, 54, 50, 0, loc) + previousTS := time.Date(2026, 7, 11, 23, 58, 0, 0, loc) + mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?"). + WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}). + AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0)) + tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE"). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", + }).AddRow( + "LMRKH9AC0R1004086", + "", + "LMRKH9AC0R1004086", + "mqtt://yutong/ytforward/shln/4", + previousTS, + `{"data":{"TOTAL_MILEAGE":11500000}}`, + int64(12), + )) + + aggregates := map[string]*metricAgg{} + added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{ + TDengineDatabase: "lingniu_vehicle_ts", + DateFrom: "2026-07-12", + DateTo: "2026-07-12", + Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT}, + Location: loc, + }, aggregates) + if err != nil { + t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err) + } + if added != 1 || len(aggregates) != 1 { + t.Fatalf("added=%d aggregates=%d", added, len(aggregates)) + } + for _, agg := range aggregates { + if agg.FirstKM != 11500 || agg.LatestKM != 11578 { + t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM) + } + if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") { + t.Fatalf("source key = %q", agg.SourceKey) + } + if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", agg.SourceCode, agg.PlatformName, agg.SourceKind) + } + if agg.QualityReason != "realtime_location_fallback_historical_baseline" { + t.Fatalf("quality reason = %q", agg.QualityReason) + } + } + if err := mysqlMock.ExpectationsWereMet(); err != nil { + t.Fatalf("mysql sql expectations: %v", err) + } + if err := tdMock.ExpectationsWereMet(); err != nil { + t.Fatalf("td sql expectations: %v", err) + } +} + +func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *testing.T) { + mysqlDB, mysqlMock, err := sqlmock.New() + if err != nil { + t.Fatalf("mysql sqlmock.New() error = %v", err) + } + defer mysqlDB.Close() + tdDB, tdMock, err := sqlmock.New() + if err != nil { + t.Fatalf("td sqlmock.New() error = %v", err) + } + defer tdDB.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + vin := "LMRKH9AC2R1004087" + endpoint := "mqtt://yutong/ytforward/shln/3" + sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", vin, "mqtt", "PLATFORM", "yutong") + dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc) + dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc) + + tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", + })) + for _, date := range []string{"2026-07-09", "2026-07-10"} { + mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?"). + WithArgs("YUTONG_MQTT", date, date). + WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"})) + } + mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?"). + WithArgs("YUTONG_MQTT", "2026-07-11", "2026-07-11"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}). + AddRow(vin, endpoint, dayThreeTS, 120.0)) + + aggregates := map[string]*metricAgg{ + vin + "|2026-07-09|YUTONG_MQTT|" + sourceKey: { + VIN: vin, + Date: "2026-07-09", + Protocol: envelope.ProtocolYutongMQTT, + LatestKM: 100, + Count: 4, + SourceKey: sourceKey, + DeviceID: vin, + SourceEndpoint: endpoint, + SourceCode: "yutong", + PlatformName: "宇通", + SourceKind: "PLATFORM", + LatestEventTime: dayOneTS, + }, + } + added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{ + TDengineDatabase: "lingniu_vehicle_ts", + DateFrom: "2026-07-09", + DateTo: "2026-07-11", + Protocols: []envelope.Protocol{envelope.ProtocolYutongMQTT}, + Location: loc, + }, aggregates) + if err != nil { + t.Fatalf("addRealtimeLocationFallbackAggregates() error = %v", err) + } + if added != 1 || len(aggregates) != 2 { + t.Fatalf("added=%d aggregates=%d, want 1 and 2", added, len(aggregates)) + } + agg := aggregates[vin+"|2026-07-11|YUTONG_MQTT|"+sourceKey] + if agg == nil { + t.Fatal("missing realtime-location fallback aggregate") + } + if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) { + t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM) + } + if agg.QualityReason != "realtime_location_fallback_historical_baseline" { + t.Fatalf("quality reason = %q", agg.QualityReason) + } + if err := mysqlMock.ExpectationsWereMet(); err != nil { + t.Fatalf("mysql sql expectations: %v", err) + } + if err := tdMock.ExpectationsWereMet(); err != nil { + t.Fatalf("td sql expectations: %v", err) + } +} + func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -140,9 +481,9 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) firstTS := time.Date(2026, 7, 8, 8, 0, 0, 0, loc) lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc) - mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'"). + mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'"). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "phone", "device_id", "source_endpoint", "FIRST(ts)", "FIRST(parsed_json)", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)", + "vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", }).AddRow( "LMRKH9AC6R1004108", "", @@ -183,9 +524,9 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc) - mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'"). + mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'"). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "phone", "device_id", "source_endpoint", "LAST(ts)", "LAST(parsed_json)", "COUNT(*)", + "vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", }).AddRow( "LMRKH9AC6R1004108", "", @@ -215,6 +556,89 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) { } } +func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) { + where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ") + if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") { + t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where) + } + if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") { + t.Fatalf("pre-window predicate must not stop at the previous day: %s", where) + } +} + +func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC) + mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'"). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)", + }).AddRow( + "LMRKH9AC7R1004098", + "", + "LMRKH9AC7R1004098", + "mqtt://yutong/ytforward/shln/4", + utcInstant, + `{"yutong_mqtt.data.total_mileage":"41249000"}`, + int64(1), + )) + + rows, err := queryPreviousLastSourceRows(context.Background(), db, config{ + TDengineDatabase: "lingniu_vehicle_ts", + Location: loc, + }, envelope.ProtocolYutongMQTT, "2026-07-13") + if err != nil { + t.Fatalf("queryPreviousLastSourceRows() error = %v", err) + } + sourceRows := rows["LMRKH9AC7R1004098"] + if len(sourceRows) != 1 { + t.Fatalf("rows = %d, want 1", len(sourceRows)) + } + want := time.Date(2026, 7, 12, 23, 59, 59, 0, loc) + if !sourceRows[0].TS.Equal(want) || sourceRows[0].TS.Location().String() != loc.String() { + t.Fatalf("previous event time = %s (%s), want %s (%s)", sourceRows[0].TS, sourceRows[0].TS.Location(), want, want.Location()) + } +} + +func TestFieldsForStatsExtractsRawYutongTotalMileage(t *testing.T) { + fields := fieldsForStats(envelope.ProtocolYutongMQTT, "LMRKH9AC6R1004108", `{ + "data": { + "TOTAL_MILEAGE": 65423000, + "METER_SPEED": 12.3 + }, + "root": { + "device": "LMRKH9AC6R1004108" + } + }`) + + if got := fields["yutong_mqtt.data.total_mileage"]; got == nil { + t.Fatalf("fields missing raw yutong total mileage: %#v", fields) + } + + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC6R1004108", + EventTimeMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + ReceivedAtMS: time.Date(2026, 7, 8, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: fields, + } + samples, err := stats.SamplesFromEnvelope(env, time.FixedZone("Asia/Shanghai", 8*3600)) + if err != nil { + t.Fatalf("SamplesFromEnvelope() error = %v", err) + } + if len(samples) != 1 { + t.Fatalf("samples = %d, want 1", len(samples)) + } + if samples[0].TotalMileageKM != 65423 { + t.Fatalf("total mileage km = %v, want 65423", samples[0].TotalMileageKM) + } +} + func TestClearBackfillTargetMileageClearsExactKey(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -269,7 +693,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T) WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808"). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`INSERT INTO vehicle_data_source`). - WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). WithArgs( @@ -292,18 +716,16 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T) "outside_daily_range", ). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808"). - WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectBegin() mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(2500)). WillReturnResult(sqlmock.NewResult(1, 0)) mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). WithArgs( "LA9GG64L7PBAF4001", "2026-07-08", "JT808", - int64(1000), + int64(2500), "LA9GG64L7PBAF4001", "2026-07-08", "JT808", @@ -319,6 +741,7 @@ func TestWriteAggregatesClearsTargetRowsBeforeStaleCandidateUpsert(t *testing.T) "JT808", ). WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() written, err := writeAggregates(context.Background(), db, aggregates, 500) if err != nil { @@ -419,6 +842,8 @@ func TestAddSamplesUsesCurrentDayFirstSampleBaseline(t *testing.T) { func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) { t.Setenv("BACKFILL_METHOD", "") + t.Setenv("BACKFILL_DAYS_BACK", "") + t.Setenv("BACKFILL_WINDOW_DAYS", "") t.Setenv("BACKFILL_DATE_FROM", "2026-07-08") t.Setenv("BACKFILL_DATE_TO", "2026-07-08") t.Setenv("BACKFILL_PROTOCOLS", "JT808") @@ -431,4 +856,101 @@ func TestLoadConfigDefaultsBackfillMethodToLastDiff(t *testing.T) { if cfg.Method != "last_diff" { t.Fatalf("method = %q, want last_diff", cfg.Method) } + if cfg.EventTimeFullScan { + t.Fatal("event-time full scan should be opt-in") + } +} + +func TestBackfillTimePredicatesUsePrimaryTimeForCoarseScanAndEventTimeForBusinessDay(t *testing.T) { + where := strings.Join(backfillTimePredicates(config{}, "2026-07-13", "2026-07-14"), " AND ") + for _, want := range []string{ + "ts >= '2026-07-12 00:00:00'", + "ts < '2026-07-15 00:00:00'", + "event_time >= '2026-07-13 00:00:00'", + "event_time < '2026-07-14 00:00:00'", + } { + if !strings.Contains(where, want) { + t.Fatalf("time predicates missing %q: %s", want, where) + } + } +} + +func TestBackfillTimePredicatesAllowExplicitDeepEventTimeScan(t *testing.T) { + where := strings.Join(backfillTimePredicates(config{EventTimeFullScan: true}, "2026-07-13", "2026-07-14"), " AND ") + if strings.Contains(where, "ts >=") || strings.Contains(where, "ts <") { + t.Fatalf("deep event-time scan must not apply storage-time bounds: %s", where) + } + for _, want := range []string{ + "event_time >= '2026-07-13 00:00:00'", + "event_time < '2026-07-14 00:00:00'", + } { + if !strings.Contains(where, want) { + t.Fatalf("deep event-time scan missing %q: %s", want, where) + } + } +} + +func TestResolveBackfillDateRangeDefaultsToToday(t *testing.T) { + t.Setenv("BACKFILL_DATE_FROM", "") + t.Setenv("BACKFILL_DATE_TO", "") + t.Setenv("BACKFILL_DAYS_BACK", "") + t.Setenv("BACKFILL_WINDOW_DAYS", "") + loc := time.FixedZone("Asia/Shanghai", 8*3600) + + dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc) + if dateFrom != "2026-07-12" || dateTo != "2026-07-12" { + t.Fatalf("date range = %s -> %s", dateFrom, dateTo) + } +} + +func TestResolveBackfillDateRangeUsesRelativeWindow(t *testing.T) { + t.Setenv("BACKFILL_DATE_FROM", "") + t.Setenv("BACKFILL_DATE_TO", "") + t.Setenv("BACKFILL_DAYS_BACK", "1") + t.Setenv("BACKFILL_WINDOW_DAYS", "3") + loc := time.FixedZone("Asia/Shanghai", 8*3600) + + dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc) + if dateFrom != "2026-07-09" || dateTo != "2026-07-11" { + t.Fatalf("date range = %s -> %s", dateFrom, dateTo) + } +} + +func TestResolveBackfillDateRangePrefersExplicitDates(t *testing.T) { + t.Setenv("BACKFILL_DATE_FROM", "2026-07-01") + t.Setenv("BACKFILL_DATE_TO", "2026-07-03") + t.Setenv("BACKFILL_DAYS_BACK", "1") + t.Setenv("BACKFILL_WINDOW_DAYS", "3") + loc := time.FixedZone("Asia/Shanghai", 8*3600) + + dateFrom, dateTo := resolveBackfillDateRange(time.Date(2026, 7, 12, 10, 0, 0, 0, loc), loc) + if dateFrom != "2026-07-01" || dateTo != "2026-07-03" { + t.Fatalf("date range = %s -> %s", dateFrom, dateTo) + } +} + +func TestBackfillEnvFilesPrefersExplicitList(t *testing.T) { + got := backfillEnvFiles("/tmp/a.env,/tmp/b.env", "/tmp/legacy.env", []string{"/tmp/default.env"}) + if got != "/tmp/a.env,/tmp/b.env" { + t.Fatalf("env files = %q", got) + } +} + +func TestBackfillEnvFilesUsesExistingDefaults(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "missing.env") + base := filepath.Join(dir, "base.env") + stat := filepath.Join(dir, "stat-writer.env") + if err := os.WriteFile(base, []byte("A=1\n"), 0600); err != nil { + t.Fatalf("write base env: %v", err) + } + if err := os.WriteFile(stat, []byte("B=2\n"), 0600); err != nil { + t.Fatalf("write stat env: %v", err) + } + + got := backfillEnvFiles("", "", []string{missing, base, stat}) + want := base + "," + stat + if got != want { + t.Fatalf("env files = %q, want %q", got, want) + } } diff --git a/go/vehicle-gateway/go.mod b/go/vehicle-gateway/go.mod index ae277b33..9480347c 100644 --- a/go/vehicle-gateway/go.mod +++ b/go/vehicle-gateway/go.mod @@ -11,7 +11,8 @@ require ( github.com/redis/go-redis/v9 v9.17.2 github.com/segmentio/kafka-go v0.4.49 github.com/taosdata/driver-go/v3 v3.8.1 - golang.org/x/text v0.35.0 + github.com/xuri/excelize/v2 v2.11.0 + golang.org/x/text v0.38.0 ) require ( @@ -27,9 +28,14 @@ require ( github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/richardlehane/mscfb v1.0.7 // indirect + github.com/richardlehane/msoleps v1.0.6 // indirect + github.com/tiendc/go-deepcopy v1.7.2 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect ) diff --git a/go/vehicle-gateway/go.sum b/go/vehicle-gateway/go.sum index baf711b1..12d1929f 100644 --- a/go/vehicle-gateway/go.sum +++ b/go/vehicle-gateway/go.sum @@ -46,6 +46,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/richardlehane/mscfb v1.0.7 h1:oeoiM0WE79vHwE8RpIYYvIAc8ajTH2mb6UZm55/+EB0= +github.com/richardlehane/mscfb v1.0.7/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= +github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= +github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk= github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -55,28 +59,39 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/taosdata/driver-go/v3 v3.8.1 h1:kkd4ABsGiU+oXDbsw/sic985LKAvnpF9Gb/TEunTnLE= github.com/taosdata/driver-go/v3 v3.8.1/go.mod h1:S6OGOinfR0xxxaMGsvBi9cLkYxEIW1p6qqr8QJATTlg= +github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44= +github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG88= +github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= +golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/vehicle-gateway/internal/authentication/authentication.go b/go/vehicle-gateway/internal/authentication/authentication.go new file mode 100644 index 00000000..b511b617 --- /dev/null +++ b/go/vehicle-gateway/internal/authentication/authentication.go @@ -0,0 +1,215 @@ +package authentication + +import ( + "crypto/subtle" + "fmt" + "strings" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +type Mode string + +const ( + ModeDisabled Mode = "disabled" + ModeObserve Mode = "observe" + ModeEnforce Mode = "enforce" +) + +const ( + StatusAccepted = "accepted" + StatusRejected = "rejected" + StatusUnknownAccount = "unknown_account" + StatusMissingCredential = "missing_credential" + StatusUnconfigured = "unconfigured" +) + +type Result struct { + Applicable bool + Allowed bool + Mode Mode + Status string + Source string +} + +type Authenticator interface { + Authenticate(envelope.FrameEnvelope) Result +} + +func ParseMode(value string, fallback Mode) (Mode, error) { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + value = string(fallback) + } + switch Mode(value) { + case ModeDisabled, ModeObserve, ModeEnforce: + return Mode(value), nil + default: + return "", fmt.Errorf("unsupported authentication mode %q", value) + } +} + +type GB32960PlatformAuthenticator struct { + mode Mode + credentials map[string][]string +} + +func NewGB32960PlatformAuthenticator(mode Mode, credentials map[string][]string) *GB32960PlatformAuthenticator { + normalized := make(map[string][]string, len(credentials)) + for username, passwords := range credentials { + username = strings.TrimSpace(username) + if username == "" { + continue + } + for _, password := range passwords { + if password == "" { + continue + } + normalized[username] = append(normalized[username], password) + } + } + return &GB32960PlatformAuthenticator{mode: mode, credentials: normalized} +} + +func (a *GB32960PlatformAuthenticator) Authenticate(env envelope.FrameEnvelope) Result { + if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x05" || a == nil || a.mode == ModeDisabled { + return Result{} + } + login := nestedMap(env.Parsed, "platform_login") + username := strings.TrimSpace(textValue(login, "username")) + password := textValue(login, "password") + status := StatusRejected + switch { + case len(a.credentials) == 0: + status = StatusUnconfigured + case username == "" || password == "": + status = StatusMissingCredential + default: + expected, ok := a.credentials[username] + if !ok || len(expected) == 0 { + status = StatusUnknownAccount + } else if anyConstantTimeEqual(expected, password) { + status = StatusAccepted + } + } + source := "none" + if status == StatusAccepted { + source = "configured" + } + return resultForMode(a.mode, status, source) +} + +type JT808Authenticator struct { + mode Mode + authCode string + deviceTokens JT808DeviceTokenProvider +} + +type JT808DeviceTokenProvider interface { + JT808AuthToken(phone string) (string, bool) +} + +func NewJT808Authenticator(mode Mode, authCode string, deviceTokens JT808DeviceTokenProvider) *JT808Authenticator { + return &JT808Authenticator{mode: mode, authCode: authCode, deviceTokens: deviceTokens} +} + +func (a *JT808Authenticator) Authenticate(env envelope.FrameEnvelope) Result { + if env.Protocol != envelope.ProtocolJT808 || env.MessageID != "0x0102" || a == nil || a.mode == ModeDisabled { + return Result{} + } + token := textValue(nestedMap(env.Parsed, "authentication"), "token") + status := StatusRejected + source := "none" + switch { + case token == "": + status = StatusMissingCredential + default: + if a.authCode != "" && constantTimeEqual(a.authCode, token) { + status = StatusAccepted + source = "configured" + break + } + deviceToken, knownDevice := "", false + if a.deviceTokens != nil { + deviceToken, knownDevice = a.deviceTokens.JT808AuthToken(env.Phone) + } + if knownDevice && deviceToken != "" && constantTimeEqual(deviceToken, token) { + status = StatusAccepted + source = "device" + } else if a.authCode == "" && !knownDevice { + status = StatusUnconfigured + } + } + return resultForMode(a.mode, status, source) +} + +func Apply(env *envelope.FrameEnvelope, result Result) { + if env == nil || !result.Applicable { + return + } + env.AuthenticationMode = string(result.Mode) + env.AuthenticationStatus = result.Status + env.AuthenticationEnforced = result.Mode == ModeEnforce +} + +// RedactParsedCredentials removes convenience copies of secrets before parsed +// fields are flattened and published. The original protocol frame remains in +// raw_hex for restricted forensic access. +func RedactParsedCredentials(env *envelope.FrameEnvelope) { + if env == nil || env.Protocol != envelope.ProtocolGB32960 { + return + } + login := nestedMap(env.Parsed, "platform_login") + if login == nil { + return + } + if password := textValue(login, "password"); password != "" { + login["password_present"] = true + } + delete(login, "password") +} + +func resultForMode(mode Mode, status string, source ...string) Result { + allowed := status == StatusAccepted || mode != ModeEnforce + credentialSource := "none" + if len(source) > 0 && strings.TrimSpace(source[0]) != "" { + credentialSource = strings.TrimSpace(source[0]) + } + return Result{Applicable: true, Allowed: allowed, Mode: mode, Status: status, Source: credentialSource} +} + +func constantTimeEqual(expected string, actual string) bool { + if len(expected) != len(actual) { + return false + } + return subtle.ConstantTimeCompare([]byte(expected), []byte(actual)) == 1 +} + +func anyConstantTimeEqual(expected []string, actual string) bool { + matched := 0 + for _, candidate := range expected { + if len(candidate) == len(actual) { + matched |= subtle.ConstantTimeCompare([]byte(candidate), []byte(actual)) + } + } + return matched == 1 +} + +func nestedMap(parent map[string]any, key string) map[string]any { + if parent == nil { + return nil + } + value, _ := parent[key].(map[string]any) + return value +} + +func textValue(values map[string]any, key string) string { + if values == nil { + return "" + } + value, ok := values[key] + if !ok || value == nil { + return "" + } + return strings.TrimSpace(fmt.Sprint(value)) +} diff --git a/go/vehicle-gateway/internal/authentication/authentication_test.go b/go/vehicle-gateway/internal/authentication/authentication_test.go new file mode 100644 index 00000000..e2198bac --- /dev/null +++ b/go/vehicle-gateway/internal/authentication/authentication_test.go @@ -0,0 +1,111 @@ +package authentication + +import ( + "strings" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestGB32960PlatformAuthenticatorModes(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x05", + Parsed: map[string]any{ + "platform_login": map[string]any{"username": "platform-a", "password": "secret-a"}, + }, + } + + accepted := NewGB32960PlatformAuthenticator(ModeEnforce, map[string][]string{"platform-a": {"secret-a"}}).Authenticate(env) + if !accepted.Applicable || !accepted.Allowed || accepted.Status != StatusAccepted { + t.Fatalf("accepted result = %#v", accepted) + } + + rejected := NewGB32960PlatformAuthenticator(ModeEnforce, map[string][]string{"platform-a": {"other"}}).Authenticate(env) + if !rejected.Applicable || rejected.Allowed || rejected.Status != StatusRejected { + t.Fatalf("enforced rejected result = %#v", rejected) + } + + observed := NewGB32960PlatformAuthenticator(ModeObserve, map[string][]string{"platform-a": {"other"}}).Authenticate(env) + if !observed.Allowed || observed.Status != StatusRejected { + t.Fatalf("observed rejected result = %#v", observed) + } +} + +func TestGB32960PlatformAuthenticatorReportsUnknownAccount(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x05", + Parsed: map[string]any{ + "platform_login": map[string]any{"username": "unknown", "password": "secret"}, + }, + } + result := NewGB32960PlatformAuthenticator(ModeEnforce, map[string][]string{"known": {"secret"}}).Authenticate(env) + if result.Allowed || result.Status != StatusUnknownAccount { + t.Fatalf("result = %#v", result) + } +} + +func TestJT808AuthenticatorValidatesAuthenticationFrame(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0102", + Parsed: map[string]any{ + "authentication": map[string]any{"token": "issued-code"}, + }, + } + accepted := NewJT808Authenticator(ModeEnforce, "issued-code", nil).Authenticate(env) + if !accepted.Allowed || accepted.Status != StatusAccepted { + t.Fatalf("accepted result = %#v", accepted) + } + rejected := NewJT808Authenticator(ModeEnforce, "different-code", nil).Authenticate(env) + if rejected.Allowed || rejected.Status != StatusRejected { + t.Fatalf("rejected result = %#v", rejected) + } +} + +func TestJT808AuthenticatorAcceptsDeviceSnapshotToken(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0102", + Phone: "013307795425", + Parsed: map[string]any{ + "authentication": map[string]any{"token": "device-code"}, + }, + } + provider := staticJT808DeviceTokens{"13307795425": "device-code"} + result := NewJT808Authenticator(ModeEnforce, "configured-code", provider).Authenticate(env) + if !result.Allowed || result.Status != StatusAccepted || result.Source != "device" { + t.Fatalf("device token result = %#v", result) + } +} + +type staticJT808DeviceTokens map[string]string + +func (p staticJT808DeviceTokens) JT808AuthToken(phone string) (string, bool) { + phone = strings.TrimLeft(phone, "0") + token, ok := p[phone] + return token, ok +} + +func TestRedactParsedCredentialsRemovesGB32960Password(t *testing.T) { + login := map[string]any{"username": "platform-a", "password": "secret-a"} + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + Parsed: map[string]any{"platform_login": login}, + } + + RedactParsedCredentials(&env) + if _, exists := login["password"]; exists { + t.Fatalf("password remained in parsed login: %#v", login) + } + if login["password_present"] != true { + t.Fatalf("password presence marker missing: %#v", login) + } +} + +func TestParseModeRejectsUnknownValue(t *testing.T) { + if _, err := ParseMode("strict-ish", ModeObserve); err == nil { + t.Fatal("expected unsupported mode error") + } +} diff --git a/go/vehicle-gateway/internal/capacity/check.go b/go/vehicle-gateway/internal/capacity/check.go index 991f884a..a9b29f0a 100644 --- a/go/vehicle-gateway/internal/capacity/check.go +++ b/go/vehicle-gateway/internal/capacity/check.go @@ -3,10 +3,14 @@ package capacity import ( "bufio" "fmt" + "math" "sort" "strconv" "strings" "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) type Status string @@ -17,24 +21,47 @@ const ( ) type Report struct { - Status Status `json:"status"` - CheckedAt time.Time `json:"checked_at"` - Totals Totals `json:"totals"` - Services []ServiceCheck `json:"services"` - Findings []string `json:"findings"` + Status Status `json:"status"` + CheckedAt time.Time `json:"checked_at"` + Totals Totals `json:"totals"` + Services []ServiceCheck `json:"services"` + GatewayFields *GatewayFieldsDiagnostics `json:"gateway_fields,omitempty"` + ProtocolFlow *ProtocolFlowDiagnostics `json:"protocol_flow,omitempty"` + IdentityResolution *IdentityResolutionDiagnostics `json:"identity_resolution,omitempty"` + PipelineLatency *PipelineLatencyDiagnostics `json:"pipeline_latency,omitempty"` + DailyMileageDiagnostics *DailyMileageDiagnostics `json:"daily_mileage_diagnostics,omitempty"` + DailyMileageSourceQuality *DailyMileageSourceQualityDiagnostics `json:"daily_mileage_source_quality,omitempty"` + DailyMileageSourceSelection *DailyMileageSourceSelectionDiagnostics `json:"daily_mileage_source_selection,omitempty"` + DataSourceDiagnostics *DataSourceDiagnostics `json:"data_source_diagnostics,omitempty"` + DataSourceKindDiagnostics *DataSourceDiagnostics `json:"data_source_kind_diagnostics,omitempty"` + JT808IdentityGaps *JT808IdentityGapDiagnostics `json:"jt808_identity_gaps,omitempty"` + Findings []string `json:"findings"` } type Totals struct { - ActiveConnections int64 `json:"active_connections"` - KafkaLag float64 `json:"kafka_lag"` - BridgeConsumerPending float64 `json:"bridge_consumer_pending"` - BridgeAckPending float64 `json:"bridge_ack_pending"` - BridgeBatchPendingMessages float64 `json:"bridge_batch_pending_messages"` - FastWriterConsumerPending float64 `json:"fast_writer_consumer_pending"` - FastWriterAckPending float64 `json:"fast_writer_ack_pending"` - FastWriterBatchPending float64 `json:"fast_writer_batch_pending_messages"` - HistoryBatchPending float64 `json:"history_batch_pending_messages"` - HistoryRowsPending float64 `json:"history_rows_pending"` + ActiveConnections int64 `json:"active_connections"` + KafkaLag float64 `json:"kafka_lag"` + BridgeConsumerPending float64 `json:"bridge_consumer_pending"` + BridgeAckPending float64 `json:"bridge_ack_pending"` + BridgeBatchPendingMessages float64 `json:"bridge_batch_pending_messages"` + FieldsProjectorKafkaLag float64 `json:"fields_projector_kafka_lag"` + FieldsProjectorBatchPending float64 `json:"fields_projector_batch_pending_messages"` + FieldsProjectorRetryPending float64 `json:"fields_projector_retry_pending_messages"` + FastWriterConsumerPending float64 `json:"fast_writer_consumer_pending"` + FastWriterAckPending float64 `json:"fast_writer_ack_pending"` + FastWriterBatchPending float64 `json:"fast_writer_batch_pending_messages"` + HistoryBatchPending float64 `json:"history_batch_pending_messages"` + HistoryRowsPending float64 `json:"history_rows_pending"` + HistoryRetryPending float64 `json:"history_retry_pending_messages"` + RealtimeBatchPending float64 `json:"realtime_batch_pending_messages"` + RealtimeAsyncQueuePending float64 `json:"realtime_async_queue_pending"` + RealtimeRetryPending float64 `json:"realtime_retry_pending_messages"` + StatRetryPending float64 `json:"stat_retry_pending_messages"` + IdentityBatchPending float64 `json:"identity_batch_pending_messages"` + IdentityFactsPending float64 `json:"identity_batch_pending_facts"` + IdentityRetryPending float64 `json:"identity_retry_pending_messages"` + JT808RegistrationWriteQueuePending float64 `json:"jt808_registration_write_queue_pending"` + JT808RegistrationWriteErrors float64 `json:"jt808_registration_write_errors"` } type ServiceCheck struct { @@ -42,47 +69,914 @@ type ServiceCheck struct { Metrics map[string]float64 `json:"metrics"` } +type GatewayFieldsDiagnostics struct { + Protocols []GatewayProtocolFields `json:"protocols,omitempty"` +} + +type GatewayProtocolFields struct { + Protocol string `json:"protocol"` + RawOK float64 `json:"raw_ok"` + RealtimeCandidates float64 `json:"realtime_candidates"` + FieldsOK float64 `json:"fields_ok"` + FieldsPublished float64 `json:"fields_published"` + FieldsDelegated float64 `json:"fields_delegated"` + FieldsSkippedNonRealtime float64 `json:"fields_skipped_non_realtime"` + FieldsSkippedMissing float64 `json:"fields_skipped_missing"` + FieldsPublishError float64 `json:"fields_publish_error"` + MissingRatio float64 `json:"missing_ratio"` + LatestFieldCount *float64 `json:"latest_field_count,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +type ProtocolFlowDiagnostics struct { + Protocols []ProtocolFlowProtocol `json:"protocols,omitempty"` +} + +type ProtocolFlowProtocol struct { + Protocol string `json:"protocol"` + RawOK float64 `json:"raw_ok"` + RealtimeCandidates float64 `json:"realtime_candidates"` + FieldsPublished float64 `json:"fields_published"` + LatestFieldCount *float64 `json:"latest_field_count,omitempty"` + IdentityTotal int64 `json:"identity_total"` + IdentitySkipped int64 `json:"identity_skipped"` + IdentityResolved int64 `json:"identity_resolved"` + IdentityNonResolved int64 `json:"identity_non_resolved"` + IdentityNonResolvedRatio float64 `json:"identity_non_resolved_ratio"` + StatWritesOK float64 `json:"stat_writes_ok"` + StatMileageCandidateWrites float64 `json:"stat_mileage_candidate_writes"` + StatSamplesFound float64 `json:"stat_samples_found"` + StatSamplesWritten float64 `json:"stat_samples_written"` + StatSamplesActionableSkipped float64 `json:"stat_samples_actionable_skipped"` + StatSkippedMissingFields float64 `json:"stat_skipped_missing_fields"` + StatSkippedMissingVIN float64 `json:"stat_skipped_missing_vin"` + StatSkippedMissingMileage float64 `json:"stat_skipped_missing_mileage"` + StatSkippedNonMileageFrame float64 `json:"stat_skipped_non_mileage_frame"` + StatSkippedNonPositiveMileage float64 `json:"stat_skipped_non_positive_mileage"` + StatSkippedMissingTime float64 `json:"stat_skipped_missing_time"` + StatEventTimeFutureAdjusted float64 `json:"stat_event_time_future_adjusted"` + StatSkippedMissingSource float64 `json:"stat_skipped_missing_source"` + StatSkippedSameMileage float64 `json:"stat_skipped_same_mileage"` + StatSourceWritten float64 `json:"stat_source_written"` + StatSourceThrottled float64 `json:"stat_source_throttled"` + StatSourceMissingEndpoint float64 `json:"stat_source_missing_endpoint"` + StatSourceSkippedUnmanaged float64 `json:"stat_source_skipped_unmanaged"` + StatProjectionAttempted float64 `json:"stat_projection_attempted"` + StatProjectionWritten float64 `json:"stat_projection_written"` + StatProjectionThrottled float64 `json:"stat_projection_throttled"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +type IdentityResolutionDiagnostics struct { + Total int64 `json:"total"` + SkippedCount int64 `json:"skipped_count"` + ResolvedCount int64 `json:"resolved_count"` + NonResolvedCount int64 `json:"non_resolved_count"` + UnresolvedCount int64 `json:"unresolved_count"` + TimeoutCount int64 `json:"timeout_count"` + ErrorCount int64 `json:"error_count"` + OtherNonResolvedCount int64 `json:"other_non_resolved_count"` + NonResolvedRatio float64 `json:"non_resolved_ratio"` + StaleCacheCount int64 `json:"stale_cache_count"` + StaleCacheRatio float64 `json:"stale_cache_ratio"` + CacheEntries map[string]int64 `json:"cache_entries,omitempty"` + Issues []IdentityIssueCount `json:"issues,omitempty"` + Protocols []IdentityProtocolResolution `json:"protocols,omitempty"` +} + +type IdentityIssueCount struct { + Protocol string `json:"protocol"` + Status string `json:"status"` + MessageID string `json:"message_id"` + Reason string `json:"reason"` + Count int64 `json:"count"` +} + +type IdentityProtocolResolution struct { + Protocol string `json:"protocol"` + Total int64 `json:"total"` + SkippedCount int64 `json:"skipped_count"` + ResolvedCount int64 `json:"resolved_count"` + NonResolvedCount int64 `json:"non_resolved_count"` + UnresolvedCount int64 `json:"unresolved_count"` + TimeoutCount int64 `json:"timeout_count"` + ErrorCount int64 `json:"error_count"` + OtherNonResolvedCount int64 `json:"other_non_resolved_count"` + NonResolvedRatio float64 `json:"non_resolved_ratio"` + StaleCacheCount int64 `json:"stale_cache_count"` + StaleCacheRatio float64 `json:"stale_cache_ratio"` + CacheStatusCounts map[string]int64 `json:"cache_status_counts,omitempty"` +} + +type PipelineLatencyDiagnostics struct { + TotalSamples int64 `json:"total_samples"` + MaxP99MS float64 `json:"max_p99_ms"` + Items []PipelineLatencyItem `json:"items,omitempty"` +} + +type PipelineLatencyItem struct { + Stage string `json:"stage"` + Service string `json:"service"` + Topic string `json:"topic,omitempty"` + Subject string `json:"subject,omitempty"` + P99MS float64 `json:"p99_ms"` + Samples int64 `json:"samples"` + ThresholdMS float64 `json:"threshold_ms,omitempty"` + Status string `json:"status"` +} + +type DailyMileageDiagnostics struct { + URL string `json:"url,omitempty"` + VehicleTotal int64 `json:"vehicle_total"` + ActionableIssueTotal int64 `json:"actionable_issue_total"` + PipelineIssueTotal int64 `json:"pipeline_issue_total"` + SourceDataIssueTotal int64 `json:"source_data_issue_total"` + Items []DailyMileageDiagnosisReasonItem `json:"items,omitempty"` + Samples []DailyMileageDiagnosisSample `json:"samples,omitempty"` + FieldStatus *DailyMileageFieldStatusDiagnostics `json:"field_status,omitempty"` +} + +type DailyMileageDiagnosisReasonItem struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + Count int64 `json:"count"` + Severity string `json:"severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` +} + +type DailyMileageFieldStatusDiagnostics struct { + URL string `json:"url,omitempty"` + SummaryTotal int64 `json:"summary_total"` + VehicleTotal int64 `json:"vehicle_total"` + ActionableIssueTotal int64 `json:"actionable_issue_total"` + PipelineIssueTotal int64 `json:"pipeline_issue_total"` + SourceDataIssueTotal int64 `json:"source_data_issue_total"` + Items []DailyMileageFieldStatusSummaryItem `json:"items,omitempty"` +} + +type DailyMileageFieldStatusSummaryItem struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status"` + Count int64 `json:"count"` + Severity string `json:"severity,omitempty"` + FieldStatusSeverity string `json:"field_status_severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` + FieldStatusAction string `json:"field_status_action,omitempty"` +} + +type DailyMileageDiagnosisSample struct { + VIN string `json:"vin"` + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Plate string `json:"plate,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + Peer string `json:"peer,omitempty"` + SnapshotEventTime string `json:"snapshot_event_time,omitempty"` + LocationEventTime string `json:"location_event_time,omitempty"` + SnapshotUpdatedAt string `json:"snapshot_updated_at,omitempty"` + LocationUpdatedAt string `json:"location_updated_at,omitempty"` + RealtimeTotalMileageKM *float64 `json:"realtime_total_mileage_km,omitempty"` + RealtimeTotalMileageAt string `json:"realtime_total_mileage_event_time,omitempty"` + DailyMileageKM *float64 `json:"daily_mileage_km,omitempty"` + DailyLatestTotalMileageKM *float64 `json:"daily_latest_total_mileage_km,omitempty"` + SourceSampleCount int64 `json:"source_sample_count"` + OKSourceCount int64 `json:"ok_source_count"` + SelectableSourceCount int64 `json:"selectable_source_count"` + RealtimeFieldCount int64 `json:"realtime_field_count,omitempty"` + RealtimeSampleFields []string `json:"realtime_sample_fields,omitempty"` + RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status,omitempty"` + RealtimeMileageEvidence string `json:"realtime_mileage_evidence,omitempty"` + LatestStatEventTime string `json:"latest_stat_event_time,omitempty"` + QualityStatuses []string `json:"quality_statuses,omitempty"` + RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` + SourceSampleQueryPath string `json:"source_sample_query_path,omitempty"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + Severity string `json:"severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` +} + +type DailyMileageSourceQualityDiagnostics struct { + URL string `json:"url,omitempty"` + SummaryTotal int64 `json:"summary_total"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SampleCount int64 `json:"sample_count"` + OKSourceCount int64 `json:"ok_source_count"` + InvalidDeltaSourceCount int64 `json:"invalid_delta_source_count"` + NoBaselineSourceCount int64 `json:"no_baseline_source_count"` + FallbackAfterInvalidBaselineSourceCount int64 `json:"fallback_after_invalid_baseline_source_count"` + RealtimeLocationFallbackSourceCount int64 `json:"realtime_location_fallback_source_count"` + Items []DailyMileageSourceQualityItem `json:"items,omitempty"` + ItemsTruncated bool `json:"items_truncated,omitempty"` +} + +type DailyMileageSourceQualityItem struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + QualityStatus string `json:"quality_status"` + QualityReason string `json:"quality_reason,omitempty"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SelectedCount int64 `json:"selected_count"` + SampleCount int64 `json:"sample_count"` + DailyMileageKM float64 `json:"daily_mileage_km"` +} + +type DailyMileageSourceSelectionDiagnostics struct { + URL string `json:"url,omitempty"` + SummaryTotal int64 `json:"summary_total"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SampleCount int64 `json:"sample_count"` + SelectedSourceCount int64 `json:"selected_source_count"` + NotSelectedSourceCount int64 `json:"not_selected_source_count"` + NotSelectedMileageKM float64 `json:"not_selected_mileage_km"` + ExcludedSourceCount int64 `json:"excluded_source_count"` + UnmanagedSourceCount int64 `json:"unmanaged_source_count"` + UnknownKindSourceCount int64 `json:"unknown_kind_source_count"` + Protocols []DailyMileageSourceSelectionProtocol `json:"protocols,omitempty"` + Items []DailyMileageSourceSelectionItem `json:"items,omitempty"` + Samples []DailyMileageSourceSelectionSample `json:"samples,omitempty"` + ItemsTruncated bool `json:"items_truncated,omitempty"` +} + +type DailyMileageSourceSelectionProtocol struct { + Protocol string `json:"protocol"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SampleCount int64 `json:"sample_count"` + SelectedSourceCount int64 `json:"selected_source_count"` + NotSelectedSourceCount int64 `json:"not_selected_source_count"` + NotSelectedMileageKM float64 `json:"not_selected_mileage_km"` + ExcludedSourceCount int64 `json:"excluded_source_count"` + UnmanagedSourceCount int64 `json:"unmanaged_source_count"` + UnknownKindSourceCount int64 `json:"unknown_kind_source_count"` +} + +type DailyMileageSourceSelectionItem struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + SelectionStatus string `json:"selection_status"` + SelectionReason string `json:"selection_reason,omitempty"` + SelectionAction string `json:"selection_action,omitempty"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SelectedCount int64 `json:"selected_count"` + SampleCount int64 `json:"sample_count"` + DailyMileageKM float64 `json:"daily_mileage_km"` +} + +type DailyMileageSourceSelectionSample struct { + VIN string `json:"vin"` + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + SourceKey string `json:"source_key"` + SourceIP string `json:"source_ip,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + Phone string `json:"phone,omitempty"` + DeviceID string `json:"device_id,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceID *int64 `json:"source_id,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + SourceEnabled *bool `json:"source_enabled,omitempty"` + TrustPriority *int `json:"trust_priority,omitempty"` + LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` + DailyMileageKM float64 `json:"daily_mileage_km"` + SampleCount int64 `json:"sample_count"` + QualityStatus string `json:"quality_status,omitempty"` + QualityReason string `json:"quality_reason,omitempty"` + SelectionStatus string `json:"selection_status,omitempty"` + SelectionReason string `json:"selection_reason,omitempty"` + SelectionAction string `json:"selection_action,omitempty"` + SelectedSource *DailyMileageSourceSelectionSample `json:"selected_source,omitempty"` + SelectedSourceQueryPath string `json:"selected_source_query_path,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +type DataSourceDiagnostics struct { + URL string `json:"url,omitempty"` + MissingSourceCodeTotal int64 `json:"missing_source_code_total"` + RecentMissingSourceCodeTotal int64 `json:"recent_missing_source_code_total"` + UnknownSourceKindTotal int64 `json:"unknown_source_kind_total,omitempty"` + RecentUnknownSourceKindTotal int64 `json:"recent_unknown_source_kind_total,omitempty"` + PlatformKindCandidateTotal int64 `json:"platform_kind_candidate_total,omitempty"` + RecentPlatformKindCandidateTotal int64 `json:"recent_platform_kind_candidate_total,omitempty"` + RecentAgeThresholdSeconds int64 `json:"recent_age_threshold_seconds,omitempty"` + ActionableCandidateTotal int64 `json:"actionable_candidate_total"` + MappingIssueTotal int64 `json:"mapping_issue_total,omitempty"` + RecentMappingIssueTotal int64 `json:"recent_mapping_issue_total,omitempty"` + ReasonCounts map[string]int64 `json:"reason_counts,omitempty"` + MappingIssueReasonCounts map[string]int64 `json:"mapping_issue_reason_counts,omitempty"` + ReasonCountsTruncated bool `json:"reason_counts_truncated,omitempty"` + MappingIssueReasonCountsTruncated bool `json:"mapping_issue_reason_counts_truncated,omitempty"` + Samples []DataSourceDiagnosticSample `json:"samples,omitempty"` + MappingIssueSamples []DataSourceDiagnosticSample `json:"mapping_issue_samples,omitempty"` +} + +type DataSourceDiagnosticSample struct { + ID int64 `json:"id"` + Protocol string `json:"protocol"` + SourceIP string `json:"source_ip"` + LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind"` + FirstSeenAt string `json:"first_seen_at,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + ActiveSpanSeconds int64 `json:"active_span_seconds"` + LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` + RegistrationRows int64 `json:"registration_rows"` + PhoneCount int64 `json:"phone_count"` + UnknownVINRows int64 `json:"unknown_vin_rows"` + IdentifierMatchedPhones int64 `json:"identifier_matched_phones"` + UnmappedPhoneCount int64 `json:"unmapped_phone_count"` + IdentifierMatchRatio float64 `json:"identifier_match_ratio"` + ConfiguredSourceCodeMatchedPhones int64 `json:"configured_source_code_matched_phones"` + ConfiguredSourceCodePlatformName string `json:"configured_source_code_platform_name,omitempty"` + ConfiguredSourceCodeConflict bool `json:"configured_source_code_conflict"` + SourcePlatformNameMismatch bool `json:"source_platform_name_mismatch"` + MatchedSourceCodeCount int64 `json:"matched_source_code_count"` + CandidateSourceCode string `json:"candidate_source_code,omitempty"` + CandidatePlatformName string `json:"candidate_platform_name,omitempty"` + MatchedSourceCodes []string `json:"matched_source_codes,omitempty"` + MatchedPlatformNames []string `json:"matched_platform_names,omitempty"` + SamplePhones []string `json:"sample_phones,omitempty"` + Reason string `json:"reason"` + RecommendedOperatorAction string `json:"recommended_operator_action"` + SuggestedSourceKind string `json:"suggested_source_kind"` + SuggestionConfidence string `json:"suggestion_confidence"` + SuggestionReason string `json:"suggestion_reason"` +} + +type JT808IdentityGapDiagnostics struct { + URL string `json:"url,omitempty"` + Total int64 `json:"total"` + RecentSeconds int64 `json:"recent_seconds"` + Samples []JT808IdentityGapSample `json:"samples,omitempty"` +} + +type JT808IdentityGapSample struct { + Phone string `json:"phone"` + DeviceID string `json:"device_id,omitempty"` + Plate string `json:"plate,omitempty"` + VIN string `json:"vin,omitempty"` + SourceIP string `json:"source_ip,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + SourceCode string `json:"source_code,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + FirstRegisteredAt string `json:"first_registered_at,omitempty"` + LatestRegisteredAt string `json:"latest_registered_at,omitempty"` + LatestAuthenticatedAt string `json:"latest_authenticated_at,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` + Reason string `json:"reason"` + RecommendedOperatorAction string `json:"recommended_operator_action"` + RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` + DataSourceQueryPath string `json:"data_source_query_path,omitempty"` +} + +const ( + DailyMileageSeverityOK = "ok" + DailyMileageSeverityPipeline = "pipeline" + DailyMileageSeveritySourceData = "source_data" +) + +func AnnotateDailyMileageDiagnostics(items []DailyMileageDiagnosisReasonItem) (annotated []DailyMileageDiagnosisReasonItem, pipelineIssues int64, sourceDataIssues int64) { + annotated = make([]DailyMileageDiagnosisReasonItem, 0, len(items)) + for _, item := range items { + severity := strings.ToLower(strings.TrimSpace(item.Severity)) + if severity != DailyMileageSeverityOK && severity != DailyMileageSeverityPipeline && severity != DailyMileageSeveritySourceData { + severity = DailyMileageIssueSeverity(item) + } + item.Severity = severity + if strings.TrimSpace(item.RecommendedOperatorAction) == "" { + item.RecommendedOperatorAction = DailyMileageRecommendedOperatorAction(item.Diagnosis, item.Reason) + } + switch item.Severity { + case DailyMileageSeverityPipeline: + pipelineIssues += item.Count + case DailyMileageSeveritySourceData: + sourceDataIssues += item.Count + } + annotated = append(annotated, item) + } + return annotated, pipelineIssues, sourceDataIssues +} + +func DailyMileageIssueSeverity(item DailyMileageDiagnosisReasonItem) string { + diagnosis := strings.ToUpper(strings.TrimSpace(item.Diagnosis)) + reason := strings.ToLower(strings.TrimSpace(item.Reason)) + switch diagnosis { + case "", "OK": + return DailyMileageSeverityOK + case "MISSING_DAILY": + if reason == "source_samples_all_invalid" { + return DailyMileageSeveritySourceData + } + return DailyMileageSeverityPipeline + case "NO_SOURCE_SAMPLE": + return DailyMileageSeverityPipeline + case "NO_TOTAL_MILEAGE": + switch reason { + case "realtime_total_mileage_time_missing": + return DailyMileageSeverityPipeline + default: + return DailyMileageSeveritySourceData + } + default: + return DailyMileageSeverityPipeline + } +} + +func DailyMileageRecommendedOperatorAction(diagnosis, reason string) string { + diagnosis = strings.ToUpper(strings.TrimSpace(diagnosis)) + reason = strings.ToLower(strings.TrimSpace(reason)) + switch diagnosis { + case "", "OK": + return "无需处理，车辆当日里程已生成" + case "MISSING_DAILY": + return "统计源样本已存在但最终日表缺失，优先排查 stat-writer 投影任务、MySQL 写入和 source 选举" + case "NO_SOURCE_SAMPLE": + return "实时位置已有有效总里程但统计字段样本缺失，优先排查字段流 topic、stat-writer 消费和协议字段映射" + case "NO_TOTAL_MILEAGE": + switch reason { + case "realtime_total_mileage_time_missing": + return "总里程值存在但事件时间缺失，优先排查解析器和 history/realtime 写入的时间字段" + case "realtime_total_mileage_missing": + return "当日实时数据缺少总里程字段，核对平台是否上报该字段以及协议字段映射是否覆盖" + case "realtime_total_mileage_non_positive": + return "终端上报总里程小于等于 0，不参与差值统计，需核对设备或平台里程字段来源" + case "realtime_total_mileage_not_reported_on_stat_date": + return "最新总里程不是统计日期内上报，等待当日含总里程数据或检查平台是否只上报部分字段" + default: + return "车辆当日在线但没有可用于里程统计的总里程，优先核对源平台报文字段和解析映射" + } + default: + return "诊断状态未知，检查统计诊断 SQL 和上游字段流" + } +} + +type Thresholds struct { + GatewayFrameP99MS float64 + GatewayIdentityP99MS float64 + GatewayResponseRecentP99MS float64 + AsyncSinkP99MS float64 + AsyncSinkQueueWaitRecentP99MS float64 + BridgeBatchP99MS float64 + BridgeKafkaWriteP99MS float64 + BridgeKafkaE2EP99MS float64 + BridgeKafkaRecentE2EP99MS float64 + BridgeFetchWaitMaxMS float64 + BridgeWorkersMin float64 + BridgeRouteErrorMax float64 + BridgeKafkaWriteErrorMax float64 + BridgeMessageMinReceived float64 + BridgeMessageMaxUnackedRatio float64 + FieldsProjectorWorkersPerProtocolMin float64 + FieldsProjectorInvalidJSONMax float64 + FieldsProjectorWriteErrorMax float64 + FieldsProjectorBatchPendingMax float64 + FastWriterStageP99MS float64 + FastWriterRedisE2EP99MS float64 + FastWriterRedisRecentE2EP99MS float64 + FastWriterFetchWaitMaxMS float64 + FastWriterWorkersMin float64 + FastWriterInvalidJSONMax float64 + FastWriterFallbackErrorMax float64 + FastWriterDecoupledUpdateErrorMax float64 + FastWriterTDengineEnabledMax float64 + FastWriterMessageMinReceived float64 + FastWriterMessageMaxUnackedRatio float64 + HistoryFlushP99MS float64 + HistoryWriteE2EP99MS float64 + HistoryWriteRecentE2EP99MS float64 + HistoryWorkersMin float64 + HistoryInvalidJSONMax float64 + HistoryLocationErrorMax float64 + HistoryLocationAccountingMaxRatio float64 + HistoryFallbackErrorMax float64 + HistoryRetryMax float64 + HistoryRetryExhaustedMax float64 + RealtimeBatchP99MS float64 + RealtimeStoreP99MS float64 + RealtimeWorkersMin float64 + RealtimeStoreErrorMax float64 + RealtimeInvalidJSONMax float64 + StatWriteP99MS float64 + StatWriteE2EP99MS float64 + StatWriteRecentE2EP99MS float64 + StatWorkersMin float64 + IdentityWorkersMin float64 + StatInvalidJSONMax float64 + StatProtocolTopicMismatchMax float64 + MessageContractMismatchMax float64 + StatRetryMax float64 + StatRetryExhaustedMax float64 + ProcessUptimeMinSec float64 + ProcessHeapAllocMaxBytes float64 + ProcessHeapSysMaxBytes float64 + ProcessGoroutinesMax float64 + KafkaLagMax float64 + KafkaConsumerCommitErrorMax float64 + KafkaConsumerMessageMinReceived float64 + KafkaConsumerMaxUncommittedRatio float64 + DurableSpoolBacklogMax float64 + DurableSpoolOldestAgeMaxSec float64 + DurableSpoolErrorMax float64 + DurableSpoolReplayErrorMax float64 + DurableOutboxInflightMax float64 + DurableOutboxErrorMax float64 + AsyncSinkEnqueueErrorMax float64 + AsyncSinkPublishErrorMax float64 + AsyncSinkQueueDepthMax float64 + AsyncSinkQueueMaxUsageRatio float64 + FastWriterBatchPendingMax float64 + HistoryBatchPendingMax float64 + HistoryRowsPendingMax float64 + RealtimeBatchPendingMax float64 + RealtimeAsyncQueueDepthMax float64 + RealtimeAsyncQueueMaxUsageRatio float64 + RealtimeAsyncQueueDroppedMax float64 + HistoryParsedFieldMinFrames float64 + HistoryParsedFieldMaxMissingRatio float64 + MinHistogramSamples float64 + StatSampleMinWrites float64 + StatSampleMaxActionableSkipRatio float64 + StatSampleFutureAdjustmentMax float64 + StatSourceMinSamples float64 + StatSourceMaxMissingRatio float64 + StatProjectionMinSampleWrites float64 + StatBatchPendingMax float64 + ConsumerRetryPendingMax float64 + StatCacheEntriesMax float64 + RealtimeThrottleMinSamples float64 + RealtimeThrottleCacheEntriesMax float64 + RealtimePlateCacheEntriesMax float64 + FastWriterRedisEnvelopeMinSeen float64 + FastWriterRedisEnvelopeMaxBadRatio float64 + FastWriterRedisFieldMinSeen float64 + FastWriterRedisFieldMaxStaleRatio float64 + GatewayFieldMinRaw float64 + GatewayFieldMaxMissingRatio float64 + GatewayFieldMinCount float64 + GatewayBridgeMinPublish float64 + GatewayFastWriterMinRawPublish float64 + GatewayParseMinFrames float64 + GatewayParseMaxBadRatio float64 + GatewayResponseMinSamples float64 + GatewayResponseMaxErrorRatio float64 + GatewayIdentityMinSamples float64 + GatewayIdentityMaxUnresolvedRatio float64 + GatewayIdentityCacheEntriesMax float64 + GatewayIdentitySnapshotRequired float64 + GatewayIdentitySnapshotMaxAgeSec float64 + GatewayIdentityLocationTouchFailureMax float64 + GatewayIdentityRegistrationWriteQueueDepthMax float64 + GatewayIdentityRegistrationWriteQueueMaxUsageRatio float64 + GatewayIdentityRegistrationWriteErrorMax float64 + GatewayIdentityStaleMinSamples float64 + GatewayIdentityMaxStaleCacheRatio float64 + StatTopicMinBridge float64 + RawFanoutMinBridge float64 + LastActivityStaleSec float64 + RequiredConsumerTopics map[string][]string +} + +func DefaultThresholds() Thresholds { + return Thresholds{ + GatewayFrameP99MS: 250, + GatewayIdentityP99MS: 100, + GatewayResponseRecentP99MS: 100, + AsyncSinkP99MS: 100, + AsyncSinkQueueWaitRecentP99MS: 100, + BridgeBatchP99MS: 5000, + BridgeKafkaWriteP99MS: 500, + BridgeKafkaE2EP99MS: 0, + BridgeKafkaRecentE2EP99MS: 300, + BridgeFetchWaitMaxMS: 50, + BridgeWorkersMin: 2, + BridgeRouteErrorMax: 0, + BridgeKafkaWriteErrorMax: 0, + BridgeMessageMinReceived: 5000, + BridgeMessageMaxUnackedRatio: 0.20, + FieldsProjectorWorkersPerProtocolMin: 1, + FieldsProjectorInvalidJSONMax: 0, + FieldsProjectorWriteErrorMax: 0, + FieldsProjectorBatchPendingMax: 1000, + FastWriterStageP99MS: 100, + FastWriterRedisE2EP99MS: 0, + FastWriterRedisRecentE2EP99MS: 100, + FastWriterFetchWaitMaxMS: 50, + FastWriterWorkersMin: 4, + FastWriterInvalidJSONMax: 0, + FastWriterFallbackErrorMax: 0, + FastWriterDecoupledUpdateErrorMax: 0, + FastWriterTDengineEnabledMax: 0, + FastWriterMessageMinReceived: 5000, + FastWriterMessageMaxUnackedRatio: 0.20, + HistoryFlushP99MS: 1000, + HistoryWriteE2EP99MS: 0, + HistoryWriteRecentE2EP99MS: 2000, + HistoryWorkersMin: 3, + HistoryInvalidJSONMax: 0, + HistoryLocationErrorMax: 0, + HistoryLocationAccountingMaxRatio: 0.02, + HistoryFallbackErrorMax: 0, + HistoryRetryMax: 100, + HistoryRetryExhaustedMax: 0, + RealtimeBatchP99MS: 500, + RealtimeStoreP99MS: 100, + RealtimeWorkersMin: 3, + RealtimeStoreErrorMax: 0, + RealtimeInvalidJSONMax: 0, + StatWriteP99MS: 100, + StatWriteE2EP99MS: 0, + StatWriteRecentE2EP99MS: 1000, + StatWorkersMin: 3, + IdentityWorkersMin: 3, + StatInvalidJSONMax: 0, + StatProtocolTopicMismatchMax: 0, + MessageContractMismatchMax: 0, + StatRetryMax: 100, + StatRetryExhaustedMax: 0, + ProcessUptimeMinSec: 0, + ProcessHeapAllocMaxBytes: 0, + ProcessHeapSysMaxBytes: 0, + ProcessGoroutinesMax: 0, + KafkaLagMax: 1000, + KafkaConsumerCommitErrorMax: 0, + KafkaConsumerMessageMinReceived: 100, + KafkaConsumerMaxUncommittedRatio: 0.20, + DurableSpoolBacklogMax: 10000, + DurableSpoolOldestAgeMaxSec: 300, + DurableSpoolErrorMax: 0, + DurableSpoolReplayErrorMax: 0, + DurableOutboxInflightMax: 10000, + DurableOutboxErrorMax: 0, + AsyncSinkEnqueueErrorMax: 0, + AsyncSinkPublishErrorMax: 0, + AsyncSinkQueueDepthMax: 10000, + AsyncSinkQueueMaxUsageRatio: 0.80, + FastWriterBatchPendingMax: 1000, + HistoryBatchPendingMax: 1000, + HistoryRowsPendingMax: 5000, + RealtimeBatchPendingMax: 1000, + RealtimeAsyncQueueDepthMax: 10000, + RealtimeAsyncQueueMaxUsageRatio: 0.80, + RealtimeAsyncQueueDroppedMax: 0, + HistoryParsedFieldMinFrames: 100, + HistoryParsedFieldMaxMissingRatio: 0.05, + MinHistogramSamples: 100, + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.60, + StatSampleFutureAdjustmentMax: 0, + StatSourceMinSamples: 100, + StatSourceMaxMissingRatio: 0.60, + StatProjectionMinSampleWrites: 100, + StatBatchPendingMax: 1000, + ConsumerRetryPendingMax: 0, + StatCacheEntriesMax: 1000000, + RealtimeThrottleMinSamples: 100, + RealtimeThrottleCacheEntriesMax: 1000000, + RealtimePlateCacheEntriesMax: 200000, + FastWriterRedisEnvelopeMinSeen: 100, + FastWriterRedisEnvelopeMaxBadRatio: 0.50, + FastWriterRedisFieldMinSeen: 1000, + FastWriterRedisFieldMaxStaleRatio: 0.20, + GatewayFieldMinRaw: 100, + GatewayFieldMaxMissingRatio: 0.20, + GatewayFieldMinCount: 1, + GatewayBridgeMinPublish: 1000, + GatewayFastWriterMinRawPublish: 1000, + GatewayParseMinFrames: 100, + GatewayParseMaxBadRatio: 0.05, + GatewayResponseMinSamples: 100, + GatewayResponseMaxErrorRatio: 0.05, + GatewayIdentityMinSamples: 100, + GatewayIdentityMaxUnresolvedRatio: 0.20, + GatewayIdentityCacheEntriesMax: 300000, + GatewayIdentitySnapshotRequired: 1, + GatewayIdentitySnapshotMaxAgeSec: 180, + GatewayIdentityLocationTouchFailureMax: 0, + GatewayIdentityRegistrationWriteQueueDepthMax: 10000, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80, + GatewayIdentityRegistrationWriteErrorMax: 0, + GatewayIdentityStaleMinSamples: 100, + GatewayIdentityMaxStaleCacheRatio: 0.05, + StatTopicMinBridge: 1000, + RawFanoutMinBridge: 1000, + LastActivityStaleSec: 300, + } +} + +func DefaultRequiredConsumerTopics() map[string][]string { + return map[string][]string{ + "fields-projector": { + topics.RawGB32960, + topics.RawJT808, + topics.RawYutongMQTT, + }, + "history": { + topics.RawGB32960, + topics.RawJT808, + topics.RawYutongMQTT, + }, + "realtime": { + topics.RawGB32960, + topics.RawJT808, + topics.RawYutongMQTT, + }, + "stat": { + topics.FieldsGB32960, + topics.FieldsJT808, + topics.FieldsYutongMQTT, + }, + "identity": { + topics.RawJT808, + }, + } +} + func Evaluate(metricsByService map[string]string) Report { + return EvaluateWithThresholds(metricsByService, DefaultThresholds()) +} + +func EvaluateWithThresholds(metricsByService map[string]string, thresholds Thresholds) Report { report := Report{ Status: StatusOK, CheckedAt: time.Now().UTC(), Services: make([]ServiceCheck, 0, len(metricsByService)), } + parsedByService := make(map[string]parsedMetricSet, len(metricsByService)) var names []string for name := range metricsByService { names = append(names, name) } sort.Strings(names) for _, name := range names { - metrics := parseMetrics(metricsByService[name]) + parsed := parseMetricSet(metricsByService[name]) + parsedByService[name] = parsed + metrics := parsed.totals + addLastActivityAges(metrics, report.CheckedAt) report.Services = append(report.Services, ServiceCheck{Name: name, Metrics: metrics}) report.Totals.ActiveConnections += int64(metrics["vehicle_gateway_active_connections"]) report.Totals.KafkaLag += metrics["vehicle_history_kafka_lag"] report.Totals.KafkaLag += metrics["vehicle_stat_kafka_lag"] report.Totals.KafkaLag += metrics["vehicle_realtime_kafka_lag"] + report.Totals.KafkaLag += metrics["vehicle_identity_writer_kafka_lag"] report.Totals.BridgeConsumerPending += metrics["vehicle_bridge_nats_consumer_pending"] report.Totals.BridgeAckPending += metrics["vehicle_bridge_nats_consumer_ack_pending"] report.Totals.BridgeBatchPendingMessages += metrics["vehicle_bridge_batch_pending_messages"] + report.Totals.FieldsProjectorKafkaLag += metrics["vehicle_fields_projector_kafka_lag"] + report.Totals.FieldsProjectorBatchPending += metrics["vehicle_fields_projector_batch_pending_messages"] + report.Totals.FieldsProjectorRetryPending += metrics["vehicle_fields_projector_retry_pending_messages"] report.Totals.FastWriterConsumerPending += metrics["vehicle_fast_writer_nats_consumer_pending"] report.Totals.FastWriterAckPending += metrics["vehicle_fast_writer_nats_consumer_ack_pending"] report.Totals.FastWriterBatchPending += metrics["vehicle_fast_writer_batch_pending_messages"] report.Totals.HistoryBatchPending += metrics["vehicle_history_batch_pending_messages"] report.Totals.HistoryRowsPending += metrics["vehicle_history_batch_pending_rows"] - report.Findings = append(report.Findings, findingsForMetrics(metrics)...) + report.Totals.HistoryRetryPending += metrics["vehicle_history_retry_pending_messages"] + report.Totals.RealtimeBatchPending += metrics["vehicle_realtime_batch_pending_messages"] + report.Totals.RealtimeAsyncQueuePending += metrics["vehicle_realtime_async_queue_depth"] + report.Totals.RealtimeRetryPending += metrics["vehicle_realtime_retry_pending_messages"] + report.Totals.StatRetryPending += metrics["vehicle_stat_retry_pending_messages"] + report.Totals.IdentityBatchPending += metrics["vehicle_identity_writer_batch_pending_messages"] + report.Totals.IdentityFactsPending += metrics["vehicle_identity_writer_batch_pending_facts"] + report.Totals.IdentityRetryPending += metrics["vehicle_identity_writer_retry_pending_messages"] + report.Totals.JT808RegistrationWriteQueuePending += gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries") + report.Totals.JT808RegistrationWriteErrors += jt808RegistrationWriteErrors(parsed) + report.Findings = append(report.Findings, findingsForService(name, metrics)...) + report.Findings = append(report.Findings, findingsForMetrics(name, metrics, thresholds)...) + report.Findings = append(report.Findings, findingsForRuntimeConfig(name, parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForAsyncSinkQueueUsage(name, parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForAsyncSinkErrors(name, parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForDurableSpool(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForDurableOutbox(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistograms(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForKafkaConsumerCompletion(name, parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForMessageContractMismatches(name, parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForBridgeRouteErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForBridgeKafkaWriteErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForBridgeMessageCompletion(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFieldsProjector(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistoryInvalidJSON(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistoryLocationErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistoryLocationAccounting(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistoryFallbackErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForWriteRetries(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForHistoryParsedFields(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatInvalidJSON(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatProtocolTopicMismatches(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatSamples(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatSources(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatProjections(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForStatCache(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterRedisEnvelopes(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterRedisFields(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterInvalidJSON(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterFallbackErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterDecoupledUpdateErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForFastWriterMessageCompletion(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimeInvalidJSON(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimeStoreErrors(parsed, thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimeAsyncQueue(parsed, thresholds)...) } + report.Findings = append(report.Findings, findingsForGatewayIdentityQuality(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayIdentityCache(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayIdentitySnapshot(parsedByService["gateway"], thresholds, report.CheckedAt)...) + report.Findings = append(report.Findings, findingsForGatewayIdentityLocationTouchFailures(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteQueue(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayRegistrationWriteErrors(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayIdentityStaleCache(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForIdentityWriterOwnership(parsedByService["gateway"], parsedByService["identity"])...) + report.GatewayFields = buildGatewayFieldsDiagnostics(parsedByService["gateway"], thresholds) + report.IdentityResolution = buildIdentityResolutionDiagnostics(parsedByService["gateway"]) + report.ProtocolFlow = buildProtocolFlowDiagnostics(parsedByService, thresholds) + report.PipelineLatency = buildPipelineLatencyDiagnostics(parsedByService, thresholds) + report.Findings = append(report.Findings, findingsForGatewayParseQuality(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForGatewayResponseQuality(parsedByService["gateway"], thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimeThrottle(parsedByService["realtime"], thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimeThrottleCache(parsedByService["realtime"], thresholds)...) + report.Findings = append(report.Findings, findingsForRealtimePlateCache(parsedByService["realtime"], thresholds)...) + report.Findings = append(report.Findings, findingsForRequiredConsumerTopics(parsedByService, thresholds)...) + report.Findings = append(report.Findings, findingsForTopicFlow(parsedByService, thresholds)...) + report.Findings = append(report.Findings, findingsForLastActivity(parsedByService, thresholds, report.CheckedAt)...) if len(report.Findings) > 0 { report.Status = StatusDegraded } return report } -func findingsForMetrics(metrics map[string]float64) []string { +func findingsForService(name string, metrics map[string]float64) []string { + if metrics["vehicle_service_info"] <= 0 { + return []string{fmt.Sprintf("%s service info metric missing", name)} + } + if requiresKafkaConsumerInfo(name) && metrics["vehicle_kafka_consumer_info"] <= 0 { + return []string{fmt.Sprintf("%s kafka consumer info metric missing", name)} + } + return nil +} + +func requiresKafkaConsumerInfo(name string) bool { + switch name { + case "history", "stat", "realtime", "fields-projector": + return true + default: + return false + } +} + +func findingsForMetrics(service string, metrics map[string]float64, thresholds Thresholds) []string { var findings []string + if thresholds.ProcessUptimeMinSec > 0 { + value, ok := metrics["vehicle_process_uptime_seconds"] + if !ok { + findings = append(findings, fmt.Sprintf("%s process uptime metric missing", service)) + } else if value < thresholds.ProcessUptimeMinSec { + findings = append(findings, fmt.Sprintf("%s process uptime %.0fs below %.0fs", service, value, thresholds.ProcessUptimeMinSec)) + } + } + if thresholds.ProcessHeapAllocMaxBytes > 0 { + value, ok := metrics["vehicle_process_heap_alloc_bytes"] + if !ok { + findings = append(findings, fmt.Sprintf("%s process heap alloc metric missing", service)) + } else if value > thresholds.ProcessHeapAllocMaxBytes { + findings = append(findings, fmt.Sprintf("%s process heap alloc %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapAllocMaxBytes)) + } + } + if thresholds.ProcessHeapSysMaxBytes > 0 { + value, ok := metrics["vehicle_process_heap_sys_bytes"] + if !ok { + findings = append(findings, fmt.Sprintf("%s process heap sys metric missing", service)) + } else if value > thresholds.ProcessHeapSysMaxBytes { + findings = append(findings, fmt.Sprintf("%s process heap sys %.0f bytes exceeds %.0f", service, value, thresholds.ProcessHeapSysMaxBytes)) + } + } + if thresholds.ProcessGoroutinesMax > 0 { + value, ok := metrics["vehicle_process_goroutines"] + if !ok { + findings = append(findings, fmt.Sprintf("%s process goroutines metric missing", service)) + } else if value > thresholds.ProcessGoroutinesMax { + findings = append(findings, fmt.Sprintf("%s process goroutines %.0f exceeds %.0f", service, value, thresholds.ProcessGoroutinesMax)) + } + } if value := metrics["vehicle_gateway_connection_rejections_total"]; value > 0 { findings = append(findings, fmt.Sprintf("gateway connection rejections %.0f", value)) } - if value := metrics["vehicle_async_sink_queue_depth"]; value > 10_000 { - findings = append(findings, fmt.Sprintf("async sink queue depth %.0f exceeds 10000", value)) + if thresholds.AsyncSinkQueueDepthMax > 0 { + if value := metrics["vehicle_async_sink_queue_depth"]; value > thresholds.AsyncSinkQueueDepthMax { + findings = append(findings, fmt.Sprintf("async sink queue depth %.0f exceeds %.0f", value, thresholds.AsyncSinkQueueDepthMax)) + } } if value := metrics["vehicle_bridge_nats_consumer_ack_pending"]; value > 100 { findings = append(findings, fmt.Sprintf("bridge ack pending %.0f exceeds 100", value)) @@ -93,56 +987,3932 @@ func findingsForMetrics(metrics map[string]float64) []string { if value := metrics["vehicle_bridge_batch_pending_messages"]; value > 1000 { findings = append(findings, fmt.Sprintf("bridge batch pending %.0f exceeds 1000", value)) } - if value := metrics["vehicle_fast_writer_nats_consumer_ack_pending"]; value > 10 { - findings = append(findings, fmt.Sprintf("fast writer ack pending %.0f exceeds 10", value)) + if value := metrics["vehicle_fast_writer_nats_consumer_ack_pending"]; value > 100 { + findings = append(findings, fmt.Sprintf("fast writer ack pending %.0f exceeds 100", value)) } if value := metrics["vehicle_fast_writer_nats_consumer_pending"]; value > 10_000 { findings = append(findings, fmt.Sprintf("fast writer consumer pending %.0f exceeds 10000", value)) } - if value := metrics["vehicle_fast_writer_batch_pending_messages"]; value > 0 { - findings = append(findings, fmt.Sprintf("fast writer batch pending %.0f", value)) + if value := metrics["vehicle_fast_writer_batch_pending_messages"]; thresholds.FastWriterBatchPendingMax > 0 && value > thresholds.FastWriterBatchPendingMax { + findings = append(findings, fmt.Sprintf("fast writer batch pending %.0f exceeds %.0f", value, thresholds.FastWriterBatchPendingMax)) } - if value := metrics["vehicle_history_batch_pending_messages"]; value > 0 { - findings = append(findings, fmt.Sprintf("history batch pending %.0f", value)) + if value := metrics["vehicle_fast_writer_tdengine_enabled"]; thresholds.FastWriterTDengineEnabledMax >= 0 && value > thresholds.FastWriterTDengineEnabledMax { + findings = append(findings, fmt.Sprintf("fast writer tdengine enabled %.0f exceeds %.0f", value, thresholds.FastWriterTDengineEnabledMax)) } - if value := metrics["vehicle_history_batch_pending_rows"]; value > 0 { - findings = append(findings, fmt.Sprintf("history rows pending %.0f", value)) + if value := metrics["vehicle_realtime_redis_projector_enabled"]; value > 0 { + findings = append(findings, fmt.Sprintf("realtime redis projector enabled %.0f", value)) } - if value := metrics["vehicle_history_kafka_lag"] + metrics["vehicle_stat_kafka_lag"] + metrics["vehicle_realtime_kafka_lag"]; value > 0 { - findings = append(findings, fmt.Sprintf("kafka lag %.0f", value)) + if value := metrics["vehicle_history_batch_pending_messages"]; thresholds.HistoryBatchPendingMax > 0 && value > thresholds.HistoryBatchPendingMax { + findings = append(findings, fmt.Sprintf("history batch pending %.0f exceeds %.0f", value, thresholds.HistoryBatchPendingMax)) + } + if value := metrics["vehicle_history_batch_pending_rows"]; thresholds.HistoryRowsPendingMax > 0 && value > thresholds.HistoryRowsPendingMax { + findings = append(findings, fmt.Sprintf("history rows pending %.0f exceeds %.0f", value, thresholds.HistoryRowsPendingMax)) + } + if value := metrics["vehicle_stat_batch_pending_messages"]; thresholds.StatBatchPendingMax > 0 && value > thresholds.StatBatchPendingMax { + findings = append(findings, fmt.Sprintf("stat batch pending %.0f exceeds %.0f", value, thresholds.StatBatchPendingMax)) + } + if value := metrics["vehicle_realtime_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax { + findings = append(findings, fmt.Sprintf("realtime batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax)) + } + if value := metrics["vehicle_realtime_async_queue_depth"]; thresholds.RealtimeAsyncQueueDepthMax > 0 && value > thresholds.RealtimeAsyncQueueDepthMax { + findings = append(findings, fmt.Sprintf("realtime async queue depth %.0f exceeds %.0f", value, thresholds.RealtimeAsyncQueueDepthMax)) + } + for _, retry := range []struct { + name string + metric string + }{ + {name: "history", metric: "vehicle_history_retry_pending_messages"}, + {name: "realtime", metric: "vehicle_realtime_retry_pending_messages"}, + {name: "stat", metric: "vehicle_stat_retry_pending_messages"}, + {name: "identity", metric: "vehicle_identity_writer_retry_pending_messages"}, + {name: "fields projector", metric: "vehicle_fields_projector_retry_pending_messages"}, + } { + if value := metrics[retry.metric]; thresholds.ConsumerRetryPendingMax >= 0 && value > thresholds.ConsumerRetryPendingMax { + findings = append(findings, fmt.Sprintf("%s retry pending %.0f exceeds %.0f", retry.name, value, thresholds.ConsumerRetryPendingMax)) + } + } + if value := metrics["vehicle_history_kafka_lag"] + metrics["vehicle_stat_kafka_lag"] + metrics["vehicle_realtime_kafka_lag"] + metrics["vehicle_identity_writer_kafka_lag"] + metrics["vehicle_fields_projector_kafka_lag"]; thresholds.KafkaLagMax > 0 && value > thresholds.KafkaLagMax { + findings = append(findings, fmt.Sprintf("kafka lag %.0f exceeds %.0f", value, thresholds.KafkaLagMax)) + } + if value := metrics["vehicle_identity_writer_batch_pending_messages"]; thresholds.RealtimeBatchPendingMax > 0 && value > thresholds.RealtimeBatchPendingMax { + findings = append(findings, fmt.Sprintf("identity batch pending %.0f exceeds %.0f", value, thresholds.RealtimeBatchPendingMax)) + } + if value := metrics["vehicle_fields_projector_batch_pending_messages"]; thresholds.FieldsProjectorBatchPendingMax > 0 && value > thresholds.FieldsProjectorBatchPendingMax { + findings = append(findings, fmt.Sprintf("fields projector batch pending %.0f exceeds %.0f", value, thresholds.FieldsProjectorBatchPendingMax)) } return findings } -func parseMetrics(text string) map[string]float64 { +func findingsForFieldsProjector(parsed parsedMetricSet, thresholds Thresholds) []string { + var invalidJSON float64 + var writeErrors float64 + for _, sample := range parsed.samples { + switch { + case sample.name == "vehicle_fields_projector_kafka_messages_total" && sample.labels["status"] == "invalid_json": + invalidJSON += sample.value + case sample.name == "vehicle_fields_projector_kafka_writes_total" && sample.labels["status"] == "error": + writeErrors += sample.value + } + } + var findings []string + if thresholds.FieldsProjectorInvalidJSONMax >= 0 && invalidJSON > thresholds.FieldsProjectorInvalidJSONMax { + findings = append(findings, fmt.Sprintf("fields projector invalid json %.0f exceeds %.0f", invalidJSON, thresholds.FieldsProjectorInvalidJSONMax)) + } + if thresholds.FieldsProjectorWriteErrorMax >= 0 && writeErrors > thresholds.FieldsProjectorWriteErrorMax { + findings = append(findings, fmt.Sprintf("fields projector kafka write errors %.0f exceeds %.0f", writeErrors, thresholds.FieldsProjectorWriteErrorMax)) + } + return findings +} + +func findingsForRuntimeConfig(service string, parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + if service == "bridge" || hasServiceInfo(parsed, "nats-kafka-bridge") { + findings = append(findings, configMaxFinding(parsed, "vehicle_bridge_config", "fetch_wait_ms", "bridge fetch wait", "ms", thresholds.BridgeFetchWaitMaxMS)...) + findings = append(findings, configMinFinding(parsed, "vehicle_bridge_config", "workers", "bridge workers", "", thresholds.BridgeWorkersMin)...) + } + if service == "fields-projector" || hasServiceInfo(parsed, "vehicle-fields-projector") { + findings = append(findings, configMinFinding(parsed, "vehicle_fields_projector_config", "workers_per_protocol", "fields projector workers per protocol", "", thresholds.FieldsProjectorWorkersPerProtocolMin)...) + } + if service == "fast-writer" || hasServiceInfo(parsed, "nats-fast-writer") { + findings = append(findings, configMaxFinding(parsed, "vehicle_fast_writer_config", "fetch_wait_ms", "fast writer fetch wait", "ms", thresholds.FastWriterFetchWaitMaxMS)...) + findings = append(findings, configMinFinding(parsed, "vehicle_fast_writer_config", "workers", "fast writer workers", "", thresholds.FastWriterWorkersMin)...) + } + if service == "history" || hasServiceInfo(parsed, "vehicle-history-writer") { + findings = append(findings, configMinFinding(parsed, "vehicle_history_config", "workers", "history workers", "", thresholds.HistoryWorkersMin)...) + } + if service == "realtime" || hasServiceInfo(parsed, "vehicle-realtime-writer") { + findings = append(findings, configMinFinding(parsed, "vehicle_realtime_config", "workers", "realtime workers", "", thresholds.RealtimeWorkersMin)...) + } + if service == "stat" || hasServiceInfo(parsed, "vehicle-stat-writer") { + findings = append(findings, configMinFinding(parsed, "vehicle_stat_config", "workers", "stat workers", "", thresholds.StatWorkersMin)...) + } + if service == "identity" || hasServiceInfo(parsed, "vehicle-identity-writer") { + findings = append(findings, configMinFinding(parsed, "vehicle_identity_writer_config", "workers", "identity workers", "", thresholds.IdentityWorkersMin)...) + } + sort.Strings(findings) + return findings +} + +func configMaxFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string { + if threshold <= 0 { + return nil + } + value, ok := maxConfigGaugeValue(parsed, metricName, setting) + if !ok { + return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)} + } + if value <= threshold { + return nil + } + return []string{fmt.Sprintf("%s %.0f%s exceeds %.0f%s", label, value, unit, threshold, unit)} +} + +func configMinFinding(parsed parsedMetricSet, metricName string, setting string, label string, unit string, threshold float64) []string { + if threshold <= 0 { + return nil + } + value, ok := minConfigGaugeValue(parsed, metricName, setting) + if !ok { + return []string{fmt.Sprintf("%s config metric missing (%s setting=%s)", label, metricName, setting)} + } + if value >= threshold { + return nil + } + return []string{fmt.Sprintf("%s %.0f%s below %.0f%s", label, value, unit, threshold, unit)} +} + +func hasServiceInfo(parsed parsedMetricSet, service string) bool { + service = strings.TrimSpace(service) + if service == "" { + return false + } + for _, sample := range parsed.samples { + if sample.name == "vehicle_service_info" && sample.labels["service"] == service && sample.value > 0 { + return true + } + } + return false +} + +func maxConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) { + var out float64 + found := false + for _, sample := range parsed.samples { + if sample.name != metricName || sample.labels["setting"] != setting { + continue + } + if !found || sample.value > out { + out = sample.value + found = true + } + } + return out, found +} + +func minConfigGaugeValue(parsed parsedMetricSet, metricName string, setting string) (float64, bool) { + var out float64 + found := false + for _, sample := range parsed.samples { + if sample.name != metricName || sample.labels["setting"] != setting { + continue + } + if !found || sample.value < out { + out = sample.value + found = true + } + } + return out, found +} + +func findingsForAsyncSinkQueueUsage(service string, parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.AsyncSinkQueueMaxUsageRatio <= 0 { + return nil + } + type queueSample struct { + labels map[string]string + depth float64 + capacity float64 + } + queues := map[string]*queueSample{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_async_sink_queue_depth": + key := metricLabelsKey(sample.labels) + item := queues[key] + if item == nil { + item = &queueSample{labels: sample.labels} + queues[key] = item + } + item.depth += sample.value + case "vehicle_async_sink_queue_capacity": + key := metricLabelsKey(sample.labels) + item := queues[key] + if item == nil { + item = &queueSample{labels: sample.labels} + queues[key] = item + } + item.capacity += sample.value + } + } + var keys []string + for key := range queues { + keys = append(keys, key) + } + sort.Strings(keys) + var findings []string + for _, key := range keys { + item := queues[key] + if item.capacity <= 0 || item.depth <= 0 { + continue + } + ratio := item.depth / item.capacity + if ratio <= thresholds.AsyncSinkQueueMaxUsageRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "%s async sink queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)", + service, + ratio, + thresholds.AsyncSinkQueueMaxUsageRatio, + key, + item.depth, + item.capacity, + )) + } + return findings +} + +func findingsForAsyncSinkErrors(service string, parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + if thresholds.AsyncSinkEnqueueErrorMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_async_sink_enqueue_total" { + continue + } + status := strings.TrimSpace(sample.labels["status"]) + if status == "" || status == "queued" { + continue + } + if sample.value <= thresholds.AsyncSinkEnqueueErrorMax { + continue + } + sink := strings.TrimSpace(sample.labels["sink"]) + if sink == "" { + sink = "unknown" + } + kind := strings.TrimSpace(sample.labels["kind"]) + if kind == "" { + kind = "unknown" + } + findings = append(findings, fmt.Sprintf( + "%s async sink enqueue %s %.0f exceeds %.0f for sink %s kind %s", + service, + status, + sample.value, + thresholds.AsyncSinkEnqueueErrorMax, + sink, + kind, + )) + } + } + if thresholds.AsyncSinkPublishErrorMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_async_sink_publish_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.AsyncSinkPublishErrorMax { + continue + } + sink := strings.TrimSpace(sample.labels["sink"]) + if sink == "" { + sink = "unknown" + } + kind := strings.TrimSpace(sample.labels["kind"]) + if kind == "" { + kind = "unknown" + } + findings = append(findings, fmt.Sprintf( + "%s async sink publish errors %.0f exceeds %.0f for sink %s kind %s", + service, + sample.value, + thresholds.AsyncSinkPublishErrorMax, + sink, + kind, + )) + } + } + sort.Strings(findings) + return findings +} + +func findingsForDurableSpool(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + if thresholds.DurableSpoolBacklogMax > 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_spool_backlog_files" || sample.value <= thresholds.DurableSpoolBacklogMax { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + findings = append(findings, fmt.Sprintf( + "durable spool backlog %.0f exceeds %.0f for name %s", + sample.value, + thresholds.DurableSpoolBacklogMax, + name, + )) + } + } + if thresholds.DurableSpoolOldestAgeMaxSec > 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_spool_oldest_age_seconds" || sample.value <= thresholds.DurableSpoolOldestAgeMaxSec { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + findings = append(findings, fmt.Sprintf( + "durable spool oldest age %.0fs exceeds %.0fs for name %s", + sample.value, + thresholds.DurableSpoolOldestAgeMaxSec, + name, + )) + } + } + if thresholds.DurableSpoolErrorMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_spool_records_total" || sample.labels["status"] != "error" || sample.value <= thresholds.DurableSpoolErrorMax { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + kind := strings.TrimSpace(sample.labels["kind"]) + if kind == "" { + kind = "unknown" + } + findings = append(findings, fmt.Sprintf( + "durable spool record errors %.0f exceeds %.0f for name %s kind %s", + sample.value, + thresholds.DurableSpoolErrorMax, + name, + kind, + )) + } + } + if thresholds.DurableSpoolReplayErrorMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_spool_replay_total" || !strings.HasSuffix(sample.labels["status"], "_error") || sample.value <= thresholds.DurableSpoolReplayErrorMax { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + status := strings.TrimSpace(sample.labels["status"]) + if status == "" { + status = "unknown" + } + findings = append(findings, fmt.Sprintf( + "durable spool replay %s %.0f exceeds %.0f for name %s", + status, + sample.value, + thresholds.DurableSpoolReplayErrorMax, + name, + )) + } + } + sort.Strings(findings) + return findings +} + +func findingsForDurableOutbox(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + if thresholds.DurableOutboxInflightMax > 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_outbox_inflight" || sample.value <= thresholds.DurableOutboxInflightMax { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + findings = append(findings, fmt.Sprintf( + "durable outbox inflight %.0f exceeds %.0f for name %s", + sample.value, + thresholds.DurableOutboxInflightMax, + name, + )) + } + } + if thresholds.DurableOutboxErrorMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_durable_outbox_publish_total" || !durableOutboxErrorStatus(sample.labels["status"]) || sample.value <= thresholds.DurableOutboxErrorMax { + continue + } + name := strings.TrimSpace(sample.labels["name"]) + if name == "" { + name = "unknown" + } + kind := strings.TrimSpace(sample.labels["kind"]) + if kind == "" { + kind = "unknown" + } + status := strings.TrimSpace(sample.labels["status"]) + findings = append(findings, fmt.Sprintf( + "durable outbox %s %.0f exceeds %.0f for name %s kind %s", + status, + sample.value, + thresholds.DurableOutboxErrorMax, + name, + kind, + )) + } + } + sort.Strings(findings) + return findings +} + +func durableOutboxErrorStatus(status string) bool { + switch strings.TrimSpace(status) { + case "validation_error", "submit_error", "ack_error", "remove_error", "close_timeout", "quarantined": + return true + default: + return false + } +} + +func findingsForHistograms(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_frame_duration_ms_histogram", "gateway frame", thresholds.GatewayFrameP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_gateway_identity_duration_ms_histogram", "gateway identity", thresholds.GatewayIdentityP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_gateway_response_e2e_recent_p99_ms", "vehicle_gateway_response_e2e_recent_samples", "gateway response e2e", thresholds.GatewayResponseRecentP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_async_sink_publish_duration_ms_histogram", "async sink publish", thresholds.AsyncSinkP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_async_sink_queue_wait_recent_p99_ms", "vehicle_async_sink_queue_wait_recent_samples", "async sink queue wait", thresholds.AsyncSinkQueueWaitRecentP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_batch_duration_ms_histogram", "bridge batch", thresholds.BridgeBatchP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_write_duration_ms_histogram", "bridge kafka write", thresholds.BridgeKafkaWriteP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_bridge_kafka_e2e_duration_ms_histogram", "bridge kafka e2e", thresholds.BridgeKafkaE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_bridge_kafka_e2e_recent_p99_ms", "vehicle_bridge_kafka_e2e_recent_samples", "bridge kafka e2e", thresholds.BridgeKafkaRecentE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_stage_duration_ms_histogram", "fast writer stage", thresholds.FastWriterStageP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_fast_writer_redis_e2e_duration_ms_histogram", "fast writer redis e2e", thresholds.FastWriterRedisE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_fast_writer_redis_e2e_recent_p99_ms", "vehicle_fast_writer_redis_e2e_recent_samples", "fast writer redis e2e", thresholds.FastWriterRedisRecentE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_history_batch_flush_duration_ms_histogram", "history flush", thresholds.HistoryFlushP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_history_write_e2e_duration_ms_histogram", "history write e2e", thresholds.HistoryWriteE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_history_write_e2e_recent_p99_ms", "vehicle_history_write_e2e_recent_samples", "history write e2e", thresholds.HistoryWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_batch_flush_duration_ms_histogram", "realtime batch flush", thresholds.RealtimeBatchP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_realtime_store_update_duration_ms_histogram", "realtime store update", thresholds.RealtimeStoreP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_duration_ms_histogram", "stat write", thresholds.StatWriteP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, histogramP99Findings(parsed, "vehicle_stat_write_e2e_duration_ms_histogram", "stat write e2e", thresholds.StatWriteE2EP99MS, thresholds.MinHistogramSamples)...) + findings = append(findings, recentP99GaugeFindings(parsed, "vehicle_stat_write_e2e_recent_p99_ms", "vehicle_stat_write_e2e_recent_samples", "stat write e2e", thresholds.StatWriteRecentE2EP99MS, thresholds.MinHistogramSamples)...) + return findings +} + +type gatewayPublishCounters struct { + rawOK float64 + fieldsOK float64 + fieldsPublished float64 + fieldsDelegated float64 + fieldsSkippedNonRealtime float64 + fieldsSkippedMissing float64 + fieldsPublishError float64 +} + +type gatewayFieldFlowGroup struct { + gatewayPublishCounters + latestCount float64 + hasLatestCount bool +} + +type gatewayIdentityCounters struct { + total float64 + resolved float64 + unresolved float64 +} + +type identityResolutionGroup struct { + protocol string + total int64 + skipped int64 + resolved int64 + unresolved int64 + timeout int64 + errorCount int64 + otherNonResolved int64 + staleCache int64 + cacheStatusCounts map[string]int64 +} + +type pipelineLatencySpec struct { + service string + stage string + metricName string + samplesMetricName string + keyLabel string + thresholdMS float64 +} + +type gatewayParseCounters struct { + total float64 + ok float64 + partial float64 + badFrame float64 + other float64 +} + +type gatewayResponseCounters struct { + total float64 + ok float64 + buildError float64 + writeError float64 +} + +type historyParsedFieldCounters struct { + present float64 + missing float64 +} + +type kafkaConsumerCompletionCounters struct { + received float64 + committedOK float64 + commitError float64 +} + +type kafkaConsumerCompletionSpec struct { + serviceName string + messageMetric string + commitMetric string +} + +func findingsForKafkaConsumerCompletion(service string, parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.KafkaConsumerMessageMinReceived <= 0 || thresholds.KafkaConsumerMaxUncommittedRatio <= 0 { + return nil + } + spec, ok := kafkaConsumerCompletionSpecForService(service) + if !ok { + return nil + } + groups := map[string]*kafkaConsumerCompletionCounters{} + for _, sample := range parsed.samples { + switch sample.name { + case spec.messageMetric, spec.commitMetric: + default: + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" || topic == "mixed" || topic == "unknown" { + continue + } + group := groups[topic] + if group == nil { + group = &kafkaConsumerCompletionCounters{} + groups[topic] = group + } + switch { + case sample.name == spec.messageMetric && sample.labels["status"] == "received": + group.received += sample.value + case sample.name == spec.commitMetric && sample.labels["status"] == "ok": + group.committedOK += sample.value + case sample.name == spec.commitMetric && sample.labels["status"] == "error": + group.commitError += sample.value + } + } + var topics []string + for topic := range groups { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + group := groups[topic] + if thresholds.KafkaConsumerCommitErrorMax >= 0 && group.commitError > thresholds.KafkaConsumerCommitErrorMax { + findings = append(findings, fmt.Sprintf( + "%s kafka consumer commit errors %.0f exceeds %.0f for topic %s", + spec.serviceName, + group.commitError, + thresholds.KafkaConsumerCommitErrorMax, + topic, + )) + } + if group.received < thresholds.KafkaConsumerMessageMinReceived { + continue + } + uncommitted := group.received - group.committedOK + if uncommitted < 0 { + uncommitted = 0 + } + ratio := uncommitted / group.received + if ratio <= thresholds.KafkaConsumerMaxUncommittedRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "%s kafka consumer uncommitted message ratio %.2f exceeds %.2f for topic %s: received %.0f, commit ok %.0f, commit errors %.0f, uncommitted %.0f", + spec.serviceName, + ratio, + thresholds.KafkaConsumerMaxUncommittedRatio, + topic, + group.received, + group.committedOK, + group.commitError, + uncommitted, + )) + } + return findings +} + +func kafkaConsumerCompletionSpecForService(service string) (kafkaConsumerCompletionSpec, bool) { + switch service { + case "history": + return kafkaConsumerCompletionSpec{ + serviceName: "history", + messageMetric: "vehicle_history_kafka_messages_total", + commitMetric: "vehicle_history_kafka_commits_total", + }, true + case "realtime": + return kafkaConsumerCompletionSpec{ + serviceName: "realtime", + messageMetric: "vehicle_realtime_kafka_messages_total", + commitMetric: "vehicle_realtime_kafka_commits_total", + }, true + case "stat": + return kafkaConsumerCompletionSpec{ + serviceName: "stat", + messageMetric: "vehicle_stat_kafka_messages_total", + commitMetric: "vehicle_stat_kafka_commits_total", + }, true + case "identity": + return kafkaConsumerCompletionSpec{ + serviceName: "identity", + messageMetric: "vehicle_identity_writer_kafka_messages_total", + commitMetric: "vehicle_identity_writer_kafka_commits_total", + }, true + default: + return kafkaConsumerCompletionSpec{}, false + } +} + +func findingsForMessageContractMismatches(service string, parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.MessageContractMismatchMax < 0 { + return nil + } + specs := messageContractMismatchSpecs(service) + if len(specs) == 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + status := strings.TrimSpace(sample.labels["status"]) + if status == "" { + continue + } + for _, spec := range specs { + if sample.name != spec.metric || status != spec.status { + continue + } + if sample.value <= thresholds.MessageContractMismatchMax { + continue + } + key := strings.TrimSpace(sample.labels[spec.keyLabel]) + if key == "" { + key = "unknown" + } + findings = append(findings, fmt.Sprintf( + "%s message contract %s %.0f exceeds %.0f for %s %s", + spec.serviceLabel, + status, + sample.value, + thresholds.MessageContractMismatchMax, + spec.keyLabel, + key, + )) + } + } + sort.Strings(findings) + return findings +} + +type messageContractMismatchSpec struct { + serviceLabel string + metric string + status string + keyLabel string +} + +func messageContractMismatchSpecs(service string) []messageContractMismatchSpec { + switch service { + case "history": + return []messageContractMismatchSpec{ + {serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, + {serviceLabel: "history", metric: "vehicle_history_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, + } + case "realtime": + return []messageContractMismatchSpec{ + {serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, + {serviceLabel: "realtime", metric: "vehicle_realtime_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, + } + case "stat": + return []messageContractMismatchSpec{ + {serviceLabel: "stat", metric: "vehicle_stat_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, + } + case "identity": + return []messageContractMismatchSpec{ + {serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "event_kind_mismatch", keyLabel: "topic"}, + {serviceLabel: "identity", metric: "vehicle_identity_writer_kafka_messages_total", status: "protocol_topic_mismatch", keyLabel: "topic"}, + } + case "fast-writer": + return []messageContractMismatchSpec{ + {serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "event_kind_mismatch", keyLabel: "subject"}, + {serviceLabel: "fast writer", metric: "vehicle_fast_writer_messages_total", status: "protocol_topic_mismatch", keyLabel: "subject"}, + } + default: + return nil + } +} + +func findingsForIdentityWriterOwnership(gateway parsedMetricSet, identityWriter parsedMetricSet) []string { + if len(identityWriter.samples) == 0 { + return nil + } + for _, sample := range gateway.samples { + if sample.name != "vehicle_gateway_jt808_registration_gateway_writes_enabled" { + continue + } + if sample.value > 0 { + return []string{"jt808 registration has duplicate writers: gateway writes remain enabled while identity writer is active"} + } + return nil + } + return []string{"gateway jt808 registration ownership metric missing while identity writer is active"} +} + +func findingsForHistoryParsedFields(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.HistoryParsedFieldMinFrames <= 0 || thresholds.HistoryParsedFieldMaxMissingRatio <= 0 { + return nil + } + groups := map[string]*historyParsedFieldCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_history_parsed_fields_total" { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "UNKNOWN" + } + key := topic + "\x00" + protocol + group := groups[key] + if group == nil { + group = &historyParsedFieldCounters{} + groups[key] = group + } + switch sample.labels["status"] { + case "present": + group.present += sample.value + case "missing": + group.missing += sample.value + } + } + var keys []string + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + var findings []string + for _, key := range keys { + group := groups[key] + total := group.present + group.missing + if total < thresholds.HistoryParsedFieldMinFrames || total <= 0 { + continue + } + ratio := group.missing / total + if ratio <= thresholds.HistoryParsedFieldMaxMissingRatio { + continue + } + topic, protocol, _ := strings.Cut(key, "\x00") + findings = append(findings, fmt.Sprintf( + "history parsed fields missing ratio %.2f exceeds %.2f for topic %s protocol %s (total %.0f, present %.0f, missing %.0f)", + ratio, + thresholds.HistoryParsedFieldMaxMissingRatio, + topic, + protocol, + total, + group.present, + group.missing, + )) + } + return findings +} + +func findingsForHistoryInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.HistoryInvalidJSONMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_history_kafka_messages_total" || sample.labels["status"] != "invalid_json" { + continue + } + if sample.value <= thresholds.HistoryInvalidJSONMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "history invalid json %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.HistoryInvalidJSONMax, + topic, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForRealtimeInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RealtimeInvalidJSONMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_realtime_kafka_messages_total" || sample.labels["status"] != "invalid_json" { + continue + } + if sample.value <= thresholds.RealtimeInvalidJSONMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "realtime invalid json %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.RealtimeInvalidJSONMax, + topic, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForRealtimeStoreErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RealtimeStoreErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_realtime_store_updates_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.RealtimeStoreErrorMax { + continue + } + store := strings.TrimSpace(sample.labels["store"]) + if store == "" { + store = "unknown" + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "UNKNOWN" + } + findings = append(findings, fmt.Sprintf( + "realtime store update errors %.0f exceeds %.0f for store %s protocol %s", + sample.value, + thresholds.RealtimeStoreErrorMax, + store, + protocol, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForRealtimeAsyncQueue(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + if thresholds.RealtimeAsyncQueueDroppedMax >= 0 { + for _, sample := range parsed.samples { + if sample.name != "vehicle_realtime_async_queue_total" { + continue + } + status := sample.labels["status"] + if status != "dropped" && status != "closed" { + continue + } + if sample.value <= thresholds.RealtimeAsyncQueueDroppedMax { + continue + } + store := strings.TrimSpace(sample.labels["store"]) + if store == "" { + store = "unknown" + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "UNKNOWN" + } + findings = append(findings, fmt.Sprintf( + "realtime async queue %s %.0f exceeds %.0f for store %s protocol %s", + status, + sample.value, + thresholds.RealtimeAsyncQueueDroppedMax, + store, + protocol, + )) + } + } + if thresholds.RealtimeAsyncQueueMaxUsageRatio <= 0 { + sort.Strings(findings) + return findings + } + type asyncQueueSample struct { + labels map[string]string + depth float64 + } + queues := map[string]*asyncQueueSample{} + capacityByKey := map[string]float64{} + capacityByStore := map[string]float64{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_realtime_async_queue_depth": + key := metricLabelsKey(sample.labels) + item := queues[key] + if item == nil { + item = &asyncQueueSample{labels: sample.labels} + queues[key] = item + } + item.depth += sample.value + case "vehicle_realtime_async_queue_capacity": + key := metricLabelsKey(sample.labels) + capacityByKey[key] += sample.value + store := strings.TrimSpace(sample.labels["store"]) + if store != "" { + capacityByStore[store] += sample.value + } + } + } + var keys []string + for key := range queues { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + item := queues[key] + capacity := capacityByKey[key] + if capacity <= 0 { + capacity = capacityByStore[strings.TrimSpace(item.labels["store"])] + } + if capacity <= 0 || item.depth <= 0 { + continue + } + ratio := item.depth / capacity + if ratio <= thresholds.RealtimeAsyncQueueMaxUsageRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "realtime async queue usage %.2f exceeds %.2f (%s, depth %.0f, capacity %.0f)", + ratio, + thresholds.RealtimeAsyncQueueMaxUsageRatio, + key, + item.depth, + capacity, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForGatewayResponseQuality(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayResponseMinSamples <= 0 || thresholds.GatewayResponseMaxErrorRatio <= 0 { + return nil + } + groups := map[string]*gatewayResponseCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_response_total" { + continue + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "unknown" + } + group := groups[protocol] + if group == nil { + group = &gatewayResponseCounters{} + groups[protocol] = group + } + switch sample.labels["status"] { + case "ok": + group.ok += sample.value + group.total += sample.value + case "build_error": + group.buildError += sample.value + group.total += sample.value + case "write_error": + group.writeError += sample.value + group.total += sample.value + } + } + var protocols []string + for protocol := range groups { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + var findings []string + for _, protocol := range protocols { + group := groups[protocol] + if group.total < thresholds.GatewayResponseMinSamples || group.total <= 0 { + continue + } + errors := group.buildError + group.writeError + ratio := errors / group.total + if ratio <= thresholds.GatewayResponseMaxErrorRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway response error ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, errors %.0f, build_error %.0f, write_error %.0f)", + ratio, + thresholds.GatewayResponseMaxErrorRatio, + protocol, + group.total, + group.ok, + errors, + group.buildError, + group.writeError, + )) + } + return findings +} + +func findingsForGatewayParseQuality(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayParseMinFrames <= 0 || thresholds.GatewayParseMaxBadRatio <= 0 { + return nil + } + groups := map[string]*gatewayParseCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_frames_total" { + continue + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "unknown" + } + group := groups[protocol] + if group == nil { + group = &gatewayParseCounters{} + groups[protocol] = group + } + group.total += sample.value + switch envelope.ParseStatus(sample.labels["status"]) { + case envelope.ParseOK: + group.ok += sample.value + case envelope.ParsePartial: + group.partial += sample.value + case envelope.ParseBadFrame: + group.badFrame += sample.value + default: + group.other += sample.value + } + } + var protocols []string + for protocol := range groups { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + var findings []string + for _, protocol := range protocols { + group := groups[protocol] + if group.total < thresholds.GatewayParseMinFrames || group.total <= 0 { + continue + } + bad := group.partial + group.badFrame + group.other + ratio := bad / group.total + if ratio <= thresholds.GatewayParseMaxBadRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway parse bad ratio %.2f exceeds %.2f for protocol %s (total %.0f, ok %.0f, bad %.0f, partial %.0f, bad_frame %.0f, other %.0f)", + ratio, + thresholds.GatewayParseMaxBadRatio, + protocol, + group.total, + group.ok, + bad, + group.partial, + group.badFrame, + group.other, + )) + } + return findings +} + +func findingsForGatewayIdentityQuality(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayIdentityMinSamples <= 0 || thresholds.GatewayIdentityMaxUnresolvedRatio <= 0 { + return nil + } + groups := map[string]*gatewayIdentityCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_identity_total" { + continue + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "unknown" + } + group := groups[protocol] + if group == nil { + group = &gatewayIdentityCounters{} + groups[protocol] = group + } + group.total += sample.value + if sample.labels["status"] == "resolved" { + group.resolved += sample.value + } else { + group.unresolved += sample.value + } + } + var protocols []string + for protocol := range groups { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + var findings []string + for _, protocol := range protocols { + group := groups[protocol] + if group.total < thresholds.GatewayIdentityMinSamples || group.total <= 0 { + continue + } + ratio := group.unresolved / group.total + if ratio <= thresholds.GatewayIdentityMaxUnresolvedRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway identity unresolved ratio %.2f exceeds %.2f for protocol %s (total %.0f, unresolved %.0f, resolved %.0f)", + ratio, + thresholds.GatewayIdentityMaxUnresolvedRatio, + protocol, + group.total, + group.unresolved, + group.resolved, + )) + } + return findings +} + +func buildIdentityResolutionDiagnostics(parsed parsedMetricSet) *IdentityResolutionDiagnostics { + groups := map[string]*identityResolutionGroup{} + cacheEntries := map[string]int64{} + issues := map[string]IdentityIssueCount{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_gateway_identity_total": + protocol := normalizedMetricLabel(sample.labels, "protocol") + status := normalizedMetricLabel(sample.labels, "status") + group := identityResolutionGroupForProtocol(groups, protocol) + value := metricCount(sample.value) + group.total += value + switch status { + case "resolved": + group.resolved += value + case "unresolved": + group.unresolved += value + case "timeout": + group.timeout += value + case "error": + group.errorCount += value + default: + group.otherNonResolved += value + } + case "vehicle_gateway_identity_skips_total": + protocol := normalizedMetricLabel(sample.labels, "protocol") + group := identityResolutionGroupForProtocol(groups, protocol) + group.skipped += metricCount(sample.value) + case "vehicle_gateway_identity_issues_total": + issue := IdentityIssueCount{ + Protocol: normalizedMetricLabel(sample.labels, "protocol"), + Status: normalizedMetricLabel(sample.labels, "status"), + MessageID: normalizedMetricLabel(sample.labels, "message_id"), + Reason: normalizedMetricLabel(sample.labels, "reason"), + Count: metricCount(sample.value), + } + if issue.Count <= 0 { + continue + } + key := strings.Join([]string{issue.Protocol, issue.Status, issue.MessageID, issue.Reason}, "\x00") + existing := issues[key] + if existing.Count == 0 { + existing = issue + } else { + existing.Count += issue.Count + } + issues[key] = existing + case "vehicle_gateway_identity_cache_total": + protocol := normalizedMetricLabel(sample.labels, "protocol") + cacheStatus := normalizedMetricLabel(sample.labels, "cache_status") + group := identityResolutionGroupForProtocol(groups, protocol) + value := metricCount(sample.value) + if group.cacheStatusCounts == nil { + group.cacheStatusCounts = map[string]int64{} + } + group.cacheStatusCounts[cacheStatus] += value + if cacheStatus == "stale" { + group.staleCache += value + } + case "vehicle_gateway_identity_cache_entries": + cacheName := normalizedMetricLabel(sample.labels, "cache") + cacheEntries[cacheName] += metricCount(sample.value) + } + } + if len(groups) == 0 && len(cacheEntries) == 0 && len(issues) == 0 { + return nil + } + diagnostics := &IdentityResolutionDiagnostics{} + if len(cacheEntries) > 0 { + diagnostics.CacheEntries = cacheEntries + } + if len(issues) > 0 { + diagnostics.Issues = sortedIdentityIssues(issues) + } + var protocols []string + for protocol := range groups { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + for _, protocol := range protocols { + group := groups[protocol] + row := identityProtocolResolution(group) + diagnostics.Total += row.Total + diagnostics.SkippedCount += row.SkippedCount + diagnostics.ResolvedCount += row.ResolvedCount + diagnostics.NonResolvedCount += row.NonResolvedCount + diagnostics.UnresolvedCount += row.UnresolvedCount + diagnostics.TimeoutCount += row.TimeoutCount + diagnostics.ErrorCount += row.ErrorCount + diagnostics.OtherNonResolvedCount += row.OtherNonResolvedCount + diagnostics.StaleCacheCount += row.StaleCacheCount + diagnostics.Protocols = append(diagnostics.Protocols, row) + } + diagnostics.NonResolvedRatio = ratioInt64(diagnostics.NonResolvedCount, diagnostics.Total) + diagnostics.StaleCacheRatio = ratioInt64(diagnostics.StaleCacheCount, diagnostics.Total) + return diagnostics +} + +func sortedIdentityIssues(issues map[string]IdentityIssueCount) []IdentityIssueCount { + rows := make([]IdentityIssueCount, 0, len(issues)) + for _, row := range issues { + rows = append(rows, row) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].Protocol != rows[j].Protocol { + return rows[i].Protocol < rows[j].Protocol + } + if rows[i].Status != rows[j].Status { + return rows[i].Status < rows[j].Status + } + if rows[i].MessageID != rows[j].MessageID { + return rows[i].MessageID < rows[j].MessageID + } + return rows[i].Reason < rows[j].Reason + }) + return rows +} + +func identityProtocolResolution(group *identityResolutionGroup) IdentityProtocolResolution { + if group == nil { + return IdentityProtocolResolution{Protocol: "unknown"} + } + nonResolved := group.unresolved + group.timeout + group.errorCount + group.otherNonResolved + row := IdentityProtocolResolution{ + Protocol: group.protocol, + Total: group.total, + SkippedCount: group.skipped, + ResolvedCount: group.resolved, + NonResolvedCount: nonResolved, + UnresolvedCount: group.unresolved, + TimeoutCount: group.timeout, + ErrorCount: group.errorCount, + OtherNonResolvedCount: group.otherNonResolved, + NonResolvedRatio: ratioInt64(nonResolved, group.total), + StaleCacheCount: group.staleCache, + StaleCacheRatio: ratioInt64(group.staleCache, group.total), + CacheStatusCounts: group.cacheStatusCounts, + } + return row +} + +func identityResolutionGroupForProtocol(groups map[string]*identityResolutionGroup, protocol string) *identityResolutionGroup { + if protocol == "" { + protocol = "unknown" + } + group := groups[protocol] + if group == nil { + group = &identityResolutionGroup{protocol: protocol} + groups[protocol] = group + } + return group +} + +func normalizedMetricLabel(labels map[string]string, key string) string { + value := strings.TrimSpace(labels[key]) + if value == "" { + return "unknown" + } + return value +} + +func metricCount(value float64) int64 { + if value <= 0 { + return 0 + } + return int64(math.Round(value)) +} + +func ratioInt64(numerator int64, denominator int64) float64 { + if denominator <= 0 { + return 0 + } + return float64(numerator) / float64(denominator) +} + +func buildPipelineLatencyDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *PipelineLatencyDiagnostics { + specs := []pipelineLatencySpec{ + { + service: "gateway", + stage: "gateway_protocol_response_e2e", + metricName: "vehicle_gateway_response_e2e_recent_p99_ms", + samplesMetricName: "vehicle_gateway_response_e2e_recent_samples", + keyLabel: "protocol", + thresholdMS: thresholds.GatewayResponseRecentP99MS, + }, + { + service: "bridge", + stage: "bridge_kafka_e2e", + metricName: "vehicle_bridge_kafka_e2e_recent_p99_ms", + samplesMetricName: "vehicle_bridge_kafka_e2e_recent_samples", + keyLabel: "topic", + thresholdMS: thresholds.BridgeKafkaRecentE2EP99MS, + }, + { + service: "fast-writer", + stage: "fast_writer_redis_e2e", + metricName: "vehicle_fast_writer_redis_e2e_recent_p99_ms", + samplesMetricName: "vehicle_fast_writer_redis_e2e_recent_samples", + keyLabel: "subject", + thresholdMS: thresholds.FastWriterRedisRecentE2EP99MS, + }, + { + service: "history", + stage: "history_tdengine_write_e2e", + metricName: "vehicle_history_write_e2e_recent_p99_ms", + samplesMetricName: "vehicle_history_write_e2e_recent_samples", + keyLabel: "topic", + thresholdMS: thresholds.HistoryWriteRecentE2EP99MS, + }, + { + service: "stat", + stage: "stat_mysql_write_e2e", + metricName: "vehicle_stat_write_e2e_recent_p99_ms", + samplesMetricName: "vehicle_stat_write_e2e_recent_samples", + keyLabel: "topic", + thresholdMS: thresholds.StatWriteRecentE2EP99MS, + }, + } + var items []PipelineLatencyItem + for _, spec := range specs { + items = append(items, pipelineLatencyItemsForSpec(parsedByService[spec.service], spec, thresholds.MinHistogramSamples)...) + } + if len(items) == 0 { + return nil + } + sort.Slice(items, func(i, j int) bool { + if items[i].Stage != items[j].Stage { + return items[i].Stage < items[j].Stage + } + if items[i].Service != items[j].Service { + return items[i].Service < items[j].Service + } + if items[i].Topic != items[j].Topic { + return items[i].Topic < items[j].Topic + } + return items[i].Subject < items[j].Subject + }) + diagnostics := &PipelineLatencyDiagnostics{Items: items} + for _, item := range items { + diagnostics.TotalSamples += item.Samples + if item.P99MS > diagnostics.MaxP99MS { + diagnostics.MaxP99MS = item.P99MS + } + } + return diagnostics +} + +func pipelineLatencyItemsForSpec(parsed parsedMetricSet, spec pipelineLatencySpec, minSamples float64) []PipelineLatencyItem { + samplesByKey := map[string]float64{} + for _, sample := range parsed.samples { + if sample.name != spec.samplesMetricName { + continue + } + samplesByKey[metricLabelsKey(sample.labels)] = sample.value + } + var items []PipelineLatencyItem + for _, sample := range parsed.samples { + if sample.name != spec.metricName { + continue + } + key := metricLabelsKey(sample.labels) + row := PipelineLatencyItem{ + Stage: spec.stage, + Service: spec.service, + P99MS: sample.value, + Samples: metricCount(samplesByKey[key]), + ThresholdMS: spec.thresholdMS, + } + switch spec.keyLabel { + case "subject": + row.Subject = normalizedMetricLabel(sample.labels, spec.keyLabel) + default: + row.Topic = normalizedMetricLabel(sample.labels, spec.keyLabel) + } + row.Status = pipelineLatencyStatus(row.P99MS, float64(row.Samples), spec.thresholdMS, minSamples) + items = append(items, row) + } + return items +} + +func pipelineLatencyStatus(p99MS float64, samples float64, thresholdMS float64, minSamples float64) string { + if minSamples > 0 && samples < minSamples { + return "insufficient_samples" + } + if thresholdMS > 0 && p99MS > thresholdMS { + return "degraded" + } + return "ok" +} + +func findingsForGatewayIdentityCache(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayIdentityCacheEntriesMax <= 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_identity_cache_entries" || sample.value <= thresholds.GatewayIdentityCacheEntriesMax { + continue + } + cacheName := strings.TrimSpace(sample.labels["cache"]) + if cacheName == "" { + cacheName = "unknown" + } + if cacheName == "registration_write_queue" { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway identity cache entries %.0f exceeds %.0f for cache %s", + sample.value, + thresholds.GatewayIdentityCacheEntriesMax, + cacheName, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForGatewayIdentitySnapshot(parsed parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string { + if thresholds.GatewayIdentitySnapshotRequired <= 0 && thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 { + return nil + } + if !hasServiceInfo(parsed, "vehicle-gateway") { + return nil + } + ready, readyFound := parsed.totals["vehicle_gateway_identity_snapshot_ready"] + var findings []string + if thresholds.GatewayIdentitySnapshotRequired > 0 { + switch { + case !readyFound: + findings = append(findings, "gateway identity snapshot readiness metric missing") + case ready < thresholds.GatewayIdentitySnapshotRequired: + findings = append(findings, fmt.Sprintf("gateway identity snapshot ready %.0f below %.0f", ready, thresholds.GatewayIdentitySnapshotRequired)) + } + } + if thresholds.GatewayIdentitySnapshotMaxAgeSec <= 0 || !readyFound || ready <= 0 { + return findings + } + lastSuccess, ok := parsed.totals["vehicle_gateway_identity_snapshot_last_success_unix_seconds"] + if !ok || lastSuccess <= 0 { + return append(findings, "gateway identity snapshot last-success metric missing") + } + age := checkedAt.Unix() - int64(lastSuccess) + if age < 0 { + age = 0 + } + if float64(age) > thresholds.GatewayIdentitySnapshotMaxAgeSec { + findings = append(findings, fmt.Sprintf( + "gateway identity snapshot age %ds exceeds %.0fs", + age, + thresholds.GatewayIdentitySnapshotMaxAgeSec, + )) + } + return findings +} + +func findingsForGatewayRegistrationWriteQueue(parsed parsedMetricSet, thresholds Thresholds) []string { + depth := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_entries") + capacity := gatewayIdentityCacheMetric(parsed, "registration_write_queue", "vehicle_gateway_identity_cache_max_entries") + var findings []string + if thresholds.GatewayIdentityRegistrationWriteQueueDepthMax > 0 && depth > thresholds.GatewayIdentityRegistrationWriteQueueDepthMax { + findings = append(findings, fmt.Sprintf( + "gateway jt808 registration write queue depth %.0f exceeds %.0f", + depth, + thresholds.GatewayIdentityRegistrationWriteQueueDepthMax, + )) + } + if thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio > 0 && depth > 0 && capacity > 0 { + ratio := depth / capacity + if ratio > thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio { + findings = append(findings, fmt.Sprintf( + "gateway jt808 registration write queue usage %.2f exceeds %.2f (depth %.0f, capacity %.0f)", + ratio, + thresholds.GatewayIdentityRegistrationWriteQueueMaxUsageRatio, + depth, + capacity, + )) + } + } + sort.Strings(findings) + return findings +} + +func gatewayIdentityCacheMetric(parsed parsedMetricSet, cacheName string, metricName string) float64 { + cacheName = strings.TrimSpace(cacheName) + if cacheName == "" || metricName == "" { + return 0 + } + var total float64 + for _, sample := range parsed.samples { + if sample.name != metricName || strings.TrimSpace(sample.labels["cache"]) != cacheName { + continue + } + total += sample.value + } + return total +} + +func findingsForGatewayRegistrationWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayIdentityRegistrationWriteErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_jt808_registration_write_total" { + continue + } + status := strings.TrimSpace(sample.labels["status"]) + if status == "" || status == "ok" { + continue + } + if sample.value <= thresholds.GatewayIdentityRegistrationWriteErrorMax { + continue + } + mode := strings.TrimSpace(sample.labels["mode"]) + if mode == "" { + mode = "unknown" + } + findings = append(findings, fmt.Sprintf( + "gateway jt808 registration write %s count %.0f exceeds %.0f for mode %s", + status, + sample.value, + thresholds.GatewayIdentityRegistrationWriteErrorMax, + mode, + )) + } + sort.Strings(findings) + return findings +} + +func jt808RegistrationWriteErrors(parsed parsedMetricSet) float64 { + var total float64 + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_jt808_registration_write_total" { + continue + } + status := strings.TrimSpace(sample.labels["status"]) + if status == "" || status == "ok" { + continue + } + total += sample.value + } + return total +} + +func findingsForGatewayIdentityLocationTouchFailures(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayIdentityLocationTouchFailureMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_gateway_identity_cache_entries" || strings.TrimSpace(sample.labels["cache"]) != "location_touch_failure" { + continue + } + if sample.value <= thresholds.GatewayIdentityLocationTouchFailureMax { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway identity location touch failure cache %.0f exceeds %.0f; jt808 registration writes are failing or in protective backoff", + sample.value, + thresholds.GatewayIdentityLocationTouchFailureMax, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForGatewayIdentityStaleCache(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayIdentityStaleMinSamples <= 0 || thresholds.GatewayIdentityMaxStaleCacheRatio <= 0 { + return nil + } + totalsByProtocol := map[string]float64{} + staleByProtocol := map[string]float64{} + for _, sample := range parsed.samples { + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "unknown" + } + switch sample.name { + case "vehicle_gateway_identity_total": + totalsByProtocol[protocol] += sample.value + case "vehicle_gateway_identity_cache_total": + if sample.labels["cache_status"] == "stale" { + staleByProtocol[protocol] += sample.value + } + } + } + var protocols []string + for protocol := range totalsByProtocol { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + var findings []string + for _, protocol := range protocols { + total := totalsByProtocol[protocol] + if total < thresholds.GatewayIdentityStaleMinSamples || total <= 0 { + continue + } + stale := staleByProtocol[protocol] + ratio := stale / total + if ratio <= thresholds.GatewayIdentityMaxStaleCacheRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "gateway identity stale cache ratio %.2f exceeds %.2f for protocol %s (total %.0f, stale %.0f)", + ratio, + thresholds.GatewayIdentityMaxStaleCacheRatio, + protocol, + total, + stale, + )) + } + return findings +} + +func findingsForBridgeRouteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.BridgeRouteErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_bridge_messages_total" || sample.labels["status"] != "route_error" { + continue + } + if sample.value <= thresholds.BridgeRouteErrorMax { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" { + subject = "unknown" + } + findings = append(findings, fmt.Sprintf( + "bridge route errors %.0f exceeds %.0f for subject %s", + sample.value, + thresholds.BridgeRouteErrorMax, + subject, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForBridgeKafkaWriteErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.BridgeKafkaWriteErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_bridge_kafka_writes_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.BridgeKafkaWriteErrorMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "bridge kafka write errors %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.BridgeKafkaWriteErrorMax, + topic, + )) + } + sort.Strings(findings) + return findings +} + +type bridgeMessageCounters struct { + received float64 + ackOK float64 + droppedRouteError float64 +} + +func findingsForBridgeMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.BridgeMessageMinReceived <= 0 || thresholds.BridgeMessageMaxUnackedRatio <= 0 { + return nil + } + groups := map[string]*bridgeMessageCounters{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_bridge_messages_total", "vehicle_bridge_nats_acks_total": + default: + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" || subject == "mixed" || subject == "unknown" { + continue + } + group := groups[subject] + if group == nil { + group = &bridgeMessageCounters{} + groups[subject] = group + } + switch { + case sample.name == "vehicle_bridge_messages_total" && sample.labels["status"] == "received": + group.received += sample.value + case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "ok": + group.ackOK += sample.value + case sample.name == "vehicle_bridge_nats_acks_total" && sample.labels["status"] == "dropped_route_error": + group.droppedRouteError += sample.value + } + } + var subjects []string + for subject := range groups { + subjects = append(subjects, subject) + } + sort.Strings(subjects) + var findings []string + for _, subject := range subjects { + group := groups[subject] + if group.received < thresholds.BridgeMessageMinReceived { + continue + } + completed := group.ackOK + group.droppedRouteError + unacked := group.received - completed + if unacked < 0 { + unacked = 0 + } + ratio := unacked / group.received + if ratio <= thresholds.BridgeMessageMaxUnackedRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "bridge unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ack ok %.0f, dropped route error %.0f, unacked %.0f", + ratio, + thresholds.BridgeMessageMaxUnackedRatio, + subject, + group.received, + group.ackOK, + group.droppedRouteError, + unacked, + )) + } + return findings +} + +func findingsForTopicFlow(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string { + var findings []string + findings = append(findings, findingsForGatewayFieldFlow(parsedByService["gateway"], thresholds)...) + findings = append(findings, findingsForGatewayFieldCounts(parsedByService["gateway"], thresholds)...) + findings = append(findings, findingsForGatewayToBridgeFlow(parsedByService["gateway"], parsedByService["bridge"], thresholds)...) + findings = append(findings, findingsForGatewayToFastWriterFlow(parsedByService["gateway"], parsedByService["fast-writer"], thresholds)...) + findings = append(findings, findingsForBridgeRawFanout(parsedByService["bridge"], parsedByService["history"], parsedByService["realtime"], thresholds)...) + findings = append(findings, findingsForBridgeToStatFieldFlow(parsedByService["bridge"], parsedByService["stat"], thresholds)...) + return findings +} + +func findingsForGatewayFieldFlow(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayFieldMinRaw <= 0 { + return nil + } + groups := gatewayFieldFlowGroups(parsed) + protocols := gatewayFieldFlowProtocols(groups) + var findings []string + for _, protocol := range protocols { + group := groups[protocol] + realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters) + if realtimeCandidates < thresholds.GatewayFieldMinRaw { + continue + } + if group.fieldsOK <= 0 { + findings = append(findings, fmt.Sprintf( + "gateway fields missing for protocol %s: raw ok %.0f, realtime candidates %.0f, fields ok %.0f, skipped non-realtime %.0f, skipped missing fields %.0f, publish errors %.0f", + protocol, + group.rawOK, + realtimeCandidates, + group.fieldsOK, + group.fieldsSkippedNonRealtime, + group.fieldsSkippedMissing, + group.fieldsPublishError, + )) + continue + } + missingRatio := (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates + if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio { + findings = append(findings, fmt.Sprintf( + "gateway fields missing ratio %.2f exceeds %.2f for protocol %s: realtime candidates %.0f, fields ok %.0f, skipped missing fields %.0f, publish errors %.0f", + missingRatio, + thresholds.GatewayFieldMaxMissingRatio, + protocol, + realtimeCandidates, + group.fieldsOK, + group.fieldsSkippedMissing, + group.fieldsPublishError, + )) + } + } + return findings +} + +func findingsForGatewayFieldCounts(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayFieldMinCount <= 0 { + return nil + } + minPublished := thresholds.GatewayFieldMinRaw + if minPublished <= 0 { + minPublished = 1 + } + groups := gatewayFieldFlowGroups(parsed) + protocols := gatewayFieldFlowProtocols(groups) + var findings []string + for _, protocol := range protocols { + group := groups[protocol] + fieldsAccepted := group.fieldsPublished + group.fieldsDelegated + if fieldsAccepted < minPublished { + continue + } + if !group.hasLatestCount { + findings = append(findings, fmt.Sprintf( + "gateway fields count metric missing for protocol %s after %.0f published fields events", + protocol, + fieldsAccepted, + )) + continue + } + if group.latestCount < thresholds.GatewayFieldMinCount { + findings = append(findings, fmt.Sprintf( + "gateway fields count %.0f below %.0f for protocol %s after %.0f published fields events", + group.latestCount, + thresholds.GatewayFieldMinCount, + protocol, + fieldsAccepted, + )) + } + } + return findings +} + +func buildGatewayFieldsDiagnostics(parsed parsedMetricSet, thresholds Thresholds) *GatewayFieldsDiagnostics { + groups := gatewayFieldFlowGroups(parsed) + protocols := gatewayFieldFlowProtocols(groups) + if len(protocols) == 0 { + return nil + } + out := &GatewayFieldsDiagnostics{Protocols: make([]GatewayProtocolFields, 0, len(protocols))} + for _, protocol := range protocols { + group := groups[protocol] + realtimeCandidates := gatewayRealtimeCandidates(group.gatewayPublishCounters) + missingRatio := 0.0 + if realtimeCandidates > 0 { + missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates + } + status, reason := gatewayFieldDiagnosticsStatus(group, realtimeCandidates, thresholds) + var latestCount *float64 + if group.hasLatestCount { + value := group.latestCount + latestCount = &value + } + out.Protocols = append(out.Protocols, GatewayProtocolFields{ + Protocol: protocol, + RawOK: group.rawOK, + RealtimeCandidates: realtimeCandidates, + FieldsOK: group.fieldsOK, + FieldsPublished: group.fieldsPublished, + FieldsDelegated: group.fieldsDelegated, + FieldsSkippedNonRealtime: group.fieldsSkippedNonRealtime, + FieldsSkippedMissing: group.fieldsSkippedMissing, + FieldsPublishError: group.fieldsPublishError, + MissingRatio: missingRatio, + LatestFieldCount: latestCount, + Status: status, + Reason: reason, + }) + } + return out +} + +func gatewayFieldDiagnosticsStatus(group *gatewayFieldFlowGroup, realtimeCandidates float64, thresholds Thresholds) (string, string) { + if thresholds.GatewayFieldMinRaw > 0 && realtimeCandidates >= thresholds.GatewayFieldMinRaw { + if group.fieldsOK <= 0 { + return "degraded", "missing_fields_publish" + } + missingRatio := 0.0 + if realtimeCandidates > 0 { + missingRatio = (group.fieldsSkippedMissing + group.fieldsPublishError) / realtimeCandidates + } + if thresholds.GatewayFieldMaxMissingRatio > 0 && missingRatio > thresholds.GatewayFieldMaxMissingRatio { + return "degraded", "high_missing_ratio" + } + } + minPublished := thresholds.GatewayFieldMinRaw + if minPublished <= 0 { + minPublished = 1 + } + if thresholds.GatewayFieldMinCount > 0 && group.fieldsPublished+group.fieldsDelegated >= minPublished { + if !group.hasLatestCount { + return "degraded", "missing_field_count_metric" + } + if group.latestCount < thresholds.GatewayFieldMinCount { + return "degraded", "low_field_count" + } + } + return "ok", "" +} + +func gatewayFieldFlowGroups(parsed parsedMetricSet) map[string]*gatewayFieldFlowGroup { + groups := map[string]*gatewayFieldFlowGroup{} + groupForProtocol := func(protocol string) *gatewayFieldFlowGroup { + protocol = strings.TrimSpace(protocol) + if protocol == "" { + protocol = "unknown" + } + group := groups[protocol] + if group == nil { + group = &gatewayFieldFlowGroup{} + groups[protocol] = group + } + return group + } + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_gateway_publish_total": + if sample.labels["status"] != "ok" && sample.labels["status"] != "delegated" { + continue + } + group := groupForProtocol(sample.labels["protocol"]) + switch sample.labels["kind"] { + case "raw": + group.rawOK += sample.value + case "fields": + group.fieldsOK += sample.value + } + case "vehicle_gateway_fields_total": + group := groupForProtocol(sample.labels["protocol"]) + switch sample.labels["status"] { + case "published": + group.fieldsPublished += sample.value + case "delegated_to_bridge": + group.fieldsDelegated += sample.value + case "skipped_non_realtime": + group.fieldsSkippedNonRealtime += sample.value + case "skipped_missing_fields": + group.fieldsSkippedMissing += sample.value + case "publish_error": + group.fieldsPublishError += sample.value + } + case "vehicle_gateway_fields_count": + if sample.labels["status"] != "published" && sample.labels["status"] != "delegated_to_bridge" { + continue + } + group := groupForProtocol(sample.labels["protocol"]) + group.latestCount = sample.value + group.hasLatestCount = true + } + } + return groups +} + +func gatewayFieldFlowProtocols(groups map[string]*gatewayFieldFlowGroup) []string { + var protocols []string + for protocol := range groups { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + return protocols +} + +func gatewayRealtimeCandidates(group gatewayPublishCounters) float64 { + realtimeCandidates := group.rawOK - group.fieldsSkippedNonRealtime + if realtimeCandidates < 0 { + return 0 + } + return realtimeCandidates +} + +func findingsForGatewayToFastWriterFlow(gateway parsedMetricSet, fastWriter parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayFastWriterMinRawPublish <= 0 { + return nil + } + gatewayPublishes := gatewayPublishGroups(gateway.samples) + fastWriterMessages := metricValuesBySubject(fastWriter.samples, "vehicle_fast_writer_messages_total", "received", "vehicle.raw.go.") + var findings []string + for _, group := range gatewayPublishes { + if group.kind != "raw" || group.count < thresholds.GatewayFastWriterMinRawPublish { + continue + } + subject, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind) + if !ok { + continue + } + if fastWriterMessages[subject] <= 0 { + findings = append(findings, fmt.Sprintf( + "fast writer received messages missing for gateway raw publish protocol %s: gateway ok %.0f, expected subject %s, fast writer received %.0f", + group.protocol, + group.count, + subject, + fastWriterMessages[subject], + )) + } + } + return findings +} + +type gatewayPublishGroup struct { + protocol string + kind string + count float64 +} + +func findingsForGatewayToBridgeFlow(gateway parsedMetricSet, bridge parsedMetricSet, thresholds Thresholds) []string { + if thresholds.GatewayBridgeMinPublish <= 0 { + return nil + } + gatewayPublishes := gatewayPublishGroups(gateway.samples) + bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.") + var findings []string + for _, group := range gatewayPublishes { + if group.count < thresholds.GatewayBridgeMinPublish { + continue + } + topic, ok := expectedKafkaTopicForGatewayPublish(group.protocol, group.kind) + if !ok { + findings = append(findings, fmt.Sprintf( + "gateway %s publish topic mapping missing for protocol %s: gateway ok %.0f", + group.kind, + group.protocol, + group.count, + )) + continue + } + if bridgeWrites[topic] <= 0 { + findings = append(findings, fmt.Sprintf( + "bridge writes missing for gateway %s publish protocol %s: gateway ok %.0f, expected topic %s, bridge writes ok %.0f", + group.kind, + group.protocol, + group.count, + topic, + bridgeWrites[topic], + )) + } + } + return findings +} + +func gatewayPublishGroups(samples []metricSample) []gatewayPublishGroup { + groups := map[string]*gatewayPublishGroup{} + for _, sample := range samples { + if sample.name != "vehicle_gateway_publish_total" || (sample.labels["status"] != "ok" && sample.labels["status"] != "delegated") { + continue + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if protocol == "" { + protocol = "unknown" + } + kind := strings.TrimSpace(sample.labels["kind"]) + if kind == "" { + kind = "unknown" + } + key := protocol + "\x00" + kind + group := groups[key] + if group == nil { + group = &gatewayPublishGroup{protocol: protocol, kind: kind} + groups[key] = group + } + group.count += sample.value + } + var keys []string + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]gatewayPublishGroup, 0, len(keys)) + for _, key := range keys { + out = append(out, *groups[key]) + } + return out +} + +func expectedKafkaTopicForGatewayPublish(protocol string, kind string) (string, bool) { + switch kind { + case "raw": + switch envelope.Protocol(protocol) { + case envelope.ProtocolGB32960: + return topics.RawGB32960, true + case envelope.ProtocolJT808: + return topics.RawJT808, true + case envelope.ProtocolYutongMQTT: + return topics.RawYutongMQTT, true + default: + return "", false + } + case "fields": + switch envelope.Protocol(protocol) { + case envelope.ProtocolGB32960: + return topics.FieldsGB32960, true + case envelope.ProtocolJT808: + return topics.FieldsJT808, true + case envelope.ProtocolYutongMQTT: + return topics.FieldsYutongMQTT, true + default: + return "", false + } + default: + return "", false + } +} + +func findingsForBridgeRawFanout(bridge parsedMetricSet, history parsedMetricSet, realtime parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RawFanoutMinBridge <= 0 { + return nil + } + bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.raw.go.") + historyConsumers := metricValuesByTopic(history.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.") + historyReceived := metricValuesByTopic(history.samples, "vehicle_history_kafka_messages_total", "received", "vehicle.raw.go.") + historyWrites := metricValuesByTopic(history.samples, "vehicle_history_writes_total", "ok", "vehicle.raw.go.") + realtimeConsumers := metricValuesByTopic(realtime.samples, "vehicle_kafka_consumer_info", "", "vehicle.raw.go.") + realtimeReceived := metricValuesByTopic(realtime.samples, "vehicle_realtime_kafka_messages_total", "received", "vehicle.raw.go.") + realtimeUpdates := metricValuesByTopic(realtime.samples, "vehicle_realtime_updates_total", "ok", "vehicle.raw.go.") + var topics []string + for topic := range bridgeWrites { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + writes := bridgeWrites[topic] + if writes < thresholds.RawFanoutMinBridge { + continue + } + if historyConsumers[topic] <= 0 { + findings = append(findings, fmt.Sprintf("history consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes)) + } else if historyReceived[topic] <= 0 { + findings = append(findings, fmt.Sprintf("history messages missing for %s: bridge raw writes ok %.0f, history received %.0f", topic, writes, historyReceived[topic])) + } else if historyWrites[topic] <= 0 { + findings = append(findings, fmt.Sprintf("history writes missing for %s: bridge raw writes ok %.0f, history received %.0f, history writes ok %.0f", topic, writes, historyReceived[topic], historyWrites[topic])) + } + if realtimeConsumers[topic] <= 0 { + findings = append(findings, fmt.Sprintf("realtime consumer topic missing for %s: bridge raw writes ok %.0f", topic, writes)) + } else if realtimeReceived[topic] <= 0 { + findings = append(findings, fmt.Sprintf("realtime messages missing for %s: bridge raw writes ok %.0f, realtime received %.0f", topic, writes, realtimeReceived[topic])) + } else if realtimeUpdates[topic] <= 0 { + findings = append(findings, fmt.Sprintf("realtime updates missing for %s: bridge raw writes ok %.0f, realtime received %.0f, realtime updates ok %.0f", topic, writes, realtimeReceived[topic], realtimeUpdates[topic])) + } + } + return findings +} + +func findingsForBridgeToStatFieldFlow(bridge parsedMetricSet, stat parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatTopicMinBridge <= 0 { + return nil + } + bridgeWrites := metricValuesByTopic(bridge.samples, "vehicle_bridge_kafka_writes_total", "ok", "vehicle.fields.go.") + statReceived := metricValuesByTopic(stat.samples, "vehicle_stat_kafka_messages_total", "received", "vehicle.fields.go.") + statConsumers := metricValuesByTopic(stat.samples, "vehicle_kafka_consumer_info", "", "vehicle.fields.go.") + var topics []string + for topic := range bridgeWrites { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + writes := bridgeWrites[topic] + if writes < thresholds.StatTopicMinBridge { + continue + } + if statConsumers[topic] <= 0 { + findings = append(findings, fmt.Sprintf("stat consumer topic missing for %s: bridge writes ok %.0f", topic, writes)) + continue + } + if statReceived[topic] <= 0 { + findings = append(findings, fmt.Sprintf("stat messages missing for %s: bridge writes ok %.0f, stat received %.0f", topic, writes, statReceived[topic])) + } + } + return findings +} + +func findingsForRequiredConsumerTopics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) []string { + if len(thresholds.RequiredConsumerTopics) == 0 { + return nil + } + var services []string + for service := range thresholds.RequiredConsumerTopics { + services = append(services, service) + } + sort.Strings(services) + var findings []string + for _, service := range services { + required := normalizeTopicSet(thresholds.RequiredConsumerTopics[service]) + if len(required) == 0 { + continue + } + seen := kafkaConsumerTopicSet(parsedByService[service]) + var topics []string + for topic := range required { + topics = append(topics, topic) + } + sort.Strings(topics) + for _, topic := range topics { + if seen[topic] > 0 { + continue + } + findings = append(findings, fmt.Sprintf("%s required kafka consumer topic missing: %s", service, topic)) + } + } + return findings +} + +func normalizeTopicSet(values []string) map[string]struct{} { + out := map[string]struct{}{} + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + out[value] = struct{}{} + } + return out +} + +func kafkaConsumerTopicSet(parsed parsedMetricSet) map[string]float64 { out := map[string]float64{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_kafka_consumer_info" || sample.value <= 0 { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + continue + } + out[topic] += sample.value + } + return out +} + +func metricValuesByTopic(samples []metricSample, metricName string, status string, topicPrefix string) map[string]float64 { + out := map[string]float64{} + for _, sample := range samples { + if sample.name != metricName { + continue + } + if status != "" && sample.labels["status"] != status { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" || (topicPrefix != "" && !strings.HasPrefix(topic, topicPrefix)) { + continue + } + out[topic] += sample.value + } + return out +} + +func metricValuesBySubject(samples []metricSample, metricName string, status string, subjectPrefix string) map[string]float64 { + out := map[string]float64{} + for _, sample := range samples { + if sample.name != metricName { + continue + } + if status != "" && sample.labels["status"] != status { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" || (subjectPrefix != "" && !strings.HasPrefix(subject, subjectPrefix)) { + continue + } + out[subject] += sample.value + } + return out +} + +func findingsForLastActivity(parsedByService map[string]parsedMetricSet, thresholds Thresholds, checkedAt time.Time) []string { + if thresholds.LastActivityStaleSec <= 0 { + return nil + } + specs := []lastActivitySpec{ + {service: "gateway", metric: "vehicle_gateway_last_frame_unix_seconds", label: "gateway last frame", status: "OK"}, + {service: "gateway", metric: "vehicle_gateway_last_publish_unix_seconds", label: "gateway last publish", status: "ok"}, + {service: "bridge", metric: "vehicle_bridge_last_message_unix_seconds", label: "bridge last message", status: "received"}, + {service: "bridge", metric: "vehicle_bridge_last_kafka_write_unix_seconds", label: "bridge last kafka write", status: "ok"}, + {service: "bridge", metric: "vehicle_bridge_last_ack_unix_seconds", label: "bridge last ack", status: "ok"}, + {service: "fast-writer", metric: "vehicle_fast_writer_last_message_unix_seconds", label: "fast writer last message", status: "received"}, + {service: "fast-writer", metric: "vehicle_fast_writer_last_stage_unix_seconds", label: "fast writer last stage", status: "ok"}, + {service: "history", metric: "vehicle_history_last_message_unix_seconds", label: "history last message", status: "received"}, + {service: "history", metric: "vehicle_history_last_write_unix_seconds", label: "history last write", status: "ok"}, + {service: "history", metric: "vehicle_history_last_commit_unix_seconds", label: "history last commit", status: "ok"}, + {service: "stat", metric: "vehicle_stat_last_message_unix_seconds", label: "stat last message", status: "received"}, + {service: "stat", metric: "vehicle_stat_last_write_unix_seconds", label: "stat last write", status: "ok"}, + {service: "stat", metric: "vehicle_stat_last_commit_unix_seconds", label: "stat last commit", status: "ok"}, + {service: "realtime", metric: "vehicle_realtime_last_message_unix_seconds", label: "realtime last message", status: "received"}, + {service: "realtime", metric: "vehicle_realtime_last_update_unix_seconds", label: "realtime last update", status: "ok"}, + {service: "realtime", metric: "vehicle_realtime_last_store_update_unix_seconds", label: "realtime last store update", status: "ok"}, + {service: "realtime", metric: "vehicle_realtime_last_commit_unix_seconds", label: "realtime last commit", status: "ok"}, + {service: "identity", metric: "vehicle_identity_writer_last_message_unix_seconds", label: "identity last message", status: "received"}, + {service: "identity", metric: "vehicle_identity_writer_last_write_unix_seconds", label: "identity last write", status: "ok"}, + {service: "identity", metric: "vehicle_identity_writer_last_commit_unix_seconds", label: "identity last commit", status: "ok"}, + } + var findings []string + for _, spec := range specs { + findings = append(findings, staleActivityFindings(parsedByService[spec.service], spec, thresholds.LastActivityStaleSec, checkedAt)...) + } + sort.Strings(findings) + return findings +} + +type lastActivitySpec struct { + service string + metric string + label string + status string +} + +func staleActivityFindings(parsed parsedMetricSet, spec lastActivitySpec, thresholdSec float64, checkedAt time.Time) []string { + var findings []string + for _, sample := range parsed.samples { + if sample.name != spec.metric { + continue + } + if spec.status != "" && sample.labels["status"] != spec.status { + continue + } + if sample.value <= 0 { + continue + } + if isRollupLastActivity(sample) { + continue + } + ageSec := checkedAt.Sub(time.Unix(int64(sample.value), 0).UTC()).Seconds() + if ageSec <= thresholdSec { + continue + } + findings = append(findings, fmt.Sprintf("%s stale %.0fs exceeds %.0fs (%s)", spec.label, ageSec, thresholdSec, metricLabelsKey(sample.labels))) + } + return findings +} + +func isRollupLastActivity(sample metricSample) bool { + for _, label := range []string{"subject", "topic"} { + switch strings.TrimSpace(sample.labels[label]) { + case "mixed", "unknown": + return true + } + } + return false +} + +type statSampleCounters struct { + writesOK float64 + found float64 + written float64 + skippedMissingFields float64 + skippedMissingVIN float64 + skippedMissingMileage float64 + skippedNonMileageFrame float64 + skippedNonPositiveMileage float64 + skippedMissingTime float64 + futureAdjusted float64 + skippedSameMileage float64 + skippedMissingSource float64 +} + +type statSourceCounters struct { + found float64 + attempted float64 + written float64 + skippedThrottled float64 + skippedMissingEndpoint float64 + skippedUnmanaged float64 +} + +type statProjectionCounters struct { + samplesWritten float64 + attempted float64 + written float64 + skippedThrottled float64 +} + +type protocolFlowStatCounters struct { + samples statSampleCounters + sources statSourceCounters + projections statProjectionCounters +} + +func buildProtocolFlowDiagnostics(parsedByService map[string]parsedMetricSet, thresholds Thresholds) *ProtocolFlowDiagnostics { + gatewayGroups := gatewayFieldFlowGroups(parsedByService["gateway"]) + identityDiagnostics := buildIdentityResolutionDiagnostics(parsedByService["gateway"]) + identityByProtocol := map[string]IdentityProtocolResolution{} + if identityDiagnostics != nil { + for _, row := range identityDiagnostics.Protocols { + identityByProtocol[row.Protocol] = row + } + } + statGroups := protocolFlowStatGroups(parsedByService["stat"]) + + protocolSet := map[string]struct{}{} + for protocol := range gatewayGroups { + protocolSet[protocol] = struct{}{} + } + for protocol := range identityByProtocol { + protocolSet[protocol] = struct{}{} + } + for protocol := range statGroups { + protocolSet[protocol] = struct{}{} + } + if len(protocolSet) == 0 { + return nil + } + protocols := make([]string, 0, len(protocolSet)) + for protocol := range protocolSet { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + + out := &ProtocolFlowDiagnostics{Protocols: make([]ProtocolFlowProtocol, 0, len(protocols))} + for _, protocol := range protocols { + gatewayGroup := gatewayGroups[protocol] + statGroup := statGroups[protocol] + identityRow := identityByProtocol[protocol] + row := ProtocolFlowProtocol{Protocol: protocol} + if gatewayGroup != nil { + row.RawOK = gatewayGroup.rawOK + row.RealtimeCandidates = gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters) + row.FieldsPublished = gatewayGroup.fieldsPublished + if gatewayGroup.hasLatestCount { + value := gatewayGroup.latestCount + row.LatestFieldCount = &value + } + } + row.IdentityTotal = identityRow.Total + row.IdentitySkipped = identityRow.SkippedCount + row.IdentityResolved = identityRow.ResolvedCount + row.IdentityNonResolved = identityRow.NonResolvedCount + row.IdentityNonResolvedRatio = identityRow.NonResolvedRatio + if statGroup != nil { + row.StatWritesOK = statGroup.samples.writesOK + row.StatMileageCandidateWrites = statMileageCandidateWrites(statGroup.samples) + row.StatSamplesFound = statGroup.samples.found + row.StatSamplesWritten = statGroup.samples.written + row.StatSamplesActionableSkipped = statActionableSkips(statGroup.samples) + row.StatSkippedMissingFields = statGroup.samples.skippedMissingFields + row.StatSkippedMissingVIN = statGroup.samples.skippedMissingVIN + row.StatSkippedMissingMileage = statGroup.samples.skippedMissingMileage + row.StatSkippedNonMileageFrame = statGroup.samples.skippedNonMileageFrame + row.StatSkippedNonPositiveMileage = statGroup.samples.skippedNonPositiveMileage + row.StatSkippedMissingTime = statGroup.samples.skippedMissingTime + row.StatEventTimeFutureAdjusted = statGroup.samples.futureAdjusted + row.StatSkippedMissingSource = statGroup.samples.skippedMissingSource + row.StatSkippedSameMileage = statGroup.samples.skippedSameMileage + row.StatSourceWritten = statGroup.sources.written + row.StatSourceThrottled = statGroup.sources.skippedThrottled + row.StatSourceMissingEndpoint = statGroup.sources.skippedMissingEndpoint + row.StatSourceSkippedUnmanaged = statGroup.sources.skippedUnmanaged + row.StatProjectionAttempted = statGroup.projections.attempted + row.StatProjectionWritten = statGroup.projections.written + row.StatProjectionThrottled = statGroup.projections.skippedThrottled + } + row.Status, row.Reason = protocolFlowStatus(gatewayGroup, identityRow, statGroup, thresholds) + out.Protocols = append(out.Protocols, row) + } + return out +} + +func protocolFlowStatGroups(parsed parsedMetricSet) map[string]*protocolFlowStatCounters { + groups := map[string]*protocolFlowStatCounters{} + groupForSample := func(sample metricSample) *protocolFlowStatCounters { + protocol := protocolForFieldsMetricSample(sample) + group := groups[protocol] + if group == nil { + group = &protocolFlowStatCounters{} + groups[protocol] = group + } + return group + } + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_stat_writes_total": + if sample.labels["status"] == "ok" { + groupForSample(sample).samples.writesOK += sample.value + } + case "vehicle_stat_samples_total": + group := groupForSample(sample) + switch sample.labels["status"] { + case "found": + group.samples.found += sample.value + group.sources.found += sample.value + case "written": + group.samples.written += sample.value + group.projections.samplesWritten += sample.value + case "skipped_missing_fields": + group.samples.skippedMissingFields += sample.value + case "skipped_missing_vin": + group.samples.skippedMissingVIN += sample.value + case "skipped_missing_mileage": + group.samples.skippedMissingMileage += sample.value + case "skipped_non_mileage_frame": + group.samples.skippedNonMileageFrame += sample.value + case "skipped_non_positive_mileage": + group.samples.skippedNonPositiveMileage += sample.value + case "skipped_missing_time": + group.samples.skippedMissingTime += sample.value + case "event_time_future_adjusted": + group.samples.futureAdjusted += sample.value + case "skipped_same_mileage": + group.samples.skippedSameMileage += sample.value + case "skipped_missing_source": + group.samples.skippedMissingSource += sample.value + } + case "vehicle_stat_sources_total": + group := groupForSample(sample) + switch sample.labels["status"] { + case "attempted": + group.sources.attempted += sample.value + case "written": + group.sources.written += sample.value + case "skipped_throttled": + group.sources.skippedThrottled += sample.value + case "skipped_missing_endpoint": + group.sources.skippedMissingEndpoint += sample.value + case "skipped_unmanaged": + group.sources.skippedUnmanaged += sample.value + } + case "vehicle_stat_projections_total": + group := groupForSample(sample) + switch sample.labels["status"] { + case "attempted": + group.projections.attempted += sample.value + case "written": + group.projections.written += sample.value + case "skipped_throttled": + group.projections.skippedThrottled += sample.value + } + } + } + return groups +} + +func protocolForFieldsMetricSample(sample metricSample) string { + if protocol := strings.TrimSpace(sample.labels["protocol"]); protocol != "" { + return protocol + } + if protocol, ok := topics.ProtocolForKnownFieldsTopic(sample.labels["topic"]); ok { + return protocol + } + return "unknown" +} + +func statActionableSkips(group statSampleCounters) float64 { + return group.skippedMissingFields + + group.skippedMissingVIN + + group.skippedMissingMileage + + group.skippedNonPositiveMileage + + group.skippedMissingTime + + group.skippedMissingSource +} + +func statOnlyMissingMileageSkips(group statSampleCounters) bool { + return group.skippedMissingMileage > 0 && + group.skippedMissingFields == 0 && + group.skippedMissingVIN == 0 && + group.skippedNonPositiveMileage == 0 && + group.skippedMissingTime == 0 && + group.skippedMissingSource == 0 +} + +func statMileageCandidateWrites(group statSampleCounters) float64 { + candidates := group.writesOK - group.skippedNonMileageFrame + if candidates < 0 { + return 0 + } + return candidates +} + +func protocolFlowStatus(gatewayGroup *gatewayFieldFlowGroup, identityRow IdentityProtocolResolution, statGroup *protocolFlowStatCounters, thresholds Thresholds) (string, string) { + if gatewayGroup != nil { + realtimeCandidates := gatewayRealtimeCandidates(gatewayGroup.gatewayPublishCounters) + status, reason := gatewayFieldDiagnosticsStatus(gatewayGroup, realtimeCandidates, thresholds) + if status != "ok" { + return status, "gateway_fields_" + reason + } + } + if thresholds.GatewayIdentityMinSamples > 0 && + thresholds.GatewayIdentityMaxUnresolvedRatio > 0 && + float64(identityRow.Total) >= thresholds.GatewayIdentityMinSamples && + identityRow.NonResolvedRatio > thresholds.GatewayIdentityMaxUnresolvedRatio { + return "degraded", "identity_unresolved_ratio" + } + if statGroup == nil { + return "ok", "" + } + if thresholds.StatSampleMinWrites > 0 && statMileageCandidateWrites(statGroup.samples) >= thresholds.StatSampleMinWrites { + if thresholds.StatSampleFutureAdjustmentMax >= 0 && + statGroup.samples.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax { + return "degraded", "stat_future_event_time_adjusted" + } + if statGroup.samples.found <= 0 { + if statActionableSkips(statGroup.samples) <= 0 && statGroup.samples.skippedNonMileageFrame > 0 { + return "ok", "" + } + if statOnlyMissingMileageSkips(statGroup.samples) { + return "ok", "" + } + return "degraded", "stat_samples_missing" + } + if thresholds.StatSampleMaxActionableSkipRatio > 0 { + ratio := statActionableSkips(statGroup.samples) / statMileageCandidateWrites(statGroup.samples) + if ratio > thresholds.StatSampleMaxActionableSkipRatio { + return "degraded", "stat_actionable_skip_ratio" + } + } + if statGroup.samples.written <= 0 && statGroup.samples.skippedMissingSource > 0 { + return "degraded", "stat_samples_missing_source" + } + } + if thresholds.StatSourceMinSamples > 0 && statGroup.sources.found >= thresholds.StatSourceMinSamples { + activeSourceTouches := statGroup.sources.written + statGroup.sources.skippedThrottled + statGroup.sources.skippedUnmanaged + if activeSourceTouches <= 0 && statGroup.sources.skippedMissingEndpoint <= 0 { + return "degraded", "stat_source_tracking_missing" + } + if thresholds.StatSourceMaxMissingRatio > 0 { + ratio := statGroup.sources.skippedMissingEndpoint / statGroup.sources.found + if ratio > thresholds.StatSourceMaxMissingRatio { + return "degraded", "stat_source_missing_endpoint_ratio" + } + } + } + if thresholds.StatProjectionMinSampleWrites > 0 && + statGroup.projections.samplesWritten >= thresholds.StatProjectionMinSampleWrites && + statGroup.projections.written <= 0 { + return "degraded", "stat_projection_missing" + } + return "ok", "" +} + +func findingsForStatProjections(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatProjectionMinSampleWrites <= 0 { + return nil + } + groups := map[string]*statProjectionCounters{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_stat_samples_total": + if sample.labels["status"] == "written" { + statProjectionGroup(groups, sample.labels["topic"]).samplesWritten += sample.value + } + case "vehicle_stat_projections_total": + group := statProjectionGroup(groups, sample.labels["topic"]) + switch sample.labels["status"] { + case "attempted": + group.attempted += sample.value + case "written": + group.written += sample.value + case "skipped_throttled": + group.skippedThrottled += sample.value + } + } + } + var topics []string + for topic := range groups { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + group := groups[topic] + if group.samplesWritten < thresholds.StatProjectionMinSampleWrites { + continue + } + if group.written > 0 { + continue + } + findings = append(findings, fmt.Sprintf( + "stat final projection missing for %s: samples written %.0f, projections attempted %.0f, written %.0f, skipped throttled %.0f", + topic, + group.samplesWritten, + group.attempted, + group.written, + group.skippedThrottled, + )) + } + return findings +} + +func statProjectionGroup(groups map[string]*statProjectionCounters, topic string) *statProjectionCounters { + topic = strings.TrimSpace(topic) + if topic == "" { + topic = "unknown" + } + group := groups[topic] + if group == nil { + group = &statProjectionCounters{} + groups[topic] = group + } + return group +} + +func findingsForStatCache(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatCacheEntriesMax <= 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_stat_cache_entries" || sample.value <= thresholds.StatCacheEntriesMax { + continue + } + cache := strings.TrimSpace(sample.labels["cache"]) + if cache == "" { + cache = "unknown" + } + findings = append(findings, fmt.Sprintf( + "stat cache entries %.0f exceeds %.0f for cache %s", + sample.value, + thresholds.StatCacheEntriesMax, + cache, + )) + } + sort.Strings(findings) + return findings +} + +type realtimeThrottleCounters struct { + passed float64 + skippedInterval float64 + skippedNonRealtime float64 + skippedMissingVIN float64 +} + +func findingsForRealtimeThrottle(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RealtimeThrottleMinSamples <= 0 { + return nil + } + groups := map[string]*realtimeThrottleCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_realtime_throttle_total" { + continue + } + group := realtimeThrottleGroup(groups, sample.labels["store"], sample.labels["protocol"]) + switch sample.labels["status"] { + case "passed": + group.passed += sample.value + case "skipped_interval": + group.skippedInterval += sample.value + case "skipped_non_realtime": + group.skippedNonRealtime += sample.value + case "skipped_missing_vin": + group.skippedMissingVIN += sample.value + } + } + var keys []string + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + var findings []string + for _, key := range keys { + group := groups[key] + total := group.passed + group.skippedInterval + group.skippedNonRealtime + group.skippedMissingVIN + if total < thresholds.RealtimeThrottleMinSamples || group.skippedInterval <= 0 || group.passed > 0 { + continue + } + findings = append(findings, fmt.Sprintf( + "realtime throttle passing updates missing for %s: total %.0f, passed %.0f, skipped interval %.0f, skipped non-realtime %.0f, skipped missing vin %.0f", + key, + total, + group.passed, + group.skippedInterval, + group.skippedNonRealtime, + group.skippedMissingVIN, + )) + } + return findings +} + +func findingsForRealtimeThrottleCache(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RealtimeThrottleCacheEntriesMax <= 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_realtime_throttle_cache_entries" || sample.value <= thresholds.RealtimeThrottleCacheEntriesMax { + continue + } + store := strings.TrimSpace(sample.labels["store"]) + if store == "" { + store = "unknown" + } + findings = append(findings, fmt.Sprintf( + "realtime throttle cache entries %.0f exceeds %.0f for store %s", + sample.value, + thresholds.RealtimeThrottleCacheEntriesMax, + store, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForRealtimePlateCache(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.RealtimePlateCacheEntriesMax <= 0 { + return nil + } + value := parsed.totals["vehicle_realtime_plate_cache_entries"] + if value <= thresholds.RealtimePlateCacheEntriesMax { + return nil + } + return []string{fmt.Sprintf( + "realtime plate cache entries %.0f exceeds %.0f", + value, + thresholds.RealtimePlateCacheEntriesMax, + )} +} + +func realtimeThrottleGroup(groups map[string]*realtimeThrottleCounters, store string, protocol string) *realtimeThrottleCounters { + store = strings.TrimSpace(store) + if store == "" { + store = "unknown" + } + protocol = strings.TrimSpace(protocol) + if protocol == "" { + protocol = "unknown" + } + key := "store=" + store + ",protocol=" + protocol + group := groups[key] + if group == nil { + group = &realtimeThrottleCounters{} + groups[key] = group + } + return group +} + +type fastWriterRedisFieldCounters struct { + seen float64 + written float64 + skippedStale float64 +} + +type fastWriterRedisEnvelopeCounters struct { + seen float64 + updated float64 + skippedNonRealtime float64 + skippedMissingVIN float64 + skippedMissingVehicleKey float64 + skippedMissingFields float64 +} + +type fastWriterMessageCounters struct { + received float64 + ok float64 + invalidJSON float64 +} + +func findingsForFastWriterMessageCompletion(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterMessageMinReceived <= 0 || thresholds.FastWriterMessageMaxUnackedRatio <= 0 { + return nil + } + groups := map[string]*fastWriterMessageCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_messages_total" { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" || subject == "mixed" || subject == "unknown" { + continue + } + group := groups[subject] + if group == nil { + group = &fastWriterMessageCounters{} + groups[subject] = group + } + switch sample.labels["status"] { + case "received": + group.received += sample.value + case "ok": + group.ok += sample.value + case "invalid_json": + group.invalidJSON += sample.value + } + } + var subjects []string + for subject := range groups { + subjects = append(subjects, subject) + } + sort.Strings(subjects) + var findings []string + for _, subject := range subjects { + group := groups[subject] + if group.received < thresholds.FastWriterMessageMinReceived { + continue + } + completed := group.ok + group.invalidJSON + unacked := group.received - completed + if unacked < 0 { + unacked = 0 + } + ratio := unacked / group.received + if ratio <= thresholds.FastWriterMessageMaxUnackedRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "fast writer unacked message ratio %.2f exceeds %.2f for subject %s: received %.0f, ok %.0f, invalid_json %.0f, unacked %.0f", + ratio, + thresholds.FastWriterMessageMaxUnackedRatio, + subject, + group.received, + group.ok, + group.invalidJSON, + unacked, + )) + } + return findings +} + +func findingsForFastWriterRedisEnvelopes(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterRedisEnvelopeMinSeen <= 0 { + return nil + } + groups := map[string]*fastWriterRedisEnvelopeCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_redis_envelopes_total" { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" || subject == "mixed" || subject == "unknown" { + continue + } + group := groups[subject] + if group == nil { + group = &fastWriterRedisEnvelopeCounters{} + groups[subject] = group + } + switch sample.labels["status"] { + case "seen": + group.seen += sample.value + case "updated": + group.updated += sample.value + case "skipped_non_realtime": + group.skippedNonRealtime += sample.value + case "skipped_missing_vin": + group.skippedMissingVIN += sample.value + case "skipped_missing_vehicle_key": + group.skippedMissingVehicleKey += sample.value + case "skipped_missing_fields": + group.skippedMissingFields += sample.value + } + } + var subjects []string + for subject := range groups { + subjects = append(subjects, subject) + } + sort.Strings(subjects) + var findings []string + for _, subject := range subjects { + group := groups[subject] + if group.seen < thresholds.FastWriterRedisEnvelopeMinSeen { + continue + } + if group.updated <= 0 { + findings = append(findings, fmt.Sprintf( + "fast writer redis realtime updates missing for subject %s: envelopes seen %.0f, updated %.0f, skipped non-realtime %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f", + subject, + group.seen, + group.updated, + group.skippedNonRealtime, + group.skippedMissingVIN, + group.skippedMissingVehicleKey, + group.skippedMissingFields, + )) + continue + } + if thresholds.FastWriterRedisEnvelopeMaxBadRatio <= 0 { + continue + } + actionableSkips := group.skippedMissingVIN + group.skippedMissingVehicleKey + group.skippedMissingFields + ratio := actionableSkips / group.seen + if ratio <= thresholds.FastWriterRedisEnvelopeMaxBadRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "fast writer redis envelope skip ratio %.2f exceeds %.2f for subject %s: envelopes seen %.0f, updated %.0f, missing vin %.0f, missing vehicle key %.0f, missing parsed fields %.0f", + ratio, + thresholds.FastWriterRedisEnvelopeMaxBadRatio, + subject, + group.seen, + group.updated, + group.skippedMissingVIN, + group.skippedMissingVehicleKey, + group.skippedMissingFields, + )) + } + return findings +} + +func findingsForFastWriterRedisFields(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterRedisFieldMinSeen <= 0 || thresholds.FastWriterRedisFieldMaxStaleRatio <= 0 { + return nil + } + groups := map[string]*fastWriterRedisFieldCounters{} + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_redis_fields_total" { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" || subject == "mixed" || subject == "unknown" { + continue + } + group := groups[subject] + if group == nil { + group = &fastWriterRedisFieldCounters{} + groups[subject] = group + } + switch sample.labels["status"] { + case "seen": + group.seen += sample.value + case "written": + group.written += sample.value + case "skipped_stale": + group.skippedStale += sample.value + } + } + var subjects []string + for subject := range groups { + subjects = append(subjects, subject) + } + sort.Strings(subjects) + var findings []string + for _, subject := range subjects { + group := groups[subject] + if group.seen < thresholds.FastWriterRedisFieldMinSeen || group.seen <= 0 { + continue + } + ratio := group.skippedStale / group.seen + if ratio <= thresholds.FastWriterRedisFieldMaxStaleRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "fast writer redis stale field ratio %.2f exceeds %.2f for subject %s (seen %.0f, written %.0f, skipped_stale %.0f)", + ratio, + thresholds.FastWriterRedisFieldMaxStaleRatio, + subject, + group.seen, + group.written, + group.skippedStale, + )) + } + return findings +} + +func findingsForHistoryLocationErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.HistoryLocationErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_history_location_writes_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.HistoryLocationErrorMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "history location writes error %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.HistoryLocationErrorMax, + topic, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForHistoryLocationAccounting(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.HistoryLocationAccountingMaxRatio <= 0 { + return nil + } + writesOKByTopic := map[string]float64{} + locationStatusByTopic := map[string]float64{} + for _, sample := range parsed.samples { + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" || topic == "mixed" || topic == "unknown" { + continue + } + switch { + case sample.name == "vehicle_history_writes_total" && sample.labels["status"] == "ok": + writesOKByTopic[topic] += sample.value + case sample.name == "vehicle_history_location_writes_total": + locationStatusByTopic[topic] += sample.value + } + } + var topics []string + for topic := range writesOKByTopic { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + writesOK := writesOKByTopic[topic] + if writesOK < thresholds.KafkaConsumerMessageMinReceived { + continue + } + accounted := locationStatusByTopic[topic] + diff := math.Abs(writesOK - accounted) + ratio := diff / writesOK + if ratio <= thresholds.HistoryLocationAccountingMaxRatio { + continue + } + findings = append(findings, fmt.Sprintf( + "history location accounting mismatch ratio %.2f exceeds %.2f for topic %s: history writes ok %.0f, location statuses %.0f, diff %.0f", + ratio, + thresholds.HistoryLocationAccountingMaxRatio, + topic, + writesOK, + accounted, + diff, + )) + } + return findings +} + +func findingsForHistoryFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.HistoryFallbackErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_history_batch_fallback_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.HistoryFallbackErrorMax { + continue + } + findings = append(findings, fmt.Sprintf( + "history batch fallback errors %.0f exceeds %.0f", + sample.value, + thresholds.HistoryFallbackErrorMax, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForFastWriterFallbackErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterFallbackErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_batch_fallback_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.FastWriterFallbackErrorMax { + continue + } + stage := strings.TrimSpace(sample.labels["stage"]) + if stage == "" { + stage = "unknown" + } + findings = append(findings, fmt.Sprintf( + "fast writer batch fallback errors %.0f exceeds %.0f for stage %s", + sample.value, + thresholds.FastWriterFallbackErrorMax, + stage, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForFastWriterDecoupledUpdateErrors(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterDecoupledUpdateErrorMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_decoupled_updates_total" || sample.labels["status"] != "error" { + continue + } + if sample.value <= thresholds.FastWriterDecoupledUpdateErrorMax { + continue + } + reason := strings.TrimSpace(sample.labels["reason"]) + if reason == "" { + reason = "unknown" + } + findings = append(findings, fmt.Sprintf( + "fast writer decoupled update errors %.0f exceeds %.0f for reason %s", + sample.value, + thresholds.FastWriterDecoupledUpdateErrorMax, + reason, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForWriteRetries(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + findings = append(findings, writeRetryFindings(parsed, "vehicle_history_write_retries_total", "history", thresholds.HistoryRetryMax, thresholds.HistoryRetryExhaustedMax)...) + findings = append(findings, writeRetryFindings(parsed, "vehicle_stat_write_retries_total", "stat", thresholds.StatRetryMax, thresholds.StatRetryExhaustedMax)...) + sort.Strings(findings) + return findings +} + +func writeRetryFindings(parsed parsedMetricSet, metricName string, serviceLabel string, retryMax float64, exhaustedMax float64) []string { + var findings []string + for _, sample := range parsed.samples { + if sample.name != metricName { + continue + } + status := strings.TrimSpace(sample.labels["status"]) + operation := strings.TrimSpace(sample.labels["operation"]) + if operation == "" { + operation = "unknown" + } + switch status { + case "retry": + if retryMax < 0 || sample.value <= retryMax { + continue + } + findings = append(findings, fmt.Sprintf( + "%s write retries %.0f exceeds %.0f for operation %s", + serviceLabel, + sample.value, + retryMax, + operation, + )) + case "exhausted": + if exhaustedMax < 0 || sample.value <= exhaustedMax { + continue + } + findings = append(findings, fmt.Sprintf( + "%s write retry exhausted %.0f exceeds %.0f for operation %s", + serviceLabel, + sample.value, + exhaustedMax, + operation, + )) + } + } + return findings +} + +func findingsForStatInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatInvalidJSONMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "invalid_json" { + continue + } + if sample.value <= thresholds.StatInvalidJSONMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "stat invalid json %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.StatInvalidJSONMax, + topic, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForStatProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatProtocolTopicMismatchMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_stat_kafka_messages_total" || sample.labels["status"] != "protocol_topic_mismatch" { + continue + } + if sample.value <= thresholds.StatProtocolTopicMismatchMax { + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + if topic == "" { + topic = "unknown" + } + findings = append(findings, fmt.Sprintf( + "stat protocol topic mismatches %.0f exceeds %.0f for topic %s", + sample.value, + thresholds.StatProtocolTopicMismatchMax, + topic, + )) + } + findings = append(findings, findingsForStatMetricProtocolTopicMismatches(parsed, thresholds)...) + sort.Strings(findings) + return findings +} + +func findingsForStatMetricProtocolTopicMismatches(parsed parsedMetricSet, thresholds Thresholds) []string { + var findings []string + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_stat_samples_total", "vehicle_stat_sources_total", "vehicle_stat_projections_total": + default: + continue + } + topic := strings.TrimSpace(sample.labels["topic"]) + expectedProtocol, ok := topics.ProtocolForKnownFieldsTopic(topic) + if !ok { + continue + } + protocol := strings.TrimSpace(sample.labels["protocol"]) + if sample.value <= thresholds.StatProtocolTopicMismatchMax { + continue + } + status := strings.TrimSpace(sample.labels["status"]) + if status == "" { + status = "unknown" + } + if protocol == "" { + findings = append(findings, fmt.Sprintf( + "stat metric protocol label missing %.0f exceeds %.0f for %s topic %s (status %s)", + sample.value, + thresholds.StatProtocolTopicMismatchMax, + sample.name, + topic, + status, + )) + continue + } + if protocol == expectedProtocol { + continue + } + findings = append(findings, fmt.Sprintf( + "stat metric protocol topic mismatches %.0f exceeds %.0f for %s topic %s: expected %s, got %s (status %s)", + sample.value, + thresholds.StatProtocolTopicMismatchMax, + sample.name, + topic, + expectedProtocol, + protocol, + status, + )) + } + return findings +} + +func findingsForFastWriterInvalidJSON(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.FastWriterInvalidJSONMax < 0 { + return nil + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != "vehicle_fast_writer_messages_total" || sample.labels["status"] != "invalid_json" { + continue + } + if sample.value <= thresholds.FastWriterInvalidJSONMax { + continue + } + subject := strings.TrimSpace(sample.labels["subject"]) + if subject == "" { + subject = "unknown" + } + findings = append(findings, fmt.Sprintf( + "fast writer invalid json %.0f exceeds %.0f for subject %s", + sample.value, + thresholds.FastWriterInvalidJSONMax, + subject, + )) + } + sort.Strings(findings) + return findings +} + +func findingsForStatSamples(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatSampleMinWrites <= 0 { + return nil + } + groups := map[string]*statSampleCounters{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_stat_writes_total": + if sample.labels["status"] != "ok" { + continue + } + statSampleGroup(groups, sample.labels["topic"]).writesOK += sample.value + case "vehicle_stat_samples_total": + group := statSampleGroup(groups, sample.labels["topic"]) + switch sample.labels["status"] { + case "found": + group.found += sample.value + case "written": + group.written += sample.value + case "skipped_missing_fields": + group.skippedMissingFields += sample.value + case "skipped_missing_vin": + group.skippedMissingVIN += sample.value + case "skipped_missing_mileage": + group.skippedMissingMileage += sample.value + case "skipped_non_mileage_frame": + group.skippedNonMileageFrame += sample.value + case "skipped_non_positive_mileage": + group.skippedNonPositiveMileage += sample.value + case "skipped_missing_time": + group.skippedMissingTime += sample.value + case "event_time_future_adjusted": + group.futureAdjusted += sample.value + case "skipped_same_mileage": + group.skippedSameMileage += sample.value + case "skipped_missing_source": + group.skippedMissingSource += sample.value + } + } + } + var topics []string + for topic := range groups { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + group := groups[topic] + mileageCandidates := statMileageCandidateWrites(*group) + if mileageCandidates < thresholds.StatSampleMinWrites { + continue + } + if thresholds.StatSampleFutureAdjustmentMax >= 0 && + group.futureAdjusted > thresholds.StatSampleFutureAdjustmentMax { + findings = append(findings, fmt.Sprintf( + "stat future event time adjustments %.0f exceeds %.0f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f", + group.futureAdjusted, + thresholds.StatSampleFutureAdjustmentMax, + topic, + group.writesOK, + mileageCandidates, + group.found, + group.written, + )) + } + actionableSkips := statActionableSkips(*group) + if group.found <= 0 { + if actionableSkips <= 0 && group.skippedNonMileageFrame > 0 { + continue + } + if statOnlyMissingMileageSkips(*group) { + continue + } + findings = append(findings, fmt.Sprintf( + "stat samples missing for %s: writes ok %.0f, mileage candidates %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f", + topic, + group.writesOK, + mileageCandidates, + group.skippedMissingFields, + group.skippedMissingVIN, + group.skippedMissingMileage, + group.skippedNonMileageFrame, + group.skippedNonPositiveMileage, + group.skippedMissingTime, + group.skippedMissingSource, + )) + continue + } + if thresholds.StatSampleMaxActionableSkipRatio > 0 { + ratio := actionableSkips / mileageCandidates + if ratio > thresholds.StatSampleMaxActionableSkipRatio { + findings = append(findings, fmt.Sprintf( + "stat actionable skip ratio %.2f exceeds %.2f for %s: writes ok %.0f, mileage candidates %.0f, found %.0f, written %.0f, missing fields %.0f, skipped missing vin %.0f, missing mileage %.0f, non-mileage frames %.0f, non-positive mileage %.0f, missing time %.0f, missing source %.0f, same mileage %.0f", + ratio, + thresholds.StatSampleMaxActionableSkipRatio, + topic, + group.writesOK, + mileageCandidates, + group.found, + group.written, + group.skippedMissingFields, + group.skippedMissingVIN, + group.skippedMissingMileage, + group.skippedNonMileageFrame, + group.skippedNonPositiveMileage, + group.skippedMissingTime, + group.skippedMissingSource, + group.skippedSameMileage, + )) + } + } + if group.written <= 0 && group.skippedMissingSource > 0 { + findings = append(findings, fmt.Sprintf( + "stat samples not written for %s: found %.0f, skipped missing source %.0f, same mileage %.0f", + topic, + group.found, + group.skippedMissingSource, + group.skippedSameMileage, + )) + } + } + return findings +} + +func findingsForStatSources(parsed parsedMetricSet, thresholds Thresholds) []string { + if thresholds.StatSourceMinSamples <= 0 { + return nil + } + groups := map[string]*statSourceCounters{} + for _, sample := range parsed.samples { + switch sample.name { + case "vehicle_stat_samples_total": + if sample.labels["status"] == "found" { + statSourceGroup(groups, sample.labels["topic"]).found += sample.value + } + case "vehicle_stat_sources_total": + group := statSourceGroup(groups, sample.labels["topic"]) + switch sample.labels["status"] { + case "attempted": + group.attempted += sample.value + case "written": + group.written += sample.value + case "skipped_throttled": + group.skippedThrottled += sample.value + case "skipped_missing_endpoint": + group.skippedMissingEndpoint += sample.value + case "skipped_unmanaged": + group.skippedUnmanaged += sample.value + } + } + } + var topics []string + for topic := range groups { + topics = append(topics, topic) + } + sort.Strings(topics) + var findings []string + for _, topic := range topics { + group := groups[topic] + if group.found < thresholds.StatSourceMinSamples || group.found <= 0 { + continue + } + activeSourceTouches := group.written + group.skippedThrottled + group.skippedUnmanaged + if activeSourceTouches <= 0 && group.skippedMissingEndpoint <= 0 { + findings = append(findings, fmt.Sprintf( + "stat source tracking missing for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f", + topic, + group.found, + group.written, + group.skippedThrottled, + group.skippedUnmanaged, + group.skippedMissingEndpoint, + )) + continue + } + if thresholds.StatSourceMaxMissingRatio <= 0 { + continue + } + ratio := group.skippedMissingEndpoint / group.found + if ratio > thresholds.StatSourceMaxMissingRatio { + findings = append(findings, fmt.Sprintf( + "stat source missing endpoint ratio %.2f exceeds %.2f for %s: samples found %.0f, source written %.0f, source throttled %.0f, source unmanaged %.0f, missing endpoint %.0f", + ratio, + thresholds.StatSourceMaxMissingRatio, + topic, + group.found, + group.written, + group.skippedThrottled, + group.skippedUnmanaged, + group.skippedMissingEndpoint, + )) + } + } + return findings +} + +func statSampleGroup(groups map[string]*statSampleCounters, topic string) *statSampleCounters { + topic = strings.TrimSpace(topic) + if topic == "" { + topic = "unknown" + } + group := groups[topic] + if group == nil { + group = &statSampleCounters{} + groups[topic] = group + } + return group +} + +func statSourceGroup(groups map[string]*statSourceCounters, topic string) *statSourceCounters { + topic = strings.TrimSpace(topic) + if topic == "" { + topic = "unknown" + } + group := groups[topic] + if group == nil { + group = &statSourceCounters{} + groups[topic] = group + } + return group +} + +func histogramP99Findings(parsed parsedMetricSet, histogramName string, label string, thresholdMS float64, minSamples float64) []string { + if thresholdMS <= 0 { + return nil + } + groups := histogramGroups(parsed.samples, histogramName) + var findings []string + var keys []string + for key := range groups { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + estimate, count, ok := histogramQuantile(groups[key], 0.99) + if !ok || count < minSamples || estimate <= thresholdMS { + continue + } + findings = append(findings, fmt.Sprintf("%s p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, estimate, thresholdMS, key, count)) + } + return findings +} + +func recentP99GaugeFindings(parsed parsedMetricSet, metricName string, samplesMetricName string, label string, thresholdMS float64, minSamples float64) []string { + if thresholdMS <= 0 { + return nil + } + samplesByKey := map[string]float64{} + for _, sample := range parsed.samples { + if sample.name != samplesMetricName { + continue + } + samplesByKey[metricLabelsKey(sample.labels)] = sample.value + } + var findings []string + for _, sample := range parsed.samples { + if sample.name != metricName { + continue + } + key := metricLabelsKey(sample.labels) + samples := samplesByKey[key] + if samples < minSamples || sample.value <= thresholdMS { + continue + } + findings = append(findings, fmt.Sprintf("%s recent p99 %.0fms exceeds %.0fms (%s, samples %.0f)", label, sample.value, thresholdMS, key, samples)) + } + sort.Strings(findings) + return findings +} + +func parseMetrics(text string) map[string]float64 { + return parseMetricSet(text).totals +} + +type parsedMetricSet struct { + totals map[string]float64 + samples []metricSample +} + +type metricSample struct { + name string + labels map[string]string + value float64 +} + +func parseMetricSet(text string) parsedMetricSet { + out := map[string]float64{} + var samples []metricSample scanner := bufio.NewScanner(strings.NewReader(text)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } - name, value, ok := parseMetricLine(line) + sample, ok := parseMetricSample(line) if !ok { continue } - out[name] += value + if aggregateMetricByMax(sample.name) { + if current, ok := out[sample.name]; !ok || sample.value > current { + out[sample.name] = sample.value + } + } else { + out[sample.name] += sample.value + } + samples = append(samples, sample) + } + return parsedMetricSet{totals: out, samples: samples} +} + +func aggregateMetricByMax(name string) bool { + return isLastActivityUnixMetric(name) || + strings.HasSuffix(name, "_p99_ms") +} + +func isLastActivityUnixMetric(name string) bool { + return strings.Contains(name, "_last_") && strings.HasSuffix(name, "_unix_seconds") +} + +func addLastActivityAges(metrics map[string]float64, checkedAt time.Time) { + if len(metrics) == 0 { + return + } + for name, value := range metrics { + if !isLastActivityUnixMetric(name) || value <= 0 { + continue + } + age := checkedAt.Sub(time.Unix(int64(value), 0).UTC()).Seconds() + if age < 0 { + age = 0 + } + metrics[strings.TrimSuffix(name, "_unix_seconds")+"_age_seconds"] = age } - return out } func parseMetricLine(line string) (string, float64, bool) { + sample, ok := parseMetricSample(line) + return sample.name, sample.value, ok +} + +func parseMetricSample(line string) (metricSample, bool) { parts := strings.Fields(line) if len(parts) < 2 { - return "", 0, false - } - name := parts[0] - if index := strings.IndexByte(name, '{'); index >= 0 { - name = name[:index] + return metricSample{}, false } + name, labels := parseMetricNameAndLabels(parts[0]) value, err := strconv.ParseFloat(parts[1], 64) if err != nil { - return "", 0, false + return metricSample{}, false } - return name, value, true + return metricSample{name: name, labels: labels, value: value}, true +} + +func parseMetricNameAndLabels(value string) (string, map[string]string) { + index := strings.IndexByte(value, '{') + if index < 0 { + return value, nil + } + end := strings.LastIndexByte(value, '}') + if end < index { + return value[:index], nil + } + return value[:index], parseMetricLabels(value[index+1 : end]) +} + +func parseMetricLabels(value string) map[string]string { + labels := map[string]string{} + for len(value) > 0 { + value = strings.TrimLeft(value, " \t,") + if value == "" { + break + } + index := strings.IndexByte(value, '=') + if index <= 0 { + break + } + key := strings.TrimSpace(value[:index]) + value = strings.TrimLeft(value[index+1:], " \t") + if key == "" || value == "" { + break + } + if value[0] != '"' { + next := strings.IndexByte(value, ',') + if next < 0 { + labels[key] = strings.TrimSpace(value) + break + } + labels[key] = strings.TrimSpace(value[:next]) + value = value[next+1:] + continue + } + labelValue, rest, ok := readQuotedLabelValue(value) + if !ok { + break + } + labels[key] = labelValue + value = rest + } + if len(labels) == 0 { + return nil + } + return labels +} + +func readQuotedLabelValue(value string) (string, string, bool) { + if value == "" || value[0] != '"' { + return "", value, false + } + var b strings.Builder + escaped := false + for index := 1; index < len(value); index++ { + ch := value[index] + if escaped { + switch ch { + case 'n': + b.WriteByte('\n') + default: + b.WriteByte(ch) + } + escaped = false + continue + } + if ch == '\\' { + escaped = true + continue + } + if ch == '"' { + return b.String(), value[index+1:], true + } + b.WriteByte(ch) + } + return "", value, false +} + +type histogramBucket struct { + upperBound float64 + count float64 +} + +func histogramGroups(samples []metricSample, histogramName string) map[string][]histogramBucket { + groups := map[string][]histogramBucket{} + for _, sample := range samples { + if sample.name != histogramName+"_bucket" { + continue + } + upperBound, ok := parseBucketUpperBound(sample.labels["le"]) + if !ok { + continue + } + key := metricLabelsKey(sample.labels, "le") + groups[key] = append(groups[key], histogramBucket{upperBound: upperBound, count: sample.value}) + } + return groups +} + +func parseBucketUpperBound(value string) (float64, bool) { + if value == "+Inf" { + return math.Inf(1), true + } + parsed, err := strconv.ParseFloat(value, 64) + return parsed, err == nil +} + +func histogramQuantile(buckets []histogramBucket, quantile float64) (float64, float64, bool) { + if len(buckets) == 0 || quantile <= 0 || quantile > 1 { + return 0, 0, false + } + sort.Slice(buckets, func(i, j int) bool { + return buckets[i].upperBound < buckets[j].upperBound + }) + total := buckets[len(buckets)-1].count + if total <= 0 { + return 0, 0, false + } + target := total * quantile + for _, bucket := range buckets { + if bucket.count >= target { + return bucket.upperBound, total, true + } + } + return buckets[len(buckets)-1].upperBound, total, true +} + +func metricLabelsKey(labels map[string]string, exclude ...string) string { + if len(labels) == 0 { + return "all" + } + excluded := map[string]struct{}{} + for _, key := range exclude { + excluded[key] = struct{}{} + } + keys := make([]string, 0, len(labels)) + for key := range labels { + if _, skip := excluded[key]; skip { + continue + } + keys = append(keys, key) + } + if len(keys) == 0 { + return "all" + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, key+"="+labels[key]) + } + return strings.Join(parts, ",") } diff --git a/go/vehicle-gateway/internal/capacity/check_test.go b/go/vehicle-gateway/internal/capacity/check_test.go index 32553fde..8bbdaf7d 100644 --- a/go/vehicle-gateway/internal/capacity/check_test.go +++ b/go/vehicle-gateway/internal/capacity/check_test.go @@ -1,20 +1,35 @@ package capacity import ( + "fmt" + "math" + "strconv" "strings" "testing" + "time" ) func TestEvaluateReportsOKWhenCriticalBacklogsAreZero(t *testing.T) { report := Evaluate(map[string]string{ - "gateway": `vehicle_gateway_active_connections{protocol="JT808"} 50000 + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_active_connections{protocol="JT808"} 50000 vehicle_gateway_connection_rejections_total{protocol="JT808",reason="max_connections"} 0 +vehicle_gateway_identity_snapshot_ready 1 +vehicle_gateway_identity_snapshot_last_success_unix_seconds 99999999999 vehicle_async_sink_queue_depth{sink="nats"} 0`, - "fast-writer": `vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 0 + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 0 vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 0 +vehicle_fast_writer_tdengine_enabled 0 vehicle_fast_writer_batch_pending_messages 0`, - "history": `vehicle_history_batch_pending_messages 0 + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_pending_messages 0 vehicle_history_batch_pending_rows 0 +vehicle_history_retry_pending_messages 0 vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`, }) @@ -31,18 +46,44 @@ vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`, func TestEvaluateReportsDegradedForPendingAndRejects(t *testing.T) { report := Evaluate(map[string]string{ - "gateway": `vehicle_gateway_active_connections{protocol="JT808"} 120000 + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_active_connections{protocol="JT808"} 120000 vehicle_gateway_connection_rejections_total{protocol="JT808",reason="max_connections"} 3 vehicle_async_sink_queue_depth{sink="nats"} 25000`, - "bridge": `vehicle_bridge_nats_consumer_ack_pending{consumer="vehicle-kafka-bridge",stream="VEHICLE_INGEST"} 101 + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_nats_consumer_ack_pending{consumer="vehicle-kafka-bridge",stream="VEHICLE_INGEST"} 101 vehicle_bridge_nats_consumer_pending{consumer="vehicle-kafka-bridge",stream="VEHICLE_INGEST"} 12001 vehicle_bridge_batch_pending_messages 1001`, - "fast-writer": `vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 11 + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 101 vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 12001 -vehicle_fast_writer_batch_pending_messages 9`, - "history": `vehicle_history_batch_pending_messages 6 -vehicle_history_batch_pending_rows 7 -vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 42`, +vehicle_fast_writer_tdengine_enabled 1 +vehicle_fast_writer_batch_pending_messages 1009`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_pending_messages 1006 +vehicle_history_batch_pending_rows 5007 +vehicle_history_retry_pending_messages 2 +vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 1042`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_batch_pending_messages 1008 +vehicle_stat_retry_pending_messages 3`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_batch_pending_messages 1009 +vehicle_realtime_retry_pending_messages 4 +vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 10001`, + "identity": `vehicle_service_info{service="vehicle-identity-writer"} 1 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_identity_writer_retry_pending_messages 5`, }) if report.Status != StatusDegraded { @@ -57,8 +98,17 @@ vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 42`, "bridge batch pending", "fast writer ack pending", "fast writer consumer pending", + "fast writer batch pending", "history batch pending", - "kafka lag", + "history rows pending", + "stat batch pending", + "realtime batch pending", + "realtime async queue depth", + "history retry pending 2 exceeds 0", + "realtime retry pending 4 exceeds 0", + "stat retry pending 3 exceeds 0", + "identity retry pending 5 exceeds 0", + "kafka lag 1042 exceeds 1000", } { if !strings.Contains(joined, want) { t.Fatalf("finding missing %q in:\n%s", want, joined) @@ -70,16 +120,4517 @@ vehicle_history_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 42`, if report.Totals.BridgeConsumerPending != 12001 { t.Fatalf("bridge consumer pending = %v, want 12001", report.Totals.BridgeConsumerPending) } - if report.Totals.FastWriterAckPending != 11 { - t.Fatalf("fast writer ack pending = %v, want 11", report.Totals.FastWriterAckPending) + if report.Totals.FastWriterAckPending != 101 { + t.Fatalf("fast writer ack pending = %v, want 101", report.Totals.FastWriterAckPending) } - if report.Totals.FastWriterBatchPending != 9 { - t.Fatalf("fast writer batch pending = %v, want 9", report.Totals.FastWriterBatchPending) + if report.Totals.FastWriterBatchPending != 1009 { + t.Fatalf("fast writer batch pending = %v, want 1009", report.Totals.FastWriterBatchPending) } - if report.Totals.HistoryBatchPending != 6 { - t.Fatalf("history batch pending = %v, want 6", report.Totals.HistoryBatchPending) + if report.Totals.HistoryBatchPending != 1006 { + t.Fatalf("history batch pending = %v, want 1006", report.Totals.HistoryBatchPending) } - if report.Totals.HistoryRowsPending != 7 { - t.Fatalf("history rows pending = %v, want 7", report.Totals.HistoryRowsPending) + if report.Totals.HistoryRowsPending != 5007 { + t.Fatalf("history rows pending = %v, want 5007", report.Totals.HistoryRowsPending) + } + if report.Totals.RealtimeBatchPending != 1009 { + t.Fatalf("realtime batch pending = %v, want 1009", report.Totals.RealtimeBatchPending) + } + if report.Totals.RealtimeAsyncQueuePending != 10001 { + t.Fatalf("realtime async queue pending = %v, want 10001", report.Totals.RealtimeAsyncQueuePending) + } + if report.Totals.HistoryRetryPending != 2 || report.Totals.RealtimeRetryPending != 4 || report.Totals.StatRetryPending != 3 || report.Totals.IdentityRetryPending != 5 { + t.Fatalf("retry pending totals = history:%v realtime:%v stat:%v identity:%v", report.Totals.HistoryRetryPending, report.Totals.RealtimeRetryPending, report.Totals.StatRetryPending, report.Totals.IdentityRetryPending) + } +} + +func TestEvaluateCanDisableConsumerRetryPendingLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_retry_pending_messages 9`, + }, Thresholds{ConsumerRetryPendingMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateDoesNotDegradeForTransientBatchPending(t *testing.T) { + report := Evaluate(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_batch_pending_messages 470`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_batch_pending_messages 470`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_batch_pending_messages 470`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_pending_messages 470 +vehicle_history_batch_pending_rows 470`, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } + if report.Totals.FastWriterBatchPending != 470 { + t.Fatalf("fast writer batch pending = %v, want 470", report.Totals.FastWriterBatchPending) + } + if report.Totals.HistoryBatchPending != 470 { + t.Fatalf("history batch pending = %v, want 470", report.Totals.HistoryBatchPending) + } + if report.Totals.HistoryRowsPending != 470 { + t.Fatalf("history rows pending = %v, want 470", report.Totals.HistoryRowsPending) + } + if report.Totals.RealtimeBatchPending != 470 { + t.Fatalf("realtime batch pending = %v, want 470", report.Totals.RealtimeBatchPending) + } +} + +func TestEvaluateReportsFastWriterTDengineStageEnabled(t *testing.T) { + report := Evaluate(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_tdengine_enabled 1`, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "fast writer tdengine enabled 1 exceeds 0") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableFastWriterTDengineStageEnabledLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_tdengine_enabled 1`, + }, Thresholds{FastWriterTDengineEnabledMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRecentlyStartedProcess(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_process_uptime_seconds 12`, + }, Thresholds{ProcessUptimeMinSec: 60}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway process uptime 12s below 60s") { + t.Fatalf("uptime finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsMissingProcessUptimeWhenRequired(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1`, + }, Thresholds{ProcessUptimeMinSec: 60}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway process uptime metric missing") { + t.Fatalf("missing uptime finding not found in:\n%s", joined) + } +} + +func TestEvaluateCanDisableProcessUptimeLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_process_uptime_seconds 1`, + }, Thresholds{ProcessUptimeMinSec: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsProcessResourceLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_process_heap_alloc_bytes 2048 +vehicle_process_heap_sys_bytes 4096 +vehicle_process_goroutines 301`, + }, Thresholds{ + ProcessHeapAllocMaxBytes: 1024, + ProcessHeapSysMaxBytes: 2048, + ProcessGoroutinesMax: 300, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway process heap alloc 2048 bytes exceeds 1024", + "gateway process heap sys 4096 bytes exceeds 2048", + "gateway process goroutines 301 exceeds 300", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsMissingProcessResourceMetricsWhenRequired(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1`, + }, Thresholds{ + ProcessHeapAllocMaxBytes: 1024, + ProcessHeapSysMaxBytes: 2048, + ProcessGoroutinesMax: 300, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway process heap alloc metric missing", + "gateway process heap sys metric missing", + "gateway process goroutines metric missing", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableProcessResourceLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_process_heap_alloc_bytes 2048 +vehicle_process_heap_sys_bytes 4096 +vehicle_process_goroutines 301`, + }, Thresholds{ + ProcessHeapAllocMaxBytes: 0, + ProcessHeapSysMaxBytes: 0, + ProcessGoroutinesMax: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateUsesConfigurableAsyncSinkQueueDepthLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_depth{sink="nats"} 1500`, + }, Thresholds{AsyncSinkQueueDepthMax: 1000}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "async sink queue depth 1500 exceeds 1000") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableAsyncSinkQueueDepthLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_depth{sink="nats"} 999999`, + }, Thresholds{AsyncSinkQueueDepthMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsSlowAsyncSinkQueueWaitRecentP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_wait_recent_p99_ms{sink="nats",queue="raw",kind="raw"} 125 +vehicle_async_sink_queue_wait_recent_samples{sink="nats",queue="raw",kind="raw"} 512`, + }, Thresholds{ + AsyncSinkQueueWaitRecentP99MS: 100, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "async sink queue wait recent p99 125ms exceeds 100ms", + "kind=raw,queue=raw,sink=nats", + "samples 512", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableAsyncSinkQueueWaitRecentP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_wait_recent_p99_ms{sink="nats",queue="raw",kind="raw"} 999 +vehicle_async_sink_queue_wait_recent_samples{sink="nats",queue="raw",kind="raw"} 512`, + }, Thresholds{ + AsyncSinkQueueWaitRecentP99MS: 0, + MinHistogramSamples: 100, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } +} + +func TestEvaluateReportsAsyncSinkEnqueueAndPublishErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_enqueue_total{sink="nats",kind="raw",status="queued"} 100 +vehicle_async_sink_enqueue_total{sink="nats",kind="fields",status="timeout"} 2 +vehicle_async_sink_enqueue_total{sink="kafka",kind="raw",status="closed"} 1 +vehicle_async_sink_publish_total{sink="nats",kind="raw",status="error"} 3 +vehicle_async_sink_publish_total{sink="nats",kind="fields",status="ok"} 100`, + }, Thresholds{AsyncSinkEnqueueErrorMax: 0, AsyncSinkPublishErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway async sink enqueue timeout 2 exceeds 0 for sink nats kind fields", + "gateway async sink enqueue closed 1 exceeds 0 for sink kafka kind raw", + "gateway async sink publish errors 3 exceeds 0 for sink nats kind raw", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if strings.Contains(joined, "queued") || strings.Contains(joined, "publish errors 100") { + t.Fatalf("successful enqueue/publish samples should not be reported:\n%s", joined) + } +} + +func TestEvaluateCanDisableAsyncSinkErrorLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_enqueue_total{sink="nats",kind="fields",status="timeout"} 2 +vehicle_async_sink_publish_total{sink="nats",kind="raw",status="error"} 3`, + }, Thresholds{AsyncSinkEnqueueErrorMax: -1, AsyncSinkPublishErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRuntimeConfigDrift(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 1000 +vehicle_bridge_config{setting="workers"} 1`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 100 +vehicle_fast_writer_config{setting="workers"} 2`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_config{setting="workers"} 1`, + "identity": `vehicle_service_info{service="vehicle-identity-writer"} 1 +vehicle_identity_writer_config{setting="workers"} 1 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{ + BridgeFetchWaitMaxMS: 50, + BridgeWorkersMin: 2, + FastWriterFetchWaitMaxMS: 50, + FastWriterWorkersMin: 4, + HistoryWorkersMin: 3, + RealtimeWorkersMin: 3, + StatWorkersMin: 3, + IdentityWorkersMin: 3, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge fetch wait 1000ms exceeds 50ms", + "bridge workers 1 below 2", + "fast writer fetch wait 100ms exceeds 50ms", + "fast writer workers 2 below 4", + "history workers 1 below 3", + "realtime workers 1 below 3", + "stat workers 1 below 3", + "identity workers 1 below 3", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateAcceptsRuntimeConfigWithinLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_jt808_registration_gateway_writes_enabled 0`, + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + "identity": `vehicle_service_info{service="vehicle-identity-writer"} 1 +vehicle_identity_writer_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{ + BridgeFetchWaitMaxMS: 50, + BridgeWorkersMin: 2, + FastWriterFetchWaitMaxMS: 50, + FastWriterWorkersMin: 4, + HistoryWorkersMin: 3, + RealtimeWorkersMin: 3, + StatWorkersMin: 3, + IdentityWorkersMin: 3, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsMissingRuntimeConfigMetrics(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="bridge-test"} 1`, + "fast-writer": `vehicle_service_info{service="fast-writer-test"} 1`, + "history": `vehicle_service_info{service="history-test"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="realtime-test"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`, + "stat": `vehicle_service_info{service="stat-test"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + "identity": `vehicle_service_info{service="identity-test"} 1 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{ + BridgeFetchWaitMaxMS: 50, + BridgeWorkersMin: 2, + FastWriterFetchWaitMaxMS: 50, + FastWriterWorkersMin: 4, + HistoryWorkersMin: 3, + RealtimeWorkersMin: 3, + StatWorkersMin: 3, + IdentityWorkersMin: 3, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge fetch wait config metric missing (vehicle_bridge_config setting=fetch_wait_ms)", + "bridge workers config metric missing (vehicle_bridge_config setting=workers)", + "fast writer fetch wait config metric missing (vehicle_fast_writer_config setting=fetch_wait_ms)", + "fast writer workers config metric missing (vehicle_fast_writer_config setting=workers)", + "history workers config metric missing (vehicle_history_config setting=workers)", + "realtime workers config metric missing (vehicle_realtime_config setting=workers)", + "stat workers config metric missing (vehicle_stat_config setting=workers)", + "identity workers config metric missing (vehicle_identity_writer_config setting=workers)", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableRuntimeConfigMetricRequirements(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{ + BridgeFetchWaitMaxMS: 0, + BridgeWorkersMin: 0, + FastWriterFetchWaitMaxMS: 0, + FastWriterWorkersMin: 0, + HistoryWorkersMin: 0, + RealtimeWorkersMin: 0, + StatWorkersMin: 0, + IdentityWorkersMin: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsAsyncSinkQueueUsageOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_depth{sink="nats",queue="raw"} 90 +vehicle_async_sink_queue_capacity{sink="nats",queue="raw"} 100 +vehicle_async_sink_queue_depth{sink="nats",queue="derived"} 10 +vehicle_async_sink_queue_capacity{sink="nats",queue="derived"} 100`, + }, Thresholds{AsyncSinkQueueMaxUsageRatio: 0.80}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway async sink queue usage 0.90 exceeds 0.80", + "queue=raw", + "sink=nats", + "depth 90", + "capacity 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if strings.Contains(joined, "queue=derived") { + t.Fatalf("derived queue should not be reported:\n%s", joined) + } +} + +func TestEvaluateAllowsAsyncSinkQueueUsageUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_depth{sink="kafka",queue="raw"} 79 +vehicle_async_sink_queue_capacity{sink="kafka",queue="raw"} 100`, + }, Thresholds{AsyncSinkQueueMaxUsageRatio: 0.80}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableAsyncSinkQueueUsageLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_async_sink_queue_depth{sink="nats",queue="derived"} 100 +vehicle_async_sink_queue_capacity{sink="nats",queue="derived"} 100`, + }, Thresholds{AsyncSinkQueueMaxUsageRatio: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDurableSpoolBacklogAndErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_durable_spool_backlog_files{name="nats"} 10001 +vehicle_durable_spool_oldest_age_seconds{name="nats"} 601 +vehicle_durable_spool_records_total{kind="raw",name="nats",status="error"} 2 +vehicle_durable_spool_replay_total{name="nats",status="publish_error"} 1`, + }, Thresholds{ + DurableSpoolBacklogMax: 10000, + DurableSpoolOldestAgeMaxSec: 300, + DurableSpoolErrorMax: 0, + DurableSpoolReplayErrorMax: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "durable spool backlog 10001 exceeds 10000 for name nats", + "durable spool oldest age 601s exceeds 300s for name nats", + "durable spool record errors 2 exceeds 0 for name nats kind raw", + "durable spool replay publish_error 1 exceeds 0 for name nats", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableDurableSpoolLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_durable_spool_backlog_files{name="kafka"} 999999 +vehicle_durable_spool_oldest_age_seconds{name="kafka"} 999999 +vehicle_durable_spool_records_total{kind="fields",name="kafka",status="error"} 99 +vehicle_durable_spool_replay_total{name="kafka",status="delete_error"} 99`, + }, Thresholds{ + DurableSpoolBacklogMax: 0, + DurableSpoolOldestAgeMaxSec: 0, + DurableSpoolErrorMax: -1, + DurableSpoolReplayErrorMax: -1, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDurableOutboxInflightAndErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_durable_outbox_inflight{name="nats-outbox"} 10001 +vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="submitted"} 50000 +vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="acked"} 49998 +vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="ack_error"} 2 +vehicle_durable_outbox_publish_total{kind="all",name="nats-outbox",status="close_timeout"} 1`, + }, Thresholds{ + DurableOutboxInflightMax: 10000, + DurableOutboxErrorMax: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "durable outbox inflight 10001 exceeds 10000 for name nats-outbox", + "durable outbox ack_error 2 exceeds 0 for name nats-outbox kind raw", + "durable outbox close_timeout 1 exceeds 0 for name nats-outbox kind all", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if strings.Contains(joined, "submitted") || strings.Contains(joined, "acked") { + t.Fatalf("successful outbox statuses must not create findings:\n%s", joined) + } +} + +func TestEvaluateCanDisableDurableOutboxLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_durable_outbox_inflight{name="nats-outbox"} 999999 +vehicle_durable_outbox_publish_total{kind="raw",name="nats-outbox",status="submit_error"} 99`, + }, Thresholds{ + DurableOutboxInflightMax: 0, + DurableOutboxErrorMax: -1, + }) + + if report.Status != StatusOK || len(report.Findings) != 0 { + t.Fatalf("report = %#v, want ok without findings", report) + } +} + +func TestEvaluateCanDisableStatBatchPendingLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_batch_pending_messages 999999`, + }, Thresholds{StatBatchPendingMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableRealtimeBatchPendingLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_batch_pending_messages 999999`, + }, Thresholds{RealtimeBatchPendingMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsBridgeRouteErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_messages_total{status="route_error",subject="vehicle.unconfigured.v1"} 1`, + }, Thresholds{BridgeRouteErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "bridge route errors 1 exceeds 0 for subject vehicle.unconfigured.v1") { + t.Fatalf("route error finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableBridgeRouteErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_messages_total{status="route_error",subject="vehicle.unconfigured.v1"} 999`, + }, Thresholds{BridgeRouteErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsBridgeKafkaWriteErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="error",topic="vehicle.fields.go.jt808.v1"} 2`, + }, Thresholds{BridgeKafkaWriteErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "bridge kafka write errors 2 exceeds 0 for topic vehicle.fields.go.jt808.v1") { + t.Fatalf("kafka write error finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableBridgeKafkaWriteErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 99`, + }, Thresholds{BridgeKafkaWriteErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsBridgeUnackedMessageRatio(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1000 +vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 700 +vehicle_bridge_nats_acks_total{status="dropped_route_error",subject="vehicle.raw.go.jt808.v1"} 50`, + }, Thresholds{ + BridgeMessageMinReceived: 100, + BridgeMessageMaxUnackedRatio: 0.20, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge unacked message ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.jt808.v1", + "received 1000", + "ack ok 700", + "dropped route error 50", + "unacked 250", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateAllowsBridgeCompletedAndLowSampleMessages(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.gb32960.v1"} 100 +vehicle_bridge_nats_acks_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 80 +vehicle_bridge_nats_acks_total{status="dropped_route_error",subject="vehicle.raw.go.gb32960.v1"} 20 +vehicle_bridge_messages_total{status="received",subject="vehicle.raw.go.yutong-mqtt.v1"} 99`, + }, Thresholds{ + BridgeMessageMinReceived: 100, + BridgeMessageMaxUnackedRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsKafkaConsumerUncommittedMessageRatio(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_kafka_messages_total{status="received",topic="vehicle.fields.go.jt808.v1"} 1000 +vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 700 +vehicle_stat_kafka_commits_total{status="error",topic="vehicle.fields.go.jt808.v1"} 10`, + }, Thresholds{ + KafkaConsumerMessageMinReceived: 100, + KafkaConsumerMaxUncommittedRatio: 0.20, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat kafka consumer uncommitted message ratio 0.30 exceeds 0.20 for topic vehicle.fields.go.jt808.v1", + "received 1000", + "commit ok 700", + "commit errors 10", + "uncommitted 300", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsKafkaConsumerCommitErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 3 +vehicle_realtime_kafka_commits_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`, + }, Thresholds{ + KafkaConsumerCommitErrorMax: 0, + KafkaConsumerMessageMinReceived: 100, + KafkaConsumerMaxUncommittedRatio: 0.20, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime kafka consumer commit errors 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1") { + t.Fatalf("commit error finding missing in:\n%s", joined) + } + if strings.Contains(joined, "uncommitted message ratio") { + t.Fatalf("low-sample uncommitted ratio should not be enforced:\n%s", joined) + } +} + +func TestEvaluateCanDisableKafkaConsumerCommitErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 3 +vehicle_realtime_kafka_commits_total{status="error",topic="vehicle.raw.go.gb32960.v1"} 1`, + }, Thresholds{ + KafkaConsumerCommitErrorMax: -1, + KafkaConsumerMessageMinReceived: 100, + KafkaConsumerMaxUncommittedRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateAllowsKafkaConsumerCompletedAndLowSampleMessages(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 100 +vehicle_history_kafka_commits_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 80 +vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 99`, + }, Thresholds{ + KafkaConsumerMessageMinReceived: 100, + KafkaConsumerMaxUncommittedRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsFastWriterInvalidJSON(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{FastWriterInvalidJSONMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "fast writer invalid json 1 exceeds 0 for subject vehicle.raw.go.jt808.v1") { + t.Fatalf("invalid json finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableFastWriterInvalidJSONLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{FastWriterInvalidJSONMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsFastWriterBatchFallbackErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_batch_fallback_total{stage="redis",status="error"} 2`, + }, Thresholds{FastWriterFallbackErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "fast writer batch fallback errors 2 exceeds 0 for stage redis") { + t.Fatalf("fallback finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableFastWriterBatchFallbackErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 99`, + }, Thresholds{FastWriterFallbackErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsFastWriterDecoupledUpdateErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 5 +vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="error"} 1`, + }, Thresholds{FastWriterDecoupledUpdateErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "fast writer decoupled update errors 1 exceeds 0 for reason tdengine_transient") { + t.Fatalf("decoupled update finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableFastWriterDecoupledUpdateErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="error"} 99`, + }, Thresholds{FastWriterDecoupledUpdateErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsHistoryInvalidJSON(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{HistoryInvalidJSONMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "history invalid json 1 exceeds 0 for topic vehicle.raw.go.jt808.v1") { + t.Fatalf("invalid json finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableHistoryInvalidJSONLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_history_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{HistoryInvalidJSONMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsHistoryLocationWriteErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 2`, + }, Thresholds{HistoryLocationErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history location writes error 2 exceeds 0", + "vehicle.raw.go.jt808.v1", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableHistoryLocationWriteErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_location_writes_total{status="error",topic="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{HistoryLocationErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsHistoryLocationAccountingMismatch(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 70 +vehicle_history_location_writes_total{status="skipped_non_realtime",topic="vehicle.raw.go.jt808.v1"} 10`, + }, Thresholds{HistoryLocationAccountingMaxRatio: 0.02}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history location accounting mismatch ratio 0.20 exceeds 0.02", + "history writes ok 100, location statuses 80, diff 20", + "vehicle.raw.go.jt808.v1", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateAcceptsHistoryLocationAccountingWhenStatusesMatchWrites(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 70 +vehicle_history_location_writes_total{status="skipped_non_realtime",topic="vehicle.raw.go.jt808.v1"} 20 +vehicle_history_location_writes_total{status="skipped_missing_coordinates",topic="vehicle.raw.go.jt808.v1"} 10`, + }, Thresholds{HistoryLocationAccountingMaxRatio: 0.02}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableHistoryLocationAccountingLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_location_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{HistoryLocationAccountingMaxRatio: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsHistoryBatchFallbackErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_fallback_total{status="error"} 2`, + }, Thresholds{HistoryFallbackErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "history batch fallback errors 2 exceeds 0") { + t.Fatalf("fallback finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableHistoryBatchFallbackErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_fallback_total{status="error"} 99`, + }, Thresholds{HistoryFallbackErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsWriteRetriesOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_write_retries_total{operation="batch",status="retry"} 11 +vehicle_history_write_retries_total{operation="batch",status="exhausted"} 1`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_write_retries_total{operation="single",status="retry"} 12 +vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`, + }, Thresholds{ + HistoryRetryMax: 10, + HistoryRetryExhaustedMax: 0, + StatRetryMax: 10, + StatRetryExhaustedMax: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history write retries 11 exceeds 10 for operation batch", + "history write retry exhausted 1 exceeds 0 for operation batch", + "stat write retries 12 exceeds 10 for operation single", + "stat write retry exhausted 1 exceeds 0 for operation single", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableWriteRetryLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_write_retries_total{operation="batch",status="retry"} 99 +vehicle_history_write_retries_total{operation="batch",status="exhausted"} 99`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_write_retries_total{operation="single",status="retry"} 99 +vehicle_stat_write_retries_total{operation="single",status="exhausted"} 99`, + }, Thresholds{ + HistoryRetryMax: -1, + HistoryRetryExhaustedMax: -1, + StatRetryMax: -1, + StatRetryExhaustedMax: -1, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeInvalidJSON(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_realtime_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.gb32960.v1"} 1`, + }, Thresholds{RealtimeInvalidJSONMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime invalid json 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1") { + t.Fatalf("invalid json finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableRealtimeInvalidJSONLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_realtime_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.gb32960.v1"} 99`, + }, Thresholds{RealtimeInvalidJSONMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsStatInvalidJSON(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{StatInvalidJSONMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "stat invalid json 1 exceeds 0 for topic vehicle.fields.go.jt808.v1") { + t.Fatalf("invalid json finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableStatInvalidJSONLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 99`, + }, Thresholds{StatInvalidJSONMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsStatProtocolTopicMismatches(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`, + }, Thresholds{StatProtocolTopicMismatchMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "stat protocol topic mismatches 1 exceeds 0 for topic vehicle.fields.go.gb32960.v1") { + t.Fatalf("protocol topic mismatch finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsMessageContractMismatches(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_realtime_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.raw.go.gb32960.v1"} 1`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.yutong-mqtt.v1"} 1`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{MessageContractMismatchMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history message contract event_kind_mismatch 1 exceeds 0 for topic vehicle.raw.go.jt808.v1", + "realtime message contract protocol_topic_mismatch 1 exceeds 0 for topic vehicle.raw.go.gb32960.v1", + "fast writer message contract event_kind_mismatch 1 exceeds 0 for subject vehicle.raw.go.yutong-mqtt.v1", + "stat message contract event_kind_mismatch 1 exceeds 0 for topic vehicle.fields.go.jt808.v1", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableMessageContractMismatchLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{MessageContractMismatchMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableStatProtocolTopicMismatchLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`, + }, Thresholds{StatProtocolTopicMismatchMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsStatMetricProtocolTopicMismatches(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.jt808.v1"} 12 +vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.gb32960.v1"} 7 +vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 5`, + }, Thresholds{StatProtocolTopicMismatchMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat metric protocol topic mismatches 12 exceeds 0 for vehicle_stat_samples_total topic vehicle.fields.go.jt808.v1: expected JT808, got GB32960 (status found)", + "stat metric protocol topic mismatches 7 exceeds 0 for vehicle_stat_sources_total topic vehicle.fields.go.gb32960.v1: expected GB32960, got YUTONG_MQTT (status written)", + "stat metric protocol topic mismatches 5 exceeds 0 for vehicle_stat_projections_total topic vehicle.fields.go.yutong-mqtt.v1: expected YUTONG_MQTT, got JT808 (status written)", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsStatMetricProtocolLabelMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_samples_total{status="found",topic="vehicle.fields.go.jt808.v1"} 12`, + }, Thresholds{StatProtocolTopicMismatchMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "stat metric protocol label missing 12 exceeds 0 for vehicle_stat_samples_total topic vehicle.fields.go.jt808.v1 (status found)") { + t.Fatalf("protocol label missing finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsDegradedWhenServiceInfoMetricIsMissing(t *testing.T) { + report := Evaluate(map[string]string{ + "realtime": ``, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime service info metric missing") { + t.Fatalf("service info finding missing in:\n%s", joined) + } +} + +func TestEvaluateAggregatesLastActivityTimestampsByMax(t *testing.T) { + report := Evaluate(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="GB32960",status="ok"} 100 +vehicle_gateway_last_publish_unix_seconds{kind="fields",protocol="GB32960",status="ok"} 120 +vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 3 +vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 4`, + }) + + if got := report.Services[0].Metrics["vehicle_gateway_last_publish_unix_seconds"]; got != 120 { + t.Fatalf("last publish aggregate = %v, want max timestamp 120", got) + } + if got := report.Services[0].Metrics["vehicle_gateway_last_publish_age_seconds"]; got <= 0 { + t.Fatalf("last publish age = %v, want positive age", got) + } + if got := report.Services[0].Metrics["vehicle_gateway_publish_total"]; got != 7 { + t.Fatalf("publish total aggregate = %v, want sum 7", got) + } +} + +func TestEvaluateReportsDegradedWhenObservedLastActivityIsStale(t *testing.T) { + stale := time.Now().UTC().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="ok"} ` + strconv.FormatInt(stale, 10), + }, Thresholds{LastActivityStaleSec: 300}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway last publish stale", + "exceeds 300s", + "kind=raw,protocol=JT808,status=ok", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeForFreshOrErrorLastActivity(t *testing.T) { + fresh := time.Now().UTC().Unix() + staleError := time.Now().UTC().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="ok"} ` + strconv.FormatInt(fresh, 10) + ` +vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="error"} ` + strconv.FormatInt(staleError, 10), + }, Thresholds{LastActivityStaleSec: 300}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenFastWriterLastActivityIsStale(t *testing.T) { + stale := time.Now().UTC().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(stale, 10) + ` +vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} ` + strconv.FormatInt(stale, 10), + }, Thresholds{LastActivityStaleSec: 300}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer last message stale", + "fast writer last stage stale", + "subject=vehicle.raw.go.jt808.v1", + "stage=redis", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateSkipsRollupLastActivitySubjects(t *testing.T) { + stale := time.Now().UTC().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="mixed"} ` + strconv.FormatInt(stale, 10) + ` +vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="unknown"} ` + strconv.FormatInt(stale, 10), + }, Thresholds{LastActivityStaleSec: 300}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeStoreLastActivityStale(t *testing.T) { + stale := time.Now().UTC().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_realtime_last_store_update_unix_seconds{protocol="JT808",status="ok",store="mysql"} ` + strconv.FormatInt(stale, 10), + }, Thresholds{LastActivityStaleSec: 300}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "realtime last store update stale", + "exceeds 300s", + "protocol=JT808,status=ok,store=mysql", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenKafkaConsumerInfoIsMissing(t *testing.T) { + report := Evaluate(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1`, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "stat kafka consumer info metric missing") { + t.Fatalf("kafka consumer info finding missing in:\n%s", joined) + } +} + +func TestEvaluateDoesNotDegradeForSmallKafkaLag(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_kafka_lag{topic="vehicle.fields.go.jt808.v1",partition="0"} 8`, + }, Thresholds{KafkaLagMax: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } + if report.Totals.KafkaLag != 8 { + t.Fatalf("kafka lag total = %v, want 8", report.Totals.KafkaLag) + } +} + +func TestEvaluateReportsDegradedWhenRealtimeRedisProjectorIsEnabled(t *testing.T) { + report := Evaluate(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_redis_projector_enabled 1 +vehicle_realtime_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime redis projector enabled") { + t.Fatalf("realtime redis projector finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsDegradedWhenGatewayRawHasNoFieldsPublish(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`, + }, Thresholds{GatewayFieldMinRaw: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway fields missing for protocol JT808", + "raw ok 120", + "realtime candidates 120", + "fields ok 0", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotRequireFieldsForNonRealtimeGatewayRaw(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120 +vehicle_gateway_fields_total{protocol="GB32960",status="skipped_non_realtime"} 120`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateAcceptsFieldsDelegatedToBridge(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120 +vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="delegated"} 120 +vehicle_gateway_fields_total{protocol="JT808",status="delegated_to_bridge"} 120 +vehicle_gateway_fields_count{protocol="JT808",status="delegated_to_bridge"} 31`, + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 120 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2, GatewayFieldMinCount: 1, GatewayBridgeMinPublish: 100}) + + if report.Status != StatusOK || len(report.Findings) != 0 { + t.Fatalf("report = %#v, want delegated fields flow ok", report) + } + if report.GatewayFields == nil || len(report.GatewayFields.Protocols) != 1 { + t.Fatalf("gateway fields diagnostics = %#v", report.GatewayFields) + } + row := report.GatewayFields.Protocols[0] + if row.FieldsOK != 120 || row.FieldsPublished != 0 || row.FieldsDelegated != 120 { + t.Fatalf("delegated fields diagnostics = %#v", row) + } +} + +func TestEvaluateReportsDegradedWhenGatewayFieldsMissingRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 200 +vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 130 +vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 130 +vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_non_realtime"} 20 +vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_missing_fields"} 50`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway fields missing ratio 0.28 exceeds 0.20 for protocol YUTONG_MQTT", + "realtime candidates 180", + "skipped missing fields 50", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayPublishesSomeFields(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120 +vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 1`, + }, Thresholds{GatewayFieldMinRaw: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateIncludesGatewayFieldsDiagnostics(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 200 +vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 170 +vehicle_gateway_fields_total{protocol="JT808",status="published"} 170 +vehicle_gateway_fields_total{protocol="JT808",status="skipped_non_realtime"} 20 +vehicle_gateway_fields_total{protocol="JT808",status="skipped_missing_fields"} 10 +vehicle_gateway_fields_count{protocol="JT808",status="published"} 31`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMaxMissingRatio: 0.2, GatewayFieldMinCount: 1}) + + if report.GatewayFields == nil { + t.Fatal("gateway fields diagnostics = nil") + } + if got, want := len(report.GatewayFields.Protocols), 1; got != want { + t.Fatalf("gateway fields protocols = %d, want %d: %#v", got, want, report.GatewayFields.Protocols) + } + row := report.GatewayFields.Protocols[0] + if row.Protocol != "JT808" || row.Status != "ok" || row.Reason != "" { + t.Fatalf("gateway fields row identity/status = %#v", row) + } + if row.RawOK != 200 || row.RealtimeCandidates != 180 || row.FieldsOK != 170 || row.FieldsPublished != 170 { + t.Fatalf("gateway fields counters = %#v", row) + } + if row.FieldsSkippedNonRealtime != 20 || row.FieldsSkippedMissing != 10 || row.FieldsPublishError != 0 { + t.Fatalf("gateway fields skip counters = %#v", row) + } + if math.Abs(row.MissingRatio-(10.0/180.0)) > 0.0001 { + t.Fatalf("missing ratio = %v, want %v", row.MissingRatio, 10.0/180.0) + } + if row.LatestFieldCount == nil || *row.LatestFieldCount != 31 { + t.Fatalf("latest field count = %#v, want 31", row.LatestFieldCount) + } +} + +func TestEvaluateIncludesProtocolFlowDiagnostics(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 200 +vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 170 +vehicle_gateway_fields_total{protocol="JT808",status="published"} 170 +vehicle_gateway_fields_total{protocol="JT808",status="skipped_non_realtime"} 20 +vehicle_gateway_fields_count{protocol="JT808",status="published"} 31 +vehicle_gateway_identity_skips_total{protocol="JT808",reason="non_vehicle_frame"} 7 +vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 160 +vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 40`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 170 +vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 160 +vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 140 +vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_mileage",topic="vehicle.fields.go.jt808.v1"} 10 +vehicle_stat_sources_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_sources_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 159 +vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 139`, + }, Thresholds{ + GatewayFieldMinRaw: 100, + GatewayFieldMaxMissingRatio: 0.2, + GatewayFieldMinCount: 1, + GatewayIdentityMinSamples: 100, + GatewayIdentityMaxUnresolvedRatio: 0.25, + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.20, + StatSourceMinSamples: 100, + StatSourceMaxMissingRatio: 0.60, + StatProjectionMinSampleWrites: 100, + }) + + if report.ProtocolFlow == nil { + t.Fatal("protocol flow diagnostics = nil") + } + if got, want := len(report.ProtocolFlow.Protocols), 1; got != want { + t.Fatalf("protocol flow rows = %d, want %d: %#v", got, want, report.ProtocolFlow.Protocols) + } + row := report.ProtocolFlow.Protocols[0] + if row.Protocol != "JT808" || row.Status != "ok" || row.Reason != "" { + t.Fatalf("protocol flow row identity/status = %#v", row) + } + if row.RawOK != 200 || row.RealtimeCandidates != 180 || row.FieldsPublished != 170 { + t.Fatalf("gateway counters = %#v", row) + } + if row.LatestFieldCount == nil || *row.LatestFieldCount != 31 { + t.Fatalf("latest field count = %#v, want 31", row.LatestFieldCount) + } + if row.IdentityTotal != 200 || row.IdentitySkipped != 7 || row.IdentityResolved != 160 || row.IdentityNonResolved != 40 { + t.Fatalf("identity counters = %#v", row) + } + if math.Abs(row.IdentityNonResolvedRatio-0.2) > 0.0001 { + t.Fatalf("identity ratio = %v, want 0.2", row.IdentityNonResolvedRatio) + } + if row.StatWritesOK != 170 || row.StatMileageCandidateWrites != 170 || row.StatSamplesFound != 160 || row.StatSamplesWritten != 140 { + t.Fatalf("stat sample counters = %#v", row) + } + if row.StatSamplesActionableSkipped != 10 || row.StatSkippedMissingMileage != 10 { + t.Fatalf("stat skip counters = %#v", row) + } + if row.StatSourceWritten != 1 || row.StatSourceThrottled != 159 { + t.Fatalf("stat source counters = %#v", row) + } + if row.StatProjectionWritten != 1 || row.StatProjectionThrottled != 139 { + t.Fatalf("stat projection counters = %#v", row) + } +} + +func TestProtocolFlowDiagnosticsHighlightsStatProjectionBreak(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_sources_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_sources_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 119 +vehicle_stat_projections_total{protocol="GB32960",status="attempted",topic="vehicle.fields.go.gb32960.v1"} 120`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSourceMinSamples: 100, + StatProjectionMinSampleWrites: 100, + }) + + if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 { + t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow) + } + row := report.ProtocolFlow.Protocols[0] + if row.Protocol != "GB32960" || row.Status != "degraded" || row.Reason != "stat_projection_missing" { + t.Fatalf("protocol flow row = %#v", row) + } + if row.StatProjectionAttempted != 120 || row.StatProjectionWritten != 0 { + t.Fatalf("projection counters = %#v", row) + } +} + +func TestEvaluateReportsDegradedWhenGatewayPublishedFieldsAreEmpty(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120 +vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 120 +vehicle_gateway_fields_total{protocol="JT808",status="published"} 120 +vehicle_gateway_fields_count{protocol="JT808",status="published"} 0`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway fields count 0 below 1 for protocol JT808", + "published fields events", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if report.GatewayFields == nil || len(report.GatewayFields.Protocols) != 1 { + t.Fatalf("gateway fields diagnostics missing: %#v", report.GatewayFields) + } + row := report.GatewayFields.Protocols[0] + if row.Status != "degraded" || row.Reason != "low_field_count" { + t.Fatalf("gateway fields diagnostic row = %#v", row) + } +} + +func TestEvaluateReportsDegradedWhenGatewayFieldsCountMetricIsMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120 +vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 120 +vehicle_gateway_fields_total{protocol="GB32960",status="published"} 120`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway fields count metric missing for protocol GB32960 after 120 published fields events") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateSkipsGatewayFieldCountWhenPublishedSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 2 +vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 2 +vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 2 +vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="published"} 0`, + }, Thresholds{GatewayFieldMinRaw: 100, GatewayFieldMinCount: 1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenGatewayRawPublishDoesNotReachBridge(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`, + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4`, + }, Thresholds{GatewayBridgeMinPublish: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge writes missing for gateway raw publish protocol JT808", + "gateway ok 120", + "expected topic vehicle.raw.go.jt808.v1", + "bridge writes ok 0", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenGatewayFieldsPublishDoesNotReachBridge(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="fields",protocol="GB32960",status="ok"} 120`, + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4`, + }, Thresholds{GatewayBridgeMinPublish: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge writes missing for gateway fields publish protocol GB32960", + "expected topic vehicle.fields.go.gb32960.v1", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayPublishReachesBridge(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 120 +vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 120`, + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 1`, + }, Thresholds{GatewayBridgeMinPublish: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsGatewayBridgeFlowWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 99`, + }, Thresholds{GatewayBridgeMinPublish: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenGatewayPublishTopicMappingIsMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="UNKNOWN_PROTO",status="ok"} 120`, + }, Thresholds{GatewayBridgeMinPublish: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway raw publish topic mapping missing for protocol UNKNOWN_PROTO") { + t.Fatalf("mapping finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsDegradedWhenGatewayRawPublishDoesNotReachFastWriter(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="GB32960",status="ok"} 120`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8`, + }, Thresholds{GatewayFastWriterMinRawPublish: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer received messages missing for gateway raw publish protocol GB32960", + "gateway ok 120", + "expected subject vehicle.raw.go.gb32960.v1", + "fast writer received 0", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayRawPublishReachesFastWriter(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 120`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{GatewayFastWriterMinRawPublish: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsFastWriterUnackedMessageRatio(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.gb32960.v1"} 1000 +vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 700 +vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.gb32960.v1"} 50`, + }, Thresholds{ + FastWriterMessageMinReceived: 100, + FastWriterMessageMaxUnackedRatio: 0.20, + FastWriterInvalidJSONMax: -1, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer unacked message ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.gb32960.v1", + "received 1000", + "ok 700", + "invalid_json 50", + "unacked 250", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateAllowsFastWriterCompletedAndLowSampleMessages(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 90 +vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 10 +vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.yutong-mqtt.v1"} 99`, + }, Thresholds{ + FastWriterMessageMinReceived: 100, + FastWriterMessageMaxUnackedRatio: 0.20, + FastWriterInvalidJSONMax: -1, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateDoesNotRequireFastWriterForGatewayFieldsPublish(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 120`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8`, + }, Thresholds{GatewayFastWriterMinRawPublish: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenFastWriterRedisStaleFieldRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 2000 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="written"} 1500 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_stale"} 500`, + }, Thresholds{ + FastWriterRedisFieldMinSeen: 1000, + FastWriterRedisFieldMaxStaleRatio: 0.20, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis stale field ratio 0.25 exceeds 0.20 for subject vehicle.raw.go.gb32960.v1", + "seen 2000", + "written 1500", + "skipped_stale 500", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenFastWriterRedisEnvelopeUpdatesAreMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 120 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_non_realtime"} 120`, + }, Thresholds{ + FastWriterRedisEnvelopeMinSeen: 100, + FastWriterRedisEnvelopeMaxBadRatio: 0.50, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis realtime updates missing for subject vehicle.raw.go.gb32960.v1", + "envelopes seen 120", + "updated 0", + "skipped non-realtime 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenFastWriterRedisEnvelopeIdentitySkipsAreHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="seen"} 200 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="updated"} 70 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.jt808.v1",status="skipped_missing_vin"} 130`, + }, Thresholds{ + FastWriterRedisEnvelopeMinSeen: 100, + FastWriterRedisEnvelopeMaxBadRatio: 0.50, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis envelope skip ratio 0.65 exceeds 0.50 for subject vehicle.raw.go.jt808.v1", + "envelopes seen 200", + "updated 70", + "missing vin 130", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenFastWriterRedisParsedFieldSkipsAreHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="seen"} 200 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="updated"} 70 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.gb32960.v1",status="skipped_missing_fields"} 130`, + }, Thresholds{ + FastWriterRedisEnvelopeMinSeen: 100, + FastWriterRedisEnvelopeMaxBadRatio: 0.50, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis envelope skip ratio 0.65 exceeds 0.50 for subject vehicle.raw.go.gb32960.v1", + "missing parsed fields 130", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenFastWriterRedisEnvelopeSkipsAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="seen"} 200 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="updated"} 180 +vehicle_fast_writer_redis_envelopes_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="skipped_missing_vin"} 20`, + }, Thresholds{ + FastWriterRedisEnvelopeMinSeen: 100, + FastWriterRedisEnvelopeMaxBadRatio: 0.50, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateDoesNotDegradeWhenFastWriterRedisStaleFieldRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="seen"} 2000 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="written"} 1950 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.jt808.v1",status="skipped_stale"} 50`, + }, Thresholds{ + FastWriterRedisFieldMinSeen: 1000, + FastWriterRedisFieldMaxStaleRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsFastWriterRedisStaleFieldRatioWhenSamplesAreLowOrRollup(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="seen"} 999 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="written"} 1 +vehicle_fast_writer_redis_fields_total{subject="vehicle.raw.go.yutong-mqtt.v1",status="skipped_stale"} 998 +vehicle_fast_writer_redis_fields_total{subject="mixed",status="seen"} 2000 +vehicle_fast_writer_redis_fields_total{subject="mixed",status="written"} 1 +vehicle_fast_writer_redis_fields_total{subject="mixed",status="skipped_stale"} 1999`, + }, Thresholds{ + FastWriterRedisFieldMinSeen: 1000, + FastWriterRedisFieldMaxStaleRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsGatewayFastWriterFlowWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 99`, + }, Thresholds{GatewayFastWriterMinRawPublish: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenHistoryParsedFieldsAreMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_history_parsed_fields_total{protocol="GB32960",status="present",topic="vehicle.raw.go.gb32960.v1"} 90 +vehicle_history_parsed_fields_total{protocol="GB32960",status="missing",topic="vehicle.raw.go.gb32960.v1"} 10`, + }, Thresholds{ + HistoryParsedFieldMinFrames: 100, + HistoryParsedFieldMaxMissingRatio: 0.05, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history parsed fields missing ratio 0.10 exceeds 0.05 for topic vehicle.raw.go.gb32960.v1 protocol GB32960", + "total 100", + "present 90", + "missing 10", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenHistoryParsedFieldMissingRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_parsed_fields_total{protocol="JT808",status="present",topic="vehicle.raw.go.jt808.v1"} 98 +vehicle_history_parsed_fields_total{protocol="JT808",status="missing",topic="vehicle.raw.go.jt808.v1"} 2`, + }, Thresholds{ + HistoryParsedFieldMinFrames: 100, + HistoryParsedFieldMaxMissingRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsHistoryParsedFieldCheckWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_history_parsed_fields_total{protocol="YUTONG_MQTT",status="present",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_history_parsed_fields_total{protocol="YUTONG_MQTT",status="missing",topic="vehicle.raw.go.yutong-mqtt.v1"} 98`, + }, Thresholds{ + HistoryParsedFieldMinFrames: 100, + HistoryParsedFieldMaxMissingRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenGatewayParseBadRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_frames_total{protocol="JT808",status="OK"} 90 +vehicle_gateway_frames_total{protocol="JT808",status="PARTIAL"} 5 +vehicle_gateway_frames_total{protocol="JT808",status="BAD_FRAME"} 10`, + }, Thresholds{ + GatewayParseMinFrames: 100, + GatewayParseMaxBadRatio: 0.05, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway parse bad ratio 0.14 exceeds 0.05 for protocol JT808", + "total 105", + "ok 90", + "bad 15", + "partial 5", + "bad_frame 10", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayParseBadRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_frames_total{protocol="GB32960",status="OK"} 99 +vehicle_gateway_frames_total{protocol="GB32960",status="BAD_FRAME"} 1`, + }, Thresholds{ + GatewayParseMinFrames: 100, + GatewayParseMaxBadRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsGatewayParseQualityWhenSampleCountIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1 +vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="BAD_FRAME"} 9`, + }, Thresholds{ + GatewayParseMinFrames: 100, + GatewayParseMaxBadRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenGatewayResponseErrorRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_response_total{message_id="0x0100",protocol="JT808",status="ok"} 90 +vehicle_gateway_response_total{message_id="0x0100",protocol="JT808",status="build_error"} 5 +vehicle_gateway_response_total{message_id="0x0200",protocol="JT808",status="write_error"} 10 +vehicle_gateway_response_total{message_id="0x0001",protocol="JT808",status="skipped"} 1000`, + }, Thresholds{ + GatewayResponseMinSamples: 100, + GatewayResponseMaxErrorRatio: 0.05, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway response error ratio 0.14 exceeds 0.05 for protocol JT808", + "total 105", + "ok 90", + "errors 15", + "build_error 5", + "write_error 10", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayResponseErrorRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_response_total{message_id="0x05",protocol="GB32960",status="ok"} 99 +vehicle_gateway_response_total{message_id="0x05",protocol="GB32960",status="write_error"} 1`, + }, Thresholds{ + GatewayResponseMinSamples: 100, + GatewayResponseMaxErrorRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsGatewayResponseQualityWhenSampleCountIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_response_total{message_id="0x0200",protocol="JT808",status="write_error"} 9 +vehicle_gateway_response_total{message_id="0x0001",protocol="JT808",status="skipped"} 1000`, + }, Thresholds{ + GatewayResponseMinSamples: 100, + GatewayResponseMaxErrorRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenGatewayIdentityUnresolvedRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 60 +vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 30 +vehicle_gateway_identity_total{protocol="JT808",status="timeout"} 10`, + }, Thresholds{ + GatewayIdentityMinSamples: 100, + GatewayIdentityMaxUnresolvedRatio: 0.20, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway identity unresolved ratio 0.40 exceeds 0.20 for protocol JT808", + "total 100", + "unresolved 40", + "resolved 60", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateIncludesIdentityResolutionDiagnostics(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 60 +vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 30 +vehicle_gateway_identity_total{protocol="JT808",status="timeout"} 7 +vehicle_gateway_identity_total{protocol="JT808",status="error"} 3 +vehicle_gateway_identity_issues_total{protocol="JT808",status="unresolved",message_id="0x0200",reason="no_binding"} 30 +vehicle_gateway_identity_issues_total{protocol="JT808",status="timeout",message_id="0x0200",reason="timeout"} 7 +vehicle_gateway_identity_issues_total{protocol="JT808",status="error",message_id="0x0100",reason="resolver_error"} 3 +vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="hit"} 80 +vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 5 +vehicle_gateway_identity_cache_entries{cache="lookup"} 123 +vehicle_gateway_identity_cache_entries{cache="registration"} 45 +vehicle_gateway_identity_skips_total{protocol="GB32960",reason="non_vehicle_frame"} 12 +vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 100`, + }, Thresholds{}) + + if report.IdentityResolution == nil { + t.Fatal("identity resolution diagnostics = nil") + } + got := report.IdentityResolution + if got.Total != 200 || got.ResolvedCount != 160 || got.NonResolvedCount != 40 { + t.Fatalf("identity totals = %#v", got) + } + if got.SkippedCount != 12 { + t.Fatalf("identity skipped = %d, want 12", got.SkippedCount) + } + if got.UnresolvedCount != 30 || got.TimeoutCount != 7 || got.ErrorCount != 3 { + t.Fatalf("identity error split = %#v", got) + } + if len(got.Issues) != 3 { + t.Fatalf("identity issues = %#v, want 3 rows", got.Issues) + } + if got.Issues[0] != (IdentityIssueCount{Protocol: "JT808", Status: "error", MessageID: "0x0100", Reason: "resolver_error", Count: 3}) { + t.Fatalf("first identity issue = %#v", got.Issues[0]) + } + if got.Issues[1] != (IdentityIssueCount{Protocol: "JT808", Status: "timeout", MessageID: "0x0200", Reason: "timeout", Count: 7}) { + t.Fatalf("second identity issue = %#v", got.Issues[1]) + } + if got.Issues[2] != (IdentityIssueCount{Protocol: "JT808", Status: "unresolved", MessageID: "0x0200", Reason: "no_binding", Count: 30}) { + t.Fatalf("third identity issue = %#v", got.Issues[2]) + } + if got.CacheEntries["lookup"] != 123 || got.CacheEntries["registration"] != 45 { + t.Fatalf("cache entries = %#v", got.CacheEntries) + } + if len(got.Protocols) != 2 || got.Protocols[0].Protocol != "GB32960" || got.Protocols[1].Protocol != "JT808" { + t.Fatalf("protocol rows = %#v", got.Protocols) + } + if got.Protocols[0].SkippedCount != 12 || got.Protocols[0].NonResolvedCount != 0 { + t.Fatalf("gb32960 skipped row = %#v", got.Protocols[0]) + } + jt808 := got.Protocols[1] + if jt808.Total != 100 || jt808.NonResolvedCount != 40 || jt808.NonResolvedRatio != 0.40 { + t.Fatalf("jt808 row = %#v", jt808) + } + if jt808.StaleCacheCount != 5 || jt808.StaleCacheRatio != 0.05 || jt808.CacheStatusCounts["stale"] != 5 { + t.Fatalf("jt808 cache row = %#v", jt808) + } +} + +func TestEvaluateDoesNotDegradeWhenGatewayIdentityUnresolvedRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 98 +vehicle_gateway_identity_total{protocol="GB32960",status="unresolved"} 2`, + }, Thresholds{ + GatewayIdentityMinSamples: 100, + GatewayIdentityMaxUnresolvedRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsGatewayIdentityQualityWhenSampleCountIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 2 +vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="unresolved"} 8`, + }, Thresholds{ + GatewayIdentityMinSamples: 100, + GatewayIdentityMaxUnresolvedRatio: 0.20, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsGatewayIdentityCacheEntriesOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="lookup"} 300001`, + }, Thresholds{GatewayIdentityCacheEntriesMax: 300000}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway identity cache entries 300001 exceeds 300000 for cache lookup") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsGatewayIdentitySnapshotNotReady(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_snapshot_ready 0`, + }, Thresholds{ + GatewayIdentitySnapshotRequired: 1, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway identity snapshot ready 0 below 1") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsGatewayIdentitySnapshotTooOld(t *testing.T) { + lastSuccess := time.Now().Add(-10 * time.Minute).Unix() + report := EvaluateWithThresholds(map[string]string{ + "gateway": fmt.Sprintf(`vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_snapshot_ready 1 +vehicle_gateway_identity_snapshot_last_success_unix_seconds %d`, lastSuccess), + }, Thresholds{ + GatewayIdentitySnapshotRequired: 1, + GatewayIdentitySnapshotMaxAgeSec: 180, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway identity snapshot age") || !strings.Contains(joined, "exceeds 180s") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateAllowsFreshGatewayIdentitySnapshot(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": fmt.Sprintf(`vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_snapshot_ready 1 +vehicle_gateway_identity_snapshot_last_success_unix_seconds %d`, time.Now().Unix()), + }, Thresholds{ + GatewayIdentitySnapshotRequired: 1, + GatewayIdentitySnapshotMaxAgeSec: 180, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } +} + +func TestEvaluateReportsGatewayIdentityLocationTouchFailures(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="location_touch_failure"} 2`, + }, Thresholds{GatewayIdentityLocationTouchFailureMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway identity location touch failure cache 2 exceeds 0") { + t.Fatalf("finding missing in:\n%s", joined) + } + if !strings.Contains(joined, "jt808 registration writes are failing or in protective backoff") { + t.Fatalf("finding should explain registration write backoff in:\n%s", joined) + } +} + +func TestEvaluateReportsGatewayRegistrationWriteQueueDepth(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="lookup"} 500000 +vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 12001 +vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100000`, + }, Thresholds{ + GatewayIdentityRegistrationWriteQueueDepthMax: 10000, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80, + GatewayIdentityCacheEntriesMax: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + if report.Totals.JT808RegistrationWriteQueuePending != 12001 { + t.Fatalf("registration write queue pending = %v, want 12001", report.Totals.JT808RegistrationWriteQueuePending) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway jt808 registration write queue depth 12001 exceeds 10000") { + t.Fatalf("depth finding missing in:\n%s", joined) + } +} + +func TestEvaluateReportsGatewayRegistrationWriteQueueUsage(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 90 +vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100`, + }, Thresholds{ + GatewayIdentityRegistrationWriteQueueDepthMax: 0, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0.80, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway jt808 registration write queue usage 0.90 exceeds 0.80", + "depth 90", + "capacity 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableGatewayRegistrationWriteQueueLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 100 +vehicle_gateway_identity_cache_max_entries{cache="registration_write_queue"} 100`, + }, Thresholds{ + GatewayIdentityRegistrationWriteQueueDepthMax: 0, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if report.Totals.JT808RegistrationWriteQueuePending != 100 { + t.Fatalf("registration write queue pending = %v, want 100", report.Totals.JT808RegistrationWriteQueuePending) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsGatewayRegistrationWriteErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_jt808_registration_write_total{mode="async_enqueue",status="ok"} 100 +vehicle_gateway_jt808_registration_write_total{mode="async_background",status="error"} 2 +vehicle_gateway_jt808_registration_write_total{mode="sync",status="error"} 1`, + }, Thresholds{ + GatewayIdentityRegistrationWriteErrorMax: 0, + GatewayIdentityRegistrationWriteQueueDepthMax: 0, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + if report.Totals.JT808RegistrationWriteErrors != 3 { + t.Fatalf("registration write errors = %v, want 3", report.Totals.JT808RegistrationWriteErrors) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway jt808 registration write error count 2 exceeds 0 for mode async_background", + "gateway jt808 registration write error count 1 exceeds 0 for mode sync", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if strings.Contains(joined, "async_enqueue") { + t.Fatalf("ok status should not produce finding:\n%s", joined) + } +} + +func TestEvaluateCanDisableGatewayRegistrationWriteErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_jt808_registration_write_total{mode="async_background",status="error"} 2`, + }, Thresholds{ + GatewayIdentityRegistrationWriteErrorMax: -1, + GatewayIdentityRegistrationWriteQueueDepthMax: 0, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if report.Totals.JT808RegistrationWriteErrors != 2 { + t.Fatalf("registration write errors = %v, want 2", report.Totals.JT808RegistrationWriteErrors) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestGatewayIdentityCacheLimitIgnoresRegistrationWriteQueue(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="registration_write_queue"} 500000`, + }, Thresholds{ + GatewayIdentityCacheEntriesMax: 300000, + GatewayIdentityRegistrationWriteQueueDepthMax: 0, + GatewayIdentityRegistrationWriteQueueMaxUsageRatio: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableGatewayIdentityLocationTouchFailureLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="location_touch_failure"} 99`, + }, Thresholds{GatewayIdentityLocationTouchFailureMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateAllowsGatewayIdentityCacheEntriesUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="registration"} 299999`, + }, Thresholds{GatewayIdentityCacheEntriesMax: 300000}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableGatewayIdentityCacheEntryLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_cache_entries{cache="lookup"} 9999999`, + }, Thresholds{GatewayIdentityCacheEntriesMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsGatewayIdentityStaleCacheRatioOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 90 +vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 10 +vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 9`, + }, Thresholds{ + GatewayIdentityStaleMinSamples: 100, + GatewayIdentityMaxStaleCacheRatio: 0.05, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "gateway identity stale cache ratio 0.09 exceeds 0.05 for protocol JT808") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateAllowsGatewayIdentityStaleCacheRatioUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 100 +vehicle_gateway_identity_cache_total{protocol="GB32960",cache_status="stale"} 3`, + }, Thresholds{ + GatewayIdentityStaleMinSamples: 100, + GatewayIdentityMaxStaleCacheRatio: 0.05, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableGatewayIdentityStaleCacheRatio(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_total{protocol="JT808",status="resolved"} 100 +vehicle_gateway_identity_cache_total{protocol="JT808",cache_status="stale"} 100`, + }, Thresholds{ + GatewayIdentityStaleMinSamples: 0, + GatewayIdentityMaxStaleCacheRatio: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenRequiredConsumerTopicIsMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1`, + }, Thresholds{ + RequiredConsumerTopics: map[string][]string{ + "history": { + "vehicle.raw.go.gb32960.v1", + "vehicle.raw.go.jt808.v1", + }, + }, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "history required kafka consumer topic missing: vehicle.raw.go.jt808.v1") { + t.Fatalf("required topic finding missing in:\n%s", joined) + } +} + +func TestEvaluateDoesNotDegradeWhenRequiredConsumerTopicsArePresent(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{ + RequiredConsumerTopics: map[string][]string{ + "stat": { + "vehicle.fields.go.gb32960.v1", + "vehicle.fields.go.jt808.v1", + }, + }, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestDefaultRequiredConsumerTopicsIncludesIdentityWriter(t *testing.T) { + required := DefaultRequiredConsumerTopics() + if got := strings.Join(required["identity"], ","); got != "vehicle.raw.go.jt808.v1" { + t.Fatalf("identity required topics = %q, want JT808 raw", got) + } +} + +func TestEvaluateRejectsDuplicateJT808RegistrationWriters(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_jt808_registration_gateway_writes_enabled 1`, + "identity": `vehicle_service_info{service="vehicle-identity-writer"} 1 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + if joined := strings.Join(report.Findings, "\n"); !strings.Contains(joined, "duplicate writers") { + t.Fatalf("duplicate writer finding missing in:\n%s", joined) + } +} + +func TestEvaluateAcceptsDelegatedJT808RegistrationWriter(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_jt808_registration_gateway_writes_enabled 0`, + "identity": `vehicle_service_info{service="vehicle-identity-writer"} 1 +vehicle_kafka_consumer_info{service="vehicle-identity-writer",group="go-identity-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_identity_writer_batch_pending_messages 0 +vehicle_identity_writer_batch_pending_facts 0 +vehicle_identity_writer_kafka_lag{topic="vehicle.raw.go.jt808.v1",partition="0"} 0`, + }, Thresholds{}) + + if report.Status != StatusOK || len(report.Findings) != 0 { + t.Fatalf("report = %#v, want ok without findings", report) + } +} + +func TestEvaluateReportsDegradedWhenRawFanoutDoesNotReachHistoryOrRealtime(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 120`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1`, + }, Thresholds{RawFanoutMinBridge: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history messages missing for vehicle.raw.go.jt808.v1", + "realtime messages missing for vehicle.raw.go.jt808.v1", + "bridge raw writes ok 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenRawFanoutIsConsumedButNotWritten(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.gb32960.v1"} 120`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 120`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.gb32960.v1"} 120`, + }, Thresholds{RawFanoutMinBridge: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history writes missing for vehicle.raw.go.gb32960.v1", + "realtime updates missing for vehicle.raw.go.gb32960.v1", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenRawFanoutIsComplete(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_history_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 120 +vehicle_history_writes_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.yutong-mqtt.v1"} 120 +vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.yutong-mqtt.v1"} 120`, + }, Thresholds{RawFanoutMinBridge: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsRawFanoutWhenBridgeSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{RawFanoutMinBridge: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenBridgeFieldsTopicIsNotConsumedByStat(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{StatTopicMinBridge: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat messages missing for vehicle.fields.go.jt808.v1", + "bridge writes ok 120", + "stat received 0", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenBridgeFieldsTopicIsNotConfiguredByStat(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + }, Thresholds{StatTopicMinBridge: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat consumer topic missing for vehicle.fields.go.gb32960.v1", + "bridge writes ok 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenStatWritesHaveNoExtractedSamples(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 120 +vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_fields",topic="vehicle.fields.go.jt808.v1"} 120`, + }, Thresholds{StatSampleMinWrites: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat samples missing for vehicle.fields.go.jt808.v1", + "writes ok 120", + "missing fields 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenStatWritesAreOnlyNonMileageFrames(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_non_mileage_frame",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`, + }, Thresholds{StatSampleMinWrites: 100, StatSampleMaxActionableSkipRatio: 0.20}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } + if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 { + t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow) + } + row := report.ProtocolFlow.Protocols[0] + if row.Protocol != "YUTONG_MQTT" || row.Status != "ok" || row.StatMileageCandidateWrites != 0 || row.StatSkippedNonMileageFrame != 120 || row.StatSamplesActionableSkipped != 0 { + t.Fatalf("protocol flow row = %#v", row) + } +} + +func TestEvaluateDoesNotDegradeWhenStatWritesOnlyMissSourceMileage(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_missing_mileage",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`, + }, Thresholds{StatSampleMinWrites: 100, StatSampleMaxActionableSkipRatio: 0.20}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } + if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 { + t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow) + } + row := report.ProtocolFlow.Protocols[0] + if row.Protocol != "YUTONG_MQTT" || row.Status != "ok" || row.StatMileageCandidateWrites != 120 || row.StatSkippedMissingMileage != 120 || row.StatSamplesActionableSkipped != 120 { + t.Fatalf("protocol flow row = %#v", row) + } +} + +func TestEvaluateUsesMileageCandidatesForStatActionableSkipRatio(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1000 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 50 +vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 40 +vehicle_stat_samples_total{protocol="GB32960",status="skipped_non_mileage_frame",topic="vehicle.fields.go.gb32960.v1"} 900 +vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_mileage",topic="vehicle.fields.go.gb32960.v1"} 50`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.40, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat actionable skip ratio 0.50 exceeds 0.40 for vehicle.fields.go.gb32960.v1", + "writes ok 1000", + "mileage candidates 100", + "non-mileage frames 900", + "missing mileage 50", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 { + t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow) + } + row := report.ProtocolFlow.Protocols[0] + if row.StatMileageCandidateWrites != 100 || row.StatSamplesActionableSkipped != 50 || row.Status != "degraded" || row.Reason != "stat_actionable_skip_ratio" { + t.Fatalf("protocol flow row = %#v", row) + } +} + +func TestEvaluateDoesNotDegradeWhenStatSamplesAreOnlyDuplicateMileage(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.yutong-mqtt.v1"} 120 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="found",topic="vehicle.fields.go.yutong-mqtt.v1"} 120 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="skipped_same_mileage",topic="vehicle.fields.go.yutong-mqtt.v1"} 120`, + }, Thresholds{StatSampleMinWrites: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenStatActionableSkipRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 200 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 80 +vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 70 +vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_mileage",topic="vehicle.fields.go.gb32960.v1"} 100 +vehicle_stat_samples_total{protocol="GB32960",status="skipped_non_positive_mileage",topic="vehicle.fields.go.gb32960.v1"} 20`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.50, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat actionable skip ratio 0.60 exceeds 0.50 for vehicle.fields.go.gb32960.v1", + "writes ok 200", + "missing mileage 100", + "non-positive mileage 20", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateTreatsMissingTimeAsActionableStatSkip(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 200 +vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 80 +vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 70 +vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_time",topic="vehicle.fields.go.jt808.v1"} 120`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.50, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat actionable skip ratio 0.60 exceeds 0.50 for vehicle.fields.go.jt808.v1", + "missing time 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsFutureEventTimeAdjustments(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 200 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 190 +vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 180 +vehicle_stat_samples_total{protocol="GB32960",status="event_time_future_adjusted",topic="vehicle.fields.go.gb32960.v1"} 3`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSampleFutureAdjustmentMax: 0, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "stat future event time adjustments 3 exceeds 0 for vehicle.fields.go.gb32960.v1") { + t.Fatalf("future adjustment finding missing in:\n%s", joined) + } + if report.ProtocolFlow == nil || len(report.ProtocolFlow.Protocols) != 1 { + t.Fatalf("protocol flow diagnostics missing: %#v", report.ProtocolFlow) + } + row := report.ProtocolFlow.Protocols[0] + if row.Protocol != "GB32960" || row.StatEventTimeFutureAdjusted != 3 || row.Status != "degraded" || row.Reason != "stat_future_event_time_adjusted" { + t.Fatalf("protocol flow row = %#v", row) + } +} + +func TestEvaluateDoesNotDegradeWhenStatActionableSkipRatioIsLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 200 +vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 180 +vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 150 +vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_mileage",topic="vehicle.fields.go.jt808.v1"} 20`, + }, Thresholds{ + StatSampleMinWrites: 100, + StatSampleMaxActionableSkipRatio: 0.50, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenStatSamplesHaveNoSource(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_samples_total{protocol="GB32960",status="skipped_missing_source",topic="vehicle.fields.go.gb32960.v1"} 120`, + }, Thresholds{StatSampleMinWrites: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat samples not written for vehicle.fields.go.gb32960.v1", + "found 120", + "skipped missing source 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenStatSourceTrackingIsMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 120`, + }, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat source tracking missing for vehicle.fields.go.jt808.v1", + "samples found 120", + "source written 0", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsDegradedWhenStatSourceMissingEndpointRatioIsHigh(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="found",topic="vehicle.fields.go.yutong-mqtt.v1"} 200 +vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 10 +vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="skipped_throttled",topic="vehicle.fields.go.yutong-mqtt.v1"} 50 +vehicle_stat_sources_total{protocol="YUTONG_MQTT",status="skipped_missing_endpoint",topic="vehicle.fields.go.yutong-mqtt.v1"} 140`, + }, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat source missing endpoint ratio 0.70 exceeds 0.60 for vehicle.fields.go.yutong-mqtt.v1", + "samples found 200", + "missing endpoint 140", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenStatSourceTrackingIsHealthy(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_samples_total{protocol="GB32960",status="found",topic="vehicle.fields.go.gb32960.v1"} 200 +vehicle_stat_sources_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_sources_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 199`, + }, Thresholds{StatSourceMinSamples: 100, StatSourceMaxMissingRatio: 0.60}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedWhenStatFinalProjectionIsMissing(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 120 +vehicle_stat_projections_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 120 +vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 30`, + }, Thresholds{StatProjectionMinSampleWrites: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat final projection missing for vehicle.fields.go.jt808.v1", + "samples written 120", + "projections attempted 120", + "written 0", + "skipped throttled 30", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateDoesNotDegradeWhenStatFinalProjectionExists(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_samples_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 120 +vehicle_stat_projections_total{protocol="GB32960",status="written",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_projections_total{protocol="GB32960",status="skipped_throttled",topic="vehicle.fields.go.gb32960.v1"} 119`, + }, Thresholds{StatProjectionMinSampleWrites: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateSkipsStatFinalProjectionCheckWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.yutong-mqtt.v1"} 1 +vehicle_stat_samples_total{protocol="YUTONG_MQTT",status="written",topic="vehicle.fields.go.yutong-mqtt.v1"} 99 +vehicle_stat_projections_total{protocol="YUTONG_MQTT",status="attempted",topic="vehicle.fields.go.yutong-mqtt.v1"} 99`, + }, Thresholds{StatProjectionMinSampleWrites: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsStatCacheEntriesOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_cache_entries{cache="last_total_mileage"} 1200 +vehicle_stat_cache_entries{cache="source_seen"} 12`, + }, Thresholds{StatCacheEntriesMax: 1000}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat cache entries 1200 exceeds 1000 for cache last_total_mileage", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateAllowsStatCacheEntriesUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_cache_entries{cache="last_total_mileage"} 999 +vehicle_stat_cache_entries{cache="source_seen"} 12`, + }, Thresholds{StatCacheEntriesMax: 1000}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableStatCacheEntryLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_cache_entries{cache="baseline"} 9999999`, + }, Thresholds{StatCacheEntriesMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateAllowsRealtimeThrottleWhenSomeWritesPass(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_throttle_total{protocol="JT808",status="passed",store="mysql"} 20 +vehicle_realtime_throttle_total{protocol="JT808",status="skipped_interval",store="mysql"} 180`, + }, Thresholds{RealtimeThrottleMinSamples: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeThrottleWhenWritesNeverPass(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_realtime_throttle_total{protocol="GB32960",status="skipped_interval",store="mysql"} 120`, + }, Thresholds{RealtimeThrottleMinSamples: 100}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "realtime throttle passing updates missing for store=mysql,protocol=GB32960", + "total 120", + "passed 0", + "skipped interval 120", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateSkipsRealtimeThrottleWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.yutong-mqtt.v1"} 1 +vehicle_realtime_throttle_total{protocol="YUTONG_MQTT",status="skipped_interval",store="mysql"} 99`, + }, Thresholds{RealtimeThrottleMinSamples: 100}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeThrottleCacheEntriesOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_throttle_cache_entries{store="mysql"} 1200`, + }, Thresholds{RealtimeThrottleCacheEntriesMax: 1000}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime throttle cache entries 1200 exceeds 1000 for store mysql") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateAllowsRealtimeThrottleCacheEntriesUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_throttle_cache_entries{store="mysql"} 999`, + }, Thresholds{RealtimeThrottleCacheEntriesMax: 1000}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableRealtimeThrottleCacheEntryLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_throttle_cache_entries{store="mysql"} 9999999`, + }, Thresholds{RealtimeThrottleCacheEntriesMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeStoreUpdateErrors(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_store_updates_total{protocol="JT808",status="error",store="mysql"} 2`, + }, Thresholds{RealtimeStoreErrorMax: 0}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime store update errors 2 exceeds 0 for store mysql protocol JT808") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateCanDisableRealtimeStoreUpdateErrorLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_store_updates_total{protocol="JT808",status="error",store="mysql"} 99`, + }, Thresholds{RealtimeStoreErrorMax: -1}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimeAsyncQueueDroppedAndUsage(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_async_queue_total{protocol="JT808",status="dropped",store="mysql"} 1 +vehicle_realtime_async_queue_total{protocol="JT808",status="closed",store="mysql"} 1 +vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 90 +vehicle_realtime_async_queue_capacity{store="mysql"} 100`, + }, Thresholds{ + RealtimeAsyncQueueDroppedMax: 0, + RealtimeAsyncQueueMaxUsageRatio: 0.80, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "realtime async queue dropped 1 exceeds 0 for store mysql protocol JT808", + "realtime async queue closed 1 exceeds 0 for store mysql protocol JT808", + "realtime async queue usage 0.90 exceeds 0.80", + "protocol=JT808,store=mysql", + "depth 90", + "capacity 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateCanDisableRealtimeAsyncQueueLimits(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_async_queue_total{protocol="JT808",status="dropped",store="mysql"} 99 +vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 999 +vehicle_realtime_async_queue_capacity{store="mysql"} 100`, + }, Thresholds{ + RealtimeAsyncQueueDepthMax: 0, + RealtimeAsyncQueueDroppedMax: -1, + RealtimeAsyncQueueMaxUsageRatio: 0, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsRealtimePlateCacheEntriesOverLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_plate_cache_entries 250000`, + }, Thresholds{RealtimePlateCacheEntriesMax: 200000}) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + if !strings.Contains(joined, "realtime plate cache entries 250000 exceeds 200000") { + t.Fatalf("finding missing in:\n%s", joined) + } +} + +func TestEvaluateAllowsRealtimePlateCacheEntriesUnderLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_plate_cache_entries 199999`, + }, Thresholds{RealtimePlateCacheEntriesMax: 200000}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateCanDisableRealtimePlateCacheEntryLimit(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_plate_cache_entries 9999999`, + }, Thresholds{RealtimePlateCacheEntriesMax: 0}) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Findings) != 0 { + t.Fatalf("findings = %#v, want none", report.Findings) + } +} + +func TestEvaluateReportsDegradedForSlowGatewayIdentityP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_identity_duration_ms_histogram_bucket{le="1",protocol="JT808",status="resolved"} 10 +vehicle_gateway_identity_duration_ms_histogram_bucket{le="5",protocol="JT808",status="resolved"} 80 +vehicle_gateway_identity_duration_ms_histogram_bucket{le="100",protocol="JT808",status="resolved"} 98 +vehicle_gateway_identity_duration_ms_histogram_bucket{le="250",protocol="JT808",status="resolved"} 100 +vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="resolved"} 100 +vehicle_gateway_identity_duration_ms_histogram_count{protocol="JT808",status="resolved"} 100 +vehicle_gateway_identity_duration_ms_histogram_sum{protocol="JT808",status="resolved"} 350`, + }, Thresholds{ + GatewayIdentityP99MS: 100, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway identity p99 250ms exceeds 100ms", + "protocol=JT808,status=resolved", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateSkipsLatencyFindingsWhenHistogramSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_frame_duration_ms_histogram_bucket{le="250",protocol="GB32960",status="OK"} 1 +vehicle_gateway_frame_duration_ms_histogram_bucket{le="+Inf",protocol="GB32960",status="OK"} 1 +vehicle_gateway_frame_duration_ms_histogram_count{protocol="GB32960",status="OK"} 1`, + }, Thresholds{ + GatewayFrameP99MS: 100, + MinHistogramSamples: 10, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } +} + +func TestEvaluateReportsSlowFastWriterRedisE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="100",subject="vehicle.raw.go.jt808.v1"} 90 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="1000",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + FastWriterRedisE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis e2e p99 1000ms exceeds 300ms", + "subject=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowFastWriterRedisRecentE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} 180 +vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + FastWriterRedisRecentE2EP99MS: 100, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "fast writer redis e2e recent p99 180ms exceeds 100ms", + "subject=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowGatewayResponseRecentP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "gateway": `vehicle_service_info{service="vehicle-gateway"} 1 +vehicle_gateway_response_e2e_recent_p99_ms{protocol="JT808"} 180 +vehicle_gateway_response_e2e_recent_samples{protocol="JT808"} 100`, + }, Thresholds{ + GatewayResponseRecentP99MS: 100, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "gateway response e2e recent p99 180ms exceeds 100ms", + "protocol=JT808", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } + if report.PipelineLatency == nil || len(report.PipelineLatency.Items) != 1 { + t.Fatalf("gateway response pipeline diagnostics missing: %#v", report.PipelineLatency) + } + if got := report.PipelineLatency.Items[0].Stage; got != "gateway_protocol_response_e2e" { + t.Fatalf("pipeline stage = %q", got) + } +} + +func TestEvaluateServiceSummaryAggregatesP99GaugesByMax(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} 49 +vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 106 +vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.fields.go.gb32960.v1"} 512 +vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 512`, + }, Thresholds{ + BridgeKafkaRecentE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if len(report.Services) != 1 { + t.Fatalf("services = %d, want 1", len(report.Services)) + } + if got, want := report.Services[0].Metrics["vehicle_bridge_kafka_e2e_recent_p99_ms"], 106.0; got != want { + t.Fatalf("summary p99 = %v, want max %v instead of label sum", got, want) + } +} + +func TestEvaluateServiceSummaryDoesNotCreateP99AgeMetric(t *testing.T) { + parsed := parseMetricSet(`vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_last_kafka_e2e_unix_seconds{topic="vehicle.raw.go.jt808.v1"} 1000 +vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 106 +vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 512`) + addLastActivityAges(parsed.totals, time.Unix(1010, 0).UTC()) + + metrics := parsed.totals + if got, want := metrics["vehicle_bridge_last_kafka_e2e_age_seconds"], 10.0; got != want { + t.Fatalf("last activity age = %v, want %v", got, want) + } + if _, ok := metrics["vehicle_bridge_kafka_e2e_recent_p99_ms_age_seconds"]; ok { + t.Fatalf("p99 gauge should not produce an age metric: %#v", metrics) + } +} + +func TestEvaluateIncludesPipelineLatencyDiagnostics(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.gb32960.v1"} 80 +vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.gb32960.v1"} 128`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_tdengine_enabled 0 +vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} 42 +vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.gb32960.v1"} 128`, + "history": `vehicle_service_info{service="history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.gb32960.v1"} 1 +vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.gb32960.v1"} 1500 +vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.gb32960.v1"} 128`, + "stat": `vehicle_service_info{service="stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1 +vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} 900 +vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.gb32960.v1"} 8`, + }, Thresholds{ + BridgeKafkaRecentE2EP99MS: 300, + FastWriterRedisRecentE2EP99MS: 100, + HistoryWriteRecentE2EP99MS: 1000, + StatWriteRecentE2EP99MS: 1000, + MinHistogramSamples: 100, + }) + + if report.PipelineLatency == nil { + t.Fatal("pipeline latency diagnostics missing") + } + if got, want := len(report.PipelineLatency.Items), 4; got != want { + t.Fatalf("pipeline latency items = %d, want %d", got, want) + } + if got, want := report.PipelineLatency.TotalSamples, int64(392); got != want { + t.Fatalf("total samples = %d, want %d", got, want) + } + if got, want := report.PipelineLatency.MaxP99MS, 1500.0; got != want { + t.Fatalf("max p99 = %v, want %v", got, want) + } + statusByStage := map[string]string{} + for _, item := range report.PipelineLatency.Items { + statusByStage[item.Stage] = item.Status + } + if statusByStage["bridge_kafka_e2e"] != "ok" { + t.Fatalf("bridge status = %q", statusByStage["bridge_kafka_e2e"]) + } + if statusByStage["fast_writer_redis_e2e"] != "ok" { + t.Fatalf("fast writer status = %q", statusByStage["fast_writer_redis_e2e"]) + } + if statusByStage["history_tdengine_write_e2e"] != "degraded" { + t.Fatalf("history status = %q", statusByStage["history_tdengine_write_e2e"]) + } + if statusByStage["stat_mysql_write_e2e"] != "insufficient_samples" { + t.Fatalf("stat status = %q", statusByStage["stat_mysql_write_e2e"]) + } +} + +func TestEvaluateReportsSlowBridgeKafkaWriteP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="100",status="ok",topic="vehicle.raw.go.jt808.v1"} 90 +vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="1000",status="ok",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_write_duration_ms_histogram_count{status="ok",topic="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + BridgeKafkaWriteP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge kafka write p99 1000ms exceeds 300ms", + "topic=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowBridgeKafkaE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + BridgeKafkaE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge kafka e2e p99 1000ms exceeds 300ms", + "topic=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowBridgeKafkaRecentE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 450 +vehicle_bridge_kafka_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + BridgeKafkaRecentE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge kafka e2e recent p99 450ms exceeds 300ms", + "topic=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowHistoryWriteE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + HistoryWriteE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history write e2e p99 1000ms exceeds 300ms", + "topic=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateReportsSlowStatWriteE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.fields.go.jt808.v1"} 90 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="1000",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 100`, + }, Thresholds{ + StatWriteE2EP99MS: 300, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat write e2e p99 1000ms exceeds 300ms", + "topic=vehicle.fields.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestDefaultThresholdsDoNotGateCumulativeE2EHistograms(t *testing.T) { + report := Evaluate(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_bridge_kafka_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_tdengine_enabled 0 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="100",subject="vehicle.raw.go.jt808.v1"} 90 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="10000",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 100`, + "history": `vehicle_service_info{service="history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.raw.go.jt808.v1"} 90 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.raw.go.jt808.v1"} 100 +vehicle_history_write_e2e_duration_ms_histogram_count{topic="vehicle.raw.go.jt808.v1"} 100`, + "stat": `vehicle_service_info{service="stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="100",topic="vehicle.fields.go.jt808.v1"} 90 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="10000",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_e2e_duration_ms_histogram_bucket{le="+Inf",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 100`, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } + if joined := strings.Join(report.Findings, "\n"); strings.Contains(joined, "e2e p99") { + t.Fatalf("cumulative e2e p99 should be observe-only by default:\n%s", joined) + } +} + +func TestEvaluateReportsSlowHistoryWriteRecentE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 2500 +vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 100`, + }, Thresholds{ + HistoryWriteRecentE2EP99MS: 1000, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "history write e2e recent p99 2500ms exceeds 1000ms", + "topic=vehicle.raw.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestEvaluateSkipsHistoryWriteRecentE2EWhenSamplesAreLow(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "history": `vehicle_service_info{service="history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_write_e2e_recent_p99_ms{topic="vehicle.raw.go.jt808.v1"} 2500 +vehicle_history_write_e2e_recent_samples{topic="vehicle.raw.go.jt808.v1"} 99`, + }, Thresholds{ + HistoryWriteRecentE2EP99MS: 1000, + MinHistogramSamples: 100, + }) + + if report.Status != StatusOK { + t.Fatalf("status = %s, want ok: %#v", report.Status, report) + } +} + +func TestEvaluateReportsSlowStatWriteRecentE2EP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "stat": `vehicle_service_info{service="stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} 1500 +vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.jt808.v1"} 100`, + }, Thresholds{ + StatWriteRecentE2EP99MS: 1000, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "stat write e2e recent p99 1500ms exceeds 1000ms", + "topic=vehicle.fields.go.jt808.v1", + "samples 100", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } + } +} + +func TestAnnotateDailyMileageDiagnosticsSplitsPipelineAndSourceDataIssues(t *testing.T) { + items, pipelineIssues, sourceDataIssues := AnnotateDailyMileageDiagnostics([]DailyMileageDiagnosisReasonItem{ + {Diagnosis: "OK", Reason: "daily_metric_exists", Count: 234}, + {Diagnosis: "NO_SOURCE_SAMPLE", Reason: "realtime_location_has_total_mileage_but_no_stat_sample", Count: 7}, + {Diagnosis: "MISSING_DAILY", Reason: "source_samples_exist_but_final_metric_missing", Count: 1}, + {Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_time_missing", Count: 2}, + {Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_non_positive", Count: 3}, + {Diagnosis: "NO_TOTAL_MILEAGE", Reason: "realtime_total_mileage_missing", Count: 4}, + }) + + if pipelineIssues != 10 || sourceDataIssues != 7 { + t.Fatalf("pipeline=%d source=%d", pipelineIssues, sourceDataIssues) + } + severities := make([]string, 0, len(items)) + for _, item := range items { + severities = append(severities, item.Severity) + } + got := strings.Join(severities, ",") + want := "ok,pipeline,pipeline,pipeline,source_data,source_data" + if got != want { + t.Fatalf("severities = %q, want %q", got, want) + } + if !strings.Contains(items[4].RecommendedOperatorAction, "总里程小于等于 0") { + t.Fatalf("recommended action = %q", items[4].RecommendedOperatorAction) + } + if !strings.Contains(items[5].RecommendedOperatorAction, "缺少总里程字段") { + t.Fatalf("recommended action = %q", items[5].RecommendedOperatorAction) + } +} + +func TestAnnotateDailyMileageDiagnosticsPreservesAPISeverityAndClassifiesInvalidSamples(t *testing.T) { + items, pipelineIssues, sourceDataIssues := AnnotateDailyMileageDiagnostics([]DailyMileageDiagnosisReasonItem{ + {Diagnosis: "MISSING_DAILY", Reason: "source_samples_all_invalid", Severity: DailyMileageSeveritySourceData, Count: 25}, + {Diagnosis: "MISSING_DAILY", Reason: "source_samples_all_invalid", Count: 17}, + {Diagnosis: "NO_SOURCE_SAMPLE", Reason: "realtime_location_has_total_mileage_but_no_stat_sample", Severity: DailyMileageSeverityPipeline, Count: 2}, + }) + + if pipelineIssues != 2 || sourceDataIssues != 42 { + t.Fatalf("pipeline=%d source=%d", pipelineIssues, sourceDataIssues) + } + if items[0].Severity != DailyMileageSeveritySourceData || items[1].Severity != DailyMileageSeveritySourceData { + t.Fatalf("severities = %#v", []string{items[0].Severity, items[1].Severity}) + } +} + +func TestEvaluateReportsDegradedForSlowDownstreamP99(t *testing.T) { + report := EvaluateWithThresholds(map[string]string{ + "bridge": `vehicle_service_info{service="nats-kafka-bridge"} 1 +vehicle_bridge_config{setting="fetch_wait_ms"} 20 +vehicle_bridge_config{setting="workers"} 4 +vehicle_bridge_batch_duration_ms_histogram_bucket{le="1000",status="ok"} 98 +vehicle_bridge_batch_duration_ms_histogram_bucket{le="5000",status="ok"} 100 +vehicle_bridge_batch_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100 +vehicle_bridge_batch_duration_ms_histogram_count{status="ok"} 100`, + "fast-writer": `vehicle_service_info{service="nats-fast-writer"} 1 +vehicle_fast_writer_config{setting="fetch_wait_ms"} 20 +vehicle_fast_writer_config{setting="workers"} 8 +vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="25",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 90 +vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="250",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100 +vehicle_fast_writer_stage_duration_ms_histogram_count{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 100`, + "history": `vehicle_service_info{service="vehicle-history-writer"} 1 +vehicle_history_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-history-writer",group="go-history-writer",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_history_batch_flush_duration_ms_histogram_bucket{le="100",status="ok"} 95 +vehicle_history_batch_flush_duration_ms_histogram_bucket{le="1000",status="ok"} 100 +vehicle_history_batch_flush_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100 +vehicle_history_batch_flush_duration_ms_histogram_count{status="ok"} 100`, + "realtime": `vehicle_service_info{service="vehicle-realtime-api"} 1 +vehicle_realtime_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-realtime-api",group="go-realtime-api",topic="vehicle.raw.go.jt808.v1"} 1 +vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="50",status="ok"} 50 +vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="250",status="ok"} 100 +vehicle_realtime_batch_flush_duration_ms_histogram_bucket{le="+Inf",status="ok"} 100 +vehicle_realtime_batch_flush_duration_ms_histogram_count{status="ok"} 100 +vehicle_realtime_store_update_duration_ms_histogram_bucket{le="10",protocol="JT808",status="ok",store="mysql"} 50 +vehicle_realtime_store_update_duration_ms_histogram_bucket{le="250",protocol="JT808",status="ok",store="mysql"} 100 +vehicle_realtime_store_update_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="ok",store="mysql"} 100 +vehicle_realtime_store_update_duration_ms_histogram_count{protocol="JT808",status="ok",store="mysql"} 100`, + "stat": `vehicle_service_info{service="vehicle-stat-writer"} 1 +vehicle_stat_config{setting="workers"} 3 +vehicle_kafka_consumer_info{service="vehicle-stat-writer",group="go-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1 +vehicle_stat_write_duration_ms_histogram_bucket{le="5",status="ok",topic="vehicle.fields.go.jt808.v1"} 98 +vehicle_stat_write_duration_ms_histogram_bucket{le="250",status="ok",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.fields.go.jt808.v1"} 100 +vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.fields.go.jt808.v1"} 100`, + }, Thresholds{ + BridgeBatchP99MS: 1000, + FastWriterStageP99MS: 100, + HistoryFlushP99MS: 500, + RealtimeBatchP99MS: 100, + RealtimeStoreP99MS: 100, + StatWriteP99MS: 100, + MinHistogramSamples: 100, + }) + + if report.Status != StatusDegraded { + t.Fatalf("status = %s, want degraded: %#v", report.Status, report) + } + joined := strings.Join(report.Findings, "\n") + for _, want := range []string{ + "bridge batch p99 5000ms exceeds 1000ms", + "fast writer stage p99 250ms exceeds 100ms", + "history flush p99 1000ms exceeds 500ms", + "realtime batch flush p99 250ms exceeds 100ms", + "realtime store update p99 250ms exceeds 100ms", + "stat write p99 250ms exceeds 100ms", + } { + if !strings.Contains(joined, want) { + t.Fatalf("finding missing %q in:\n%s", want, joined) + } } } diff --git a/go/vehicle-gateway/internal/envelope/envelope.go b/go/vehicle-gateway/internal/envelope/envelope.go index 6fb4ad33..9077537d 100644 --- a/go/vehicle-gateway/internal/envelope/envelope.go +++ b/go/vehicle-gateway/internal/envelope/envelope.go @@ -24,6 +24,14 @@ const ( ParseBadFrame ParseStatus = "BAD_FRAME" ) +type EventKind string + +const ( + EventKindRaw EventKind = "RAW" + EventKindFields EventKind = "FIELDS" + EventKindUnified EventKind = "UNIFIED" +) + const ( FieldSpeedKMH = "speed_kmh" FieldTotalMileageKM = "total_mileage_km" @@ -33,27 +41,36 @@ const ( ) type FrameEnvelope struct { - EventID string `json:"event_id"` - TraceID string `json:"trace_id"` - Protocol Protocol `json:"protocol"` - MessageID string `json:"message_id"` - Sequence uint16 `json:"sequence"` - VIN string `json:"vin,omitempty"` - VehicleKeyHint string `json:"vehicle_key,omitempty"` - Phone string `json:"phone,omitempty"` - DeviceID string `json:"device_id,omitempty"` - Plate string `json:"plate,omitempty"` - SourceEndpoint string `json:"source_endpoint,omitempty"` - EventTimeMS int64 `json:"event_time_ms"` - ReceivedAtMS int64 `json:"received_at_ms"` - RawHex string `json:"raw_hex,omitempty"` - RawText string `json:"raw_text,omitempty"` - Parsed map[string]any `json:"parsed,omitempty"` - ParsedFields map[string]any `json:"parsed_fields,omitempty"` - ParsedFieldTypes map[string]string `json:"parsed_field_types,omitempty"` - Fields map[string]any `json:"fields,omitempty"` - ParseStatus ParseStatus `json:"parse_status"` - ParseError string `json:"parse_error,omitempty"` + EventID string `json:"event_id"` + TraceID string `json:"trace_id"` + EventKind EventKind `json:"event_kind,omitempty"` + SourceEventID string `json:"source_event_id,omitempty"` + FieldMapping string `json:"field_mapping,omitempty"` + Protocol Protocol `json:"protocol"` + MessageID string `json:"message_id"` + Sequence uint16 `json:"sequence"` + VIN string `json:"vin,omitempty"` + VehicleKeyHint string `json:"vehicle_key,omitempty"` + Phone string `json:"phone,omitempty"` + DeviceID string `json:"device_id,omitempty"` + Plate string `json:"plate,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + SourceCode string `json:"source_code,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + AuthenticationMode string `json:"authentication_mode,omitempty"` + AuthenticationStatus string `json:"authentication_status,omitempty"` + AuthenticationEnforced bool `json:"authentication_enforced,omitempty"` + EventTimeMS int64 `json:"event_time_ms"` + ReceivedAtMS int64 `json:"received_at_ms"` + RawHex string `json:"raw_hex,omitempty"` + RawText string `json:"raw_text,omitempty"` + Parsed map[string]any `json:"parsed,omitempty"` + ParsedFields map[string]any `json:"parsed_fields,omitempty"` + ParsedFieldTypes map[string]string `json:"parsed_field_types,omitempty"` + Fields map[string]any `json:"fields,omitempty"` + ParseStatus ParseStatus `json:"parse_status"` + ParseError string `json:"parse_error,omitempty"` } func (e FrameEnvelope) VehicleKey() string { @@ -93,5 +110,8 @@ func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) { if e.ParseStatus == "" { e.ParseStatus = ParseOK } + if e.EventKind == "" { + e.EventKind = EventKindRaw + } return json.Marshal(e) } diff --git a/go/vehicle-gateway/internal/envelope/envelope_test.go b/go/vehicle-gateway/internal/envelope/envelope_test.go index 520430fa..300310e7 100644 --- a/go/vehicle-gateway/internal/envelope/envelope_test.go +++ b/go/vehicle-gateway/internal/envelope/envelope_test.go @@ -3,6 +3,7 @@ package envelope import ( "encoding/json" "testing" + "time" ) func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) { @@ -52,4 +53,160 @@ func TestFrameEnvelopeMarshalDefaults(t *testing.T) { if decoded.ParseStatus != ParseOK { t.Fatalf("parse status = %q", decoded.ParseStatus) } + if decoded.EventKind != EventKindRaw { + t.Fatalf("event kind = %q, want %q", decoded.EventKind, EventKindRaw) + } +} + +func TestNormalizedEventTimeMSFallsBackToReceivedWhenEventIsFarFuture(t *testing.T) { + received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli() + event := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC).UnixMilli() + + got, ok := NormalizedEventTimeMS(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received}) + if !ok { + t.Fatal("NormalizedEventTimeMS() ok = false") + } + if got != received { + t.Fatalf("normalized event time = %d, want received %d", got, received) + } + _, reason, ok := NormalizedEventTimeMSWithReason(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received}) + if !ok || reason != EventTimeReasonReceivedFutureEvent { + t.Fatalf("reason = %q ok=%v, want future fallback", reason, ok) + } +} + +func TestNormalizedEventTimeMSKeepsSmallFutureSkew(t *testing.T) { + received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli() + event := time.Date(2026, 7, 12, 9, 35, 0, 0, time.UTC).UnixMilli() + + got, ok := NormalizedEventTimeMS(FrameEnvelope{EventTimeMS: event, ReceivedAtMS: received}) + if !ok { + t.Fatal("NormalizedEventTimeMS() ok = false") + } + if got != event { + t.Fatalf("normalized event time = %d, want event %d", got, event) + } +} + +func TestIsRealtimeTelemetryFrameRejectsKnownNonRealtimeMessageIDs(t *testing.T) { + fields := map[string]any{ + FieldLatitude: 30.590151, + FieldLongitude: 121.069881, + FieldTotalMileageKM: 10241.2, + } + tests := []FrameEnvelope{ + {Protocol: ProtocolJT808, MessageID: "0x0100", Fields: fields, ParseStatus: ParseOK}, + {Protocol: ProtocolGB32960, MessageID: "0x01", Fields: fields, Parsed: map[string]any{"data_units": []any{}}, ParseStatus: ParseOK}, + } + for _, env := range tests { + if IsRealtimeTelemetryFrame(env) { + t.Fatalf("IsRealtimeTelemetryFrame(%s/%s) = true, want false", env.Protocol, env.MessageID) + } + } +} + +func TestIsRealtimeTelemetryFrameAcceptsKnownRealtimeMessageIDs(t *testing.T) { + tests := []FrameEnvelope{ + {Protocol: ProtocolJT808, MessageID: "0x0200", ParsedFields: map[string]any{"jt808.location.speed_kmh": 1}, ParseStatus: ParseOK}, + {Protocol: ProtocolGB32960, MessageID: "0x02", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 1}, ParseStatus: ParseOK}, + {Protocol: ProtocolGB32960, MessageID: "0x03", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 1}, ParseStatus: ParseOK}, + } + for _, env := range tests { + if !IsRealtimeTelemetryFrame(env) { + t.Fatalf("IsRealtimeTelemetryFrame(%s/%s) = false, want true", env.Protocol, env.MessageID) + } + } +} + +func TestIsRealtimeTelemetryFrameUsesCanonicalMQTTParsedFields(t *testing.T) { + realtime := FrameEnvelope{ + Protocol: ProtocolYutongMQTT, + MessageID: "MQTT", + ParseStatus: ParseOK, + ParsedFields: map[string]any{ + "yutong_mqtt.data.latitude": 30.590151, + }, + } + if !IsRealtimeTelemetryFrame(realtime) { + t.Fatal("canonical MQTT data field should be classified as realtime") + } + + metadataOnly := realtime + metadataOnly.ParsedFields = map[string]any{"yutong_mqtt.metadata.topic": "/ytforward/shln/3"} + if IsRealtimeTelemetryFrame(metadataOnly) { + t.Fatal("MQTT metadata-only field must not be classified as realtime") + } +} + +func TestRequiresVehicleIdentity(t *testing.T) { + tests := []struct { + name string + env FrameEnvelope + want bool + }{ + { + name: "gb32960 realtime upload", + env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x02", Parsed: map[string]any{"data_units": []any{map[string]any{"name": "vehicle"}}}, ParseStatus: ParseOK}, + want: true, + }, + { + name: "gb32960 platform login", + env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x05", ParseStatus: ParseOK}, + want: false, + }, + { + name: "gb32960 command response", + env: FrameEnvelope{Protocol: ProtocolGB32960, MessageID: "0x07", ParseStatus: ParseOK}, + want: false, + }, + { + name: "jt808 registration", + env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0100", ParseStatus: ParseOK}, + want: true, + }, + { + name: "jt808 location", + env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0200", Parsed: map[string]any{"location": map[string]any{"speed_kmh": 1}}, ParseStatus: ParseOK}, + want: true, + }, + { + name: "yutong empty data", + env: FrameEnvelope{Protocol: ProtocolYutongMQTT, MessageID: "MQTT", Parsed: map[string]any{"data": map[string]any{}}, ParseStatus: ParseOK}, + want: false, + }, + { + name: "yutong telemetry data", + env: FrameEnvelope{Protocol: ProtocolYutongMQTT, MessageID: "MQTT", Parsed: map[string]any{"data": map[string]any{"TOTAL_MILEAGE": 123}}, ParseStatus: ParseOK}, + want: true, + }, + { + name: "bad frame", + env: FrameEnvelope{Protocol: ProtocolJT808, MessageID: "0x0200", ParseStatus: ParseBadFrame}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RequiresVehicleIdentity(tt.env); got != tt.want { + t.Fatalf("RequiresVehicleIdentity() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNormalizeSourceEndpointKey(t *testing.T) { + tests := map[string]string{ + "115.231.168.135:20215": "115.231.168.135", + "115.231.168.135": "115.231.168.135", + " 115.159.85.149:28316 ": "115.159.85.149", + "mqtt://yutong/ytforward/shln/3": "mqtt", + "MQTT://YUTONG/topic": "mqtt", + "": "", + " ": "", + } + for input, want := range tests { + if got := NormalizeSourceEndpointKey(input); got != want { + t.Fatalf("NormalizeSourceEndpointKey(%q) = %q, want %q", input, got, want) + } + } } diff --git a/go/vehicle-gateway/internal/envelope/realtime.go b/go/vehicle-gateway/internal/envelope/realtime.go index 04ca875e..31f0f5fe 100644 --- a/go/vehicle-gateway/internal/envelope/realtime.go +++ b/go/vehicle-gateway/internal/envelope/realtime.go @@ -14,33 +14,93 @@ func IsRealtimeTelemetryFrame(env FrameEnvelope) bool { command = strings.TrimSpace(stringAny(header["command"])) } } - if strings.EqualFold(command, "0x02") || strings.EqualFold(command, "0x03") { - return true + if command != "" && !strings.EqualFold(command, "0x02") && !strings.EqualFold(command, "0x03") { + return false } _, hasDataUnits := env.Parsed["data_units"] - return hasDataUnits || hasRealtimeField(env) + return hasDataUnits || hasGB32960ParsedTelemetryField(env) || hasRealtimeField(env) case ProtocolJT808: - if strings.EqualFold(strings.TrimSpace(env.MessageID), "0x0200") { - return true + messageID := strings.TrimSpace(env.MessageID) + if messageID != "" && !strings.EqualFold(messageID, "0x0200") { + return false } if _, ok := env.Parsed["location"]; ok { return true } - return hasRealtimeField(env) + return hasParsedFieldPrefix(env, "jt808.location.") || hasRealtimeField(env) case ProtocolYutongMQTT: data, ok := env.Parsed["data"].(map[string]any) - return ok && len(data) > 0 + if ok && len(data) > 0 { + return true + } + return hasParsedFieldPrefix(env, "yutong_mqtt.data.") || + hasParsedFieldPrefix(env, "yutong_mqtt.root.data.") + default: + return false + } +} + +func hasGB32960ParsedTelemetryField(env FrameEnvelope) bool { + for field := range env.ParsedFields { + if !strings.HasPrefix(field, "gb32960.") { + continue + } + if strings.HasPrefix(field, "gb32960.header.") || + strings.HasPrefix(field, "gb32960.platform.") || + strings.HasPrefix(field, "gb32960.identity.") || + strings.HasPrefix(field, "gb32960.device_time.") { + continue + } + return true + } + return false +} + +// RequiresVehicleIdentity keeps platform/control frames out of vehicle identity +// quality metrics while preserving JT808's phone-centric registration/auth flow. +func RequiresVehicleIdentity(env FrameEnvelope) bool { + if env.ParseStatus == ParseBadFrame { + return false + } + switch env.Protocol { + case ProtocolGB32960: + return IsRealtimeTelemetryFrame(env) + case ProtocolJT808: + return true + case ProtocolYutongMQTT: + return IsRealtimeTelemetryFrame(env) default: return false } } func hasRealtimeField(env FrameEnvelope) bool { - return env.Fields[FieldLatitude] != nil || + if env.Fields[FieldLatitude] != nil || env.Fields[FieldLongitude] != nil || env.Fields[FieldTotalMileageKM] != nil || env.Fields[FieldSpeedKMH] != nil || - env.Fields[FieldSOCPercent] != nil + env.Fields[FieldSOCPercent] != nil { + return true + } + for field := range env.ParsedFields { + if strings.HasSuffix(field, ".latitude") || + strings.HasSuffix(field, ".longitude") || + strings.HasSuffix(field, ".total_mileage_km") || + strings.HasSuffix(field, ".speed_kmh") || + strings.HasSuffix(field, ".soc_percent") { + return true + } + } + return false +} + +func hasParsedFieldPrefix(env FrameEnvelope, prefix string) bool { + for field := range env.ParsedFields { + if strings.HasPrefix(field, prefix) { + return true + } + } + return false } func stringAny(value any) string { diff --git a/go/vehicle-gateway/internal/envelope/source.go b/go/vehicle-gateway/internal/envelope/source.go new file mode 100644 index 00000000..5911d954 --- /dev/null +++ b/go/vehicle-gateway/internal/envelope/source.go @@ -0,0 +1,19 @@ +package envelope + +import "strings" + +// NormalizeSourceEndpointKey returns the stable, low-cardinality source key used +// by identity lookup and statistics source tracking. +func NormalizeSourceEndpointKey(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(endpoint), "mqtt://") { + return "mqtt" + } + if host, _, ok := strings.Cut(endpoint, ":"); ok { + return strings.TrimSpace(host) + } + return endpoint +} diff --git a/go/vehicle-gateway/internal/envelope/time.go b/go/vehicle-gateway/internal/envelope/time.go new file mode 100644 index 00000000..ce420c30 --- /dev/null +++ b/go/vehicle-gateway/internal/envelope/time.go @@ -0,0 +1,31 @@ +package envelope + +import "time" + +const ( + MaxFutureEventSkew = 10 * time.Minute + MinPlausibleReceivedTimeMS = 1577836800000 // 2020-01-01T00:00:00Z +) + +const ( + EventTimeReasonEventTime = "event_time" + EventTimeReasonReceivedMissingEvent = "received_time_missing_event" + EventTimeReasonReceivedFutureEvent = "received_time_future_event" +) + +func NormalizedEventTimeMS(env FrameEnvelope) (int64, bool) { + eventMS, _, ok := NormalizedEventTimeMSWithReason(env) + return eventMS, ok +} + +func NormalizedEventTimeMSWithReason(env FrameEnvelope) (int64, string, bool) { + eventMS := env.EventTimeMS + receivedMS := env.ReceivedAtMS + if eventMS <= 0 { + return receivedMS, EventTimeReasonReceivedMissingEvent, receivedMS > 0 + } + if receivedMS >= MinPlausibleReceivedTimeMS && eventMS > receivedMS+MaxFutureEventSkew.Milliseconds() { + return receivedMS, EventTimeReasonReceivedFutureEvent, true + } + return eventMS, EventTimeReasonEventTime, true +} diff --git a/go/vehicle-gateway/internal/eventbus/async_sink.go b/go/vehicle-gateway/internal/eventbus/async_sink.go index faa0605f..0848aafd 100644 --- a/go/vehicle-gateway/internal/eventbus/async_sink.go +++ b/go/vehicle-gateway/internal/eventbus/async_sink.go @@ -13,6 +13,7 @@ import ( type AsyncConfig struct { QueueSize int Workers int + EnqueueTimeout time.Duration OperationTimeout time.Duration OnError func(error) Metrics *metrics.Registry @@ -20,12 +21,14 @@ type AsyncConfig struct { } type AsyncSink struct { - delegate Sink - jobs chan asyncJob - timeout time.Duration - onError func(error) - metrics *metrics.Registry - name string + delegate Sink + jobs chan asyncJob + enqueueTimeout time.Duration + timeout time.Duration + onError func(error) + metrics *metrics.Registry + name string + queueWait *metrics.RecentLatencyByKey closeOnce sync.Once closed chan struct{} @@ -34,11 +37,13 @@ type AsyncSink struct { } type asyncJob struct { - kind string - env envelope.FrameEnvelope + kind string + env envelope.FrameEnvelope + enqueuedAt time.Time } var ErrAsyncSinkClosed = errors.New("async sink is closed") +var ErrAsyncSinkEnqueueTimeout = errors.New("async sink enqueue timeout") var asyncSinkPublishDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} @@ -52,6 +57,12 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink { if cfg.Workers <= 0 { cfg.Workers = 1 } + if cfg.EnqueueTimeout == 0 { + cfg.EnqueueTimeout = time.Second + } + if cfg.EnqueueTimeout < 0 { + cfg.EnqueueTimeout = 0 + } if cfg.OperationTimeout <= 0 { cfg.OperationTimeout = 30 * time.Second } @@ -59,19 +70,23 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink { cfg.Name = "async" } s := &AsyncSink{ - delegate: delegate, - jobs: make(chan asyncJob, cfg.QueueSize), - timeout: cfg.OperationTimeout, - onError: cfg.OnError, - metrics: cfg.Metrics, - name: cfg.Name, - closed: make(chan struct{}), - done: make(chan struct{}), + delegate: delegate, + jobs: make(chan asyncJob, cfg.QueueSize), + enqueueTimeout: cfg.EnqueueTimeout, + timeout: cfg.OperationTimeout, + onError: cfg.OnError, + metrics: cfg.Metrics, + name: cfg.Name, + queueWait: metrics.NewRecentLatencyByKey(512), + closed: make(chan struct{}), + done: make(chan struct{}), } s.wg.Add(cfg.Workers) for i := 0; i < cfg.Workers; i++ { go s.worker() } + s.recordQueueCapacity() + s.recordWorkers("default", cfg.Workers) go func() { s.wg.Wait() close(s.done) @@ -94,7 +109,6 @@ func (s *AsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelop func (s *AsyncSink) Close() error { s.closeOnce.Do(func() { close(s.closed) - close(s.jobs) }) <-s.done return s.delegate.Close() @@ -107,6 +121,14 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error { return ErrAsyncSinkClosed default: } + var timeoutC <-chan time.Time + var timer *time.Timer + if s.enqueueTimeout > 0 { + timer = time.NewTimer(s.enqueueTimeout) + timeoutC = timer.C + defer timer.Stop() + } + job.enqueuedAt = time.Now() select { case s.jobs <- job: s.recordEnqueue(job.kind, "queued") @@ -119,37 +141,58 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error { s.recordEnqueue(job.kind, "timeout") s.recordQueueDepth() return ctx.Err() + case <-timeoutC: + s.recordEnqueue(job.kind, "timeout") + s.recordQueueDepth() + return ErrAsyncSinkEnqueueTimeout } } func (s *AsyncSink) worker() { defer s.wg.Done() - for job := range s.jobs { - s.recordQueueDepth() - ctx, cancel := context.WithTimeout(context.Background(), s.timeout) - started := time.Now() - var err error - switch job.kind { - case "raw": - err = s.delegate.PublishRaw(ctx, job.env) - case "unified": - err = s.delegate.PublishUnified(ctx, job.env) - case "fields": - err = s.delegate.PublishFields(ctx, job.env) + for { + select { + case job := <-s.jobs: + s.publishJob(job) + case <-s.closed: + for { + select { + case job := <-s.jobs: + s.publishJob(job) + default: + return + } + } } - cancel() - status := "ok" - if err != nil { - status = "error" - } - s.recordPublish(job.kind, status, time.Since(started)) - if err != nil && s.onError != nil { - s.onError(err) - } - s.recordQueueDepth() } } +func (s *AsyncSink) publishJob(job asyncJob) { + s.recordQueueDepth() + s.recordQueueWait("default", job) + ctx, cancel := context.WithTimeout(context.Background(), s.timeout) + started := time.Now() + var err error + switch job.kind { + case "raw": + err = s.delegate.PublishRaw(ctx, job.env) + case "unified": + err = s.delegate.PublishUnified(ctx, job.env) + case "fields": + err = s.delegate.PublishFields(ctx, job.env) + } + cancel() + status := "ok" + if err != nil { + status = "error" + } + s.recordPublish(job.kind, status, time.Since(started)) + if err != nil && s.onError != nil { + s.onError(err) + } + s.recordQueueDepth() +} + func (s *AsyncSink) recordEnqueue(kind string, status string) { if s.metrics == nil { return @@ -184,3 +227,38 @@ func (s *AsyncSink) recordQueueDepth() { "sink": s.name, }, float64(len(s.jobs))) } + +func (s *AsyncSink) recordQueueCapacity() { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_async_sink_queue_capacity", metrics.Labels{ + "sink": s.name, + }, float64(cap(s.jobs))) +} + +func (s *AsyncSink) recordQueueWait(queueName string, job asyncJob) { + if s.metrics == nil || job.enqueuedAt.IsZero() { + return + } + elapsedMS := float64(time.Since(job.enqueuedAt)) / float64(time.Millisecond) + labels := metrics.Labels{ + "sink": s.name, + "queue": queueName, + "kind": job.kind, + } + s.metrics.ObserveHistogram("vehicle_async_sink_queue_wait_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS) + p99, samples := s.queueWait.Observe(queueName+"\x00"+job.kind, elapsedMS) + s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_p99_ms", labels, p99) + s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_samples", labels, float64(samples)) +} + +func (s *AsyncSink) recordWorkers(queueName string, workers int) { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_async_sink_workers", metrics.Labels{ + "sink": s.name, + "queue": queueName, + }, float64(workers)) +} diff --git a/go/vehicle-gateway/internal/eventbus/async_sink_test.go b/go/vehicle-gateway/internal/eventbus/async_sink_test.go index 05af6aec..10bfd967 100644 --- a/go/vehicle-gateway/internal/eventbus/async_sink_test.go +++ b/go/vehicle-gateway/internal/eventbus/async_sink_test.go @@ -2,6 +2,7 @@ package eventbus import ( "context" + "errors" "strings" "testing" "time" @@ -80,12 +81,17 @@ func TestAsyncSinkRecordsQueueMetrics(t *testing.T) { for _, want := range []string{ `vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`, `vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`, + `vehicle_async_sink_queue_capacity{sink="nats"} 2`, `vehicle_async_sink_queue_depth{sink="nats"}`, + `vehicle_async_sink_workers{queue="default",sink="nats"} 1`, `vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`, `vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`, `vehicle_async_sink_publish_duration_ms_histogram_bucket{le="+Inf",kind="raw",sink="nats",status="ok"} 1`, `vehicle_async_sink_publish_duration_ms_histogram_count{kind="raw",sink="nats",status="ok"} 1`, `vehicle_async_sink_publish_duration_ms_histogram_sum{kind="raw",sink="nats",status="ok"}`, + `vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="default",sink="nats"} 1`, + `vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="default",sink="nats"}`, + `vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="default",sink="nats"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("async sink metric missing %s:\n%s", want, text) @@ -135,6 +141,145 @@ func TestAsyncSinkRecordsEnqueueTimeoutWhenQueueIsFull(t *testing.T) { delegate.release() } +func TestAsyncSinkEnqueueTimeoutDoesNotRequireCallerDeadline(t *testing.T) { + registry := metrics.NewRegistry() + delegate := newBlockingSink() + sink := NewAsyncSink(delegate, AsyncConfig{ + QueueSize: 1, + Workers: 1, + EnqueueTimeout: 10 * time.Millisecond, + OperationTimeout: time.Second, + Metrics: registry, + Name: "nats", + }) + defer sink.Close() + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("first PublishRaw() error = %v", err) + } + select { + case <-delegate.rawStarted: + case <-time.After(time.Second): + t.Fatal("delegate raw publish was not started") + } + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("second PublishFields() error = %v", err) + } + if err := sink.PublishUnified(context.Background(), env); !errors.Is(err, ErrAsyncSinkEnqueueTimeout) { + t.Fatalf("third PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_async_sink_enqueue_total{kind="unified",sink="nats",status="timeout"} 1`, + `vehicle_async_sink_queue_depth{sink="nats"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("async sink enqueue timeout metric missing %s:\n%s", want, text) + } + } + delegate.release() +} + +func TestAsyncSinkCloseDoesNotPanicWithConcurrentBlockedEnqueue(t *testing.T) { + delegate := newBlockingSink() + sink := NewAsyncSink(delegate, AsyncConfig{ + QueueSize: 1, + Workers: 1, + EnqueueTimeout: 20 * time.Millisecond, + OperationTimeout: time.Second, + }) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("first PublishRaw() error = %v", err) + } + select { + case <-delegate.rawStarted: + case <-time.After(time.Second): + t.Fatal("delegate raw publish was not started") + } + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("second PublishFields() error = %v", err) + } + + publishErr := make(chan error, 1) + go func() { + defer func() { + if recovered := recover(); recovered != nil { + publishErr <- errors.New("publish panicked") + } + }() + publishErr <- sink.PublishUnified(context.Background(), env) + }() + closeErr := make(chan error, 1) + go func() { + closeErr <- sink.Close() + }() + + if err := <-publishErr; !errors.Is(err, ErrAsyncSinkEnqueueTimeout) && !errors.Is(err, ErrAsyncSinkClosed) { + t.Fatalf("concurrent PublishUnified() error = %v, want timeout or closed", err) + } + delegate.release() + if err := <-closeErr; err != nil { + t.Fatalf("Close() error = %v", err) + } + if err := sink.PublishRaw(context.Background(), env); !errors.Is(err, ErrAsyncSinkClosed) { + t.Fatalf("PublishRaw() after Close error = %v, want ErrAsyncSinkClosed", err) + } +} + +func TestAsyncSinkCloseUnblocksPublishWhenEnqueueTimeoutIsDisabled(t *testing.T) { + delegate := newBlockingSink() + sink := NewAsyncSink(delegate, AsyncConfig{ + QueueSize: 1, + Workers: 1, + EnqueueTimeout: -1, + OperationTimeout: time.Second, + }) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("first PublishRaw() error = %v", err) + } + select { + case <-delegate.rawStarted: + case <-time.After(time.Second): + t.Fatal("delegate raw publish was not started") + } + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("second PublishFields() error = %v", err) + } + + publishErr := make(chan error, 1) + go func() { + publishErr <- sink.PublishUnified(context.Background(), env) + }() + closeErr := make(chan error, 1) + go func() { + closeErr <- sink.Close() + }() + + select { + case err := <-publishErr: + if !errors.Is(err, ErrAsyncSinkClosed) { + t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err) + } + case <-time.After(time.Second): + t.Fatal("blocked PublishUnified() was not released by Close") + } + delegate.release() + select { + case err := <-closeErr: + if err != nil { + t.Fatalf("Close() error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close() did not finish after delegate release") + } +} + type blockingSink struct { rawStarted chan struct{} releaseRaw chan struct{} diff --git a/go/vehicle-gateway/internal/eventbus/durable_outbox_sink.go b/go/vehicle-gateway/internal/eventbus/durable_outbox_sink.go new file mode 100644 index 00000000..55098d82 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/durable_outbox_sink.go @@ -0,0 +1,348 @@ +package eventbus + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" +) + +type DurableOutboxConfig struct { + Directory string + ReplayBatchSize int + SyncWrites bool + CloseTimeout time.Duration + WALSegmentBytes int64 + WALSegmentAge time.Duration + WALAppendQueue int + WALCommitBatch int + WALCommitWait time.Duration + Metrics *metrics.Registry + Name string + OnError func(error) +} + +type asyncRecordPublishingSink interface { + Sink + ValidateRecord(durableRecord) error + PublishRecordAsync(durableRecord, func(error)) error +} + +// DurableOutboxSink accepts a record only after its WAL commit is durable. +// Publishing is asynchronous; the WAL record remains replayable until the +// broker returns PubAck. Stable event IDs make crash-window replays idempotent. +type DurableOutboxSink struct { + delegate asyncRecordPublishingSink + store *durableOutboxWAL + replayBatchSize int + closeTimeout time.Duration + metrics *metrics.Registry + name string + onError func(error) + + acceptMu sync.RWMutex + mu sync.Mutex + closed bool + inflight map[outboxWALRecordRef]struct{} + pending sync.WaitGroup + closeOne sync.Once + closeErr error +} + +var ErrDurableOutboxClosed = errors.New("durable outbox is closed") + +func NewDurableOutboxSink(delegate Sink, cfg DurableOutboxConfig) (*DurableOutboxSink, error) { + publisher, ok := delegate.(asyncRecordPublishingSink) + if !ok || publisher == nil { + return nil, errors.New("durable outbox delegate must support async record publishing") + } + dir := strings.TrimSpace(cfg.Directory) + if dir == "" { + return nil, errors.New("durable outbox directory is required") + } + if cfg.ReplayBatchSize <= 0 { + cfg.ReplayBatchSize = 1000 + } + if cfg.CloseTimeout <= 0 { + cfg.CloseTimeout = 5 * time.Second + } + store, err := newDurableOutboxWAL(durableOutboxWALConfig{ + Directory: dir, + SyncWrites: cfg.SyncWrites, + SegmentBytes: cfg.WALSegmentBytes, + SegmentAge: cfg.WALSegmentAge, + AppendQueue: cfg.WALAppendQueue, + CommitBatch: cfg.WALCommitBatch, + CommitInterval: cfg.WALCommitWait, + }) + if err != nil { + return nil, err + } + s := &DurableOutboxSink{ + delegate: publisher, + store: store, + replayBatchSize: cfg.ReplayBatchSize, + closeTimeout: cfg.CloseTimeout, + metrics: cfg.Metrics, + name: durableMetricName(cfg.Name), + onError: cfg.OnError, + inflight: map[outboxWALRecordRef]struct{}{}, + } + s.recordBacklog() + return s, nil +} + +func (s *DurableOutboxSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { + return s.persistAndSubmit(ctx, "raw", env) +} + +func (s *DurableOutboxSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error { + return s.persistAndSubmit(ctx, "unified", env) +} + +func (s *DurableOutboxSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error { + return s.persistAndSubmit(ctx, "fields", env) +} + +func (s *DurableOutboxSink) Close() error { + s.closeOne.Do(func() { + s.acceptMu.Lock() + s.mu.Lock() + s.closed = true + s.mu.Unlock() + s.acceptMu.Unlock() + + walErr := s.store.Close() + done := make(chan struct{}) + go func() { + s.pending.Wait() + close(done) + }() + var waitErr error + select { + case <-done: + case <-time.After(s.closeTimeout): + waitErr = fmt.Errorf("durable outbox close timed out after %s", s.closeTimeout) + s.recordPublish("all", "close_timeout") + } + s.closeErr = errors.Join(walErr, waitErr, s.delegate.Close()) + }) + return s.closeErr +} + +// ReplayOnce claims at most one configured batch. A bounded claim prevents a +// large recovered backlog from creating an unbounded number of PubAck futures. +func (s *DurableOutboxSink) ReplayOnce(ctx context.Context) error { + s.acceptMu.RLock() + defer s.acceptMu.RUnlock() + if s.isClosed() { + return ErrDurableOutboxClosed + } + return s.replay(ctx, s.replayBatchSize) +} + +func (s *DurableOutboxSink) ReplayLoop(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = time.Second + } + if err := s.ReplayOnce(ctx); err != nil && !errors.Is(err, ErrDurableOutboxClosed) { + s.reportError(err) + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := s.ReplayOnce(ctx); err != nil && !errors.Is(err, ErrDurableOutboxClosed) { + s.reportError(err) + } + } + } +} + +func (s *DurableOutboxSink) persistAndSubmit(ctx context.Context, kind string, env envelope.FrameEnvelope) error { + s.acceptMu.RLock() + defer s.acceptMu.RUnlock() + if s.isClosed() { + return ErrDurableOutboxClosed + } + record := durableRecord{Kind: kind, Envelope: normalizeDurableEnvelope(env)} + if err := s.delegate.ValidateRecord(record); err != nil { + s.recordPublish(kind, "validation_error") + return err + } + stored, err := s.store.Append(ctx, record) + if err != nil { + s.recordSpool(kind, "error") + return err + } + s.recordSpool(kind, "ok") + s.recordBacklog() + if err := s.submit(stored); err != nil { + // The durable commit is the device-facing acceptance boundary. Broker + // submission errors are surfaced operationally and recovered by replay. + s.reportError(err) + } + return nil +} + +func normalizeDurableEnvelope(env envelope.FrameEnvelope) envelope.FrameEnvelope { + if env.EventID == "" { + env.EventID = env.StableEventID() + } + if env.ParseStatus == "" { + env.ParseStatus = envelope.ParseOK + } + return env +} + +func (s *DurableOutboxSink) submit(stored storedOutboxRecord) error { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + s.store.Release(stored.Ref) + return ErrDurableOutboxClosed + } + if _, exists := s.inflight[stored.Ref]; exists { + s.mu.Unlock() + return nil + } + s.inflight[stored.Ref] = struct{}{} + s.pending.Add(1) + inflight := len(s.inflight) + s.mu.Unlock() + s.recordInflight(inflight) + + err := s.delegate.PublishRecordAsync(stored.Record, func(publishErr error) { + s.complete(stored, publishErr) + }) + if err == nil { + s.recordPublish(stored.Record.Kind, "submitted") + return nil + } + s.mu.Lock() + delete(s.inflight, stored.Ref) + inflight = len(s.inflight) + s.mu.Unlock() + s.pending.Done() + s.store.Release(stored.Ref) + s.recordInflight(inflight) + s.recordPublish(stored.Record.Kind, "submit_error") + return fmt.Errorf("submit durable outbox record: %w", err) +} + +func (s *DurableOutboxSink) complete(stored storedOutboxRecord, publishErr error) { + defer s.pending.Done() + status := "acked" + if publishErr != nil { + status = "ack_error" + s.store.Release(stored.Ref) + } else if err := s.store.Ack(stored.Ref); err != nil { + publishErr = err + status = "remove_error" + } + s.mu.Lock() + delete(s.inflight, stored.Ref) + inflight := len(s.inflight) + s.mu.Unlock() + s.recordInflight(inflight) + s.recordBacklog() + s.recordPublish(stored.Record.Kind, status) + if publishErr != nil { + s.reportError(publishErr) + } +} + +func (s *DurableOutboxSink) replay(ctx context.Context, limit int) error { + if ctx != nil && ctx.Err() != nil { + return ctx.Err() + } + records, err := s.store.ClaimPending(limit) + if err != nil { + return fmt.Errorf("claim durable outbox records: %w", err) + } + for index, stored := range records { + if ctx != nil && ctx.Err() != nil { + for _, remaining := range records[index:] { + s.store.Release(remaining.Ref) + } + return ctx.Err() + } + if err := s.delegate.ValidateRecord(stored.Record); err != nil { + s.store.Release(stored.Ref) + for _, remaining := range records[index+1:] { + s.store.Release(remaining.Ref) + } + s.recordPublish(stored.Record.Kind, "validation_error") + return fmt.Errorf("validate durable outbox replay record: %w", err) + } + if err := s.submit(stored); err != nil && !errors.Is(err, ErrDurableOutboxClosed) { + s.reportError(err) + } + } + return nil +} + +func (s *DurableOutboxSink) isClosed() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.closed +} + +func (s *DurableOutboxSink) reportError(err error) { + if err != nil && s.onError != nil { + s.onError(err) + } +} + +func (s *DurableOutboxSink) recordSpool(kind string, status string) { + if s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{ + "name": s.name, "kind": kind, "status": status, + }) +} + +func (s *DurableOutboxSink) recordPublish(kind string, status string) { + if s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_durable_outbox_publish_total", metrics.Labels{ + "name": s.name, "kind": kind, "status": status, + }) +} + +func (s *DurableOutboxSink) recordInflight(value int) { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_durable_outbox_inflight", metrics.Labels{"name": s.name}, float64(value)) +} + +func (s *DurableOutboxSink) recordBacklog() { + if s.metrics == nil { + return + } + count, oldest := s.store.Stats() + labels := metrics.Labels{"name": s.name} + // Keep the legacy gauge during migration because capacity gates already + // consume it; its value now represents durable WAL records, not files. + s.metrics.SetGauge("vehicle_durable_spool_backlog_files", labels, float64(count)) + s.metrics.SetGauge("vehicle_durable_outbox_backlog_records", labels, float64(count)) + ageSeconds := 0.0 + if count > 0 && !oldest.IsZero() { + ageSeconds = time.Since(oldest).Seconds() + if ageSeconds < 0 { + ageSeconds = 0 + } + } + s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", labels, ageSeconds) +} diff --git a/go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go b/go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go new file mode 100644 index 00000000..4533e6c9 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go @@ -0,0 +1,380 @@ +package eventbus + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestDurableOutboxPersistsBeforeAsyncSubmitAndDeletesAfterAck(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{} + var sink *DurableOutboxSink + delegate.onSubmit = func(record durableRecord) { + if got := outboxBacklog(sink); got != 1 { + t.Fatalf("durable backlog visible at submit = %d, want 1", got) + } + if record.Envelope.EventID == "" { + t.Fatal("submitted record must have a stable event id") + } + } + var err error + sink, err = NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + if got := outboxBacklog(sink); got != 1 { + t.Fatalf("backlog before ack = %d, want 1", got) + } + delegate.completeNext(t, nil) + if got := outboxBacklog(sink); got != 0 { + t.Fatalf("backlog after ack = %d, want 0", got) + } +} + +func TestDurableOutboxImmediateSubmitErrorKeepsAcceptedRecord(t *testing.T) { + dir := t.TempDir() + submitErr := errors.New("nats pending limit") + delegate := &outboxAsyncSink{submitErrors: []error{submitErr}} + var reported error + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ + Directory: dir, + OnError: func(err error) { + reported = err + }, + }) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("accepted durable record should hide submit error, got %v", err) + } + if got := outboxBacklog(sink); got != 1 { + t.Fatalf("backlog after submit error = %d, want 1", got) + } + if reported == nil || !strings.Contains(reported.Error(), submitErr.Error()) { + t.Fatalf("reported error = %v", reported) + } +} + +func TestDurableOutboxAckErrorIsRetriedAndStableRecordIsRemoved(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + delegate.completeNext(t, errors.New("ack timeout")) + if got := outboxBacklog(sink); got != 1 { + t.Fatalf("backlog after ack error = %d, want 1", got) + } + if err := sink.ReplayOnce(context.Background()); err != nil { + t.Fatalf("ReplayOnce() error = %v", err) + } + if got := delegate.recordCount(); got != 2 { + t.Fatalf("async submissions = %d, want 2", got) + } + first, second := delegate.recordAt(0), delegate.recordAt(1) + if first.Envelope.EventID == "" || first.Envelope.EventID != second.Envelope.EventID { + t.Fatalf("replay event ids = %q and %q", first.Envelope.EventID, second.Envelope.EventID) + } + delegate.completeNext(t, nil) + if got := outboxBacklog(sink); got != 0 { + t.Fatalf("backlog after replay ack = %d, want 0", got) + } +} + +func TestDurableOutboxReplaysPreexistingRecordAfterRestart(t *testing.T) { + dir := t.TempDir() + first := &outboxAsyncSink{submitErrors: []error{errors.New("nats unavailable")}} + writer, err := NewDurableOutboxSink(first, DurableOutboxConfig{Directory: dir}) + if err != nil { + t.Fatalf("create first outbox: %v", err) + } + if err := writer.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("persist restart record: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close first outbox: %v", err) + } + delegate := &outboxAsyncSink{} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir, ReplayBatchSize: 10}) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + if err := sink.ReplayOnce(context.Background()); err != nil { + t.Fatalf("ReplayOnce() error = %v", err) + } + if got := delegate.recordCount(); got != 1 { + t.Fatalf("replayed records = %d, want 1", got) + } + delegate.completeNext(t, nil) + if got := outboxBacklog(sink); got != 0 { + t.Fatalf("backlog after replay ack = %d, want 0", got) + } +} + +func TestDurableOutboxDoesNotResubmitInflightRecord(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + if err := sink.ReplayOnce(context.Background()); err != nil { + t.Fatalf("ReplayOnce() error = %v", err) + } + if got := delegate.recordCount(); got != 1 { + t.Fatalf("submissions while inflight = %d, want 1", got) + } + delegate.completeNext(t, nil) +} + +func TestDurableOutboxRejectsInvalidRecordBeforePersistence(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{validateErr: errors.New("subject not configured")} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + + err = sink.PublishRaw(context.Background(), durableTestEnvelope()) + if err == nil || !strings.Contains(err.Error(), "subject not configured") { + t.Fatalf("PublishRaw() error = %v", err) + } + if got := outboxBacklog(sink); got != 0 { + t.Fatalf("invalid record backlog = %d, want 0", got) + } + if got := delegate.recordCount(); got != 0 { + t.Fatalf("invalid record submissions = %d, want 0", got) + } +} + +func TestDurableOutboxCloseWaitsForPendingAck(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ + Directory: dir, + CloseTimeout: time.Second, + }) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + + done := make(chan error, 1) + go func() { done <- sink.Close() }() + select { + case err := <-done: + t.Fatalf("Close() returned before ack: %v", err) + case <-time.After(20 * time.Millisecond): + } + delegate.completeNext(t, nil) + select { + case err := <-done: + if err != nil { + t.Fatalf("Close() error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close() did not return after ack") + } +} + +func TestDurableOutboxCloseTimeoutRetainsRecord(t *testing.T) { + dir := t.TempDir() + delegate := &outboxAsyncSink{} + sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ + Directory: dir, + CloseTimeout: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("NewDurableOutboxSink() error = %v", err) + } + if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + + err = sink.Close() + if err == nil || !strings.Contains(err.Error(), "close timed out") { + t.Fatalf("Close() error = %v", err) + } + if got := outboxBacklog(sink); got != 1 { + t.Fatalf("backlog after close timeout = %d, want 1", got) + } + delegate.completeNext(t, errors.New("connection closed")) +} + +func BenchmarkDurableOutboxPublishRaw(b *testing.B) { + for _, syncWrites := range []bool{false, true} { + name := "fsync_off" + if syncWrites { + name = "fsync_on" + } + b.Run(name, func(b *testing.B) { + sink, err := NewDurableOutboxSink(autoAckOutboxSink{}, DurableOutboxConfig{ + Directory: b.TempDir(), + SyncWrites: syncWrites, + }) + if err != nil { + b.Fatalf("NewDurableOutboxSink() error = %v", err) + } + var sequence atomic.Uint32 + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + env := durableTestEnvelope() + env.EventID = "" + env.Sequence = uint16(sequence.Add(1)) + if err := sink.PublishRaw(context.Background(), env); err != nil { + b.Fatalf("PublishRaw() error = %v", err) + } + } + }) + b.StopTimer() + if err := sink.Close(); err != nil { + b.Fatalf("Close() error = %v", err) + } + }) + } +} + +func outboxBacklog(sink *DurableOutboxSink) int { + count, _ := sink.store.Stats() + return count +} + +type outboxAsyncSink struct { + mu sync.Mutex + validateErr error + submitErrors []error + records []durableRecord + callbacks []func(error) + onSubmit func(durableRecord) + closeCalls int +} + +func (s *outboxAsyncSink) ValidateRecord(record durableRecord) error { + if s.validateErr != nil { + return s.validateErr + } + switch record.Kind { + case "raw", "unified", "fields": + return nil + default: + return errUnknownRecordKind(record.Kind) + } +} + +func (s *outboxAsyncSink) PublishRecordAsync(record durableRecord, complete func(error)) error { + if s.onSubmit != nil { + s.onSubmit(record) + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.submitErrors) > 0 { + err := s.submitErrors[0] + s.submitErrors = s.submitErrors[1:] + if err != nil { + return err + } + } + s.records = append(s.records, record) + s.callbacks = append(s.callbacks, complete) + return nil +} + +func (s *outboxAsyncSink) PublishRaw(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (s *outboxAsyncSink) PublishUnified(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (s *outboxAsyncSink) PublishFields(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (s *outboxAsyncSink) Close() error { + s.mu.Lock() + s.closeCalls++ + s.mu.Unlock() + return nil +} + +func (s *outboxAsyncSink) completeNext(t *testing.T, err error) { + t.Helper() + s.mu.Lock() + if len(s.callbacks) == 0 { + s.mu.Unlock() + t.Fatal("no pending async callback") + } + callback := s.callbacks[0] + s.callbacks = s.callbacks[1:] + s.mu.Unlock() + callback(err) +} + +func (s *outboxAsyncSink) recordCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.records) +} + +func (s *outboxAsyncSink) recordAt(index int) durableRecord { + s.mu.Lock() + defer s.mu.Unlock() + return s.records[index] +} + +type autoAckOutboxSink struct{} + +func (autoAckOutboxSink) ValidateRecord(record durableRecord) error { + switch record.Kind { + case "raw", "unified", "fields": + return nil + default: + return errUnknownRecordKind(record.Kind) + } +} + +func (autoAckOutboxSink) PublishRecordAsync(_ durableRecord, complete func(error)) error { + complete(nil) + return nil +} + +func (autoAckOutboxSink) PublishRaw(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (autoAckOutboxSink) PublishUnified(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (autoAckOutboxSink) PublishFields(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (autoAckOutboxSink) Close() error { return nil } diff --git a/go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go b/go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go new file mode 100644 index 00000000..1e63b5b7 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go @@ -0,0 +1,804 @@ +package eventbus + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const ( + outboxWALMagic = uint32(0x4c4e5731) // LNW1 + outboxWALHeaderSize = 12 + outboxWALMaxRecordBytes = 16 << 20 + defaultWALSegmentBytes = 16 << 20 + defaultWALAppendQueue = 100_000 + defaultWALCommitBatch = 256 + defaultWALCommitInterval = time.Millisecond + defaultWALSegmentAge = 5 * time.Second +) + +var ErrDurableOutboxWALClosed = errors.New("durable outbox wal is closed") + +type durableOutboxWALConfig struct { + Directory string + SyncWrites bool + SegmentBytes int64 + SegmentAge time.Duration + AppendQueue int + CommitBatch int + CommitInterval time.Duration +} + +type outboxWALRecordRef struct { + segmentID uint64 + index int +} + +type storedOutboxRecord struct { + Ref outboxWALRecordRef + Record durableRecord +} + +type outboxWALRecordStatus uint8 + +const ( + outboxWALPending outboxWALRecordStatus = iota + outboxWALClaimed + outboxWALAcknowledged +) + +type outboxWALRecordState struct { + payloadOffset int64 + payloadLength uint32 + status outboxWALRecordStatus +} + +type outboxWALSegment struct { + id uint64 + path string + createdAt time.Time + size int64 + closed bool + deleting bool + acked int + records []*outboxWALRecordState +} + +type outboxWALAppendRequest struct { + record durableRecord + payload []byte + frame []byte + result chan outboxWALAppendResult +} + +type outboxWALAppendResult struct { + stored storedOutboxRecord + err error +} + +type durableOutboxWAL struct { + dir string + syncWrites bool + segmentBytes int64 + segmentAge time.Duration + commitBatch int + commitInterval time.Duration + + mu sync.Mutex + segments []*outboxWALSegment + segmentByID map[uint64]*outboxWALSegment + current *outboxWALSegment + currentFile *os.File + backlog int + fatalErr error + + appendMu sync.RWMutex + closed bool + queue chan outboxWALAppendRequest + writerWG sync.WaitGroup + closeOne sync.Once +} + +func newDurableOutboxWAL(cfg durableOutboxWALConfig) (*durableOutboxWAL, error) { + dir := strings.TrimSpace(cfg.Directory) + if dir == "" { + return nil, errors.New("durable outbox wal directory is required") + } + if cfg.SegmentBytes <= 0 { + cfg.SegmentBytes = defaultWALSegmentBytes + } + if cfg.SegmentAge <= 0 { + cfg.SegmentAge = defaultWALSegmentAge + } + if cfg.AppendQueue <= 0 { + cfg.AppendQueue = defaultWALAppendQueue + } + if cfg.CommitBatch <= 0 { + cfg.CommitBatch = defaultWALCommitBatch + } + if cfg.CommitInterval <= 0 { + cfg.CommitInterval = defaultWALCommitInterval + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, fmt.Errorf("create durable outbox wal directory: %w", err) + } + w := &durableOutboxWAL{ + dir: dir, + syncWrites: cfg.SyncWrites, + segmentBytes: cfg.SegmentBytes, + segmentAge: cfg.SegmentAge, + commitBatch: cfg.CommitBatch, + commitInterval: cfg.CommitInterval, + segmentByID: map[uint64]*outboxWALSegment{}, + queue: make(chan outboxWALAppendRequest, cfg.AppendQueue), + } + if err := w.loadSegments(); err != nil { + return nil, err + } + if err := w.createCurrentSegment(); err != nil { + return nil, err + } + w.writerWG.Add(1) + go w.appendLoop() + return w, nil +} + +func (w *durableOutboxWAL) Append(ctx context.Context, record durableRecord) (storedOutboxRecord, error) { + payload, err := json.Marshal(record) + if err != nil { + return storedOutboxRecord{}, fmt.Errorf("marshal durable outbox wal record: %w", err) + } + if len(payload) > outboxWALMaxRecordBytes { + return storedOutboxRecord{}, fmt.Errorf("durable outbox wal record is %d bytes, max %d", len(payload), outboxWALMaxRecordBytes) + } + request := outboxWALAppendRequest{ + record: record, + payload: payload, + frame: encodeOutboxWALFrame(payload), + result: make(chan outboxWALAppendResult, 1), + } + if ctx == nil { + ctx = context.Background() + } + w.appendMu.RLock() + if w.closed { + w.appendMu.RUnlock() + return storedOutboxRecord{}, ErrDurableOutboxWALClosed + } + select { + case w.queue <- request: + w.appendMu.RUnlock() + case <-ctx.Done(): + w.appendMu.RUnlock() + return storedOutboxRecord{}, ctx.Err() + } + // Once admitted to the WAL queue, wait for the durability result even if + // the caller context is cancelled. Otherwise a committed record could be + // left claimed with no publisher responsible for it. + result := <-request.result + return result.stored, result.err +} + +func (w *durableOutboxWAL) ClaimPending(limit int) ([]storedOutboxRecord, error) { + if limit <= 0 { + limit = defaultWALCommitBatch + } + w.mu.Lock() + refs := make([]outboxWALRecordRef, 0, limit) + for _, segment := range w.segments { + for index, state := range segment.records { + if state.status != outboxWALPending { + continue + } + state.status = outboxWALClaimed + refs = append(refs, outboxWALRecordRef{segmentID: segment.id, index: index}) + if len(refs) >= limit { + break + } + } + if len(refs) >= limit { + break + } + } + w.mu.Unlock() + if len(refs) == 0 { + return nil, nil + } + records, err := w.readClaimedRecords(refs) + if err != nil { + for _, ref := range refs { + w.Release(ref) + } + return nil, err + } + return records, nil +} + +func (w *durableOutboxWAL) Release(ref outboxWALRecordRef) { + w.mu.Lock() + defer w.mu.Unlock() + state := w.recordStateLocked(ref) + if state != nil && state.status == outboxWALClaimed { + state.status = outboxWALPending + } +} + +func (w *durableOutboxWAL) Ack(ref outboxWALRecordRef) error { + w.mu.Lock() + segment := w.segmentByID[ref.segmentID] + if segment == nil || ref.index < 0 || ref.index >= len(segment.records) { + w.mu.Unlock() + return nil + } + state := segment.records[ref.index] + if state.status == outboxWALAcknowledged { + w.mu.Unlock() + return nil + } + state.status = outboxWALAcknowledged + segment.acked++ + if w.backlog > 0 { + w.backlog-- + } + shouldDelete := segment.closed && segment.acked == len(segment.records) && !segment.deleting + if shouldDelete { + segment.deleting = true + } + w.mu.Unlock() + if !shouldDelete { + return nil + } + return w.deleteAcknowledgedSegment(segment) +} + +func (w *durableOutboxWAL) Stats() (backlog int, oldest time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + backlog = w.backlog + if backlog == 0 { + return backlog, time.Time{} + } + for _, segment := range w.segments { + if segment.acked < len(segment.records) { + return backlog, segment.createdAt + } + } + return backlog, time.Time{} +} + +func (w *durableOutboxWAL) Close() error { + w.closeOne.Do(func() { + w.appendMu.Lock() + w.closed = true + close(w.queue) + w.appendMu.Unlock() + w.writerWG.Wait() + }) + w.mu.Lock() + err := w.fatalErr + w.mu.Unlock() + return err +} + +func (w *durableOutboxWAL) appendLoop() { + defer w.writerWG.Done() + maintenanceInterval := minDuration(w.segmentAge/2, time.Second) + if maintenanceInterval < 10*time.Millisecond { + maintenanceInterval = 10 * time.Millisecond + } + maintenance := time.NewTicker(maintenanceInterval) + defer maintenance.Stop() + for { + select { + case request, ok := <-w.queue: + if !ok { + w.finishWriter() + return + } + batch := w.collectAppendBatch(request) + w.commitAppendBatch(batch) + case <-maintenance.C: + if err := w.rotateIfAged(); err != nil { + w.setFatal(err) + } + } + } +} + +func (w *durableOutboxWAL) collectAppendBatch(first outboxWALAppendRequest) []outboxWALAppendRequest { + batch := make([]outboxWALAppendRequest, 0, w.commitBatch) + batch = append(batch, first) + timer := time.NewTimer(w.commitInterval) + defer timer.Stop() + for len(batch) < w.commitBatch { + select { + case request, ok := <-w.queue: + if !ok { + return batch + } + batch = append(batch, request) + case <-timer.C: + return batch + } + } + return batch +} + +func (w *durableOutboxWAL) commitAppendBatch(batch []outboxWALAppendRequest) { + if len(batch) == 0 { + return + } + w.mu.Lock() + fatalErr := w.fatalErr + w.mu.Unlock() + if fatalErr != nil { + w.completeAppendErrors(batch, fatalErr) + return + } + for len(batch) > 0 { + if err := w.rotateBeforeAppend(len(batch[0].frame)); err != nil { + w.setFatal(err) + w.completeAppendErrors(batch, err) + return + } + count := w.batchCountForCurrentSegment(batch) + if count <= 0 { + count = 1 + } + group := batch[:count] + if err := w.commitAppendGroup(group); err != nil { + w.setFatal(err) + w.completeAppendErrors(batch, err) + return + } + batch = batch[count:] + } +} + +func (w *durableOutboxWAL) commitAppendGroup(group []outboxWALAppendRequest) error { + start := w.current.size + totalBytes := 0 + for _, request := range group { + totalBytes += len(request.frame) + } + buffer := make([]byte, 0, totalBytes) + for _, request := range group { + buffer = append(buffer, request.frame...) + } + written, err := w.currentFile.Write(buffer) + if err != nil || written != len(buffer) { + if err == nil { + err = io.ErrShortWrite + } + w.rollbackAppend(start) + return fmt.Errorf("append durable outbox wal: %w", err) + } + if w.syncWrites { + if err := w.currentFile.Sync(); err != nil { + w.rollbackAppend(start) + return fmt.Errorf("sync durable outbox wal: %w", err) + } + } + + w.mu.Lock() + segment := w.current + offset := start + results := make([]outboxWALAppendResult, 0, len(group)) + for _, request := range group { + state := &outboxWALRecordState{ + payloadOffset: offset + outboxWALHeaderSize, + payloadLength: uint32(len(request.payload)), + status: outboxWALClaimed, + } + index := len(segment.records) + segment.records = append(segment.records, state) + w.backlog++ + results = append(results, outboxWALAppendResult{stored: storedOutboxRecord{ + Ref: outboxWALRecordRef{segmentID: segment.id, index: index}, + Record: request.record, + }}) + offset += int64(len(request.frame)) + } + segment.size += int64(len(buffer)) + w.mu.Unlock() + for index, request := range group { + request.result <- results[index] + } + return nil +} + +func (w *durableOutboxWAL) rollbackAppend(size int64) { + if w.currentFile == nil { + return + } + _ = w.currentFile.Truncate(size) + if w.syncWrites { + _ = w.currentFile.Sync() + } +} + +func (w *durableOutboxWAL) rotateBeforeAppend(frameBytes int) error { + if w.current == nil { + return w.createCurrentSegment() + } + if len(w.current.records) == 0 { + return nil + } + tooLarge := w.current.size+int64(frameBytes) > w.segmentBytes + tooOld := time.Since(w.current.createdAt) >= w.segmentAge + if !tooLarge && !tooOld { + return nil + } + return w.rotateCurrentSegment() +} + +func (w *durableOutboxWAL) rotateIfAged() error { + w.mu.Lock() + current := w.current + shouldRotate := current != nil && len(current.records) > 0 && time.Since(current.createdAt) >= w.segmentAge + w.mu.Unlock() + if !shouldRotate { + return nil + } + return w.rotateCurrentSegment() +} + +func (w *durableOutboxWAL) rotateCurrentSegment() error { + if w.currentFile == nil || w.current == nil { + return w.createCurrentSegment() + } + if w.syncWrites { + if err := w.currentFile.Sync(); err != nil { + return fmt.Errorf("sync closing durable outbox wal segment: %w", err) + } + } + if err := w.currentFile.Close(); err != nil { + return fmt.Errorf("close durable outbox wal segment: %w", err) + } + w.mu.Lock() + old := w.current + old.closed = true + w.current = nil + w.currentFile = nil + deleteOld := old.acked == len(old.records) && !old.deleting + if deleteOld { + old.deleting = true + } + w.mu.Unlock() + if deleteOld { + if err := w.deleteAcknowledgedSegment(old); err != nil { + return err + } + } + return w.createCurrentSegment() +} + +func (w *durableOutboxWAL) createCurrentSegment() error { + w.mu.Lock() + lastID := uint64(0) + if len(w.segments) > 0 { + lastID = w.segments[len(w.segments)-1].id + } + w.mu.Unlock() + id := uint64(time.Now().UnixNano()) + if id <= lastID { + id = lastID + 1 + } + path := filepath.Join(w.dir, outboxWALFileName(id)) + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR|os.O_APPEND, 0o640) + if err != nil { + return fmt.Errorf("create durable outbox wal segment: %w", err) + } + if w.syncWrites { + if err := syncDirectory(w.dir); err != nil { + _ = file.Close() + return fmt.Errorf("sync durable outbox wal directory after create: %w", err) + } + } + segment := &outboxWALSegment{id: id, path: path, createdAt: time.Now()} + w.mu.Lock() + w.segments = append(w.segments, segment) + w.segmentByID[id] = segment + w.current = segment + w.currentFile = file + w.mu.Unlock() + return nil +} + +func (w *durableOutboxWAL) finishWriter() { + if w.currentFile == nil || w.current == nil { + return + } + var finishErr error + if w.syncWrites { + finishErr = w.currentFile.Sync() + } + if closeErr := w.currentFile.Close(); finishErr == nil { + finishErr = closeErr + } + w.mu.Lock() + current := w.current + current.closed = true + w.current = nil + w.currentFile = nil + deleteCurrent := current.acked == len(current.records) && !current.deleting + if deleteCurrent { + current.deleting = true + } + w.mu.Unlock() + if deleteCurrent { + if err := w.deleteAcknowledgedSegment(current); finishErr == nil { + finishErr = err + } + } + if finishErr != nil { + w.setFatal(finishErr) + } +} + +func (w *durableOutboxWAL) batchCountForCurrentSegment(batch []outboxWALAppendRequest) int { + remaining := w.segmentBytes - w.current.size + count := 0 + for _, request := range batch { + if count > 0 && int64(len(request.frame)) > remaining { + break + } + remaining -= int64(len(request.frame)) + count++ + if remaining <= 0 { + break + } + } + return count +} + +func (w *durableOutboxWAL) completeAppendErrors(batch []outboxWALAppendRequest, err error) { + for _, request := range batch { + request.result <- outboxWALAppendResult{err: err} + } +} + +func (w *durableOutboxWAL) readClaimedRecords(refs []outboxWALRecordRef) ([]storedOutboxRecord, error) { + files := map[uint64]*os.File{} + defer func() { + for _, file := range files { + _ = file.Close() + } + }() + records := make([]storedOutboxRecord, 0, len(refs)) + for _, ref := range refs { + w.mu.Lock() + segment := w.segmentByID[ref.segmentID] + state := w.recordStateLocked(ref) + w.mu.Unlock() + if segment == nil || state == nil { + return nil, fmt.Errorf("durable outbox wal record reference not found: segment=%d index=%d", ref.segmentID, ref.index) + } + file := files[segment.id] + if file == nil { + opened, err := os.Open(segment.path) + if err != nil { + return nil, fmt.Errorf("open durable outbox wal segment for replay: %w", err) + } + files[segment.id] = opened + file = opened + } + payload := make([]byte, state.payloadLength) + if _, err := file.ReadAt(payload, state.payloadOffset); err != nil { + return nil, fmt.Errorf("read durable outbox wal record: %w", err) + } + var record durableRecord + if err := json.Unmarshal(payload, &record); err != nil { + return nil, fmt.Errorf("decode durable outbox wal record: %w", err) + } + records = append(records, storedOutboxRecord{Ref: ref, Record: record}) + } + return records, nil +} + +func (w *durableOutboxWAL) loadSegments() error { + paths, err := filepath.Glob(filepath.Join(w.dir, "outbox-*.wal")) + if err != nil { + return fmt.Errorf("list durable outbox wal segments: %w", err) + } + sort.Strings(paths) + for _, path := range paths { + segment, err := loadOutboxWALSegment(path, w.syncWrites) + if err != nil { + return err + } + if len(segment.records) == 0 { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove empty durable outbox wal segment: %w", err) + } + if w.syncWrites { + if err := syncDirectory(w.dir); err != nil { + return fmt.Errorf("sync durable outbox wal directory after removing empty segment: %w", err) + } + } + continue + } + w.segments = append(w.segments, segment) + w.segmentByID[segment.id] = segment + w.backlog += len(segment.records) + } + return nil +} + +func loadOutboxWALSegment(path string, syncWrites bool) (*outboxWALSegment, error) { + id, err := parseOutboxWALFileName(filepath.Base(path)) + if err != nil { + return nil, err + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open durable outbox wal segment: %w", err) + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat durable outbox wal segment: %w", err) + } + segment := &outboxWALSegment{ + id: id, + path: path, + createdAt: info.ModTime(), + closed: true, + } + offset := int64(0) + header := make([]byte, outboxWALHeaderSize) + for offset < info.Size() { + n, readErr := file.ReadAt(header, offset) + if readErr != nil { + if readErr == io.EOF && n < outboxWALHeaderSize { + if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil { + return nil, err + } + break + } + return nil, fmt.Errorf("read durable outbox wal header at %d: %w", offset, readErr) + } + if binary.BigEndian.Uint32(header[0:4]) != outboxWALMagic { + return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid magic", path, offset) + } + length := binary.BigEndian.Uint32(header[4:8]) + checksum := binary.BigEndian.Uint32(header[8:12]) + if length == 0 || length > outboxWALMaxRecordBytes { + return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid length %d", path, offset, length) + } + frameEnd := offset + outboxWALHeaderSize + int64(length) + if frameEnd > info.Size() { + if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil { + return nil, err + } + break + } + payload := make([]byte, length) + if _, err := file.ReadAt(payload, offset+outboxWALHeaderSize); err != nil { + return nil, fmt.Errorf("read durable outbox wal payload at %d: %w", offset, err) + } + if crc32.ChecksumIEEE(payload) != checksum { + return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: checksum mismatch", path, offset) + } + segment.records = append(segment.records, &outboxWALRecordState{ + payloadOffset: offset + outboxWALHeaderSize, + payloadLength: length, + status: outboxWALPending, + }) + offset = frameEnd + } + segment.size = offset + return segment, nil +} + +func truncateOutboxWALTail(file *os.File, size int64, syncWrites bool) error { + if err := file.Truncate(size); err != nil { + return fmt.Errorf("truncate incomplete durable outbox wal tail: %w", err) + } + if syncWrites { + if err := file.Sync(); err != nil { + return fmt.Errorf("sync truncated durable outbox wal tail: %w", err) + } + } + return nil +} + +func (w *durableOutboxWAL) deleteAcknowledgedSegment(segment *outboxWALSegment) error { + err := os.Remove(segment.path) + if os.IsNotExist(err) { + err = nil + } + if err == nil && w.syncWrites { + err = syncDirectory(w.dir) + } + w.mu.Lock() + defer w.mu.Unlock() + if err != nil { + segment.deleting = false + return fmt.Errorf("delete acknowledged durable outbox wal segment: %w", err) + } + delete(w.segmentByID, segment.id) + for index, candidate := range w.segments { + if candidate != segment { + continue + } + w.segments = append(w.segments[:index], w.segments[index+1:]...) + break + } + return nil +} + +func (w *durableOutboxWAL) recordStateLocked(ref outboxWALRecordRef) *outboxWALRecordState { + segment := w.segmentByID[ref.segmentID] + if segment == nil || ref.index < 0 || ref.index >= len(segment.records) { + return nil + } + return segment.records[ref.index] +} + +func (w *durableOutboxWAL) setFatal(err error) { + if err == nil { + return + } + w.mu.Lock() + if w.fatalErr == nil { + w.fatalErr = err + } + w.mu.Unlock() +} + +func encodeOutboxWALFrame(payload []byte) []byte { + frame := make([]byte, outboxWALHeaderSize+len(payload)) + binary.BigEndian.PutUint32(frame[0:4], outboxWALMagic) + binary.BigEndian.PutUint32(frame[4:8], uint32(len(payload))) + binary.BigEndian.PutUint32(frame[8:12], crc32.ChecksumIEEE(payload)) + copy(frame[outboxWALHeaderSize:], payload) + return frame +} + +func outboxWALFileName(id uint64) string { + return fmt.Sprintf("outbox-%020d.wal", id) +} + +func parseOutboxWALFileName(name string) (uint64, error) { + if !strings.HasPrefix(name, "outbox-") || !strings.HasSuffix(name, ".wal") { + return 0, fmt.Errorf("invalid durable outbox wal file name %q", name) + } + value := strings.TrimSuffix(strings.TrimPrefix(name, "outbox-"), ".wal") + id, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("parse durable outbox wal file name %q: %w", name, err) + } + return id, nil +} + +func syncDirectory(dir string) error { + handle, err := os.Open(dir) + if err != nil { + return err + } + syncErr := handle.Sync() + closeErr := handle.Close() + return errors.Join(syncErr, closeErr) +} + +func minDuration(left, right time.Duration) time.Duration { + if left <= 0 { + return right + } + if right <= 0 || left < right { + return left + } + return right +} diff --git a/go/vehicle-gateway/internal/eventbus/durable_outbox_wal_test.go b/go/vehicle-gateway/internal/eventbus/durable_outbox_wal_test.go new file mode 100644 index 00000000..c01b7880 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/durable_outbox_wal_test.go @@ -0,0 +1,229 @@ +package eventbus + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestDurableOutboxWALConcurrentGroupCommitRecoversEveryRecord(t *testing.T) { + dir := t.TempDir() + wal, err := newDurableOutboxWAL(durableOutboxWALConfig{ + Directory: dir, + SyncWrites: true, + CommitBatch: 64, + CommitInterval: 5 * time.Millisecond, + }) + if err != nil { + t.Fatalf("new WAL: %v", err) + } + + const total = 256 + refs := make(chan outboxWALRecordRef, total) + errs := make(chan error, total) + var workers sync.WaitGroup + for i := 0; i < total; i++ { + workers.Add(1) + go func(sequence int) { + defer workers.Done() + record := walTestRecord(sequence) + stored, appendErr := wal.Append(context.Background(), record) + if appendErr != nil { + errs <- appendErr + return + } + refs <- stored.Ref + }(i) + } + workers.Wait() + close(errs) + for appendErr := range errs { + t.Fatalf("concurrent append: %v", appendErr) + } + close(refs) + for ref := range refs { + wal.Release(ref) + } + if got, _ := wal.Stats(); got != total { + t.Fatalf("backlog before restart = %d, want %d", got, total) + } + if err := wal.Close(); err != nil { + t.Fatalf("close first WAL: %v", err) + } + + recovered, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir, SyncWrites: true}) + if err != nil { + t.Fatalf("reopen WAL: %v", err) + } + records, err := recovered.ClaimPending(total + 1) + if err != nil { + t.Fatalf("claim recovered records: %v", err) + } + if len(records) != total { + t.Fatalf("recovered records = %d, want %d", len(records), total) + } + seen := make(map[string]struct{}, total) + for _, record := range records { + seen[record.Record.Envelope.EventID] = struct{}{} + if err := recovered.Ack(record.Ref); err != nil { + t.Fatalf("ack recovered record: %v", err) + } + } + if len(seen) != total { + t.Fatalf("unique recovered event ids = %d, want %d", len(seen), total) + } + if got, _ := recovered.Stats(); got != 0 { + t.Fatalf("backlog after ack = %d, want 0", got) + } + if err := recovered.Close(); err != nil { + t.Fatalf("close recovered WAL: %v", err) + } +} + +func TestDurableOutboxWALTruncatesIncompleteTrailingFrame(t *testing.T) { + dir := t.TempDir() + first := encodedWALTestFrame(t, 1) + second := encodedWALTestFrame(t, 2) + path := filepath.Join(dir, outboxWALFileName(1)) + payload := append(append([]byte{}, first...), second[:len(second)/2]...) + if err := os.WriteFile(path, payload, 0o640); err != nil { + t.Fatalf("write incomplete WAL: %v", err) + } + + wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir, SyncWrites: true}) + if err != nil { + t.Fatalf("recover incomplete WAL: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat recovered segment: %v", err) + } + if got, want := info.Size(), int64(len(first)); got != want { + t.Fatalf("truncated size = %d, want %d", got, want) + } + if got, _ := wal.Stats(); got != 1 { + t.Fatalf("recovered backlog = %d, want 1", got) + } + if err := wal.Close(); err != nil { + t.Fatalf("close WAL: %v", err) + } +} + +func TestDurableOutboxWALRejectsChecksumCorruption(t *testing.T) { + dir := t.TempDir() + frame := encodedWALTestFrame(t, 1) + frame[len(frame)-1] ^= 0xff + path := filepath.Join(dir, outboxWALFileName(1)) + if err := os.WriteFile(path, frame, 0o640); err != nil { + t.Fatalf("write corrupt WAL: %v", err) + } + + _, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir}) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("corrupt WAL error = %v", err) + } +} + +func TestDurableOutboxWALAckDeletesClosedSegment(t *testing.T) { + dir := t.TempDir() + frameSize := int64(len(encodedWALTestFrame(t, 1))) + wal, err := newDurableOutboxWAL(durableOutboxWALConfig{ + Directory: dir, + SegmentBytes: frameSize, + CommitBatch: 1, + }) + if err != nil { + t.Fatalf("new WAL: %v", err) + } + first, err := wal.Append(context.Background(), walTestRecord(1)) + if err != nil { + t.Fatalf("append first: %v", err) + } + firstPath := filepath.Join(dir, outboxWALFileName(first.Ref.segmentID)) + second, err := wal.Append(context.Background(), walTestRecord(2)) + if err != nil { + t.Fatalf("append second: %v", err) + } + if first.Ref.segmentID == second.Ref.segmentID { + t.Fatal("second append should rotate to a new segment") + } + if err := wal.Ack(first.Ref); err != nil { + t.Fatalf("ack first: %v", err) + } + if _, err := os.Stat(firstPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("closed acknowledged segment still exists: %v", err) + } + wal.Release(second.Ref) + if err := wal.Close(); err != nil { + t.Fatalf("close WAL: %v", err) + } +} + +func TestDurableOutboxWALClaimIsBoundedAndNotDuplicated(t *testing.T) { + wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: t.TempDir(), CommitBatch: 1}) + if err != nil { + t.Fatalf("new WAL: %v", err) + } + for i := 0; i < 10; i++ { + stored, err := wal.Append(context.Background(), walTestRecord(i)) + if err != nil { + t.Fatalf("append %d: %v", i, err) + } + wal.Release(stored.Ref) + } + first, err := wal.ClaimPending(3) + if err != nil || len(first) != 3 { + t.Fatalf("first claim = %d, error = %v", len(first), err) + } + second, err := wal.ClaimPending(3) + if err != nil || len(second) != 3 { + t.Fatalf("second claim = %d, error = %v", len(second), err) + } + claimed := map[outboxWALRecordRef]struct{}{} + for _, record := range append(first, second...) { + if _, duplicate := claimed[record.Ref]; duplicate { + t.Fatalf("record claimed twice: %#v", record.Ref) + } + claimed[record.Ref] = struct{}{} + wal.Release(record.Ref) + } + if err := wal.Close(); err != nil { + t.Fatalf("close WAL: %v", err) + } +} + +func TestDurableOutboxWALRejectsAppendAfterClose(t *testing.T) { + wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: t.TempDir()}) + if err != nil { + t.Fatalf("new WAL: %v", err) + } + if err := wal.Close(); err != nil { + t.Fatalf("close WAL: %v", err) + } + _, err = wal.Append(context.Background(), walTestRecord(1)) + if !errors.Is(err, ErrDurableOutboxWALClosed) { + t.Fatalf("append after close error = %v", err) + } +} + +func walTestRecord(sequence int) durableRecord { + env := normalizeDurableEnvelope(durableTestEnvelope()) + env.EventID = "wal-event-" + time.Unix(0, int64(sequence)+1).UTC().Format("150405.000000000") + env.Sequence = uint16(sequence) + return durableRecord{Kind: "raw", Envelope: env} +} + +func encodedWALTestFrame(t *testing.T, sequence int) []byte { + t.Helper() + payload, err := json.Marshal(walTestRecord(sequence)) + if err != nil { + t.Fatalf("marshal WAL test record: %v", err) + } + return encodeOutboxWALFrame(payload) +} diff --git a/go/vehicle-gateway/internal/eventbus/durable_sink.go b/go/vehicle-gateway/internal/eventbus/durable_sink.go index 8dd193c9..1f4e2136 100644 --- a/go/vehicle-gateway/internal/eventbus/durable_sink.go +++ b/go/vehicle-gateway/internal/eventbus/durable_sink.go @@ -13,16 +13,21 @@ import ( "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) type DurableConfig struct { Directory string ReplayBatchSize int + Metrics *metrics.Registry + Name string } type DurableSink struct { delegate Sink dir string + metrics *metrics.Registry + name string mu sync.Mutex seq uint64 @@ -50,12 +55,16 @@ func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink { if delegate == nil { panic("durable delegate sink must not be nil") } - return &DurableSink{ + s := &DurableSink{ delegate: delegate, dir: strings.TrimSpace(cfg.Directory), + metrics: cfg.Metrics, + name: durableMetricName(cfg.Name), rawPending: map[string]struct{}{}, replayBatchSize: cfg.ReplayBatchSize, } + s.recordBacklogAfterReplay() + return s } func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { @@ -96,16 +105,29 @@ func (s *DurableSink) ReplayOnce(ctx context.Context) error { func (s *DurableSink) replay(ctx context.Context, limit int) error { files, err := durableFiles(s.dir, limit) if err != nil { + s.recordReplay("list_error", 0) return err } + s.recordBacklogAfterReplay() records := make([]durableRecordFile, 0, len(files)) for _, file := range files { record, err := readDurableRecord(file) if err != nil { - return err + s.recordReplay("read_error", 1) + if quarantineErr := quarantineDurableFile(file); quarantineErr != nil { + s.recordReplay("quarantine_error", 1) + return fmt.Errorf("quarantine unreadable durable record %s: read error: %w; quarantine error: %v", file, err, quarantineErr) + } + s.recordReplay("quarantined", 1) + continue } records = append(records, durableRecordFile{path: file, record: record}) } + if len(records) == 0 { + s.recordReplay("empty", 0) + s.recordBacklogAfterReplay() + return nil + } sortDurableRecords(records) if publisher, ok := s.delegate.(recordPublishingSink); ok { durableRecords := make([]durableRecord, 0, len(records)) @@ -113,29 +135,41 @@ func (s *DurableSink) replay(ctx context.Context, limit int) error { durableRecords = append(durableRecords, item.record) } if err := publisher.PublishRecords(ctx, durableRecords); err != nil { + s.recordReplay("publish_error", len(records)) + s.recordReplayRecords(records, "publish_error") return err } for _, item := range records { if err := os.Remove(item.path); err != nil { + s.recordReplay("delete_error", len(records)) return err } if item.record.Kind == "raw" { s.clearRawPending(item.record.Envelope) } } + s.recordReplay("ok", len(records)) + s.recordReplayRecords(records, "ok") + s.recordBacklogAfterReplay() return nil } for _, item := range records { if err := s.publishRecord(ctx, item.record); err != nil { + s.recordReplay("publish_error", len(records)) + s.recordReplayRecord(item.record, "publish_error") return err } if err := os.Remove(item.path); err != nil { + s.recordReplay("delete_error", len(records)) return err } if item.record.Kind == "raw" { s.clearRawPending(item.record.Envelope) } + s.recordReplayRecord(item.record, "ok") } + s.recordReplay("ok", len(records)) + s.recordBacklogAfterReplay() return nil } @@ -210,7 +244,7 @@ func errUnknownRecordKind(kind string) error { func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error { if s.dir == "" { - return fmt.Errorf("durable spool directory is empty") + return s.spoolError(kind, fmt.Errorf("durable spool directory is empty")) } if env.EventID == "" { env.EventID = env.StableEventID() @@ -219,19 +253,24 @@ func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error { env.ParseStatus = envelope.ParseOK } if err := os.MkdirAll(s.dir, 0o750); err != nil { - return err + return s.spoolError(kind, err) } payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env}) if err != nil { - return err + return s.spoolError(kind, err) } name := s.nextFileName(env, kind) path := filepath.Join(s.dir, name) tmp := path + ".tmp" if err := os.WriteFile(tmp, payload, 0o640); err != nil { - return err + return s.spoolError(kind, err) } - return os.Rename(tmp, path) + if err := os.Rename(tmp, path); err != nil { + return s.spoolError(kind, err) + } + s.recordSpool(kind, "ok") + s.recordBacklogAfterReplay() + return nil } func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string { @@ -273,6 +312,16 @@ func readDurableRecord(path string) (durableRecord, error) { return record, json.Unmarshal(payload, &record) } +func quarantineDurableFile(path string) error { + target := path + ".bad" + if _, err := os.Stat(target); err == nil { + target = fmt.Sprintf("%s.%d.bad", path, time.Now().UnixNano()) + } else if !os.IsNotExist(err) { + return err + } + return os.Rename(path, target) +} + func durableFiles(dir string, limit int) ([]string, error) { if limit > 0 { handle, err := os.Open(dir) @@ -330,3 +379,108 @@ func durableFilesFromReader(dir string, limit int, reader durableNameReader) ([] func errorsIsEOF(err error) bool { return err == io.EOF } + +func durableMetricName(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "default" + } + return name +} + +func (s *DurableSink) spoolError(kind string, err error) error { + s.recordSpool(kind, "error") + return err +} + +func (s *DurableSink) recordSpool(kind string, status string) { + if s == nil || s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{ + "name": s.name, + "kind": kind, + "status": status, + }) +} + +func (s *DurableSink) recordReplay(status string, records int) { + if s == nil || s.metrics == nil { + return + } + labels := metrics.Labels{"name": s.name, "status": status} + s.metrics.IncCounter("vehicle_durable_spool_replay_total", labels) + if records > 0 { + s.metrics.AddCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{ + "name": s.name, + "kind": "all", + "status": status, + }, float64(records)) + } +} + +func (s *DurableSink) recordReplayRecords(records []durableRecordFile, status string) { + for _, item := range records { + s.recordReplayRecord(item.record, status) + } +} + +func (s *DurableSink) recordReplayRecord(record durableRecord, status string) { + if s == nil || s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{ + "name": s.name, + "kind": record.Kind, + "status": status, + }) +} + +func (s *DurableSink) recordBacklogAfterReplay() { + if s == nil || s.metrics == nil { + return + } + if strings.TrimSpace(s.dir) == "" { + s.recordBacklog(0, 0) + return + } + files, oldestAge, err := durableBacklogStats(s.dir, time.Now()) + if err != nil { + return + } + s.recordBacklog(files, oldestAge) +} + +func (s *DurableSink) recordBacklog(files int, oldestAge time.Duration) { + if s == nil || s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_durable_spool_backlog_files", metrics.Labels{"name": s.name}, float64(files)) + ageSeconds := 0.0 + if files > 0 && oldestAge > 0 { + ageSeconds = oldestAge.Seconds() + } + s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", metrics.Labels{"name": s.name}, ageSeconds) +} + +func durableBacklogStats(dir string, now time.Time) (count int, oldestAge time.Duration, err error) { + files, err := durableFiles(dir, 0) + if err != nil { + return 0, 0, err + } + for _, file := range files { + info, statErr := os.Stat(file) + if statErr != nil { + return 0, 0, statErr + } + age := now.Sub(info.ModTime()) + if age < 0 { + age = 0 + } + if count == 0 || age > oldestAge { + oldestAge = age + } + count++ + } + return count, oldestAge, nil +} diff --git a/go/vehicle-gateway/internal/eventbus/durable_sink_test.go b/go/vehicle-gateway/internal/eventbus/durable_sink_test.go index 902600a3..d6f5be04 100644 --- a/go/vehicle-gateway/internal/eventbus/durable_sink_test.go +++ b/go/vehicle-gateway/internal/eventbus/durable_sink_test.go @@ -8,8 +8,10 @@ import ( "os" "path/filepath" "testing" + "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) { @@ -167,6 +169,156 @@ func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) { } } +func TestDurableSinkRecordsSpoolAndReplayMetrics(t *testing.T) { + dir := t.TempDir() + registry := metrics.NewRegistry() + delegate := &scriptedSink{rawErrors: []error{errSpoolTest}} + sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"}) + env := durableTestEnvelope() + + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_durable_spool_records_total{kind="raw",name="nats",status="ok"} 1`, + `vehicle_durable_spool_backlog_files{name="nats"} 1`, + `vehicle_durable_spool_oldest_age_seconds{name="nats"}`, + } { + if !containsString(text, want) { + t.Fatalf("spool metric missing %s:\n%s", want, text) + } + } + + delegate.rawErrors = nil + if err := sink.ReplayOnce(context.Background()); err != nil { + t.Fatalf("ReplayOnce() error = %v", err) + } + text = registry.Render() + for _, want := range []string{ + `vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`, + `vehicle_durable_spool_replay_records_total{kind="all",name="nats",status="ok"} 1`, + `vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`, + `vehicle_durable_spool_backlog_files{name="nats"} 0`, + `vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`, + } { + if !containsString(text, want) { + t.Fatalf("replay metric missing %s:\n%s", want, text) + } + } +} + +func TestDurableSinkInitializesBacklogMetrics(t *testing.T) { + dir := t.TempDir() + registry := metrics.NewRegistry() + _ = NewDurableSink(&scriptedSink{}, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"}) + + text := registry.Render() + for _, want := range []string{ + `vehicle_durable_spool_backlog_files{name="nats"} 0`, + `vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`, + } { + if !containsString(text, want) { + t.Fatalf("initial spool metric missing %s:\n%s", want, text) + } + } +} + +func TestDurableSinkRecordsReplayPublishErrorMetrics(t *testing.T) { + dir := t.TempDir() + registry := metrics.NewRegistry() + env := durableTestEnvelope() + writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env}) + delegate := &scriptedSink{rawErrors: []error{errSpoolTest}} + sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "kafka"}) + + if err := sink.ReplayOnce(context.Background()); err == nil { + t.Fatal("ReplayOnce() error = nil, want delegate publish error") + } + text := registry.Render() + for _, want := range []string{ + `vehicle_durable_spool_replay_total{name="kafka",status="publish_error"} 1`, + `vehicle_durable_spool_replay_records_total{kind="all",name="kafka",status="publish_error"} 1`, + `vehicle_durable_spool_replay_records_total{kind="raw",name="kafka",status="publish_error"} 1`, + `vehicle_durable_spool_backlog_files{name="kafka"} 1`, + } { + if !containsString(text, want) { + t.Fatalf("replay error metric missing %s:\n%s", want, text) + } + } + if files := spoolFiles(t, dir); len(files) != 1 { + t.Fatalf("failed replay should keep spool file, files=%#v", files) + } +} + +func TestDurableBacklogStatsCountsAllFilesAndOldestAge(t *testing.T) { + dir := t.TempDir() + env := durableTestEnvelope() + now := time.Date(2026, 7, 12, 15, 0, 0, 0, time.UTC) + oldFile := filepath.Join(dir, "0001-raw.json") + newFile := filepath.Join(dir, "0002-fields.json") + writeDurableRecord(t, oldFile, durableRecord{Kind: "raw", Envelope: env}) + writeDurableRecord(t, newFile, durableRecord{Kind: "fields", Envelope: env}) + if err := os.Chtimes(oldFile, now.Add(-10*time.Minute), now.Add(-10*time.Minute)); err != nil { + t.Fatalf("chtimes old file: %v", err) + } + if err := os.Chtimes(newFile, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil { + t.Fatalf("chtimes new file: %v", err) + } + + count, oldestAge, err := durableBacklogStats(dir, now) + if err != nil { + t.Fatalf("durableBacklogStats() error = %v", err) + } + if count != 2 { + t.Fatalf("count = %d, want 2", count) + } + if oldestAge != 10*time.Minute { + t.Fatalf("oldestAge = %s, want 10m", oldestAge) + } +} + +func TestDurableSinkQuarantinesBadRecordAndReplaysRemainingFiles(t *testing.T) { + dir := t.TempDir() + registry := metrics.NewRegistry() + env := durableTestEnvelope() + if err := os.WriteFile(filepath.Join(dir, "0001-bad.json"), []byte("{bad json"), 0o640); err != nil { + t.Fatalf("write bad durable record: %v", err) + } + writeDurableRecord(t, filepath.Join(dir, "0002-raw.json"), durableRecord{Kind: "raw", Envelope: env}) + delegate := &scriptedSink{} + sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"}) + + if err := sink.ReplayOnce(context.Background()); err != nil { + t.Fatalf("ReplayOnce() error = %v", err) + } + if delegate.rawCalls != 1 { + t.Fatalf("raw calls = %d, want 1", delegate.rawCalls) + } + if files := spoolFiles(t, dir); len(files) != 0 { + t.Fatalf("normal spool files after replay = %#v, want none", files) + } + badFiles, err := filepath.Glob(filepath.Join(dir, "*.bad")) + if err != nil { + t.Fatalf("glob bad files: %v", err) + } + if len(badFiles) != 1 { + t.Fatalf("bad files = %#v, want one quarantined file", badFiles) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_durable_spool_replay_total{name="nats",status="read_error"} 1`, + `vehicle_durable_spool_replay_total{name="nats",status="quarantined"} 1`, + `vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`, + `vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`, + `vehicle_durable_spool_backlog_files{name="nats"} 0`, + } { + if !containsString(text, want) { + t.Fatalf("quarantine metric missing %s:\n%s", want, text) + } + } +} + func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) { reader := &fakeNameReader{ batches: [][]string{ diff --git a/go/vehicle-gateway/internal/eventbus/kafka_retry.go b/go/vehicle-gateway/internal/eventbus/kafka_retry.go new file mode 100644 index 00000000..8b20b0b4 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/kafka_retry.go @@ -0,0 +1,30 @@ +package eventbus + +import "github.com/segmentio/kafka-go" + +// MessagesAfterCommittedPrefixes keeps every fetched message that is not +// covered by the highest committed offset for its topic partition. This is +// used when a batch partially succeeds: later poison or valid messages must +// remain in memory until the failed offset ahead of them is durably handled. +func MessagesAfterCommittedPrefixes(messages []kafka.Message, committed []kafka.Message) []kafka.Message { + type partitionKey struct { + topic string + partition int + } + highest := make(map[partitionKey]int64, len(committed)) + for _, message := range committed { + key := partitionKey{topic: message.Topic, partition: message.Partition} + if offset, ok := highest[key]; !ok || message.Offset > offset { + highest[key] = message.Offset + } + } + remaining := make([]kafka.Message, 0, len(messages)) + for _, message := range messages { + key := partitionKey{topic: message.Topic, partition: message.Partition} + if offset, ok := highest[key]; ok && message.Offset <= offset { + continue + } + remaining = append(remaining, message) + } + return remaining +} diff --git a/go/vehicle-gateway/internal/eventbus/kafka_retry_test.go b/go/vehicle-gateway/internal/eventbus/kafka_retry_test.go new file mode 100644 index 00000000..1d7d3e69 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/kafka_retry_test.go @@ -0,0 +1,34 @@ +package eventbus + +import ( + "testing" + + "github.com/segmentio/kafka-go" +) + +func TestMessagesAfterCommittedPrefixesKeepsPartitionGaps(t *testing.T) { + messages := []kafka.Message{ + {Topic: "raw", Partition: 0, Offset: 10}, + {Topic: "raw", Partition: 1, Offset: 20}, + {Topic: "raw", Partition: 0, Offset: 11}, + {Topic: "raw", Partition: 1, Offset: 21}, + {Topic: "raw", Partition: 0, Offset: 12}, + } + committed := []kafka.Message{ + {Topic: "raw", Partition: 0, Offset: 10}, + {Topic: "raw", Partition: 1, Offset: 21}, + } + + remaining := MessagesAfterCommittedPrefixes(messages, committed) + if len(remaining) != 2 || remaining[0].Partition != 0 || remaining[0].Offset != 11 || remaining[1].Offset != 12 { + t.Fatalf("remaining = %#v", remaining) + } +} + +func TestMessagesAfterCommittedPrefixesWithoutCommitKeepsWholeBatch(t *testing.T) { + messages := []kafka.Message{{Topic: "raw", Partition: 0, Offset: 10}} + remaining := MessagesAfterCommittedPrefixes(messages, nil) + if len(remaining) != 1 || remaining[0].Offset != 10 { + t.Fatalf("remaining = %#v", remaining) + } +} diff --git a/go/vehicle-gateway/internal/eventbus/kafka_sink.go b/go/vehicle-gateway/internal/eventbus/kafka_sink.go index a81a34d3..a34c305f 100644 --- a/go/vehicle-gateway/internal/eventbus/kafka_sink.go +++ b/go/vehicle-gateway/internal/eventbus/kafka_sink.go @@ -41,16 +41,38 @@ func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) { if len(cfg.Brokers) == 0 { return nil, errors.New("kafka brokers are required") } + if err := ValidateKafkaConfig(cfg); err != nil { + return nil, err + } + rawTopics, fieldsTopics := kafkaTopicMaps(cfg) return newKafkaSinkWithWriter(&kafka.Writer{ Addr: kafka.TCP(cfg.Brokers...), Balancer: &kafka.Hash{}, AllowAutoTopicCreation: false, RequiredAcks: kafka.RequireAll, Async: false, - }, cfg), nil + }, KafkaConfig{ + RawTopics: rawTopics, + FieldsTopics: fieldsTopics, + UnifiedTopic: cfg.UnifiedTopic, + }), nil +} + +func ValidateKafkaConfig(cfg KafkaConfig) error { + rawTopics, fieldsTopics := kafkaTopicMaps(cfg) + return topics.ValidateKafkaRawFields(protocolTopicLabels(rawTopics), protocolTopicLabels(fieldsTopics)) } func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink { + rawTopics, fieldsTopics := kafkaTopicMaps(cfg) + unifiedTopic := cfg.UnifiedTopic + if unifiedTopic == "" { + unifiedTopic = topics.Unified + } + return &KafkaSink{writer: writer, rawTopics: rawTopics, fieldsTopics: fieldsTopics, unifiedTopic: unifiedTopic} +} + +func kafkaTopicMaps(cfg KafkaConfig) (map[envelope.Protocol]string, map[envelope.Protocol]string) { rawTopics := map[envelope.Protocol]string{ envelope.ProtocolGB32960: topics.RawGB32960, envelope.ProtocolJT808: topics.RawJT808, @@ -71,11 +93,15 @@ func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink { fieldsTopics[protocol] = topic } } - unifiedTopic := cfg.UnifiedTopic - if unifiedTopic == "" { - unifiedTopic = topics.Unified + return rawTopics, fieldsTopics +} + +func protocolTopicLabels(values map[envelope.Protocol]string) map[string]string { + out := make(map[string]string, len(values)) + for protocol, topic := range values { + out[string(protocol)] = topic } - return &KafkaSink{writer: writer, rawTopics: rawTopics, fieldsTopics: fieldsTopics, unifiedTopic: unifiedTopic} + return out } func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { diff --git a/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go b/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go index 77087962..ee60db21 100644 --- a/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go +++ b/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "reflect" + "strings" "testing" "github.com/segmentio/kafka-go" @@ -112,6 +113,49 @@ func TestNewKafkaSinkUsesProductionDeliveryGuarantees(t *testing.T) { } } +func TestValidateKafkaConfigRejectsRawFieldsTopicOverlap(t *testing.T) { + err := ValidateKafkaConfig(KafkaConfig{ + RawTopics: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1", + }, + FieldsTopics: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1", + }, + }) + if err == nil { + t.Fatal("ValidateKafkaConfig() error = nil, want overlap rejection") + } + if !strings.Contains(err.Error(), "fields kafka topic") { + t.Fatalf("error = %q, want fields topic family hint", err) + } +} + +func TestValidateKafkaConfigRejectsKnownProtocolTopicMismatch(t *testing.T) { + err := ValidateKafkaConfig(KafkaConfig{ + RawTopics: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.raw.go.gb32960.v1", + }, + }) + if err == nil { + t.Fatal("ValidateKafkaConfig() error = nil, want raw protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } + + err = ValidateKafkaConfig(KafkaConfig{ + FieldsTopics: map[envelope.Protocol]string{ + envelope.ProtocolYutongMQTT: "vehicle.fields.go.jt808.v1", + }, + }) + if err == nil { + t.Fatal("ValidateKafkaConfig() error = nil, want fields protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } +} + type recordingWriter struct { messages []kafka.Message writeCalls int diff --git a/go/vehicle-gateway/internal/eventbus/nats_sink.go b/go/vehicle-gateway/internal/eventbus/nats_sink.go index 46d216a2..1954514c 100644 --- a/go/vehicle-gateway/internal/eventbus/nats_sink.go +++ b/go/vehicle-gateway/internal/eventbus/nats_sink.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/nats-io/nats.go" @@ -13,16 +14,19 @@ import ( ) type NATSConfig struct { - URL string - Name string - RawSubjects map[envelope.Protocol]string - FieldsSubjects map[envelope.Protocol]string - UnifiedSubject string + URL string + Name string + RawSubjects map[envelope.Protocol]string + FieldsSubjects map[envelope.Protocol]string + UnifiedSubject string + AsyncMaxPending int + AsyncAckTimeout time.Duration } type NATSSink struct { conn *nats.Conn publisher natsPublisher + asyncPublisher natsAsyncPublisher rawSubjects map[envelope.Protocol]string fieldsSubjects map[envelope.Protocol]string unifiedSubject string @@ -34,10 +38,18 @@ type natsPublisher interface { Publish(context.Context, string, []byte, ...NATSPublishOption) error } +type natsAsyncPublisher interface { + PublishAsync(string, []byte, ...NATSPublishOption) (nats.PubAckFuture, error) +} + func NewNATSSink(cfg NATSConfig) (*NATSSink, error) { if cfg.URL == "" { return nil, errors.New("nats url is required") } + if err := ValidateNATSConfig(cfg); err != nil { + return nil, err + } + rawSubjects, fieldsSubjects := natsSubjectMaps(cfg) name := cfg.Name if name == "" { name = "lingniu-vehicle-gateway" @@ -46,17 +58,58 @@ func NewNATSSink(cfg NATSConfig) (*NATSSink, error) { if err != nil { return nil, err } - js, err := conn.JetStream() + var jsOptions []nats.JSOpt + if cfg.AsyncMaxPending > 0 { + jsOptions = append(jsOptions, nats.PublishAsyncMaxPending(cfg.AsyncMaxPending)) + } + if cfg.AsyncAckTimeout > 0 { + jsOptions = append(jsOptions, nats.PublishAsyncTimeout(cfg.AsyncAckTimeout)) + } + js, err := conn.JetStream(jsOptions...) if err != nil { conn.Close() return nil, err } - sink := newNATSSinkWithPublisher(natsJetStreamPublisher{js: js}, cfg) + publisher := natsJetStreamPublisher{js: js} + sink := newNATSSinkWithPublishers(publisher, publisher, NATSConfig{ + RawSubjects: rawSubjects, + FieldsSubjects: fieldsSubjects, + UnifiedSubject: cfg.UnifiedSubject, + }) sink.conn = conn return sink, nil } +func ValidateNATSConfig(cfg NATSConfig) error { + rawSubjects, fieldsSubjects := natsSubjectMaps(cfg) + raw := protocolTopicLabels(rawSubjects) + fields := protocolTopicLabels(fieldsSubjects) + if err := topics.ValidateKnownRawFieldsProtocols(raw, fields, "nats subject"); err != nil { + return err + } + return topics.ValidateRawFieldsDisjoint(raw, fields, "nats subject") +} + func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink { + return newNATSSinkWithPublishers(publisher, nil, cfg) +} + +func newNATSSinkWithPublishers(publisher natsPublisher, asyncPublisher natsAsyncPublisher, cfg NATSConfig) *NATSSink { + rawSubjects, fieldsSubjects := natsSubjectMaps(cfg) + unifiedSubject := cfg.UnifiedSubject + if unifiedSubject == "" { + unifiedSubject = topics.Unified + } + return &NATSSink{ + publisher: publisher, + asyncPublisher: asyncPublisher, + rawSubjects: rawSubjects, + fieldsSubjects: fieldsSubjects, + unifiedSubject: unifiedSubject, + } +} + +func natsSubjectMaps(cfg NATSConfig) (map[envelope.Protocol]string, map[envelope.Protocol]string) { rawSubjects := map[envelope.Protocol]string{ envelope.ProtocolGB32960: topics.RawGB32960, envelope.ProtocolJT808: topics.RawJT808, @@ -77,16 +130,7 @@ func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink fieldsSubjects[protocol] = subject } } - unifiedSubject := cfg.UnifiedSubject - if unifiedSubject == "" { - unifiedSubject = topics.Unified - } - return &NATSSink{ - publisher: publisher, - rawSubjects: rawSubjects, - fieldsSubjects: fieldsSubjects, - unifiedSubject: unifiedSubject, - } + return rawSubjects, fieldsSubjects } func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { @@ -94,14 +138,14 @@ func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) e if !ok || subject == "" { return fmt.Errorf("raw subject not configured for protocol %s", env.Protocol) } - return s.publish(ctx, subject, env) + return s.publish(ctx, subject, "raw", env) } func (s *NATSSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error { if s.unifiedSubject == "" { return errors.New("unified subject is empty") } - return s.publish(ctx, s.unifiedSubject, env) + return s.publish(ctx, s.unifiedSubject, "unified", env) } func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error { @@ -109,7 +153,62 @@ func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope if !ok || subject == "" { return fmt.Errorf("fields subject not configured for protocol %s", env.Protocol) } - return s.publish(ctx, subject, env) + return s.publish(ctx, subject, "fields", env) +} + +func (s *NATSSink) PublishRecords(ctx context.Context, records []durableRecord) error { + for _, record := range records { + subject, err := s.subjectForRecord(record) + if err != nil { + return err + } + if err := s.publish(ctx, subject, record.Kind, record.Envelope); err != nil { + return err + } + } + return nil +} + +func (s *NATSSink) PublishRecordAsync(record durableRecord, complete func(error)) error { + if s == nil || s.asyncPublisher == nil { + return errors.New("nats async publisher is not configured") + } + if complete == nil { + return errors.New("nats async publish completion callback is required") + } + subject, err := s.subjectForRecord(record) + if err != nil { + return err + } + payload, err := record.Envelope.MarshalJSONBytes() + if err != nil { + return err + } + future, err := s.asyncPublisher.PublishAsync( + subject, + payload, + nats.MsgId(natsMessageID(record.Kind, subject, record.Envelope)), + ) + if err != nil { + return err + } + go func() { + select { + case <-future.Ok(): + complete(nil) + case asyncErr := <-future.Err(): + if asyncErr == nil { + asyncErr = errors.New("nats async publish failed without error detail") + } + complete(asyncErr) + } + }() + return nil +} + +func (s *NATSSink) ValidateRecord(record durableRecord) error { + _, err := s.subjectForRecord(record) + return err } func (s *NATSSink) Close() error { @@ -121,12 +220,48 @@ func (s *NATSSink) Close() error { return nil } -func (s *NATSSink) publish(ctx context.Context, subject string, env envelope.FrameEnvelope) error { +func (s *NATSSink) publish(ctx context.Context, subject string, kind string, env envelope.FrameEnvelope) error { payload, err := env.MarshalJSONBytes() if err != nil { return err } - return s.publisher.Publish(ctx, subject, payload, nats.MsgId(env.StableEventID())) + return s.publisher.Publish(ctx, subject, payload, nats.MsgId(natsMessageID(kind, subject, env))) +} + +func natsMessageID(kind string, subject string, env envelope.FrameEnvelope) string { + kind = strings.TrimSpace(kind) + if kind == "" { + kind = "unknown" + } + subject = strings.TrimSpace(subject) + if subject == "" { + subject = "unknown" + } + return kind + ":" + subject + ":" + env.StableEventID() +} + +func (s *NATSSink) subjectForRecord(record durableRecord) (string, error) { + switch record.Kind { + case "raw": + subject, ok := s.rawSubjects[record.Envelope.Protocol] + if !ok || subject == "" { + return "", fmt.Errorf("raw subject not configured for protocol %s", record.Envelope.Protocol) + } + return subject, nil + case "unified": + if s.unifiedSubject == "" { + return "", errors.New("unified subject is empty") + } + return s.unifiedSubject, nil + case "fields": + subject, ok := s.fieldsSubjects[record.Envelope.Protocol] + if !ok || subject == "" { + return "", fmt.Errorf("fields subject not configured for protocol %s", record.Envelope.Protocol) + } + return subject, nil + default: + return "", errUnknownRecordKind(record.Kind) + } } type natsJetStreamPublisher struct { @@ -137,3 +272,7 @@ func (p natsJetStreamPublisher) Publish(ctx context.Context, subject string, dat _, err := p.js.Publish(subject, data, append(opts, nats.Context(ctx))...) return err } + +func (p natsJetStreamPublisher) PublishAsync(subject string, data []byte, opts ...NATSPublishOption) (nats.PubAckFuture, error) { + return p.js.PublishAsync(subject, data, opts...) +} diff --git a/go/vehicle-gateway/internal/eventbus/nats_sink_test.go b/go/vehicle-gateway/internal/eventbus/nats_sink_test.go index 1eb501bb..955b8f6e 100644 --- a/go/vehicle-gateway/internal/eventbus/nats_sink_test.go +++ b/go/vehicle-gateway/internal/eventbus/nats_sink_test.go @@ -3,7 +3,12 @@ package eventbus import ( "context" "encoding/json" + "errors" + "strings" "testing" + "time" + + "github.com/nats-io/nats.go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) @@ -57,6 +62,190 @@ func TestNATSSinkDefaultsToGoRawSubjects(t *testing.T) { } } +func TestNATSSinkPublishesDurableRecords(t *testing.T) { + publisher := &recordingNATSPublisher{} + sink := newNATSSinkWithPublisher(publisher, NATSConfig{ + RawSubjects: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1", + }, + FieldsSubjects: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.fields.go.jt808.v1", + }, + UnifiedSubject: "vehicle.event.go.unified.v1", + }) + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"} + + err := sink.PublishRecords(context.Background(), []durableRecord{ + {Kind: "raw", Envelope: env}, + {Kind: "fields", Envelope: env}, + {Kind: "unified", Envelope: env}, + }) + if err != nil { + t.Fatalf("PublishRecords() error = %v", err) + } + if len(publisher.messages) != 3 { + t.Fatalf("published messages = %d, want 3", len(publisher.messages)) + } + for i, want := range []string{ + "vehicle.raw.go.jt808.v1", + "vehicle.fields.go.jt808.v1", + "vehicle.event.go.unified.v1", + } { + if got := publisher.messages[i].subject; got != want { + t.Fatalf("message %d subject = %q, want %q", i, got, want) + } + } +} + +func TestNATSMessageIDSeparatesKindAndSubject(t *testing.T) { + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", Sequence: 7} + + raw := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env) + rawRetry := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env) + fields := natsMessageID("fields", "vehicle.fields.go.jt808.v1", env) + unified := natsMessageID("unified", "vehicle.event.go.unified.v1", env) + rawOtherSubject := natsMessageID("raw", "vehicle.raw.go.gb32960.v1", env) + + if raw != rawRetry { + t.Fatalf("same kind/subject/event id should be stable: %q vs %q", raw, rawRetry) + } + if raw == fields || raw == unified || raw == rawOtherSubject { + t.Fatalf("message ids should be unique per kind and subject: raw=%q fields=%q unified=%q rawOther=%q", raw, fields, unified, rawOtherSubject) + } + if !strings.Contains(raw, env.StableEventID()) { + t.Fatalf("message id %q should retain stable event id %q", raw, env.StableEventID()) + } +} + +func TestNATSSinkPublishRecordsRejectsUnknownKind(t *testing.T) { + sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{}) + + err := sink.PublishRecords(context.Background(), []durableRecord{ + {Kind: "unknown", Envelope: envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}}, + }) + if err == nil || !strings.Contains(err.Error(), "unknown durable record kind") { + t.Fatalf("PublishRecords() error = %v, want unknown kind", err) + } +} + +func TestNATSSinkAsyncPublishCompletesOnlyAfterPubAck(t *testing.T) { + future := newTestPubAckFuture() + asyncPublisher := &recordingNATSAsyncPublisher{future: future} + sink := newNATSSinkWithPublishers(&recordingNATSPublisher{}, asyncPublisher, NATSConfig{}) + record := durableRecord{Kind: "raw", Envelope: normalizeDurableEnvelope(durableTestEnvelope())} + completed := make(chan error, 1) + + if err := sink.PublishRecordAsync(record, func(err error) { completed <- err }); err != nil { + t.Fatalf("PublishRecordAsync() error = %v", err) + } + select { + case err := <-completed: + t.Fatalf("completion fired before PubAck: %v", err) + case <-time.After(10 * time.Millisecond): + } + if got, want := asyncPublisher.subject, "vehicle.raw.go.jt808.v1"; got != want { + t.Fatalf("async subject = %q, want %q", got, want) + } + var decoded envelope.FrameEnvelope + if err := json.Unmarshal(asyncPublisher.data, &decoded); err != nil { + t.Fatalf("decode async payload: %v", err) + } + if decoded.EventID != record.Envelope.EventID { + t.Fatalf("async event id = %q, want %q", decoded.EventID, record.Envelope.EventID) + } + + future.ok <- &nats.PubAck{Stream: "VEHICLE_RAW", Sequence: 1} + select { + case err := <-completed: + if err != nil { + t.Fatalf("completion error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("completion did not fire after PubAck") + } +} + +func TestNATSSinkAsyncPublishPropagatesFutureError(t *testing.T) { + future := newTestPubAckFuture() + sink := newNATSSinkWithPublishers( + &recordingNATSPublisher{}, + &recordingNATSAsyncPublisher{future: future}, + NATSConfig{}, + ) + completed := make(chan error, 1) + errWant := errors.New("jetstream ack timeout") + + if err := sink.PublishRecordAsync(durableRecord{ + Kind: "raw", + Envelope: normalizeDurableEnvelope(durableTestEnvelope()), + }, func(err error) { completed <- err }); err != nil { + t.Fatalf("PublishRecordAsync() error = %v", err) + } + future.err <- errWant + select { + case err := <-completed: + if !errors.Is(err, errWant) { + t.Fatalf("completion error = %v, want %v", err, errWant) + } + case <-time.After(time.Second): + t.Fatal("completion did not fire after future error") + } +} + +func TestNATSSinkValidatesDurableRecordBeforeOutboxPersistence(t *testing.T) { + sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{}) + err := sink.ValidateRecord(durableRecord{ + Kind: "raw", + Envelope: envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")}, + }) + if err == nil || !strings.Contains(err.Error(), "raw subject not configured") { + t.Fatalf("ValidateRecord() error = %v", err) + } +} + +func TestValidateNATSConfigRejectsRawFieldsSubjectOverlap(t *testing.T) { + err := ValidateNATSConfig(NATSConfig{ + RawSubjects: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.same.jt808", + }, + FieldsSubjects: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.same.jt808", + }, + }) + if err == nil { + t.Fatal("ValidateNATSConfig() error = nil, want overlap rejection") + } + if !strings.Contains(err.Error(), "nats subject") { + t.Fatalf("error = %q, want nats subject hint", err) + } +} + +func TestValidateNATSConfigRejectsKnownProtocolSubjectMismatch(t *testing.T) { + err := ValidateNATSConfig(NATSConfig{ + RawSubjects: map[envelope.Protocol]string{ + envelope.ProtocolJT808: "vehicle.raw.go.gb32960.v1", + }, + }) + if err == nil { + t.Fatal("ValidateNATSConfig() error = nil, want raw protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } + + err = ValidateNATSConfig(NATSConfig{ + FieldsSubjects: map[envelope.Protocol]string{ + envelope.ProtocolGB32960: "vehicle.fields.go.yutong-mqtt.v1", + }, + }) + if err == nil { + t.Fatal("ValidateNATSConfig() error = nil, want fields protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } +} + type recordingNATSPublisher struct { messages []recordedNATSMessage } @@ -70,3 +259,36 @@ func (p *recordingNATSPublisher) Publish(_ context.Context, subject string, data p.messages = append(p.messages, recordedNATSMessage{subject: subject, data: append([]byte(nil), data...)}) return nil } + +type recordingNATSAsyncPublisher struct { + subject string + data []byte + future nats.PubAckFuture + err error +} + +func (p *recordingNATSAsyncPublisher) PublishAsync(subject string, data []byte, _ ...NATSPublishOption) (nats.PubAckFuture, error) { + p.subject = subject + p.data = append([]byte(nil), data...) + return p.future, p.err +} + +type testPubAckFuture struct { + ok chan *nats.PubAck + err chan error + msg *nats.Msg +} + +func newTestPubAckFuture() *testPubAckFuture { + return &testPubAckFuture{ + ok: make(chan *nats.PubAck, 1), + err: make(chan error, 1), + msg: &nats.Msg{}, + } +} + +func (f *testPubAckFuture) Ok() <-chan *nats.PubAck { return f.ok } + +func (f *testPubAckFuture) Err() <-chan error { return f.err } + +func (f *testPubAckFuture) Msg() *nats.Msg { return f.msg } diff --git a/go/vehicle-gateway/internal/eventbus/partitioned_async_sink.go b/go/vehicle-gateway/internal/eventbus/partitioned_async_sink.go new file mode 100644 index 00000000..d8a71d05 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/partitioned_async_sink.go @@ -0,0 +1,293 @@ +package eventbus + +import ( + "context" + "sync" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" +) + +type PartitionedAsyncConfig struct { + RawQueueSize int + DerivedQueueSize int + RawWorkers int + DerivedWorkers int + EnqueueTimeout time.Duration + RawEnqueueTimeout time.Duration + DerivedEnqueueTimeout time.Duration + OperationTimeout time.Duration + OnError func(error) + Metrics *metrics.Registry + Name string +} + +type PartitionedAsyncSink struct { + delegate Sink + rawJobs chan asyncJob + derivedJobs chan asyncJob + rawEnqueueTimeout time.Duration + derivedEnqueueTimeout time.Duration + timeout time.Duration + onError func(error) + metrics *metrics.Registry + name string + queueWait *metrics.RecentLatencyByKey + + closeOnce sync.Once + closed chan struct{} + done chan struct{} + wg sync.WaitGroup +} + +func NewPartitionedAsyncSink(delegate Sink, cfg PartitionedAsyncConfig) *PartitionedAsyncSink { + if delegate == nil { + panic("partitioned async delegate sink must not be nil") + } + if cfg.RawQueueSize <= 0 { + cfg.RawQueueSize = 100_000 + } + if cfg.DerivedQueueSize <= 0 { + cfg.DerivedQueueSize = 50_000 + } + if cfg.RawWorkers <= 0 { + cfg.RawWorkers = 4 + } + if cfg.DerivedWorkers <= 0 { + cfg.DerivedWorkers = 2 + } + enqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.EnqueueTimeout, time.Second) + rawEnqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.RawEnqueueTimeout, enqueueTimeout) + derivedEnqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.DerivedEnqueueTimeout, enqueueTimeout) + if cfg.OperationTimeout <= 0 { + cfg.OperationTimeout = 30 * time.Second + } + if cfg.Name == "" { + cfg.Name = "partitioned-async" + } + s := &PartitionedAsyncSink{ + delegate: delegate, + rawJobs: make(chan asyncJob, cfg.RawQueueSize), + derivedJobs: make(chan asyncJob, cfg.DerivedQueueSize), + rawEnqueueTimeout: rawEnqueueTimeout, + derivedEnqueueTimeout: derivedEnqueueTimeout, + timeout: cfg.OperationTimeout, + onError: cfg.OnError, + metrics: cfg.Metrics, + name: cfg.Name, + queueWait: metrics.NewRecentLatencyByKey(512), + closed: make(chan struct{}), + done: make(chan struct{}), + } + s.startWorkers("raw", s.rawJobs, cfg.RawWorkers) + s.startWorkers("derived", s.derivedJobs, cfg.DerivedWorkers) + s.recordWorkers("raw", cfg.RawWorkers) + s.recordWorkers("derived", cfg.DerivedWorkers) + s.recordQueueCapacity("raw", cap(s.rawJobs)) + s.recordQueueCapacity("derived", cap(s.derivedJobs)) + go func() { + s.wg.Wait() + close(s.done) + }() + return s +} + +func (s *PartitionedAsyncSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { + return s.enqueue(ctx, "raw", s.rawJobs, s.rawEnqueueTimeout, asyncJob{kind: "raw", env: env}) +} + +func (s *PartitionedAsyncSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error { + return s.enqueue(ctx, "derived", s.derivedJobs, s.derivedEnqueueTimeout, asyncJob{kind: "unified", env: env}) +} + +func (s *PartitionedAsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error { + return s.enqueue(ctx, "derived", s.derivedJobs, s.derivedEnqueueTimeout, asyncJob{kind: "fields", env: env}) +} + +func (s *PartitionedAsyncSink) Close() error { + s.closeOnce.Do(func() { + close(s.closed) + }) + <-s.done + return s.delegate.Close() +} + +func (s *PartitionedAsyncSink) startWorkers(queueName string, jobs <-chan asyncJob, workers int) { + s.wg.Add(workers) + for i := 0; i < workers; i++ { + go s.worker(queueName, jobs) + } +} + +func (s *PartitionedAsyncSink) enqueue(ctx context.Context, queueName string, jobs chan<- asyncJob, enqueueTimeout time.Duration, job asyncJob) error { + select { + case <-s.closed: + s.recordEnqueue(job.kind, "closed") + return ErrAsyncSinkClosed + default: + } + var timeoutC <-chan time.Time + var timer *time.Timer + if enqueueTimeout > 0 { + timer = time.NewTimer(enqueueTimeout) + timeoutC = timer.C + defer timer.Stop() + } + job.enqueuedAt = time.Now() + select { + case jobs <- job: + s.recordEnqueue(job.kind, "queued") + s.recordQueueDepth(queueName) + return nil + case <-s.closed: + s.recordEnqueue(job.kind, "closed") + return ErrAsyncSinkClosed + case <-ctx.Done(): + s.recordEnqueue(job.kind, "timeout") + s.recordQueueDepth(queueName) + return ctx.Err() + case <-timeoutC: + s.recordEnqueue(job.kind, "timeout") + s.recordQueueDepth(queueName) + return ErrAsyncSinkEnqueueTimeout + } +} + +func normalizePartitionedEnqueueTimeout(value time.Duration, fallback time.Duration) time.Duration { + if value == 0 { + value = fallback + } + if value < 0 { + return 0 + } + return value +} + +func (s *PartitionedAsyncSink) worker(queueName string, jobs <-chan asyncJob) { + defer s.wg.Done() + for { + select { + case job := <-jobs: + s.publishJob(queueName, job) + case <-s.closed: + for { + select { + case job := <-jobs: + s.publishJob(queueName, job) + default: + return + } + } + } + } +} + +func (s *PartitionedAsyncSink) publishJob(queueName string, job asyncJob) { + s.recordQueueDepth(queueName) + s.recordQueueWait(queueName, job) + ctx, cancel := context.WithTimeout(context.Background(), s.timeout) + started := time.Now() + var err error + switch job.kind { + case "raw": + err = s.delegate.PublishRaw(ctx, job.env) + case "unified": + err = s.delegate.PublishUnified(ctx, job.env) + case "fields": + err = s.delegate.PublishFields(ctx, job.env) + } + cancel() + status := "ok" + if err != nil { + status = "error" + } + s.recordPublish(job.kind, status, time.Since(started)) + if err != nil && s.onError != nil { + s.onError(err) + } + s.recordQueueDepth(queueName) +} + +func (s *PartitionedAsyncSink) recordEnqueue(kind string, status string) { + if s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_async_sink_enqueue_total", metrics.Labels{ + "sink": s.name, + "kind": kind, + "status": status, + }) +} + +func (s *PartitionedAsyncSink) recordPublish(kind string, status string, elapsed time.Duration) { + if s.metrics == nil { + return + } + labels := metrics.Labels{ + "sink": s.name, + "kind": kind, + "status": status, + } + s.metrics.IncCounter("vehicle_async_sink_publish_total", labels) + elapsedMS := float64(elapsed.Milliseconds()) + s.metrics.SetGauge("vehicle_async_sink_publish_duration_ms", labels, elapsedMS) + s.metrics.ObserveHistogram("vehicle_async_sink_publish_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS) +} + +func (s *PartitionedAsyncSink) recordQueueDepth(queueName string) { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_async_sink_queue_depth", metrics.Labels{ + "sink": s.name, + "queue": queueName, + }, float64(s.queueDepth(queueName))) +} + +func (s *PartitionedAsyncSink) recordQueueCapacity(queueName string, value int) { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_async_sink_queue_capacity", metrics.Labels{ + "sink": s.name, + "queue": queueName, + }, float64(value)) +} + +func (s *PartitionedAsyncSink) recordQueueWait(queueName string, job asyncJob) { + if s.metrics == nil || job.enqueuedAt.IsZero() { + return + } + elapsedMS := float64(time.Since(job.enqueuedAt)) / float64(time.Millisecond) + labels := metrics.Labels{ + "sink": s.name, + "queue": queueName, + "kind": job.kind, + } + s.metrics.ObserveHistogram("vehicle_async_sink_queue_wait_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS) + p99, samples := s.queueWait.Observe(queueName+"\x00"+job.kind, elapsedMS) + s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_p99_ms", labels, p99) + s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_samples", labels, float64(samples)) +} + +func (s *PartitionedAsyncSink) recordWorkers(queueName string, workers int) { + if s.metrics == nil { + return + } + s.metrics.SetGauge("vehicle_async_sink_workers", metrics.Labels{ + "sink": s.name, + "queue": queueName, + }, float64(workers)) +} + +func (s *PartitionedAsyncSink) queueDepth(queueName string) int { + switch queueName { + case "raw": + return len(s.rawJobs) + case "derived": + return len(s.derivedJobs) + default: + return 0 + } +} diff --git a/go/vehicle-gateway/internal/eventbus/partitioned_async_sink_test.go b/go/vehicle-gateway/internal/eventbus/partitioned_async_sink_test.go new file mode 100644 index 00000000..379127b6 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/partitioned_async_sink_test.go @@ -0,0 +1,230 @@ +package eventbus + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" +) + +func TestPartitionedAsyncSinkRawQueueIsIsolatedFromDerivedBacklog(t *testing.T) { + delegate := newBlockingDerivedSink() + sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{ + RawQueueSize: 1, + DerivedQueueSize: 1, + RawWorkers: 1, + DerivedWorkers: 1, + EnqueueTimeout: 20 * time.Millisecond, + OperationTimeout: time.Second, + }) + defer sink.Close() + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("PublishFields() error = %v", err) + } + select { + case <-delegate.fieldsStarted: + case <-time.After(time.Second): + t.Fatal("delegate fields publish was not started") + } + if err := sink.PublishUnified(context.Background(), env); err != nil { + t.Fatalf("PublishUnified() error = %v", err) + } + + start := time.Now() + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + if elapsed := time.Since(start); elapsed > 50*time.Millisecond { + t.Fatalf("PublishRaw() blocked behind derived backlog for %s", elapsed) + } + delegate.release() +} + +func TestPartitionedAsyncSinkRecordsPerQueueMetrics(t *testing.T) { + registry := metrics.NewRegistry() + delegate := newBlockingDerivedSink() + sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{ + RawQueueSize: 2, + DerivedQueueSize: 3, + RawWorkers: 1, + DerivedWorkers: 1, + OperationTimeout: time.Second, + Metrics: registry, + Name: "nats", + }) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBSCB3D4R1234567"} + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("PublishFields() error = %v", err) + } + delegate.release() + if err := sink.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + text := registry.Render() + for _, want := range []string{ + `vehicle_async_sink_queue_capacity{queue="raw",sink="nats"} 2`, + `vehicle_async_sink_queue_capacity{queue="derived",sink="nats"} 3`, + `vehicle_async_sink_workers{queue="raw",sink="nats"} 1`, + `vehicle_async_sink_workers{queue="derived",sink="nats"} 1`, + `vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`, + `vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`, + `vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`, + `vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`, + `vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="raw",sink="nats"} 1`, + `vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="raw",sink="nats"}`, + `vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="raw",sink="nats"} 1`, + `vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="fields",queue="derived",sink="nats"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("partitioned async metric missing %s:\n%s", want, text) + } + } +} + +func TestPartitionedAsyncSinkUsesIndependentDerivedEnqueueTimeout(t *testing.T) { + delegate := newBlockingDerivedSink() + sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{ + RawQueueSize: 1, + DerivedQueueSize: 1, + RawWorkers: 1, + DerivedWorkers: 1, + RawEnqueueTimeout: 200 * time.Millisecond, + DerivedEnqueueTimeout: 10 * time.Millisecond, + OperationTimeout: time.Second, + }) + defer sink.Close() + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("PublishFields() error = %v", err) + } + select { + case <-delegate.fieldsStarted: + case <-time.After(time.Second): + t.Fatal("delegate fields publish was not started") + } + if err := sink.PublishUnified(context.Background(), env); err != nil { + t.Fatalf("first PublishUnified() error = %v", err) + } + + start := time.Now() + err := sink.PublishUnified(context.Background(), env) + elapsed := time.Since(start) + if !errors.Is(err, ErrAsyncSinkEnqueueTimeout) { + t.Fatalf("second PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err) + } + if elapsed > 100*time.Millisecond { + t.Fatalf("derived enqueue timeout took %s, want quick failure", elapsed) + } + delegate.release() +} + +func TestPartitionedAsyncSinkCloseUnblocksBlockedDerivedEnqueue(t *testing.T) { + delegate := newBlockingDerivedSink() + sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{ + RawQueueSize: 1, + DerivedQueueSize: 1, + RawWorkers: 1, + DerivedWorkers: 1, + EnqueueTimeout: -1, + OperationTimeout: time.Second, + }) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"} + if err := sink.PublishFields(context.Background(), env); err != nil { + t.Fatalf("PublishFields() error = %v", err) + } + select { + case <-delegate.fieldsStarted: + case <-time.After(time.Second): + t.Fatal("delegate fields publish was not started") + } + if err := sink.PublishUnified(context.Background(), env); err != nil { + t.Fatalf("PublishUnified() error = %v", err) + } + + publishErr := make(chan error, 1) + go func() { + publishErr <- sink.PublishUnified(context.Background(), env) + }() + closeErr := make(chan error, 1) + go func() { + closeErr <- sink.Close() + }() + + select { + case err := <-publishErr: + if !errors.Is(err, ErrAsyncSinkClosed) { + t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err) + } + case <-time.After(time.Second): + t.Fatal("blocked PublishUnified() was not released by Close") + } + delegate.release() + select { + case err := <-closeErr: + if err != nil { + t.Fatalf("Close() error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close() did not finish after delegate release") + } +} + +type blockingDerivedSink struct { + fieldsStarted chan struct{} + releaseFields chan struct{} +} + +func newBlockingDerivedSink() *blockingDerivedSink { + return &blockingDerivedSink{ + fieldsStarted: make(chan struct{}), + releaseFields: make(chan struct{}), + } +} + +func (s *blockingDerivedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (s *blockingDerivedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error { + return nil +} + +func (s *blockingDerivedSink) PublishFields(context.Context, envelope.FrameEnvelope) error { + s.signalFieldsStarted() + <-s.releaseFields + return nil +} + +func (s *blockingDerivedSink) Close() error { + s.release() + return nil +} + +func (s *blockingDerivedSink) signalFieldsStarted() { + select { + case <-s.fieldsStarted: + default: + close(s.fieldsStarted) + } +} + +func (s *blockingDerivedSink) release() { + select { + case <-s.releaseFields: + default: + close(s.releaseFields) + } +} diff --git a/go/vehicle-gateway/internal/gateway/fields.go b/go/vehicle-gateway/internal/gateway/fields.go new file mode 100644 index 00000000..e789502f --- /dev/null +++ b/go/vehicle-gateway/internal/gateway/fields.go @@ -0,0 +1,42 @@ +package gateway + +import ( + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" +) + +const ( + gatewayFieldsPublished = "published" + gatewayFieldsDelegated = "delegated_to_bridge" + gatewayFieldsSkippedNonRealtime = "skipped_non_realtime" + gatewayFieldsSkippedMissing = "skipped_missing_fields" + gatewayFieldsPublishError = "publish_error" +) + +func gatewayFieldsEnvelope(env envelope.FrameEnvelope) (envelope.FrameEnvelope, string, bool) { + if !envelope.IsRealtimeTelemetryFrame(env) { + return envelope.FrameEnvelope{}, gatewayFieldsSkippedNonRealtime, false + } + fieldsEnv, ok := realtime.BuildFieldsEnvelope(env) + if !ok { + return envelope.FrameEnvelope{}, gatewayFieldsSkippedMissing, false + } + return fieldsEnv, gatewayFieldsPublished, true +} + +func gatewayDelegatedFields(env envelope.FrameEnvelope) (int, string, bool) { + if !envelope.IsRealtimeTelemetryFrame(env) { + return 0, gatewayFieldsSkippedNonRealtime, false + } + if len(env.ParsedFields) == 0 { + return 0, gatewayFieldsSkippedMissing, false + } + return len(env.ParsedFields), gatewayFieldsDelegated, true +} + +func canonicalRawEnvelope(env envelope.FrameEnvelope) envelope.FrameEnvelope { + env.EventKind = envelope.EventKindRaw + env.Parsed = nil + env.Fields = nil + return env +} diff --git a/go/vehicle-gateway/internal/gateway/fields_test.go b/go/vehicle-gateway/internal/gateway/fields_test.go new file mode 100644 index 00000000..3cbaf184 --- /dev/null +++ b/go/vehicle-gateway/internal/gateway/fields_test.go @@ -0,0 +1,76 @@ +package gateway + +import ( + "strings" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestCanonicalRawEnvelopeDropsBareStandardizedFields(t *testing.T) { + original := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + Parsed: map[string]any{ + "location": map[string]any{"latitude": 30.5, "speed_kmh": 20}, + }, + Fields: map[string]any{ + envelope.FieldLatitude: 30.5, + envelope.FieldSpeedKMH: 20, + }, + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.5, + "jt808.location.speed_kmh": 20, + }, + } + + canonical := canonicalRawEnvelope(original) + if canonical.EventKind != envelope.EventKindRaw { + t.Fatalf("event kind = %q", canonical.EventKind) + } + if len(canonical.Fields) != 0 { + t.Fatalf("canonical raw leaked bare fields: %#v", canonical.Fields) + } + if len(canonical.Parsed) != 0 { + t.Fatalf("canonical raw leaked duplicate parsed tree: %#v", canonical.Parsed) + } + if len(canonical.ParsedFields) != 2 { + t.Fatalf("canonical raw lost protocol fields: %#v", canonical.ParsedFields) + } + if len(original.Fields) != 2 { + t.Fatalf("canonical copy mutated in-process parser fields: %#v", original.Fields) + } + if len(original.Parsed) != 1 { + t.Fatalf("canonical copy mutated in-process parsed tree: %#v", original.Parsed) + } +} + +func TestCanonicalRawEnvelopeReducesDuplicateParsedPayload(t *testing.T) { + original := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x02", + VIN: "LTESTVIN000000001", + ParseStatus: envelope.ParseOK, + Parsed: map[string]any{ + "data_units": []any{map[string]any{ + "name": "vendor", + "value": strings.Repeat("x", 4096), + }}, + }, + ParsedFields: map[string]any{ + "gb32960.vendor.value": strings.Repeat("x", 4096), + }, + } + before, err := original.MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + after, err := canonicalRawEnvelope(original).MarshalJSONBytes() + if err != nil { + t.Fatal(err) + } + if len(after)*100 >= len(before)*65 { + t.Fatalf("canonical raw should remove the duplicate parsed tree: before=%d after=%d", len(before), len(after)) + } + t.Logf("canonical raw bytes before=%d after=%d reduction=%.1f%%", len(before), len(after), 100*(1-float64(len(after))/float64(len(before)))) +} diff --git a/go/vehicle-gateway/internal/gateway/identity_metrics.go b/go/vehicle-gateway/internal/gateway/identity_metrics.go new file mode 100644 index 00000000..e1fb57b8 --- /dev/null +++ b/go/vehicle-gateway/internal/gateway/identity_metrics.go @@ -0,0 +1,104 @@ +package gateway + +import ( + "context" + "errors" + "strings" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" +) + +var gatewayIdentityDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var gatewayFieldsCountBuckets = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000} + +func identityErrorStatus(err error) string { + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + return "error" +} + +func recordGatewayIdentityDuration(registry *metrics.Registry, protocol envelope.Protocol, status string, elapsed time.Duration) { + if registry == nil { + return + } + elapsedMS := float64(elapsed.Nanoseconds()) / float64(time.Millisecond) + labels := metrics.Labels{ + "protocol": string(protocol), + "status": status, + } + registry.SetGauge("vehicle_gateway_identity_duration_ms", labels, elapsedMS) + registry.ObserveHistogram("vehicle_gateway_identity_duration_ms_histogram", labels, gatewayIdentityDurationBucketsMS, elapsedMS) +} + +func recordGatewayIdentityCacheStatus(registry *metrics.Registry, protocol envelope.Protocol, env envelope.FrameEnvelope) { + if registry == nil { + return + } + status := identityCacheStatus(env) + if status == "" { + return + } + registry.IncCounter("vehicle_gateway_identity_cache_total", metrics.Labels{ + "protocol": string(protocol), + "cache_status": status, + }) +} + +func identityCacheStatus(env envelope.FrameEnvelope) string { + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok { + return "" + } + status, ok := identity["cache_status"].(string) + if !ok { + return "" + } + return strings.TrimSpace(status) +} + +func annotateIdentityError(env *envelope.FrameEnvelope, err error) { + if env == nil || err == nil { + return + } + if env.Parsed == nil { + env.Parsed = map[string]any{} + } + identity, _ := env.Parsed["identity"].(map[string]any) + if identity == nil { + identity = map[string]any{} + } + if _, ok := identity["resolved"]; !ok { + identity["resolved"] = strings.TrimSpace(env.VIN) != "" + } + identity["error"] = err.Error() + env.Parsed["identity"] = identity +} + +func recordGatewayFieldsMetric(registry *metrics.Registry, protocol envelope.Protocol, status string) { + if registry == nil { + return + } + registry.IncCounter("vehicle_gateway_fields_total", metrics.Labels{ + "protocol": string(protocol), + "status": status, + }) +} + +func recordGatewayFieldsCount(registry *metrics.Registry, protocol envelope.Protocol, status string, fieldCount int) { + if registry == nil { + return + } + if fieldCount < 0 { + fieldCount = 0 + } + labels := metrics.Labels{ + "protocol": string(protocol), + "status": status, + } + value := float64(fieldCount) + registry.SetGauge("vehicle_gateway_fields_count", labels, value) + registry.ObserveHistogram("vehicle_gateway_fields_count_histogram", labels, gatewayFieldsCountBuckets, value) +} diff --git a/go/vehicle-gateway/internal/gateway/identity_metrics_test.go b/go/vehicle-gateway/internal/gateway/identity_metrics_test.go new file mode 100644 index 00000000..2dc95a0d --- /dev/null +++ b/go/vehicle-gateway/internal/gateway/identity_metrics_test.go @@ -0,0 +1,27 @@ +package gateway + +import ( + "strings" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" +) + +func TestRecordGatewayIdentityCacheStatus(t *testing.T) { + registry := metrics.NewRegistry() + recordGatewayIdentityCacheStatus(registry, envelope.ProtocolJT808, envelope.FrameEnvelope{ + Parsed: map[string]any{ + "identity": map[string]any{ + "resolved": true, + "cache_status": "stale", + }, + }, + }) + + got := registry.Render() + want := `vehicle_gateway_identity_cache_total{cache_status="stale",protocol="JT808"} 1` + if !strings.Contains(got, want) { + t.Fatalf("metrics missing %q:\n%s", want, got) + } +} diff --git a/go/vehicle-gateway/internal/gateway/mqtt_client.go b/go/vehicle-gateway/internal/gateway/mqtt_client.go index 6b19e805..3ca31912 100644 --- a/go/vehicle-gateway/internal/gateway/mqtt_client.go +++ b/go/vehicle-gateway/internal/gateway/mqtt_client.go @@ -41,6 +41,7 @@ type MQTTClientConfig struct { Logger *slog.Logger Metrics *metrics.Registry PublishUnified bool + DelegateFields bool } type MQTTClient struct { @@ -204,28 +205,42 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload [] } env.EventID = env.StableEventID() c.recordParseErrorMetric(err) - } else { + } else if envelope.RequiresVehicleIdentity(env) { + resolveStarted := time.Now() resolved, resolveErr := c.cfg.Resolver.Resolve(messageCtx, env) if resolveErr != nil { - c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr) - if env.Parsed == nil { - env.Parsed = map[string]any{} + if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" { + env = resolved } - env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()} + identityStatus := identityErrorStatus(resolveErr) + c.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) + c.cfg.Logger.Warn("mqtt identity resolve failed", "topic", topic, "event_id", env.StableEventID(), "error", resolveErr) + annotateIdentityError(&env, resolveErr) env.ParseStatus = envelope.ParsePartial - c.recordIdentityMetric("error") + c.recordIdentityMetric(identityStatus) + c.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr)) } else { env = resolved annotateIdentityUnresolved(&env) - c.recordIdentityMetric(identityStatus(env)) + identityStatus := identityStatus(env) + c.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) + c.recordIdentityMetric(identityStatus) + if identityStatus != "resolved" { + c.recordIdentityIssueMetric(identityStatus, env, "no_binding") + } + recordGatewayIdentityCacheStatus(c.cfg.Metrics, envelope.ProtocolYutongMQTT, env) } + } else { + c.recordIdentitySkipMetric("non_vehicle_frame") } frameStatus = env.ParseStatus + env.EventKind = envelope.EventKindRaw c.recordFrameMetric(env.ParseStatus) if env.ParseStatus != envelope.ParseBadFrame { realtime.EnsureParsedFields(&env) } - if err := c.cfg.Sink.PublishRaw(messageCtx, env); err != nil { + canonicalRaw := canonicalRawEnvelope(env) + if err := c.cfg.Sink.PublishRaw(messageCtx, canonicalRaw); err != nil { c.recordPublishMetric("raw", "error") c.cfg.Logger.Error("publish mqtt raw failed", "topic", topic, "event_id", env.StableEventID(), "error", err) return @@ -234,21 +249,37 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload [] if env.ParseStatus == envelope.ParseBadFrame { return } - if fieldsEnv, ok := realtime.BuildFieldsEnvelope(env); ok { - if err := c.cfg.Sink.PublishFields(messageCtx, fieldsEnv); err != nil { - c.recordPublishMetric("fields", "error") - c.cfg.Logger.Error("publish mqtt fields failed", "topic", topic, "event_id", env.StableEventID(), "error", err) - return + if c.cfg.DelegateFields { + fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env) + c.recordFieldsMetric(fieldsStatus) + if ok { + c.recordFieldsCount(fieldsStatus, fieldCount) + c.recordPublishMetric("fields", "delegated") + } + } else { + fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env) + if !ok { + c.recordFieldsMetric(fieldsStatus) + } else { + if err := c.cfg.Sink.PublishFields(messageCtx, fieldsEnv); err != nil { + c.recordFieldsMetric(gatewayFieldsPublishError) + c.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields)) + c.recordPublishMetric("fields", "error") + c.cfg.Logger.Error("publish mqtt fields failed", "topic", topic, "event_id", env.StableEventID(), "error", err) + } else { + c.recordFieldsMetric(fieldsStatus) + c.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields)) + c.recordPublishMetric("fields", "ok") + } } - c.recordPublishMetric("fields", "ok") } if c.cfg.PublishUnified { - if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil { + if err := c.cfg.Sink.PublishUnified(messageCtx, canonicalRaw); err != nil { c.recordPublishMetric("unified", "error") c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err) - return + } else { + c.recordPublishMetric("unified", "ok") } - c.recordPublishMetric("unified", "ok") } } @@ -256,10 +287,12 @@ func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) { if c.cfg.Metrics == nil { return } - c.cfg.Metrics.IncCounter("vehicle_gateway_frames_total", metrics.Labels{ + labels := metrics.Labels{ "protocol": string(envelope.ProtocolYutongMQTT), "status": string(status), - }) + } + c.cfg.Metrics.IncCounter("vehicle_gateway_frames_total", labels) + metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_frame_unix_seconds", labels) } func (c *MQTTClient) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) { @@ -282,11 +315,21 @@ func (c *MQTTClient) recordPublishMetric(kind string, status string) { if c.cfg.Metrics == nil { return } - c.cfg.Metrics.IncCounter("vehicle_gateway_publish_total", metrics.Labels{ + labels := metrics.Labels{ "protocol": string(envelope.ProtocolYutongMQTT), "kind": kind, "status": status, - }) + } + c.cfg.Metrics.IncCounter("vehicle_gateway_publish_total", labels) + metrics.RecordLastActivity(c.cfg.Metrics, "vehicle_gateway_last_publish_unix_seconds", labels) +} + +func (c *MQTTClient) recordFieldsMetric(status string) { + recordGatewayFieldsMetric(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status) +} + +func (c *MQTTClient) recordFieldsCount(status string, fieldCount int) { + recordGatewayFieldsCount(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, fieldCount) } func (c *MQTTClient) recordParseErrorMetric(err error) { @@ -308,3 +351,37 @@ func (c *MQTTClient) recordIdentityMetric(status string) { "status": status, }) } + +func (c *MQTTClient) recordIdentitySkipMetric(reason string) { + if c.cfg.Metrics == nil { + return + } + c.cfg.Metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{ + "protocol": string(envelope.ProtocolYutongMQTT), + "reason": reason, + }) +} + +func (c *MQTTClient) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) { + if c.cfg.Metrics == nil { + return + } + messageID := strings.TrimSpace(env.MessageID) + if messageID == "" { + messageID = "unknown" + } + reason = strings.TrimSpace(reason) + if reason == "" { + reason = "unknown" + } + c.cfg.Metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{ + "protocol": string(envelope.ProtocolYutongMQTT), + "status": status, + "message_id": messageID, + "reason": reason, + }) +} + +func (c *MQTTClient) recordIdentityDuration(status string, elapsed time.Duration) { + recordGatewayIdentityDuration(c.cfg.Metrics, envelope.ProtocolYutongMQTT, status, elapsed) +} diff --git a/go/vehicle-gateway/internal/gateway/mqtt_client_test.go b/go/vehicle-gateway/internal/gateway/mqtt_client_test.go index 8ab059af..6404900b 100644 --- a/go/vehicle-gateway/internal/gateway/mqtt_client_test.go +++ b/go/vehicle-gateway/internal/gateway/mqtt_client_test.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "log/slog" "math/big" "os" @@ -19,7 +20,7 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) -func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) { +func TestMQTTClientHandleMessagePublishesRawAndFieldsByDefault(t *testing.T) { sink := &recordingSink{} client, err := NewMQTTClient(MQTTClientConfig{ EndpointName: "endpoint-a", @@ -39,18 +40,80 @@ func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) { "data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7} }`)) - if len(sink.raw) != 1 || len(sink.unified) != 0 { - t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified)) + if len(sink.raw) != 1 || len(sink.fields) != 1 || len(sink.unified) != 0 { + t.Fatalf("raw=%d fields=%d unified=%d", len(sink.raw), len(sink.fields), len(sink.unified)) } if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" { t.Fatalf("unexpected raw envelope: %#v", sink.raw[0]) } + if sink.raw[0].EventKind != envelope.EventKindRaw { + t.Fatalf("raw event kind = %q, want %q", sink.raw[0].EventKind, envelope.EventKindRaw) + } if sink.raw[0].RawText == "" { t.Fatal("mqtt raw envelope should keep text payload") } if sink.raw[0].RawHex != "" { t.Fatalf("mqtt raw envelope should not duplicate text payload as hex: %q", sink.raw[0].RawHex) } + if len(sink.raw[0].Fields) != 0 { + t.Fatalf("canonical raw must not carry bare standardized fields: %#v", sink.raw[0].Fields) + } + if got, want := sink.fields[0].Fields["yutong_mqtt.data.meter_speed"], sink.raw[0].ParsedFields["yutong_mqtt.data.meter_speed"]; got != want { + t.Fatalf("fields event should reuse raw parsed field, got %#v want %#v", got, want) + } + if sink.fields[0].EventKind != envelope.EventKindFields { + t.Fatalf("fields event kind = %q, want %q", sink.fields[0].EventKind, envelope.EventKindFields) + } + if got, want := sink.fields[0].SourceEventID, sink.raw[0].StableEventID(); got != want { + t.Fatalf("fields source event id = %#v, want %s", got, want) + } + if sink.fields[0].FieldMapping == "" { + t.Fatal("fields event should expose field mapping version") + } + if len(sink.fields[0].Parsed) != 0 || len(sink.fields[0].ParsedFields) != 0 { + t.Fatalf("fields envelope should not duplicate parsed payload: parsed=%#v parsed_fields=%#v", sink.fields[0].Parsed, sink.fields[0].ParsedFields) + } +} + +func TestMQTTClientDelegatesFieldsProjectionWhenConfigured(t *testing.T) { + sink := &recordingSink{} + registry := metrics.NewRegistry() + client, err := NewMQTTClient(MQTTClientConfig{ + EndpointName: "endpoint-a", + Broker: "tcp://127.0.0.1:1883", + ClientID: "test-client", + Topics: []string{"/ytforward/shln/+"}, + Sink: sink, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + DelegateFields: true, + }) + if err != nil { + t.Fatalf("NewMQTTClient() error = %v", err) + } + + client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{ + "device":"LTEST000000000001", + "time":"20260413100000", + "data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7} + }`)) + + if len(sink.raw) != 1 || len(sink.fields) != 0 { + t.Fatalf("raw=%d fields=%d, want canonical raw only", len(sink.raw), len(sink.fields)) + } + if len(sink.raw[0].Fields) != 0 { + t.Fatalf("delegated canonical raw must not carry bare standardized fields: %#v", sink.raw[0].Fields) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="delegated_to_bridge"} 1`, + `vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="delegated_to_bridge"} `, + `vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="delegated"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("delegated fields metric missing %s:\n%s", want, text) + } + } } func TestMQTTClientRecordsMessageMetrics(t *testing.T) { @@ -77,8 +140,18 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) { text := registry.Render() for _, want := range []string{ `vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`, + `vehicle_gateway_last_frame_unix_seconds{protocol="YUTONG_MQTT",status="OK"} `, `vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="resolved"} 1`, + `vehicle_gateway_identity_duration_ms{protocol="YUTONG_MQTT",status="resolved"}`, + `vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="YUTONG_MQTT",status="resolved"} 1`, + `vehicle_gateway_identity_duration_ms_histogram_count{protocol="YUTONG_MQTT",status="resolved"} 1`, + `vehicle_gateway_identity_duration_ms_histogram_sum{protocol="YUTONG_MQTT",status="resolved"}`, `vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`, + `vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="YUTONG_MQTT",status="ok"} `, + `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 1`, + `vehicle_gateway_fields_count{protocol="YUTONG_MQTT",status="published"} `, + `vehicle_gateway_fields_count_histogram_count{protocol="YUTONG_MQTT",status="published"} 1`, + `vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="ok"} 1`, `vehicle_gateway_frame_duration_ms{protocol="YUTONG_MQTT",status="OK"}`, `vehicle_gateway_frame_duration_ms_histogram_bucket{le="+Inf",protocol="YUTONG_MQTT",status="OK"} 1`, `vehicle_gateway_frame_duration_ms_histogram_count{protocol="YUTONG_MQTT",status="OK"} 1`, @@ -93,6 +166,36 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) { } } +func TestMQTTClientRecordsNonRealtimeFieldsSkipMetric(t *testing.T) { + registry := metrics.NewRegistry() + client, err := NewMQTTClient(MQTTClientConfig{ + EndpointName: "endpoint-a", + Broker: "tcp://127.0.0.1:1883", + ClientID: "test-client", + Topics: []string{"/ytforward/shln/+"}, + Sink: &recordingSink{}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewMQTTClient() error = %v", err) + } + + client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{ + "device":"LTEST000000000001", + "time":"20260413100000", + "data":{} + }`)) + + text := registry.Render() + if !strings.Contains(text, `vehicle_gateway_identity_skips_total{protocol="YUTONG_MQTT",reason="non_vehicle_frame"} 1`) { + t.Fatalf("identity skip metric missing:\n%s", text) + } + if !strings.Contains(text, `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="skipped_non_realtime"} 1`) { + t.Fatalf("non realtime fields skip metric missing:\n%s", text) + } +} + func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) { sink := &recordingSink{} registry := metrics.NewRegistry() @@ -167,6 +270,66 @@ func TestMQTTClientUsesUncancelledMessageContextForReceivedMessage(t *testing.T) } } +func TestMQTTClientPreservesResolvedEnvelopeWhenIdentitySideEffectFails(t *testing.T) { + sink := &recordingSink{} + registry := metrics.NewRegistry() + wantErr := errors.New("registration upsert failed") + client, err := NewMQTTClient(MQTTClientConfig{ + EndpointName: "endpoint-a", + Broker: "tcp://127.0.0.1:1883", + ClientID: "test-client", + Topics: []string{"/ytforward/shln/+"}, + Sink: sink, + Resolver: resolvedErrorResolver{vin: "LRESOLVED00000001", err: wantErr}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewMQTTClient() error = %v", err) + } + + client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{ + "device":"LTEST000000000001", + "time":"20260413100000", + "data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7} + }`)) + + if len(sink.raw) != 1 { + t.Fatalf("raw=%d, want 1", len(sink.raw)) + } + if sink.raw[0].VIN != "LRESOLVED00000001" { + t.Fatalf("raw vin = %q, want resolver vin", sink.raw[0].VIN) + } + if sink.raw[0].ParseStatus != envelope.ParsePartial { + t.Fatalf("parse status = %q, want PARTIAL", sink.raw[0].ParseStatus) + } + if len(sink.raw[0].Parsed) != 0 { + t.Fatalf("canonical raw should not duplicate parsed tree: %#v", sink.raw[0].Parsed) + } + if got := sink.raw[0].ParsedFields["yutong_mqtt.data.meter_speed"]; got != "52.3" { + t.Fatalf("protocol field lost after identity side-effect failure: %#v", got) + } + for field := range sink.raw[0].ParsedFields { + if strings.HasPrefix(field, "yutong_mqtt.identity.") { + t.Fatalf("derived identity annotation leaked into protocol fields: %s", field) + } + } + if len(sink.fields) != 1 || sink.fields[0].VIN != "LRESOLVED00000001" { + t.Fatalf("fields should preserve resolved vin, fields=%#v", sink.fields) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_identity_total{protocol="YUTONG_MQTT",status="error"} 1`, + `vehicle_gateway_identity_issues_total{message_id="MQTT",protocol="YUTONG_MQTT",reason="resolver_error",status="error"} 1`, + `vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="PARTIAL"} 1`, + `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="published"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metric missing %s:\n%s", want, text) + } + } +} + func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) { sink := &recordingSink{} client, err := NewMQTTClient(MQTTClientConfig{ @@ -193,6 +356,44 @@ func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) { } } +func TestMQTTClientPublishesUnifiedWhenFieldsPublishFails(t *testing.T) { + sink := &recordingSink{fieldsErr: errors.New("fields queue full")} + registry := metrics.NewRegistry() + client, err := NewMQTTClient(MQTTClientConfig{ + EndpointName: "endpoint-a", + Broker: "tcp://127.0.0.1:1883", + ClientID: "test-client", + Topics: []string{"/ytforward/shln/+"}, + PublishUnified: true, + Sink: sink, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewMQTTClient() error = %v", err) + } + + client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{ + "device":"LTEST000000000001", + "time":"20260413100000", + "data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7} + }`)) + + if len(sink.raw) != 1 || len(sink.fields) != 1 || len(sink.unified) != 1 { + t.Fatalf("raw=%d fields=%d unified=%d", len(sink.raw), len(sink.fields), len(sink.unified)) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_fields_total{protocol="YUTONG_MQTT",status="publish_error"} 1`, + `vehicle_gateway_publish_total{kind="fields",protocol="YUTONG_MQTT",status="error"} 1`, + `vehicle_gateway_publish_total{kind="unified",protocol="YUTONG_MQTT",status="ok"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metric missing %s:\n%s", want, text) + } + } +} + func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) { dir := t.TempDir() caPath, certPath, keyPath := writeTestTLSMaterial(t, dir) diff --git a/go/vehicle-gateway/internal/gateway/tcp_server.go b/go/vehicle-gateway/internal/gateway/tcp_server.go index 615fc23d..9c401581 100644 --- a/go/vehicle-gateway/internal/gateway/tcp_server.go +++ b/go/vehicle-gateway/internal/gateway/tcp_server.go @@ -10,8 +10,10 @@ import ( "net" "strings" "sync" + "syscall" "time" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" @@ -26,11 +28,12 @@ type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (en type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error) type TCPProtocol struct { - Protocol envelope.Protocol - Addr string - Extract FrameExtractor - Parse FrameParser - Respond FrameResponder + Protocol envelope.Protocol + Addr string + Extract FrameExtractor + Parse FrameParser + Authenticate authentication.Authenticator + Respond FrameResponder } type connectionState struct { @@ -47,6 +50,9 @@ type TCPServer struct { idleTimeout time.Duration maxConnections int publishUnified bool + delegateFields bool + activeMu sync.Mutex + activeConns map[net.Conn]struct{} } type TCPServerConfig struct { @@ -59,11 +65,14 @@ type TCPServerConfig struct { IdleTimeout time.Duration MaxConnections int PublishUnified bool + DelegateFields bool } const frameOperationTimeout = 30 * time.Second var gatewayFrameDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var gatewayResponseDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} +var gatewayResponseE2ERecent = metrics.NewRecentLatencyByKey(512) func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) { if cfg.Protocol.Protocol == "" { @@ -106,6 +115,8 @@ func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) { idleTimeout: cfg.IdleTimeout, maxConnections: cfg.MaxConnections, publishUnified: cfg.PublishUnified, + delegateFields: cfg.DelegateFields, + activeConns: map[net.Conn]struct{}{}, }, nil } @@ -120,6 +131,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error { go func() { <-ctx.Done() _ = listener.Close() + s.closeActiveConnections() }() s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String()) @@ -155,12 +167,15 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error { func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { defer conn.Close() + s.trackConnection(conn) + defer s.untrackConnection(conn) + source := conn.RemoteAddr().String() log := s.logger.With("protocol", s.protocol.Protocol, "remote", source) s.recordConnectionMetric(1) defer s.recordConnectionMetric(-1) - log.Info("tcp connection opened") - defer log.Info("tcp connection closed") + log.Debug("tcp connection opened") + defer log.Debug("tcp connection closed") readBuffer := make([]byte, s.readBufferSize) var pending []byte @@ -182,16 +197,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { } } if err != nil { - if errors.Is(err, io.EOF) { - s.recordConnectionClose("eof") + if ctx.Err() != nil { + s.recordConnectionClose("context_cancelled") return } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { - log.Warn("tcp connection idle timeout") + log.Debug("tcp connection idle timeout") s.recordConnectionClose("read_timeout") return } + if isRoutineTCPReadClose(err) { + log.Debug("tcp connection closed by peer", "error", err) + s.recordConnectionClose("remote_closed") + return + } log.Warn("tcp read failed", "error", err) s.recordConnectionClose("read_error") return @@ -203,6 +223,55 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { } } +func isRoutineTCPReadClose(err error) bool { + if err == nil { + return false + } + if errors.Is(err, io.EOF) || + errors.Is(err, net.ErrClosed) || + errors.Is(err, syscall.ECONNRESET) || + errors.Is(err, syscall.EPIPE) { + return true + } + text := strings.ToLower(err.Error()) + return strings.Contains(text, "connection reset by peer") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "use of closed network connection") +} + +func (s *TCPServer) trackConnection(conn net.Conn) { + if conn == nil { + return + } + s.activeMu.Lock() + defer s.activeMu.Unlock() + if s.activeConns == nil { + s.activeConns = map[net.Conn]struct{}{} + } + s.activeConns[conn] = struct{}{} +} + +func (s *TCPServer) untrackConnection(conn net.Conn) { + if conn == nil { + return + } + s.activeMu.Lock() + defer s.activeMu.Unlock() + delete(s.activeConns, conn) +} + +func (s *TCPServer) closeActiveConnections() { + s.activeMu.Lock() + conns := make([]net.Conn, 0, len(s.activeConns)) + for conn := range s.activeConns { + conns = append(conns, conn) + } + s.activeMu.Unlock() + for _, conn := range conns { + _ = conn.Close() + } +} + func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) { started := time.Now() frameStatus := envelope.ParseBadFrame @@ -228,29 +297,53 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, env.EventID = env.StableEventID() s.recordParseErrorMetric(err) } else { - resolved, resolveErr := s.resolver.Resolve(frameCtx, env) - if resolveErr != nil { - s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr) - if env.Parsed == nil { - env.Parsed = map[string]any{} + if s.protocol.Authenticate != nil { + result := s.protocol.Authenticate.Authenticate(env) + authentication.Apply(&env, result) + if result.Applicable { + s.recordAuthenticationMetric(result) } - env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()} - env.ParseStatus = envelope.ParsePartial - s.recordIdentityMetric("error") - } else { - env = resolved - annotateIdentityUnresolved(&env) - s.recordIdentityMetric(identityStatus(env)) } - enrichConnectionPlatform(&env, state) + authentication.RedactParsedCredentials(&env) + if envelope.RequiresVehicleIdentity(env) { + resolveStarted := time.Now() + resolved, resolveErr := s.resolver.Resolve(frameCtx, env) + if resolveErr != nil { + if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" { + env = resolved + } + identityStatus := identityErrorStatus(resolveErr) + s.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) + s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr) + annotateIdentityError(&env, resolveErr) + env.ParseStatus = envelope.ParsePartial + s.recordIdentityMetric(identityStatus) + s.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr)) + } else { + env = resolved + annotateIdentityUnresolved(&env) + identityStatus := identityStatus(env) + s.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) + s.recordIdentityMetric(identityStatus) + if identityStatus != "resolved" { + s.recordIdentityIssueMetric(identityStatus, env, "no_binding") + } + recordGatewayIdentityCacheStatus(s.metrics, s.protocol.Protocol, env) + } + } else { + s.recordIdentitySkipMetric("non_vehicle_frame") + } } + enrichConnectionPlatform(&env, state) frameStatus = env.ParseStatus + env.EventKind = envelope.EventKindRaw s.recordFrameMetric(env.ParseStatus) if env.ParseStatus != envelope.ParseBadFrame { realtime.EnsureParsedFields(&env) } - if err := s.sink.PublishRaw(frameCtx, env); err != nil { + canonicalRaw := canonicalRawEnvelope(env) + if err := s.sink.PublishRaw(frameCtx, canonicalRaw); err != nil { s.recordPublishMetric("raw", "error") s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) return @@ -259,37 +352,79 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, if env.ParseStatus == envelope.ParseBadFrame { return } - if fieldsEnv, ok := realtime.BuildFieldsEnvelope(env); ok { - if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil { - s.recordPublishMetric("fields", "error") - s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) - return + s.writeProtocolResponse(conn, raw, env, started) + if shouldCloseAfterAuthentication(env) && conn != nil { + _ = conn.Close() + return + } + if s.delegateFields { + fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env) + s.recordFieldsMetric(fieldsStatus) + if ok { + s.recordFieldsCount(fieldsStatus, fieldCount) + s.recordPublishMetric("fields", "delegated") + } + } else { + fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env) + if !ok { + s.recordFieldsMetric(fieldsStatus) + } else { + if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil { + s.recordFieldsMetric(gatewayFieldsPublishError) + s.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields)) + s.recordPublishMetric("fields", "error") + s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) + } else { + s.recordFieldsMetric(fieldsStatus) + s.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields)) + s.recordPublishMetric("fields", "ok") + } } - s.recordPublishMetric("fields", "ok") } if s.publishUnified { - if err := s.sink.PublishUnified(frameCtx, env); err != nil { + if err := s.sink.PublishUnified(frameCtx, canonicalRaw); err != nil { s.recordPublishMetric("unified", "error") s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) - return + } else { + s.recordPublishMetric("unified", "ok") } - s.recordPublishMetric("unified", "ok") } +} + +func (s *TCPServer) writeProtocolResponse(conn net.Conn, raw []byte, env envelope.FrameEnvelope, frameStarted time.Time) { if s.protocol.Respond == nil { return } + status := "skipped" + defer func() { + s.recordResponseDuration(env.MessageID, status, time.Since(frameStarted)) + }() response, ok, err := s.protocol.Respond(raw, env) if err != nil { + status = "build_error" + s.recordResponseMetric(env.MessageID, "build_error") s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) return } if !ok || len(response) == 0 { + s.recordResponseMetric(env.MessageID, "skipped") + return + } + if conn == nil { + status = "write_error" + s.recordResponseMetric(env.MessageID, "write_error") + s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", "nil connection") return } _ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) if _, err := conn.Write(response); err != nil { + status = "write_error" + s.recordResponseMetric(env.MessageID, "write_error") s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) + return } + status = "ok" + s.recordResponseMetric(env.MessageID, "ok") } func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionState) { @@ -299,6 +434,9 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat if env.Fields == nil { env.Fields = map[string]any{} } + if shouldCloseAfterAuthentication(*env) { + return + } if current := strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])); current != "" && current != "" { state.platformName = current } @@ -308,6 +446,15 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat if strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])) == "" || fmt.Sprint(env.Fields["platform_account"]) == "" { env.Fields["platform_account"] = state.platformName } + if strings.TrimSpace(env.PlatformName) == "" { + env.PlatformName = state.platformName + } + if strings.TrimSpace(env.SourceCode) == "" { + env.SourceCode = sourceCodeFromPlatformName(state.platformName) + } + if strings.TrimSpace(env.SourceKind) == "" || strings.EqualFold(strings.TrimSpace(env.SourceKind), "UNKNOWN") { + env.SourceKind = "PLATFORM" + } if env.Parsed == nil { env.Parsed = map[string]any{} } @@ -316,14 +463,37 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat } } +func sourceCodeFromPlatformName(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + builder.WriteRune(r) + case r >= 'A' && r <= 'Z': + builder.WriteRune(r) + case r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '_' || r == '-' || r == '.': + builder.WriteRune(r) + } + } + return builder.String() +} + func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) { if s.metrics == nil { return } - s.metrics.IncCounter("vehicle_gateway_frames_total", metrics.Labels{ + labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": string(status), - }) + } + s.metrics.IncCounter("vehicle_gateway_frames_total", labels) + metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_frame_unix_seconds", labels) } func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) { @@ -346,10 +516,74 @@ func (s *TCPServer) recordPublishMetric(kind string, status string) { if s.metrics == nil { return } - s.metrics.IncCounter("vehicle_gateway_publish_total", metrics.Labels{ + labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "kind": kind, "status": status, + } + s.metrics.IncCounter("vehicle_gateway_publish_total", labels) + metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_publish_unix_seconds", labels) +} + +func (s *TCPServer) recordFieldsMetric(status string) { + recordGatewayFieldsMetric(s.metrics, s.protocol.Protocol, status) +} + +func (s *TCPServer) recordFieldsCount(status string, fieldCount int) { + recordGatewayFieldsCount(s.metrics, s.protocol.Protocol, status, fieldCount) +} + +func (s *TCPServer) recordResponseMetric(messageID string, status string) { + if s.metrics == nil { + return + } + messageID = strings.TrimSpace(messageID) + if messageID == "" { + messageID = "unknown" + } + labels := metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "message_id": messageID, + "status": status, + } + s.metrics.IncCounter("vehicle_gateway_response_total", labels) + metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_response_unix_seconds", labels) +} + +func (s *TCPServer) recordResponseDuration(messageID string, status string, elapsed time.Duration) { + if s.metrics == nil { + return + } + messageID = strings.TrimSpace(messageID) + if messageID == "" { + messageID = "unknown" + } + elapsedMS := float64(elapsed.Microseconds()) / 1000 + labels := metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "message_id": messageID, + "status": status, + } + s.metrics.ObserveHistogram("vehicle_gateway_response_duration_ms_histogram", labels, gatewayResponseDurationBucketsMS, elapsedMS) + if status != "ok" { + return + } + protocol := string(s.protocol.Protocol) + p99, samples := gatewayResponseE2ERecent.Observe(protocol, elapsedMS) + protocolLabels := metrics.Labels{"protocol": protocol} + s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_p99_ms", protocolLabels, p99) + s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_samples", protocolLabels, float64(samples)) +} + +func (s *TCPServer) recordAuthenticationMetric(result authentication.Result) { + if s.metrics == nil || !result.Applicable { + return + } + s.metrics.IncCounter("vehicle_gateway_authentication_total", metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "mode": string(result.Mode), + "source": result.Source, + "status": result.Status, }) } @@ -373,6 +607,40 @@ func (s *TCPServer) recordIdentityMetric(status string) { }) } +func (s *TCPServer) recordIdentitySkipMetric(reason string) { + if s.metrics == nil { + return + } + s.metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "reason": reason, + }) +} + +func (s *TCPServer) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) { + if s.metrics == nil { + return + } + messageID := strings.TrimSpace(env.MessageID) + if messageID == "" { + messageID = "unknown" + } + reason = strings.TrimSpace(reason) + if reason == "" { + reason = "unknown" + } + s.metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{ + "protocol": string(s.protocol.Protocol), + "status": status, + "message_id": messageID, + "reason": reason, + }) +} + +func (s *TCPServer) recordIdentityDuration(status string, elapsed time.Duration) { + recordGatewayIdentityDuration(s.metrics, s.protocol.Protocol, status, elapsed) +} + func (s *TCPServer) recordConnectionMetric(delta float64) { if s.metrics == nil { return @@ -402,6 +670,12 @@ func (s *TCPServer) recordConnectionClose(reason string) { }) } +func shouldCloseAfterAuthentication(env envelope.FrameEnvelope) bool { + return env.AuthenticationEnforced && + strings.TrimSpace(env.AuthenticationStatus) != "" && + env.AuthenticationStatus != authentication.StatusAccepted +} + func identityStatus(env envelope.FrameEnvelope) string { if strings.TrimSpace(env.VIN) != "" { return "resolved" @@ -412,6 +686,28 @@ func identityStatus(env envelope.FrameEnvelope) string { return "unresolved" } +func identityIssueReason(status string, err error) string { + switch status { + case "timeout": + return "timeout" + case "error": + if err == nil { + return "resolver_error" + } + text := strings.ToLower(strings.TrimSpace(err.Error())) + switch { + case strings.Contains(text, "connection"), strings.Contains(text, "tcp"), strings.Contains(text, "network"): + return "identity_store_connection" + case strings.Contains(text, "timeout"), strings.Contains(text, "deadline"): + return "timeout" + default: + return "resolver_error" + } + default: + return "no_binding" + } +} + func annotateIdentityUnresolved(env *envelope.FrameEnvelope) { if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" { return diff --git a/go/vehicle-gateway/internal/gateway/tcp_server_test.go b/go/vehicle-gateway/internal/gateway/tcp_server_test.go index 146e819c..be28d5f5 100644 --- a/go/vehicle-gateway/internal/gateway/tcp_server_test.go +++ b/go/vehicle-gateway/internal/gateway/tcp_server_test.go @@ -3,20 +3,48 @@ package gateway import ( "context" "encoding/hex" + "errors" + "fmt" "io" "log/slog" "net" + "os" "strings" + "syscall" "testing" "time" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808" ) -func TestTCPServerPublishesGoodFrameOnlyToRawByDefault(t *testing.T) { +func TestEnrichConnectionPlatformPromotesSourceMetadata(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x02", + ParseStatus: envelope.ParseOK, + Fields: map[string]any{}, + Parsed: map[string]any{}, + } + state := &connectionState{platformName: "Hyundai"} + + enrichConnectionPlatform(&env, state) + + if env.PlatformName != "Hyundai" || env.SourceCode != "Hyundai" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } + if got := fmt.Sprint(env.Fields["platform_account"]); got != "Hyundai" { + t.Fatalf("platform_account = %q", got) + } + if got := fmt.Sprint(env.Parsed["platform_name"]); got != "Hyundai" { + t.Fatalf("parsed platform_name = %q", got) + } +} + +func TestTCPServerPublishesGoodFrameToRawAndFieldsByDefault(t *testing.T) { frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E") if err != nil { t.Fatal(err) @@ -28,6 +56,7 @@ func TestTCPServerPublishesGoodFrameOnlyToRawByDefault(t *testing.T) { Extract: jt808.ExtractFrames, Parse: jt808.ParseFrame, }, sink) + server.resolver = vinResolver{vin: "LNBVIN00000000001"} client, done := runPipe(t, server) if _, err := client.Write(frame); err != nil { @@ -39,11 +68,71 @@ func TestTCPServerPublishesGoodFrameOnlyToRawByDefault(t *testing.T) { if len(sink.raw) != 1 || len(sink.unified) != 0 { t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified)) } + if len(sink.fields) != 1 { + t.Fatalf("fields=%d, want 1", len(sink.fields)) + } if sink.raw[0].Phone != "13307795425" { t.Fatalf("phone = %q", sink.raw[0].Phone) } - if sink.raw[0].Fields[envelope.FieldTotalMileageKM] != 10241.2 { - t.Fatalf("total mileage = %#v", sink.raw[0].Fields[envelope.FieldTotalMileageKM]) + if sink.raw[0].EventKind != envelope.EventKindRaw { + t.Fatalf("raw event kind = %q, want %q", sink.raw[0].EventKind, envelope.EventKindRaw) + } + if len(sink.raw[0].Fields) != 0 { + t.Fatalf("canonical raw must not carry bare standardized fields: %#v", sink.raw[0].Fields) + } + if got := sink.raw[0].ParsedFields["jt808.location.total_mileage_km"]; got == nil { + t.Fatalf("raw parsed fields missing total mileage: %#v", sink.raw[0].ParsedFields) + } + if got, want := sink.fields[0].Fields["jt808.location.total_mileage_km"], sink.raw[0].ParsedFields["jt808.location.total_mileage_km"]; got != want { + t.Fatalf("fields event should reuse raw parsed field, got %#v want %#v", got, want) + } + if sink.fields[0].EventKind != envelope.EventKindFields { + t.Fatalf("fields event kind = %q, want %q", sink.fields[0].EventKind, envelope.EventKindFields) + } + if got, want := sink.fields[0].SourceEventID, sink.raw[0].StableEventID(); got != want { + t.Fatalf("fields source event id = %#v, want %s", got, want) + } + if sink.fields[0].FieldMapping == "" { + t.Fatal("fields event should expose field mapping version") + } + if len(sink.fields[0].Parsed) != 0 || len(sink.fields[0].ParsedFields) != 0 { + t.Fatalf("fields envelope should not duplicate parsed payload: parsed=%#v parsed_fields=%#v", sink.fields[0].Parsed, sink.fields[0].ParsedFields) + } +} + +func TestTCPServerRecordsFieldsPublishedMetric(t *testing.T) { + frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E") + if err != nil { + t.Fatal(err) + } + sink := &recordingSink{} + registry := metrics.NewRegistry() + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: jt808.ExtractFrames, + Parse: jt808.ParseFrame, + }, sink) + server.metrics = registry + server.resolver = vinResolver{vin: "LNBVIN00000000001"} + + client, done := runPipe(t, server) + if _, err := client.Write(frame); err != nil { + t.Fatalf("client.Write() error = %v", err) + } + _ = client.Close() + <-done + + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_fields_total{protocol="JT808",status="published"} 1`, + `vehicle_gateway_fields_count{protocol="JT808",status="published"} `, + `vehicle_gateway_fields_count_histogram_count{protocol="JT808",status="published"} 1`, + `vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="ok"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metrics missing %s:\n%s", want, text) + } } } @@ -79,8 +168,16 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) { text := registry.Render() for _, want := range []string{ `vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`, + `vehicle_gateway_last_frame_unix_seconds{protocol="JT808",status="OK"} `, `vehicle_gateway_identity_total{protocol="JT808",status="unresolved"} 1`, + `vehicle_gateway_identity_issues_total{message_id="0x0200",protocol="JT808",reason="no_binding",status="unresolved"} 1`, + `vehicle_gateway_identity_duration_ms{protocol="JT808",status="unresolved"}`, + `vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="unresolved"} 1`, + `vehicle_gateway_identity_duration_ms_histogram_count{protocol="JT808",status="unresolved"} 1`, + `vehicle_gateway_identity_duration_ms_histogram_sum{protocol="JT808",status="unresolved"}`, `vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`, + `vehicle_gateway_last_publish_unix_seconds{kind="raw",protocol="JT808",status="ok"} `, + `vehicle_gateway_fields_total{protocol="JT808",status="skipped_non_realtime"} 1`, `vehicle_gateway_frame_duration_ms{protocol="JT808",status="OK"}`, `vehicle_gateway_frame_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="OK"} 1`, `vehicle_gateway_frame_duration_ms_histogram_count{protocol="JT808",status="OK"} 1`, @@ -95,6 +192,244 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) { } } +func TestTCPServerRecordsNonRealtimeFieldsSkipMetric(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{} + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolGB32960, + Addr: ":0", + Extract: gb32960.ExtractFrames, + Parse: gb32960.ParseFrame, + }, sink) + server.metrics = registry + + server.handleFrame(context.Background(), nil, buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil), "127.0.0.1:32960", &connectionState{}) + + if len(sink.raw) != 1 { + t.Fatalf("raw=%d, want 1", len(sink.raw)) + } + if len(sink.fields) != 0 { + t.Fatalf("fields=%d, want 0", len(sink.fields)) + } + if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_fields_total{protocol="GB32960",status="skipped_non_realtime"} 1`) { + t.Fatalf("non realtime fields skip metric missing:\n%s", text) + } +} + +func TestTCPServerSkipsIdentityForGB32960PlatformControlFrame(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{} + resolver := &countingResolver{vin: "LNBVIN00000000001"} + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolGB32960, + Addr: ":0", + Extract: gb32960.ExtractFrames, + Parse: gb32960.ParseFrame, + }, sink) + server.metrics = registry + server.resolver = resolver + + server.handleFrame(context.Background(), nil, buildGBFrame(0x05, 0xfe, "PLATFORMLOGIN0001", nil), "127.0.0.1:32960", &connectionState{}) + + if resolver.calls != 0 { + t.Fatalf("resolver calls = %d, want 0 for platform control frame", resolver.calls) + } + if len(sink.raw) != 1 { + t.Fatalf("raw=%d, want 1", len(sink.raw)) + } + text := registry.Render() + if strings.Contains(text, "vehicle_gateway_identity_total") { + t.Fatalf("identity metric should not be recorded for platform control frame:\n%s", text) + } + if !strings.Contains(text, `vehicle_gateway_identity_skips_total{protocol="GB32960",reason="non_vehicle_frame"} 1`) { + t.Fatalf("identity skip metric missing:\n%s", text) + } + if !strings.Contains(text, `vehicle_gateway_fields_total{protocol="GB32960",status="skipped_non_realtime"} 1`) { + t.Fatalf("non realtime fields skip metric missing:\n%s", text) + } +} + +func TestTCPServerAuthenticatesAndRedactsGB32960PlatformLogin(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{} + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolGB32960, + Addr: ":0", + Extract: gb32960.ExtractFrames, + Parse: gb32960.ParseFrame, + Authenticate: authentication.NewGB32960PlatformAuthenticator(authentication.ModeObserve, map[string][]string{"platform-a": {"secret-a"}}), + }, sink) + server.metrics = registry + body := []byte{0x1a, 0x07, 0x0d, 0x14, 0x00, 0x00, 0x00, 0x01} + body = append(body, fixedASCIIBytes("platform-a", 12)...) + body = append(body, fixedASCIIBytes("secret-a", 20)...) + body = append(body, 0x01) + + server.handleFrame(context.Background(), nil, buildGBFrame(0x05, 0xfe, "12345678901234567", body), "127.0.0.1:32960", &connectionState{}) + + if len(sink.raw) != 1 { + t.Fatalf("raw=%d, want 1", len(sink.raw)) + } + raw := sink.raw[0] + if raw.AuthenticationMode != "observe" || raw.AuthenticationStatus != authentication.StatusAccepted || raw.AuthenticationEnforced { + t.Fatalf("authentication metadata = mode:%q status:%q enforced:%v", raw.AuthenticationMode, raw.AuthenticationStatus, raw.AuthenticationEnforced) + } + if _, exists := raw.ParsedFields["gb32960.platform_login.password"]; exists { + t.Fatalf("plaintext password leaked into parsed fields: %#v", raw.ParsedFields) + } + if got := raw.ParsedFields["gb32960.platform_login.password_present"]; got != "true" { + t.Fatalf("password presence marker = %#v", got) + } + if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_authentication_total{mode="observe",protocol="GB32960",source="configured",status="accepted"} 1`) { + t.Fatalf("authentication metric missing:\n%s", text) + } +} + +func TestTCPServerResolvesIdentityForGB32960RealtimeFrame(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{} + resolver := &countingResolver{vin: "LNBVIN00000000001"} + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolGB32960, + Addr: ":0", + Extract: gb32960.ExtractFrames, + Parse: gb32960.ParseFrame, + }, sink) + server.metrics = registry + server.resolver = resolver + + server.handleFrame(context.Background(), nil, buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil), "127.0.0.1:32960", &connectionState{}) + + if resolver.calls != 1 { + t.Fatalf("resolver calls = %d, want 1 for realtime frame", resolver.calls) + } + if len(sink.raw) != 1 { + t.Fatalf("raw=%d, want 1", len(sink.raw)) + } + if sink.raw[0].VIN != "LNBVIN00000000001" { + t.Fatalf("raw vin = %q, want resolved vin", sink.raw[0].VIN) + } + if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_identity_total{protocol="GB32960",status="resolved"} 1`) { + t.Fatalf("identity resolved metric missing:\n%s", text) + } +} + +func TestTCPServerRecordsIdentityTimeoutMetrics(t *testing.T) { + registry := metrics.NewRegistry() + server, err := NewTCPServer(TCPServerConfig{ + Protocol: TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: jt808.ExtractFrames, + Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) { + return envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "13307795425", + SourceEndpoint: sourceEndpoint, + ReceivedAtMS: receivedAtMS, + EventTimeMS: receivedAtMS, + ParseStatus: envelope.ParseOK, + }, nil + }, + }, + Sink: &recordingSink{}, + Resolver: errorResolver{err: context.DeadlineExceeded}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewTCPServer() error = %v", err) + } + + server.handleFrame(context.Background(), nil, []byte{0x01}, "127.0.0.1:808", &connectionState{}) + + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_identity_total{protocol="JT808",status="timeout"} 1`, + `vehicle_gateway_identity_issues_total{message_id="0x0200",protocol="JT808",reason="timeout",status="timeout"} 1`, + `vehicle_gateway_identity_duration_ms{protocol="JT808",status="timeout"}`, + `vehicle_gateway_identity_duration_ms_histogram_bucket{le="+Inf",protocol="JT808",status="timeout"} 1`, + `vehicle_gateway_identity_duration_ms_histogram_count{protocol="JT808",status="timeout"} 1`, + `vehicle_gateway_frames_total{protocol="JT808",status="PARTIAL"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metrics missing %s:\n%s", want, text) + } + } +} + +func TestTCPServerPreservesResolvedEnvelopeWhenIdentitySideEffectFails(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{} + wantErr := errors.New("registration upsert failed") + server, err := NewTCPServer(TCPServerConfig{ + Protocol: TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: jt808.ExtractFrames, + Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) { + return envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "13307795425", + SourceEndpoint: sourceEndpoint, + ReceivedAtMS: receivedAtMS, + EventTimeMS: receivedAtMS, + ParsedFields: map[string]any{ + "jt808.location.total_mileage_km": "10241.2", + }, + ParseStatus: envelope.ParseOK, + }, nil + }, + }, + Sink: sink, + Resolver: resolvedErrorResolver{vin: "LNBVIN00000000001", err: wantErr}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + }) + if err != nil { + t.Fatalf("NewTCPServer() error = %v", err) + } + + server.handleFrame(context.Background(), nil, []byte{0x01}, "127.0.0.1:808", &connectionState{}) + + if len(sink.raw) != 1 { + t.Fatalf("raw count = %d, want 1", len(sink.raw)) + } + if sink.raw[0].VIN != "LNBVIN00000000001" { + t.Fatalf("raw vin = %q, want resolved vin", sink.raw[0].VIN) + } + if sink.raw[0].ParseStatus != envelope.ParsePartial { + t.Fatalf("parse status = %q, want PARTIAL", sink.raw[0].ParseStatus) + } + if len(sink.raw[0].Parsed) != 0 { + t.Fatalf("canonical raw should not duplicate parsed tree: %#v", sink.raw[0].Parsed) + } + if got := sink.raw[0].ParsedFields["jt808.location.total_mileage_km"]; got != "10241.2" { + t.Fatalf("protocol field lost after identity side-effect failure: %#v", got) + } + for field := range sink.raw[0].ParsedFields { + if strings.HasPrefix(field, "jt808.identity.") { + t.Fatalf("derived identity annotation leaked into protocol fields: %s", field) + } + } + if len(sink.fields) != 1 || sink.fields[0].VIN != "LNBVIN00000000001" { + t.Fatalf("fields should preserve resolved vin, fields=%#v", sink.fields) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_identity_total{protocol="JT808",status="error"} 1`, + `vehicle_gateway_identity_issues_total{message_id="0x0200",protocol="JT808",reason="resolver_error",status="error"} 1`, + `vehicle_gateway_frames_total{protocol="JT808",status="PARTIAL"} 1`, + `vehicle_gateway_fields_total{protocol="JT808",status="published"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metric missing %s:\n%s", want, text) + } + } +} + func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) { registry := metrics.NewRegistry() server, err := NewTCPServer(TCPServerConfig{ @@ -140,6 +475,45 @@ func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) { } } +func TestTCPServerClosesActiveConnectionWhenContextCancelled(t *testing.T) { + registry := metrics.NewRegistry() + server, err := NewTCPServer(TCPServerConfig{ + Protocol: TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: jt808.ExtractFrames, + Parse: jt808.ParseFrame, + }, + Sink: &recordingSink{}, + Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)), + Metrics: registry, + IdleTimeout: time.Minute, + }) + if err != nil { + t.Fatalf("NewTCPServer() error = %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + client, srv := net.Pipe() + done := make(chan struct{}) + go func() { + server.handleConnection(ctx, srv) + close(done) + }() + waitForMetric(t, registry, `vehicle_gateway_active_connections{protocol="JT808"} 1`) + + cancel() + server.closeActiveConnections() + defer client.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("connection handler did not exit after context cancellation") + } + if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_closes_total{protocol="JT808",reason="context_cancelled"} 1`) { + t.Fatalf("context cancellation close metric missing:\n%s", text) + } +} + func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) { registry := metrics.NewRegistry() server, err := NewTCPServer(TCPServerConfig{ @@ -188,6 +562,57 @@ func TestTCPServerRecordsConnectionCloseReasonMetric(t *testing.T) { } } +func TestTCPServerConnectionLifecycleLogsStayDebug(t *testing.T) { + source, err := os.ReadFile("tcp_server.go") + if err != nil { + t.Fatal(err) + } + text := string(source) + for _, forbidden := range []string{ + `Info("tcp connection opened"`, + `Info("tcp connection closed"`, + `Warn("tcp connection idle timeout"`, + `Warn("tcp connection closed by peer"`, + } { + if strings.Contains(text, forbidden) { + t.Fatalf("high-volume connection lifecycle log should not be info/warn: %s", forbidden) + } + } + for _, want := range []string{ + `Debug("tcp connection opened"`, + `Debug("tcp connection closed"`, + `Debug("tcp connection idle timeout"`, + `Debug("tcp connection closed by peer"`, + } { + if !strings.Contains(text, want) { + t.Fatalf("connection lifecycle log should remain debug: %s", want) + } + } +} + +func TestIsRoutineTCPReadCloseClassifiesRemoteCloseNoise(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "eof", err: io.EOF, want: true}, + {name: "net closed", err: net.ErrClosed, want: true}, + {name: "reset wrapped", err: fmt.Errorf("read tcp: %w", syscall.ECONNRESET), want: true}, + {name: "pipe wrapped", err: fmt.Errorf("write tcp: %w", syscall.EPIPE), want: true}, + {name: "reset text", err: errors.New("read tcp 172.17.111.55:808->117.132.196.176:22187: read: connection reset by peer"), want: true}, + {name: "unexpected", err: errors.New("checksum parser exploded"), want: false}, + {name: "nil", err: nil, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isRoutineTCPReadClose(tt.err); got != tt.want { + t.Fatalf("isRoutineTCPReadClose(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) { good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil) good[len(good)-1] ^= 0xff @@ -248,14 +673,14 @@ func TestTCPServerAnnotatesUnresolvedIdentity(t *testing.T) { if len(sink.raw) != 1 { t.Fatalf("raw count = %d", len(sink.raw)) } - identity, ok := sink.raw[0].Parsed["identity"].(map[string]any) - if !ok || identity["resolved"] != false || identity["reason"] != "no_binding" { - t.Fatalf("identity metadata = %#v", sink.raw[0].Parsed["identity"]) + if len(sink.raw[0].Parsed) != 0 || len(sink.raw[0].ParsedFields) != 0 { + t.Fatalf("unresolved derived metadata must not enter canonical protocol payload: parsed=%#v fields=%#v", sink.raw[0].Parsed, sink.raw[0].ParsedFields) } } func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) { frame := buildGBFrame(0x07, 0xfe, "LNBSCB3D4R1234567", nil) + registry := metrics.NewRegistry() sink := &recordingSink{} server := newTestServer(t, TCPProtocol{ Protocol: envelope.ProtocolGB32960, @@ -269,6 +694,7 @@ func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) { return []byte("ACK"), true, nil }, }, sink) + server.metrics = registry client, done := runPipe(t, server) if _, err := client.Write(frame); err != nil { @@ -283,6 +709,79 @@ func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) { } _ = client.Close() <-done + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_response_total{message_id="0x07",protocol="GB32960",status="ok"} 1`, + `vehicle_gateway_last_response_unix_seconds{message_id="0x07",protocol="GB32960",status="ok"} `, + `vehicle_gateway_response_duration_ms_histogram_count{message_id="0x07",protocol="GB32960",status="ok"} 1`, + `vehicle_gateway_response_e2e_recent_p99_ms{protocol="GB32960"} `, + `vehicle_gateway_response_e2e_recent_samples{protocol="GB32960"} `, + } { + if !strings.Contains(text, want) { + t.Fatalf("response metric missing %s:\n%s", want, text) + } + } +} + +func TestTCPServerWritesProtocolResponseWhenFieldsPublishFails(t *testing.T) { + registry := metrics.NewRegistry() + sink := &recordingSink{fieldsErr: errors.New("fields queue full")} + server := newTestServer(t, TCPProtocol{ + Protocol: envelope.ProtocolJT808, + Addr: ":0", + Extract: func(raw []byte) ([][]byte, []byte, error) { + return [][]byte{raw}, nil, nil + }, + Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) { + return envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "LNBVIN00000000001", + Phone: "13307795425", + SourceEndpoint: sourceEndpoint, + ReceivedAtMS: receivedAtMS, + EventTimeMS: receivedAtMS, + ParsedFields: map[string]any{ + "jt808.location.total_mileage_km": "10241.2", + }, + ParseStatus: envelope.ParseOK, + }, nil + }, + Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) { + if len(sink.raw) != 1 || sink.raw[0].EventID != env.EventID { + t.Fatalf("response built before raw publish: raw=%d", len(sink.raw)) + } + return []byte("ACK"), true, nil + }, + }, sink) + server.metrics = registry + + client, done := runPipe(t, server) + if _, err := client.Write([]byte{0x01}); err != nil { + t.Fatalf("client.Write() error = %v", err) + } + buf := make([]byte, 3) + if _, err := io.ReadFull(client, buf); err != nil { + t.Fatalf("read response error = %v", err) + } + if string(buf) != "ACK" { + t.Fatalf("response = %q", string(buf)) + } + _ = client.Close() + <-done + if len(sink.fields) != 1 { + t.Fatalf("fields publish attempts = %d, want 1", len(sink.fields)) + } + text := registry.Render() + for _, want := range []string{ + `vehicle_gateway_response_total{message_id="0x0200",protocol="JT808",status="ok"} 1`, + `vehicle_gateway_fields_total{protocol="JT808",status="publish_error"} 1`, + `vehicle_gateway_publish_total{kind="fields",protocol="JT808",status="error"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("metric missing %s:\n%s", want, text) + } + } } func TestTCPServerUsesUncancelledFrameContextForAlreadyReadFrame(t *testing.T) { @@ -368,6 +867,49 @@ func (r *contextCheckingResolver) Resolve(ctx context.Context, env envelope.Fram return env, r.ctxErr } +type vinResolver struct { + vin string +} + +func (r vinResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + env.VIN = r.vin + return env, nil +} + +type countingResolver struct { + calls int + vin string +} + +func (r *countingResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + r.calls++ + env.VIN = r.vin + return env, nil +} + +type errorResolver struct { + err error +} + +func (r errorResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + return env, r.err +} + +type resolvedErrorResolver struct { + vin string + err error +} + +func (r resolvedErrorResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + env.VIN = r.vin + if env.Parsed == nil { + env.Parsed = map[string]any{} + } + env.Parsed["identity"] = map[string]any{"resolved": true, "source": "test"} + env.EventID = env.StableEventID() + return env, r.err +} + type contextCheckingSink struct { rawCtxErr error unifiedCtxErr error @@ -426,25 +968,40 @@ func runPipe(t *testing.T, server *TCPServer) (net.Conn, <-chan struct{}) { return client, done } +func waitForMetric(t *testing.T, registry *metrics.Registry, want string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if strings.Contains(registry.Render(), want) { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("metric missing %s:\n%s", want, registry.Render()) +} + type recordingSink struct { - raw []envelope.FrameEnvelope - unified []envelope.FrameEnvelope - fields []envelope.FrameEnvelope + raw []envelope.FrameEnvelope + unified []envelope.FrameEnvelope + fields []envelope.FrameEnvelope + rawErr error + unifiedErr error + fieldsErr error } func (s *recordingSink) PublishRaw(_ context.Context, env envelope.FrameEnvelope) error { s.raw = append(s.raw, env) - return nil + return s.rawErr } func (s *recordingSink) PublishUnified(_ context.Context, env envelope.FrameEnvelope) error { s.unified = append(s.unified, env) - return nil + return s.unifiedErr } func (s *recordingSink) PublishFields(_ context.Context, env envelope.FrameEnvelope) error { s.fields = append(s.fields, env) - return nil + return s.fieldsErr } func (s *recordingSink) Close() error { @@ -475,3 +1032,9 @@ func buildGBFrame(command byte, response byte, vin string, body []byte) []byte { } return append(frame, bcc) } + +func fixedASCIIBytes(value string, size int) []byte { + out := make([]byte, size) + copy(out, []byte(value)) + return out +} diff --git a/go/vehicle-gateway/internal/health/health.go b/go/vehicle-gateway/internal/health/health.go index 0d25476f..796dd86c 100644 --- a/go/vehicle-gateway/internal/health/health.go +++ b/go/vehicle-gateway/internal/health/health.go @@ -40,6 +40,7 @@ func NewMux(service string, checks []Check, registry *metrics.Registry) *http.Se mux.Handle("/healthz", handler) mux.Handle("/readyz", handler) if registry != nil { + metrics.RegisterServiceInfo(registry, handler.service) mux.Handle("/metrics", metrics.NewHandler(registry)) } return mux diff --git a/go/vehicle-gateway/internal/health/health_test.go b/go/vehicle-gateway/internal/health/health_test.go index 2764cc24..bc666c2d 100644 --- a/go/vehicle-gateway/internal/health/health_test.go +++ b/go/vehicle-gateway/internal/health/health_test.go @@ -80,6 +80,13 @@ func TestNewMuxRegistersHealthAndReadinessRoutes(t *testing.T) { t.Fatalf("%s status = %d body=%s", path, response.Code, response.Body.String()) } } + + request := httptest.NewRequest(http.MethodGet, "/metrics", nil) + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + if !strings.Contains(response.Body.String(), `vehicle_service_info{service="vehicle-stat-writer"} 1`) { + t.Fatalf("service info metric missing: %s", response.Body.String()) + } } func TestNewServerReturnsNilWhenAddressIsEmpty(t *testing.T) { diff --git a/go/vehicle-gateway/internal/history/query.go b/go/vehicle-gateway/internal/history/query.go index 2084b663..e72a8b33 100644 --- a/go/vehicle-gateway/internal/history/query.go +++ b/go/vehicle-gateway/internal/history/query.go @@ -76,6 +76,7 @@ type LocationRow struct { Latitude float64 `json:"latitude"` AltitudeM *float64 `json:"altitude_m,omitempty"` SpeedKMH *float64 `json:"speed_kmh,omitempty"` + SOCPercent *float64 `json:"soc_percent,omitempty"` DirectionDeg *int64 `json:"direction_deg,omitempty"` AlarmFlag *int64 `json:"alarm_flag,omitempty"` StatusFlag *int64 `json:"status_flag,omitempty"` @@ -218,6 +219,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([] var receivedAt scanDateTime var altitude sql.NullFloat64 var speed sql.NullFloat64 + var soc sql.NullFloat64 var direction sql.NullInt64 var alarm sql.NullInt64 var status sql.NullInt64 @@ -230,6 +232,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([] &row.Latitude, &altitude, &speed, + &soc, &direction, &alarm, &status, @@ -243,6 +246,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([] row.ReceivedAt = receivedAt.String row.AltitudeM = nullableFloat(altitude) row.SpeedKMH = nullableFloat(speed) + row.SOCPercent = nullableFloat(soc) row.DirectionDeg = nullableInt(direction) row.AlarmFlag = nullableInt(alarm) row.StatusFlag = nullableInt(status) @@ -563,7 +567,7 @@ func quotedList(values []string) string { func buildLocationSQL(table string, query LocationQuery) (string, []any) { where := locationWhere(query) - sqlText := `SELECT ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vin FROM ` + table + sqlText := `SELECT ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vin FROM ` + table if len(where) > 0 { sqlText += " WHERE " + strings.Join(where, " AND ") } @@ -954,11 +958,11 @@ func normalizeDateTimeLiteral(value string) string { shanghai := time.FixedZone("Asia/Shanghai", 8*3600) for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} { if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil { - return parsed.UTC().Format("2006-01-02 15:04:05") + return parsed.In(shanghai).Format("2006-01-02 15:04:05") } } if parsed, err := time.Parse(time.RFC3339, value); err == nil { - return parsed.UTC().Format("2006-01-02 15:04:05") + return parsed.In(shanghai).Format("2006-01-02 15:04:05") } return value } diff --git a/go/vehicle-gateway/internal/history/query_test.go b/go/vehicle-gateway/internal/history/query_test.go index 3c827144..7afeb414 100644 --- a/go/vehicle-gateway/internal/history/query_test.go +++ b/go/vehicle-gateway/internal/history/query_test.go @@ -391,10 +391,10 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) { mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'"). WillReturnRows(sqlmock.NewRows([]string{ "ts", "event_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh", - "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin", + "soc_percent", "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin", }).AddRow( "2026-07-02 00:18:22", "event-3", "2026-07-02 00:22:43", - 121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8, + 121.07764, 30.585928, 11.0, 8.0, 82.5, 171, 0, 4718595, 8792.8, "JT808", "LKLG7C4E3NA774736", )) @@ -408,7 +408,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } body := response.Body.String() - for _, want := range []string{`"vin":"LKLG7C4E3NA774736"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} { + for _, want := range []string{`"vin":"LKLG7C4E3NA774736"`, `"longitude":121.07764`, `"soc_percent":82.5`, `"total_mileage_km":8792.8`, `"total":17`} { if !strings.Contains(body, want) { t.Fatalf("response missing %s: %s", want, body) } @@ -432,10 +432,10 @@ func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) { mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'"). WillReturnRows(sqlmock.NewRows([]string{ "ts", "event_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh", - "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin", + "soc_percent", "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin", }).AddRow( "2026-07-02 00:18:22", "event-3", "2026-07-02 00:22:43", - 121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8, + 121.07764, 30.585928, 11.0, 8.0, 82.5, 171, 0, 4718595, 8792.8, "JT808", "LKLG7C4E3NA774736", )) @@ -476,17 +476,17 @@ func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) { if err != nil { t.Fatalf("parseRawFrameQuery() error = %v", err) } - if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" { + if query.DateFrom != "2026-07-01 00:00:00" || query.DateTo != "2026-07-02 00:00:00" { t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo) } } -func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) { +func TestNormalizeDateTimeLiteralUsesAsiaShanghaiQueryTime(t *testing.T) { for raw, want := range map[string]string{ - "2026-07-01T00:00:00": "2026-06-30 16:00:00", - "2026-07-01 00:00:00": "2026-06-30 16:00:00", - "2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00", - "2026-06-30T16:00:00Z": "2026-06-30 16:00:00", + "2026-07-01T00:00:00": "2026-07-01 00:00:00", + "2026-07-01 00:00:00": "2026-07-01 00:00:00", + "2026-07-01T00:00:00+08:00": "2026-07-01 00:00:00", + "2026-06-30T16:00:00Z": "2026-07-01 00:00:00", } { if got := normalizeDateTimeLiteral(raw); got != want { t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want) @@ -522,7 +522,7 @@ func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) { for _, want := range []string{ "protocol = 'JT808'", "vin = 'LKLG7C4E3NA774736'", - "ts >= '2026-07-01 16:00:00'", + "ts >= '2026-07-02 00:00:00'", "LIMIT 20 OFFSET 5", } { if !strings.Contains(sqlText, want) { @@ -556,8 +556,8 @@ func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) { "vehicle_key = 'JT808:013307811350'", "vin = 'VIN''1'", "message_id = 512", - "ts >= '2026-06-30 16:00:00'", - "ts <= '2026-07-01 15:59:59'", + "ts >= '2026-07-01 00:00:00'", + "ts <= '2026-07-01 23:59:59'", "LIMIT 20 OFFSET 5", } { if !strings.Contains(sqlText, want) { diff --git a/go/vehicle-gateway/internal/history/schema.go b/go/vehicle-gateway/internal/history/schema.go index 01a43d40..8d5abc92 100644 --- a/go/vehicle-gateway/internal/history/schema.go +++ b/go/vehicle-gateway/internal/history/schema.go @@ -54,6 +54,7 @@ func SchemaStatements(database string) []string { latitude DOUBLE, altitude_m DOUBLE, speed_kmh DOUBLE, + soc_percent DOUBLE, direction_deg INT, alarm_flag BIGINT, status_flag BIGINT, @@ -64,3 +65,13 @@ func SchemaStatements(database string) []string { )`, } } + +// SchemaMigrationStatements contains additive TDengine changes that must also +// be applied to an already existing stable. The writer treats duplicate-column +// errors as success so startup remains idempotent across releases. +func SchemaMigrationStatements(database string) []string { + if database == "" { + database = DefaultDatabase + } + return []string{"ALTER STABLE " + database + ".vehicle_locations ADD COLUMN soc_percent DOUBLE"} +} diff --git a/go/vehicle-gateway/internal/history/writer.go b/go/vehicle-gateway/internal/history/writer.go index cf192e44..cee2312d 100644 --- a/go/vehicle-gateway/internal/history/writer.go +++ b/go/vehicle-gateway/internal/history/writer.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "sort" "strconv" "strings" "sync" @@ -15,6 +16,7 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry" ) type Execer interface { @@ -27,9 +29,23 @@ type Writer struct { cache tableCache } +type AppendResult struct { + RawRows int + LocationRows int + LocationError error +} + +const ( + LocationStatusOK = "ok" + LocationStatusSkippedNonRealtime = "skipped_non_realtime" + LocationStatusSkippedMissingVIN = "skipped_missing_vin" + LocationStatusSkippedMissingCoordinates = "skipped_missing_coordinates" +) + const ( rawFramePayloadInlineLimit = 12_000 rawFramePayloadChunkSize = 16_000 + tdengineInsertSoftLimit = 6 * 1024 * 1024 ) type payloadChunk struct { @@ -100,9 +116,22 @@ func (w *Writer) EnsureSchema(ctx context.Context, database string) error { return err } } + for _, statement := range SchemaMigrationStatements(database) { + if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateTDengineColumnError(err) { + return err + } + } return nil } +func isDuplicateTDengineColumnError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "duplicate column") || strings.Contains(message, "duplicated column") || strings.Contains(message, "column already exists") +} + func (w *Writer) qualify(table string) string { table = normalizeIdentifier(table) if w.database == "" { @@ -112,20 +141,52 @@ func (w *Writer) qualify(table string) string { } func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error { - if err := w.AppendRawFrame(ctx, env); err != nil { + result, err := w.AppendAllWithResult(ctx, env) + if err != nil { return err } - return w.AppendLocation(ctx, env) + return result.LocationError +} + +func (w *Writer) AppendAllWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) { + var result AppendResult + if err := w.AppendRawFrame(ctx, env); err != nil { + return result, err + } + result.RawRows = 1 + rows, err := w.appendLocationWithCount(ctx, env) + if err != nil { + result.LocationError = err + return result, nil + } + result.LocationRows = rows + return result, nil } func (w *Writer) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error { - if len(envelopes) == 0 { - return nil - } - if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil { + result, err := w.AppendAllBatchWithResult(ctx, envelopes) + if err != nil { return err } - return w.AppendLocationBatch(ctx, envelopes) + return result.LocationError +} + +func (w *Writer) AppendAllBatchWithResult(ctx context.Context, envelopes []envelope.FrameEnvelope) (AppendResult, error) { + var result AppendResult + if len(envelopes) == 0 { + return result, nil + } + if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil { + return result, err + } + result.RawRows = len(envelopes) + rows, err := w.appendLocationBatchWithCount(ctx, envelopes) + if err != nil { + result.LocationError = err + return result, nil + } + result.LocationRows = rows + return result, nil } func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error { @@ -165,7 +226,6 @@ VALUES (%s)`, w.qualify(chunkTable), joinLiterals(chunkValues(env, chunk)))); er func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error { rowsByTable := map[string][]string{} chunkRowsByTable := map[string][]string{} - chunkEnvByTable := map[string]envelope.FrameEnvelope{} for _, env := range envelopes { table := tableName("raw", env) if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil { @@ -184,87 +244,141 @@ func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.F if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil { return err } - chunkEnvByTable[chunkTable] = env for _, chunk := range chunks { chunkRowsByTable[chunkTable] = append(chunkRowsByTable[chunkTable], "("+joinLiterals(chunkValues(env, chunk))+")") } } - for table, rows := range rowsByTable { - if len(rows) == 0 { - continue - } - if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s - (ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, - raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint) -VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil { - return err - } + if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, + raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint`); err != nil { + return err } - for table, rows := range chunkRowsByTable { - if len(rows) == 0 { - continue - } - _ = chunkEnvByTable[table] - if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s - (ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text) -VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil { - return err - } + if err := w.execMultiTableInsert(ctx, chunkRowsByTable, `ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text`); err != nil { + return err } return nil } func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error { - if strings.TrimSpace(env.VIN) == "" { - return nil - } - longitude, okLon := floatField(env, envelope.FieldLongitude) - latitude, okLat := floatField(env, envelope.FieldLatitude) - if !okLon || !okLat { - return nil - } - table := locationTableName(env) - if err := w.ensureLocationChild(ctx, table, env); err != nil { - return err - } - _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s - (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, - direction_deg, alarm_flag, status_flag, total_mileage_km) -VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude)))) + _, err := w.appendLocationWithCount(ctx, env) return err } +func (w *Writer) appendLocationWithCount(ctx context.Context, env envelope.FrameEnvelope) (int, error) { + longitude, latitude, status := locationCandidate(env) + if status != LocationStatusOK { + return 0, nil + } + table := locationTableName(env) + if err := w.ensureLocationChild(ctx, table, env); err != nil { + return 0, err + } + _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s + (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, + soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km) +VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude)))) + if err != nil { + return 0, err + } + return 1, nil +} + func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error { + _, err := w.appendLocationBatchWithCount(ctx, envelopes) + return err +} + +func (w *Writer) appendLocationBatchWithCount(ctx context.Context, envelopes []envelope.FrameEnvelope) (int, error) { rowsByTable := map[string][]string{} + rowCount := 0 for _, env := range envelopes { - if strings.TrimSpace(env.VIN) == "" { - continue - } - longitude, okLon := floatField(env, envelope.FieldLongitude) - latitude, okLat := floatField(env, envelope.FieldLatitude) - if !okLon || !okLat { + longitude, latitude, status := locationCandidate(env) + if status != LocationStatusOK { continue } table := locationTableName(env) if err := w.ensureLocationChild(ctx, table, env); err != nil { - return err + return rowCount, err } rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(locationValues(env, longitude, latitude))+")") + rowCount++ } - for table, rows := range rowsByTable { - if len(rows) == 0 { - continue - } - if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s - (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, - direction_deg, alarm_flag, status_flag, total_mileage_km) -VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil { + if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, + soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km`); err != nil { + return rowCount, err + } + return rowCount, nil +} + +func (w *Writer) execMultiTableInsert(ctx context.Context, rowsByTable map[string][]string, columns string) error { + statements := buildMultiTableInsertStatements(rowsByTable, w.qualify, columns, tdengineInsertSoftLimit) + for _, statement := range statements { + if _, err := w.exec.ExecContext(ctx, statement); err != nil { return err } } return nil } +func buildMultiTableInsertStatements(rowsByTable map[string][]string, qualify func(string) string, columns string, softLimit int) []string { + if len(rowsByTable) == 0 { + return nil + } + keys := make([]string, 0, len(rowsByTable)) + for table, rows := range rowsByTable { + if len(rows) > 0 { + keys = append(keys, table) + } + } + sort.Strings(keys) + if len(keys) == 0 { + return nil + } + if qualify == nil { + qualify = func(table string) string { return table } + } + var statements []string + current := "INSERT INTO " + parts := 0 + for _, table := range keys { + part := fmt.Sprintf(`%s + (%s) +VALUES %s`, qualify(table), columns, strings.Join(rowsByTable[table], ",")) + if parts > 0 && softLimit > 0 && len(current)+1+len(part) > softLimit { + statements = append(statements, current) + current = "INSERT INTO " + parts = 0 + } + if parts > 0 { + current += " " + } + current += part + parts++ + } + if parts > 0 { + statements = append(statements, current) + } + return statements +} + +func LocationStatus(env envelope.FrameEnvelope) string { + _, _, status := locationCandidate(env) + return status +} + +func locationCandidate(env envelope.FrameEnvelope) (float64, float64, string) { + if !envelope.IsRealtimeTelemetryFrame(env) { + return 0, 0, LocationStatusSkippedNonRealtime + } + if strings.TrimSpace(env.VIN) == "" { + return 0, 0, LocationStatusSkippedMissingVIN + } + location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields) + if !ok { + return 0, 0, LocationStatusSkippedMissingCoordinates + } + return location.Longitude, location.Latitude, LocationStatusOK +} + func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error { key := stable + "." + table return w.cache.doOnce(key, func() error { @@ -325,7 +439,7 @@ func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsed func chunkValues(env envelope.FrameEnvelope, chunk payloadChunk) []any { received := millis(env.ReceivedAtMS) return []any{ - received, + received.Add(time.Duration(chunk.Index) * time.Millisecond), env.StableEventID(), frameID(env), received, @@ -379,18 +493,21 @@ func safeChunkEnd(value string, start int, maxBytes int) int { func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any { received := millis(env.ReceivedAtMS) + location, _ := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields) + totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields) return []any{ eventTimeOrReceived(env), env.StableEventID(), received, longitude, latitude, - floatFieldOrNil(env, "altitude_m"), - floatFieldOrNil(env, envelope.FieldSpeedKMH), - intFieldOrNil(env, "direction_deg"), - intFieldOrNil(env, "alarm_flag"), - intFieldOrNil(env, "status_flag"), - floatFieldOrNil(env, envelope.FieldTotalMileageKM), + optionalFloat(location.AltitudeM), + optionalFloat(location.SpeedKMH), + optionalFloat(location.SOCPercent), + optionalInt(location.DirectionDeg), + optionalInt64(location.AlarmFlag), + optionalInt64(location.StatusFlag), + optionalPositiveFloat(totalMileageKM, hasTotalMileage), } } @@ -463,76 +580,37 @@ func parsedFieldsJSONString(env envelope.FrameEnvelope) string { return jsonString(fields) } -func floatField(env envelope.FrameEnvelope, key string) (float64, bool) { - if env.Fields == nil { - return 0, false - } - value, ok := env.Fields[key] - if !ok || value == nil { - return 0, false - } - switch typed := value.(type) { - case float64: - return typed, true - case float32: - return float64(typed), true - case int: - return float64(typed), true - case int64: - return float64(typed), true - case uint16: - return float64(typed), true - case uint32: - return float64(typed), true - case string: - parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64) - return parsed, err == nil - default: - return 0, false +func optionalFloat(value *float64) any { + if value == nil { + return nil } + return *value } -func floatFieldOrNil(env envelope.FrameEnvelope, key string) any { - value, ok := floatField(env, key) - if !ok { +func optionalInt(value *float64) any { + if value == nil { + return nil + } + return int64(*value) +} + +func optionalInt64(value *int64) any { + if value == nil { + return nil + } + return *value +} + +func optionalPositiveFloat(value float64, ok bool) any { + if !ok || value <= 0 { return nil } return value } -func intFieldOrNil(env envelope.FrameEnvelope, key string) any { - if env.Fields == nil { - return nil - } - value, ok := env.Fields[key] - if !ok || value == nil { - return nil - } - switch typed := value.(type) { - case int: - return typed - case int64: - return typed - case uint16: - return int64(typed) - case uint32: - return int64(typed) - case float64: - return int64(typed) - case string: - parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64) - if err == nil { - return parsed - } - } - return nil -} - func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time { - if env.EventTimeMS > 0 { - return millis(env.EventTimeMS) - } - return millis(env.ReceivedAtMS) + eventMS, _ := envelope.NormalizedEventTimeMS(env) + return millis(eventMS) } func millis(value int64) time.Time { @@ -543,7 +621,10 @@ func millis(value int64) time.Time { } func quote(value string) string { - return strings.ReplaceAll(value, "'", "''") + return strings.NewReplacer( + `\`, `\\`, + `'`, `''`, + ).Replace(value) } func joinLiterals(values []any) string { diff --git a/go/vehicle-gateway/internal/history/writer_test.go b/go/vehicle-gateway/internal/history/writer_test.go index 935d676f..6b949d37 100644 --- a/go/vehicle-gateway/internal/history/writer_test.go +++ b/go/vehicle-gateway/internal/history/writer_test.go @@ -3,6 +3,8 @@ package history import ( "context" "database/sql" + "errors" + "strconv" "strings" "sync" "testing" @@ -32,6 +34,18 @@ func TestSchemaStatementsCreateCoreStables(t *testing.T) { } } +func TestSchemaMigrationAddsTrackSOCIdempotently(t *testing.T) { + statements := strings.Join(SchemaMigrationStatements("test_ts"), "\n") + if !strings.Contains(statements, "ALTER STABLE test_ts.vehicle_locations ADD COLUMN soc_percent DOUBLE") { + t.Fatalf("SOC migration missing: %s", statements) + } + for _, message := range []string{"duplicate column name", "Duplicated column names", "column already exists"} { + if !isDuplicateTDengineColumnError(errors.New(message)) { + t.Fatalf("duplicate TDengine column error should be idempotent: %s", message) + } + } +} + func TestWriterAppendsRawAndLocationOnly(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec) @@ -130,6 +144,16 @@ func TestWriterChunksOversizedParsedFields(t *testing.T) { if got := countSQL(exec.calls, "INSERT INTO chunk_"); got < 2 { t.Fatalf("chunk insert count = %d, calls=%v", got, exec.calls) } + chunkInserts := matchingSQL(exec.calls, "INSERT INTO chunk_") + if len(chunkInserts) != 2 { + t.Fatalf("chunk inserts = %d, calls=%v", len(chunkInserts), exec.calls) + } + if !strings.Contains(chunkInserts[0], "VALUES (1782745114999, '") { + t.Fatalf("first chunk ts should use received_at: %s", chunkInserts[0]) + } + if !strings.Contains(chunkInserts[1], "VALUES (1782745115000, '") { + t.Fatalf("second chunk ts should be offset by chunk_index ms: %s", chunkInserts[1]) + } } func TestWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T) { @@ -185,11 +209,20 @@ func TestTimeLiteralsUseEpochMilliseconds(t *testing.T) { } } +func TestStringLiteralPreservesJSONEscapesForTDengine(t *testing.T) { + value := `{"bits":"{\"abs\":false}"}` + want := `'{"bits":"{\\"abs\\":false}"}'` + + if got := literal(value); got != want { + t.Fatalf("literal() = %q, want %q", got, want) + } +} + func TestWriterSkipsSparseDerivedRows(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec) env := sampleEnvelope() - env.Fields = map[string]any{} + env.ParsedFields = map[string]any{} if err := writer.AppendAll(context.Background(), env); err != nil { t.Fatalf("AppendAll() error = %v", err) @@ -227,6 +260,86 @@ func TestWriterSkipsLocationWhenVINIsMissing(t *testing.T) { } } +func TestLocationStatusClassifiesDerivedLocationEligibility(t *testing.T) { + realtimeWithoutCoordinates := sampleEnvelope() + realtimeWithoutCoordinates.ParsedFields = map[string]any{ + "jt808.location.speed_kmh": 30, + } + bareFieldsOnly := realtimeWithoutCoordinates + bareFieldsOnly.ParsedFields = nil + bareFieldsOnly.Fields = map[string]any{ + envelope.FieldLatitude: 30.590151, + envelope.FieldLongitude: 121.069881, + } + withoutVIN := sampleEnvelope() + withoutVIN.VIN = "" + nonRealtimeWithCoordinates := sampleEnvelope() + nonRealtimeWithCoordinates.MessageID = "0x0100" + + tests := []struct { + name string + env envelope.FrameEnvelope + want string + }{ + {name: "ok", env: sampleEnvelope(), want: LocationStatusOK}, + {name: "non realtime", env: nonRealtimeWithCoordinates, want: LocationStatusSkippedNonRealtime}, + {name: "missing vin", env: withoutVIN, want: LocationStatusSkippedMissingVIN}, + {name: "missing coordinates", env: realtimeWithoutCoordinates, want: LocationStatusSkippedMissingCoordinates}, + {name: "bare fields are not canonical", env: bareFieldsOnly, want: LocationStatusSkippedMissingCoordinates}, + } + for _, test := range tests { + if got := LocationStatus(test.env); got != test.want { + t.Fatalf("%s LocationStatus() = %q, want %q", test.name, got, test.want) + } + } +} + +func TestWriterSkipsLocationForNonRealtimeFrame(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec) + env := sampleEnvelope() + env.MessageID = "0x0100" + env.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}} + + if err := writer.AppendAll(context.Background(), env); err != nil { + t.Fatalf("AppendAll() error = %v", err) + } + + if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { + t.Fatalf("raw insert count = %d", got) + } + if got := countSQL(exec.calls, "USING vehicle_locations"); got != 0 { + t.Fatalf("location child create count = %d", got) + } + if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 { + t.Fatalf("location insert count = %d", got) + } +} + +func TestWriterAppendAllWithResultKeepsRawSuccessWhenLocationFails(t *testing.T) { + locationErr := errors.New("location insert failed") + exec := &recordingExec{errs: []error{nil, nil, nil, nil, locationErr}} + writer := NewWriter(exec) + env := sampleEnvelope() + + result, err := writer.AppendAllWithResult(context.Background(), env) + if err != nil { + t.Fatalf("AppendAllWithResult() raw error = %v", err) + } + if !errors.Is(result.LocationError, locationErr) { + t.Fatalf("location error = %v, want %v", result.LocationError, locationErr) + } + if result.RawRows != 1 || result.LocationRows != 0 { + t.Fatalf("result = %+v, want raw row retained and no location rows", result) + } + if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { + t.Fatalf("raw insert count = %d", got) + } + if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 { + t.Fatalf("location insert attempted count = %d", got) + } +} + func TestWriterAppendsBatchRowsByChildTable(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec) @@ -262,6 +375,94 @@ func TestWriterAppendsBatchRowsByChildTable(t *testing.T) { } } +func TestWriterAppendsBatchAcrossChildTablesWithSingleMultiTableInsert(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec) + first := sampleEnvelope() + second := sampleEnvelope() + second.Sequence = 2 + second.EventID = "second-event" + second.VIN = "LNBVIN00000000002" + second.Phone = "013307795426" + second.EventTimeMS += 1000 + second.ReceivedAtMS += 1000 + + if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil { + t.Fatalf("AppendAllBatch() error = %v", err) + } + + if got := countSQL(exec.calls, "USING raw_frames"); got != 2 { + t.Fatalf("raw child create count = %d", got) + } + if got := countSQL(exec.calls, "USING vehicle_locations"); got != 2 { + t.Fatalf("location child create count = %d", got) + } + if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { + t.Fatalf("raw multi-table insert count = %d", got) + } + if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 { + t.Fatalf("location multi-table insert count = %d", got) + } + rawInsert := findSQL(exec.calls, "INSERT INTO raw_") + if got := strings.Count(rawInsert, "\nVALUES "); got != 2 { + t.Fatalf("raw multi-table VALUES sections = %d, sql=%s", got, rawInsert) + } + locationInsert := findSQL(exec.calls, "INSERT INTO loc_") + if got := strings.Count(locationInsert, "\nVALUES "); got != 2 { + t.Fatalf("location multi-table VALUES sections = %d, sql=%s", got, locationInsert) + } +} + +func TestWriterAppendAllBatchSkipsLocationForNonRealtimeFrames(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec) + first := sampleEnvelope() + first.MessageID = "0x0100" + first.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}} + second := sampleEnvelope() + second.Sequence = 2 + second.EventTimeMS += 1000 + second.ReceivedAtMS += 1000 + + if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil { + t.Fatalf("AppendAllBatch() error = %v", err) + } + + if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { + t.Fatalf("raw batch insert count = %d", got) + } + locationInsert := findSQL(exec.calls, "INSERT INTO loc_") + if got := strings.Count(locationInsert, "),(") + 1; got != 1 { + t.Fatalf("location batch row count = %d, sql=%s", got, locationInsert) + } +} + +func TestWriterNormalizesFarFutureEventTimeForLocationButKeepsRawEvidence(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec) + env := sampleEnvelope() + received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC) + futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC) + env.ReceivedAtMS = received.UnixMilli() + env.EventTimeMS = futureEvent.UnixMilli() + + if err := writer.AppendAll(context.Background(), env); err != nil { + t.Fatalf("AppendAll() error = %v", err) + } + + rawInsert := findSQL(exec.calls, "INSERT INTO raw_") + locationInsert := findSQL(exec.calls, "INSERT INTO loc_") + if !strings.Contains(rawInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) { + t.Fatalf("raw insert should keep original event time %d: %s", futureEvent.UnixMilli(), rawInsert) + } + if strings.Contains(locationInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) { + t.Fatalf("location insert should not use far future event time: %s", locationInsert) + } + if !strings.Contains(locationInsert, strconv.FormatInt(received.UnixMilli(), 10)) { + t.Fatalf("location insert should use received time %d: %s", received.UnixMilli(), locationInsert) + } +} + func TestWriterWithDatabaseQualifiesTDengineTables(t *testing.T) { exec := &recordingExec{} writer := NewWriterWithDatabase(exec, "vehicle_ts") @@ -372,14 +573,14 @@ func sampleEnvelope() envelope.FrameEnvelope { ReceivedAtMS: 1782745114999, RawHex: "7E0200", Parsed: map[string]any{"message": "location"}, - Fields: map[string]any{ - envelope.FieldLongitude: 121.069881, - envelope.FieldLatitude: 30.590151, - envelope.FieldSpeedKMH: 23.0, - envelope.FieldTotalMileageKM: 10241.2, - "direction_deg": uint16(79), - "alarm_flag": uint32(0), - "status_flag": uint32(72), + ParsedFields: map[string]any{ + "jt808.location.longitude": 121.069881, + "jt808.location.latitude": 30.590151, + "jt808.location.speed_kmh": 23.0, + "jt808.location.total_mileage_km": 10241.2, + "jt808.location.direction_deg": uint16(79), + "jt808.location.alarm_flag": uint32(0), + "jt808.location.status_flag": uint32(72), }, ParseStatus: envelope.ParseOK, } @@ -404,6 +605,16 @@ func findSQL(calls []execCall, pattern string) string { return "" } +func matchingSQL(calls []execCall, pattern string) []string { + out := []string{} + for _, call := range calls { + if strings.Contains(call.query, pattern) { + out = append(out, call.query) + } + } + return out +} + func containsSQL(calls []execCall, pattern string) bool { return findSQL(calls, pattern) != "" } @@ -428,10 +639,16 @@ type execCall struct { type recordingExec struct { calls []execCall + errs []error } func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) { e.calls = append(e.calls, execCall{query: query, args: args}) + if len(e.errs) > 0 { + err := e.errs[0] + e.errs = e.errs[1:] + return nil, err + } return nil, nil } diff --git a/go/vehicle-gateway/internal/identity/mapping_import.go b/go/vehicle-gateway/internal/identity/mapping_import.go new file mode 100644 index 00000000..318d2d01 --- /dev/null +++ b/go/vehicle-gateway/internal/identity/mapping_import.go @@ -0,0 +1,773 @@ +package identity + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/xuri/excelize/v2" +) + +const ( + IdentifierTypeJT808Phone = "JT808_PHONE" + IdentifierTypePlate = "PLATE" +) + +type MappingRecord struct { + File string `json:"file"` + Sheet string `json:"sheet"` + Row int `json:"row"` + SourceCode string `json:"source_code"` + SourceName string `json:"source_name"` + Protocol string `json:"protocol"` + IdentifierType string `json:"identifier_type"` + IdentifierValue string `json:"identifier_value"` + RawValue string `json:"raw_value,omitempty"` + Plate string `json:"plate,omitempty"` + OEM string `json:"oem,omitempty"` +} + +type MappingScanReport struct { + Files int `json:"files"` + Sheets int `json:"sheets"` + Rows int `json:"rows"` + Records int `json:"records"` + Skipped int `json:"skipped"` + UnsupportedFiles int `json:"unsupported_files,omitempty"` + Sources []MappingSourceScanReport `json:"sources,omitempty"` + UnsupportedItems []MappingUnsupportedFileReport `json:"unsupported_items,omitempty"` + Errors []string `json:"errors,omitempty"` +} + +type MappingSourceScanReport struct { + SourceCode string `json:"source_code"` + SourceName string `json:"source_name"` + Files int `json:"files"` + Sheets int `json:"sheets"` + Rows int `json:"rows"` + Records int `json:"records"` + PhoneRecords int `json:"phone_records"` + PlateRecords int `json:"plate_records"` + Skipped int `json:"skipped"` +} + +type MappingUnsupportedFileReport struct { + File string `json:"file"` + Ext string `json:"ext"` + Reason string `json:"reason"` +} + +type MappingImportOptions struct { + Apply bool + LegacyTable string + ReportItemLimit int +} + +type MappingImportReport struct { + Scan MappingScanReport `json:"scan"` + Records int `json:"records"` + Deduplicated int `json:"deduplicated"` + Resolved int `json:"resolved"` + Unresolved int `json:"unresolved"` + Conflicts int `json:"conflicts"` + WouldInsert int `json:"would_insert,omitempty"` + WouldUpdate int `json:"would_update,omitempty"` + Inserted int `json:"inserted,omitempty"` + Updated int `json:"updated,omitempty"` + SourceResults []MappingSourceImportStat `json:"source_results,omitempty"` + UnresolvedItems []MappingRecord `json:"unresolved_items,omitempty"` + ConflictItems []MappingConflict `json:"conflict_items,omitempty"` +} + +type MappingSourceImportStat struct { + SourceCode string `json:"source_code"` + SourceName string `json:"source_name"` + Records int `json:"records"` + Deduplicated int `json:"deduplicated"` + Resolved int `json:"resolved"` + Unresolved int `json:"unresolved"` + Conflicts int `json:"conflicts"` + WouldInsert int `json:"would_insert,omitempty"` + WouldUpdate int `json:"would_update,omitempty"` + Inserted int `json:"inserted,omitempty"` + Updated int `json:"updated,omitempty"` +} + +type MappingConflict struct { + Record MappingRecord `json:"record"` + ExistingVIN string `json:"existing_vin,omitempty"` + NewVIN string `json:"new_vin,omitempty"` + Reason string `json:"reason"` +} + +type resolvedMappingRecord struct { + MappingRecord + VIN string +} + +type mappingStore interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +var digitPattern = regexp.MustCompile(`\D+`) + +func EnsureVehicleIdentifierSchema(ctx context.Context, db *sql.DB) error { + if db == nil { + return errors.New("identity db must not be nil") + } + if _, err := db.ExecContext(ctx, vehicleTableSQL); err != nil { + return err + } + _, err := db.ExecContext(ctx, vehicleIdentifierTableSQL) + return err +} + +func ReadMappingDirectory(root string) ([]MappingRecord, MappingScanReport, error) { + root = strings.TrimSpace(root) + if root == "" { + return nil, MappingScanReport{}, errors.New("mapping input directory is empty") + } + var records []MappingRecord + report := MappingScanReport{} + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + report.Errors = append(report.Errors, err.Error()) + return nil + } + if entry == nil || entry.IsDir() { + return nil + } + name := entry.Name() + if strings.HasPrefix(name, "~$") || strings.HasPrefix(name, "._") { + return nil + } + ext := strings.ToLower(filepath.Ext(name)) + if !isSupportedMappingWorkbookExt(ext) { + if isUnsupportedMappingWorkbookExt(ext) { + report.UnsupportedFiles++ + report.UnsupportedItems = append(report.UnsupportedItems, MappingUnsupportedFileReport{ + File: path, + Ext: ext, + Reason: "convert legacy workbook to .xlsx before import", + }) + } + return nil + } + fileRecords, fileReport, err := readMappingWorkbook(root, path) + report.Files++ + report.Sheets += fileReport.Sheets + report.Rows += fileReport.Rows + report.Records += fileReport.Records + report.Skipped += fileReport.Skipped + report.Errors = append(report.Errors, fileReport.Errors...) + for _, source := range fileReport.Sources { + mergeMappingSourceScan(&report, source) + } + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("%s: %v", path, err)) + return nil + } + records = append(records, fileRecords...) + return nil + }) + if err != nil { + return records, report, err + } + sortMappingSourceScans(report.Sources) + return records, report, nil +} + +func readMappingWorkbook(root string, path string) ([]MappingRecord, MappingScanReport, error) { + workbook, err := excelize.OpenFile(path) + if err != nil { + return nil, MappingScanReport{}, err + } + defer func() { _ = workbook.Close() }() + + sourceCode, sourceName := mappingSource(root, path) + sourceReport := MappingSourceScanReport{ + SourceCode: sourceCode, + SourceName: sourceName, + Files: 1, + } + var records []MappingRecord + report := MappingScanReport{} + for _, sheet := range workbook.GetSheetList() { + rows, err := workbook.GetRows(sheet) + if err != nil { + report.Errors = append(report.Errors, fmt.Sprintf("%s/%s: %v", path, sheet, err)) + continue + } + report.Sheets++ + sourceReport.Sheets++ + report.Rows += len(rows) + sourceReport.Rows += len(rows) + if len(rows) == 0 { + continue + } + header, dataStart := mappingHeader(rows) + for rowIndex := dataStart; rowIndex < len(rows); rowIndex++ { + row := rows[rowIndex] + plate := normalizePlate(cellByHeader(row, header, "plate")) + rawPhone := cellByHeader(row, header, "phone") + phone := normalizeMappingPhone(rawPhone) + if len(header) == 0 { + plate = normalizePlate(cell(row, 0)) + rawPhone = cell(row, 1) + phone = normalizeMappingPhone(rawPhone) + } + if plate == "" && phone == "" { + report.Skipped++ + sourceReport.Skipped++ + continue + } + if phone != "" { + records = append(records, MappingRecord{ + File: path, + Sheet: sheet, + Row: rowIndex + 1, + SourceCode: sourceCode, + SourceName: sourceName, + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: phone, + RawValue: strings.TrimSpace(rawPhone), + Plate: plate, + OEM: sourceName, + }) + sourceReport.PhoneRecords++ + sourceReport.Records++ + } + if plate != "" { + records = append(records, MappingRecord{ + File: path, + Sheet: sheet, + Row: rowIndex + 1, + SourceCode: sourceCode, + SourceName: sourceName, + Protocol: "JT808", + IdentifierType: IdentifierTypePlate, + IdentifierValue: plate, + RawValue: plate, + Plate: plate, + OEM: sourceName, + }) + sourceReport.PlateRecords++ + sourceReport.Records++ + } + } + } + report.Records = len(records) + report.Sources = []MappingSourceScanReport{sourceReport} + return records, report, nil +} + +func mergeMappingSourceScan(report *MappingScanReport, source MappingSourceScanReport) { + if report == nil || strings.TrimSpace(source.SourceCode) == "" { + return + } + for index := range report.Sources { + if report.Sources[index].SourceCode != source.SourceCode { + continue + } + report.Sources[index].Files += source.Files + report.Sources[index].Sheets += source.Sheets + report.Sources[index].Rows += source.Rows + report.Sources[index].Records += source.Records + report.Sources[index].PhoneRecords += source.PhoneRecords + report.Sources[index].PlateRecords += source.PlateRecords + report.Sources[index].Skipped += source.Skipped + if report.Sources[index].SourceName == "" { + report.Sources[index].SourceName = source.SourceName + } + return + } + report.Sources = append(report.Sources, source) +} + +func sortMappingSourceScans(sources []MappingSourceScanReport) { + sort.SliceStable(sources, func(i, j int) bool { + return sources[i].SourceCode < sources[j].SourceCode + }) +} + +func isSupportedMappingWorkbookExt(ext string) bool { + switch strings.ToLower(strings.TrimSpace(ext)) { + case ".xlsx", ".xlsm": + return true + default: + return false + } +} + +func isUnsupportedMappingWorkbookExt(ext string) bool { + switch strings.ToLower(strings.TrimSpace(ext)) { + case ".xls", ".xlsb": + return true + default: + return false + } +} + +func ImportMappingRecords(ctx context.Context, db *sql.DB, records []MappingRecord, scan MappingScanReport, opts MappingImportOptions) (MappingImportReport, error) { + if db == nil { + return MappingImportReport{}, errors.New("identity db must not be nil") + } + legacyTable := strings.TrimSpace(opts.LegacyTable) + if legacyTable == "" || !safeIdentifier(legacyTable) { + legacyTable = "vehicle_identity_binding" + } + report := MappingImportReport{ + Scan: scan, + Records: len(records), + } + sourceStats := map[string]*MappingSourceImportStat{} + for _, record := range records { + sourceImportStat(sourceStats, record).Records++ + } + reportLimit := opts.ReportItemLimit + if reportLimit == 0 { + reportLimit = 50 + } + deduped, conflicts := dedupeMappingRecords(records) + report.Deduplicated = len(deduped) + report.Conflicts += len(conflicts) + for _, conflict := range conflicts { + sourceImportStat(sourceStats, conflict.Record).Conflicts++ + appendConflictItem(&report, conflict, reportLimit) + } + + store := mappingStore(db) + var tx *sql.Tx + if opts.Apply { + var err error + tx, err = db.BeginTx(ctx, nil) + if err != nil { + return report, err + } + store = tx + defer func() { + if tx != nil { + _ = tx.Rollback() + } + }() + } + + for _, record := range deduped { + sourceStat := sourceImportStat(sourceStats, record) + sourceStat.Deduplicated++ + vin, err := resolveMappingVIN(ctx, store, legacyTable, record) + if err != nil { + return report, err + } + if vin == "" { + report.Unresolved++ + sourceStat.Unresolved++ + appendUnresolvedItem(&report, record, reportLimit) + continue + } + resolved := resolvedMappingRecord{MappingRecord: record, VIN: vin} + existingVIN, exists, err := existingIdentifierVIN(ctx, store, resolved) + if err != nil { + return report, err + } + if exists && !strings.EqualFold(existingVIN, vin) { + report.Conflicts++ + sourceStat.Conflicts++ + appendConflictItem(&report, MappingConflict{ + Record: record, + ExistingVIN: existingVIN, + NewVIN: vin, + Reason: "identifier already points to another vin", + }, reportLimit) + continue + } + report.Resolved++ + sourceStat.Resolved++ + if exists { + if opts.Apply { + if err := updateVehicleIdentifier(ctx, store, resolved); err != nil { + return report, err + } + report.Updated++ + sourceStat.Updated++ + } else { + report.WouldUpdate++ + sourceStat.WouldUpdate++ + } + continue + } + if opts.Apply { + if err := upsertVehicle(ctx, store, resolved); err != nil { + return report, err + } + if err := insertVehicleIdentifier(ctx, store, resolved); err != nil { + return report, err + } + report.Inserted++ + sourceStat.Inserted++ + } else { + report.WouldInsert++ + sourceStat.WouldInsert++ + } + } + if tx != nil { + if err := tx.Commit(); err != nil { + return report, err + } + tx = nil + } + report.SourceResults = sortedMappingSourceImportStats(sourceStats) + return report, nil +} + +func sourceImportStat(stats map[string]*MappingSourceImportStat, record MappingRecord) *MappingSourceImportStat { + sourceCode := strings.TrimSpace(record.SourceCode) + if sourceCode == "" { + sourceCode = "unknown" + } + stat := stats[sourceCode] + if stat != nil { + if stat.SourceName == "" { + stat.SourceName = strings.TrimSpace(record.SourceName) + } + return stat + } + stat = &MappingSourceImportStat{ + SourceCode: sourceCode, + SourceName: strings.TrimSpace(record.SourceName), + } + stats[sourceCode] = stat + return stat +} + +func sortedMappingSourceImportStats(stats map[string]*MappingSourceImportStat) []MappingSourceImportStat { + if len(stats) == 0 { + return nil + } + keys := make([]string, 0, len(stats)) + for key := range stats { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]MappingSourceImportStat, 0, len(keys)) + for _, key := range keys { + out = append(out, *stats[key]) + } + return out +} + +func appendUnresolvedItem(report *MappingImportReport, record MappingRecord, limit int) { + if report == nil { + return + } + if limit >= 0 && len(report.UnresolvedItems) >= limit { + return + } + report.UnresolvedItems = append(report.UnresolvedItems, record) +} + +func appendConflictItem(report *MappingImportReport, conflict MappingConflict, limit int) { + if report == nil { + return + } + if limit >= 0 && len(report.ConflictItems) >= limit { + return + } + report.ConflictItems = append(report.ConflictItems, conflict) +} + +func dedupeMappingRecords(records []MappingRecord) ([]MappingRecord, []MappingConflict) { + seen := map[string]MappingRecord{} + indexByKey := map[string]int{} + var out []MappingRecord + var conflicts []MappingConflict + for _, record := range records { + record.IdentifierValue = normalizeIdentifierValue(record.IdentifierType, record.IdentifierValue) + record.Plate = normalizePlate(record.Plate) + if record.Protocol == "" { + record.Protocol = "JT808" + } + if record.IdentifierValue == "" || record.IdentifierType == "" { + continue + } + key := strings.Join([]string{record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue}, "\x00") + existing, ok := seen[key] + if !ok { + seen[key] = record + indexByKey[key] = len(out) + out = append(out, record) + continue + } + if existing.Plate != "" && record.Plate != "" && existing.Plate != record.Plate { + conflicts = append(conflicts, MappingConflict{ + Record: record, + Reason: fmt.Sprintf("same source identifier maps to multiple plates: %s/%s", existing.Plate, record.Plate), + }) + continue + } + merged := mergeMappingRecord(existing, record) + seen[key] = merged + if index, ok := indexByKey[key]; ok && index >= 0 && index < len(out) { + out[index] = merged + } + } + return out, conflicts +} + +func mergeMappingRecord(existing MappingRecord, incoming MappingRecord) MappingRecord { + merged := existing + if merged.File == "" { + merged.File = incoming.File + } + if merged.Sheet == "" { + merged.Sheet = incoming.Sheet + } + if merged.Row == 0 { + merged.Row = incoming.Row + } + if merged.SourceName == "" { + merged.SourceName = incoming.SourceName + } + if merged.Protocol == "" { + merged.Protocol = incoming.Protocol + } + if merged.IdentifierType == "" { + merged.IdentifierType = incoming.IdentifierType + } + if merged.IdentifierValue == "" { + merged.IdentifierValue = incoming.IdentifierValue + } + if merged.RawValue == "" { + merged.RawValue = incoming.RawValue + } + if merged.Plate == "" { + merged.Plate = incoming.Plate + } + if merged.OEM == "" { + merged.OEM = incoming.OEM + } + return merged +} + +func resolveMappingVIN(ctx context.Context, db mappingStore, legacyTable string, record MappingRecord) (string, error) { + if record.Plate != "" { + vin, err := lookupLegacyVIN(ctx, db, legacyTable, "plate", record.Plate) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return "", err + } + if vin != "" { + return vin, nil + } + } + if record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue != "" { + vin, err := lookupLegacyVIN(ctx, db, legacyTable, "phone", record.IdentifierValue) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return "", err + } + return vin, nil + } + return "", nil +} + +func lookupLegacyVIN(ctx context.Context, db mappingStore, table string, column string, value string) (string, error) { + if !safeIdentifier(table) || !safeIdentifier(column) { + return "", sql.ErrNoRows + } + query := "SELECT vin FROM " + table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> '' LIMIT 1" + var vin string + err := db.QueryRowContext(ctx, query, value).Scan(&vin) + if err != nil { + return "", err + } + return strings.TrimSpace(vin), nil +} + +func existingIdentifierVIN(ctx context.Context, db mappingStore, record resolvedMappingRecord) (string, bool, error) { + var vin string + err := db.QueryRowContext(ctx, `SELECT vin FROM vehicle_identifier +WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`, + record.Protocol, record.SourceCode, record.IdentifierType, record.IdentifierValue).Scan(&vin) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return strings.TrimSpace(vin), true, nil +} + +func upsertVehicle(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { + _, err := db.ExecContext(ctx, `INSERT INTO vehicle (vin, plate, oem, enabled) +VALUES (?, ?, ?, 1) +ON DUPLICATE KEY UPDATE + plate = IF(VALUES(plate) <> '', VALUES(plate), plate), + oem = IF(VALUES(oem) <> '', VALUES(oem), oem), + enabled = 1`, + record.VIN, record.Plate, record.OEM) + return err +} + +func insertVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { + _, err := db.ExecContext(ctx, `INSERT INTO vehicle_identifier + (protocol, source_code, identifier_type, identifier_value, vin, plate, oem, raw_value, enabled, latest_import_file) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`, + record.Protocol, + record.SourceCode, + record.IdentifierType, + record.IdentifierValue, + record.VIN, + record.Plate, + record.OEM, + record.RawValue, + record.File, + ) + return err +} + +func updateVehicleIdentifier(ctx context.Context, db mappingStore, record resolvedMappingRecord) error { + _, err := db.ExecContext(ctx, `UPDATE vehicle_identifier +SET plate = IF(? <> '', ?, plate), + oem = IF(? <> '', ?, oem), + raw_value = IF(? <> '', ?, raw_value), + latest_import_file = ?, + enabled = 1 +WHERE protocol = ? AND source_code = ? AND identifier_type = ? AND identifier_value = ?`, + record.Plate, record.Plate, + record.OEM, record.OEM, + record.RawValue, record.RawValue, + record.File, + record.Protocol, + record.SourceCode, + record.IdentifierType, + record.IdentifierValue, + ) + return err +} + +func mappingSource(root string, path string) (string, string) { + rel, err := filepath.Rel(root, path) + if err != nil { + rel = filepath.Base(path) + } + parts := strings.Split(filepath.ToSlash(rel), "/") + name := strings.TrimSpace(parts[0]) + if name == "" || strings.EqualFold(name, ".") { + name = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + } + code := sourceCode(name) + return code, name +} + +func sourceCode(name string) string { + switch strings.TrimSpace(strings.ToLower(name)) { + case "g7s": + return "g7s" + case "信达": + return "xinda" + case "广安北斗", "广安车联": + return "guangan_beidou" + case "东方北斗": + return "dongfang_beidou" + case "赛格": + return "saige" + default: + return normalizeASCIIKey(name) + } +} + +func mappingHeader(rows [][]string) (map[string]int, int) { + for index, row := range rows { + header := map[string]int{} + for columnIndex, value := range row { + key := normalizeHeader(value) + switch key { + case "车牌", "车牌号", "车牌号码": + header["plate"] = columnIndex + case "sim", "sim卡号", "手机号", "终端手机号", "终端id", "终端标识": + header["phone"] = columnIndex + } + } + if len(header) > 0 { + return header, index + 1 + } + if index >= 5 { + break + } + } + return nil, 0 +} + +func cellByHeader(row []string, header map[string]int, key string) string { + if len(header) == 0 { + return "" + } + index, ok := header[key] + if !ok { + return "" + } + return cell(row, index) +} + +func cell(row []string, index int) string { + if index < 0 || index >= len(row) { + return "" + } + return strings.TrimSpace(row[index]) +} + +func normalizeHeader(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + value = strings.ReplaceAll(value, " ", "") + value = strings.ReplaceAll(value, "\t", "") + value = strings.ReplaceAll(value, "（", "(") + value = strings.ReplaceAll(value, "）", ")") + return value +} + +func normalizePlate(value string) string { + value = strings.TrimSpace(value) + value = strings.ReplaceAll(value, " ", "") + value = strings.ReplaceAll(value, "\t", "") + return value +} + +func normalizeMappingPhone(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if strings.ContainsAny(value, ".eE") { + if parsed, err := strconv.ParseFloat(value, 64); err == nil && parsed > 0 { + return normalizePhone(strconv.FormatFloat(parsed, 'f', 0, 64)) + } + } + digits := digitPattern.ReplaceAllString(value, "") + return normalizePhone(digits) +} + +func normalizeASCIIKey(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + var b strings.Builder + lastUnderscore := false + for _, r := range value { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + lastUnderscore = false + continue + } + if !lastUnderscore { + b.WriteByte('_') + lastUnderscore = true + } + } + return strings.Trim(b.String(), "_") +} diff --git a/go/vehicle-gateway/internal/identity/mapping_import_test.go b/go/vehicle-gateway/internal/identity/mapping_import_test.go new file mode 100644 index 00000000..8035f20d --- /dev/null +++ b/go/vehicle-gateway/internal/identity/mapping_import_test.go @@ -0,0 +1,345 @@ +package identity + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/xuri/excelize/v2" +) + +func TestReadMappingDirectoryExtractsPhoneAndPlate(t *testing.T) { + dir := t.TempDir() + writeWorkbook(t, filepath.Join(dir, "G7s", "宇速全量.xlsx"), [][]string{ + {"车牌号", "sim卡号", "设备号"}, + {"粤AG18312", "013307795425", "DEV001"}, + }) + writeWorkbook(t, filepath.Join(dir, "东方北斗", "无标题0703.xlsx"), [][]string{ + {"沪A01559F", "64341233712"}, + }) + if err := os.MkdirAll(filepath.Join(dir, "信达"), 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "信达", "旧格式.xls"), []byte("legacy xls"), 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "信达", "说明.txt"), []byte("ignored"), 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + records, report, err := ReadMappingDirectory(dir) + if err != nil { + t.Fatalf("ReadMappingDirectory() error = %v", err) + } + if report.Files != 2 { + t.Fatalf("files = %d, report=%#v", report.Files, report) + } + if report.UnsupportedFiles != 1 || len(report.UnsupportedItems) != 1 { + t.Fatalf("unsupported files = %d, items=%#v", report.UnsupportedFiles, report.UnsupportedItems) + } + if got := report.UnsupportedItems[0]; got.Ext != ".xls" || got.Reason == "" { + t.Fatalf("unsupported item = %#v", got) + } + sources := map[string]MappingSourceScanReport{} + for _, source := range report.Sources { + sources[source.SourceCode] = source + } + if got := sources["g7s"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 { + t.Fatalf("g7s source report = %#v", got) + } + if got := sources["dongfang_beidou"]; got.Files != 1 || got.Sheets != 1 || got.PhoneRecords != 1 || got.PlateRecords != 1 || got.Records != 2 { + t.Fatalf("dongfang source report = %#v", got) + } + var phoneSeen bool + var plateSeen bool + var headerlessSeen bool + for _, record := range records { + if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "13307795425" && record.Plate == "粤AG18312" { + phoneSeen = true + } + if record.SourceCode == "g7s" && record.IdentifierType == IdentifierTypePlate && record.IdentifierValue == "粤AG18312" { + plateSeen = true + } + if record.SourceCode == "dongfang_beidou" && record.IdentifierType == IdentifierTypeJT808Phone && record.IdentifierValue == "64341233712" && record.Plate == "沪A01559F" { + headerlessSeen = true + } + } + if !phoneSeen || !plateSeen || !headerlessSeen { + t.Fatalf("records missing expected mappings: %#v", records) + } +} + +func TestImportMappingRecordsDryRunResolvesVINFromLegacyPlate(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18312"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230")) + mock.ExpectQuery("SELECT vin FROM vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + + report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{ + { + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "013307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + }, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"}) + if err != nil { + t.Fatalf("ImportMappingRecords() error = %v", err) + } + if report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 { + t.Fatalf("report = %#v", report) + } + if len(report.SourceResults) != 1 { + t.Fatalf("source results = %#v, want one source", report.SourceResults) + } + if got := report.SourceResults[0]; got.SourceCode != "g7s" || got.Records != 1 || got.Deduplicated != 1 || got.Resolved != 1 || got.WouldInsert != 1 { + t.Fatalf("source result = %#v", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestImportMappingRecordsReportsSourceResults(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18312"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230")) + mock.ExpectQuery("SELECT vin FROM vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤B00000"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). + WithArgs("14400000000"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + + report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{ + { + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + { + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + Plate: "粤B99999", + OEM: "G7s", + }, + { + SourceCode: "xinda", + SourceName: "信达", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "14400000000", + Plate: "粤B00000", + OEM: "信达", + }, + }, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"}) + if err != nil { + t.Fatalf("ImportMappingRecords() error = %v", err) + } + if report.Records != 3 || report.Deduplicated != 2 || report.Resolved != 1 || report.Unresolved != 1 || report.Conflicts != 1 || report.WouldInsert != 1 { + t.Fatalf("report = %#v", report) + } + got := map[string]MappingSourceImportStat{} + for _, source := range report.SourceResults { + got[source.SourceCode] = source + } + if source := got["g7s"]; source.Records != 2 || source.Deduplicated != 1 || source.Resolved != 1 || source.Conflicts != 1 || source.WouldInsert != 1 { + t.Fatalf("g7s source result = %#v", source) + } + if source := got["xinda"]; source.Records != 1 || source.Deduplicated != 1 || source.Unresolved != 1 { + t.Fatalf("xinda source result = %#v", source) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestImportMappingRecordsMergesDuplicateIdentifierDetails(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18312"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230")) + mock.ExpectQuery("SELECT vin FROM vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + + report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{ + { + File: "G7s/no-plate.xlsx", + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + OEM: "G7s", + }, + { + File: "G7s/with-plate.xlsx", + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + }, MappingScanReport{}, MappingImportOptions{LegacyTable: "vehicle_identity_binding"}) + if err != nil { + t.Fatalf("ImportMappingRecords() error = %v", err) + } + if report.Records != 2 || report.Deduplicated != 1 || report.Resolved != 1 || report.WouldInsert != 1 || report.Unresolved != 0 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestImportMappingRecordsApplyCommitsSingleTransaction(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectBegin() + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18312"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230")) + mock.ExpectQuery("SELECT vin FROM vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + mock.ExpectExec("INSERT INTO vehicle"). + WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("INSERT INTO vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425", "LB9A32A22P0LS1230", "粤AG18312", "G7s", "13307795425", "G7s/example.xlsx"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{ + { + File: "G7s/example.xlsx", + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + RawValue: "13307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + }, MappingScanReport{}, MappingImportOptions{ + Apply: true, + LegacyTable: "vehicle_identity_binding", + }) + if err != nil { + t.Fatalf("ImportMappingRecords() error = %v", err) + } + if report.Inserted != 1 || report.Resolved != 1 || report.WouldInsert != 0 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestImportMappingRecordsApplyRollsBackOnWriteError(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + errWrite := errors.New("insert vehicle failed") + mock.ExpectBegin() + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18312"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LB9A32A22P0LS1230")) + mock.ExpectQuery("SELECT vin FROM vehicle_identifier"). + WithArgs("JT808", "g7s", IdentifierTypeJT808Phone, "13307795425"). + WillReturnRows(sqlmock.NewRows([]string{"vin"})) + mock.ExpectExec("INSERT INTO vehicle"). + WithArgs("LB9A32A22P0LS1230", "粤AG18312", "G7s"). + WillReturnError(errWrite) + mock.ExpectRollback() + + report, err := ImportMappingRecords(context.Background(), db, []MappingRecord{ + { + SourceCode: "g7s", + SourceName: "G7s", + Protocol: "JT808", + IdentifierType: IdentifierTypeJT808Phone, + IdentifierValue: "13307795425", + RawValue: "13307795425", + Plate: "粤AG18312", + OEM: "G7s", + }, + }, MappingScanReport{}, MappingImportOptions{ + Apply: true, + LegacyTable: "vehicle_identity_binding", + }) + if !errors.Is(err, errWrite) { + t.Fatalf("ImportMappingRecords() error = %v, want %v", err, errWrite) + } + if report.Inserted != 0 || report.Resolved != 1 { + t.Fatalf("report = %#v", report) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestNormalizeMappingPhoneHandlesExcelNumericFormats(t *testing.T) { + tests := map[string]string{ + "013307795425": "13307795425", + "13307795425.0": "13307795425", + "1.3307795425E10": "13307795425", + } + for input, want := range tests { + if got := normalizeMappingPhone(input); got != want { + t.Fatalf("normalizeMappingPhone(%q) = %q, want %q", input, got, want) + } + } +} + +func writeWorkbook(t *testing.T, path string, rows [][]string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + workbook := excelize.NewFile() + sheet := "Sheet1" + for rowIndex, row := range rows { + for columnIndex, value := range row { + cellName, err := excelize.CoordinatesToCellName(columnIndex+1, rowIndex+1) + if err != nil { + t.Fatalf("CoordinatesToCellName() error = %v", err) + } + if err := workbook.SetCellValue(sheet, cellName, value); err != nil { + t.Fatalf("SetCellValue() error = %v", err) + } + } + } + if err := workbook.SaveAs(path); err != nil { + t.Fatalf("SaveAs() error = %v", err) + } + if err := workbook.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } +} diff --git a/go/vehicle-gateway/internal/identity/registration_writer.go b/go/vehicle-gateway/internal/identity/registration_writer.go new file mode 100644 index 00000000..d18ffe31 --- /dev/null +++ b/go/vehicle-gateway/internal/identity/registration_writer.go @@ -0,0 +1,357 @@ +package identity + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +const ( + JT808RegisterMessageID = "0x0100" + JT808AuthMessageID = "0x0102" + JT808LocationMessageID = "0x0200" +) + +// JT808RegistrationFact is the durable identity projection carried by a raw +// JT808 envelope. SeenAt uses gateway receive time so replay cannot move a +// terminal to a future date because its device clock was wrong. +type JT808RegistrationFact struct { + Phone string + DeviceID string + Plate string + VIN string + Province string + City string + Manufacturer string + DeviceType string + PlateColor string + AuthToken string + AuthIMEI string + AuthSoftwareVersion string + SourceEndpoint string + SourceIP string + FirstRegisteredAt *time.Time + LatestRegisteredAt *time.Time + LatestAuthenticated *time.Time + SeenAt time.Time +} + +// JT808RegistrationProjector throttles ordinary location touches in memory. +// Registration and authentication frames are never throttled. +type JT808RegistrationProjector struct { + location *time.Location + touchInterval time.Duration + retention time.Duration + + mu sync.Mutex + lastTouches map[string]time.Time + nextCleanup time.Time +} + +func NewJT808RegistrationProjector(location *time.Location, touchInterval time.Duration) *JT808RegistrationProjector { + if location == nil { + location = time.Local + } + if touchInterval <= 0 { + touchInterval = 10 * time.Minute + } + return &JT808RegistrationProjector{ + location: location, + touchInterval: touchInterval, + retention: 24 * time.Hour, + lastTouches: map[string]time.Time{}, + } +} + +// ProjectBatch returns at most one merged fact per phone. Call MarkPersisted +// only after the database transaction succeeds; otherwise replay must remain +// eligible immediately. +func (p *JT808RegistrationProjector) ProjectBatch(envelopes []envelope.FrameEnvelope) []JT808RegistrationFact { + if p == nil || len(envelopes) == 0 { + return nil + } + p.mu.Lock() + defer p.mu.Unlock() + + byPhone := make(map[string]JT808RegistrationFact) + order := make([]string, 0, len(envelopes)) + for _, env := range envelopes { + fact, ok := p.projectLocked(env) + if !ok { + continue + } + if current, exists := byPhone[fact.Phone]; exists { + byPhone[fact.Phone] = mergeJT808RegistrationFact(current, fact) + continue + } + byPhone[fact.Phone] = fact + order = append(order, fact.Phone) + } + result := make([]JT808RegistrationFact, 0, len(order)) + for _, phone := range order { + result = append(result, byPhone[phone]) + } + return result +} + +func (p *JT808RegistrationProjector) MarkPersisted(facts []JT808RegistrationFact) { + if p == nil || len(facts) == 0 { + return + } + p.mu.Lock() + defer p.mu.Unlock() + for _, fact := range facts { + phone := normalizePhone(fact.Phone) + if phone == "" || fact.SeenAt.IsZero() { + continue + } + if current := p.lastTouches[phone]; fact.SeenAt.After(current) { + p.lastTouches[phone] = fact.SeenAt + } + } + p.cleanupLocked(time.Now()) +} + +func (p *JT808RegistrationProjector) projectLocked(env envelope.FrameEnvelope) (JT808RegistrationFact, bool) { + if env.Protocol != envelope.ProtocolJT808 || env.ParseStatus == envelope.ParseBadFrame { + return JT808RegistrationFact{}, false + } + messageID := strings.TrimSpace(env.MessageID) + if messageID != JT808RegisterMessageID && messageID != JT808AuthMessageID && messageID != JT808LocationMessageID { + return JT808RegistrationFact{}, false + } + phone := normalizePhone(env.Phone) + if phone == "" { + return JT808RegistrationFact{}, false + } + seenAt := p.receivedAt(env) + if seenAt.IsZero() { + return JT808RegistrationFact{}, false + } + if messageID == JT808LocationMessageID { + if last := p.lastTouches[phone]; !last.IsZero() && seenAt.Before(last.Add(p.touchInterval)) { + return JT808RegistrationFact{}, false + } + } + + registration := mapValue(env.Parsed, "registration") + authentication := mapValue(env.Parsed, "authentication") + authenticationAccepted := messageID != JT808AuthMessageID || + !env.AuthenticationEnforced || env.AuthenticationStatus == "accepted" + vin := strings.TrimSpace(env.VIN) + if vin == "" { + vin = "unknown" + } + fact := JT808RegistrationFact{ + Phone: phone, + DeviceID: firstNonEmpty(env.DeviceID, textValue(registration, "device_id"), parsedFieldText(env.ParsedFields, "jt808.registration.device_id")), + Plate: firstNonEmpty(env.Plate, textValue(registration, "plate"), parsedFieldText(env.ParsedFields, "jt808.registration.plate")), + VIN: vin, + Province: firstNonEmpty(textValue(registration, "province"), parsedFieldText(env.ParsedFields, "jt808.registration.province")), + City: firstNonEmpty(textValue(registration, "city"), parsedFieldText(env.ParsedFields, "jt808.registration.city")), + Manufacturer: firstNonEmpty(textValue(registration, "manufacturer"), parsedFieldText(env.ParsedFields, "jt808.registration.manufacturer")), + DeviceType: firstNonEmpty(textValue(registration, "device_type"), parsedFieldText(env.ParsedFields, "jt808.registration.device_type")), + PlateColor: firstNonEmpty(textValue(registration, "plate_color"), parsedFieldText(env.ParsedFields, "jt808.registration.plate_color")), + AuthToken: firstNonEmpty(textValue(authentication, "token"), parsedFieldText(env.ParsedFields, "jt808.authentication.token")), + AuthIMEI: firstNonEmpty(textValue(authentication, "imei"), parsedFieldText(env.ParsedFields, "jt808.authentication.imei")), + AuthSoftwareVersion: firstNonEmpty(textValue(authentication, "software_version"), parsedFieldText(env.ParsedFields, "jt808.authentication.software_version")), + SourceEndpoint: strings.TrimSpace(env.SourceEndpoint), + SourceIP: normalizeEndpointIP(env.SourceEndpoint), + SeenAt: seenAt, + } + if !authenticationAccepted { + fact.AuthToken = "" + fact.AuthIMEI = "" + fact.AuthSoftwareVersion = "" + } + if messageID == JT808RegisterMessageID { + fact.FirstRegisteredAt = timePointer(seenAt) + fact.LatestRegisteredAt = timePointer(seenAt) + } + if messageID == JT808AuthMessageID && authenticationAccepted { + fact.LatestAuthenticated = timePointer(seenAt) + } + return fact, true +} + +func parsedFieldText(fields map[string]any, key string) string { + value, ok := fields[key] + if !ok || value == nil { + return "" + } + text := strings.TrimSpace(fmt.Sprint(value)) + if text == "" { + return "" + } + return text +} + +func (p *JT808RegistrationProjector) receivedAt(env envelope.FrameEnvelope) time.Time { + milliseconds := env.ReceivedAtMS + if milliseconds <= 0 { + milliseconds = env.EventTimeMS + } + if milliseconds <= 0 { + return time.Time{} + } + return time.UnixMilli(milliseconds).In(p.location).Truncate(time.Second) +} + +func (p *JT808RegistrationProjector) cleanupLocked(now time.Time) { + if !p.nextCleanup.IsZero() && now.Before(p.nextCleanup) { + return + } + p.nextCleanup = now.Add(time.Hour) + cutoff := now.Add(-p.retention) + for phone, touchedAt := range p.lastTouches { + if touchedAt.Before(cutoff) { + delete(p.lastTouches, phone) + } + } +} + +func mergeJT808RegistrationFact(current JT808RegistrationFact, candidate JT808RegistrationFact) JT808RegistrationFact { + if candidate.SeenAt.After(current.SeenAt) || candidate.SeenAt.Equal(current.SeenAt) { + current.DeviceID = firstNonEmpty(candidate.DeviceID, current.DeviceID) + current.Plate = firstNonEmpty(candidate.Plate, current.Plate) + current.VIN = preferKnownVIN(candidate.VIN, current.VIN) + current.Province = firstNonEmpty(candidate.Province, current.Province) + current.City = firstNonEmpty(candidate.City, current.City) + current.Manufacturer = firstNonEmpty(candidate.Manufacturer, current.Manufacturer) + current.DeviceType = firstNonEmpty(candidate.DeviceType, current.DeviceType) + current.PlateColor = firstNonEmpty(candidate.PlateColor, current.PlateColor) + current.AuthToken = firstNonEmpty(candidate.AuthToken, current.AuthToken) + current.AuthIMEI = firstNonEmpty(candidate.AuthIMEI, current.AuthIMEI) + current.AuthSoftwareVersion = firstNonEmpty(candidate.AuthSoftwareVersion, current.AuthSoftwareVersion) + current.SourceEndpoint = firstNonEmpty(candidate.SourceEndpoint, current.SourceEndpoint) + current.SourceIP = firstNonEmpty(candidate.SourceIP, current.SourceIP) + current.SeenAt = candidate.SeenAt + } else { + current.DeviceID = firstNonEmpty(current.DeviceID, candidate.DeviceID) + current.Plate = firstNonEmpty(current.Plate, candidate.Plate) + current.VIN = preferKnownVIN(current.VIN, candidate.VIN) + current.Province = firstNonEmpty(current.Province, candidate.Province) + current.City = firstNonEmpty(current.City, candidate.City) + current.Manufacturer = firstNonEmpty(current.Manufacturer, candidate.Manufacturer) + current.DeviceType = firstNonEmpty(current.DeviceType, candidate.DeviceType) + current.PlateColor = firstNonEmpty(current.PlateColor, candidate.PlateColor) + current.AuthToken = firstNonEmpty(current.AuthToken, candidate.AuthToken) + current.AuthIMEI = firstNonEmpty(current.AuthIMEI, candidate.AuthIMEI) + current.AuthSoftwareVersion = firstNonEmpty(current.AuthSoftwareVersion, candidate.AuthSoftwareVersion) + } + current.FirstRegisteredAt = earlierTimePointer(current.FirstRegisteredAt, candidate.FirstRegisteredAt) + current.LatestRegisteredAt = laterTimePointer(current.LatestRegisteredAt, candidate.LatestRegisteredAt) + current.LatestAuthenticated = laterTimePointer(current.LatestAuthenticated, candidate.LatestAuthenticated) + return current +} + +func timePointer(value time.Time) *time.Time { + copy := value + return © +} + +func earlierTimePointer(left *time.Time, right *time.Time) *time.Time { + if left == nil { + return right + } + if right == nil || left.Before(*right) || left.Equal(*right) { + return left + } + return right +} + +func laterTimePointer(left *time.Time, right *time.Time) *time.Time { + if left == nil { + return right + } + if right == nil || left.After(*right) || left.Equal(*right) { + return left + } + return right +} + +type JT808RegistrationStore struct { + db *sql.DB +} + +func NewJT808RegistrationStore(db *sql.DB) *JT808RegistrationStore { + if db == nil { + panic("jt808 registration db must not be nil") + } + return &JT808RegistrationStore{db: db} +} + +func EnsureJT808RegistrationSchema(ctx context.Context, db *sql.DB) error { + if db == nil { + return fmt.Errorf("jt808 registration db is nil") + } + if _, err := db.ExecContext(ctx, jt808RegistrationTableSQL); err != nil { + return err + } + for _, statement := range jt808RegistrationAlterSQL { + if _, err := db.ExecContext(ctx, statement); err != nil && !isIgnoredJT808RegistrationAlterError(err) { + return err + } + } + _, err := db.ExecContext(ctx, jt808RegistrationSourceIPBackfillSQL) + return err +} + +func (s *JT808RegistrationStore) UpsertBatch(ctx context.Context, facts []JT808RegistrationFact) error { + if s == nil || s.db == nil || len(facts) == 0 { + return nil + } + const columns = `phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color, + auth_token, auth_imei, auth_software_version, source_endpoint, source_ip, + first_registered_at, latest_registered_at, latest_authenticated_at, latest_seen_at` + values := make([]string, 0, len(facts)) + args := make([]any, 0, len(facts)*18) + for _, fact := range facts { + phone := normalizePhone(fact.Phone) + if phone == "" || fact.SeenAt.IsZero() { + continue + } + vin := strings.TrimSpace(fact.VIN) + if vin == "" { + vin = "unknown" + } + values = append(values, "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + args = append(args, + phone, strings.TrimSpace(fact.DeviceID), strings.TrimSpace(fact.Plate), vin, + strings.TrimSpace(fact.Province), strings.TrimSpace(fact.City), strings.TrimSpace(fact.Manufacturer), + strings.TrimSpace(fact.DeviceType), strings.TrimSpace(fact.PlateColor), strings.TrimSpace(fact.AuthToken), + strings.TrimSpace(fact.AuthIMEI), strings.TrimSpace(fact.AuthSoftwareVersion), + strings.TrimSpace(fact.SourceEndpoint), strings.TrimSpace(fact.SourceIP), + fact.FirstRegisteredAt, fact.LatestRegisteredAt, fact.LatestAuthenticated, fact.SeenAt, + ) + } + if len(values) == 0 { + return nil + } + query := `INSERT INTO jt808_registration (` + columns + `) VALUES ` + strings.Join(values, ",") + ` +ON DUPLICATE KEY UPDATE + device_id = IF(VALUES(device_id) <> '' AND (device_id IS NULL OR device_id = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(device_id), device_id), + plate = IF(VALUES(plate) <> '' AND (plate IS NULL OR plate = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(plate), plate), + vin = IF(VALUES(vin) <> '' AND VALUES(vin) <> 'unknown' AND (vin IS NULL OR vin = '' OR vin = 'unknown' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(vin), vin), + province = IF(VALUES(province) <> '' AND (province IS NULL OR province = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(province), province), + city = IF(VALUES(city) <> '' AND (city IS NULL OR city = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(city), city), + manufacturer = IF(VALUES(manufacturer) <> '' AND (manufacturer IS NULL OR manufacturer = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(manufacturer), manufacturer), + device_type = IF(VALUES(device_type) <> '' AND (device_type IS NULL OR device_type = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(device_type), device_type), + plate_color = IF(VALUES(plate_color) <> '' AND (plate_color IS NULL OR plate_color = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(plate_color), plate_color), + auth_token = IF(VALUES(auth_token) <> '' AND (auth_token IS NULL OR auth_token = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_token), auth_token), + auth_imei = IF(VALUES(auth_imei) <> '' AND (auth_imei IS NULL OR auth_imei = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_imei), auth_imei), + auth_software_version = IF(VALUES(auth_software_version) <> '' AND (auth_software_version IS NULL OR auth_software_version = '' OR latest_authenticated_at IS NULL OR VALUES(latest_authenticated_at) >= latest_authenticated_at), VALUES(auth_software_version), auth_software_version), + source_endpoint = IF(VALUES(source_endpoint) <> '' AND (source_endpoint IS NULL OR source_endpoint = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(source_endpoint), source_endpoint), + source_ip = IF(VALUES(source_ip) <> '' AND (source_ip IS NULL OR source_ip = '' OR VALUES(latest_seen_at) >= latest_seen_at), VALUES(source_ip), source_ip), + first_registered_at = CASE WHEN VALUES(first_registered_at) IS NULL THEN first_registered_at WHEN first_registered_at IS NULL THEN VALUES(first_registered_at) ELSE LEAST(first_registered_at, VALUES(first_registered_at)) END, + latest_registered_at = CASE WHEN VALUES(latest_registered_at) IS NULL THEN latest_registered_at WHEN latest_registered_at IS NULL THEN VALUES(latest_registered_at) ELSE GREATEST(latest_registered_at, VALUES(latest_registered_at)) END, + latest_authenticated_at = CASE WHEN VALUES(latest_authenticated_at) IS NULL THEN latest_authenticated_at WHEN latest_authenticated_at IS NULL THEN VALUES(latest_authenticated_at) ELSE GREATEST(latest_authenticated_at, VALUES(latest_authenticated_at)) END, + latest_seen_at = GREATEST(latest_seen_at, VALUES(latest_seen_at))` + _, err := s.db.ExecContext(ctx, query, args...) + return err +} diff --git a/go/vehicle-gateway/internal/identity/registration_writer_test.go b/go/vehicle-gateway/internal/identity/registration_writer_test.go new file mode 100644 index 00000000..012029e3 --- /dev/null +++ b/go/vehicle-gateway/internal/identity/registration_writer_test.go @@ -0,0 +1,221 @@ +package identity + +import ( + "context" + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestJT808RegistrationProjectorProjectsRegistrationUsingReceiveTime(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*60*60) + projector := NewJT808RegistrationProjector(loc, 10*time.Minute) + receivedAt := time.Date(2026, 7, 13, 17, 20, 30, 987000000, loc) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808RegisterMessageID, + Phone: "0013307795425", + VIN: "LTESTVIN000000001", + DeviceID: "DEV-1", + Plate: "粤A00001", + SourceEndpoint: "115.231.168.135:43625", + ReceivedAtMS: receivedAt.UnixMilli(), + EventTimeMS: receivedAt.Add(24 * time.Hour).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "jt808.registration.province": "44", + "jt808.registration.city": "1", + "jt808.registration.manufacturer": "YUTNG", + "jt808.registration.device_type": "TBOX-1", + "jt808.registration.plate_color": "2", + }, + } + + facts := projector.ProjectBatch([]envelope.FrameEnvelope{env}) + if len(facts) != 1 { + t.Fatalf("facts = %d, want 1", len(facts)) + } + fact := facts[0] + if fact.Phone != "13307795425" || fact.VIN != env.VIN || fact.Plate != env.Plate { + t.Fatalf("identity fact = %+v", fact) + } + wantTime := receivedAt.Truncate(time.Second) + if !fact.SeenAt.Equal(wantTime) || fact.FirstRegisteredAt == nil || !fact.FirstRegisteredAt.Equal(wantTime) { + t.Fatalf("fact times = seen %s first %#v, want %s", fact.SeenAt, fact.FirstRegisteredAt, wantTime) + } + if fact.SourceIP != "115.231.168.135" || fact.Manufacturer != "YUTNG" || fact.PlateColor != "2" { + t.Fatalf("registration details = %+v", fact) + } +} + +func TestJT808RegistrationProjectorThrottlesLocationOnlyAfterPersist(t *testing.T) { + loc := time.UTC + projector := NewJT808RegistrationProjector(loc, 10*time.Minute) + base := time.Date(2026, 7, 13, 8, 0, 0, 0, loc) + location := func(at time.Time) envelope.FrameEnvelope { + return envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808LocationMessageID, + Phone: "013307795425", + VIN: "LTESTVIN000000001", + ReceivedAtMS: at.UnixMilli(), + ParseStatus: envelope.ParseOK, + } + } + + first := projector.ProjectBatch([]envelope.FrameEnvelope{location(base)}) + if len(first) != 1 { + t.Fatalf("first facts = %d, want 1", len(first)) + } + // A failed database attempt must remain immediately replayable. + if replay := projector.ProjectBatch([]envelope.FrameEnvelope{location(base)}); len(replay) != 1 { + t.Fatalf("uncommitted replay facts = %d, want 1", len(replay)) + } + projector.MarkPersisted(first) + if throttled := projector.ProjectBatch([]envelope.FrameEnvelope{location(base.Add(9 * time.Minute))}); len(throttled) != 0 { + t.Fatalf("throttled facts = %d, want 0", len(throttled)) + } + if due := projector.ProjectBatch([]envelope.FrameEnvelope{location(base.Add(10 * time.Minute))}); len(due) != 1 { + t.Fatalf("due facts = %d, want 1", len(due)) + } +} + +func TestJT808RegistrationProjectorMergesRegisterAndAuthForPhone(t *testing.T) { + projector := NewJT808RegistrationProjector(time.UTC, 10*time.Minute) + base := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC) + register := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808RegisterMessageID, + Phone: "013307795425", + VIN: "LTESTVIN000000001", + Plate: "粤A00001", + ReceivedAtMS: base.UnixMilli(), + ParseStatus: envelope.ParseOK, + } + auth := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808AuthMessageID, + Phone: "13307795425", + ReceivedAtMS: base.Add(time.Second).UnixMilli(), + ParseStatus: envelope.ParseOK, + ParsedFields: map[string]any{ + "jt808.authentication.token": "g7gps", + "jt808.authentication.imei": "123456789012345", + }, + } + + facts := projector.ProjectBatch([]envelope.FrameEnvelope{register, auth}) + if len(facts) != 1 { + t.Fatalf("facts = %d, want 1", len(facts)) + } + fact := facts[0] + if fact.VIN != register.VIN || fact.Plate != register.Plate || fact.AuthToken != "g7gps" { + t.Fatalf("merged fact = %+v", fact) + } + if fact.FirstRegisteredAt == nil || fact.LatestRegisteredAt == nil || fact.LatestAuthenticated == nil { + t.Fatalf("merged timestamps missing: %+v", fact) + } +} + +func TestJT808RegistrationProjectorDoesNotTrustEnforcedRejectedAuth(t *testing.T) { + projector := NewJT808RegistrationProjector(time.UTC, 10*time.Minute) + seenAt := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC) + facts := projector.ProjectBatch([]envelope.FrameEnvelope{{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808AuthMessageID, + Phone: "13307795425", + ReceivedAtMS: seenAt.UnixMilli(), + ParseStatus: envelope.ParseOK, + AuthenticationEnforced: true, + AuthenticationStatus: "rejected", + Parsed: map[string]any{ + "authentication": map[string]any{ + "token": "untrusted-token", + "imei": "untrusted-imei", + "software_version": "untrusted-version", + }, + }, + }}) + if len(facts) != 1 { + t.Fatalf("facts = %d, want 1 audit touch", len(facts)) + } + fact := facts[0] + if fact.AuthToken != "" || fact.AuthIMEI != "" || fact.AuthSoftwareVersion != "" || fact.LatestAuthenticated != nil { + t.Fatalf("rejected credential leaked into registration fact: %+v", fact) + } +} + +func TestJT808RegistrationStoreUsesIdempotentEventTimeUpsert(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewJT808RegistrationStore(db) + seenAt := time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC) + mock.ExpectExec(regexp.QuoteMeta("INSERT INTO jt808_registration")). + WithArgs( + "13307795425", "DEV-1", "粤A00001", "LTESTVIN000000001", "", "", "YUTNG", "", "", + "g7gps", "", "", "115.231.168.135:43625", "115.231.168.135", + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), seenAt, + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + err = store.UpsertBatch(context.Background(), []JT808RegistrationFact{{ + Phone: "013307795425", + DeviceID: "DEV-1", + Plate: "粤A00001", + VIN: "LTESTVIN000000001", + Manufacturer: "YUTNG", + AuthToken: "g7gps", + SourceEndpoint: "115.231.168.135:43625", + SourceIP: "115.231.168.135", + FirstRegisteredAt: timePointer(seenAt), + LatestRegisteredAt: timePointer(seenAt), + LatestAuthenticated: timePointer(seenAt), + SeenAt: seenAt, + }}) + if err != nil { + t.Fatalf("UpsertBatch() error = %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestRegistrationUpsertProtectsLatestValuesFromOldReplay(t *testing.T) { + var query string + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherFunc(func(_ string, actual string) error { + query = actual + return nil + }))) + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewJT808RegistrationStore(db) + mock.ExpectExec("INSERT INTO jt808_registration").WillReturnResult(sqlmock.NewResult(1, 1)) + if err := store.UpsertBatch(context.Background(), []JT808RegistrationFact{{ + Phone: "13307795425", + VIN: "unknown", + SeenAt: time.Date(2026, 7, 13, 8, 0, 0, 0, time.UTC), + }}); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + for _, required := range []string{ + "VALUES(latest_seen_at) >= latest_seen_at", + "LEAST(first_registered_at, VALUES(first_registered_at))", + "GREATEST(latest_authenticated_at, VALUES(latest_authenticated_at))", + "latest_seen_at = GREATEST(latest_seen_at, VALUES(latest_seen_at))", + } { + if !regexp.MustCompile(regexp.QuoteMeta(required)).MatchString(query) { + t.Fatalf("upsert SQL missing %q", required) + } + } +} diff --git a/go/vehicle-gateway/internal/identity/resolver.go b/go/vehicle-gateway/internal/identity/resolver.go index 29113776..4433c2e9 100644 --- a/go/vehicle-gateway/internal/identity/resolver.go +++ b/go/vehicle-gateway/internal/identity/resolver.go @@ -7,6 +7,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" @@ -22,37 +23,156 @@ func (NoopResolver) Resolve(_ context.Context, env envelope.FrameEnvelope) (enve return env, nil } +type TimeoutResolver struct { + Delegate Resolver + Timeout time.Duration +} + +func (r TimeoutResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + if r.Delegate == nil { + return env, nil + } + if r.Timeout <= 0 { + return r.Delegate.Resolve(ctx, env) + } + resolveCtx, cancel := context.WithTimeout(ctx, r.Timeout) + defer cancel() + return r.Delegate.Resolve(resolveCtx, env) +} + type MySQLResolver struct { - db *sql.DB - table string - locationTouchInterval time.Duration - lookupCacheTTL time.Duration - lookupMu sync.Mutex - lookupCache map[string]lookupCacheEntry - registrationCache map[string]registrationCacheEntry - touchMu sync.Mutex - locationTouches map[string]time.Time + db *sql.DB + table string + snapshotOnlyLookups bool + snapshotRefreshMu sync.Mutex + snapshot atomic.Pointer[identitySnapshot] + locationTouchInterval time.Duration + locationTouchRetryInterval time.Duration + registrationWriteAttempts int + registrationWriteRetryDelay time.Duration + lookupCacheTTL time.Duration + staleLookupTTL time.Duration + cacheCleanupInterval time.Duration + maxCacheEntries int + sourceCodeLookup bool + lookupMu sync.Mutex + lookupCache map[string]lookupCacheEntry + registrationCache map[string]registrationCacheEntry + sourceCodeCache map[string]sourceCodeCacheEntry + nextLookupCleanup time.Time + touchMu sync.Mutex + locationTouches map[string]time.Time + locationTouchFailures map[string]time.Time + nextTouchCleanup time.Time + registrationWriteQueue chan jt808RegistrationWriteJob + registrationWriteClosed chan struct{} + registrationWriteDone chan struct{} + registrationWriteOnce sync.Once + registrationWriteWG sync.WaitGroup + registrationWriteTimeout time.Duration + registrationEnqueueTimeout time.Duration + disableRegistrationWrites bool + onRegistrationWriteError func(error) + onRegistrationWriteResult func(RegistrationWriteResult) } type MySQLResolverOptions struct { - LocationTouchInterval time.Duration - LookupCacheTTL time.Duration + SnapshotOnlyLookups bool + LocationTouchInterval time.Duration + LocationTouchRetryInterval time.Duration + RegistrationWriteAttempts int + RegistrationWriteRetryDelay time.Duration + LookupCacheTTL time.Duration + StaleLookupTTL time.Duration + CacheCleanupInterval time.Duration + MaxCacheEntries int + SourceCodeLookup bool + AsyncRegistrationWrites bool + RegistrationWriteQueueSize int + RegistrationWriteWorkers int + RegistrationWriteTimeout time.Duration + RegistrationEnqueueTimeout time.Duration + DisableRegistrationWrites bool + OnRegistrationWriteError func(error) + OnRegistrationWriteResult func(RegistrationWriteResult) } +type RegistrationWriteResult struct { + Mode string + Status string + Err error +} + +type CacheStats struct { + LookupEntries int + RegistrationEntries int + SourceCodeEntries int + LocationTouchEntries int + LocationTouchFailureEntries int + RegistrationWriteQueueDepth int + RegistrationWriteQueueCap int + SnapshotBindingEntries int + SnapshotIdentifierEntries int + SnapshotRegistrationEntries int + SnapshotSourceEntries int + SnapshotReady bool + SnapshotRefreshedAt time.Time + MaxEntries int +} + +type jt808RegistrationWriteJob struct { + query string + args []any + locationTouchPhone string + locationTouchAt time.Time +} + +var ErrRegistrationWriteQueueClosed = errors.New("jt808 registration write queue is closed") +var ErrRegistrationWriteQueueTimeout = errors.New("jt808 registration write queue enqueue timeout") + type lookupCacheEntry struct { - vin string - notFound bool - expiresAt time.Time + vin string + sourceCode string + platformName string + sourceKind string + notFound bool + expiresAt time.Time + staleAt time.Time } type registrationCacheEntry struct { vin string deviceID string plate string + authToken string notFound bool expiresAt time.Time + staleAt time.Time } +type sourceCodeCacheEntry struct { + sourceCode string + platformName string + sourceKind string + notFound bool + expiresAt time.Time + staleAt time.Time +} + +type sourceMetadata struct { + SourceCode string + PlatformName string + SourceKind string +} + +type vehicleIdentifierMatch struct { + VIN string + SourceCode string + PlatformName string +} + +var errAmbiguousIdentifier = errors.New("ambiguous vehicle identifier") + func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver { return NewMySQLResolverWithOptions(db, table, MySQLResolverOptions{}) } @@ -69,19 +189,145 @@ func NewMySQLResolverWithOptions(db *sql.DB, table string, opts MySQLResolverOpt if interval <= 0 { interval = 10 * time.Minute } + touchRetryInterval := opts.LocationTouchRetryInterval + if touchRetryInterval == 0 { + touchRetryInterval = 5 * time.Second + } + if touchRetryInterval < 0 { + touchRetryInterval = 0 + } + if touchRetryInterval > interval { + touchRetryInterval = interval + } + writeAttempts := opts.RegistrationWriteAttempts + if writeAttempts <= 0 { + writeAttempts = 2 + } + writeRetryDelay := opts.RegistrationWriteRetryDelay + if writeRetryDelay == 0 { + writeRetryDelay = 20 * time.Millisecond + } + if writeRetryDelay < 0 { + writeRetryDelay = 0 + } lookupTTL := opts.LookupCacheTTL if lookupTTL <= 0 { lookupTTL = 10 * time.Minute } - return &MySQLResolver{ - db: db, - table: table, - locationTouchInterval: interval, - lookupCacheTTL: lookupTTL, - lookupCache: map[string]lookupCacheEntry{}, - registrationCache: map[string]registrationCacheEntry{}, - locationTouches: map[string]time.Time{}, + staleLookupTTL := opts.StaleLookupTTL + if staleLookupTTL == 0 { + staleLookupTTL = time.Hour } + if staleLookupTTL < 0 { + staleLookupTTL = 0 + } + cleanupInterval := opts.CacheCleanupInterval + if cleanupInterval <= 0 { + cleanupInterval = time.Minute + } + maxEntries := opts.MaxCacheEntries + if maxEntries <= 0 { + maxEntries = 300000 + } + resolver := &MySQLResolver{ + db: db, + table: table, + snapshotOnlyLookups: opts.SnapshotOnlyLookups, + locationTouchInterval: interval, + locationTouchRetryInterval: touchRetryInterval, + registrationWriteAttempts: writeAttempts, + registrationWriteRetryDelay: writeRetryDelay, + lookupCacheTTL: lookupTTL, + staleLookupTTL: staleLookupTTL, + cacheCleanupInterval: cleanupInterval, + maxCacheEntries: maxEntries, + sourceCodeLookup: opts.SourceCodeLookup, + lookupCache: map[string]lookupCacheEntry{}, + registrationCache: map[string]registrationCacheEntry{}, + sourceCodeCache: map[string]sourceCodeCacheEntry{}, + locationTouches: map[string]time.Time{}, + locationTouchFailures: map[string]time.Time{}, + onRegistrationWriteError: opts.OnRegistrationWriteError, + onRegistrationWriteResult: opts.OnRegistrationWriteResult, + disableRegistrationWrites: opts.DisableRegistrationWrites, + } + if opts.AsyncRegistrationWrites && !opts.DisableRegistrationWrites { + queueSize := opts.RegistrationWriteQueueSize + if queueSize <= 0 { + queueSize = 100000 + } + workers := opts.RegistrationWriteWorkers + if workers <= 0 { + workers = 4 + } + timeout := opts.RegistrationWriteTimeout + if timeout <= 0 { + timeout = 5 * time.Second + } + enqueueTimeout := opts.RegistrationEnqueueTimeout + if enqueueTimeout == 0 { + enqueueTimeout = 50 * time.Millisecond + } + if enqueueTimeout < 0 { + enqueueTimeout = 0 + } + resolver.registrationWriteQueue = make(chan jt808RegistrationWriteJob, queueSize) + resolver.registrationWriteClosed = make(chan struct{}) + resolver.registrationWriteDone = make(chan struct{}) + resolver.registrationWriteTimeout = timeout + resolver.registrationEnqueueTimeout = enqueueTimeout + resolver.registrationWriteWG.Add(workers) + for i := 0; i < workers; i++ { + go resolver.registrationWriteWorker() + } + go func() { + resolver.registrationWriteWG.Wait() + close(resolver.registrationWriteDone) + }() + } + return resolver +} + +func (r *MySQLResolver) CacheStats() CacheStats { + if r == nil { + return CacheStats{} + } + r.lookupMu.Lock() + stats := CacheStats{ + LookupEntries: len(r.lookupCache), + RegistrationEntries: len(r.registrationCache), + SourceCodeEntries: len(r.sourceCodeCache), + MaxEntries: r.maxCacheEntries, + } + if snapshot := r.snapshot.Load(); snapshot != nil { + stats.SnapshotBindingEntries = len(snapshot.bindings) + stats.SnapshotIdentifierEntries = len(snapshot.identifiers) + stats.SnapshotRegistrationEntries = len(snapshot.registrations) + stats.SnapshotSourceEntries = len(snapshot.sources) + stats.SnapshotReady = true + stats.SnapshotRefreshedAt = snapshot.refreshedAt + } + r.lookupMu.Unlock() + r.touchMu.Lock() + stats.LocationTouchEntries = len(r.locationTouches) + stats.LocationTouchFailureEntries = len(r.locationTouchFailures) + r.touchMu.Unlock() + if r.registrationWriteQueue != nil { + stats.RegistrationWriteQueueDepth = len(r.registrationWriteQueue) + stats.RegistrationWriteQueueCap = cap(r.registrationWriteQueue) + } + return stats +} + +func (r *MySQLResolver) Close() error { + if r == nil || r.registrationWriteClosed == nil { + return nil + } + r.registrationWriteOnce.Do(func() { + close(r.registrationWriteClosed) + }) + <-r.registrationWriteDone + return nil } func (r *MySQLResolver) EnsureSchema(ctx context.Context) error { @@ -95,8 +341,13 @@ func (r *MySQLResolver) EnsureSchema(ctx context.Context) error { } } } - _, err := r.db.ExecContext(ctx, jt808RegistrationTableSQL) - return err + if _, err := r.db.ExecContext(ctx, vehicleTableSQL); err != nil { + return err + } + if _, err := r.db.ExecContext(ctx, vehicleIdentifierTableSQL); err != nil { + return err + } + return EnsureJT808RegistrationSchema(ctx, r.db) } func identityBindingTableSQL(table string) string { @@ -134,6 +385,34 @@ func isIgnoredIdentityBindingAlterError(err error) bool { strings.Contains(text, "error 1091") } +const vehicleTableSQL = `CREATE TABLE IF NOT EXISTS vehicle ( + vin VARCHAR(32) PRIMARY KEY, + plate VARCHAR(32) NULL, + oem VARCHAR(64) NULL, + enabled TINYINT(1) NOT NULL DEFAULT 1, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY idx_vehicle_plate (plate), + KEY idx_vehicle_enabled (enabled) +)` + +const vehicleIdentifierTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_identifier ( + protocol VARCHAR(32) NOT NULL, + source_code VARCHAR(64) NOT NULL DEFAULT '', + identifier_type VARCHAR(32) NOT NULL, + identifier_value VARCHAR(128) NOT NULL, + vin VARCHAR(32) NOT NULL, + plate VARCHAR(32) NULL, + oem VARCHAR(64) NULL, + raw_value VARCHAR(128) NULL, + enabled TINYINT(1) NOT NULL DEFAULT 1, + latest_import_file VARCHAR(255) NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (protocol, source_code, identifier_type, identifier_value), + KEY idx_identifier_lookup (protocol, identifier_type, identifier_value, enabled), + KEY idx_identifier_vin (vin), + KEY idx_identifier_plate (plate) +)` + const jt808RegistrationTableSQL = `CREATE TABLE IF NOT EXISTS jt808_registration ( phone VARCHAR(32) PRIMARY KEY, device_id VARCHAR(64) NULL, @@ -148,6 +427,7 @@ const jt808RegistrationTableSQL = `CREATE TABLE IF NOT EXISTS jt808_registration auth_imei VARCHAR(64) NULL, auth_software_version VARCHAR(64) NULL, source_endpoint VARCHAR(64) NULL, + source_ip VARCHAR(64) NULL, first_registered_at DATETIME NULL, latest_registered_at DATETIME NULL, latest_authenticated_at DATETIME NULL, @@ -156,9 +436,29 @@ const jt808RegistrationTableSQL = `CREATE TABLE IF NOT EXISTS jt808_registration KEY idx_jt808_registration_device (device_id), KEY idx_jt808_registration_plate (plate), KEY idx_jt808_registration_vin (vin), + KEY idx_jt808_registration_source_ip (source_ip), KEY idx_jt808_registration_seen (latest_seen_at) )` +var jt808RegistrationAlterSQL = []string{ + "ALTER TABLE jt808_registration ADD COLUMN source_ip VARCHAR(64) NULL AFTER source_endpoint", + "ALTER TABLE jt808_registration ADD KEY idx_jt808_registration_source_ip (source_ip)", +} + +const jt808RegistrationSourceIPBackfillSQL = `UPDATE jt808_registration +SET source_ip = SUBSTRING_INDEX(TRIM(source_endpoint), ':', 1) +WHERE (source_ip IS NULL OR TRIM(source_ip) = '') + AND source_endpoint IS NOT NULL + AND TRIM(source_endpoint) <> ''` + +func isIgnoredJT808RegistrationAlterError(err error) bool { + text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err))) + return strings.Contains(text, "duplicate column") || + strings.Contains(text, "duplicate key name") || + strings.Contains(text, "error 1060") || + strings.Contains(text, "error 1061") +} + func safeIdentifier(value string) bool { if value == "" { return false @@ -180,9 +480,31 @@ func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) } return env, nil } + source, err := r.sourceMetadataForEnvelope(ctx, env) + if err != nil { + return env, err + } + applyEnvelopeSourceMetadata(&env, source) candidates := CandidateKeys(env) for _, candidate := range candidates { - vin, err := r.lookup(ctx, candidate.Column, candidate.Value) + if env.Protocol == envelope.ProtocolJT808 { + vin, identitySource, cacheStatus, identitySourceMetadata, err := r.lookupVehicleIdentifier(ctx, env.Protocol, source.SourceCode, candidate) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) && !errors.Is(err, errAmbiguousIdentifier) { + return env, err + } + } else if strings.TrimSpace(vin) != "" { + env.VIN = strings.TrimSpace(vin) + overrideEnvelopeSourceMetadata(&env, identitySourceMetadata) + addEnvelopeIdentityMetadataWithCacheStatus(&env, identitySource, candidate.Value, cacheStatus) + env.EventID = env.StableEventID() + if err := r.trackJT808(ctx, env); err != nil { + return env, err + } + return env, nil + } + } + vin, cacheStatus, err := r.lookup(ctx, candidate.Column, candidate.Value) if err != nil { if errors.Is(err, sql.ErrNoRows) { continue @@ -191,14 +513,7 @@ func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) } if strings.TrimSpace(vin) != "" { env.VIN = strings.TrimSpace(vin) - if env.Parsed == nil { - env.Parsed = map[string]any{} - } - env.Parsed["identity"] = map[string]any{ - "resolved": true, - "source": candidate.Column, - "value": candidate.Value, - } + addEnvelopeIdentityMetadataWithCacheStatus(&env, candidate.Column, candidate.Value, cacheStatus) env.EventID = env.StableEventID() if err := r.trackJT808(ctx, env); err != nil { return env, err @@ -206,7 +521,7 @@ func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) return env, nil } } - resolved, ok, err := r.resolveFromJT808Registration(ctx, env) + resolved, ok, err := r.resolveFromJT808Registration(ctx, env, source.SourceCode) if err != nil { return env, err } @@ -223,11 +538,11 @@ func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) return env, nil } -func (r *MySQLResolver) resolveFromJT808Registration(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, bool, error) { +func (r *MySQLResolver) resolveFromJT808Registration(ctx context.Context, env envelope.FrameEnvelope, sourceCode string) (envelope.FrameEnvelope, bool, error) { if env.Protocol != envelope.ProtocolJT808 || env.MessageID != "0x0200" || strings.TrimSpace(env.Phone) == "" { return env, false, nil } - vin, deviceID, plate, err := r.lookupJT808Registration(ctx, env.Phone) + vin, deviceID, plate, cacheStatus, err := r.lookupJT808Registration(ctx, env.Phone) if err != nil { if errors.Is(err, sql.ErrNoRows) { return env, false, nil @@ -239,14 +554,26 @@ func (r *MySQLResolver) resolveFromJT808Registration(ctx context.Context, env en if strings.TrimSpace(vin) != "" && !strings.EqualFold(strings.TrimSpace(vin), "unknown") { env.VIN = strings.TrimSpace(vin) env.EventID = env.StableEventID() - addIdentityMetadata(env.Parsed, "jt808_registration.vin", env.Phone) + r.enrichResolvedJT808SourceMetadata(ctx, &env, env.VIN) + addEnvelopeIdentityMetadataWithCacheStatus(&env, "jt808_registration.vin", env.Phone, cacheStatus) return env, true, nil } value := strings.TrimSpace(env.Plate) if value == "" { return env, false, nil } - vin, err = r.lookup(ctx, "plate", value) + vin, source, cacheStatus, metadata, err := r.lookupVehicleIdentifier(ctx, env.Protocol, sourceCode, CandidateKey{Column: "plate", Value: value}) + if err == nil && strings.TrimSpace(vin) != "" { + env.VIN = strings.TrimSpace(vin) + overrideEnvelopeSourceMetadata(&env, metadata) + env.EventID = env.StableEventID() + addEnvelopeIdentityMetadataWithCacheStatus(&env, source, value, cacheStatus) + return env, true, nil + } + if err != nil && !errors.Is(err, sql.ErrNoRows) && !errors.Is(err, errAmbiguousIdentifier) { + return env, false, err + } + vin, cacheStatus, err = r.lookup(ctx, "plate", value) if err != nil { if errors.Is(err, sql.ErrNoRows) { return env, false, nil @@ -258,29 +585,64 @@ func (r *MySQLResolver) resolveFromJT808Registration(ctx context.Context, env en } env.VIN = strings.TrimSpace(vin) env.EventID = env.StableEventID() - addIdentityMetadata(env.Parsed, "jt808_registration.plate", value) + addEnvelopeIdentityMetadataWithCacheStatus(&env, "jt808_registration.plate", value, cacheStatus) return env, true, nil } -func (r *MySQLResolver) lookupJT808Registration(ctx context.Context, phone string) (vin string, deviceID string, plate string, err error) { +func (r *MySQLResolver) enrichResolvedJT808SourceMetadata(ctx context.Context, env *envelope.FrameEnvelope, vin string) { + if r == nil || env == nil || env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(vin) == "" { + return + } + for _, candidate := range CandidateKeys(*env) { + if candidateIdentifierType(candidate.Column) == "" { + continue + } + matchedVIN, _, _, metadata, err := r.lookupVehicleIdentifier(ctx, env.Protocol, "", candidate) + if err != nil || strings.TrimSpace(matchedVIN) == "" { + continue + } + if !strings.EqualFold(strings.TrimSpace(matchedVIN), strings.TrimSpace(vin)) { + continue + } + overrideEnvelopeSourceMetadata(env, metadata) + return + } +} + +func (r *MySQLResolver) lookupJT808Registration(ctx context.Context, phone string) (vin string, deviceID string, plate string, cacheStatus string, err error) { phone = normalizePhone(phone) now := time.Now() + var stale registrationCacheEntry + hasStale := false r.lookupMu.Lock() cached, ok := r.registrationCache[phone] if ok && now.Before(cached.expiresAt) { r.lookupMu.Unlock() if cached.notFound { - return "", "", "", sql.ErrNoRows + return "", "", "", "", sql.ErrNoRows } - return cached.vin, cached.deviceID, cached.plate, nil + return cached.vin, cached.deviceID, cached.plate, "", nil + } + if ok && cached.canServeStale(now) { + stale = cached + hasStale = true } r.lookupMu.Unlock() + if snapshot, ok := r.snapshotRegistration(phone); ok { + return snapshot.vin, snapshot.deviceID, snapshot.plate, "snapshot", nil + } + if r.snapshotOnlyLookups { + return "", "", "", "", sql.ErrNoRows + } var vinValue, deviceValue, plateValue sql.NullString err = r.db.QueryRowContext(ctx, "SELECT vin, device_id, plate FROM jt808_registration WHERE phone = ?", phone). Scan(&vinValue, &deviceValue, &plateValue) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return "", "", "", err + if hasStale { + return stale.vin, stale.deviceID, stale.plate, "stale", nil + } + return "", "", "", "", err } entry := registrationCacheEntry{ vin: strings.TrimSpace(vinValue.String), @@ -289,37 +651,349 @@ func (r *MySQLResolver) lookupJT808Registration(ctx context.Context, phone strin notFound: errors.Is(err, sql.ErrNoRows), expiresAt: now.Add(r.lookupCacheTTL), } - r.lookupMu.Lock() - r.registrationCache[phone] = entry - r.lookupMu.Unlock() + entry.staleAt = r.staleUntil(entry.expiresAt, entry.notFound) + r.cacheRegistration(phone, entry, now) if entry.notFound { - return "", "", "", sql.ErrNoRows + return "", "", "", "", sql.ErrNoRows } - return entry.vin, entry.deviceID, entry.plate, nil + return entry.vin, entry.deviceID, entry.plate, "", nil } func addIdentityMetadata(parsed map[string]any, source string, value string) { if parsed == nil { return } - parsed["identity"] = map[string]any{ - "resolved": true, - "source": source, - "value": value, + identity, _ := parsed["identity"].(map[string]any) + if identity == nil { + identity = map[string]any{} + } + identity["resolved"] = true + identity["source"] = source + identity["value"] = value + parsed["identity"] = identity +} + +func addEnvelopeIdentityMetadata(env *envelope.FrameEnvelope, source string, value string) { + addEnvelopeIdentityMetadataWithCacheStatus(env, source, value, "") +} + +func addEnvelopeIdentityMetadataWithCacheStatus(env *envelope.FrameEnvelope, source string, value string, cacheStatus string) { + if env == nil { + return + } + if env.Parsed == nil { + env.Parsed = map[string]any{} + } + addIdentityMetadata(env.Parsed, source, value) + if cacheStatus != "" { + if identity, ok := env.Parsed["identity"].(map[string]any); ok { + identity["cache_status"] = cacheStatus + } } } -func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) { - cacheKey := column + "\x00" + value +func (r *MySQLResolver) sourceMetadataForEnvelope(ctx context.Context, env envelope.FrameEnvelope) (sourceMetadata, error) { + if !r.sourceCodeLookup || env.Protocol != envelope.ProtocolJT808 { + return sourceMetadata{}, nil + } + sourceIP := normalizeEndpointIP(env.SourceEndpoint) + if sourceIP == "" { + return sourceMetadata{}, nil + } + protocolValue := string(env.Protocol) + cacheKey := protocolValue + "\x00" + sourceIP + if snapshot, ok := r.snapshotSource(protocolValue, sourceIP); ok { + return snapshot, nil + } + if r.snapshotOnlyLookups { + return sourceMetadata{}, nil + } now := time.Now() + var stale sourceCodeCacheEntry + hasStale := false + r.lookupMu.Lock() + cached, ok := r.sourceCodeCache[cacheKey] + if ok && now.Before(cached.expiresAt) { + r.lookupMu.Unlock() + if cached.notFound { + return sourceMetadata{}, nil + } + return cached.sourceMetadata(), nil + } + if ok && cached.canServeStale(now) { + stale = cached + hasStale = true + } + r.lookupMu.Unlock() + + var sourceCode, platformName, sourceKind sql.NullString + err := r.db.QueryRowContext(ctx, `SELECT source_code, platform_name, source_kind +FROM vehicle_data_source +WHERE protocol = ? AND source_ip = ? AND enabled = 1 + AND ( + (source_code IS NOT NULL AND TRIM(source_code) <> '') + OR source_kind IN ('PLATFORM', 'DIRECT') + OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '') + ) +ORDER BY updated_at DESC +LIMIT 1`, protocolValue, sourceIP).Scan(&sourceCode, &platformName, &sourceKind) + if err != nil { + if errors.Is(err, sql.ErrNoRows) || isOptionalSourceCodeLookupError(err) { + r.cacheSourceMetadata(cacheKey, sourceMetadata{}, true, now) + return sourceMetadata{}, nil + } + if hasStale { + return stale.sourceMetadata(), nil + } + return sourceMetadata{}, err + } + value := sourceMetadata{ + SourceCode: strings.TrimSpace(sourceCode.String), + PlatformName: strings.TrimSpace(platformName.String), + SourceKind: strings.TrimSpace(sourceKind.String), + } + r.cacheSourceMetadata(cacheKey, value, value.SourceCode == "" && value.PlatformName == "" && value.SourceKind == "", now) + return value, nil +} + +func (r *MySQLResolver) cacheSourceMetadata(cacheKey string, metadata sourceMetadata, notFound bool, now time.Time) { + r.lookupMu.Lock() + r.sourceCodeCache[cacheKey] = sourceCodeCacheEntry{ + sourceCode: strings.TrimSpace(metadata.SourceCode), + platformName: strings.TrimSpace(metadata.PlatformName), + sourceKind: strings.TrimSpace(metadata.SourceKind), + notFound: notFound, + expiresAt: now.Add(r.lookupCacheTTL), + } + entry := r.sourceCodeCache[cacheKey] + entry.staleAt = r.staleUntil(entry.expiresAt, entry.notFound) + r.sourceCodeCache[cacheKey] = entry + r.cleanupLookupCachesLocked(now, false) + r.lookupMu.Unlock() +} + +func (entry sourceCodeCacheEntry) sourceMetadata() sourceMetadata { + return sourceMetadata{ + SourceCode: strings.TrimSpace(entry.sourceCode), + PlatformName: strings.TrimSpace(entry.platformName), + SourceKind: strings.TrimSpace(entry.sourceKind), + } +} + +func applyEnvelopeSourceMetadata(env *envelope.FrameEnvelope, metadata sourceMetadata) { + applyEnvelopeSourceMetadataMode(env, metadata, false) +} + +func overrideEnvelopeSourceMetadata(env *envelope.FrameEnvelope, metadata sourceMetadata) { + applyEnvelopeSourceMetadataMode(env, metadata, true) +} + +func applyEnvelopeSourceMetadataMode(env *envelope.FrameEnvelope, metadata sourceMetadata, override bool) { + if env == nil { + return + } + if value := strings.TrimSpace(metadata.SourceCode); value != "" && (override || env.SourceCode == "") { + env.SourceCode = value + } + if value := strings.TrimSpace(metadata.PlatformName); value != "" && (override || env.PlatformName == "") { + env.PlatformName = value + } + if value := strings.TrimSpace(metadata.SourceKind); value != "" && (override || env.SourceKind == "") { + env.SourceKind = value + } + if env.Parsed == nil || (env.SourceCode == "" && env.PlatformName == "" && env.SourceKind == "") { + return + } + identity, _ := env.Parsed["identity"].(map[string]any) + if identity == nil { + identity = map[string]any{} + } + if env.SourceCode != "" { + identity["source_code"] = env.SourceCode + } + if env.PlatformName != "" { + identity["platform_name"] = env.PlatformName + } + if env.SourceKind != "" { + identity["source_kind"] = env.SourceKind + } + env.Parsed["identity"] = identity +} + +func normalizeEndpointIP(endpoint string) string { + return envelope.NormalizeSourceEndpointKey(endpoint) +} + +func isOptionalSourceCodeLookupError(err error) bool { + if err == nil { + return false + } + text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err))) + return strings.Contains(text, "error 1146") || + strings.Contains(text, "doesn't exist") || + strings.Contains(text, "unknown table") +} + +func (r *MySQLResolver) lookupVehicleIdentifier(ctx context.Context, protocol envelope.Protocol, sourceCode string, candidate CandidateKey) (string, string, string, sourceMetadata, error) { + identifierType := candidateIdentifierType(candidate.Column) + if identifierType == "" { + return "", "", "", sourceMetadata{}, sql.ErrNoRows + } + value := normalizeIdentifierValue(identifierType, candidate.Value) + if value == "" { + return "", "", "", sourceMetadata{}, sql.ErrNoRows + } + protocolValue := string(protocol) + sourceCode = strings.TrimSpace(sourceCode) + if sourceCode != "" { + vin, source, cacheStatus, metadata, err := r.lookupVehicleIdentifierInScope(ctx, protocolValue, sourceCode, identifierType, value) + if err == nil || !errors.Is(err, sql.ErrNoRows) { + return vin, source, cacheStatus, metadata, err + } + } + return r.lookupVehicleIdentifierInScope(ctx, protocolValue, "", identifierType, value) +} + +func (r *MySQLResolver) lookupVehicleIdentifierInScope(ctx context.Context, protocolValue string, sourceCode string, identifierType string, value string) (string, string, string, sourceMetadata, error) { + scope := strings.TrimSpace(sourceCode) + cacheKey := "vehicle_identifier\x00" + protocolValue + "\x00" + scope + "\x00" + identifierType + "\x00" + value + if snapshot, ok := r.snapshotIdentifier(protocolValue, scope, identifierType, value); ok { + metadata := snapshot.sourceMetadata() + return snapshot.VIN, vehicleIdentifierSource(identifierType, firstNonEmpty(scope, metadata.SourceCode)), "snapshot", metadata, nil + } + if r.snapshotOnlyLookups { + return "", "", "", sourceMetadata{}, sql.ErrNoRows + } + now := time.Now() + var stale lookupCacheEntry + hasStale := false r.lookupMu.Lock() cached, ok := r.lookupCache[cacheKey] if ok && now.Before(cached.expiresAt) { r.lookupMu.Unlock() if cached.notFound { - return "", sql.ErrNoRows + return "", "", "", sourceMetadata{}, sql.ErrNoRows } - return cached.vin, nil + return cached.vin, vehicleIdentifierSource(identifierType, firstNonEmpty(scope, cached.sourceCode)), "", cached.sourceMetadata(), nil + } + if ok && cached.canServeStale(now) { + stale = cached + hasStale = true + } + r.lookupMu.Unlock() + + query := `SELECT DISTINCT vin, source_code, COALESCE(NULLIF(TRIM(oem), ''), source_code) AS platform_name +FROM vehicle_identifier +WHERE protocol = ? AND identifier_type = ? AND identifier_value = ? AND enabled = 1 AND vin IS NOT NULL AND vin <> ''` + args := []any{protocolValue, identifierType, value} + if scope != "" { + query += ` AND source_code = ?` + args = append(args, scope) + } + query += ` +ORDER BY vin, source_code +LIMIT 2` + match, err := querySingleVehicleIdentifier(ctx, r.db, query, args...) + if err != nil { + if hasStale && isTransientLookupError(err) { + return stale.vin, vehicleIdentifierSource(identifierType, firstNonEmpty(scope, stale.sourceCode)), "stale", stale.sourceMetadata(), nil + } + if isTransientLookupError(err) { + return "", "", "", sourceMetadata{}, err + } + r.cacheLookup(cacheKey, r.newLookupCacheEntry(vehicleIdentifierMatch{}, true, now), now) + return "", "", "", sourceMetadata{}, err + } + r.cacheLookup(cacheKey, r.newLookupCacheEntry(match, false, now), now) + metadata := match.sourceMetadata() + return match.VIN, vehicleIdentifierSource(identifierType, firstNonEmpty(scope, metadata.SourceCode)), "", metadata, nil +} + +func querySingleVehicleIdentifier(ctx context.Context, db *sql.DB, query string, args ...any) (vehicleIdentifierMatch, error) { + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return vehicleIdentifierMatch{}, err + } + defer rows.Close() + vins := map[string]struct{}{} + sourceCodes := map[string]struct{}{} + platformNames := map[string]struct{}{} + for rows.Next() { + var vin, sourceCode, platformName sql.NullString + if err := rows.Scan(&vin, &sourceCode, &platformName); err != nil { + return vehicleIdentifierMatch{}, err + } + vinValue := strings.TrimSpace(vin.String) + if vinValue == "" { + continue + } + vins[vinValue] = struct{}{} + sourceCodeValue := strings.TrimSpace(sourceCode.String) + if sourceCodeValue != "" { + sourceCodes[sourceCodeValue] = struct{}{} + } + platformNameValue := strings.TrimSpace(platformName.String) + if platformNameValue != "" { + platformNames[platformNameValue] = struct{}{} + } + } + if err := rows.Err(); err != nil { + return vehicleIdentifierMatch{}, err + } + if len(vins) == 1 { + match := vehicleIdentifierMatch{} + for vin := range vins { + match.VIN = vin + } + if len(sourceCodes) == 1 { + for sourceCode := range sourceCodes { + match.SourceCode = sourceCode + } + if len(platformNames) == 1 { + for platformName := range platformNames { + match.PlatformName = platformName + } + } + } + return match, nil + } + if len(vins) > 1 { + return vehicleIdentifierMatch{}, errAmbiguousIdentifier + } + return vehicleIdentifierMatch{}, sql.ErrNoRows +} + +func vehicleIdentifierSource(identifierType string, sourceCode string) string { + source := "vehicle_identifier." + identifierType + if strings.TrimSpace(sourceCode) != "" { + source += "." + strings.TrimSpace(sourceCode) + } + return source +} + +func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, string, error) { + cacheKey := column + "\x00" + value + if vin, ok := r.snapshotBinding(column, value); ok { + return vin, "snapshot", nil + } + if r.snapshotOnlyLookups { + return "", "", sql.ErrNoRows + } + now := time.Now() + var stale lookupCacheEntry + hasStale := false + r.lookupMu.Lock() + cached, ok := r.lookupCache[cacheKey] + if ok && now.Before(cached.expiresAt) { + r.lookupMu.Unlock() + if cached.notFound { + return "", "", sql.ErrNoRows + } + return cached.vin, "", nil + } + if ok && cached.canServeStale(now) { + stale = cached + hasStale = true } r.lookupMu.Unlock() @@ -327,25 +1001,207 @@ func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) var vin string err := r.db.QueryRowContext(ctx, query, value).Scan(&vin) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return "", err + if hasStale { + return stale.vin, "stale", nil + } + return "", "", err } + r.cacheLookup(cacheKey, r.newLookupCacheEntry(vehicleIdentifierMatch{VIN: strings.TrimSpace(vin)}, errors.Is(err, sql.ErrNoRows), now), now) + return vin, "", err +} + +func (r *MySQLResolver) cacheLookup(cacheKey string, entry lookupCacheEntry, now time.Time) { r.lookupMu.Lock() - r.lookupCache[cacheKey] = lookupCacheEntry{ - vin: strings.TrimSpace(vin), - notFound: errors.Is(err, sql.ErrNoRows), - expiresAt: now.Add(r.lookupCacheTTL), + if entry.staleAt.IsZero() { + entry.staleAt = r.staleUntil(entry.expiresAt, entry.notFound) } + r.lookupCache[cacheKey] = entry + r.cleanupLookupCachesLocked(now, false) r.lookupMu.Unlock() - return vin, err +} + +func (r *MySQLResolver) cacheRegistration(phone string, entry registrationCacheEntry, now time.Time) { + r.lookupMu.Lock() + if entry.staleAt.IsZero() { + entry.staleAt = r.staleUntil(entry.expiresAt, entry.notFound) + } + r.registrationCache[phone] = entry + r.cleanupLookupCachesLocked(now, false) + r.lookupMu.Unlock() +} + +func (r *MySQLResolver) newLookupCacheEntry(match vehicleIdentifierMatch, notFound bool, now time.Time) lookupCacheEntry { + expiresAt := now.Add(r.lookupCacheTTL) + metadata := match.sourceMetadata() + return lookupCacheEntry{ + vin: strings.TrimSpace(match.VIN), + sourceCode: strings.TrimSpace(metadata.SourceCode), + platformName: strings.TrimSpace(metadata.PlatformName), + sourceKind: strings.TrimSpace(metadata.SourceKind), + notFound: notFound, + expiresAt: expiresAt, + staleAt: r.staleUntil(expiresAt, notFound), + } +} + +func (match vehicleIdentifierMatch) sourceMetadata() sourceMetadata { + sourceCode := strings.TrimSpace(match.SourceCode) + if sourceCode == "" { + return sourceMetadata{} + } + return sourceMetadata{ + SourceCode: sourceCode, + PlatformName: strings.TrimSpace(firstNonEmpty(match.PlatformName, sourceCode)), + SourceKind: "PLATFORM", + } +} + +func (e lookupCacheEntry) sourceMetadata() sourceMetadata { + return sourceMetadata{ + SourceCode: strings.TrimSpace(e.sourceCode), + PlatformName: strings.TrimSpace(e.platformName), + SourceKind: strings.TrimSpace(e.sourceKind), + } +} + +func (r *MySQLResolver) staleUntil(expiresAt time.Time, notFound bool) time.Time { + if notFound || r.staleLookupTTL <= 0 { + return expiresAt + } + return expiresAt.Add(r.staleLookupTTL) +} + +func (e lookupCacheEntry) canServeStale(now time.Time) bool { + return !e.notFound && strings.TrimSpace(e.vin) != "" && now.Before(e.staleAt) +} + +func (e registrationCacheEntry) canServeStale(now time.Time) bool { + return !e.notFound && (strings.TrimSpace(e.vin) != "" || strings.TrimSpace(e.deviceID) != "" || strings.TrimSpace(e.plate) != "") && now.Before(e.staleAt) +} + +func (e sourceCodeCacheEntry) canServeStale(now time.Time) bool { + return !e.notFound && strings.TrimSpace(e.sourceCode) != "" && now.Before(e.staleAt) +} + +func isTransientLookupError(err error) bool { + return err != nil && !errors.Is(err, sql.ErrNoRows) && !errors.Is(err, errAmbiguousIdentifier) +} + +func (r *MySQLResolver) cleanupLookupCachesLocked(now time.Time, force bool) { + if !force && r.maxCacheEntries > 0 && + len(r.lookupCache) <= r.maxCacheEntries && + len(r.registrationCache) <= r.maxCacheEntries && + len(r.sourceCodeCache) <= r.maxCacheEntries && + !r.nextLookupCleanup.IsZero() && + now.Before(r.nextLookupCleanup) { + return + } + r.nextLookupCleanup = now.Add(r.cacheCleanupInterval) + for key, entry := range r.lookupCache { + if !now.Before(entry.staleAt) { + delete(r.lookupCache, key) + } + } + for key, entry := range r.registrationCache { + if !now.Before(entry.staleAt) { + delete(r.registrationCache, key) + } + } + for key, entry := range r.sourceCodeCache { + if !now.Before(entry.staleAt) { + delete(r.sourceCodeCache, key) + } + } + r.trimLookupCacheLocked() + r.trimRegistrationCacheLocked() + r.trimSourceCodeCacheLocked() +} + +func (r *MySQLResolver) trimLookupCacheLocked() { + if r.maxCacheEntries <= 0 { + return + } + for len(r.lookupCache) > r.maxCacheEntries { + victim := "" + var victimExpiresAt time.Time + for key, entry := range r.lookupCache { + if victim == "" || entry.expiresAt.Before(victimExpiresAt) { + victim = key + victimExpiresAt = entry.expiresAt + } + } + if victim == "" { + return + } + delete(r.lookupCache, victim) + } +} + +func (r *MySQLResolver) trimRegistrationCacheLocked() { + if r.maxCacheEntries <= 0 { + return + } + for len(r.registrationCache) > r.maxCacheEntries { + victim := "" + var victimExpiresAt time.Time + for key, entry := range r.registrationCache { + if victim == "" || entry.expiresAt.Before(victimExpiresAt) { + victim = key + victimExpiresAt = entry.expiresAt + } + } + if victim == "" { + return + } + delete(r.registrationCache, victim) + } +} + +func (r *MySQLResolver) trimSourceCodeCacheLocked() { + if r.maxCacheEntries <= 0 { + return + } + for len(r.sourceCodeCache) > r.maxCacheEntries { + victim := "" + var victimExpiresAt time.Time + for key, entry := range r.sourceCodeCache { + if victim == "" || entry.expiresAt.Before(victimExpiresAt) { + victim = key + victimExpiresAt = entry.expiresAt + } + } + if victim == "" { + return + } + delete(r.sourceCodeCache, victim) + } +} + +func candidateIdentifierType(column string) string { + switch strings.TrimSpace(strings.ToLower(column)) { + case "phone": + return "JT808_PHONE" + case "plate": + return "PLATE" + default: + return "" + } +} + +func normalizeIdentifierValue(identifierType string, value string) string { + value = strings.TrimSpace(value) + switch strings.ToUpper(strings.TrimSpace(identifierType)) { + case "JT808_PHONE": + return normalizePhone(value) + default: + return value + } } func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelope) error { if env.Protocol != envelope.ProtocolJT808 || strings.TrimSpace(env.Phone) == "" || strings.TrimSpace(env.MessageID) == "" { return nil } - if env.MessageID != "0x0100" && env.MessageID != "0x0102" && !r.shouldTouchJT808Location(env) { - return nil - } registration := mapValue(env.Parsed, "registration") authentication := mapValue(env.Parsed, "authentication") deviceID := firstNonEmpty(env.DeviceID, textValue(registration, "device_id")) @@ -354,22 +1210,42 @@ func (r *MySQLResolver) trackJT808(ctx context.Context, env envelope.FrameEnvelo if vin == "" { vin = "unknown" } + if r.disableRegistrationWrites { + // The gateway still updates its local phone session immediately. Durable + // registration/authentication facts are projected from Kafka raw by the + // identity writer, keeping MySQL latency out of the ingress process. + r.cacheJT808Registration(env, deviceID, plate, vin) + if env.MessageID == JT808RegisterMessageID || env.MessageID == JT808AuthMessageID { + r.observeRegistrationWrite("delegated", "ok", nil) + } + return nil + } + + locationTouchPhone := "" + locationTouchAt := time.Time{} + if env.MessageID != JT808RegisterMessageID && env.MessageID != JT808AuthMessageID { + var ok bool + locationTouchPhone, locationTouchAt, ok = r.reserveJT808LocationTouch(env) + if !ok { + return nil + } + } firstRegisteredAt := nilTime() latestRegisteredAt := nilTime() - if env.MessageID == "0x0100" { + if env.MessageID == JT808RegisterMessageID { firstRegisteredAt = "CURRENT_TIMESTAMP" latestRegisteredAt = "CURRENT_TIMESTAMP" } latestAuthenticatedAt := nilTime() - if env.MessageID == "0x0102" { + if env.MessageID == JT808AuthMessageID { latestAuthenticatedAt = "CURRENT_TIMESTAMP" } query := `INSERT INTO jt808_registration (phone, device_id, plate, vin, province, city, manufacturer, device_type, plate_color, - auth_token, auth_imei, auth_software_version, source_endpoint, + auth_token, auth_imei, auth_software_version, source_endpoint, source_ip, first_registered_at, latest_registered_at, latest_authenticated_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ` + firstRegisteredAt + `, ` + latestRegisteredAt + `, ` + latestAuthenticatedAt + `) ON DUPLICATE KEY UPDATE device_id = IF(VALUES(device_id) <> '', VALUES(device_id), device_id), plate = IF(VALUES(plate) <> '', VALUES(plate), plate), @@ -383,11 +1259,12 @@ ON DUPLICATE KEY UPDATE auth_imei = IF(VALUES(auth_imei) <> '', VALUES(auth_imei), auth_imei), auth_software_version = IF(VALUES(auth_software_version) <> '', VALUES(auth_software_version), auth_software_version), source_endpoint = IF(VALUES(source_endpoint) <> '', VALUES(source_endpoint), source_endpoint), + source_ip = IF(VALUES(source_ip) <> '', VALUES(source_ip), source_ip), latest_seen_at = CURRENT_TIMESTAMP, first_registered_at = COALESCE(first_registered_at, VALUES(first_registered_at)), latest_registered_at = COALESCE(VALUES(latest_registered_at), latest_registered_at), latest_authenticated_at = COALESCE(VALUES(latest_authenticated_at), latest_authenticated_at)` - _, err := r.db.ExecContext(ctx, query, + args := []any{ env.Phone, deviceID, plate, @@ -401,30 +1278,345 @@ ON DUPLICATE KEY UPDATE textValue(authentication, "imei"), textValue(authentication, "software_version"), env.SourceEndpoint, - ) + normalizeEndpointIP(env.SourceEndpoint), + } + if r.registrationWriteQueue != nil { + if err := r.enqueueJT808RegistrationWrite(ctx, jt808RegistrationWriteJob{ + query: query, + args: args, + locationTouchPhone: locationTouchPhone, + locationTouchAt: locationTouchAt, + }); err != nil { + r.observeRegistrationWrite("async_enqueue", "error", err) + if locationTouchPhone != "" { + r.rollbackJT808LocationTouch(locationTouchPhone, locationTouchAt) + r.markJT808LocationTouchFailure(locationTouchPhone) + } + return err + } + r.observeRegistrationWrite("async_enqueue", "ok", nil) + r.cacheJT808Registration(env, deviceID, plate, vin) + return nil + } + err := r.execJT808Registration(ctx, query, args...) if err != nil { + r.observeRegistrationWrite("sync", "error", err) + if locationTouchPhone != "" { + r.rollbackJT808LocationTouch(locationTouchPhone, locationTouchAt) + r.markJT808LocationTouchFailure(locationTouchPhone) + } return err } + r.observeRegistrationWrite("sync", "ok", nil) + if locationTouchPhone != "" { + r.clearJT808LocationTouchFailure(locationTouchPhone) + } + r.cacheJT808Registration(env, deviceID, plate, vin) return nil } -func (r *MySQLResolver) shouldTouchJT808Location(env envelope.FrameEnvelope) bool { - if env.MessageID != "0x0200" { +func (r *MySQLResolver) enqueueJT808RegistrationWrite(ctx context.Context, job jt808RegistrationWriteJob) error { + if r.registrationWriteQueue == nil { + return r.execJT808Registration(ctx, job.query, job.args...) + } + select { + case <-r.registrationWriteClosed: + return ErrRegistrationWriteQueueClosed + default: + } + var timeoutC <-chan time.Time + var timer *time.Timer + if r.registrationEnqueueTimeout > 0 { + timer = time.NewTimer(r.registrationEnqueueTimeout) + timeoutC = timer.C + defer timer.Stop() + } + select { + case r.registrationWriteQueue <- job: + return nil + case <-r.registrationWriteClosed: + return ErrRegistrationWriteQueueClosed + case <-ctx.Done(): + return ctx.Err() + case <-timeoutC: + return ErrRegistrationWriteQueueTimeout + } +} + +func (r *MySQLResolver) registrationWriteWorker() { + defer r.registrationWriteWG.Done() + for { + select { + case job := <-r.registrationWriteQueue: + r.runJT808RegistrationWriteJob(job) + case <-r.registrationWriteClosed: + for { + select { + case job := <-r.registrationWriteQueue: + r.runJT808RegistrationWriteJob(job) + default: + return + } + } + } + } +} + +func (r *MySQLResolver) runJT808RegistrationWriteJob(job jt808RegistrationWriteJob) { + ctx := context.Background() + var cancel context.CancelFunc + if r.registrationWriteTimeout > 0 { + ctx, cancel = context.WithTimeout(ctx, r.registrationWriteTimeout) + } else { + ctx, cancel = context.WithCancel(ctx) + } + err := r.execJT808Registration(ctx, job.query, job.args...) + cancel() + if err != nil { + r.observeRegistrationWrite("async_background", "error", err) + if job.locationTouchPhone != "" { + r.rollbackJT808LocationTouch(job.locationTouchPhone, job.locationTouchAt) + r.markJT808LocationTouchFailure(job.locationTouchPhone) + } + return + } + r.observeRegistrationWrite("async_background", "ok", nil) + if job.locationTouchPhone != "" { + r.clearJT808LocationTouchFailure(job.locationTouchPhone) + } +} + +func (r *MySQLResolver) observeRegistrationWrite(mode string, status string, err error) { + if r == nil { + return + } + mode = strings.TrimSpace(mode) + if mode == "" { + mode = "unknown" + } + status = strings.TrimSpace(status) + if status == "" { + status = "unknown" + } + if r.onRegistrationWriteResult != nil { + r.onRegistrationWriteResult(RegistrationWriteResult{ + Mode: mode, + Status: status, + Err: err, + }) + } + if err != nil && r.onRegistrationWriteError != nil { + r.onRegistrationWriteError(err) + } +} + +func (r *MySQLResolver) execJT808Registration(ctx context.Context, query string, args ...any) error { + attempts := r.registrationWriteAttempts + if attempts <= 0 { + attempts = 1 + } + var err error + for attempt := 1; attempt <= attempts; attempt++ { + _, err = r.db.ExecContext(ctx, query, args...) + if err == nil || !isTransientMySQLIdentityWriteError(err) || attempt == attempts { + return err + } + if r.registrationWriteRetryDelay <= 0 { + continue + } + timer := time.NewTimer(r.registrationWriteRetryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + return err +} + +func isTransientMySQLIdentityWriteError(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return false } + text := strings.ToLower(strings.TrimSpace(err.Error())) + return strings.Contains(text, "deadlock") || + strings.Contains(text, "error 1213") || + strings.Contains(text, "40001") || + strings.Contains(text, "lock wait timeout") || + strings.Contains(text, "error 1205") || + strings.Contains(text, "timeout") || + strings.Contains(text, "temporary") || + strings.Contains(text, "temporarily") || + strings.Contains(text, "connection refused") || + strings.Contains(text, "connection reset") || + strings.Contains(text, "connection closed") || + strings.Contains(text, "broken pipe") || + strings.Contains(text, "bad connection") || + strings.Contains(text, "invalid connection") || + strings.Contains(text, "i/o timeout") || + text == "eof" || + strings.Contains(text, "unexpected eof") || + strings.Contains(text, "server is down") || + strings.Contains(text, "network is unreachable") || + strings.Contains(text, "no route to host") +} + +func (r *MySQLResolver) cacheJT808Registration(env envelope.FrameEnvelope, deviceID string, plate string, vin string) { + phone := normalizePhone(env.Phone) + if phone == "" { + return + } + vin = strings.TrimSpace(vin) + if vin == "" { + vin = "unknown" + } + now := time.Now() + r.lookupMu.Lock() + defer r.lookupMu.Unlock() + existing := r.registrationCache[phone] + r.registrationCache[phone] = registrationCacheEntry{ + vin: preferKnownVIN(vin, existing.vin), + deviceID: firstNonEmpty(deviceID, existing.deviceID), + plate: firstNonEmpty(plate, existing.plate), + notFound: false, + expiresAt: now.Add(r.lookupCacheTTL), + } + entry := r.registrationCache[phone] + entry.staleAt = r.staleUntil(entry.expiresAt, entry.notFound) + r.registrationCache[phone] = entry + r.cleanupLookupCachesLocked(now, false) +} + +func preferKnownVIN(candidate string, existing string) string { + candidate = strings.TrimSpace(candidate) + existing = strings.TrimSpace(existing) + if candidate != "" && !strings.EqualFold(candidate, "unknown") { + return candidate + } + if existing != "" { + return existing + } + if candidate != "" { + return candidate + } + return "unknown" +} + +func (r *MySQLResolver) reserveJT808LocationTouch(env envelope.FrameEnvelope) (string, time.Time, bool) { + if env.MessageID != "0x0200" { + return "", time.Time{}, false + } phone := normalizePhone(env.Phone) if phone == "" { - return false + return "", time.Time{}, false } now := time.Now() r.touchMu.Lock() defer r.touchMu.Unlock() + if retryAt, ok := r.locationTouchFailures[phone]; ok { + if now.Before(retryAt) { + return "", time.Time{}, false + } + delete(r.locationTouchFailures, phone) + } last, ok := r.locationTouches[phone] if ok && now.Sub(last) < r.locationTouchInterval { - return false + return "", time.Time{}, false } r.locationTouches[phone] = now - return true + r.cleanupLocationTouchesLocked(now, false) + return phone, now, true +} + +func (r *MySQLResolver) rollbackJT808LocationTouch(phone string, reservedAt time.Time) { + phone = normalizePhone(phone) + if phone == "" || reservedAt.IsZero() { + return + } + r.touchMu.Lock() + defer r.touchMu.Unlock() + if current, ok := r.locationTouches[phone]; ok && current.Equal(reservedAt) { + delete(r.locationTouches, phone) + } +} + +func (r *MySQLResolver) markJT808LocationTouchFailure(phone string) { + phone = normalizePhone(phone) + if phone == "" || r.locationTouchRetryInterval <= 0 { + return + } + r.touchMu.Lock() + defer r.touchMu.Unlock() + if r.locationTouchFailures == nil { + r.locationTouchFailures = map[string]time.Time{} + } + now := time.Now() + r.locationTouchFailures[phone] = now.Add(r.locationTouchRetryInterval) + r.cleanupLocationTouchesLocked(now, false) +} + +func (r *MySQLResolver) clearJT808LocationTouchFailure(phone string) { + phone = normalizePhone(phone) + if phone == "" { + return + } + r.touchMu.Lock() + defer r.touchMu.Unlock() + delete(r.locationTouchFailures, phone) +} + +func (r *MySQLResolver) cleanupLocationTouchesLocked(now time.Time, force bool) { + if !force && r.maxCacheEntries > 0 && + len(r.locationTouches) <= r.maxCacheEntries && + len(r.locationTouchFailures) <= r.maxCacheEntries && + !r.nextTouchCleanup.IsZero() && + now.Before(r.nextTouchCleanup) { + return + } + r.nextTouchCleanup = now.Add(r.cacheCleanupInterval) + cutoff := now.Add(-r.locationTouchInterval) + for phone, last := range r.locationTouches { + if last.Before(cutoff) || last.Equal(cutoff) { + delete(r.locationTouches, phone) + } + } + for phone, retryAt := range r.locationTouchFailures { + if !now.Before(retryAt) { + delete(r.locationTouchFailures, phone) + } + } + if r.maxCacheEntries <= 0 { + return + } + for len(r.locationTouches) > r.maxCacheEntries { + victim := "" + var oldest time.Time + for phone, last := range r.locationTouches { + if victim == "" || last.Before(oldest) { + victim = phone + oldest = last + } + } + if victim == "" { + return + } + delete(r.locationTouches, victim) + } + for len(r.locationTouchFailures) > r.maxCacheEntries { + victim := "" + var earliest time.Time + for phone, retryAt := range r.locationTouchFailures { + if victim == "" || retryAt.Before(earliest) { + victim = phone + earliest = retryAt + } + } + if victim == "" { + return + } + delete(r.locationTouchFailures, victim) + } } type CandidateKey struct { diff --git a/go/vehicle-gateway/internal/identity/resolver_test.go b/go/vehicle-gateway/internal/identity/resolver_test.go index 8f7a9755..cd8ea869 100644 --- a/go/vehicle-gateway/internal/identity/resolver_test.go +++ b/go/vehicle-gateway/internal/identity/resolver_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "strings" + "sync" "testing" "time" @@ -57,9 +58,7 @@ func TestCandidateKeysNormalizesPhone(t *testing.T) { func TestMySQLResolverFillsVINFromPhone(t *testing.T) { db, mock := newMockDB(t) defer db.Close() - mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). - WithArgs("13307795425"). - WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001")) + expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000001") resolver := NewMySQLResolver(db, "vehicle_identity_binding") env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ @@ -74,7 +73,7 @@ func TestMySQLResolverFillsVINFromPhone(t *testing.T) { t.Fatalf("vin = %q", env.VIN) } identity, ok := env.Parsed["identity"].(map[string]any) - if !ok || identity["source"] != "phone" { + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" { t.Fatalf("identity metadata = %#v", env.Parsed["identity"]) } if err := mock.ExpectationsWereMet(); err != nil { @@ -82,9 +81,254 @@ func TestMySQLResolverFillsVINFromPhone(t *testing.T) { } } +func TestMySQLResolverUsesDataSourceCodeForJT808IdentifierLookup(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectSourceCodeHit(mock, "115.231.168.135", "xinda") + expectVehicleIdentifierHitForSource(mock, "xinda", "JT808_PHONE", "13307795425", "LNBVIN00000000001") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SourceCodeLookup: true, + LookupCacheTTL: time.Hour, + }) + env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + SourceEndpoint: "115.231.168.135:43625", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if env.VIN != "LNBVIN00000000001" { + t.Fatalf("vin = %q", env.VIN) + } + if env.SourceCode != "xinda" || env.PlatformName != "G7s" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.xinda" { + t.Fatalf("identity metadata = %#v", env.Parsed["identity"]) + } + if identity["source_code"] != "xinda" || identity["platform_name"] != "G7s" || identity["source_kind"] != "PLATFORM" { + t.Fatalf("identity source metadata = %#v", identity) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverVehicleIdentifierSourceOverridesDataSourceCode(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectQuery("SELECT source_code"). + WithArgs("JT808", "115.159.85.149"). + WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow("dongfang_beidou", "G7易流", "PLATFORM")) + expectVehicleIdentifierMissForSource(mock, "dongfang_beidou", "JT808_PHONE", "64341232682") + expectVehicleIdentifierHitWithSource(mock, "JT808_PHONE", "64341232682", "LB9A32A23R0LS1045", "g7s", "G7s") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SourceCodeLookup: true, + LookupCacheTTL: time.Hour, + }) + env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "064341232682", + SourceEndpoint: "115.159.85.149:42823", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if env.VIN != "LB9A32A23R0LS1045" { + t.Fatalf("vin = %q", env.VIN) + } + if env.SourceCode != "g7s" || env.PlatformName != "G7s" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.g7s" { + t.Fatalf("identity metadata = %#v", env.Parsed["identity"]) + } + if identity["source_code"] != "g7s" || identity["platform_name"] != "G7s" || identity["source_kind"] != "PLATFORM" { + t.Fatalf("identity source metadata = %#v", identity) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverUsesStaleIdentifierCacheWhenLookupFails(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000001") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LookupCacheTTL: time.Nanosecond, + StaleLookupTTL: time.Hour, + }) + first, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("first Resolve() error = %v", err) + } + if first.VIN != "LNBVIN00000000001" { + t.Fatalf("first vin = %q", first.VIN) + } + + time.Sleep(time.Millisecond) + mock.ExpectQuery("SELECT DISTINCT vin"). + WithArgs("JT808", "JT808_PHONE", "13307795425"). + WillReturnError(errors.New("mysql temporarily unavailable")) + second, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("second Resolve() error = %v", err) + } + if second.VIN != "LNBVIN00000000001" { + t.Fatalf("second vin = %q, want stale cached vin", second.VIN) + } + identity, ok := second.Parsed["identity"].(map[string]any) + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" { + t.Fatalf("identity metadata = %#v", second.Parsed["identity"]) + } + if identity["cache_status"] != "stale" { + t.Fatalf("identity cache_status = %#v, want stale", identity["cache_status"]) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestNormalizeEndpointIPUsesSharedSourceEndpointKey(t *testing.T) { + tests := map[string]string{ + "115.231.168.135:43625": "115.231.168.135", + " 115.159.85.149:28316 ": "115.159.85.149", + "mqtt://yutong/ytforward/shln/3": "mqtt", + "MQTT://YUTONG/topic": "mqtt", + } + for input, want := range tests { + if got := normalizeEndpointIP(input); got != want { + t.Fatalf("normalizeEndpointIP(%q) = %q, want %q", input, got, want) + } + } +} + +func TestMySQLResolverKeepsDirectSourceKindWithoutSourceCode(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectQuery("SELECT source_code"). + WithArgs("JT808", "39.144.3.22"). + WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow("", "", "DIRECT")) + expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307765812", "LA9GG64L7PBAF4001") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SourceCodeLookup: true, + LookupCacheTTL: time.Hour, + }) + env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307765812", + SourceEndpoint: "39.144.3.22:60177", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if env.VIN != "LA9GG64L7PBAF4001" { + t.Fatalf("vin = %q", env.VIN) + } + if env.SourceCode != "" || env.SourceKind != "DIRECT" { + t.Fatalf("source metadata = code:%q kind:%q", env.SourceCode, env.SourceKind) + } + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok || identity["source_kind"] != "DIRECT" { + t.Fatalf("identity source metadata = %#v", env.Parsed["identity"]) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverFallsBackToGlobalIdentifierWhenSourceCodeMisses(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectSourceCodeHit(mock, "115.231.168.135", "xinda") + expectVehicleIdentifierMissForSource(mock, "xinda", "JT808_PHONE", "13307795425") + expectVehicleIdentifierHit(mock, "JT808_PHONE", "13307795425", "LNBVIN00000000002") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SourceCodeLookup: true, + LookupCacheTTL: time.Hour, + }) + env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + SourceEndpoint: "115.231.168.135:43625", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if env.VIN != "LNBVIN00000000002" { + t.Fatalf("vin = %q", env.VIN) + } + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE" { + t.Fatalf("identity metadata = %#v", env.Parsed["identity"]) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverUsesSingleGlobalIdentifierSourceCodeForJT808SourceMetadata(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectSourceCodeMiss(mock, "117.132.196.119") + expectVehicleIdentifierHitWithSource(mock, "JT808_PHONE", "41456413943", "LNXNEGRR0SR321372", "xinda", "信达") + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SourceCodeLookup: true, + LookupCacheTTL: time.Hour, + }) + env, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "41456413943", + SourceEndpoint: "117.132.196.119:3275", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if env.VIN != "LNXNEGRR0SR321372" { + t.Fatalf("vin = %q", env.VIN) + } + if env.SourceCode != "xinda" || env.PlatformName != "信达" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } + identity, ok := env.Parsed["identity"].(map[string]any) + if !ok || identity["source"] != "vehicle_identifier.JT808_PHONE.xinda" { + t.Fatalf("identity metadata = %#v", env.Parsed["identity"]) + } + if identity["source_code"] != "xinda" || identity["platform_name"] != "信达" || identity["source_kind"] != "PLATFORM" { + t.Fatalf("identity source metadata = %#v", identity) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + func TestMySQLResolverLooksUpVINByUniqueKeyWithoutSort(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\? AND vin IS NOT NULL AND vin <> ''$"). WithArgs("13307795425"). WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNBVIN00000000001")) @@ -109,6 +353,7 @@ func TestMySQLResolverLooksUpVINByUniqueKeyWithoutSort(t *testing.T) { func TestMySQLResolverDoesNotLookupBindingByDeviceID(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). WithArgs("13307795425"). WillReturnError(sql.ErrNoRows) @@ -133,9 +378,11 @@ func TestMySQLResolverDoesNotLookupBindingByDeviceID(t *testing.T) { func TestMySQLResolverTracksJT808RegistrationWithoutWritingBinding(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13079963379") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). WithArgs("13079963379"). WillReturnError(sql.ErrNoRows) + expectVehicleIdentifierMiss(mock, "PLATE", "TEST123") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). WithArgs("TEST123"). WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LKLG7C4E3NA774736")) @@ -176,6 +423,7 @@ func TestMySQLResolverTracksJT808RegistrationWithoutWritingBinding(t *testing.T) func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). WithArgs("13307795425"). WillReturnError(sql.ErrNoRows) @@ -206,12 +454,14 @@ func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) { func TestMySQLResolverUsesJT808RegistrationPlateForLocationVIN(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "40692934322") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). WithArgs("40692934322"). WillReturnError(sql.ErrNoRows) mock.ExpectQuery("SELECT vin, device_id, plate FROM jt808_registration WHERE phone = \\?"). WithArgs("40692934322"). WillReturnRows(sqlmock.NewRows([]string{"vin", "device_id", "plate"}).AddRow("unknown", "18285", "粤AG18285")) + expectVehicleIdentifierMiss(mock, "PLATE", "粤AG18285") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). WithArgs("粤AG18285"). WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNXNEGRR7SR318212")) @@ -248,9 +498,352 @@ func TestMySQLResolverUsesJT808RegistrationPlateForLocationVIN(t *testing.T) { } } +func TestMySQLResolverRefreshesRegistrationCacheAfterRegisterFrame(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "40692934322") + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). + WithArgs("40692934322"). + WillReturnError(sql.ErrNoRows) + mock.ExpectQuery("SELECT vin, device_id, plate FROM jt808_registration WHERE phone = \\?"). + WithArgs("40692934322"). + WillReturnError(sql.ErrNoRows) + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnResult(sqlmock.NewResult(1, 1)) + expectVehicleIdentifierMiss(mock, "PLATE", "粤AG18285") + mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE plate = \\?"). + WithArgs("粤AG18285"). + WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("LNXNEGRR7SR318212")) + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LocationTouchInterval: time.Hour, + LookupCacheTTL: time.Hour, + }) + firstLocation, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("first location Resolve() error = %v", err) + } + if firstLocation.VIN != "" { + t.Fatalf("first location vin = %q, want unresolved", firstLocation.VIN) + } + + registered, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0100", + Phone: "040692934322", + DeviceID: "18285", + Plate: "粤AG18285", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{ + "registration": map[string]any{ + "device_id": "18285", + "plate": "粤AG18285", + }, + }, + }) + if err != nil { + t.Fatalf("registration Resolve() error = %v", err) + } + if registered.VIN != "LNXNEGRR7SR318212" { + t.Fatalf("registered vin = %q", registered.VIN) + } + + secondLocation, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("second location Resolve() error = %v", err) + } + if secondLocation.VIN != "LNXNEGRR7SR318212" { + t.Fatalf("second location vin = %q, want cache-refreshed registration vin", secondLocation.VIN) + } + if secondLocation.DeviceID != "18285" || secondLocation.Plate != "粤AG18285" { + t.Fatalf("second location identity not copied: device=%q plate=%q", secondLocation.DeviceID, secondLocation.Plate) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverDelegatesRegistrationPersistenceButKeepsLocalSession(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + var results []RegistrationWriteResult + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + DisableRegistrationWrites: true, + OnRegistrationWriteResult: func(result RegistrationWriteResult) { + results = append(results, result) + }, + }) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: JT808RegisterMessageID, + Phone: "013307795425", + VIN: "LTESTVIN000000001", + DeviceID: "DEV-1", + Plate: "粤A00001", + SourceEndpoint: "115.231.168.135:43625", + ParseStatus: envelope.ParseOK, + } + + resolved, err := resolver.Resolve(context.Background(), env) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if resolved.VIN != env.VIN { + t.Fatalf("resolved vin = %q, want %q", resolved.VIN, env.VIN) + } + entry, ok := resolver.registrationCache["13307795425"] + if !ok || entry.vin != env.VIN || entry.plate != env.Plate || entry.deviceID != env.DeviceID { + t.Fatalf("local session = %+v exists=%v", entry, ok) + } + if len(results) != 1 || results[0].Mode != "delegated" || results[0].Status != "ok" { + t.Fatalf("registration write results = %#v", results) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unexpected mysql access: %v", err) + } +} + +func TestMySQLResolverRetriesRegistrationUpsertTransientFailure(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnError(errors.New("driver: bad connection: read tcp: connection reset by peer")) + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LocationTouchInterval: time.Hour, + RegistrationWriteAttempts: 2, + RegistrationWriteRetryDelay: -1, + LookupCacheTTL: time.Hour, + }) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + VIN: "LNXNEGRR7SR318212", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + } + if _, err := resolver.Resolve(context.Background(), env); err != nil { + t.Fatalf("Resolve() error = %v, want internal retry to recover", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverAsyncRegistrationWritesDrainOnClose(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnResult(sqlmock.NewResult(1, 1)) + results := make(chan RegistrationWriteResult, 2) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + AsyncRegistrationWrites: true, + RegistrationWriteQueueSize: 8, + RegistrationWriteWorkers: 1, + RegistrationWriteTimeout: time.Second, + OnRegistrationWriteResult: func(result RegistrationWriteResult) { + results <- result + }, + LocationTouchInterval: time.Hour, + LookupCacheTTL: time.Hour, + }) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + VIN: "LNXNEGRR7SR318212", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + } + if _, err := resolver.Resolve(context.Background(), env); err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if err := resolver.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + close(results) + got := map[string]int{} + for result := range results { + got[result.Mode+":"+result.Status]++ + } + if got["async_enqueue:ok"] != 1 || got["async_background:ok"] != 1 { + t.Fatalf("registration write results = %#v, want enqueue/background ok", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverAsyncRegistrationWriteFailureMarksLocationRetry(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnError(errors.New("driver: bad connection")) + errs := make(chan error, 1) + results := make(chan RegistrationWriteResult, 2) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + AsyncRegistrationWrites: true, + RegistrationWriteQueueSize: 8, + RegistrationWriteWorkers: 1, + RegistrationWriteTimeout: time.Second, + OnRegistrationWriteResult: func(result RegistrationWriteResult) { + results <- result + }, + OnRegistrationWriteError: func(err error) { + errs <- err + }, + LocationTouchInterval: time.Hour, + LocationTouchRetryInterval: time.Hour, + RegistrationWriteAttempts: 1, + LookupCacheTTL: time.Hour, + }) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + VIN: "LNXNEGRR7SR318212", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + } + if _, err := resolver.Resolve(context.Background(), env); err != nil { + t.Fatalf("Resolve() error = %v, async write failure should be reported out-of-band", err) + } + if err := resolver.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + select { + case err := <-errs: + if err == nil { + t.Fatal("async error callback received nil") + } + default: + t.Fatal("async write failure was not reported") + } + stats := resolver.CacheStats() + if stats.LocationTouchFailureEntries != 1 { + t.Fatalf("location touch failure cache entries = %d, want 1", stats.LocationTouchFailureEntries) + } + close(results) + got := map[string]int{} + for result := range results { + got[result.Mode+":"+result.Status]++ + } + if got["async_enqueue:ok"] != 1 || got["async_background:error"] != 1 { + t.Fatalf("registration write results = %#v, want enqueue ok/background error", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMySQLResolverBacksOffLocationTouchAfterExhaustedUpsert(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + mock.ExpectExec("INSERT INTO jt808_registration"). + WillReturnError(errors.New("driver: bad connection")) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LocationTouchInterval: time.Hour, + LocationTouchRetryInterval: time.Hour, + RegistrationWriteAttempts: 1, + LookupCacheTTL: time.Hour, + }) + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + Phone: "040692934322", + VIN: "LNXNEGRR7SR318212", + SourceEndpoint: "115.231.168.135:47822", + Parsed: map[string]any{}, + } + if _, err := resolver.Resolve(context.Background(), env); err == nil { + t.Fatal("first Resolve() error = nil, want exhausted transient registration upsert failure") + } + stats := resolver.CacheStats() + if stats.LocationTouchFailureEntries != 1 { + t.Fatalf("location touch failure cache entries = %d, want 1", stats.LocationTouchFailureEntries) + } + if _, err := resolver.Resolve(context.Background(), env); err != nil { + t.Fatalf("second Resolve() error = %v, want short backoff to skip immediate retry", err) + } + stats = resolver.CacheStats() + if stats.LocationTouchFailureEntries != 1 { + t.Fatalf("location touch failure cache entries after backoff = %d, want 1", stats.LocationTouchFailureEntries) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestIsTransientMySQLIdentityWriteError(t *testing.T) { + for _, err := range []error{ + errors.New("dial tcp 127.0.0.1:3306: connection refused"), + errors.New("read tcp: connection reset by peer"), + errors.New("write tcp: broken pipe"), + errors.New("driver: bad connection"), + errors.New("invalid connection"), + errors.New("i/o timeout"), + errors.New("EOF"), + errors.New("server is down"), + errors.New("network is unreachable"), + } { + if !isTransientMySQLIdentityWriteError(err) { + t.Fatalf("isTransientMySQLIdentityWriteError(%q) = false, want true", err.Error()) + } + } + for _, err := range []error{ + context.Canceled, + context.DeadlineExceeded, + errors.New("duplicate key conflict"), + nil, + } { + if isTransientMySQLIdentityWriteError(err) { + t.Fatalf("isTransientMySQLIdentityWriteError(%v) = true, want false", err) + } + } +} + +func TestTimeoutResolverAppliesDeadlineToDelegate(t *testing.T) { + delegate := &deadlineCheckingResolver{} + resolver := TimeoutResolver{Delegate: delegate, Timeout: 50 * time.Millisecond} + + if _, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}); err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if delegate.deadline.IsZero() { + t.Fatal("delegate did not receive a deadline") + } + if remaining := time.Until(delegate.deadline); remaining <= 0 || remaining > time.Second { + t.Fatalf("deadline remaining = %s", remaining) + } +} + func TestMySQLResolverCachesIdentityMissesForHighFrequencyFrames(t *testing.T) { db, mock := newMockDB(t) defer db.Close() + expectVehicleIdentifierMiss(mock, "JT808_PHONE", "13307795425") mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?"). WithArgs("13307795425"). WillReturnError(sql.ErrNoRows) @@ -274,6 +867,95 @@ func TestMySQLResolverCachesIdentityMissesForHighFrequencyFrames(t *testing.T) { } } +func TestMySQLResolverBoundsIdentityLookupCaches(t *testing.T) { + db, _ := newMockDB(t) + defer db.Close() + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LookupCacheTTL: time.Hour, + CacheCleanupInterval: time.Hour, + MaxCacheEntries: 2, + LocationTouchInterval: time.Hour, + }) + now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC) + for i, key := range []string{"lookup-1", "lookup-2", "lookup-3"} { + entryNow := now.Add(time.Duration(i) * time.Second) + resolver.cacheLookup(key, lookupCacheEntry{ + vin: key, + expiresAt: entryNow.Add(time.Hour), + }, entryNow) + resolver.cacheRegistration("phone-"+key, registrationCacheEntry{ + vin: key, + expiresAt: entryNow.Add(time.Hour), + }, entryNow) + resolver.cacheSourceMetadata("source-"+key, sourceMetadata{SourceCode: key}, false, entryNow) + } + + stats := resolver.CacheStats() + if stats.LookupEntries != 2 || stats.RegistrationEntries != 2 || stats.SourceCodeEntries != 2 || stats.MaxEntries != 2 { + t.Fatalf("cache stats = %+v, want all identity caches capped at 2", stats) + } + resolver.lookupMu.Lock() + _, hasOldLookup := resolver.lookupCache["lookup-1"] + _, hasOldRegistration := resolver.registrationCache["phone-lookup-1"] + _, hasOldSource := resolver.sourceCodeCache["source-lookup-1"] + resolver.lookupMu.Unlock() + if hasOldLookup || hasOldRegistration || hasOldSource { + t.Fatalf("oldest cache entries should be evicted") + } +} + +func TestMySQLResolverBoundsLocationTouchCache(t *testing.T) { + db, _ := newMockDB(t) + defer db.Close() + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + LookupCacheTTL: time.Hour, + CacheCleanupInterval: time.Hour, + MaxCacheEntries: 2, + LocationTouchInterval: time.Hour, + }) + now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC) + resolver.touchMu.Lock() + resolver.locationTouches["old-expired"] = now.Add(-2 * time.Hour) + resolver.locationTouches["phone-1"] = now.Add(-2 * time.Minute) + resolver.locationTouches["phone-2"] = now.Add(-time.Minute) + resolver.locationTouches["phone-3"] = now + resolver.locationTouchFailures["old-failure"] = now.Add(-time.Minute) + resolver.locationTouchFailures["phone-2"] = now.Add(time.Minute) + resolver.locationTouchFailures["phone-3"] = now.Add(2 * time.Minute) + resolver.locationTouchFailures["phone-4"] = now.Add(3 * time.Minute) + resolver.cleanupLocationTouchesLocked(now, false) + resolver.touchMu.Unlock() + + stats := resolver.CacheStats() + if stats.LocationTouchEntries != 2 || stats.LocationTouchFailureEntries != 2 || stats.MaxEntries != 2 { + t.Fatalf("cache stats = %+v, want location touch caches capped at 2", stats) + } + resolver.touchMu.Lock() + _, hasExpired := resolver.locationTouches["old-expired"] + _, hasOldest := resolver.locationTouches["phone-1"] + _, hasExpiredFailure := resolver.locationTouchFailures["old-failure"] + _, hasOldestFailure := resolver.locationTouchFailures["phone-2"] + resolver.touchMu.Unlock() + if hasExpired || hasOldest || hasExpiredFailure || hasOldestFailure { + t.Fatalf("expired and oldest location touch entries should be evicted") + } +} + +type deadlineCheckingResolver struct { + mu sync.Mutex + deadline time.Time +} + +func (r *deadlineCheckingResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope) (envelope.FrameEnvelope, error) { + deadline, _ := ctx.Deadline() + r.mu.Lock() + r.deadline = deadline + r.mu.Unlock() + return env, nil +} + func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) { db, mock := newMockDB(t) defer db.Close() @@ -285,8 +967,18 @@ func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("ALTER TABLE vehicle_identity_binding DROP COLUMN device_id"). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle \\("). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_identifier"). + WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS jt808_registration"). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("ALTER TABLE jt808_registration ADD COLUMN source_ip"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("ALTER TABLE jt808_registration ADD KEY idx_jt808_registration_source_ip"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE jt808_registration"). + WillReturnResult(sqlmock.NewResult(0, 0)) resolver := NewMySQLResolver(db, "vehicle_identity_binding") if err := resolver.EnsureSchema(context.Background()); err != nil { @@ -308,8 +1000,18 @@ func TestMySQLResolverIgnoresExistingOEMColumn(t *testing.T) { WillReturnError(errors.New("Error 1091 (42000): Can't DROP 'uk_identity_device'; check that column/key exists")) mock.ExpectExec("ALTER TABLE vehicle_identity_binding DROP COLUMN device_id"). WillReturnError(errors.New("Error 1091 (42000): Can't DROP 'device_id'; check that column/key exists")) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle \\("). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_identifier"). + WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE TABLE IF NOT EXISTS jt808_registration"). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("ALTER TABLE jt808_registration ADD COLUMN source_ip"). + WillReturnError(errors.New("Error 1060 (42S21): Duplicate column name 'source_ip'")) + mock.ExpectExec("ALTER TABLE jt808_registration ADD KEY idx_jt808_registration_source_ip"). + WillReturnError(errors.New("Error 1061 (42000): Duplicate key name 'idx_jt808_registration_source_ip'")) + mock.ExpectExec("UPDATE jt808_registration"). + WillReturnResult(sqlmock.NewResult(0, 0)) resolver := NewMySQLResolver(db, "vehicle_identity_binding") if err := resolver.EnsureSchema(context.Background()); err != nil { @@ -339,6 +1041,15 @@ func TestIdentitySchemaUsesBusinessKeysOnly(t *testing.T) { if !strings.Contains(registration, "phone VARCHAR(32) PRIMARY KEY") { t.Fatalf("registration table should key by phone:\n%s", registration) } + if !strings.Contains(registration, "source_ip VARCHAR(64)") || !strings.Contains(registration, "idx_jt808_registration_source_ip") { + t.Fatalf("registration table should keep indexed source_ip:\n%s", registration) + } + if !strings.Contains(vehicleIdentifierTableSQL, "PRIMARY KEY (protocol, source_code, identifier_type, identifier_value)") { + t.Fatalf("vehicle identifier should use protocol/source/type/value as key:\n%s", vehicleIdentifierTableSQL) + } + if strings.Contains(vehicleIdentifierTableSQL, "AUTO_INCREMENT") { + t.Fatalf("vehicle identifier should not use surrogate auto increment id:\n%s", vehicleIdentifierTableSQL) + } } func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) { @@ -349,3 +1060,43 @@ func newMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock) { } return db, mock } + +func expectVehicleIdentifierMiss(mock sqlmock.Sqlmock, identifierType string, value string) { + mock.ExpectQuery("SELECT DISTINCT vin"). + WithArgs("JT808", identifierType, value). + WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"})) +} + +func expectVehicleIdentifierHit(mock sqlmock.Sqlmock, identifierType string, value string, vin string) { + expectVehicleIdentifierHitWithSource(mock, identifierType, value, vin, "", "") +} + +func expectVehicleIdentifierHitWithSource(mock sqlmock.Sqlmock, identifierType string, value string, vin string, sourceCode string, platformName string) { + mock.ExpectQuery("SELECT DISTINCT vin"). + WithArgs("JT808", identifierType, value). + WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}).AddRow(vin, sourceCode, platformName)) +} + +func expectSourceCodeHit(mock sqlmock.Sqlmock, sourceIP string, sourceCode string) { + mock.ExpectQuery("SELECT source_code"). + WithArgs("JT808", sourceIP). + WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"}).AddRow(sourceCode, "G7s", "PLATFORM")) +} + +func expectSourceCodeMiss(mock sqlmock.Sqlmock, sourceIP string) { + mock.ExpectQuery("SELECT source_code"). + WithArgs("JT808", sourceIP). + WillReturnRows(sqlmock.NewRows([]string{"source_code", "platform_name", "source_kind"})) +} + +func expectVehicleIdentifierMissForSource(mock sqlmock.Sqlmock, sourceCode string, identifierType string, value string) { + mock.ExpectQuery("SELECT DISTINCT vin"). + WithArgs("JT808", identifierType, value, sourceCode). + WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"})) +} + +func expectVehicleIdentifierHitForSource(mock sqlmock.Sqlmock, sourceCode string, identifierType string, value string, vin string) { + mock.ExpectQuery("SELECT DISTINCT vin"). + WithArgs("JT808", identifierType, value, sourceCode). + WillReturnRows(sqlmock.NewRows([]string{"vin", "source_code", "platform_name"}).AddRow(vin, sourceCode, "G7s")) +} diff --git a/go/vehicle-gateway/internal/identity/snapshot.go b/go/vehicle-gateway/internal/identity/snapshot.go new file mode 100644 index 00000000..732962ea --- /dev/null +++ b/go/vehicle-gateway/internal/identity/snapshot.go @@ -0,0 +1,325 @@ +package identity + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" +) + +type SnapshotRefreshResult struct { + BindingEntries int + IdentifierEntries int + RegistrationEntries int + SourceEntries int + RefreshedAt time.Time +} + +// identitySnapshot is immutable after atomic publication, so frame handling +// performs only local map lookups and never waits for MySQL or a refresh lock. +type identitySnapshot struct { + bindings map[string]string + identifiers map[string]vehicleIdentifierMatch + registrations map[string]registrationCacheEntry + sources map[string]sourceMetadata + refreshedAt time.Time +} + +func (r *MySQLResolver) RefreshSnapshot(ctx context.Context) (SnapshotRefreshResult, error) { + if r == nil || r.db == nil { + return SnapshotRefreshResult{}, fmt.Errorf("identity snapshot database is not configured") + } + r.snapshotRefreshMu.Lock() + defer r.snapshotRefreshMu.Unlock() + + next := &identitySnapshot{ + bindings: map[string]string{}, + identifiers: map[string]vehicleIdentifierMatch{}, + registrations: map[string]registrationCacheEntry{}, + sources: map[string]sourceMetadata{}, + } + if err := r.loadBindingSnapshot(ctx, next); err != nil { + return SnapshotRefreshResult{}, err + } + if err := r.loadIdentifierSnapshot(ctx, next); err != nil { + return SnapshotRefreshResult{}, err + } + if err := r.loadRegistrationSnapshot(ctx, next); err != nil { + return SnapshotRefreshResult{}, err + } + if err := r.loadSourceSnapshot(ctx, next); err != nil { + return SnapshotRefreshResult{}, err + } + next.refreshedAt = time.Now() + + r.snapshot.Store(next) + return SnapshotRefreshResult{ + BindingEntries: len(next.bindings), + IdentifierEntries: len(next.identifiers), + RegistrationEntries: len(next.registrations), + SourceEntries: len(next.sources), + RefreshedAt: next.refreshedAt, + }, nil +} + +func (r *MySQLResolver) loadBindingSnapshot(ctx context.Context, target *identitySnapshot) error { + rows, err := r.db.QueryContext(ctx, "SELECT vin, plate, phone FROM "+r.table+" WHERE vin IS NOT NULL AND TRIM(vin) <> ''") + if err != nil { + return fmt.Errorf("load identity binding snapshot: %w", err) + } + defer rows.Close() + ambiguous := map[string]struct{}{} + for rows.Next() { + var vin, plate, phone sql.NullString + if err := rows.Scan(&vin, &plate, &phone); err != nil { + return fmt.Errorf("scan identity binding snapshot: %w", err) + } + vinValue := strings.TrimSpace(vin.String) + if vinValue == "" { + continue + } + addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("vin", vinValue), vinValue) + if value := strings.TrimSpace(plate.String); value != "" { + addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("plate", value), vinValue) + } + if value := normalizePhone(phone.String); value != "" { + addSnapshotBinding(target.bindings, ambiguous, bindingSnapshotKey("phone", value), vinValue) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate identity binding snapshot: %w", err) + } + return nil +} + +func (r *MySQLResolver) loadIdentifierSnapshot(ctx context.Context, target *identitySnapshot) error { + rows, err := r.db.QueryContext(ctx, `SELECT protocol, source_code, identifier_type, identifier_value, + vin, COALESCE(NULLIF(TRIM(oem), ''), source_code) AS platform_name +FROM vehicle_identifier +WHERE enabled = 1 AND vin IS NOT NULL AND TRIM(vin) <> '' + AND identifier_value IS NOT NULL AND TRIM(identifier_value) <> ''`) + if err != nil { + return fmt.Errorf("load vehicle identifier snapshot: %w", err) + } + defer rows.Close() + ambiguous := map[string]struct{}{} + for rows.Next() { + var protocol, sourceCode, identifierType, identifierValue, vin, platformName sql.NullString + if err := rows.Scan(&protocol, &sourceCode, &identifierType, &identifierValue, &vin, &platformName); err != nil { + return fmt.Errorf("scan vehicle identifier snapshot: %w", err) + } + protocolValue := strings.TrimSpace(protocol.String) + typeValue := strings.ToUpper(strings.TrimSpace(identifierType.String)) + value := normalizeIdentifierValue(typeValue, identifierValue.String) + vinValue := strings.TrimSpace(vin.String) + if protocolValue == "" || typeValue == "" || value == "" || vinValue == "" { + continue + } + match := vehicleIdentifierMatch{ + VIN: vinValue, + SourceCode: strings.TrimSpace(sourceCode.String), + PlatformName: strings.TrimSpace(platformName.String), + } + scopedKey := vehicleIdentifierSnapshotKey(protocolValue, match.SourceCode, typeValue, value) + addSnapshotIdentifier(target.identifiers, ambiguous, scopedKey, match) + globalKey := vehicleIdentifierSnapshotKey(protocolValue, "", typeValue, value) + addSnapshotIdentifier(target.identifiers, ambiguous, globalKey, match) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate vehicle identifier snapshot: %w", err) + } + return nil +} + +func (r *MySQLResolver) loadRegistrationSnapshot(ctx context.Context, target *identitySnapshot) error { + rows, err := r.db.QueryContext(ctx, `SELECT phone, vin, device_id, plate, auth_token +FROM jt808_registration +WHERE phone IS NOT NULL AND TRIM(phone) <> ''`) + if err != nil { + return fmt.Errorf("load jt808 registration snapshot: %w", err) + } + defer rows.Close() + for rows.Next() { + var phone, vin, deviceID, plate, authToken sql.NullString + if err := rows.Scan(&phone, &vin, &deviceID, &plate, &authToken); err != nil { + return fmt.Errorf("scan jt808 registration snapshot: %w", err) + } + phoneValue := normalizePhone(phone.String) + if phoneValue == "" { + continue + } + target.registrations[phoneValue] = registrationCacheEntry{ + vin: strings.TrimSpace(vin.String), + deviceID: strings.TrimSpace(deviceID.String), + plate: strings.TrimSpace(plate.String), + authToken: strings.TrimSpace(authToken.String), + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate jt808 registration snapshot: %w", err) + } + return nil +} + +func (r *MySQLResolver) loadSourceSnapshot(ctx context.Context, target *identitySnapshot) error { + rows, err := r.db.QueryContext(ctx, `SELECT protocol, source_ip, source_code, platform_name, source_kind +FROM vehicle_data_source +WHERE enabled = 1 AND source_ip IS NOT NULL AND TRIM(source_ip) <> '' + AND ( + (source_code IS NOT NULL AND TRIM(source_code) <> '') + OR source_kind IN ('PLATFORM', 'DIRECT') + OR (platform_name IS NOT NULL AND TRIM(platform_name) <> '') + )`) + if err != nil { + if isOptionalSourceCodeLookupError(err) { + return nil + } + return fmt.Errorf("load vehicle data source snapshot: %w", err) + } + defer rows.Close() + for rows.Next() { + var protocol, sourceIP, sourceCode, platformName, sourceKind sql.NullString + if err := rows.Scan(&protocol, &sourceIP, &sourceCode, &platformName, &sourceKind); err != nil { + return fmt.Errorf("scan vehicle data source snapshot: %w", err) + } + key := sourceSnapshotKey(protocol.String, sourceIP.String) + if key == "" { + continue + } + target.sources[key] = sourceMetadata{ + SourceCode: strings.TrimSpace(sourceCode.String), + PlatformName: strings.TrimSpace(platformName.String), + SourceKind: strings.TrimSpace(sourceKind.String), + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate vehicle data source snapshot: %w", err) + } + return nil +} + +func addSnapshotBinding(values map[string]string, ambiguous map[string]struct{}, key string, vin string) { + if key == "" { + return + } + if _, exists := ambiguous[key]; exists { + return + } + if current, exists := values[key]; exists && !strings.EqualFold(current, vin) { + delete(values, key) + ambiguous[key] = struct{}{} + return + } + values[key] = vin +} + +func addSnapshotIdentifier(values map[string]vehicleIdentifierMatch, ambiguous map[string]struct{}, key string, match vehicleIdentifierMatch) { + if key == "" { + return + } + if _, exists := ambiguous[key]; exists { + return + } + current, exists := values[key] + if !exists { + values[key] = match + return + } + if !strings.EqualFold(current.VIN, match.VIN) { + delete(values, key) + ambiguous[key] = struct{}{} + return + } + if !strings.EqualFold(current.SourceCode, match.SourceCode) { + current.SourceCode = "" + current.PlatformName = "" + } else if current.PlatformName != match.PlatformName { + current.PlatformName = "" + } + values[key] = current +} + +func (r *MySQLResolver) snapshotBinding(column string, value string) (string, bool) { + key := bindingSnapshotKey(column, value) + if key == "" { + return "", false + } + snapshot := r.snapshot.Load() + if snapshot == nil { + return "", false + } + vin, ok := snapshot.bindings[key] + return vin, ok +} + +func (r *MySQLResolver) snapshotIdentifier(protocolValue string, sourceCode string, identifierType string, value string) (vehicleIdentifierMatch, bool) { + key := vehicleIdentifierSnapshotKey(protocolValue, sourceCode, identifierType, value) + snapshot := r.snapshot.Load() + if snapshot == nil { + return vehicleIdentifierMatch{}, false + } + match, ok := snapshot.identifiers[key] + return match, ok +} + +func (r *MySQLResolver) snapshotRegistration(phone string) (registrationCacheEntry, bool) { + phone = normalizePhone(phone) + snapshot := r.snapshot.Load() + if snapshot == nil { + return registrationCacheEntry{}, false + } + entry, ok := snapshot.registrations[phone] + return entry, ok +} + +func (r *MySQLResolver) snapshotSource(protocolValue string, sourceIP string) (sourceMetadata, bool) { + key := sourceSnapshotKey(protocolValue, sourceIP) + snapshot := r.snapshot.Load() + if snapshot == nil { + return sourceMetadata{}, false + } + metadata, ok := snapshot.sources[key] + return metadata, ok +} + +// JT808AuthToken serves authentication from the same immutable snapshot used +// by identity resolution. It deliberately never falls back to a per-frame SQL +// query because authentication is on the protocol response hot path. +func (r *MySQLResolver) JT808AuthToken(phone string) (string, bool) { + entry, ok := r.snapshotRegistration(phone) + token := strings.TrimSpace(entry.authToken) + return token, ok && token != "" +} + +func bindingSnapshotKey(column string, value string) string { + column = strings.ToLower(strings.TrimSpace(column)) + value = strings.TrimSpace(value) + if column == "phone" { + value = normalizePhone(value) + } + if value == "" || (column != "vin" && column != "plate" && column != "phone") { + return "" + } + return column + "\x00" + value +} + +func vehicleIdentifierSnapshotKey(protocolValue string, sourceCode string, identifierType string, value string) string { + protocolValue = strings.TrimSpace(protocolValue) + sourceCode = strings.TrimSpace(sourceCode) + identifierType = strings.ToUpper(strings.TrimSpace(identifierType)) + value = normalizeIdentifierValue(identifierType, value) + if protocolValue == "" || identifierType == "" || value == "" { + return "" + } + return "vehicle_identifier\x00" + protocolValue + "\x00" + sourceCode + "\x00" + identifierType + "\x00" + value +} + +func sourceSnapshotKey(protocolValue string, sourceIP string) string { + protocolValue = strings.TrimSpace(protocolValue) + sourceIP = normalizeEndpointIP(sourceIP) + if protocolValue == "" || sourceIP == "" { + return "" + } + return protocolValue + "\x00" + sourceIP +} diff --git a/go/vehicle-gateway/internal/identity/snapshot_test.go b/go/vehicle-gateway/internal/identity/snapshot_test.go new file mode 100644 index 00000000..558d974b --- /dev/null +++ b/go/vehicle-gateway/internal/identity/snapshot_test.go @@ -0,0 +1,166 @@ +package identity + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestRefreshSnapshotResolvesKnownJT808WithoutPerFrameQueries(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectIdentitySnapshot(mock) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SnapshotOnlyLookups: true, + }) + result, err := resolver.RefreshSnapshot(context.Background()) + if err != nil { + t.Fatalf("RefreshSnapshot() error = %v", err) + } + if result.BindingEntries != 3 || result.IdentifierEntries != 2 || result.RegistrationEntries != 1 || result.SourceEntries != 1 { + t.Fatalf("snapshot result = %+v", result) + } + + resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + SourceEndpoint: "115.231.168.135:43625", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if resolved.VIN != "LNBVIN00000000001" { + t.Fatalf("vin = %q", resolved.VIN) + } + if resolved.SourceCode != "g7s" || resolved.PlatformName != "G7s" || resolved.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", resolved.SourceCode, resolved.PlatformName, resolved.SourceKind) + } + identityMetadata, _ := resolved.Parsed["identity"].(map[string]any) + if identityMetadata["cache_status"] != "snapshot" { + t.Fatalf("identity metadata = %#v, want snapshot cache status", identityMetadata) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestSnapshotOnlyResolverMissDoesNotQueryMySQL(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SnapshotOnlyLookups: true, + }) + resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307700000", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if resolved.VIN != "" { + t.Fatalf("vin = %q, want unresolved", resolved.VIN) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("snapshot-only miss should not query mysql: %v", err) + } +} + +func TestRefreshSnapshotFailureKeepsLastKnownGoodSnapshot(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectIdentitySnapshot(mock) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SnapshotOnlyLookups: true, + }) + first, err := resolver.RefreshSnapshot(context.Background()) + if err != nil { + t.Fatalf("first RefreshSnapshot() error = %v", err) + } + mock.ExpectQuery("SELECT vin, plate, phone"). + WillReturnError(errors.New("mysql unavailable")) + if _, err := resolver.RefreshSnapshot(context.Background()); err == nil { + t.Fatal("second RefreshSnapshot() error = nil, want failure") + } + + resolved, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "013307795425", + Parsed: map[string]any{}, + }) + if err != nil { + t.Fatalf("Resolve() after failed refresh error = %v", err) + } + if resolved.VIN != "LNBVIN00000000001" { + t.Fatalf("vin after failed refresh = %q", resolved.VIN) + } + stats := resolver.CacheStats() + if !stats.SnapshotReady || stats.SnapshotRefreshedAt.IsZero() || !stats.SnapshotRefreshedAt.Equal(first.RefreshedAt) { + t.Fatalf("snapshot stats after failed refresh = %+v, first = %+v", stats, first) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func expectIdentitySnapshot(mock sqlmock.Sqlmock) { + mock.ExpectQuery("SELECT vin, plate, phone"). + WillReturnRows(sqlmock.NewRows([]string{"vin", "plate", "phone"}). + AddRow("LNBVIN00000000001", "粤A00001", "13307795425")) + mock.ExpectQuery("SELECT protocol, source_code, identifier_type, identifier_value"). + WillReturnRows(sqlmock.NewRows([]string{"protocol", "source_code", "identifier_type", "identifier_value", "vin", "platform_name"}). + AddRow("JT808", "g7s", "JT808_PHONE", "13307795425", "LNBVIN00000000001", "G7s")) + mock.ExpectQuery("SELECT phone, vin, device_id, plate, auth_token"). + WillReturnRows(sqlmock.NewRows([]string{"phone", "vin", "device_id", "plate", "auth_token"}). + AddRow("13307795425", "LNBVIN00000000001", "DEVICE-1", "粤A00001", "device-code")) + mock.ExpectQuery("SELECT protocol, source_ip, source_code, platform_name, source_kind"). + WillReturnRows(sqlmock.NewRows([]string{"protocol", "source_ip", "source_code", "platform_name", "source_kind"}). + AddRow("JT808", "115.231.168.135", "g7s", "G7s", "PLATFORM")) +} + +func TestSnapshotServesJT808AuthenticationTokenByNormalizedPhone(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectIdentitySnapshot(mock) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SnapshotOnlyLookups: true, + }) + if _, err := resolver.RefreshSnapshot(context.Background()); err != nil { + t.Fatalf("RefreshSnapshot() error = %v", err) + } + token, ok := resolver.JT808AuthToken("0013307795425") + if !ok || token != "device-code" { + t.Fatalf("JT808AuthToken() = %q, %v", token, ok) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestSnapshotResultRefreshedAtUsesCurrentTime(t *testing.T) { + db, mock := newMockDB(t) + defer db.Close() + expectIdentitySnapshot(mock) + + resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{ + SnapshotOnlyLookups: true, + }) + before := time.Now() + result, err := resolver.RefreshSnapshot(context.Background()) + if err != nil { + t.Fatalf("RefreshSnapshot() error = %v", err) + } + if result.RefreshedAt.Before(before) || result.RefreshedAt.After(time.Now()) { + t.Fatalf("refreshed_at = %v, want current time", result.RefreshedAt) + } +} diff --git a/go/vehicle-gateway/internal/loadsim/config.go b/go/vehicle-gateway/internal/loadsim/config.go index 454798fa..4b7ef354 100644 --- a/go/vehicle-gateway/internal/loadsim/config.go +++ b/go/vehicle-gateway/internal/loadsim/config.go @@ -14,6 +14,9 @@ const ( ProtocolGB32960 Protocol = "gb32960" ProtocolJT808 Protocol = "jt808" ProtocolYutongMQTT Protocol = "yutong-mqtt" + + DefaultJT808PhoneBase int64 = 139000000000 + maxJT808Phone int64 = 999999999999 ) type Config struct { @@ -25,17 +28,21 @@ type Config struct { Duration time.Duration Template string SendFrames bool + JT808PhoneBase int64 + DrainResponses bool } type FlagConfig struct { - protocol string - addr string - connections int - connectRate int - sendInterval time.Duration - duration time.Duration - template string - sendFrames bool + protocol string + addr string + connections int + connectRate int + sendInterval time.Duration + duration time.Duration + template string + sendFrames bool + jt808PhoneBase int64 + drainResponses bool } func RegisterFlags(fs *flag.FlagSet) *FlagConfig { @@ -48,6 +55,8 @@ func RegisterFlags(fs *flag.FlagSet) *FlagConfig { fs.DurationVar(&cfg.duration, "duration", 10*time.Minute, "load test duration") fs.StringVar(&cfg.template, "template", "", "frame template name") fs.BoolVar(&cfg.sendFrames, "send", true, "send protocol frames while connections are open") + fs.Int64Var(&cfg.jt808PhoneBase, "jt808-phone-base", DefaultJT808PhoneBase, "first synthetic JT808 phone; one consecutive phone is assigned per connection") + fs.BoolVar(&cfg.drainResponses, "drain-responses", true, "continuously read protocol responses while frames are being sent") return cfg } @@ -73,6 +82,14 @@ func (c *FlagConfig) Build() (Config, error) { if c.duration <= 0 { return Config{}, errors.New("duration must be positive") } + if protocol == ProtocolJT808 { + if c.jt808PhoneBase <= 0 || c.jt808PhoneBase > maxJT808Phone { + return Config{}, errors.New("jt808-phone-base must be a positive 12-digit-or-shorter number") + } + if int64(c.connections-1) > maxJT808Phone-c.jt808PhoneBase { + return Config{}, errors.New("jt808 synthetic phone range exceeds 12 digits") + } + } return Config{ Protocol: protocol, Addr: strings.TrimSpace(c.addr), @@ -82,5 +99,7 @@ func (c *FlagConfig) Build() (Config, error) { Duration: c.duration, Template: strings.TrimSpace(c.template), SendFrames: c.sendFrames, + JT808PhoneBase: c.jt808PhoneBase, + DrainResponses: c.drainResponses, }, nil } diff --git a/go/vehicle-gateway/internal/loadsim/config_test.go b/go/vehicle-gateway/internal/loadsim/config_test.go index d3c968a7..2ce4d961 100644 --- a/go/vehicle-gateway/internal/loadsim/config_test.go +++ b/go/vehicle-gateway/internal/loadsim/config_test.go @@ -18,6 +18,7 @@ func TestConfigFromFlagsParsesCapacityKnobs(t *testing.T) { "-send-interval", "5s", "-duration", "30m", "-template", "0200", + "-jt808-phone-base", "138900000000", "-send=false", }) if err != nil { @@ -53,6 +54,12 @@ func TestConfigFromFlagsParsesCapacityKnobs(t *testing.T) { if got.SendFrames { t.Fatal("SendFrames = true, want false") } + if got.JT808PhoneBase != 138900000000 { + t.Fatalf("JT808PhoneBase = %d, want 138900000000", got.JT808PhoneBase) + } + if !got.DrainResponses { + t.Fatal("DrainResponses = false, want true by default") + } } func TestConfigFromFlagsRejectsUnsafeValues(t *testing.T) { @@ -62,6 +69,7 @@ func TestConfigFromFlagsRejectsUnsafeValues(t *testing.T) { "zero connections": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "0"}, "zero connect rate": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-connect-rate", "0"}, "zero interval": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-send-interval", "0s"}, + "phone overflow": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "2", "-jt808-phone-base", "999999999999"}, } { t.Run(name, func(t *testing.T) { fs := flag.NewFlagSet("load-sim", flag.ContinueOnError) diff --git a/go/vehicle-gateway/internal/loadsim/mutator.go b/go/vehicle-gateway/internal/loadsim/mutator.go index 97f6c6a2..25058ac5 100644 --- a/go/vehicle-gateway/internal/loadsim/mutator.go +++ b/go/vehicle-gateway/internal/loadsim/mutator.go @@ -7,22 +7,30 @@ import ( ) type FrameFactory struct { - protocol Protocol - base []byte + protocol Protocol + base []byte + jt808PhoneBase int64 } func NewFrameFactory(protocol Protocol, template string) (*FrameFactory, error) { + return NewFrameFactoryWithJT808PhoneBase(protocol, template, DefaultJT808PhoneBase) +} + +func NewFrameFactoryWithJT808PhoneBase(protocol Protocol, template string, phoneBase int64) (*FrameFactory, error) { base, err := FrameTemplate(protocol, template) if err != nil { return nil, err } - return &FrameFactory{protocol: protocol, base: base}, nil + if phoneBase <= 0 { + phoneBase = DefaultJT808PhoneBase + } + return &FrameFactory{protocol: protocol, base: base, jt808PhoneBase: phoneBase}, nil } func (f *FrameFactory) Frame(connectionIndex int, frameIndex int64) ([]byte, error) { switch f.protocol { case ProtocolJT808: - return mutateJT808Frame(f.base, connectionIndex, frameIndex) + return mutateJT808Frame(f.base, f.jt808PhoneBase, connectionIndex, frameIndex) case ProtocolGB32960: return mutateGB32960Frame(f.base, connectionIndex, frameIndex) default: @@ -31,7 +39,7 @@ func (f *FrameFactory) Frame(connectionIndex int, frameIndex int64) ([]byte, err } } -func mutateJT808Frame(base []byte, connectionIndex int, frameIndex int64) ([]byte, error) { +func mutateJT808Frame(base []byte, phoneBase int64, connectionIndex int, frameIndex int64) ([]byte, error) { if len(base) < 2 || base[0] != 0x7e || base[len(base)-1] != 0x7e { return nil, fmt.Errorf("jt808 template must include 0x7e delimiters") } @@ -42,7 +50,7 @@ func mutateJT808Frame(base []byte, connectionIndex int, frameIndex int64) ([]byt if len(payload) < 13 { return nil, fmt.Errorf("jt808 template too short: %d", len(payload)) } - phone := fmt.Sprintf("%012d", 139000000000+connectionIndex%100000000) + phone := fmt.Sprintf("%012d", phoneBase+int64(connectionIndex)) copy(payload[4:10], encodeBCD(phone, 6)) binary.BigEndian.PutUint16(payload[10:12], uint16((int(frameIndex)+connectionIndex)%65536)) bodySize := int(binary.BigEndian.Uint16(payload[2:4]) & 0x03ff) diff --git a/go/vehicle-gateway/internal/loadsim/mutator_test.go b/go/vehicle-gateway/internal/loadsim/mutator_test.go index 2b6fc6c4..6d3f1b37 100644 --- a/go/vehicle-gateway/internal/loadsim/mutator_test.go +++ b/go/vehicle-gateway/internal/loadsim/mutator_test.go @@ -38,6 +38,20 @@ func TestFrameFactoryGeneratesUniqueParsableJT808Frames(t *testing.T) { } } +func TestFrameFactoryUsesConfiguredJT808PhoneBase(t *testing.T) { + factory, err := NewFrameFactoryWithJT808PhoneBase(ProtocolJT808, "0200", 138900000000) + if err != nil { + t.Fatalf("NewFrameFactoryWithJT808PhoneBase() error = %v", err) + } + frame, err := factory.Frame(42, 0) + if err != nil { + t.Fatalf("Frame() error = %v", err) + } + if got := parseJT808Frame(t, frame).Phone; got != "138900000042" { + t.Fatalf("phone = %q, want 138900000042", got) + } +} + func TestFrameFactoryGeneratesUniqueParsableGB32960Frames(t *testing.T) { factory, err := NewFrameFactory(ProtocolGB32960, "realtime") if err != nil { diff --git a/go/vehicle-gateway/internal/loadsim/runner.go b/go/vehicle-gateway/internal/loadsim/runner.go index 1e737551..96738c35 100644 --- a/go/vehicle-gateway/internal/loadsim/runner.go +++ b/go/vehicle-gateway/internal/loadsim/runner.go @@ -2,6 +2,8 @@ package loadsim import ( "context" + "errors" + "io" "net" "sync" "sync/atomic" @@ -19,24 +21,32 @@ type Stats struct { ConnectionsFailed int64 FramesWritten int64 WriteErrors int64 + ResponseBytes int64 + ReadErrors int64 } func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) { + phoneBase := cfg.JT808PhoneBase + if phoneBase <= 0 { + phoneBase = DefaultJT808PhoneBase + } if _, err := (&FlagConfig{ - protocol: string(cfg.Protocol), - addr: cfg.Addr, - connections: cfg.Connections, - connectRate: cfg.ConnectRatePerSecond, - sendInterval: cfg.SendInterval, - duration: cfg.Duration, - template: cfg.Template, + protocol: string(cfg.Protocol), + addr: cfg.Addr, + connections: cfg.Connections, + connectRate: cfg.ConnectRatePerSecond, + sendInterval: cfg.SendInterval, + duration: cfg.Duration, + template: cfg.Template, + jt808PhoneBase: phoneBase, + drainResponses: cfg.DrainResponses, }).Build(); err != nil { return Stats{}, err } var factory *FrameFactory if cfg.SendFrames { var err error - factory, err = NewFrameFactory(cfg.Protocol, cfg.Template) + factory, err = NewFrameFactoryWithJT808PhoneBase(cfg.Protocol, cfg.Template, phoneBase) if err != nil { return Stats{}, err } @@ -74,12 +84,24 @@ func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) { connectionIndex++ go func() { defer wg.Done() - defer conn.Close() if cfg.SendFrames { + var readDone chan struct{} + if cfg.DrainResponses { + readDone = make(chan struct{}) + go func() { + drainResponses(runCtx, conn, &stats) + close(readDone) + }() + } writeLoop(runCtx, conn, factory, connIndex, cfg.SendInterval, &stats) + _ = conn.Close() + if readDone != nil { + <-readDone + } return } <-runCtx.Done() + _ = conn.Close() }() } } @@ -88,6 +110,28 @@ func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) { return stats, nil } +func drainResponses(ctx context.Context, conn net.Conn, stats *Stats) { + buffer := make([]byte, 4096) + for { + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + read, err := conn.Read(buffer) + if read > 0 { + atomic.AddInt64(&stats.ResponseBytes, int64(read)) + } + if err == nil { + continue + } + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) { + return + } + if timeout, ok := err.(net.Error); ok && timeout.Timeout() { + continue + } + atomic.AddInt64(&stats.ReadErrors, 1) + return + } +} + func writeLoop(ctx context.Context, conn net.Conn, factory *FrameFactory, connectionIndex int, interval time.Duration, stats *Stats) { for frameIndex := int64(0); ; frameIndex++ { payload, err := factory.Frame(connectionIndex, frameIndex) diff --git a/go/vehicle-gateway/internal/loadsim/runner_test.go b/go/vehicle-gateway/internal/loadsim/runner_test.go index 4823fe5d..37311be6 100644 --- a/go/vehicle-gateway/internal/loadsim/runner_test.go +++ b/go/vehicle-gateway/internal/loadsim/runner_test.go @@ -131,6 +131,48 @@ func TestRunnerCanHoldConnectionsWithoutWritingFrames(t *testing.T) { } } +func TestRunnerDrainsProtocolResponses(t *testing.T) { + runner := Runner{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + client, server := net.Pipe() + go func() { + defer server.Close() + buffer := make([]byte, 4096) + for { + if _, err := server.Read(buffer); err != nil { + return + } + if _, err := server.Write([]byte{0x7e, 0x80, 0x01, 0x7e}); err != nil { + return + } + } + }() + return client, nil + }, + } + + stats, err := runner.Run(context.Background(), Config{ + Protocol: ProtocolJT808, + Addr: "127.0.0.1:808", + Connections: 1, + ConnectRatePerSecond: 1000, + SendInterval: time.Millisecond, + Duration: 10 * time.Millisecond, + Template: "0200", + SendFrames: true, + DrainResponses: true, + }) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if stats.ResponseBytes == 0 { + t.Fatalf("ResponseBytes = 0, want drained protocol responses") + } + if stats.ReadErrors != 0 { + t.Fatalf("ReadErrors = %d, want 0", stats.ReadErrors) + } +} + type recordingConn struct { write func([]byte) (int, error) close func() error diff --git a/go/vehicle-gateway/internal/metrics/metrics.go b/go/vehicle-gateway/internal/metrics/metrics.go index b5507288..06d5ce71 100644 --- a/go/vehicle-gateway/internal/metrics/metrics.go +++ b/go/vehicle-gateway/internal/metrics/metrics.go @@ -4,12 +4,16 @@ import ( "fmt" "math" "net/http" + "runtime" "sort" "strconv" "strings" "sync" + "time" ) +var processStartedAt = time.Now() + type Labels map[string]string type Registry struct { @@ -33,6 +37,56 @@ func NewRegistry() *Registry { } } +func RegisterServiceInfo(registry *Registry, service string) { + if registry == nil { + return + } + service = strings.TrimSpace(service) + if service == "" { + service = "vehicle-service" + } + registry.SetGauge("vehicle_service_info", Labels{"service": service}, 1) + RecordProcessRuntime(registry) +} + +func RecordProcessRuntime(registry *Registry) { + if registry == nil { + return + } + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + registry.SetGauge("vehicle_process_start_time_unix_seconds", nil, float64(processStartedAt.Unix())) + registry.SetGauge("vehicle_process_uptime_seconds", nil, time.Since(processStartedAt).Seconds()) + registry.SetGauge("vehicle_process_heap_alloc_bytes", nil, float64(mem.HeapAlloc)) + registry.SetGauge("vehicle_process_heap_sys_bytes", nil, float64(mem.HeapSys)) + registry.SetGauge("vehicle_process_goroutines", nil, float64(runtime.NumGoroutine())) +} + +func RegisterKafkaConsumerInfo(registry *Registry, service string, group string, topics []string) { + if registry == nil { + return + } + service = strings.TrimSpace(service) + if service == "" { + service = "vehicle-service" + } + group = strings.TrimSpace(group) + if group == "" { + return + } + for _, topic := range topics { + topic = strings.TrimSpace(topic) + if topic == "" { + continue + } + registry.SetGauge("vehicle_kafka_consumer_info", Labels{ + "service": service, + "group": group, + "topic": topic, + }, 1) + } +} + func (r *Registry) IncCounter(name string, labels Labels) { r.AddCounter(name, labels, 1) } @@ -90,6 +144,13 @@ func (r *Registry) SetKafkaLag(name string, topic string, partition int, offset }, float64(lag)) } +func RecordLastActivity(registry *Registry, name string, labels Labels) { + if registry == nil { + return + } + registry.SetGauge(name, labels, float64(time.Now().Unix())) +} + func (r *Registry) ObserveHistogram(name string, labels Labels, buckets []float64, value float64) { name = strings.TrimSpace(name) if r == nil || name == "" { @@ -265,6 +326,7 @@ func NewHandler(registry *Registry) http.Handler { http.NotFound(w, r) return } + RecordProcessRuntime(registry) w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") _, _ = w.Write([]byte(registry.Render())) }) diff --git a/go/vehicle-gateway/internal/metrics/metrics_test.go b/go/vehicle-gateway/internal/metrics/metrics_test.go index f9b92ad7..b0569a56 100644 --- a/go/vehicle-gateway/internal/metrics/metrics_test.go +++ b/go/vehicle-gateway/internal/metrics/metrics_test.go @@ -85,6 +85,65 @@ func TestRegistryRecordsKafkaLagGauge(t *testing.T) { } } +func TestRecordLastActivityRendersUnixGauge(t *testing.T) { + registry := NewRegistry() + + RecordLastActivity(registry, "vehicle_stat_last_message_unix_seconds", Labels{"topic": "vehicle.fields.go.jt808.v1", "status": "received"}) + + text := registry.Render() + if !strings.Contains(text, `# TYPE vehicle_stat_last_message_unix_seconds gauge`) || + !strings.Contains(text, `vehicle_stat_last_message_unix_seconds{status="received",topic="vehicle.fields.go.jt808.v1"} `) { + t.Fatalf("last activity gauge missing:\n%s", text) + } +} + +func TestRegisterServiceInfoRendersStableGauge(t *testing.T) { + registry := NewRegistry() + + RegisterServiceInfo(registry, "vehicle-realtime-api") + + text := registry.Render() + if !strings.Contains(text, `vehicle_service_info{service="vehicle-realtime-api"} 1`) { + t.Fatalf("service info metric missing:\n%s", text) + } + for _, want := range []string{ + `# TYPE vehicle_process_start_time_unix_seconds gauge`, + `# TYPE vehicle_process_uptime_seconds gauge`, + `# TYPE vehicle_process_heap_alloc_bytes gauge`, + `# TYPE vehicle_process_heap_sys_bytes gauge`, + `# TYPE vehicle_process_goroutines gauge`, + `vehicle_process_start_time_unix_seconds `, + `vehicle_process_uptime_seconds `, + `vehicle_process_heap_alloc_bytes `, + `vehicle_process_heap_sys_bytes `, + `vehicle_process_goroutines `, + } { + if !strings.Contains(text, want) { + t.Fatalf("process runtime metric missing %s:\n%s", want, text) + } + } +} + +func TestRegisterKafkaConsumerInfoRendersTopics(t *testing.T) { + registry := NewRegistry() + + RegisterKafkaConsumerInfo(registry, "vehicle-stat-writer", "go-stat-writer", []string{ + "vehicle.fields.go.gb32960.v1", + "vehicle.fields.go.jt808.v1", + "", + }) + + text := registry.Render() + for _, want := range []string{ + `vehicle_kafka_consumer_info{group="go-stat-writer",service="vehicle-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1`, + `vehicle_kafka_consumer_info{group="go-stat-writer",service="vehicle-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("consumer info metric missing %s:\n%s", want, text) + } + } +} + func TestHandlerServesPrometheusText(t *testing.T) { registry := NewRegistry() registry.IncCounter("vehicle_kafka_commits_total", Labels{"service": "history"}) diff --git a/go/vehicle-gateway/internal/metrics/pending.go b/go/vehicle-gateway/internal/metrics/pending.go new file mode 100644 index 00000000..ea8c5ee3 --- /dev/null +++ b/go/vehicle-gateway/internal/metrics/pending.go @@ -0,0 +1,35 @@ +package metrics + +import "sync" + +// PendingPairGauge keeps process-wide in-flight totals correct when a service +// has multiple workers updating the same pair of pending gauges. +type PendingPairGauge struct { + mu sync.Mutex + first int + second int +} + +func (g *PendingPairGauge) Add(registry *Registry, firstMetric string, secondMetric string, firstDelta int, secondDelta int) { + if g == nil { + return + } + g.mu.Lock() + g.first += firstDelta + g.second += secondDelta + if g.first < 0 { + g.first = 0 + } + if g.second < 0 { + g.second = 0 + } + first := g.first + second := g.second + g.mu.Unlock() + + if registry == nil { + return + } + registry.SetGauge(firstMetric, nil, float64(first)) + registry.SetGauge(secondMetric, nil, float64(second)) +} diff --git a/go/vehicle-gateway/internal/metrics/recent_latency.go b/go/vehicle-gateway/internal/metrics/recent_latency.go new file mode 100644 index 00000000..1409ba17 --- /dev/null +++ b/go/vehicle-gateway/internal/metrics/recent_latency.go @@ -0,0 +1,79 @@ +package metrics + +import ( + "math" + "sort" + "strings" + "sync" +) + +// RecentLatencyByKey tracks a bounded in-memory latency window per logical key. +// It is intentionally small and process-local; Prometheus histograms keep the +// lifetime view, while this gives capacity-check a recent-window signal. +type RecentLatencyByKey struct { + mu sync.Mutex + size int + windows map[string]*recentLatencyWindow +} + +type recentLatencyWindow struct { + values []float64 + next int + count int +} + +func NewRecentLatencyByKey(size int) *RecentLatencyByKey { + if size <= 0 { + size = 1 + } + return &RecentLatencyByKey{ + size: size, + windows: map[string]*recentLatencyWindow{}, + } +} + +func (r *RecentLatencyByKey) Observe(key string, value float64) (p99 float64, samples int) { + if r == nil { + return 0, 0 + } + key = strings.TrimSpace(key) + if key == "" { + key = "unknown" + } + if value < 0 || math.IsNaN(value) { + value = 0 + } + if math.IsInf(value, 1) { + value = math.MaxFloat64 + } + + r.mu.Lock() + defer r.mu.Unlock() + window := r.windows[key] + if window == nil { + window = &recentLatencyWindow{values: make([]float64, r.size)} + r.windows[key] = window + } + window.values[window.next] = value + window.next = (window.next + 1) % len(window.values) + if window.count < len(window.values) { + window.count++ + } + return window.quantileLocked(0.99), window.count +} + +func (w *recentLatencyWindow) quantileLocked(q float64) float64 { + if w == nil || w.count == 0 { + return 0 + } + snapshot := append([]float64(nil), w.values[:w.count]...) + sort.Float64s(snapshot) + index := int(math.Ceil(q*float64(len(snapshot)))) - 1 + if index < 0 { + index = 0 + } + if index >= len(snapshot) { + index = len(snapshot) - 1 + } + return snapshot[index] +} diff --git a/go/vehicle-gateway/internal/metrics/recent_latency_test.go b/go/vehicle-gateway/internal/metrics/recent_latency_test.go new file mode 100644 index 00000000..a777a9b7 --- /dev/null +++ b/go/vehicle-gateway/internal/metrics/recent_latency_test.go @@ -0,0 +1,34 @@ +package metrics + +import "testing" + +func TestRecentLatencyByKeyTracksBoundedP99PerKey(t *testing.T) { + tracker := NewRecentLatencyByKey(3) + + p99, samples := tracker.Observe("a", 10) + if p99 != 10 || samples != 1 { + t.Fatalf("first p99=%v samples=%d", p99, samples) + } + tracker.Observe("a", 20) + p99, samples = tracker.Observe("a", 30) + if p99 != 30 || samples != 3 { + t.Fatalf("filled p99=%v samples=%d", p99, samples) + } + p99, samples = tracker.Observe("a", 5) + if p99 != 30 || samples != 3 { + t.Fatalf("bounded p99=%v samples=%d", p99, samples) + } + p99, samples = tracker.Observe("b", 7) + if p99 != 7 || samples != 1 { + t.Fatalf("second key p99=%v samples=%d", p99, samples) + } +} + +func TestRecentLatencyByKeySanitizesBadValues(t *testing.T) { + tracker := NewRecentLatencyByKey(2) + + p99, samples := tracker.Observe("", -10) + if p99 != 0 || samples != 1 { + t.Fatalf("p99=%v samples=%d", p99, samples) + } +} diff --git a/go/vehicle-gateway/internal/observability/logger.go b/go/vehicle-gateway/internal/observability/logger.go index ac54a1f8..e8cdf852 100644 --- a/go/vehicle-gateway/internal/observability/logger.go +++ b/go/vehicle-gateway/internal/observability/logger.go @@ -3,11 +3,28 @@ package observability import ( "log/slog" "os" + "strings" ) // NewLogger returns a JSON logger with a stable service field so ECS logs from // different Go processes can be filtered without relying on container names. func NewLogger(service string) *slog.Logger { - handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true}) + handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + AddSource: true, + Level: parseLogLevel(os.Getenv("LOG_LEVEL")), + }) return slog.New(handler).With("service", service) } + +func parseLogLevel(value string) slog.Level { + switch strings.ToLower(strings.TrimSpace(value)) { + case "debug": + return slog.LevelDebug + case "warn", "warning": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} diff --git a/go/vehicle-gateway/internal/observability/logger_test.go b/go/vehicle-gateway/internal/observability/logger_test.go new file mode 100644 index 00000000..72316a32 --- /dev/null +++ b/go/vehicle-gateway/internal/observability/logger_test.go @@ -0,0 +1,27 @@ +package observability + +import ( + "log/slog" + "testing" +) + +func TestParseLogLevel(t *testing.T) { + tests := []struct { + name string + value string + want slog.Level + }{ + {name: "empty defaults to info", value: "", want: slog.LevelInfo}, + {name: "unknown defaults to info", value: "trace", want: slog.LevelInfo}, + {name: "debug", value: "debug", want: slog.LevelDebug}, + {name: "warn alias", value: "warning", want: slog.LevelWarn}, + {name: "error trims and lowercases", value: " ERROR ", want: slog.LevelError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseLogLevel(tt.value); got != tt.want { + t.Fatalf("parseLogLevel(%q) = %v, want %v", tt.value, got, tt.want) + } + }) + } +} diff --git a/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go b/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go index 05c78f93..fb176df8 100644 --- a/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go +++ b/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go @@ -124,6 +124,28 @@ func TestAutoResponseEchoesRawVINAndOriginalTime(t *testing.T) { } } +func TestAutoResponseRejectsEnforcedInvalidPlatformLogin(t *testing.T) { + body := []byte{0x1a, 0x07, 0x02, 0x00, 0x26, 0x0f, 0x00, 0x01} + body = append(body, fixedASCII("platform-a", 12)...) + body = append(body, fixedASCII("wrong-password", 20)...) + body = append(body, 0x01) + request := buildFrame(0x05, 0xfe, "12345678901234567", body) + env, err := ParseFrame(request, 1782914969584, "127.0.0.1:32960") + if err != nil { + t.Fatal(err) + } + env.AuthenticationEnforced = true + env.AuthenticationStatus = "rejected" + + response, ok, err := AutoResponse(request, env) + if err != nil || !ok { + t.Fatalf("AutoResponse() ok=%v err=%v", ok, err) + } + if response[3] != responseError { + t.Fatalf("response flag = 0x%02x, want 0x%02x", response[3], responseError) + } +} + func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) { body := []byte{0x1a, 0x06, 0x1e, 0x16, 0x17, 0x39} body = append(body, 0x01) diff --git a/go/vehicle-gateway/internal/protocol/gb32960/response.go b/go/vehicle-gateway/internal/protocol/gb32960/response.go index cb320ca8..8f36fbba 100644 --- a/go/vehicle-gateway/internal/protocol/gb32960/response.go +++ b/go/vehicle-gateway/internal/protocol/gb32960/response.go @@ -12,6 +12,7 @@ var ErrResponseFrameTooShort = errors.New("gb32960 response frame too short") const ( responseSuccess = byte(0x01) + responseError = byte(0x02) responseCommand = byte(0xfe) encryptNone = byte(0x01) ) @@ -37,7 +38,11 @@ func AutoResponse(raw []byte, env envelope.FrameEnvelope) ([]byte, bool, error) if timestamp := responseTime(raw, command); !timestamp.IsZero() { body = encodeGBTime(timestamp) } - return buildResponse(raw[0], command, responseSuccess, raw[4:21], body), true, nil + responseFlag := responseSuccess + if command == 0x05 && env.AuthenticationEnforced && env.AuthenticationStatus != "accepted" { + responseFlag = responseError + } + return buildResponse(raw[0], command, responseFlag, raw[4:21], body), true, nil } func shouldRespond(command byte) bool { diff --git a/go/vehicle-gateway/internal/protocol/jt808/parser_test.go b/go/vehicle-gateway/internal/protocol/jt808/parser_test.go index bbca4f01..9b182ecc 100644 --- a/go/vehicle-gateway/internal/protocol/jt808/parser_test.go +++ b/go/vehicle-gateway/internal/protocol/jt808/parser_test.go @@ -416,6 +416,32 @@ func TestAutoResponderBuildsVersionedGeneralAck(t *testing.T) { } } +func TestAutoResponderRejectsEnforcedInvalidAuthentication(t *testing.T) { + request := buildFrame(0x0102, "064646848757", 13, []byte("wrong-code")) + env, err := ParseFrame(request, 1782918600000, "127.0.0.1:808") + if err != nil { + t.Fatal(err) + } + env.AuthenticationEnforced = true + env.AuthenticationStatus = "rejected" + + response, ok, err := NewAutoResponder("issued-code").Respond(request, env) + if err != nil || !ok { + t.Fatalf("Respond() ok=%v err=%v", ok, err) + } + frames, remainder, err := ExtractFrames(response) + if err != nil || len(frames) != 1 || len(remainder) != 0 { + t.Fatalf("ExtractFrames() frames=%d remainder=%x err=%v", len(frames), remainder, err) + } + payload := frames[0] + if got := binary.BigEndian.Uint16(payload[0:2]); got != msgPlatformGeneralResponse { + t.Fatalf("message id = 0x%04x", got) + } + if result := payload[len(payload)-2]; result != 1 { + t.Fatalf("authentication response result = %d, want 1", result) + } +} + func TestExtractFramesHandlesEscapedPayload(t *testing.T) { payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d} frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...) diff --git a/go/vehicle-gateway/internal/protocol/jt808/response.go b/go/vehicle-gateway/internal/protocol/jt808/response.go index b1f5a69b..86817651 100644 --- a/go/vehicle-gateway/internal/protocol/jt808/response.go +++ b/go/vehicle-gateway/internal/protocol/jt808/response.go @@ -49,6 +49,9 @@ func (r AutoResponder) Respond(raw []byte, env envelope.FrameEnvelope) ([]byte, binary.BigEndian.PutUint16(body[0:2], header.sequence) binary.BigEndian.PutUint16(body[2:4], header.messageID) body[4] = 0 + if header.messageID == 0x0102 && env.AuthenticationEnforced && env.AuthenticationStatus != "accepted" { + body[4] = 1 + } return encodeResponse(msgPlatformGeneralResponse, header, body), true, nil } diff --git a/go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go b/go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go index ebfc786c..07fe7e07 100644 --- a/go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go +++ b/go/vehicle-gateway/internal/protocol/yutongmqtt/parser.go @@ -52,6 +52,9 @@ func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS in DeviceID: deviceID, Plate: plate, SourceEndpoint: "mqtt://" + endpoint + topic, + SourceCode: sourceCodeFromEndpoint(endpoint), + PlatformName: platformNameFromEndpoint(endpoint), + SourceKind: "PLATFORM", EventTimeMS: eventTimeMS, ReceivedAtMS: receivedAtMS, RawText: string(payload), @@ -63,6 +66,32 @@ func ParseMessage(endpoint string, topic string, payload []byte, receivedAtMS in return env, nil } +func platformNameFromEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if strings.EqualFold(endpoint, "yutong") { + return "宇通" + } + return endpoint +} + +func sourceCodeFromEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + var builder strings.Builder + for _, r := range endpoint { + switch { + case r >= 'a' && r <= 'z': + builder.WriteRune(r) + case r >= 'A' && r <= 'Z': + builder.WriteRune(r) + case r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '_' || r == '-' || r == '.': + builder.WriteRune(r) + } + } + return builder.String() +} + func fieldsFromData(data map[string]any) map[string]any { fields := map[string]any{} if speed, ok := firstFloat(data, "METER_SPEED", "speed", "speed_kmh"); ok { diff --git a/go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go b/go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go index 7d56993f..802dfcb3 100644 --- a/go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go +++ b/go/vehicle-gateway/internal/protocol/yutongmqtt/parser_test.go @@ -47,6 +47,20 @@ func TestParseMessageMapsYutongPayloadToEnvelope(t *testing.T) { if env.RawText == "" || env.Parsed["data"] == nil { t.Fatalf("raw/parsed missing: %#v", env) } + if env.SourceCode != "endpoint-a" || env.PlatformName != "endpoint-a" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } +} + +func TestParseMessageUsesYutongEndpointDisplayName(t *testing.T) { + payload := []byte(`{"device":"LMRKH9AC3R1004101","time":"20260413100000","data":{"TOTAL_MILEAGE":56905000}}`) + env, err := ParseMessage("yutong", "/ytforward/shln/1", payload, 1782745114999) + if err != nil { + t.Fatalf("ParseMessage() error = %v", err) + } + if env.SourceCode != "yutong" || env.PlatformName != "宇通" || env.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", env.SourceCode, env.PlatformName, env.SourceKind) + } } func TestParseMessageUsesDeviceAsVehicleKeyWhenNotVIN(t *testing.T) { diff --git a/go/vehicle-gateway/internal/realtime/kv.go b/go/vehicle-gateway/internal/realtime/kv.go index 5325dcc1..a8fea939 100644 --- a/go/vehicle-gateway/internal/realtime/kv.go +++ b/go/vehicle-gateway/internal/realtime/kv.go @@ -8,6 +8,7 @@ import ( "strings" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry" ) type RealtimeKVField struct { @@ -26,32 +27,36 @@ func BuildFieldsEnvelope(env envelope.FrameEnvelope) (envelope.FrameEnvelope, bo if !envelope.IsRealtimeTelemetryFrame(env) { return envelope.FrameEnvelope{}, false } - fields, fieldTypes, ok := ParsedFieldsForEnvelope(env) + fields, _, ok := ParsedFieldsForEnvelope(env) if !ok { return envelope.FrameEnvelope{}, false } + filterInvalidRealtimeMeasurementFields(fields) + if len(fields) == 0 || !telemetry.HasRealtimeFields(env.Protocol, fields) { + return envelope.FrameEnvelope{}, false + } out := envelope.FrameEnvelope{ - EventID: env.StableEventID() + ":fields", - TraceID: env.TraceID, - Protocol: env.Protocol, - MessageID: env.MessageID, - Sequence: env.Sequence, - VIN: env.VIN, - VehicleKeyHint: env.VehicleKeyHint, - Phone: env.Phone, - DeviceID: env.DeviceID, - Plate: env.Plate, - SourceEndpoint: env.SourceEndpoint, - EventTimeMS: env.EventTimeMS, - ReceivedAtMS: env.ReceivedAtMS, - Fields: fields, - ParsedFields: fields, - ParsedFieldTypes: fieldTypes, - Parsed: map[string]any{ - "source_event_id": env.StableEventID(), - "field_mapping": realtimeFieldMappingVersion, - }, - ParseStatus: envelope.ParseOK, + EventID: env.StableEventID() + ":fields", + TraceID: env.TraceID, + EventKind: envelope.EventKindFields, + SourceEventID: env.StableEventID(), + FieldMapping: realtimeFieldMappingVersion, + Protocol: env.Protocol, + MessageID: env.MessageID, + Sequence: env.Sequence, + VIN: env.VIN, + VehicleKeyHint: env.VehicleKeyHint, + Phone: env.Phone, + DeviceID: env.DeviceID, + Plate: env.Plate, + SourceEndpoint: env.SourceEndpoint, + SourceCode: env.SourceCode, + PlatformName: env.PlatformName, + SourceKind: env.SourceKind, + EventTimeMS: env.EventTimeMS, + ReceivedAtMS: env.ReceivedAtMS, + Fields: fields, + ParseStatus: envelope.ParseOK, } return out, true } @@ -61,12 +66,34 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool { return false } if len(env.ParsedFields) > 0 { + if derived, derivedTypes, ok := computeParsedFieldsFromParsed(*env); ok { + for field, value := range derived { + if _, exists := env.ParsedFields[field]; !exists { + env.ParsedFields[field] = value + } + } + if env.ParsedFieldTypes == nil { + env.ParsedFieldTypes = map[string]string{} + } + for field, valueType := range derivedTypes { + if _, exists := env.ParsedFieldTypes[field]; !exists { + env.ParsedFieldTypes[field] = valueType + } + } + } + inferredTypes := inferParsedFieldTypes(env.ParsedFields) if env.ParsedFieldTypes == nil { - env.ParsedFieldTypes = inferParsedFieldTypes(env.ParsedFields) + env.ParsedFieldTypes = inferredTypes + } else { + for field, valueType := range inferredTypes { + if _, exists := env.ParsedFieldTypes[field]; !exists { + env.ParsedFieldTypes[field] = valueType + } + } } return true } - fields, fieldTypes, ok := ParsedFieldsForEnvelope(*env) + fields, fieldTypes, ok := ComputeParsedFieldsForEnvelope(*env) if !ok { return false } @@ -76,14 +103,27 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool { } func ParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) { - if len(env.ParsedFields) > 0 { - fields := cloneAnyMap(env.ParsedFields) - fieldTypes := cloneStringMap(env.ParsedFieldTypes) - if len(fieldTypes) == 0 { - fieldTypes = inferParsedFieldTypes(fields) - } + if len(env.ParsedFields) == 0 { + return nil, nil, false + } + fields := cloneAnyMap(env.ParsedFields) + fieldTypes := cloneStringMap(env.ParsedFieldTypes) + if len(fieldTypes) == 0 { + fieldTypes = inferParsedFieldTypes(fields) + } + return fields, fieldTypes, true +} + +// ComputeParsedFieldsForEnvelope is reserved for ingress and offline backfills. +// Runtime projections must consume the precomputed ParsedFields contract instead. +func ComputeParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) { + if fields, fieldTypes, ok := ParsedFieldsForEnvelope(env); ok { return fields, fieldTypes, true } + return computeParsedFieldsFromParsed(env) +} + +func computeParsedFieldsFromParsed(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) { rows := realtimeKVFields(env, env.Parsed) if len(rows) == 0 { return nil, nil, false @@ -119,6 +159,22 @@ func inferParsedFieldTypes(fields map[string]any) map[string]string { return out } +func filterInvalidRealtimeMeasurementFields(fields map[string]any) { + for field, value := range fields { + if isRealtimeTotalMileageField(field) && !positiveNumber(value) { + delete(fields, field) + } + } +} + +func isRealtimeTotalMileageField(field string) bool { + field = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(field), "/", ".")) + return field == envelope.FieldTotalMileageKM || + field == "total_mileage" || + strings.HasSuffix(field, "."+envelope.FieldTotalMileageKM) || + strings.HasSuffix(field, ".total_mileage") +} + func cloneAnyMap(in map[string]any) map[string]any { if len(in) == 0 { return nil @@ -191,8 +247,9 @@ const realtimeFieldMappingVersion = "2026-07-03.v1" func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []RealtimeKVField { vin := strings.TrimSpace(env.VIN) - if vin == "" { - return nil + if env.Protocol == envelope.ProtocolGB32960 && len(parsed) > 0 { + parsed = cloneMap(parsed) + normalizeGB32960RealtimeParsed(parsed) } eventID := env.StableEventID() mapping := realtimeMapping(env.Protocol) @@ -205,6 +262,9 @@ func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []Realt } sort.Strings(names) for _, name := range names { + if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) { + continue + } value, valueType, ok := stringifyKVValue(flat[name]) if !ok { continue @@ -255,6 +315,9 @@ func gb32960KVFields(env envelope.FrameEnvelope, parsed map[string]any, mapping } sort.Strings(names) for _, name := range names { + if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) { + continue + } value, valueType, ok := stringifyKVValue(flat[name]) if !ok { continue @@ -335,6 +398,12 @@ func flattenKV(prefix string, value any, out map[string]any, mapping protocolFie } flattenKV(next, typed[key], out, mapping) } + case map[string]bool: + normalized := make(map[string]any, len(typed)) + for key, item := range typed { + normalized[key] = item + } + flattenKV(prefix, normalized, out, mapping) case []any: for index, item := range typed { itemMap, ok := item.(map[string]any) @@ -397,8 +466,14 @@ func mappedTopLevelDomain(mapping protocolFieldMapping, key string) string { if key == "" { return "" } - name := mapping.TopLevelName[key] - if name == "" { + name := "" + if len(mapping.TopLevelName) > 0 { + var ok bool + name, ok = mapping.TopLevelName[key] + if !ok { + return "" + } + } else { name = sanitizeFieldPart(key) } if name == "" { diff --git a/go/vehicle-gateway/internal/realtime/kv_test.go b/go/vehicle-gateway/internal/realtime/kv_test.go index 43610d75..1fbf773f 100644 --- a/go/vehicle-gateway/internal/realtime/kv_test.go +++ b/go/vehicle-gateway/internal/realtime/kv_test.go @@ -1,6 +1,8 @@ package realtime import ( + "encoding/json" + "strings" "testing" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" @@ -40,7 +42,7 @@ func TestRealtimeKVFieldsFromGB32960ParsedDomains(t *testing.T) { } func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) { - fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{ + env := envelope.FrameEnvelope{ Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02", @@ -61,10 +63,17 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) { }}, }, }, - }) + } + if !EnsureParsedFields(&env) { + t.Fatal("EnsureParsedFields() should compute ingress fields") + } + fieldsEnv, ok := BuildFieldsEnvelope(env) if !ok { t.Fatal("BuildFieldsEnvelope() should emit mapped fields") } + if len(fieldsEnv.ParsedFields) != 0 || len(fieldsEnv.ParsedFieldTypes) != 0 || len(fieldsEnv.Parsed) != 0 { + t.Fatalf("fields envelope should keep only slim fields payload, parsed=%#v parsed_fields=%#v parsed_field_types=%#v", fieldsEnv.Parsed, fieldsEnv.ParsedFields, fieldsEnv.ParsedFieldTypes) + } for _, bareKey := range []string{"charge_status", "soc_percent", "total_mileage_km"} { if _, exists := fieldsEnv.Fields[bareKey]; exists { t.Fatalf("fields envelope should not expose non-protocol bare key %q: %#v", bareKey, fieldsEnv.Fields) @@ -81,6 +90,240 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) { } } +func TestEnsureParsedFieldsKeepsUnknownVINProtocolFieldsAndExcludesAnnotations(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + Phone: "13307795425", + Parsed: map[string]any{ + "registration": map[string]any{ + "manufacturer": "YUTNG", + }, + "identity": map[string]any{ + "resolved": false, + "reason": "no_binding", + }, + }, + ParsedFields: map[string]any{ + "jt808.location.speed_kmh": "12.3", + }, + } + if !EnsureParsedFields(&env) { + t.Fatal("unknown VIN raw frame should still retain parsed fields") + } + if got := env.ParsedFields["jt808.location.speed_kmh"]; got != "12.3" { + t.Fatalf("precomputed field = %#v", got) + } + if got := env.ParsedFields["jt808.registration.manufacturer"]; got != "YUTNG" { + t.Fatalf("merged registration field = %#v", got) + } + if _, exists := env.ParsedFields["jt808.identity.resolved"]; exists { + t.Fatalf("derived identity annotations must not enter protocol fields: %#v", env.ParsedFields) + } + for _, field := range []string{"jt808.location.speed_kmh", "jt808.registration.manufacturer"} { + if env.ParsedFieldTypes[field] == "" { + t.Fatalf("field type missing for %s: %#v", field, env.ParsedFieldTypes) + } + } +} + +func TestBuildFieldsEnvelopeCopiesSourceMetadata(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC3R1004101", + MessageID: "MQTT", + SourceEndpoint: "mqtt://yutong/ytforward/shln/1", + SourceCode: "yutong", + PlatformName: "宇通", + SourceKind: "PLATFORM", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + EventID: "event-yutong", + Fields: map[string]any{ + envelope.FieldTotalMileageKM: 56905, + }, + Parsed: map[string]any{ + "data": map[string]any{"TOTAL_MILEAGE": 56905000}, + }, + } + if !EnsureParsedFields(&env) { + t.Fatal("EnsureParsedFields() should compute ingress fields") + } + fieldsEnv, ok := BuildFieldsEnvelope(env) + if !ok { + t.Fatal("BuildFieldsEnvelope() should emit mapped fields") + } + if fieldsEnv.SourceCode != "yutong" || fieldsEnv.PlatformName != "宇通" || fieldsEnv.SourceKind != "PLATFORM" { + t.Fatalf("source metadata = code:%q platform:%q kind:%q", fieldsEnv.SourceCode, fieldsEnv.PlatformName, fieldsEnv.SourceKind) + } +} + +func TestBuildFieldsEnvelopeDropsNonPositiveTotalMileage(t *testing.T) { + fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + MessageID: "0x0200", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + EventID: "event-1", + ParsedFields: map[string]any{ + "jt808.location.latitude": "31.259555", + "jt808.location.longitude": "119.892413", + "jt808.location.total_mileage_km": "0", + }, + }) + if !ok { + t.Fatal("BuildFieldsEnvelope() should still emit valid fields") + } + if _, exists := fieldsEnv.Fields["jt808.location.total_mileage_km"]; exists { + t.Fatalf("fields envelope should drop non-positive mileage: %#v", fieldsEnv.Fields) + } + if fieldsEnv.Fields["jt808.location.latitude"] != "31.259555" { + t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields) + } +} + +func TestBuildFieldsEnvelopeDropsNonPositiveRawTotalMileage(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC3R1004101", + MessageID: "MQTT", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + EventID: "event-yutong-zero-mileage", + Parsed: map[string]any{ + "data": map[string]any{ + "LATITUDE": 30.590921, + "LONGITUDE": 121.075044, + "TOTAL_MILEAGE": 0, + }, + }, + } + if !EnsureParsedFields(&env) { + t.Fatal("EnsureParsedFields() should compute ingress fields") + } + fieldsEnv, ok := BuildFieldsEnvelope(env) + if !ok { + t.Fatal("BuildFieldsEnvelope() should still emit non-mileage fields") + } + if _, exists := fieldsEnv.Fields["yutong_mqtt.data.total_mileage"]; exists { + t.Fatalf("fields envelope should drop non-positive raw mileage: %#v", fieldsEnv.Fields) + } + if fieldsEnv.Fields["yutong_mqtt.data.latitude"] != "30.590921" { + t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields) + } +} + +func TestBuildFieldsEnvelopeJSONOmitsDuplicateParsedPayload(t *testing.T) { + fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + MessageID: "0x0200", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + EventID: "event-1", + ParsedFields: map[string]any{ + "jt808.location.total_mileage_km": "10241.2", + "jt808.location.speed_kmh": "23", + }, + ParsedFieldTypes: map[string]string{ + "jt808.location.total_mileage_km": "number", + "jt808.location.speed_kmh": "number", + }, + }) + if !ok { + t.Fatal("BuildFieldsEnvelope() should emit mapped fields") + } + data, err := json.Marshal(fieldsEnv) + if err != nil { + t.Fatalf("Marshal(fieldsEnv) error = %v", err) + } + text := string(data) + for _, duplicateKey := range []string{`"parsed_fields"`, `"parsed_field_types"`, `"parsed"`} { + if strings.Contains(text, duplicateKey) { + t.Fatalf("fields JSON should omit duplicate %s payload: %s", duplicateKey, text) + } + } + if !strings.Contains(text, `"fields"`) || !strings.Contains(text, `"source_event_id"`) || !strings.Contains(text, `"field_mapping"`) { + t.Fatalf("fields JSON missing slim payload metadata: %s", text) + } +} + +func TestBuildFieldsEnvelopeOmitsGB32960VendorFragmentFields(t *testing.T) { + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "VIN001", + MessageID: "0x02", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + Parsed: map[string]any{ + "data_units": []any{ + map[string]any{ + "type": "0x30", + "name": "gd_fc_stack", + "value": map[string]any{ + "stack_count": 1, + "summaries": []any{ + map[string]any{ + "cell_count": 432, + "stack_water_outlet_temp_c": 63, + "frame_cell_start": 401, + "frame_cell_count": 32, + "frame_max_cell_voltage_v": 1, + "frame_min_cell_voltage_v": 1, + "hydrogen_inlet_pressure_kpa": 130, + "air_inlet_pressure_kpa": 150, + "stack_water_outlet_temp_extra": "kept", + }, + }, + }, + }, + }, + }, + } + if !EnsureParsedFields(&env) { + t.Fatal("EnsureParsedFields() should compute ingress fields") + } + fieldsEnv, ok := BuildFieldsEnvelope(env) + if !ok { + t.Fatal("BuildFieldsEnvelope() should emit mapped fields") + } + for _, fragmentField := range []string{ + "gb32960.gd_fc_stack.frame_cell_start", + "gb32960.gd_fc_stack.frame_cell_count", + "gb32960.gd_fc_stack.frame_max_cell_voltage_v", + "gb32960.gd_fc_stack.frame_min_cell_voltage_v", + } { + if _, exists := fieldsEnv.Fields[fragmentField]; exists { + t.Fatalf("fields envelope should omit fragment-only field %q: %#v", fragmentField, fieldsEnv.Fields) + } + } + for _, keptField := range []string{ + "gb32960.gd_fc_stack.stack_count", + "gb32960.gd_fc_stack.stack_water_outlet_temp_c", + "gb32960.gd_fc_stack.hydrogen_inlet_pressure_kpa", + } { + if _, exists := fieldsEnv.Fields[keptField]; !exists { + t.Fatalf("fields envelope should keep current-state field %q: %#v", keptField, fieldsEnv.Fields) + } + } +} + +func TestBuildFieldsEnvelopeRequiresIngressParsedFields(t *testing.T) { + _, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + MessageID: "0x0200", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + Parsed: map[string]any{ + "location": map[string]any{"speed_kmh": 99}, + }, + }) + if ok { + t.Fatal("BuildFieldsEnvelope() must not re-flatten parsed payload downstream") + } +} + func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) { jtRows := realtimeKVFields(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, @@ -97,6 +340,10 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) { "latitude": 30.2, "total_mileage_km": 10241.2, "additional": []any{map[string]any{"id": "0x01", "value_hex": "00077235"}}, + "io_status": map[string]any{ + "value": uint16(2), + "bits": map[string]bool{"deep_sleep": false, "sleep": true}, + }, }, }) jtValues := kvMap(jtRows) @@ -105,9 +352,14 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) { } if jtValues["jt808.location/total_mileage_km"] != "10241.2" || jtValues["jt808.location/longitude"] != "121.1" || - jtValues["jt808.location/additional.additional_1.id"] != "0x01" { + jtValues["jt808.location/additional.additional_1.id"] != "0x01" || + jtValues["jt808.location/io_status.bits.deep_sleep"] != "false" || + jtValues["jt808.location/io_status.bits.sleep"] != "true" { t.Fatalf("jt808 location kv missing: %#v", jtValues) } + if _, exists := jtValues["jt808.location/io_status.bits"]; exists { + t.Fatalf("JT808 bit fields must be flattened instead of embedded JSON: %#v", jtValues) + } if jtValues["jt808.location/soc_percent"] != "" { t.Fatalf("jt808 kv should not include standardized env.Fields-only values: %#v", jtValues) } diff --git a/go/vehicle-gateway/internal/realtime/mysql_query.go b/go/vehicle-gateway/internal/realtime/mysql_query.go index 8a0c5845..e68e3369 100644 --- a/go/vehicle-gateway/internal/realtime/mysql_query.go +++ b/go/vehicle-gateway/internal/realtime/mysql_query.go @@ -46,6 +46,7 @@ type LocationRow struct { Longitude float64 `json:"longitude"` SpeedKMH *float64 `json:"speed_kmh,omitempty"` TotalMileageKM *float64 `json:"total_mileage_km,omitempty"` + TotalMileageAt string `json:"total_mileage_event_time,omitempty"` SOCPercent *float64 `json:"soc_percent,omitempty"` AltitudeM *float64 `json:"altitude_m,omitempty"` DirectionDeg *float64 `json:"direction_deg,omitempty"` @@ -124,7 +125,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable query = normalizeRealtimeTableQuery(query) sqlText, args := buildRealtimeSelectSQL( "vehicle_realtime_location", - "protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at", + "protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at", query, ) rows, err := r.db.QueryContext(ctx, sqlText, args...) @@ -136,7 +137,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable out := make([]LocationRow, 0) for rows.Next() { var row LocationRow - var eventTime, receivedAt, updatedAt scanSQLDateTime + var eventTime, mileageAt, receivedAt, updatedAt scanSQLDateTime var speed, mileage, soc, altitude, direction sql.NullFloat64 var alarm, status sql.NullInt64 if err := rows.Scan( @@ -148,6 +149,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable &row.Longitude, &speed, &mileage, + &mileageAt, &soc, &altitude, &direction, @@ -162,6 +164,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable row.EventTime = eventTime.String row.SpeedKMH = nullableFloat(speed) row.TotalMileageKM = nullableFloat(mileage) + row.TotalMileageAt = mileageAt.String row.SOCPercent = nullableFloat(soc) row.AltitudeM = nullableFloat(altitude) row.DirectionDeg = nullableFloat(direction) diff --git a/go/vehicle-gateway/internal/realtime/mysql_query_test.go b/go/vehicle-gateway/internal/realtime/mysql_query_test.go index 17d5adbe..02900437 100644 --- a/go/vehicle-gateway/internal/realtime/mysql_query_test.go +++ b/go/vehicle-gateway/internal/realtime/mysql_query_test.go @@ -61,14 +61,14 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) { mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_location"). WithArgs("JT808", "粤B98765"). WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) - mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location"). + mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location"). WithArgs("JT808", "粤B98765", 10, 10). WillReturnRows(sqlmock.NewRows([]string{ "protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km", - "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at", + "total_mileage_event_time", "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at", }).AddRow( "JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9, - nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04", + "2026-07-02 16:11:02.000", nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04", )) handler := NewLocationQueryHandler(NewLocationQueryRepository(db)) @@ -81,7 +81,7 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } body := response.Body.String() - for _, want := range []string{`"protocol":"JT808"`, `"latitude":30.123456`, `"longitude":120.654321`, `"total_mileage_km":48798.9`, `"offset":10`} { + for _, want := range []string{`"protocol":"JT808"`, `"latitude":30.123456`, `"longitude":120.654321`, `"total_mileage_km":48798.9`, `"total_mileage_event_time":"2026-07-02 16:11:02.000"`, `"offset":10`} { if !strings.Contains(body, want) { t.Fatalf("response missing %s: %s", want, body) } @@ -133,14 +133,14 @@ func TestLocationQueryHandlerSkipsTotalCountByDefault(t *testing.T) { } defer db.Close() - mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location"). + mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location"). WithArgs("JT808", 1, 0). WillReturnRows(sqlmock.NewRows([]string{ "protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km", - "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at", + "total_mileage_event_time", "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at", }).AddRow( "JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9, - nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04", + "2026-07-02 16:11:02.000", nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04", )) handler := NewLocationQueryHandler(NewLocationQueryRepository(db)) diff --git a/go/vehicle-gateway/internal/realtime/openapi.go b/go/vehicle-gateway/internal/realtime/openapi.go index f10c6b54..15c56a6b 100644 --- a/go/vehicle-gateway/internal/realtime/openapi.go +++ b/go/vehicle-gateway/internal/realtime/openapi.go @@ -79,11 +79,358 @@ func realtimeOpenAPISpec() map[string]any { }, }, }, + "/api/stats/daily-metrics": map[string]any{ + "get": map[string]any{ + "summary": "查询车辆每日里程", + "description": "从 vehicle_daily_mileage 查询最终选举后的每日里程，并通过 source_id 关联 vehicle_data_source 返回来源证据。默认不执行 COUNT(*)，total 默认表示本页返回条数；需要精确总数时传 includeTotal=true。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_daily_mileage,vehicle_data_source", + "parameters": dailyMetricParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/sources": map[string]any{ + "get": map[string]any{ + "summary": "查询每日里程候选来源", + "description": "从 vehicle_daily_mileage_source 查询每辆车、每天、每个协议下各来源独立计算出的里程候选，并关联 vehicle_data_source 返回平台、来源类型和可信优先级，用于解释最终日里程为什么选中某个来源。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_daily_mileage_source,vehicle_data_source", + "parameters": dailyMetricSourceParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric source page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourcePage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/sources/quality": map[string]any{ + "get": map[string]any{ + "summary": "汇总每日里程候选质量", + "description": "按日期、协议、quality_status、quality_reason 汇总 vehicle_daily_mileage_source，用于发现某协议或某来源类型是否正在批量产生无基线、异常跳变等候选里程质量问题。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_daily_mileage_source", + "parameters": dailyMetricSourceParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric source quality summary page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourceQualityPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/sources/selection": map[string]any{ + "get": map[string]any{ + "summary": "汇总每日里程来源选举", + "description": "按日期、协议、selection_status、selection_reason 汇总 vehicle_daily_mileage_source，并结合 vehicle_data_source 解释各来源被选中、质量淘汰、禁用淘汰或低优先级未选中的原因。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_daily_mileage_source,vehicle_data_source", + "parameters": dailyMetricSourceParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric source selection summary page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourceSelectionPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/diagnostics": map[string]any{ + "get": map[string]any{ + "summary": "诊断实时在线但日里程缺失", + "description": "从 vehicle_realtime_snapshot/location 按 event_time/received_at 找到指定日期活跃车辆，并关联 vehicle_daily_mileage 与 vehicle_daily_mileage_source，解释车辆有实时数据但日里程缺失的原因。默认不执行 COUNT(*)，total 默认表示本页返回条数；需要精确总数时传 includeTotal=true。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source", + "parameters": dailyMetricDiagnosisParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric diagnostics page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/diagnostics/summary": map[string]any{ + "get": map[string]any{ + "summary": "汇总每日里程诊断", + "description": "按协议汇总指定日期 event_time/received_at 活跃车辆的日里程诊断结果，返回 OK、MISSING_DAILY、NO_SOURCE_SAMPLE、NO_TOTAL_MILEAGE 等分类数量，用于大屏、告警和排障入口。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source", + "parameters": dailyMetricDiagnosisSummaryParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric diagnostics summary", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisSummaryPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/diagnostics/reasons": map[string]any{ + "get": map[string]any{ + "summary": "按原因汇总每日里程诊断", + "description": "按协议、诊断分类和 reason 汇总指定日期 event_time/received_at 活跃车辆的日里程诊断结果，用于大屏和告警直接区分源头未上报、总里程为 0、旧总里程未更新、统计抽取缺失等问题。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source", + "parameters": dailyMetricDiagnosisReasonSummaryParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric diagnostics reason summary", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisReasonSummaryPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/daily-metrics/diagnostics/field-status": map[string]any{ + "get": map[string]any{ + "summary": "按字段状态汇总每日里程诊断", + "description": "按协议、诊断分类、reason 和实时总里程字段状态汇总，直接区分源头没有可用总里程、疑似字段未映射、标准总里程为 0、标准总里程停留在旧日期、统计消费缺失等问题。", + "tags": []string{"Stats API"}, + "x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source", + "parameters": dailyMetricDiagnosisReasonSummaryParameters(), + "responses": map[string]any{ + "200": map[string]any{ + "description": "Daily metric diagnostics field status summary", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisFieldStatusSummaryPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources": map[string]any{ + "get": map[string]any{ + "summary": "查询数据来源", + "description": "查询 vehicle_data_source 中自动发现的协议来源，用于人工维护平台名称、source_code、可信优先级和启停状态。", + "tags": []string{"Data Source API"}, + "x-table": "vehicle_data_source", + "parameters": []map[string]any{ + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, + {"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceKind", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"UNKNOWN", "PLATFORM", "DIRECT"}}, "required": false}, + {"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false}, + {"name": "enabled", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Data source page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DataSourcePage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources/diagnostics": map[string]any{ + "get": map[string]any{ + "summary": "诊断 JT808 来源映射", + "description": "按来源 IP 汇总 jt808_registration 与 vehicle_identifier 的匹配情况，帮助判断缺失 source_code 的 808 来源应维护到哪个平台。", + "tags": []string{"Data Source API"}, + "x-table": "vehicle_data_source,jt808_registration,vehicle_identifier", + "parameters": []map[string]any{ + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"JT808"}, "default": "JT808"}, "required": false}, + {"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean", "default": true}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Data source diagnostics page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DataSourceDiagnosticsPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources/kind-suggestions": map[string]any{ + "get": map[string]any{ + "summary": "建议 JT808 来源类型", + "description": "基于来源活跃时长、注册手机号、vehicle_identifier 匹配和 source_code 情况，对 UNKNOWN 来源给出 PLATFORM/DIRECT/UNKNOWN 的只读建议。", + "tags": []string{"Data Source API"}, + "x-table": "vehicle_data_source,jt808_registration,vehicle_identifier", + "parameters": []map[string]any{ + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"JT808"}, "default": "JT808"}, "required": false}, + {"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceKind", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"UNKNOWN", "PLATFORM", "DIRECT"}, "default": "UNKNOWN"}, "required": false}, + {"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "Data source kind suggestion page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DataSourceDiagnosticsPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources/jt808-identity-gaps": map[string]any{ + "get": map[string]any{ + "summary": "诊断 JT808 未绑定设备", + "description": "列出 jt808_registration 中仍无法通过 phone/plate 关联 VIN 的设备，用于定位 0x0200 no_binding 和补充 vehicle_identifier 映射。", + "tags": []string{"Data Source API"}, + "x-table": "jt808_registration,vehicle_identifier,vehicle_identity_binding,vehicle_data_source", + "parameters": []map[string]any{ + {"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "recentSeconds", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 2592000, "default": 86400}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "JT808 identity gap page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/JT808IdentityGapPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources/jt808-mapping-gaps": map[string]any{ + "get": map[string]any{ + "summary": "诊断 JT808 来源映射缺口", + "description": "列出已配置来源平台但缺少当前 source_code 下 JT808_PHONE 到 VIN 映射的注册手机号，用于维护 vehicle_identifier 并提升 VIN 解析、在线统计和里程来源选举覆盖率。", + "tags": []string{"Data Source API"}, + "x-table": "jt808_registration,vehicle_identifier,vehicle_data_source", + "parameters": []map[string]any{ + {"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "recentSeconds", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 2592000, "default": 86400}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + }, + "responses": map[string]any{ + "200": map[string]any{ + "description": "JT808 mapping gap page", + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/JT808MappingGapPage"}, + }, + }, + }, + }, + }, + }, + "/api/stats/data-sources/{id}": map[string]any{ + "patch": map[string]any{ + "summary": "维护数据来源人工字段", + "description": "只允许更新 platform_name、source_code、source_kind、trust_priority、enabled、remark；source_ip、latest_seen_at 等运行态字段由接入链路自动维护。", + "tags": []string{"Data Source API"}, + "parameters": []map[string]any{ + {"name": "id", "in": "path", "schema": map[string]any{"type": "integer"}, "required": true}, + }, + "requestBody": map[string]any{ + "required": true, + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/DataSourceUpdate"}, + }, + }, + }, + "responses": map[string]any{ + "200": map[string]any{"description": "Updated"}, + }, + }, + }, }, "components": map[string]any{ "schemas": map[string]any{ - "SnapshotPage": pageSchema("#/components/schemas/SnapshotRow"), - "LocationPage": pageSchema("#/components/schemas/LocationRow"), + "SnapshotPage": pageSchema("#/components/schemas/SnapshotRow"), + "LocationPage": pageSchema("#/components/schemas/LocationRow"), + "DailyMetricPage": pageSchema("#/components/schemas/DailyMetricRow"), + "DailyMetricSourcePage": pageSchema("#/components/schemas/DailyMetricSourceRow"), + "DailyMetricSourceQualityPage": pageSchema("#/components/schemas/DailyMetricSourceQualityRow"), + "DailyMetricSourceSelectionPage": pageSchema("#/components/schemas/DailyMetricSourceSelectionRow"), + "DailyMetricDiagnosisPage": pageSchema("#/components/schemas/DailyMetricDiagnosisRow"), + "DailyMetricDiagnosisSummaryPage": map[string]any{ + "type": "object", + "properties": map[string]any{ + "items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisSummaryRow"}}, + "total": integerSchema(3), + "active_total": integerSchema(256), + "actionable_issue_total": integerSchema(6), + }, + }, + "DailyMetricDiagnosisReasonSummaryPage": map[string]any{ + "type": "object", + "properties": map[string]any{ + "items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisReasonSummaryRow"}}, + "total": integerSchema(5), + "vehicle_total": integerSchema(347), + "actionable_issue_total": integerSchema(9), + "pipeline_issue_total": integerSchema(0), + "source_data_issue_total": integerSchema(9), + }, + }, + "DailyMetricDiagnosisFieldStatusSummaryPage": map[string]any{ + "type": "object", + "properties": map[string]any{ + "items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisFieldStatusSummaryRow"}}, + "total": integerSchema(7), + "vehicle_total": integerSchema(347), + "actionable_issue_total": integerSchema(9), + "pipeline_issue_total": integerSchema(2), + "source_data_issue_total": integerSchema(7), + }, + }, + "DataSourcePage": pageSchema("#/components/schemas/DataSourceRow"), + "DataSourceDiagnosticsPage": pageSchema("#/components/schemas/DataSourceDiagnosticRow"), + "JT808IdentityGapPage": pageSchema("#/components/schemas/JT808IdentityGapRow"), + "JT808MappingGapPage": pageSchema("#/components/schemas/JT808MappingGapRow"), "SnapshotRow": map[string]any{ "type": "object", "properties": map[string]any{ @@ -99,22 +446,291 @@ func realtimeOpenAPISpec() map[string]any { "LocationRow": map[string]any{ "type": "object", "properties": map[string]any{ + "protocol": stringSchema("JT808"), + "vin": stringSchema("LKLG7C4E3NA774736"), + "plate": stringSchema("粤B98765"), + "event_time": stringSchema("2026-07-02 16:11:02.000"), + "latitude": numberSchema(30.123456), + "longitude": numberSchema(120.654321), + "speed_kmh": numberSchema(54.3), + "total_mileage_km": numberSchema(48798.9), + "total_mileage_event_time": stringSchema("2026-07-02 16:11:02.000"), + "soc_percent": numberSchema(81.5), + "altitude_m": numberSchema(19.0), + "direction_deg": numberSchema(88.0), + "alarm_flag": integerSchema(0), + "status_flag": integerSchema(3), + "received_at": stringSchema("2026-07-02 16:11:03.000"), + "event_id": stringSchema("event id"), + "updated_at": stringSchema("2026-07-02 16:11:04"), + }, + }, + "DailyMetricRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "vin": stringSchema("LA9GG64L7PBAF4001"), + "stat_date": stringSchema("2026-07-08"), + "protocol": stringSchema("JT808"), + "source_id": integerSchema(3), + "source_ip": stringSchema("115.231.168.135"), + "latest_source_endpoint": stringSchema("115.231.168.135:41561"), + "platform_name": stringSchema("G7 平台"), + "source_code": stringSchema("G7S"), + "source_kind": stringSchema("PLATFORM"), + "daily_mileage_km": numberSchema(23.1), + "latest_total_mileage_km": numberSchema(4123.9), + "updated_at": stringSchema("2026-07-08 13:30:57"), + }, + }, + "DailyMetricSourceRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "vin": stringSchema("LA9GG64L7PBAF4001"), + "stat_date": stringSchema("2026-07-08"), + "protocol": stringSchema("JT808"), + "source_key": stringSchema("JT808:13307795425@115.231.168.135"), + "source_ip": stringSchema("115.231.168.135"), + "source_endpoint": stringSchema("115.231.168.135:41561"), + "phone": stringSchema("13307795425"), + "platform_name": stringSchema("信达"), + "source_id": integerSchema(5), + "source_code": stringSchema("xinda"), + "source_kind": stringSchema("PLATFORM"), + "source_enabled": map[string]any{"type": "boolean", "example": true}, + "trust_priority": integerSchema(10), + "first_total_mileage_km": numberSchema(4100.8), + "latest_total_mileage_km": numberSchema(4123.9), + "daily_mileage_km": numberSchema(23.1), + "sample_count": integerSchema(128), + "first_event_time": stringSchema("2026-07-08 00:01:00"), + "latest_event_time": stringSchema("2026-07-08 23:59:00"), + "quality_status": stringSchema("OK"), + "quality_reason": stringSchema("historical_source_baseline"), + "is_selected": map[string]any{"type": "boolean", "example": true}, + "selection_status": stringSchema("selected"), + "selection_reason": stringSchema("selected_current_projection"), + "selection_action": stringSchema("当前来源已被投影到最终日里程表"), + "updated_at": stringSchema("2026-07-08 23:59:10"), + }, + }, + "DailyMetricSourceQualityRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "stat_date": stringSchema("2026-07-12"), "protocol": stringSchema("JT808"), - "vin": stringSchema("LKLG7C4E3NA774736"), - "plate": stringSchema("粤B98765"), - "event_time": stringSchema("2026-07-02 16:11:02.000"), - "latitude": numberSchema(30.123456), - "longitude": numberSchema(120.654321), - "speed_kmh": numberSchema(54.3), - "total_mileage_km": numberSchema(48798.9), - "soc_percent": numberSchema(81.5), - "altitude_m": numberSchema(19.0), - "direction_deg": numberSchema(88.0), - "alarm_flag": integerSchema(0), - "status_flag": integerSchema(3), - "received_at": stringSchema("2026-07-02 16:11:03.000"), - "event_id": stringSchema("event id"), - "updated_at": stringSchema("2026-07-02 16:11:04"), + "quality_status": stringSchema("INVALID_DELTA"), + "quality_reason": stringSchema("outside_daily_range"), + "source_count": integerSchema(3), + "vehicle_count": integerSchema(3), + "selected_count": integerSchema(0), + "sample_count": integerSchema(128), + "daily_mileage_km": numberSchema(12435.2), + }, + }, + "DailyMetricSourceSelectionRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "stat_date": stringSchema("2026-07-12"), + "protocol": stringSchema("JT808"), + "selection_status": stringSchema("not_selected"), + "selection_reason": stringSchema("lower_trust_priority_or_sample_count"), + "selection_action": stringSchema("来源质量可用，但被更高可信优先级、更多样本或更新时间更新的来源覆盖"), + "source_count": integerSchema(36), + "vehicle_count": integerSchema(31), + "selected_count": integerSchema(0), + "sample_count": integerSchema(3200), + "daily_mileage_km": numberSchema(812.5), + }, + }, + "DailyMetricDiagnosisRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "vin": stringSchema("LA9GG64L7PBAF4001"), + "stat_date": stringSchema("2026-07-12"), + "protocol": stringSchema("JT808"), + "plate": stringSchema("粤A12345"), + "platform_name": stringSchema("信达"), + "peer": stringSchema("115.231.168.135:47849"), + "snapshot_event_time": stringSchema("2026-07-12 10:01:02.000"), + "location_event_time": stringSchema("2026-07-12 10:01:02.000"), + "snapshot_updated_at": stringSchema("2026-07-12 10:01:03"), + "location_updated_at": stringSchema("2026-07-12 10:01:03"), + "realtime_total_mileage_km": numberSchema(25246.6), + "realtime_total_mileage_event_time": stringSchema("2026-07-12 10:01:02.000"), + "daily_mileage_km": numberSchema(0.3), + "daily_latest_total_mileage_km": numberSchema(25246.6), + "source_sample_count": integerSchema(36), + "ok_source_count": integerSchema(1), + "selectable_source_count": integerSchema(1), + "latest_stat_event_time": stringSchema("2026-07-12 10:01:02"), + "quality_statuses": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"OK"}}, + "realtime_field_count": integerSchema(32), + "realtime_sample_fields": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"yutong_mqtt.data.latitude", "yutong_mqtt.data.longitude", "yutong_mqtt.data.meter_speed"}}, + "realtime_mileage_field_status": map[string]any{"type": "string", "enum": dailyMetricMileageFieldStatusEnum(), "example": "no_candidate_mileage_field"}, + "realtime_mileage_candidate_fields": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"yutong_mqtt.data.odometer"}}, + "realtime_mileage_evidence": stringSchema("实时快照已有字段，但没有发现 mileage/odometer/odo 等疑似总里程字段"), + "diagnosis": stringSchema("OK"), + "reason": stringSchema("daily_metric_exists"), + "severity": stringSchema("ok"), + }, + }, + "DailyMetricDiagnosisSummaryRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "stat_date": stringSchema("2026-07-12"), + "protocol": stringSchema("JT808"), + "active_count": integerSchema(244), + "ok_count": integerSchema(244), + "missing_daily_count": integerSchema(0), + "no_source_sample_count": integerSchema(0), + "no_total_mileage_count": integerSchema(0), + "actionable_issue_count": integerSchema(0), + }, + }, + "DailyMetricDiagnosisReasonSummaryRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "stat_date": stringSchema("2026-07-12"), + "protocol": stringSchema("YUTONG_MQTT"), + "diagnosis": stringSchema("NO_TOTAL_MILEAGE"), + "reason": stringSchema("realtime_total_mileage_not_reported_on_stat_date"), + "count": integerSchema(7), + "severity": stringSchema("source_data"), + }, + }, + "DailyMetricDiagnosisFieldStatusSummaryRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "stat_date": stringSchema("2026-07-12"), + "protocol": stringSchema("YUTONG_MQTT"), + "diagnosis": stringSchema("NO_TOTAL_MILEAGE"), + "reason": stringSchema("realtime_total_mileage_missing"), + "realtime_mileage_field_status": map[string]any{"type": "string", "enum": dailyMetricMileageFieldStatusEnum(), "example": "no_candidate_mileage_field"}, + "count": integerSchema(7), + "severity": stringSchema("source_data"), + "field_status_severity": stringSchema("source_data"), + "recommended_operator_action": stringSchema("当日实时数据缺少总里程字段，核对平台是否上报该字段以及协议字段映射是否覆盖"), + "field_status_action": stringSchema("实时字段中没有疑似里程字段，优先向源平台核对是否上报总里程"), + }, + }, + "DataSourceRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": integerSchema(3), + "protocol": stringSchema("JT808"), + "source_ip": stringSchema("115.231.168.135"), + "latest_source_endpoint": stringSchema("115.231.168.135:41561"), + "platform_name": stringSchema("G7 平台"), + "source_code": stringSchema("G7S"), + "source_kind": stringSchema("PLATFORM"), + "trust_priority": integerSchema(10), + "enabled": map[string]any{"type": "boolean", "example": true}, + "first_seen_at": stringSchema("2026-07-09 14:48:30"), + "latest_seen_at": stringSchema("2026-07-12 01:16:04"), + "remark": stringSchema("可信来源"), + "updated_at": stringSchema("2026-07-12 01:16:05"), + }, + }, + "DataSourceDiagnosticRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": integerSchema(242190), + "protocol": stringSchema("JT808"), + "source_ip": stringSchema("117.132.194.31"), + "latest_source_endpoint": stringSchema("117.132.194.31:20471"), + "platform_name": stringSchema("G7 平台"), + "source_code": stringSchema("G7S"), + "source_kind": stringSchema("PLATFORM"), + "first_seen_at": stringSchema("2026-07-12 01:00:00"), + "latest_seen_at": stringSchema("2026-07-12 01:36:26"), + "active_span_seconds": integerSchema(2186), + "latest_seen_age_seconds": integerSchema(1800), + "registration_rows": integerSchema(5), + "phone_count": integerSchema(4), + "unknown_vin_rows": integerSchema(2), + "identifier_matched_phones": integerSchema(3), + "unmapped_phone_count": integerSchema(1), + "identifier_match_ratio": map[string]any{"type": "number", "example": 0.75}, + "configured_source_code_matched_phones": integerSchema(3), + "configured_source_code_platform_name": stringSchema("东方北斗"), + "configured_source_code_conflict": map[string]any{"type": "boolean", "example": false}, + "source_platform_name_mismatch": map[string]any{"type": "boolean", "example": true}, + "matched_source_code_count": integerSchema(1), + "candidate_source_code": stringSchema("g7s"), + "candidate_platform_name": stringSchema("G7s"), + "matched_source_codes": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"g7s"}}, + "matched_platform_names": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"G7s"}}, + "sample_phones": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"13307795425"}}, + "reason": stringSchema("candidate_available"), + "recommended_operator_action": stringSchema("可用候选 source_code，执行 identity-import -sync-data-sources -apply 或在来源管理中确认"), + "suggested_source_kind": stringSchema("PLATFORM"), + "suggestion_confidence": stringSchema("HIGH"), + "suggestion_reason": stringSchema("single_source_code_with_many_phones_or_long_activity"), + }, + }, + "JT808IdentityGapRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "phone": stringSchema("13307795425"), + "device_id": stringSchema("TERM-001"), + "plate": stringSchema("沪A63305F"), + "vin": stringSchema("unknown"), + "source_ip": stringSchema("117.132.194.31"), + "source_endpoint": stringSchema("117.132.194.31:20471"), + "source_code": stringSchema("guangan_beidou"), + "platform_name": stringSchema("广安北斗"), + "source_kind": stringSchema("PLATFORM"), + "first_registered_at": stringSchema("2026-07-12 09:00:00"), + "latest_registered_at": stringSchema("2026-07-12 09:00:00"), + "latest_authenticated_at": stringSchema("2026-07-12 09:00:03"), + "latest_seen_at": stringSchema("2026-07-12 22:24:53"), + "latest_seen_age_seconds": integerSchema(32), + "reason": stringSchema("missing_phone_and_plate_binding"), + "recommended_operator_action": stringSchema("将该 phone 或 plate 维护到 vehicle_identifier，并确认 VIN、source_code、oem"), + "raw_frame_query_path": stringSchema("/api/history/raw-frames?protocol=JT808&phone=13307795425&includeFields=true"), + "data_source_query_path": stringSchema("/api/stats/data-sources?protocol=JT808&sourceIP=117.132.194.31"), + }, + }, + "JT808MappingGapRow": map[string]any{ + "type": "object", + "properties": map[string]any{ + "phone": stringSchema("64646848246"), + "device_id": stringSchema("TERM-001"), + "plate": stringSchema("粤AG18312"), + "vin": stringSchema("LKLG7C4E8NA774778"), + "source_ip": stringSchema("115.159.85.149"), + "source_endpoint": stringSchema("115.159.85.149:16885"), + "source_code": stringSchema("dongfang_beidou"), + "platform_name": stringSchema("G7易流"), + "source_kind": stringSchema("PLATFORM"), + "identifier_vin": stringSchema(""), + "identifier_plate": stringSchema(""), + "matched_source_codes": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"g7s"}}, + "matched_platform_names": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"G7s"}}, + "latest_seen_at": stringSchema("2026-07-12 23:38:22"), + "latest_seen_age_seconds": integerSchema(32), + "reason": stringSchema("missing_source_phone_identifier"), + "recommended_action": stringSchema("按当前来源 source_code 维护 JT808_PHONE 到 VIN 的映射；如平台名与 source_code 不一致，先修正来源配置"), + "suggested_source_code": stringSchema("dongfang_beidou"), + "suggested_platform_name": stringSchema("G7易流"), + "suggested_identifier_type": stringSchema("JT808_PHONE"), + "suggested_identifier_value": stringSchema("64646848246"), + "suggested_vin": stringSchema("LKLG7C4E8NA774778"), + "suggested_plate": stringSchema("粤AG18312"), + "raw_frame_query_path": stringSchema("/api/history/raw-frames?protocol=JT808&phone=64646848246&includeFields=true"), + "data_source_query_path": stringSchema("/api/stats/data-sources?protocol=JT808&sourceIP=115.159.85.149"), + "vehicle_identifier_example": stringSchema("protocol=JT808, source_code=dongfang_beidou, identifier_type=JT808_PHONE, identifier_value=64646848246, vin=LKLG7C4E8NA774778, plate=粤AG18312, oem=G7易流"), + }, + }, + "DataSourceUpdate": map[string]any{ + "type": "object", + "properties": map[string]any{ + "platform_name": stringSchema("G7 平台"), + "source_code": stringSchema("G7S"), + "source_kind": stringSchema("PLATFORM"), + "trust_priority": integerSchema(10), + "enabled": map[string]any{"type": "boolean", "example": true}, + "remark": stringSchema("可信来源"), }, }, }, @@ -122,6 +738,92 @@ func realtimeOpenAPISpec() map[string]any { } } +func dailyMetricSourceParameters() []map[string]any { + parameters := dailyMetricParameters() + parameters = append(parameters, + map[string]any{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + map[string]any{"name": "qualityStatus", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "NO_PREVIOUS_BASELINE", "INVALID_DELTA"}}, "required": false}, + map[string]any{"name": "qualityReason", "in": "query", "schema": map[string]any{"type": "string", "example": "historical_source_baseline"}, "required": false}, + map[string]any{"name": "selected", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false}, + ) + return parameters +} + +func dailyMetricDiagnosisParameters() []map[string]any { + return []map[string]any{ + {"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, + {"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"}, + {"name": "diagnosis", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "MISSING_DAILY", "NO_SOURCE_SAMPLE", "NO_TOTAL_MILEAGE"}}, "required": false}, + {"name": "reason", "in": "query", "schema": map[string]any{"type": "string", "enum": dailyMetricDiagnosisReasonEnum()}, "required": false}, + {"name": "severity", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"ok", "pipeline", "source_data"}}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false, "description": "默认 false，不执行 COUNT(*)；true 时返回精确总数。"}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + } +} + +func dailyMetricDiagnosisSummaryParameters() []map[string]any { + return []map[string]any{ + {"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, + {"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"}, + } +} + +func dailyMetricDiagnosisReasonSummaryParameters() []map[string]any { + return []map[string]any{ + {"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, + {"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"}, + {"name": "diagnosis", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "MISSING_DAILY", "NO_SOURCE_SAMPLE", "NO_TOTAL_MILEAGE"}}, "required": false}, + {"name": "reason", "in": "query", "schema": map[string]any{"type": "string", "enum": dailyMetricDiagnosisReasonEnum()}, "required": false}, + {"name": "severity", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"ok", "pipeline", "source_data"}}, "required": false}, + } +} + +func dailyMetricDiagnosisReasonEnum() []string { + return []string{ + "daily_metric_exists", + "source_samples_all_invalid", + "source_samples_all_excluded", + "source_samples_exist_but_final_metric_missing", + "realtime_location_has_total_mileage_but_no_stat_sample", + "realtime_total_mileage_missing", + "realtime_total_mileage_non_positive", + "realtime_total_mileage_time_missing", + "realtime_total_mileage_not_reported_on_stat_date", + "realtime_active_without_total_mileage", + } +} + +func dailyMetricMileageFieldStatusEnum() []string { + return []string{ + "daily_metric_exists", + "source_sample_exists", + "standard_field_not_consumed", + "standard_field_non_positive", + "standard_field_time_missing", + "standard_field_stale", + "candidate_field_unmapped", + "mapped_protocol_field_without_fresh_evidence", + "no_candidate_mileage_field", + "no_realtime_fields", + } +} + +func dailyMetricParameters() []map[string]any { + return []map[string]any{ + {"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false}, + {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, + {"name": "dateFrom", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false}, + {"name": "dateTo", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false}, + {"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false, "description": "默认 false，不执行 COUNT(*)；true 时返回精确总数。"}, + {"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false}, + {"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false}, + } +} + func commonRealtimeParameters() []map[string]any { return []map[string]any{ {"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false}, diff --git a/go/vehicle-gateway/internal/realtime/openapi_test.go b/go/vehicle-gateway/internal/realtime/openapi_test.go index e86cebd3..cafa9db8 100644 --- a/go/vehicle-gateway/internal/realtime/openapi_test.go +++ b/go/vehicle-gateway/internal/realtime/openapi_test.go @@ -17,17 +17,46 @@ func TestOpenAPIHandlerDocumentsRealtimeSnapshotAndLocation(t *testing.T) { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } body := response.Body.String() - for _, want := range []string{`"/api/realtime/snapshots"`, `"/api/realtime/locations"`, `"vehicle_realtime_snapshot"`, `"vehicle_realtime_location"`} { + for _, want := range []string{ + `"/api/realtime/snapshots"`, + `"/api/realtime/locations"`, + `"/api/stats/daily-metrics"`, + `"/api/stats/daily-metrics/sources"`, + `"/api/stats/daily-metrics/sources/quality"`, + `"/api/stats/daily-metrics/sources/selection"`, + `"/api/stats/daily-metrics/diagnostics"`, + `"/api/stats/daily-metrics/diagnostics/summary"`, + `"/api/stats/daily-metrics/diagnostics/reasons"`, + `"/api/stats/daily-metrics/diagnostics/field-status"`, + `"/api/stats/data-sources"`, + `"/api/stats/data-sources/diagnostics"`, + `"/api/stats/data-sources/kind-suggestions"`, + `"/api/stats/data-sources/jt808-identity-gaps"`, + `"/api/stats/data-sources/jt808-mapping-gaps"`, + `"/api/stats/data-sources/{id}"`, + `"vehicle_realtime_snapshot"`, + `"vehicle_realtime_location"`, + `vehicle_daily_mileage`, + `vehicle_daily_mileage_source`, + `"vehicle_data_source"`, + `"Stats API"`, + `"Data Source API"`, + } { if !strings.Contains(body, want) { t.Fatalf("openapi missing %s: %s", want, body) } } + for _, want := range []string{`"daily_mileage_km"`, `"latest_total_mileage_km"`, `"source_key"`, `"source_count"`, `"vehicle_count"`, `"selected_count"`, `"sample_count"`, `"quality_status"`, `"quality_reason"`, `"is_selected"`, `"selection_status"`, `"selection_reason"`, `"selection_action"`, `"lower_trust_priority_or_sample_count"`, `"platform_name"`, `"source_code"`, `"source_kind"`, `"sourceKind"`, `"trust_priority"`, `"enabled"`, `"latest_seen_at"`, `"candidate_source_code"`, `"recommended_operator_action"`, `"suggested_source_kind"`, `"suggestion_confidence"`, `"NO_SOURCE_SAMPLE"`, `"NO_TOTAL_MILEAGE"`, `"source_sample_count"`, `"realtime_total_mileage_km"`, `"total_mileage_event_time"`, `"realtime_total_mileage_event_time"`, `"active_count"`, `"actionable_issue_count"`, `"vehicle_total"`, `"pipeline_issue_total"`, `"source_data_issue_total"`, `"severity"`, `"field_status_severity"`, `"realtime_total_mileage_not_reported_on_stat_date"`, `"realtime_mileage_field_status"`, `"field_status_action"`, `"candidate_field_unmapped"`, `"standard_field_stale"`, `"missing_phone_and_plate_binding"`, `"missing_source_phone_identifier"`, `"configured_source_code_platform_name"`, `"source_platform_name_mismatch"`, `"suggested_identifier_value"`, `"vehicle_identifier_example"`, `"raw_frame_query_path"`, `"data_source_query_path"`} { + if !strings.Contains(body, want) { + t.Fatalf("openapi should document data source field %s: %s", want, body) + } + } for _, removed := range []string{`"/api/realtime/kv"`, `"vehicle_realtime_kv"`, `"Realtime KV API"`} { if strings.Contains(body, removed) { t.Fatalf("openapi should not expose removed MySQL realtime kv API %s: %s", removed, body) } } - for _, want := range []string{`"includeTotal"`, `默认不执行 COUNT(*)`, `total 默认表示本页返回条数`} { + for _, want := range []string{`"includeTotal"`, `"sourceCodeMissing"`, `默认不执行 COUNT(*)`, `total 默认表示本页返回条数`} { if !strings.Contains(body, want) { t.Fatalf("openapi should document lightweight total semantics, missing %s: %s", want, body) } diff --git a/go/vehicle-gateway/internal/realtime/repository.go b/go/vehicle-gateway/internal/realtime/repository.go index ad491ea5..2693d62d 100644 --- a/go/vehicle-gateway/internal/realtime/repository.go +++ b/go/vehicle-gateway/internal/realtime/repository.go @@ -20,6 +20,32 @@ type Repository struct { cfg Config } +type FastUpdateResult struct { + EnvelopesSeen int + EnvelopesUpdated int + EnvelopesSkippedNonRealtime int + EnvelopesSkippedMissingVIN int + EnvelopesSkippedMissingVehicleKey int + EnvelopesSkippedMissingFields int + FieldsSeen int + FieldsWritten int + FieldsSkippedStale int +} + +func (r FastUpdateResult) Add(other FastUpdateResult) FastUpdateResult { + return FastUpdateResult{ + EnvelopesSeen: r.EnvelopesSeen + other.EnvelopesSeen, + EnvelopesUpdated: r.EnvelopesUpdated + other.EnvelopesUpdated, + EnvelopesSkippedNonRealtime: r.EnvelopesSkippedNonRealtime + other.EnvelopesSkippedNonRealtime, + EnvelopesSkippedMissingVIN: r.EnvelopesSkippedMissingVIN + other.EnvelopesSkippedMissingVIN, + EnvelopesSkippedMissingVehicleKey: r.EnvelopesSkippedMissingVehicleKey + other.EnvelopesSkippedMissingVehicleKey, + EnvelopesSkippedMissingFields: r.EnvelopesSkippedMissingFields + other.EnvelopesSkippedMissingFields, + FieldsSeen: r.FieldsSeen + other.FieldsSeen, + FieldsWritten: r.FieldsWritten + other.FieldsWritten, + FieldsSkippedStale: r.FieldsSkippedStale + other.FieldsSkippedStale, + } +} + func NewRepository(client *redis.Client, cfg Config) *Repository { if client == nil { panic("redis client must not be nil") @@ -28,48 +54,82 @@ func NewRepository(client *redis.Client, cfg Config) *Repository { } func (r *Repository) FastUpdate(ctx context.Context, env envelope.FrameEnvelope) error { + _, err := r.FastUpdateWithResult(ctx, env) + return err +} + +func (r *Repository) FastUpdateWithResult(ctx context.Context, env envelope.FrameEnvelope) (FastUpdateResult, error) { + result := FastUpdateResult{EnvelopesSeen: 1} if !envelope.IsRealtimeTelemetryFrame(env) { - return nil + result.EnvelopesSkippedNonRealtime = 1 + return result, nil } vin := strings.TrimSpace(env.VIN) if vin == "" { - return nil + result.EnvelopesSkippedMissingVIN = 1 + return result, nil } vehicleKey := strings.TrimSpace(env.VehicleKey()) if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") { - return nil + result.EnvelopesSkippedMissingVehicleKey = 1 + return result, nil + } + if len(env.ParsedFields) == 0 { + result.EnvelopesSkippedMissingFields = 1 + return result, nil } return r.setFastProjection(ctx, vehicleKey, vin, env) } func (r *Repository) FastUpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { + _, err := r.FastUpdateBatchWithResult(ctx, envs) + return err +} + +func (r *Repository) FastUpdateBatchWithResult(ctx context.Context, envs []envelope.FrameEnvelope) (FastUpdateResult, error) { if len(envs) == 0 { - return nil + return FastUpdateResult{}, nil } pipe := r.client.Pipeline() - queued := 0 + var result FastUpdateResult + queued := make([]queuedFastProjection, 0, len(envs)) for _, env := range envs { + result.EnvelopesSeen++ if !envelope.IsRealtimeTelemetryFrame(env) { + result.EnvelopesSkippedNonRealtime++ continue } vin := strings.TrimSpace(env.VIN) if vin == "" { + result.EnvelopesSkippedMissingVIN++ continue } vehicleKey := strings.TrimSpace(env.VehicleKey()) if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") { + result.EnvelopesSkippedMissingVehicleKey++ continue } - if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil { - return err + if len(env.ParsedFields) == 0 { + result.EnvelopesSkippedMissingFields++ + continue } - queued++ + queuedProjection, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env) + if err != nil { + return FastUpdateResult{}, err + } + queued = append(queued, queuedProjection) } - if queued == 0 { - return nil + if len(queued) == 0 { + return result, nil } _, err := pipe.Exec(ctx) - return err + if err != nil { + return FastUpdateResult{}, err + } + for _, item := range queued { + result = result.Add(item.result()) + } + return result, nil } func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error { @@ -85,10 +145,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err return nil } nowMS := time.Now().UnixMilli() - eventMS := env.EventTimeMS - if eventMS <= 0 { - eventMS = env.ReceivedAtMS - } + eventMS, _ := envelope.NormalizedEventTimeMS(env) protocolSnapshot := Snapshot{ VehicleKey: vehicleKey, VIN: vin, @@ -129,7 +186,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err if err := r.setJSON(ctx, realtimeRawKey(vehicleKey, env.Protocol), protocolSnapshot.Parsed, r.cfg.ttl()); err != nil { return err } - if err := r.setKV(ctx, vin, env, protocolSnapshot.Parsed); err != nil { + if err := r.setKV(ctx, vin, env); err != nil { return err } @@ -377,10 +434,7 @@ func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl tim return r.client.Set(ctx, key, payload, ttl).Err() } -func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope, parsed map[string]any) error { - if len(env.ParsedFields) == 0 && len(parsed) > 0 { - env.Parsed = parsed - } +func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope) error { values, types := realtimeKVMapsForEnvelope(env) if len(values) == 0 { return nil @@ -394,24 +448,43 @@ func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEn "source_endpoint": env.SourceEndpoint, "field_mapping": realtimeFieldMappingVersion, } - pipe := r.client.Pipeline() - pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values) - pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types) - pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta) - _, err := pipe.Exec(ctx) - return err + return evalGuardedRealtimeKV(ctx, r.client, env.Protocol, vin, eventTimeOrReceivedMS(env), values, types, meta).Err() } -func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) error { +func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) (FastUpdateResult, error) { pipe := r.client.Pipeline() - if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil { - return err + queued, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env) + if err != nil { + return FastUpdateResult{}, err } - _, err := pipe.Exec(ctx) - return err + if _, err := pipe.Exec(ctx); err != nil { + return FastUpdateResult{}, err + } + result := queued.result() + result.EnvelopesSeen = 1 + return result, nil } -func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) error { +type queuedFastProjection struct { + fieldsSeen int + writeCmd *redis.Cmd +} + +func (q queuedFastProjection) result() FastUpdateResult { + written := redisCmdInt(q.writeCmd) + skipped := q.fieldsSeen - written + if skipped < 0 { + skipped = 0 + } + return FastUpdateResult{ + EnvelopesUpdated: 1, + FieldsSeen: q.fieldsSeen, + FieldsWritten: written, + FieldsSkippedStale: skipped, + } +} + +func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) (queuedFastProjection, error) { values, types := realtimeKVMapsForEnvelope(env) eventTimeMS := eventTimeOrReceivedMS(env) offlineAfterMS := env.ReceivedAtMS + r.cfg.ttl().Milliseconds() @@ -428,7 +501,7 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin } payload, err := json.Marshal(online) if err != nil { - return err + return queuedFastProjection{}, err } meta := map[string]any{ "event_time_ms": strconv.FormatInt(eventTimeMS, 10), @@ -449,16 +522,171 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin "ttl_seconds": strconv.FormatInt(int64(r.cfg.ttl().Seconds()), 10), "source_endpoint": env.SourceEndpoint, } + queued := queuedFastProjection{fieldsSeen: len(values)} if len(values) > 0 { - pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values) - pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types) - pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta) + queued.writeCmd = evalGuardedRealtimeKV(ctx, pipe, env.Protocol, vin, eventTimeMS, values, types, meta) } - pipe.Set(ctx, onlineKey(env.Protocol, vin), payload, r.cfg.ttl()) - pipe.HSet(ctx, onlineStateKey(env.Protocol, vin), state) - pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: onlineMember(env.Protocol, vin)}) - pipe.SAdd(ctx, realtimeIndexKey(env.Protocol), vin) - return nil + if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil { + return queuedFastProjection{}, err + } + return queued, nil +} + +const guardedRealtimeKVScript = ` +local incoming = tonumber(ARGV[1]) or 0 +local value_count = tonumber(ARGV[2]) or 0 +local idx = 3 +local written = 0 + +for i = 1, value_count do + local field = ARGV[idx] + local value = ARGV[idx + 1] + idx = idx + 2 + local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0 + if current <= incoming then + redis.call('HSET', KEYS[1], field, value) + redis.call('HSET', KEYS[3], field, incoming) + written = written + 1 + end +end + +local type_count = tonumber(ARGV[idx]) or 0 +idx = idx + 1 +for i = 1, type_count do + local field = ARGV[idx] + local value_type = ARGV[idx + 1] + idx = idx + 2 + local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0 + if current <= incoming then + redis.call('HSET', KEYS[2], field, value_type) + end +end + +local meta_count = tonumber(ARGV[idx]) or 0 +idx = idx + 1 +local current_meta = tonumber(redis.call('HGET', KEYS[4], 'event_time_ms') or '') or 0 +if current_meta <= incoming then + for i = 1, meta_count do + redis.call('HSET', KEYS[4], ARGV[idx], ARGV[idx + 1]) + idx = idx + 2 + end +end + +return written +` + +const guardedOnlineStatusScript = ` +local incoming = tonumber(ARGV[1]) or 0 +local ttl_ms = tonumber(ARGV[2]) or 60000 +local payload = ARGV[3] +local member = ARGV[4] +local vin = ARGV[5] +local state_count = tonumber(ARGV[6]) or 0 +local idx = 7 +local current = tonumber(redis.call('HGET', KEYS[2], 'last_seen_ms') or '') or 0 + +redis.call('SADD', KEYS[4], vin) +if current <= incoming then + redis.call('SET', KEYS[1], payload, 'PX', ttl_ms) + for i = 1, state_count do + redis.call('HSET', KEYS[2], ARGV[idx], ARGV[idx + 1]) + idx = idx + 2 + end + redis.call('ZADD', KEYS[3], incoming, member) + return 1 +end +return 0 +` + +type redisEvaler interface { + Eval(ctx context.Context, script string, keys []string, args ...any) *redis.Cmd +} + +func redisCmdInt(cmd *redis.Cmd) int { + if cmd == nil { + return 0 + } + value, err := cmd.Int64() + if err != nil { + return 0 + } + return int(value) +} + +func evalGuardedRealtimeKV(ctx context.Context, evaler redisEvaler, protocol envelope.Protocol, vin string, eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) *redis.Cmd { + keys := []string{ + realtimeKVValuesKey(protocol, vin), + realtimeKVTypesKey(protocol, vin), + realtimeKVTimesKey(protocol, vin), + realtimeKVMetaKey(protocol, vin), + } + args := guardedRealtimeKVArgs(eventTimeMS, values, types, meta) + return evaler.Eval(ctx, guardedRealtimeKVScript, keys, args...) +} + +func evalGuardedOnlineStatus(ctx context.Context, evaler redisEvaler, online OnlineStatus, state map[string]any, payload []byte, ttl time.Duration) (*redis.Cmd, error) { + protocol := online.Protocol + if protocol == "" && len(online.Protocols) > 0 { + protocol = online.Protocols[0] + } + vin := strings.TrimSpace(online.VIN) + if protocol == "" || vin == "" { + return nil, nil + } + if len(payload) == 0 { + var err error + payload, err = json.Marshal(online) + if err != nil { + return nil, err + } + } + if ttl <= 0 { + ttl = time.Minute + } + keys := []string{ + onlineKey(protocol, vin), + onlineStateKey(protocol, vin), + "vehicle:last_seen", + realtimeIndexKey(protocol), + } + args := guardedOnlineStatusArgs(online, protocol, state, payload, ttl) + return evaler.Eval(ctx, guardedOnlineStatusScript, keys, args...), nil +} + +func guardedOnlineStatusArgs(online OnlineStatus, protocol envelope.Protocol, state map[string]any, payload []byte, ttl time.Duration) []any { + args := []any{ + strconv.FormatInt(online.LastSeenMS, 10), + strconv.FormatInt(ttl.Milliseconds(), 10), + string(payload), + onlineMember(protocol, online.VIN), + strings.TrimSpace(online.VIN), + } + return appendMapPairs(args, state) +} + +func guardedRealtimeKVArgs(eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) []any { + args := []any{strconv.FormatInt(eventTimeMS, 10)} + args = appendMapPairs(args, values) + args = appendMapPairs(args, types) + args = appendMapPairs(args, meta) + return args +} + +func appendMapPairs(args []any, values map[string]any) []any { + keys := make([]string, 0, len(values)) + for key := range values { + key = strings.TrimSpace(key) + if key == "" { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + args = append(args, strconv.Itoa(len(keys))) + for _, key := range keys { + args = append(args, key, fmt.Sprint(values[key])) + } + return args } func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]any) { @@ -469,6 +697,9 @@ func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[ if strings.TrimSpace(field) == "" { continue } + if isRealtimeTotalMileageField(field) && !positiveNumber(value) { + continue + } stringValue, valueType, ok := stringifyKVValue(value) if !ok { continue @@ -528,7 +759,6 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e if online.OfflineAfterMS <= 0 { online.OfflineAfterMS = online.LastSeenMS + r.cfg.ttl().Milliseconds() } - member := onlineMember(protocol, online.VIN) state := map[string]any{ "vehicle_key": online.VehicleKey, "vin": online.VIN, @@ -544,10 +774,9 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e if err != nil { return err } - pipe.Set(ctx, onlineKey(protocol, online.VIN), payload, r.cfg.ttl()) - pipe.HSet(ctx, onlineStateKey(protocol, online.VIN), state) - pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(online.LastSeenMS), Member: member}) - pipe.SAdd(ctx, realtimeIndexKey(protocol), online.VIN) + if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil { + return err + } _, err = pipe.Exec(ctx) return err } @@ -879,6 +1108,10 @@ func realtimeKVTypesKey(protocol envelope.Protocol, vin string) string { return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":types" } +func realtimeKVTimesKey(protocol envelope.Protocol, vin string) string { + return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":times" +} + func realtimeKVMetaKey(protocol envelope.Protocol, vin string) string { return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":meta" } @@ -897,10 +1130,8 @@ func realtimeKVFieldPath(domain string, field string) string { } func eventTimeOrReceivedMS(env envelope.FrameEnvelope) int64 { - if env.EventTimeMS > 0 { - return env.EventTimeMS - } - return env.ReceivedAtMS + eventMS, _ := envelope.NormalizedEventTimeMS(env) + return eventMS } func onlineKey(protocol envelope.Protocol, vin string) string { diff --git a/go/vehicle-gateway/internal/realtime/repository_test.go b/go/vehicle-gateway/internal/realtime/repository_test.go index f27a0db2..de318b5e 100644 --- a/go/vehicle-gateway/internal/realtime/repository_test.go +++ b/go/vehicle-gateway/internal/realtime/repository_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -133,6 +134,50 @@ func TestRepositoryStoresFullParsedOnlyInRealtimeRawAndMergesGB32960Units(t *tes } } +func TestRepositoryNormalizesFarFutureEventTime(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli() + futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC).UnixMilli() + + if err := repo.Update(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: futureEvent, + ReceivedAtMS: received, + Parsed: map[string]any{"location": map[string]any{"longitude": 121.1}}, + ParsedFields: map[string]any{ + "jt808.location.longitude": 121.1, + "jt808.location.latitude": 30.1, + "jt808.location.total_mileage_km": 10241.2, + }, + Fields: map[string]any{ + envelope.FieldLongitude: 121.1, + envelope.FieldLatitude: 30.1, + envelope.FieldTotalMileageKM: 10241.2, + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + + protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808) + if err != nil { + t.Fatalf("GetProtocol() error = %v", err) + } + if protocol.EventTimeMS != received { + t.Fatalf("protocol event time = %d, want received %d", protocol.EventTimeMS, received) + } + meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result() + if err != nil { + t.Fatalf("HGetAll meta error = %v", err) + } + if meta["event_time_ms"] != strconv.FormatInt(received, 10) { + t.Fatalf("rt-kv event_time_ms = %q, want received %d", meta["event_time_ms"], received) + } +} + func TestRepositoryMergesNestedGB32960MotorSlicesBySerialNo(t *testing.T) { repo, closeFn := newTestRepository(t) defer closeFn() @@ -337,6 +382,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) { }}, }, }, + ParsedFields: map[string]any{ + "gb32960.vehicle.type": "0x01", + "gb32960.vehicle.soc_percent": 88.0, + "gb32960.gd_fc_stack.type": "0x30", + "gb32960.gd_fc_stack.stack_count": 1, + "gb32960.gd_fc_stack.stack_water_outlet_temp_c": 63, + }, }); err != nil { t.Fatalf("Update() error = %v", err) } @@ -358,6 +410,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) { if typeKV["gb32960.vehicle.soc_percent"] != "number" || typeKV["gb32960.gd_fc_stack.type"] != "string" { t.Fatalf("types kv = %#v", typeKV) } + timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result() + if err != nil { + t.Fatalf("times kv HGetAll error = %v", err) + } + if timeKV["gb32960.vehicle.soc_percent"] != "1000" || timeKV["gb32960.gd_fc_stack.stack_water_outlet_temp_c"] != "1000" { + t.Fatalf("times kv = %#v", timeKV) + } metaKV, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:meta").Result() if err != nil { t.Fatalf("meta kv HGetAll error = %v", err) @@ -388,6 +447,10 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T) map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}, }, }, + ParsedFields: map[string]any{ + "gb32960.vehicle.type": "0x01", + "gb32960.vehicle.soc_percent": 88.0, + }, }); err != nil { t.Fatalf("FastUpdate() error = %v", err) } @@ -406,9 +469,19 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T) if typeKV["gb32960.vehicle.soc_percent"] != "number" { t.Fatalf("types kv = %#v", typeKV) } + timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result() + if err != nil { + t.Fatalf("times kv HGetAll error = %v", err) + } + if timeKV["gb32960.vehicle.soc_percent"] != "1000" { + t.Fatalf("times kv = %#v", timeKV) + } if ttl := repo.client.TTL(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val(); ttl != -1 { t.Fatalf("kv ttl should not expire, got %v", ttl) } + if ttl := repo.client.TTL(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Val(); ttl != -1 { + t.Fatalf("kv times ttl should not expire, got %v", ttl) + } if ttl := repo.client.TTL(ctx, "vehicle:online:GB32960:VIN001").Val(); ttl <= 0 || ttl > time.Minute { t.Fatalf("online ttl = %v, want within 1 minute", ttl) } @@ -430,12 +503,138 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T) } } +func TestRepositoryFastUpdateDoesNotLetOlderFramesOverwriteNewerFields(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + + firstResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 2000, + ReceivedAtMS: 2100, + ParsedFields: map[string]any{ + "jt808.location.speed_kmh": 50, + }, + }) + if err != nil { + t.Fatalf("newer FastUpdate() error = %v", err) + } + if firstResult.FieldsSeen != 1 || firstResult.FieldsWritten != 1 || firstResult.FieldsSkippedStale != 0 { + t.Fatalf("newer result = %#v, want 1/1/0", firstResult) + } + if firstResult.EnvelopesSeen != 1 || firstResult.EnvelopesUpdated != 1 { + t.Fatalf("newer envelope result = %#v, want seen=1 updated=1", firstResult) + } + secondResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 2200, + ParsedFields: map[string]any{ + "jt808.location.speed_kmh": 10, + "jt808.location.latitude": 30.5, + }, + }) + if err != nil { + t.Fatalf("older FastUpdate() error = %v", err) + } + if secondResult.FieldsSeen != 2 || secondResult.FieldsWritten != 1 || secondResult.FieldsSkippedStale != 1 { + t.Fatalf("older result = %#v, want 2/1/1", secondResult) + } + if secondResult.EnvelopesSeen != 1 || secondResult.EnvelopesUpdated != 1 { + t.Fatalf("older envelope result = %#v, want seen=1 updated=1", secondResult) + } + + values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result() + if err != nil { + t.Fatalf("values HGetAll error = %v", err) + } + if values["jt808.location.speed_kmh"] != "50" { + t.Fatalf("older frame overwrote speed: %#v", values) + } + if values["jt808.location.latitude"] != "30.5" { + t.Fatalf("older frame should still add previously unseen latitude: %#v", values) + } + times, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolJT808, "VIN001")).Result() + if err != nil { + t.Fatalf("times HGetAll error = %v", err) + } + if times["jt808.location.speed_kmh"] != "2000" || times["jt808.location.latitude"] != "1000" { + t.Fatalf("field times = %#v", times) + } + meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result() + if err != nil { + t.Fatalf("meta HGetAll error = %v", err) + } + if meta["event_time_ms"] != "2000" { + t.Fatalf("meta should keep latest frame time: %#v", meta) + } +} + +func TestRepositoryFastUpdateDoesNotLetOlderReceivedFrameOverwriteOnlineState(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + + if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + SourceEndpoint: "newer.example:808", + EventTimeMS: 2000, + ReceivedAtMS: 5000, + ParsedFields: map[string]any{ + "jt808.location.speed_kmh": 50, + }, + }); err != nil { + t.Fatalf("newer FastUpdate() error = %v", err) + } + if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + SourceEndpoint: "older.example:808", + EventTimeMS: 1000, + ReceivedAtMS: 4000, + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.5, + }, + }); err != nil { + t.Fatalf("older FastUpdate() error = %v", err) + } + + state, err := repo.client.HGetAll(ctx, "vehicle:online-state:JT808:VIN001").Result() + if err != nil { + t.Fatalf("online state HGetAll error = %v", err) + } + if state["last_seen_ms"] != "5000" || state["offline_after_ms"] != "65000" || state["source_endpoint"] != "newer.example:808" { + t.Fatalf("older received frame should not overwrite online state: %#v", state) + } + score, err := repo.client.ZScore(ctx, "vehicle:last_seen", "JT808:VIN001").Result() + if err != nil { + t.Fatalf("last_seen ZScore error = %v", err) + } + if score != 5000 { + t.Fatalf("last_seen score = %v, want 5000", score) + } + status, err := repo.IsOnline(ctx, "VIN001") + if err != nil { + t.Fatalf("IsOnline() error = %v", err) + } + if status.LastSeenMS != 5000 || status.SourceEndpoint != "newer.example:808" { + t.Fatalf("online status should keep newer received frame: %#v", status) + } +} + func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T) { repo, closeFn := newTestRepository(t) defer closeFn() ctx := context.Background() - err := repo.FastUpdateBatch(ctx, []envelope.FrameEnvelope{ + result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{ { Protocol: envelope.ProtocolGB32960, VIN: "VIN001", @@ -444,6 +643,10 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T Parsed: map[string]any{ "data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}}, }, + ParsedFields: map[string]any{ + "gb32960.vehicle.type": "0x01", + "gb32960.vehicle.soc_percent": 88.0, + }, }, { Protocol: envelope.ProtocolJT808, @@ -452,12 +655,17 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T ReceivedAtMS: 2200, MessageID: "0x0200", Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}}, + ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2}, Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2}, }, }) if err != nil { t.Fatalf("FastUpdateBatch() error = %v", err) } + if result.FieldsSeen != 4 || result.FieldsWritten != 4 || result.FieldsSkippedStale != 0 || + result.EnvelopesSeen != 2 || result.EnvelopesUpdated != 2 { + t.Fatalf("FastUpdateBatchWithResult() result = %#v, want fields 4/4/0 and envelopes 2/2", result) + } gbValues, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Result() if err != nil { @@ -481,6 +689,55 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T } } +func TestRepositoryFastUpdateReportsSkippedEnvelopesWithoutWritingRedis(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + + result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{ + { + Protocol: envelope.ProtocolGB32960, + MessageID: "0x01", + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 1000, + Parsed: map[string]any{"platform_login": map[string]any{"username": "Hyundai"}}, + }, + { + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + EventTimeMS: 2000, + ReceivedAtMS: 2000, + ParsedFields: map[string]any{"jt808.location.speed_kmh": 30}, + }, + { + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN002", + EventTimeMS: 3000, + ReceivedAtMS: 3000, + Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}}, + }, + }) + if err != nil { + t.Fatalf("FastUpdateBatchWithResult() error = %v", err) + } + if result.EnvelopesSeen != 3 || result.EnvelopesUpdated != 0 || + result.EnvelopesSkippedNonRealtime != 1 || result.EnvelopesSkippedMissingVIN != 1 || + result.EnvelopesSkippedMissingFields != 1 { + t.Fatalf("result = %#v, want non-realtime, missing vin and missing parsed fields skips", result) + } + if result.FieldsSeen != 0 || result.FieldsWritten != 0 || result.FieldsSkippedStale != 0 { + t.Fatalf("field result = %#v, want no field writes", result) + } + if repo.client.Exists(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val() != 0 { + t.Fatal("non-realtime frame should not write realtime kv") + } + if repo.client.Exists(ctx, "vehicle:online:JT808:VIN002").Val() != 0 { + t.Fatal("missing parsed fields frame should not mark vehicle online") + } +} + func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) { repo, closeFn := newTestRepository(t) defer closeFn() @@ -495,6 +752,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) { Parsed: map[string]any{ "data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}}, }, + ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0}, }, { Protocol: envelope.ProtocolJT808, @@ -503,6 +761,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) { ReceivedAtMS: 2200, MessageID: "0x0200", Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}}, + ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2}, Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2}, }, } { @@ -548,6 +807,7 @@ func TestOnlineIndexHandlerReturnsPagedOnlineStatuses(t *testing.T) { Parsed: map[string]any{ "data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}}, }, + ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0}, }); err != nil { t.Fatalf("FastUpdate() error = %v", err) } @@ -579,6 +839,7 @@ func TestPipelineDebugHandlerReturnsProtocolSummary(t *testing.T) { EventTimeMS: 3000, ReceivedAtMS: 3300, Parsed: map[string]any{"data": map[string]any{"TOTAL_MILEAGE": 1}}, + ParsedFields: map[string]any{"yutong_mqtt.data.total_mileage": 1}, Fields: map[string]any{envelope.FieldTotalMileageKM: 1}, }); err != nil { t.Fatalf("FastUpdate() error = %v", err) @@ -728,6 +989,82 @@ func TestRepositoryUsesEnvelopeParsedFieldsForRealtimeKV(t *testing.T) { } } +func TestRepositoryDropsNonPositiveTotalMileageFromRealtimeKV(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + + if err := repo.Update(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "VIN001", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + MessageID: "0x0200", + ParsedFields: map[string]any{ + "jt808.location.latitude": "31.259555", + "jt808.location.longitude": "119.892413", + "jt808.location.total_mileage_km": "0", + }, + Fields: map[string]any{ + envelope.FieldLatitude: 31.259555, + envelope.FieldLongitude: 119.892413, + envelope.FieldTotalMileageKM: 0, + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + + values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result() + if err != nil { + t.Fatalf("HGetAll() error = %v", err) + } + if _, exists := values["jt808.location.total_mileage_km"]; exists { + t.Fatalf("realtime kv should drop non-positive mileage: %#v", values) + } + if values["jt808.location.latitude"] != "31.259555" { + t.Fatalf("realtime kv should keep valid fields: %#v", values) + } +} + +func TestRepositoryDropsNonPositiveRawTotalMileageFromRealtimeKV(t *testing.T) { + repo, closeFn := newTestRepository(t) + defer closeFn() + ctx := context.Background() + + if err := repo.Update(ctx, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC3R1004101", + EventTimeMS: 1000, + ReceivedAtMS: 1100, + MessageID: "MQTT", + Parsed: map[string]any{ + "data": map[string]any{ + "LATITUDE": 30.590921, + "LONGITUDE": 121.075044, + "TOTAL_MILEAGE": 0, + }, + }, + ParsedFields: map[string]any{ + "yutong_mqtt.data.latitude": 30.590921, + "yutong_mqtt.data.longitude": 121.075044, + "yutong_mqtt.data.total_mileage": 0, + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + + values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolYutongMQTT, "LMRKH9AC3R1004101")).Result() + if err != nil { + t.Fatalf("HGetAll() error = %v", err) + } + if _, exists := values["yutong_mqtt.data.total_mileage"]; exists { + t.Fatalf("realtime kv should drop non-positive raw mileage: %#v", values) + } + if values["yutong_mqtt.data.latitude"] != "30.590921" { + t.Fatalf("realtime kv should keep valid fields: %#v", values) + } +} + func TestRepositorySkipsFramesWithoutVIN(t *testing.T) { repo, closeFn := newTestRepository(t) defer closeFn() diff --git a/go/vehicle-gateway/internal/realtime/snapshot_writer.go b/go/vehicle-gateway/internal/realtime/snapshot_writer.go index 41299afa..660a9724 100644 --- a/go/vehicle-gateway/internal/realtime/snapshot_writer.go +++ b/go/vehicle-gateway/internal/realtime/snapshot_writer.go @@ -10,6 +10,7 @@ import ( "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry" ) type SnapshotExecer interface { @@ -21,11 +22,20 @@ type PlateResolver interface { } type CachedPlateResolver struct { - delegate PlateResolver - ttl time.Duration - now func() time.Time - mu sync.Mutex - entries map[string]cachedPlateEntry + delegate PlateResolver + ttl time.Duration + maxEntries int + now func() time.Time + observe func(PlateCacheStats) + mu sync.Mutex + entries map[string]cachedPlateEntry + evictions int +} + +type PlateCacheStats struct { + Entries int + MaxEntries int + Evictions int } type cachedPlateEntry struct { @@ -35,18 +45,38 @@ type cachedPlateEntry struct { } func NewCachedPlateResolver(delegate PlateResolver, ttl time.Duration) *CachedPlateResolver { + return NewCachedPlateResolverWithMaxEntries(delegate, ttl, 200000) +} + +func NewCachedPlateResolverWithMaxEntries(delegate PlateResolver, ttl time.Duration, maxEntries int) *CachedPlateResolver { if delegate == nil { panic("cached plate resolver delegate must not be nil") } if ttl <= 0 { ttl = 10 * time.Minute } - return &CachedPlateResolver{ - delegate: delegate, - ttl: ttl, - now: time.Now, - entries: map[string]cachedPlateEntry{}, + if maxEntries < 0 { + maxEntries = 0 } + return &CachedPlateResolver{ + delegate: delegate, + ttl: ttl, + maxEntries: maxEntries, + now: time.Now, + entries: map[string]cachedPlateEntry{}, + } +} + +func (r *CachedPlateResolver) SetStatsObserver(observe func(PlateCacheStats)) { + r.mu.Lock() + r.observe = observe + r.mu.Unlock() +} + +func (r *CachedPlateResolver) CacheStats() PlateCacheStats { + r.mu.Lock() + defer r.mu.Unlock() + return PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions} } func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) { @@ -76,13 +106,46 @@ func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (strin notFound: errors.Is(err, sql.ErrNoRows), expiresAt: now.Add(r.ttl), } + r.enforceLimitLocked(now) + stats := PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions} + observe := r.observe r.mu.Unlock() + if observe != nil { + observe(stats) + } if err != nil { return "", err } return strings.TrimSpace(plate), nil } +func (r *CachedPlateResolver) enforceLimitLocked(now time.Time) { + if r.maxEntries <= 0 || len(r.entries) <= r.maxEntries { + return + } + for vin, entry := range r.entries { + if now.After(entry.expiresAt) || now.Equal(entry.expiresAt) { + delete(r.entries, vin) + r.evictions++ + } + } + for len(r.entries) > r.maxEntries { + victim := "" + var victimExpiresAt time.Time + for vin, entry := range r.entries { + if victim == "" || entry.expiresAt.Before(victimExpiresAt) { + victim = vin + victimExpiresAt = entry.expiresAt + } + } + if victim == "" { + return + } + delete(r.entries, victim) + r.evictions++ + } +} + type SnapshotWriter struct { exec SnapshotExecer plateResolver PlateResolver @@ -111,6 +174,19 @@ func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error { if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil { return err } + for _, statement := range realtimeLocationCompatibilitySQL { + if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) { + return err + } + } + for _, statement := range cleanupInvalidRealtimeTotalMileageSQL { + if _, err := w.exec.ExecContext(ctx, statement); err != nil { + return err + } + } + if _, err := w.exec.ExecContext(ctx, backfillRealtimeAccessProjectionSQL); err != nil { + return err + } return nil } @@ -129,13 +205,11 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) if err != nil { return err } - eventTime := nullableTime(env.EventTimeMS) + eventTimeMS, _ := envelope.NormalizedEventTimeMS(env) + eventTime := nullableTime(eventTimeMS) receivedAt := nullableTime(env.ReceivedAtMS) platformName := platformNameFromEnvelope(env) - parsed, err := w.snapshotFieldsForEnvelope(ctx, env, vin) - if err != nil { - return err - } + parsed := snapshotFieldsForEnvelope(env) if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL, string(env.Protocol), vin, @@ -146,6 +220,9 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) eventTime, receivedAt, env.StableEventID(), + receivedAt, + receivedAt, + env.StableEventID(), ); err != nil { return err } @@ -162,6 +239,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) location.Longitude, location.SpeedKMH, location.TotalMileageKM, + location.TotalMileageAt, location.SOCPercent, location.AltitudeM, location.DirectionDeg, @@ -173,87 +251,25 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope) return err } -func (w *SnapshotWriter) snapshotFieldsForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) { - if len(env.Parsed) == 0 { - return nil, nil - } - parsed := cloneMap(env.Parsed) - incoming := realtimeSnapshotFlatFields(env, parsed) - if queryer, ok := w.exec.(Queryer); ok { - existing, err := realtimeSnapshotParsedJSON(ctx, queryer, env.Protocol, vin) - if err != nil { - return nil, err - } - if len(existing) > 0 { - if isStructuredSnapshotParsed(env.Protocol, existing) { - parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed) - mergedEnv := env - mergedEnv.ParsedFields = nil - mergedEnv.ParsedFieldTypes = nil - return realtimeSnapshotFlatFields(mergedEnv, parsed), nil - } - return mergeRealtimeSnapshotFields(existing, incoming), nil - } - } - return incoming, nil -} - -func realtimeSnapshotFlatFields(env envelope.FrameEnvelope, parsed map[string]any) map[string]any { - if len(env.ParsedFields) > 0 { - fields := cloneMap(env.ParsedFields) - if len(fields) > 0 { - return fields - } - } - rows := realtimeKVFields(env, parsed) - if len(rows) == 0 { +func snapshotFieldsForEnvelope(env envelope.FrameEnvelope) map[string]any { + if len(env.ParsedFields) == 0 { return nil } - fields := make(map[string]any, len(rows)) - for _, row := range rows { - key := realtimeKVFieldPath(row.Domain, row.Field) - if key == "" { - continue - } - fields[key] = row.Value + return realtimeSnapshotFlatFields(env) +} + +func realtimeSnapshotFlatFields(env envelope.FrameEnvelope) map[string]any { + fields := cloneMap(env.ParsedFields) + filterInvalidRealtimeMeasurementFields(fields) + if len(fields) == 0 { + return nil } return fields } -func mergeRealtimeSnapshotFields(existing map[string]any, incoming map[string]any) map[string]any { - if len(existing) == 0 { - return cloneMap(incoming) - } - merged := cloneMap(existing) - for key, value := range incoming { - merged[key] = value - } - return merged -} - -func isStructuredSnapshotParsed(protocol envelope.Protocol, parsed map[string]any) bool { - if len(parsed) == 0 { - return false - } - if protocol == envelope.ProtocolGB32960 { - if _, ok := parsed["data_units"]; ok { - return true - } - } - mapping := realtimeMapping(protocol) - for key := range mapping.TopLevelName { - if _, ok := parsed[key]; ok { - return true - } - } - return false -} - func platformNameFromEnvelope(env envelope.FrameEnvelope) string { - if env.Fields != nil { - if value := strings.TrimSpace(stringValue(env.Fields["platform_account"])); value != "" { - return value - } + if value := strings.TrimSpace(env.PlatformName); value != "" { + return value } if env.Parsed != nil { if value := strings.TrimSpace(stringValue(env.Parsed["platform_name"])); value != "" { @@ -275,25 +291,6 @@ func stringValue(value any) string { } } -func realtimeSnapshotParsedJSON(ctx context.Context, queryer Queryer, protocol envelope.Protocol, vin string) (map[string]any, error) { - var raw sql.NullString - err := queryer.QueryRowContext(ctx, selectRealtimeSnapshotParsedJSONSQL, string(protocol), vin).Scan(&raw) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, err - } - if !raw.Valid || strings.TrimSpace(raw.String) == "" { - return nil, nil - } - var parsed map[string]any - if err := json.Unmarshal([]byte(raw.String), &parsed); err != nil { - return nil, nil - } - return parsed, nil -} - func marshalParsedJSON(parsed map[string]any) any { if len(parsed) == 0 { return nil @@ -335,6 +332,7 @@ type realtimeLocationRow struct { Longitude float64 SpeedKMH any TotalMileageKM any + TotalMileageAt any SOCPercent any AltitudeM any DirectionDeg any @@ -345,71 +343,53 @@ type realtimeLocationRow struct { } func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) { - latitude, okLat := numberField(env.Fields, envelope.FieldLatitude) - longitude, okLon := numberField(env.Fields, envelope.FieldLongitude) - if !okLat || !okLon { + location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields) + if !ok { return realtimeLocationRow{}, false } + eventTimeMS, _ := envelope.NormalizedEventTimeMS(env) + totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields) + if totalMileageKM <= 0 { + hasTotalMileage = false + } + var totalMileageValue any + var totalMileageAt any + if hasTotalMileage { + totalMileageValue = totalMileageKM + totalMileageAt = nullableTime(eventTimeMS) + } return realtimeLocationRow{ Protocol: string(env.Protocol), VIN: vin, Plate: plate, - EventTime: nullableTime(env.EventTimeMS), - Latitude: latitude, - Longitude: longitude, - SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH), - TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM), - SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent), - AltitudeM: nullableNumberField(env.Fields, "altitude_m"), - DirectionDeg: nullableNumberField(env.Fields, "direction_deg"), - AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"), - StatusFlag: nullableNumberField(env.Fields, "status_flag"), + EventTime: nullableTime(eventTimeMS), + Latitude: location.Latitude, + Longitude: location.Longitude, + SpeedKMH: optionalFloatValue(location.SpeedKMH), + TotalMileageKM: totalMileageValue, + TotalMileageAt: totalMileageAt, + SOCPercent: optionalFloatValue(location.SOCPercent), + AltitudeM: optionalFloatValue(location.AltitudeM), + DirectionDeg: optionalFloatValue(location.DirectionDeg), + AlarmFlag: optionalInt64Value(location.AlarmFlag), + StatusFlag: optionalInt64Value(location.StatusFlag), ReceivedAt: nullableTime(env.ReceivedAtMS), EventID: env.StableEventID(), }, true } -func nullableNumberField(fields map[string]any, key string) any { - value, ok := numberField(fields, key) - if !ok { +func optionalFloatValue(value *float64) any { + if value == nil { return nil } - return value + return *value } -func numberField(fields map[string]any, key string) (float64, bool) { - value, ok := fields[key] - if !ok { - return 0, false - } - switch typed := value.(type) { - case float64: - return typed, true - case float32: - return float64(typed), true - case int: - return float64(typed), true - case int8: - return float64(typed), true - case int16: - return float64(typed), true - case int32: - return float64(typed), true - case int64: - return float64(typed), true - case uint: - return float64(typed), true - case uint8: - return float64(typed), true - case uint16: - return float64(typed), true - case uint32: - return float64(typed), true - case uint64: - return float64(typed), true - default: - return 0, false +func optionalInt64Value(value *int64) any { + if value == nil { + return nil } + return *value } type BindingPlateResolver struct { @@ -469,30 +449,19 @@ func nullableTime(ms int64) any { func isRealtimeSnapshotEvent(env envelope.FrameEnvelope) bool { switch env.Protocol { case envelope.ProtocolGB32960: - return env.MessageID == "0x02" && hasGB32960RealtimeDataUnits(env) + return (env.MessageID == "0x02" || env.MessageID == "0x03") && telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields) case envelope.ProtocolJT808: return env.MessageID == "0x0200" && hasLocationFields(env) case envelope.ProtocolYutongMQTT: - return len(env.Fields) > 0 + return telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields) default: return false } } -func hasGB32960RealtimeDataUnits(env envelope.FrameEnvelope) bool { - if units, ok := env.Parsed["data_units"].([]any); ok && len(units) > 0 { - return true - } - if units, ok := env.Parsed["data_units"].([]map[string]any); ok && len(units) > 0 { - return true - } - return false -} - func hasLocationFields(env envelope.FrameEnvelope) bool { - _, okLat := numberField(env.Fields, envelope.FieldLatitude) - _, okLon := numberField(env.Fields, envelope.FieldLongitude) - return okLat && okLon + _, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields) + return ok } const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot ( @@ -505,16 +474,31 @@ const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_sn event_time DATETIME(3) NULL, received_at DATETIME(3) NULL, event_id VARCHAR(64) NOT NULL DEFAULT '', + access_first_seen_at DATETIME(3) NULL, + access_previous_received_at DATETIME(3) NULL, + access_latest_received_at DATETIME(3) NULL, + access_report_interval_ms BIGINT UNSIGNED NULL, + access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0, + access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '', + access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (protocol, vin), KEY idx_vin (vin), - KEY idx_protocol_updated (protocol, updated_at) + KEY idx_protocol_updated (protocol, updated_at), + KEY idx_access_latest (access_latest_received_at, vin) )` var realtimeSnapshotCompatibilitySQL = []string{ "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN platform_name VARCHAR(64) NOT NULL DEFAULT '' AFTER plate", "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN peer VARCHAR(128) NOT NULL DEFAULT '' AFTER platform_name", "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json LONGTEXT NULL AFTER peer", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_at DATETIME(3) NULL AFTER event_id", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_previous_received_at DATETIME(3) NULL AFTER access_first_seen_at", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_received_at DATETIME(3) NULL AFTER access_previous_received_at", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_report_interval_ms BIGINT UNSIGNED NULL AFTER access_latest_received_at", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER access_report_interval_ms", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '' AFTER access_sample_count", + "ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '' AFTER access_latest_event_id", } func isDuplicateColumnError(err error) bool { @@ -524,21 +508,27 @@ func isDuplicateColumnError(err error) bool { const upsertRealtimeSnapshotSQL = ` INSERT INTO vehicle_realtime_snapshot - (protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + (protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id, + access_first_seen_at, access_latest_received_at, access_sample_count, access_latest_event_id, access_first_seen_source) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 'live_writer') ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), platform_name = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(platform_name), platform_name), peer = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(peer), peer), - parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(parsed_json), parsed_json), + parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), + IF(VALUES(parsed_json) IS NULL OR VALUES(parsed_json) = '', parsed_json, JSON_MERGE_PATCH(COALESCE(NULLIF(vehicle_realtime_snapshot.parsed_json, ''), JSON_OBJECT()), VALUES(parsed_json))), + parsed_json), event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_time), event_time), received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(received_at), received_at), event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_id), event_id), - updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at) + updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at), + access_previous_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, vehicle_realtime_snapshot.access_latest_received_at, access_previous_received_at), + access_report_interval_ms = IF(VALUES(access_latest_received_at) IS NOT NULL AND vehicle_realtime_snapshot.access_latest_received_at IS NOT NULL AND VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, TIMESTAMPDIFF(MICROSECOND, vehicle_realtime_snapshot.access_latest_received_at, VALUES(access_latest_received_at)) DIV 1000, access_report_interval_ms), + access_sample_count = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, access_sample_count + 1, access_sample_count), + access_latest_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_received_at), access_latest_received_at), + access_latest_event_id = IF(VALUES(access_latest_received_at) IS NOT NULL AND VALUES(access_latest_received_at) = vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_event_id), access_latest_event_id) ` -const selectRealtimeSnapshotParsedJSONSQL = `SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = ? AND vin = ?` - const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location ( protocol VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL DEFAULT '', @@ -548,6 +538,7 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo longitude DECIMAL(12,6) NOT NULL, speed_kmh DECIMAL(10,3) NULL, total_mileage_km DECIMAL(18,3) NULL, + total_mileage_event_time DATETIME(3) NULL, soc_percent DECIMAL(6,2) NULL, altitude_m DECIMAL(10,3) NULL, direction_deg DECIMAL(10,3) NULL, @@ -561,11 +552,50 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo KEY idx_protocol_updated (protocol, updated_at) )` +var realtimeLocationCompatibilitySQL = []string{ + "ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time DATETIME(3) NULL AFTER total_mileage_km", +} + +var cleanupInvalidRealtimeTotalMileageSQL = []string{ + `UPDATE vehicle_realtime_location +SET total_mileage_km = NULL, + total_mileage_event_time = NULL +WHERE total_mileage_km IS NOT NULL + AND total_mileage_km <= 0`, + `UPDATE vehicle_realtime_snapshot +SET parsed_json = JSON_REMOVE(parsed_json, '$."jt808.location.total_mileage_km"') +WHERE protocol = 'JT808' + AND parsed_json IS NOT NULL + AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."jt808.location.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`, + `UPDATE vehicle_realtime_snapshot +SET parsed_json = JSON_REMOVE(parsed_json, '$."gb32960.vehicle.total_mileage_km"') +WHERE protocol = 'GB32960' + AND parsed_json IS NOT NULL + AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."gb32960.vehicle.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`, + `UPDATE vehicle_realtime_snapshot +SET parsed_json = JSON_REMOVE(parsed_json, '$."yutong_mqtt.data.total_mileage_km"') +WHERE protocol = 'YUTONG_MQTT' + AND parsed_json IS NOT NULL + AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."yutong_mqtt.data.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`, +} + +const backfillRealtimeAccessProjectionSQL = `UPDATE vehicle_realtime_snapshot +SET access_first_seen_at = COALESCE(access_first_seen_at, received_at, updated_at), + access_latest_received_at = COALESCE(access_latest_received_at, received_at, updated_at), + access_sample_count = IF(access_sample_count = 0, 1, access_sample_count), + access_latest_event_id = IF(access_latest_event_id = '', event_id, access_latest_event_id), + access_first_seen_source = IF(access_first_seen_source = '', 'snapshot_backfill', access_first_seen_source) +WHERE access_first_seen_at IS NULL + OR access_latest_received_at IS NULL + OR access_sample_count = 0 + OR access_latest_event_id = '' + OR access_first_seen_source = ''` + const upsertRealtimeLocationSQL = ` INSERT INTO vehicle_realtime_location - (protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, + (protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE plate = IF(VALUES(plate) <> '', VALUES(plate), plate), event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_time), event_time), @@ -573,6 +603,7 @@ ON DUPLICATE KEY UPDATE longitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(longitude), longitude), speed_kmh = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh), total_mileage_km = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km), + total_mileage_event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), IF(VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), soc_percent = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(soc_percent), soc_percent), soc_percent), altitude_m = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(altitude_m), altitude_m), altitude_m), direction_deg = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(direction_deg), direction_deg), direction_deg), diff --git a/go/vehicle-gateway/internal/realtime/snapshot_writer_test.go b/go/vehicle-gateway/internal/realtime/snapshot_writer_test.go index 8e8f1bbb..0c65fc97 100644 --- a/go/vehicle-gateway/internal/realtime/snapshot_writer_test.go +++ b/go/vehicle-gateway/internal/realtime/snapshot_writer_test.go @@ -37,26 +37,30 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) { map[string]any{"name": "gd_fc_vendor_tlv", "type": "vendor", "value": map[string]any{"foo": "bar"}}, }, }, + ParsedFields: map[string]any{ + "gb32960.vehicle.soc_percent": "90", + "gb32960.gd_fc_vendor_tlv.foo": "bar", + }, Fields: map[string]any{envelope.FieldSOCPercent: 90}, }); err != nil { t.Fatalf("Update() error = %v", err) } - if len(exec.calls) != 6 { - t.Fatalf("exec calls = %d, want 6", len(exec.calls)) + if len(exec.calls) != 19 { + t.Fatalf("exec calls = %d, want 19", len(exec.calls)) } if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot") { t.Fatalf("schema query = %s", exec.calls[0].query) } - if !strings.Contains(exec.calls[4].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_location") { - t.Fatalf("location schema query = %s", exec.calls[4].query) + if !strings.Contains(exec.calls[11].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_location") { + t.Fatalf("location schema query = %s", exec.calls[11].query) } for _, call := range exec.calls { if strings.Contains(call.query, "vehicle_realtime_kv") { t.Fatalf("snapshot writer should not create or write MySQL realtime kv: %s", call.query) } } - for _, call := range []snapshotExecCall{exec.calls[0], exec.calls[4]} { + for _, call := range []snapshotExecCall{exec.calls[0], exec.calls[11]} { if strings.Contains(call.query, "fields_json") { t.Fatalf("realtime schema should not contain fields_json: %s", call.query) } @@ -74,10 +78,21 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) { t.Fatalf("snapshot schema should contain %s: %s", want, exec.calls[0].query) } } - if strings.Contains(exec.calls[4].query, "idx_location") { - t.Fatalf("realtime location table should not keep unused geo index: %s", exec.calls[4].query) + if strings.Contains(exec.calls[11].query, "idx_location") { + t.Fatalf("realtime location table should not keep unused geo index: %s", exec.calls[11].query) } - upsert := exec.calls[5] + if !strings.Contains(exec.calls[12].query, "total_mileage_event_time") { + t.Fatalf("location compatibility migration should add total mileage event time: %s", exec.calls[12].query) + } + for _, call := range exec.calls[13:17] { + if !strings.Contains(call.query, "total_mileage") { + t.Fatalf("schema bootstrap should clean invalid realtime mileage: %s", call.query) + } + } + if !strings.Contains(exec.calls[17].query, "snapshot_backfill") { + t.Fatalf("access projection baseline should be backfilled explicitly: %s", exec.calls[17].query) + } + upsert := exec.calls[18] if !strings.Contains(upsert.query, "ON DUPLICATE KEY UPDATE") { t.Fatalf("upsert query = %s", upsert.query) } @@ -101,8 +116,13 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) { if got := upsert.args[5]; !strings.Contains(got.(string), `"gb32960.vehicle.soc_percent":"90"`) { t.Fatalf("parsed json arg = %#v", got) } - if len(upsert.args) != 9 { - t.Fatalf("snapshot upsert args = %d, want 9", len(upsert.args)) + if len(upsert.args) != 12 { + t.Fatalf("snapshot upsert args = %d, want 12", len(upsert.args)) + } + for _, want := range []string{"access_first_seen_at", "access_previous_received_at", "access_latest_received_at", "access_report_interval_ms", "access_sample_count", "access_latest_event_id", "live_writer", "TIMESTAMPDIFF(MICROSECOND"} { + if !strings.Contains(upsert.query, want) { + t.Fatalf("access projection upsert missing %q: %s", want, upsert.query) + } } } @@ -122,8 +142,24 @@ func TestSnapshotWriterEnsureSchemaOnlyCreatesTargetTables(t *testing.T) { WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json"). WillReturnResult(sqlmock.NewResult(0, 0)) + for _, column := range []string{"access_first_seen_at", "access_previous_received_at", "access_latest_received_at", "access_report_interval_ms", "access_sample_count", "access_latest_event_id", "access_first_seen_source"} { + mock.ExpectExec("ALTER TABLE vehicle_realtime_snapshot ADD COLUMN " + column). + WillReturnResult(sqlmock.NewResult(0, 0)) + } mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_realtime_location"). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE vehicle_realtime_location"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE vehicle_realtime_snapshot"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE vehicle_realtime_snapshot"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE vehicle_realtime_snapshot"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("UPDATE vehicle_realtime_snapshot"). + WillReturnResult(sqlmock.NewResult(0, 0)) if err := writer.EnsureSchema(context.Background()); err != nil { t.Fatalf("EnsureSchema() error = %v", err) @@ -133,6 +169,41 @@ func TestSnapshotWriterEnsureSchemaOnlyCreatesTargetTables(t *testing.T) { } } +func TestRealtimeAccessProjectionSQLProtectsDuplicateAndOutOfOrderReceipts(t *testing.T) { + accessStart := strings.Index(upsertRealtimeSnapshotSQL, "access_previous_received_at =") + if accessStart < 0 { + t.Fatal("access projection assignments are missing") + } + accessSQL := upsertRealtimeSnapshotSQL[accessStart:] + for _, want := range []string{ + "VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at", + "VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id", + "TIMESTAMPDIFF(MICROSECOND", + "access_sample_count + 1", + } { + if !strings.Contains(accessSQL, want) { + t.Fatalf("access projection SQL missing %q:\n%s", want, accessSQL) + } + } + if strings.Contains(accessSQL, "VALUES(event_time)") { + t.Fatalf("access receipt projection must not be gated by device event time:\n%s", accessSQL) + } + ordered := []string{"access_previous_received_at =", "access_report_interval_ms =", "access_sample_count =", "access_latest_received_at =", "access_latest_event_id ="} + previous := -1 + for _, assignment := range ordered { + position := strings.Index(accessSQL, assignment) + if position <= previous { + t.Fatalf("access assignment %q must preserve old latest receipt before advancing it: %s", assignment, accessSQL) + } + previous = position + } + for _, want := range []string{"COALESCE(access_first_seen_at, received_at, updated_at)", "snapshot_backfill", "access_sample_count = 0"} { + if !strings.Contains(backfillRealtimeAccessProjectionSQL, want) { + t.Fatalf("access baseline backfill missing %q: %s", want, backfillRealtimeAccessProjectionSQL) + } + } +} + func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T) { exec := &recordingSnapshotExec{} writer := NewSnapshotWriter(exec) @@ -150,16 +221,15 @@ func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T) ReceivedAtMS: 1782918601000, EventID: "event-1", Parsed: map[string]any{"location": map[string]any{"latitude": 30.590151, "longitude": 121.069881}}, - Fields: map[string]any{ - envelope.FieldLatitude: 30.590151, - envelope.FieldLongitude: 121.069881, - envelope.FieldSpeedKMH: 23.0, - envelope.FieldTotalMileageKM: 10241.2, - envelope.FieldSOCPercent: 88, - "altitude_m": uint16(5), - "direction_deg": uint16(79), - "alarm_flag": uint32(0), - "status_flag": uint32(786435), + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.590151, + "jt808.location.longitude": 121.069881, + "jt808.location.speed_kmh": 23.0, + "jt808.location.total_mileage_km": "10241.2", + "jt808.location.altitude_m": uint16(5), + "jt808.location.direction_deg": uint16(79), + "jt808.location.alarm_flag": uint32(0), + "jt808.location.status_flag": uint32(786435), }, }); err != nil { t.Fatalf("Update() error = %v", err) @@ -198,15 +268,190 @@ func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T) if got, want := locationUpsert.args[7], 10241.2; got != want { t.Fatalf("mileage arg = %#v, want %v", got, want) } - if got, want := locationUpsert.args[8], 88.0; got != want { - t.Fatalf("soc arg = %#v, want %v", got, want) + if _, ok := locationUpsert.args[8].(time.Time); !ok { + t.Fatalf("total mileage event time arg = %#v, want time.Time", locationUpsert.args[8]) } - if len(locationUpsert.args) != 15 { - t.Fatalf("location upsert args = %d, want 15", len(locationUpsert.args)) + if got := locationUpsert.args[9]; got != nil { + t.Fatalf("JT808 location should not invent SOC, got %#v", got) + } + if len(locationUpsert.args) != 16 { + t.Fatalf("location upsert args = %d, want 16", len(locationUpsert.args)) } } -func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testing.T) { +func TestSnapshotWriterDropsNonPositiveTotalMileageFromRealtimeStores(t *testing.T) { + exec := &recordingSnapshotExec{} + writer := NewSnapshotWriter(exec) + + if err := writer.Update(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918601000, + ParsedFields: map[string]any{ + "jt808.location.latitude": "30.590151", + "jt808.location.longitude": "121.069881", + "jt808.location.speed_kmh": "23", + "jt808.location.total_mileage_km": "0", + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + + if len(exec.calls) != 2 { + t.Fatalf("exec calls = %d, want snapshot + location", len(exec.calls)) + } + parsedJSON, ok := exec.calls[0].args[5].(string) + if !ok { + t.Fatalf("snapshot parsed_json arg = %#v", exec.calls[0].args[5]) + } + if strings.Contains(parsedJSON, "total_mileage_km") { + t.Fatalf("snapshot parsed_json should drop non-positive mileage: %s", parsedJSON) + } + if !strings.Contains(parsedJSON, "jt808.location.speed_kmh") { + t.Fatalf("snapshot parsed_json should keep valid fields: %s", parsedJSON) + } + if exec.calls[1].args[7] != nil { + t.Fatalf("location total mileage arg = %#v, want nil", exec.calls[1].args[7]) + } + if exec.calls[1].args[8] != nil { + t.Fatalf("location total mileage time arg = %#v, want nil", exec.calls[1].args[8]) + } +} + +func TestSnapshotWriterNormalizesFarFutureEventTime(t *testing.T) { + exec := &recordingSnapshotExec{} + writer := NewSnapshotWriter(exec) + received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC) + futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC) + + if err := writer.Update(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: "0x0200", + VIN: "VIN001", + EventTimeMS: futureEvent.UnixMilli(), + ReceivedAtMS: received.UnixMilli(), + Parsed: map[string]any{"location": map[string]any{"latitude": 30.590151, "longitude": 121.069881}}, + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.590151, + "jt808.location.longitude": 121.069881, + "jt808.location.total_mileage_km": "10241.2", + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + + if len(exec.calls) != 2 { + t.Fatalf("exec calls = %d, want snapshot + location", len(exec.calls)) + } + if got, ok := exec.calls[0].args[6].(time.Time); !ok || !got.Equal(received) { + t.Fatalf("snapshot event_time arg = %#v, want received %s", exec.calls[0].args[6], received) + } + if got, ok := exec.calls[1].args[3].(time.Time); !ok || !got.Equal(received) { + t.Fatalf("location event_time arg = %#v, want received %s", exec.calls[1].args[3], received) + } + if got, ok := exec.calls[1].args[8].(time.Time); !ok || !got.Equal(received) { + t.Fatalf("total mileage event time arg = %#v, want received %s", exec.calls[1].args[8], received) + } +} + +func TestRealtimeLocationUsesProtocolMileageMapping(t *testing.T) { + tests := []struct { + name string + protocol envelope.Protocol + fields map[string]any + wantMileage float64 + }{ + { + name: "gb32960 kilometers", + protocol: envelope.ProtocolGB32960, + fields: map[string]any{ + "gb32960.position.latitude": 30.590151, + "gb32960.position.longitude": 121.069881, + "gb32960.vehicle.total_mileage_km": "4123.9", + }, + wantMileage: 4123.9, + }, + { + name: "jt808 kilometers", + protocol: envelope.ProtocolJT808, + fields: map[string]any{ + "jt808.location.latitude": 30.590151, + "jt808.location.longitude": 121.069881, + "jt808.location.total_mileage_km": json.Number("10241.2"), + }, + wantMileage: 10241.2, + }, + { + name: "yutong meters", + protocol: envelope.ProtocolYutongMQTT, + fields: map[string]any{ + "yutong_mqtt.data.latitude": 30.590151, + "yutong_mqtt.data.longitude": 121.069881, + "yutong_mqtt.data.total_mileage": "86737000", + }, + wantMileage: 86737, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + row, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{ + Protocol: tc.protocol, + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918601000, + ParsedFields: tc.fields, + }, "VIN001", "") + if !ok { + t.Fatal("realtimeLocationFromEnvelope() should produce a location") + } + if got, ok := row.TotalMileageKM.(float64); !ok || got != tc.wantMileage { + t.Fatalf("total mileage = %#v, want %v", row.TotalMileageKM, tc.wantMileage) + } + if _, ok := row.TotalMileageAt.(time.Time); !ok { + t.Fatalf("total mileage event time = %#v, want time.Time", row.TotalMileageAt) + } + }) + } +} + +func TestRealtimeLocationDoesNotAdvanceMileageFromCoreFieldOnSparseFrame(t *testing.T) { + row, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918601000, + ParsedFields: map[string]any{ + "yutong_mqtt.data.latitude": "30.590151", + "yutong_mqtt.data.longitude": "121.069881", + }, + }, "VIN001", "") + if !ok { + t.Fatal("realtimeLocationFromEnvelope() should produce a location") + } + if row.TotalMileageKM != nil { + t.Fatalf("sparse frame total mileage = %#v, want nil", row.TotalMileageKM) + } + if row.TotalMileageAt != nil { + t.Fatalf("sparse frame total mileage event time = %#v, want nil", row.TotalMileageAt) + } +} + +func TestRealtimeLocationRejectsBareStandardizedFields(t *testing.T) { + _, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918601000, + Fields: map[string]any{ + envelope.FieldLatitude: 30.590151, + envelope.FieldLongitude: 121.069881, + }, + }, "VIN001", "") + if ok { + t.Fatal("bare standardized fields must not drive the canonical location projection") + } +} + +func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFramesInUpsert(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatal(err) @@ -214,10 +459,6 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi defer db.Close() writer := NewSnapshotWriter(db) - existing := `{"data_units":[{"type":"0x01","name":"vehicle","value":{"soc_percent":88}}]}` - mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?"). - WithArgs("GB32960", "VIN001"). - WillReturnRows(sqlmock.NewRows([]string{"parsed_json"}).AddRow(existing)) mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot"). WithArgs( "GB32960", @@ -226,14 +467,15 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi "", "", jsonFlatFieldsArg{fields: map[string]string{ - "gb32960.vehicle.soc_percent": "88", - "gb32960.vehicle.type": "0x01", "gb32960.gd_fc_stack.stack_count": "1", "gb32960.gd_fc_stack.type": "0x30", }}, sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), ). WillReturnResult(sqlmock.NewResult(0, 1)) @@ -248,6 +490,10 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": map[string]any{"stack_count": 1}}, }, }, + ParsedFields: map[string]any{ + "gb32960.gd_fc_stack.stack_count": "1", + "gb32960.gd_fc_stack.type": "0x30", + }, }) if err != nil { t.Fatalf("Update() error = %v", err) @@ -265,9 +511,6 @@ func TestSnapshotWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T) defer db.Close() writer := NewSnapshotWriter(db) - mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?"). - WithArgs("GB32960", "VIN001"). - WillReturnRows(sqlmock.NewRows([]string{"parsed_json"})) mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot"). WithArgs( "GB32960", @@ -281,6 +524,9 @@ func TestSnapshotWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T) sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), ). WillReturnResult(sqlmock.NewResult(0, 1)) @@ -316,10 +562,6 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te defer db.Close() writer := NewSnapshotWriter(db) - existing := `{"data_units":[{"type":"0x30","name":"gd_fc_stack","value":{"stack_count":1,"summaries":[{"cell_count":432,"stack_water_outlet_temp_c":65,"frame_cell_start":201,"frame_cell_count":200}]}}]}` - mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?"). - WithArgs("GB32960", "VIN001"). - WillReturnRows(sqlmock.NewRows([]string{"parsed_json"}).AddRow(existing)) mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot"). WithArgs( "GB32960", @@ -341,6 +583,9 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), + sqlmock.AnyArg(), ). WillReturnResult(sqlmock.NewResult(0, 1)) @@ -364,6 +609,9 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te }, }, }, + ParsedFields: map[string]any{ + "gb32960.gd_fc_stack.stack_water_outlet_temp_c": "63", + }, }) if err != nil { t.Fatalf("Update() error = %v", err) @@ -397,6 +645,18 @@ func TestRealtimeUpsertsDoNotOverwriteWithOlderEventTime(t *testing.T) { } } +func TestRealtimeSnapshotUpsertMergesParsedJSONInSQL(t *testing.T) { + for _, want := range []string{ + "JSON_MERGE_PATCH", + "COALESCE(NULLIF(vehicle_realtime_snapshot.parsed_json, ''), JSON_OBJECT())", + "VALUES(parsed_json)", + } { + if !strings.Contains(upsertRealtimeSnapshotSQL, want) { + t.Fatalf("snapshot upsert should merge parsed_json in SQL, missing %q:\n%s", want, upsertRealtimeSnapshotSQL) + } + } +} + func TestRealtimeLocationUpsertKeepsExistingSparseFieldsWhenMQTTOnlySendsCoordinates(t *testing.T) { for _, column := range []string{ "speed_kmh", @@ -412,6 +672,10 @@ func TestRealtimeLocationUpsertKeepsExistingSparseFieldsWhenMQTTOnlySendsCoordin t.Fatalf("location upsert should keep existing %s when incoming sparse MQTT frame omits it; missing:\n%s\nin:\n%s", column, want, upsertRealtimeLocationSQL) } } + wantMileageAt := "total_mileage_event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), IF(VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), total_mileage_event_time)" + if !strings.Contains(upsertRealtimeLocationSQL, wantMileageAt) { + t.Fatalf("location upsert should only advance total_mileage_event_time when the incoming frame carries mileage; missing:\n%s\nin:\n%s", wantMileageAt, upsertRealtimeLocationSQL) + } } func TestSnapshotWriterBackfillsPlateFromBindingByVIN(t *testing.T) { @@ -426,10 +690,7 @@ func TestSnapshotWriterBackfillsPlateFromBindingByVIN(t *testing.T) { EventTimeMS: 1782918600000, ReceivedAtMS: 1782918601000, Parsed: sampleGB32960RealtimeParsed(), - Fields: map[string]any{ - envelope.FieldLatitude: 30.590151, - envelope.FieldLongitude: 121.069881, - }, + ParsedFields: sampleGB32960LocationFields(), }); err != nil { t.Fatalf("Update() error = %v", err) } @@ -481,10 +742,7 @@ func TestSnapshotWriterCachesBindingPlateByVIN(t *testing.T) { EventTimeMS: 1782918600000, ReceivedAtMS: 1782918601000, Parsed: sampleGB32960RealtimeParsed(), - Fields: map[string]any{ - envelope.FieldLatitude: 30.590151, - envelope.FieldLongitude: 121.069881, - }, + ParsedFields: sampleGB32960LocationFields(), } if err := writer.Update(context.Background(), event); err != nil { @@ -509,6 +767,61 @@ func TestSnapshotWriterCachesBindingPlateByVIN(t *testing.T) { } } +func TestCachedPlateResolverEvictsOldestEntryWhenLimitExceeded(t *testing.T) { + delegate := &mapPlateResolver{plates: map[string]string{ + "VIN001": "沪A00001", + "VIN002": "沪A00002", + "VIN003": "沪A00003", + }} + resolver := NewCachedPlateResolverWithMaxEntries(delegate, time.Hour, 2) + now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC) + resolver.now = func() time.Time { return now } + var observed PlateCacheStats + resolver.SetStatsObserver(func(stats PlateCacheStats) { + observed = stats + }) + for _, vin := range []string{"VIN001", "VIN002", "VIN003"} { + if _, err := resolver.PlateByVIN(context.Background(), vin); err != nil { + t.Fatalf("PlateByVIN(%s) error = %v", vin, err) + } + now = now.Add(time.Second) + } + + stats := resolver.CacheStats() + if stats.Entries != 2 || stats.MaxEntries != 2 || stats.Evictions != 1 { + t.Fatalf("cache stats = %+v, want entries=2 max=2 evictions=1", stats) + } + if observed != stats { + t.Fatalf("observed stats = %+v, want %+v", observed, stats) + } + if _, ok := resolver.entries["VIN001"]; ok { + t.Fatalf("oldest VIN should be evicted") + } + if _, ok := resolver.entries["VIN002"]; !ok { + t.Fatalf("VIN002 should remain cached") + } + if _, ok := resolver.entries["VIN003"]; !ok { + t.Fatalf("VIN003 should remain cached") + } +} + +func TestCachedPlateResolverCanDisableEntryLimit(t *testing.T) { + delegate := &mapPlateResolver{plates: map[string]string{ + "VIN001": "沪A00001", + "VIN002": "沪A00002", + }} + resolver := NewCachedPlateResolverWithMaxEntries(delegate, time.Hour, 0) + for _, vin := range []string{"VIN001", "VIN002"} { + if _, err := resolver.PlateByVIN(context.Background(), vin); err != nil { + t.Fatalf("PlateByVIN(%s) error = %v", vin, err) + } + } + stats := resolver.CacheStats() + if stats.Entries != 2 || stats.MaxEntries != 0 || stats.Evictions != 0 { + t.Fatalf("cache stats = %+v, want unlimited cache with two entries", stats) + } +} + func TestSnapshotWriterKeepsEventPlateWhenPresent(t *testing.T) { exec := &recordingSnapshotExec{} resolver := &recordingPlateResolver{plate: "沪B99999"} @@ -521,9 +834,9 @@ func TestSnapshotWriterKeepsEventPlateWhenPresent(t *testing.T) { Plate: "沪A12345", EventTimeMS: 1782918600000, ReceivedAtMS: 1782918601000, - Fields: map[string]any{ - envelope.FieldLatitude: 30.590151, - envelope.FieldLongitude: 121.069881, + ParsedFields: map[string]any{ + "jt808.location.latitude": 30.590151, + "jt808.location.longitude": 121.069881, }, }); err != nil { t.Fatalf("Update() error = %v", err) @@ -548,7 +861,7 @@ func TestSnapshotWriterIgnoresMissingBindingPlate(t *testing.T) { EventTimeMS: 1782918600000, ReceivedAtMS: 1782918601000, Parsed: sampleGB32960RealtimeParsed(), - Fields: map[string]any{envelope.FieldSOCPercent: 90}, + ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 90}, }); err != nil { t.Fatalf("Update() error = %v", err) } @@ -568,7 +881,7 @@ func TestSnapshotWriterReturnsUnexpectedPlateLookupError(t *testing.T) { EventTimeMS: 1782918600000, ReceivedAtMS: 1782918601000, Parsed: sampleGB32960RealtimeParsed(), - Fields: map[string]any{envelope.FieldSOCPercent: 90}, + ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 90}, }) if err == nil || !strings.Contains(err.Error(), "db down") { t.Fatalf("Update() error = %v, want db down", err) @@ -701,6 +1014,9 @@ func TestSnapshotWriterSkipsMQTTWithoutActualDataFields(t *testing.T) { "topic": "/vehicle/VIN001/state", "data": map[string]any{}, }, + ParsedFields: map[string]any{ + "yutong_mqtt.metadata.topic": "/vehicle/VIN001/state", + }, }); err != nil { t.Fatalf("Update() error = %v", err) } @@ -709,6 +1025,27 @@ func TestSnapshotWriterSkipsMQTTWithoutActualDataFields(t *testing.T) { } } +func TestSnapshotWriterAcceptsGB32960RetransmissionFields(t *testing.T) { + exec := &recordingSnapshotExec{} + writer := NewSnapshotWriter(exec) + + if err := writer.Update(context.Background(), envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: "0x03", + VIN: "VIN001", + EventTimeMS: 1782918600000, + ReceivedAtMS: 1782918601000, + ParsedFields: map[string]any{ + "gb32960.vehicle.soc_percent": 90, + }, + }); err != nil { + t.Fatalf("Update() error = %v", err) + } + if len(exec.calls) != 1 || !strings.Contains(exec.calls[0].query, "vehicle_realtime_snapshot") { + t.Fatalf("GB32960 retransmission should update snapshot, calls=%#v", exec.calls) + } +} + func TestSnapshotWriterSkipsEmptyDataUnitsWithoutCoreFields(t *testing.T) { exec := &recordingSnapshotExec{} writer := NewSnapshotWriter(exec) @@ -740,6 +1077,14 @@ func sampleGB32960RealtimeParsed() map[string]any { } } +func sampleGB32960LocationFields() map[string]any { + return map[string]any{ + "gb32960.position.latitude": 30.590151, + "gb32960.position.longitude": 121.069881, + "gb32960.vehicle.soc_percent": 90, + } +} + type snapshotExecCall struct { query string args []any @@ -763,6 +1108,18 @@ func (r *recordingPlateResolver) PlateByVIN(_ context.Context, vin string) (stri return r.plate, r.err } +type mapPlateResolver struct { + plates map[string]string +} + +func (r *mapPlateResolver) PlateByVIN(_ context.Context, vin string) (string, error) { + plate := strings.TrimSpace(r.plates[strings.TrimSpace(vin)]) + if plate == "" { + return "", sql.ErrNoRows + } + return plate, nil +} + type jsonFlatFieldsArg struct { fields map[string]string forbidden []string diff --git a/go/vehicle-gateway/internal/stats/daily_metric.go b/go/vehicle-gateway/internal/stats/daily_metric.go index f80338b9..8b491a01 100644 --- a/go/vehicle-gateway/internal/stats/daily_metric.go +++ b/go/vehicle-gateway/internal/stats/daily_metric.go @@ -3,7 +3,6 @@ package stats import ( "context" "database/sql" - "errors" "fmt" "strconv" "strings" @@ -11,6 +10,15 @@ import ( "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry" +) + +const ( + maxNegativeMileageJitterKM = 1.0 + defaultCacheRetention = 72 * time.Hour + defaultCacheCleanupInterval = 10 * time.Minute + defaultBaselineMissTTL = time.Minute + defaultMaxCacheEntries = 1000000 ) type Execer interface { @@ -18,14 +26,26 @@ type Execer interface { } type Writer struct { - exec Execer - query Queryer - loc *time.Location - sourceTouchInterval time.Duration - mu sync.Mutex - lastTotalMileage map[string]float64 - lastSourceSeen map[string]time.Time - baselineCache map[string]sourceBaselineCacheEntry + exec Execer + query Queryer + loc *time.Location + sourceTouchInterval time.Duration + projectionInterval time.Duration + cacheRetention time.Duration + cacheCleanupInterval time.Duration + baselineMissTTL time.Duration + maxCacheEntries int + lastCacheCleanup time.Time + lastCacheCleanupStats cacheCleanupStats + cacheEvictions cacheCleanupStats + mu sync.Mutex + lastTotalMileage map[string]float64 + lastSourceSeen map[string]time.Time + lastProjection map[string]projectionCacheEntry + baselineCache map[string]sourceBaselineCacheEntry + mileageKeysByPrefix map[string]map[string]struct{} + projectionKeysByPrefix map[string]map[string]struct{} + baselineKeysByPrefix map[string]map[string]struct{} } type MetricSample struct { @@ -38,6 +58,53 @@ type MetricSample struct { Phone string DeviceID string SourceEndpoint string + PlatformName string +} + +type AppendResult struct { + SamplesFound int + SamplesWritten int + SamplesSkippedMissingFields int + SamplesSkippedMissingVIN int + SamplesSkippedMissingMileage int + SamplesSkippedNonMileageFrame int + SamplesSkippedNonPositiveMileage int + SamplesSkippedMissingTime int + SamplesSkippedSameMileage int + SamplesSkippedMissingSource int + SamplesAdjustedFutureEventTime int + SourceTouchesAttempted int + SourceTouchesWritten int + SourceTouchesSkippedThrottled int + SourceTouchesSkippedMissing int + SourceTouchesSkippedUnmanaged int + ProjectionsAttempted int + ProjectionsWritten int + ProjectionsSkippedThrottled int +} + +type CacheStats struct { + LastTotalMileageEntries int + LastSourceSeenEntries int + LastProjectionEntries int + BaselineEntries int + MaxEntries int + LastCleanupAt time.Time + LastCleanupTotalMileage int + LastCleanupSourceSeen int + LastCleanupProjection int + LastCleanupBaseline int + TotalMileageEvictions int + SourceSeenEvictions int + ProjectionEvictions int + BaselineEvictions int +} + +type cacheCleanupStats struct { + totalMileage int + sourceSeen int + projection int + baseline int } func NewWriter(exec Execer, loc *time.Location) *Writer { @@ -48,12 +115,21 @@ func NewWriter(exec Execer, loc *time.Location) *Writer { loc = time.FixedZone("Asia/Shanghai", 8*3600) } writer := &Writer{ - exec: exec, - loc: loc, - sourceTouchInterval: time.Minute, - lastTotalMileage: map[string]float64{}, - lastSourceSeen: map[string]time.Time{}, - baselineCache: map[string]sourceBaselineCacheEntry{}, + exec: exec, + loc: loc, + sourceTouchInterval: time.Minute, + projectionInterval: 15 * time.Second, + cacheRetention: defaultCacheRetention, + cacheCleanupInterval: defaultCacheCleanupInterval, + baselineMissTTL: defaultBaselineMissTTL, + maxCacheEntries: defaultMaxCacheEntries, + lastTotalMileage: map[string]float64{}, + lastSourceSeen: map[string]time.Time{}, + lastProjection: map[string]projectionCacheEntry{}, + baselineCache: map[string]sourceBaselineCacheEntry{}, + mileageKeysByPrefix: map[string]map[string]struct{}{}, + projectionKeysByPrefix: map[string]map[string]struct{}{}, + baselineKeysByPrefix: map[string]map[string]struct{}{}, } if query, ok := exec.(Queryer); ok { writer.query = query @@ -61,6 +137,82 @@ func NewWriter(exec Execer, loc *time.Location) *Writer { return writer } +func (w *Writer) SetSourceTouchInterval(interval time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + if interval < 0 { + interval = 0 + } + w.sourceTouchInterval = interval +} + +func (w *Writer) SetProjectionInterval(interval time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + if interval < 0 { + interval = 0 + } + w.projectionInterval = interval +} + +func (w *Writer) SetCacheRetention(retention time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + if retention < 0 { + retention = 0 + } + w.cacheRetention = retention +} + +func (w *Writer) SetCacheCleanupInterval(interval time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + if interval < 0 { + interval = 0 + } + w.cacheCleanupInterval = interval +} + +func (w *Writer) SetBaselineMissTTL(ttl time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + if ttl < 0 { + ttl = 0 + } + w.baselineMissTTL = ttl +} + +func (w *Writer) SetMaxCacheEntries(maxEntries int) { + w.mu.Lock() + defer w.mu.Unlock() + if maxEntries < 0 { + maxEntries = 0 + } + w.maxCacheEntries = maxEntries + w.enforceCacheLimitsLocked() +} + +func (w *Writer) CacheStats() CacheStats { + w.mu.Lock() + defer w.mu.Unlock() + return CacheStats{ + LastTotalMileageEntries: len(w.lastTotalMileage), + LastSourceSeenEntries: len(w.lastSourceSeen), + LastProjectionEntries: len(w.lastProjection), + BaselineEntries: len(w.baselineCache), + MaxEntries: w.maxCacheEntries, + LastCleanupAt: w.lastCacheCleanup, + LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage, + LastCleanupSourceSeen: w.lastCacheCleanupStats.sourceSeen, + LastCleanupProjection: w.lastCacheCleanupStats.projection, + LastCleanupBaseline: w.lastCacheCleanupStats.baseline, + TotalMileageEvictions: w.cacheEvictions.totalMileage, + SourceSeenEvictions: w.cacheEvictions.sourceSeen, + ProjectionEvictions: w.cacheEvictions.projection, + BaselineEvictions: w.cacheEvictions.baseline, + } +} + func (w *Writer) EnsureSchema(ctx context.Context) error { for _, statement := range []string{ DataSourceTableSQL, @@ -80,48 +232,121 @@ func (w *Writer) EnsureSchema(ctx context.Context) error { } func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error { - identity, hasSource := NewSourceIdentity(env.Protocol, env.SourceEndpoint) - if hasSource { - seenAt := w.sourceSeenAt(env) + _, err := w.AppendWithResult(ctx, env) + return err +} + +func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) { + var result AppendResult + if len(env.Fields) == 0 { + result.SamplesSkippedMissingFields = 1 + return result, nil + } + seenAt := w.sourceSeenAt(env) + w.maybeCleanupCaches(seenAt) + identity, hasSource := NewSourceIdentityFromEnvelope(env) + var sourceResult AppendResult + if hasSource && ShouldManageDataSource(identity) { if w.shouldTouchSource(identity, seenAt) { + sourceResult.SourceTouchesAttempted = 1 if err := UpsertDataSource(ctx, w.exec, identity, seenAt); err != nil { - return err + return sourceResult, err } w.markSourceTouched(identity, seenAt) + sourceResult.SourceTouchesWritten = 1 + } else { + sourceResult.SourceTouchesSkippedThrottled = 1 } + } else if hasSource { + sourceResult.SourceTouchesSkippedUnmanaged = 1 } - samples, err := SamplesFromEnvelope(env, w.loc) + samples, extractionResult, err := samplesFromEnvelopeWithResult(env, w.loc) + result = extractionResult + result.SourceTouchesAttempted += sourceResult.SourceTouchesAttempted + result.SourceTouchesWritten += sourceResult.SourceTouchesWritten + result.SourceTouchesSkippedThrottled += sourceResult.SourceTouchesSkippedThrottled + result.SourceTouchesSkippedMissing += sourceResult.SourceTouchesSkippedMissing + result.SourceTouchesSkippedUnmanaged += sourceResult.SourceTouchesSkippedUnmanaged if err != nil { - return err + return result, err } for _, sample := range samples { - if w.seenSameMileage(sample) { + if !hasSource { + result.SamplesSkippedMissingSource++ + result.SourceTouchesSkippedMissing++ continue } - if !hasSource { + if w.seenSameMileage(sample) { + result.SamplesSkippedSameMileage++ continue } candidate := SourceMileageSampleFromMetric(sample, identity) if err := w.applyRealtimeBaseline(ctx, &candidate); err != nil { - return err + return result, err } - if err := UpsertSourceMileage(ctx, w.exec, candidate); err != nil { - return err + projectDaily := w.shouldProjectDailyMileage(sample) + if projectDaily { + result.ProjectionsAttempted++ + } else { + result.ProjectionsSkippedThrottled++ } - if err := ProjectDailyMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil { - return err + if err := w.writeMileageSample(ctx, sample, candidate, projectDaily); err != nil { + return result, err + } + w.markBaselineWritten(sample, candidate) + if projectDaily { + w.markProjected(sample) + result.ProjectionsWritten++ } w.markMileageWritten(sample) + result.SamplesWritten++ + } + return result, nil +} + +func (w *Writer) writeMileageSample(ctx context.Context, sample MetricSample, candidate SourceMileageSample, projectDaily bool) error { + if projectDaily { + if beginner, ok := w.exec.(txBeginner); ok { + tx, err := beginner.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := UpsertSourceMileage(ctx, tx, candidate); err != nil { + _ = tx.Rollback() + return err + } + if ShouldNormalizePlatformSourceMileage(candidate) { + if err := NormalizePlatformSourceMileage(ctx, tx, sample.VIN, sample.StatDate, sample.Protocol); err != nil { + _ = tx.Rollback() + return err + } + } + if err := projectDailyMileageWithExec(ctx, tx, sample.VIN, sample.StatDate, sample.Protocol); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() + } + } + if err := UpsertSourceMileage(ctx, w.exec, candidate); err != nil { + return err + } + if projectDaily { + if ShouldNormalizePlatformSourceMileage(candidate) { + if err := NormalizePlatformSourceMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil { + return err + } + } + return ProjectDailyMileage(ctx, w.exec, sample.VIN, sample.StatDate, sample.Protocol) } return nil } func (w *Writer) sourceSeenAt(env envelope.FrameEnvelope) time.Time { - eventMS := env.EventTimeMS - if eventMS <= 0 { - eventMS = env.ReceivedAtMS + if env.ReceivedAtMS > 0 { + return time.UnixMilli(env.ReceivedAtMS).In(w.loc) } - if eventMS > 0 { + if eventMS, ok := statEventTimeMS(env); ok { return time.UnixMilli(eventMS).In(w.loc) } return time.Now().In(w.loc) @@ -131,6 +356,9 @@ func (w *Writer) shouldTouchSource(identity SourceIdentity, seenAt time.Time) bo key := string(identity.Protocol) + "|" + identity.SourceIP w.mu.Lock() defer w.mu.Unlock() + if w.sourceTouchInterval == 0 { + return true + } last, ok := w.lastSourceSeen[key] if ok && !seenAt.After(last.Add(w.sourceTouchInterval)) { return false @@ -142,9 +370,45 @@ func (w *Writer) markSourceTouched(identity SourceIdentity, seenAt time.Time) { key := string(identity.Protocol) + "|" + identity.SourceIP w.mu.Lock() w.lastSourceSeen[key] = seenAt + w.enforceCacheLimitsLocked() w.mu.Unlock() } +func (w *Writer) shouldProjectDailyMileage(sample MetricSample) bool { + key := projectionCacheKey(sample) + w.mu.Lock() + defer w.mu.Unlock() + interval := w.projectionInterval + if interval == 0 { + return true + } + entry, ok := w.lastProjection[key] + if !ok { + return true + } + if _, ok := entry.sourceKeys[sample.SourceKey]; !ok { + return true + } + return sample.EventTime.After(entry.projectedAt.Add(interval)) +} + +func (w *Writer) markProjected(sample MetricSample) { + key := projectionCacheKey(sample) + prefix := projectionCachePrefix(sample) + w.mu.Lock() + defer w.mu.Unlock() + w.deleteProjectionPrefixExceptLocked(prefix, key) + entry := w.lastProjection[key] + if entry.sourceKeys == nil { + entry.sourceKeys = map[string]struct{}{} + } + entry.sourceKeys[sample.SourceKey] = struct{}{} + entry.projectedAt = sample.EventTime + w.lastProjection[key] = entry + w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key) + w.enforceCacheLimitsLocked() +} + func (w *Writer) seenSameMileage(sample MetricSample) bool { key := mileageCacheKey(sample) w.mu.Lock() @@ -160,17 +424,59 @@ func (w *Writer) markMileageWritten(sample MetricSample) { key := mileageCacheKey(sample) w.mu.Lock() defer w.mu.Unlock() - for existing := range w.lastTotalMileage { - if strings.HasPrefix(existing, prefix) && existing != key { - delete(w.lastTotalMileage, existing) - } - } - for existing := range w.baselineCache { - if strings.HasPrefix(existing, prefix) && existing != key { - delete(w.baselineCache, existing) - } - } + w.deleteMileagePrefixExceptLocked(prefix, key) + w.deleteBaselinePrefixExceptLocked(prefix, key) w.lastTotalMileage[key] = sample.TotalMileageKM + w.addCacheKeyLocked(w.mileageKeysByPrefix, prefix, key) + w.enforceCacheLimitsLocked() +} + +func (w *Writer) maybeCleanupCaches(now time.Time) { + if now.IsZero() { + now = time.Now().In(w.loc) + } + w.mu.Lock() + defer w.mu.Unlock() + retention := w.cacheRetention + if retention <= 0 { + w.enforceCacheLimitsLocked() + return + } + interval := w.cacheCleanupInterval + if interval > 0 && !w.lastCacheCleanup.IsZero() && !now.After(w.lastCacheCleanup.Add(interval)) { + w.enforceCacheLimitsLocked() + return + } + w.lastCacheCleanup = now + cleanupStats := cacheCleanupStats{} + cutoff := now.Add(-retention) + cutoffDate := cutoff.In(w.loc).Format("2006-01-02") + for key, seenAt := range w.lastSourceSeen { + if !seenAt.IsZero() && seenAt.Before(cutoff) { + delete(w.lastSourceSeen, key) + cleanupStats.sourceSeen++ + } + } + for key, entry := range w.lastProjection { + if entry.projectedAt.IsZero() || entry.projectedAt.Before(cutoff) || cacheKeyDateBefore(key, cutoffDate) { + w.deleteProjectionKeyLocked(key) + cleanupStats.projection++ + } + } + for key := range w.lastTotalMileage { + if cacheKeyDateBefore(key, cutoffDate) { + w.deleteMileageKeyLocked(key) + cleanupStats.totalMileage++ + } + } + for key := range w.baselineCache { + if cacheKeyDateBefore(key, cutoffDate) { + w.deleteBaselineKeyLocked(key) + cleanupStats.baseline++ + } + } + w.lastCacheCleanupStats = cleanupStats + w.enforceCacheLimitsLocked() } func (w *Writer) applyRealtimeBaseline(ctx context.Context, candidate *SourceMileageSample) error { @@ -182,22 +488,22 @@ func (w *Writer) applyRealtimeBaseline(ctx context.Context, candidate *SourceMil return err } if !found { + // If no earlier odometer exists at all, use the first current-day sample. + // Later samples retain that boundary through the in-memory baseline cache. + candidate.DailyKM = 0 candidate.QualityStatus = QualityOK - candidate.QualityReason = "current_day_first_sample" + candidate.QualityReason = QualityReasonCurrentDayFirst return nil } candidate.FirstTotalKM = baseline.LatestTotalKM candidate.FirstEventTime = baseline.LatestEventTime - candidate.DailyKM = candidate.LatestTotalKM - baseline.LatestTotalKM + candidate.DailyKM = DailyMileageFromDayBoundary(baseline.LatestTotalKM, candidate.LatestTotalKM) candidate.QualityStatus = QualityOK candidate.QualityReason = baseline.QualityReason if candidate.QualityReason == "" { - candidate.QualityReason = "historical_source_baseline" - } - if candidate.DailyKM < 0 || candidate.DailyKM > maxSelectedDailyMileageKM { - candidate.QualityStatus = QualityInvalidDelta - candidate.QualityReason = "outside_daily_range" + candidate.QualityReason = QualityReasonHistorical } + ApplyMileageQualityRules(candidate) return nil } @@ -208,10 +514,15 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa StatDate: candidate.StatDate, SourceKey: candidate.SourceKey, }) + now := time.Now() w.mu.Lock() if cached, ok := w.baselineCache[cacheKey]; ok { - w.mu.Unlock() - return cached.baseline, cached.found, nil + missExpired := !cached.found && (w.baselineMissTTL == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= w.baselineMissTTL) + if !missExpired { + w.mu.Unlock() + return cached.baseline, cached.found, nil + } + w.deleteBaselineKeyLocked(cacheKey) } w.mu.Unlock() @@ -219,21 +530,165 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa if err != nil { return sourceBaseline{}, false, err } - if !found { - baseline, found, err = lookupCurrentSourceBaseline(ctx, w.query, candidate.VIN, candidate.StatDate, candidate.Protocol, candidate.SourceKey) - if err != nil { - return sourceBaseline{}, false, err - } - } if found { - w.mu.Lock() - w.baselineCache[cacheKey] = sourceBaselineCacheEntry{baseline: baseline, found: true} - w.mu.Unlock() + w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{baseline: baseline, found: true}) + } else { + w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{found: false}) } return baseline, found, nil } +func (w *Writer) markBaselineWritten(sample MetricSample, candidate SourceMileageSample) { + if candidate.QualityStatus != QualityOK { + return + } + baselineTotal := candidate.FirstTotalKM + if baselineTotal <= 0 { + baselineTotal = candidate.LatestTotalKM + } + baselineTime := candidate.FirstEventTime + if baselineTime.IsZero() { + baselineTime = candidate.LatestEventTime + } + if baselineTotal <= 0 || baselineTime.IsZero() { + return + } + cacheKey := mileageCacheKey(sample) + w.cacheBaseline(candidate, cacheKey, sourceBaselineCacheEntry{ + baseline: sourceBaseline{ + LatestTotalKM: baselineTotal, + LatestEventTime: baselineTime, + QualityReason: candidate.QualityReason, + }, + found: true, + }) +} + +func (w *Writer) cacheBaseline(candidate SourceMileageSample, cacheKey string, entry sourceBaselineCacheEntry) { + if cacheKey == "" { + return + } + w.mu.Lock() + if entry.cachedAt.IsZero() { + entry.cachedAt = time.Now() + } + w.baselineCache[cacheKey] = entry + w.addCacheKeyLocked(w.baselineKeysByPrefix, mileageCachePrefix(MetricSample{ + VIN: candidate.VIN, + Protocol: candidate.Protocol, + SourceKey: candidate.SourceKey, + }), cacheKey) + w.enforceCacheLimitsLocked() + w.mu.Unlock() +} + +func (w *Writer) addCacheKeyLocked(index map[string]map[string]struct{}, prefix string, key string) { + if prefix == "" || key == "" { + return + } + keys := index[prefix] + if keys == nil { + keys = map[string]struct{}{} + index[prefix] = keys + } + keys[key] = struct{}{} +} + +func (w *Writer) deleteMileagePrefixExceptLocked(prefix string, keep string) { + for key := range w.mileageKeysByPrefix[prefix] { + if key == keep { + continue + } + w.deleteMileageKeyLocked(key) + } +} + +func (w *Writer) deleteBaselinePrefixExceptLocked(prefix string, keep string) { + for key := range w.baselineKeysByPrefix[prefix] { + if key == keep { + continue + } + w.deleteBaselineKeyLocked(key) + } +} + +func (w *Writer) deleteProjectionPrefixExceptLocked(prefix string, keep string) { + for key := range w.projectionKeysByPrefix[prefix] { + if key == keep { + continue + } + w.deleteProjectionKeyLocked(key) + } +} + +func (w *Writer) deleteMileageKeyLocked(key string) { + delete(w.lastTotalMileage, key) + w.deleteIndexedCacheKeyLocked(w.mileageKeysByPrefix, cacheKeyPrefix(key), key) +} + +func (w *Writer) deleteBaselineKeyLocked(key string) { + delete(w.baselineCache, key) + w.deleteIndexedCacheKeyLocked(w.baselineKeysByPrefix, cacheKeyPrefix(key), key) +} + +func (w *Writer) deleteProjectionKeyLocked(key string) { + delete(w.lastProjection, key) + w.deleteIndexedCacheKeyLocked(w.projectionKeysByPrefix, cacheKeyPrefix(key), key) +} + +func (w *Writer) deleteIndexedCacheKeyLocked(index map[string]map[string]struct{}, prefix string, key string) { + if prefix == "" || key == "" { + return + } + keys := index[prefix] + if len(keys) == 0 { + return + } + delete(keys, key) + if len(keys) == 0 { + delete(index, prefix) + } +} + +func (w *Writer) enforceCacheLimitsLocked() { + if w.maxCacheEntries <= 0 { + return + } + for len(w.lastTotalMileage) > w.maxCacheEntries { + victim := oldestCacheKeyByDate(w.lastTotalMileage) + if victim == "" { + break + } + w.deleteMileageKeyLocked(victim) + w.cacheEvictions.totalMileage++ + } + for len(w.baselineCache) > w.maxCacheEntries { + victim := oldestCacheKeyByDate(w.baselineCache) + if victim == "" { + break + } + w.deleteBaselineKeyLocked(victim) + w.cacheEvictions.baseline++ + } + for len(w.lastProjection) > w.maxCacheEntries { + victim := oldestProjectionCacheKey(w.lastProjection) + if victim == "" { + break + } + w.deleteProjectionKeyLocked(victim) + w.cacheEvictions.projection++ + } + for len(w.lastSourceSeen) > w.maxCacheEntries { + victim := oldestSourceSeenKey(w.lastSourceSeen) + if victim == "" { + break + } + delete(w.lastSourceSeen, victim) + w.cacheEvictions.sourceSeen++ + } +} + func mileageCachePrefix(sample MetricSample) string { return fmt.Sprintf("%s|%s|%s|", sample.VIN, sample.Protocol, sample.SourceKey) } @@ -242,34 +697,138 @@ func mileageCacheKey(sample MetricSample) string { return mileageCachePrefix(sample) + sample.StatDate } +func projectionCacheKey(sample MetricSample) string { + return projectionCachePrefix(sample) + sample.StatDate +} + +func projectionCachePrefix(sample MetricSample) string { + return fmt.Sprintf("%s|%s|", sample.VIN, sample.Protocol) +} + +func cacheKeyDateBefore(key string, cutoffDate string) bool { + date := cacheKeyDate(key) + if len(date) != len("2006-01-02") { + return false + } + return date < cutoffDate +} + +func cacheKeyPrefix(key string) string { + index := strings.LastIndex(key, "|") + if index < 0 { + return "" + } + return key[:index+1] +} + +func cacheKeyDate(key string) string { + index := strings.LastIndex(key, "|") + if index < 0 || index == len(key)-1 { + return "" + } + return key[index+1:] +} + +func oldestCacheKeyByDate[T any](items map[string]T) string { + victim := "" + victimDate := "" + for key := range items { + date := cacheKeyDate(key) + if date == "" { + date = "9999-99-99" + } + if victim == "" || date < victimDate || (date == victimDate && key < victim) { + victim = key + victimDate = date + } + } + return victim +} + +func oldestProjectionCacheKey(items map[string]projectionCacheEntry) string { + victim := "" + var victimTime time.Time + for key, entry := range items { + if victim == "" || + (!entry.projectedAt.IsZero() && (victimTime.IsZero() || entry.projectedAt.Before(victimTime))) || + (entry.projectedAt.Equal(victimTime) && key < victim) { + victim = key + victimTime = entry.projectedAt + } + } + if victim != "" { + return victim + } + return oldestCacheKeyByDate(items) +} + +func oldestSourceSeenKey(items map[string]time.Time) string { + victim := "" + var victimTime time.Time + for key, seenAt := range items { + if victim == "" || + (!seenAt.IsZero() && (victimTime.IsZero() || seenAt.Before(victimTime))) || + (seenAt.Equal(victimTime) && key < victim) { + victim = key + victimTime = seenAt + } + } + return victim +} + +type projectionCacheEntry struct { + projectedAt time.Time + sourceKeys map[string]struct{} +} + type sourceBaselineCacheEntry struct { baseline sourceBaseline found bool + cachedAt time.Time } func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) { + samples, _, err := samplesFromEnvelopeWithResult(env, loc) + return samples, err +} + +func samplesFromEnvelopeWithResult(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, AppendResult, error) { + var result AppendResult + if len(env.Fields) == 0 { + result.SamplesSkippedMissingFields = 1 + return nil, result, nil + } vin := strings.TrimSpace(env.VIN) if vin == "" { - return nil, nil + result.SamplesSkippedMissingVIN = 1 + return nil, result, nil } totalMileage, ok := totalMileageKMFromEnvelope(env) if !ok { - return nil, nil + if isMileageCandidateEnvelope(env) { + result.SamplesSkippedMissingMileage = 1 + } else { + result.SamplesSkippedNonMileageFrame = 1 + } + return nil, result, nil } if totalMileage <= 0 { - return nil, nil + result.SamplesSkippedNonPositiveMileage = 1 + return nil, result, nil } if loc == nil { loc = time.FixedZone("Asia/Shanghai", 8*3600) } - eventMS := env.EventTimeMS - if eventMS <= 0 { - eventMS = env.ReceivedAtMS + eventMS, reason, ok := envelope.NormalizedEventTimeMSWithReason(env) + if !ok { + result.SamplesSkippedMissingTime = 1 + return nil, result, nil } - if eventMS <= 0 { - return nil, errors.New("event or received time is required") + if reason == envelope.EventTimeReasonReceivedFutureEvent { + result.SamplesAdjustedFutureEventTime = 1 } statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02") + result.SamplesFound = 1 return []MetricSample{{ VIN: vin, Protocol: env.Protocol, @@ -280,11 +839,16 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr Phone: strings.TrimSpace(env.Phone), DeviceID: strings.TrimSpace(env.DeviceID), SourceEndpoint: strings.TrimSpace(env.SourceEndpoint), - }}, nil + PlatformName: strings.TrimSpace(env.PlatformName), + }}, result, nil +} + +func statEventTimeMS(env envelope.FrameEnvelope) (int64, bool) { + return envelope.NormalizedEventTimeMS(env) } func sourceKey(env envelope.FrameEnvelope) string { - return SourceKey(env.Protocol, env.Phone, env.DeviceID, NormalizeSourceIP(env.SourceEndpoint)) + return SourceKeyForSource(env.Protocol, env.Phone, env.DeviceID, NormalizeSourceIP(env.SourceEndpoint), env.SourceKind, env.SourceCode) } func isIgnorableSchemaChangeError(err error) bool { @@ -300,41 +864,59 @@ func isIgnorableSchemaChangeError(err error) bool { strings.Contains(text, "1091") } -type mileageFieldMapping struct { - key string - scale float64 -} +type mileageFieldMapping = telemetry.MileageFieldMapping func totalMileageKMFromEnvelope(env envelope.FrameEnvelope) (float64, bool) { - if value, ok := floatField(env, envelope.FieldTotalMileageKM); ok { - return value, true - } - for _, mapping := range mileageMappingsByProtocol(env.Protocol) { - value, ok := floatField(env, mapping.key) - if !ok { - continue - } - return value * mapping.scale, true - } - return 0, false + return telemetry.TotalMileageKM(env.Protocol, env.Fields) } func mileageMappingsByProtocol(protocol envelope.Protocol) []mileageFieldMapping { - switch protocol { - case envelope.ProtocolGB32960: - return []mileageFieldMapping{{key: "gb32960.vehicle.total_mileage_km", scale: 1}} + return telemetry.MileageFieldMappings(protocol) +} + +func isMileageCandidateEnvelope(env envelope.FrameEnvelope) bool { + switch env.Protocol { case envelope.ProtocolJT808: - return []mileageFieldMapping{{key: "jt808.location.total_mileage_km", scale: 1}} + messageID := strings.TrimSpace(env.MessageID) + return strings.EqualFold(messageID, "0x0200") || + hasFieldPrefix(env.Fields, "jt808.location.") || + hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH) + case envelope.ProtocolGB32960: + return hasFieldPrefix(env.Fields, "gb32960.vehicle.") || + hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH, envelope.FieldSOCPercent) case envelope.ProtocolYutongMQTT: - return []mileageFieldMapping{ - {key: "yutong_mqtt.data.total_mileage", scale: 0.001}, - {key: "yutong_mqtt.root.data.total_mileage", scale: 0.001}, - } + return hasFieldPrefix(env.Fields, "yutong_mqtt.data.") || + hasFieldPrefix(env.Fields, "yutong_mqtt.root.data.") || + hasAnyField(env.Fields, envelope.FieldLatitude, envelope.FieldLongitude, envelope.FieldSpeedKMH, envelope.FieldSOCPercent) default: - return nil + return true } } +func hasFieldPrefix(fields map[string]any, prefix string) bool { + if len(fields) == 0 || prefix == "" { + return false + } + for key := range fields { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func hasAnyField(fields map[string]any, keys ...string) bool { + if len(fields) == 0 { + return false + } + for _, key := range keys { + if _, ok := fields[key]; ok { + return true + } + } + return false +} + func floatField(env envelope.FrameEnvelope, key string) (float64, bool) { if env.Fields == nil { return 0, false diff --git a/go/vehicle-gateway/internal/stats/daily_metric_test.go b/go/vehicle-gateway/internal/stats/daily_metric_test.go index ccaaae7d..ad048597 100644 --- a/go/vehicle-gateway/internal/stats/daily_metric_test.go +++ b/go/vehicle-gateway/internal/stats/daily_metric_test.go @@ -5,8 +5,10 @@ import ( "database/sql" "database/sql/driver" "errors" + "fmt" "math" "strings" + "sync" "testing" "time" @@ -14,6 +16,13 @@ import ( "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) +func managedTestSource(env envelope.FrameEnvelope) envelope.FrameEnvelope { + if strings.TrimSpace(env.SourceKind) == "" { + env.SourceKind = "PLATFORM" + } + return env +} + func TestSamplesFromEnvelopeDerivesDailyMileageSample(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) env := envelope.FrameEnvelope{ @@ -21,7 +30,7 @@ func TestSamplesFromEnvelopeDerivesDailyMileageSample(t *testing.T) { VIN: "LNBVIN00000000001", EventTimeMS: time.Date(2026, 7, 1, 8, 30, 0, 0, loc).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: 10241.2, + "jt808.location.total_mileage_km": 10241.2, }, } @@ -69,12 +78,24 @@ func TestSamplesFromEnvelopeMapsProtocolMileageFields(t *testing.T) { fields: map[string]any{"yutong_mqtt.data.total_mileage": "120756000"}, wantKM: 120756, }, + { + name: "yutong mqtt km", + protocol: envelope.ProtocolYutongMQTT, + fields: map[string]any{"yutong_mqtt.data.total_mileage_km": "120756"}, + wantKM: 120756, + }, { name: "yutong mqtt root meters", protocol: envelope.ProtocolYutongMQTT, fields: map[string]any{"yutong_mqtt.root.data.total_mileage": float64(22858000)}, wantKM: 22858, }, + { + name: "yutong mqtt root km", + protocol: envelope.ProtocolYutongMQTT, + fields: map[string]any{"yutong_mqtt.root.data.total_mileage_km": float64(22858)}, + wantKM: 22858, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -97,6 +118,90 @@ func TestSamplesFromEnvelopeMapsProtocolMileageFields(t *testing.T) { } } +func TestSamplesFromEnvelopeFallsBackToReceivedTimeWhenEventTimeIsFuture(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + received := time.Date(2026, 7, 8, 23, 59, 0, 0, loc) + futureEvent := received.Add(48 * time.Hour) + samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + EventTimeMS: futureEvent.UnixMilli(), + ReceivedAtMS: received.UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4136.8, + }, + }, loc) + if err != nil { + t.Fatalf("SamplesFromEnvelope() error = %v", err) + } + if len(samples) != 1 { + t.Fatalf("sample count = %d", len(samples)) + } + if samples[0].StatDate != "2026-07-08" { + t.Fatalf("stat date = %q, want received date", samples[0].StatDate) + } + if !samples[0].EventTime.Equal(received) { + t.Fatalf("event time = %s, want received time %s", samples[0].EventTime, received) + } + _, result, err := samplesFromEnvelopeWithResult(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + EventTimeMS: futureEvent.UnixMilli(), + ReceivedAtMS: received.UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4136.8, + }, + }, loc) + if err != nil { + t.Fatalf("samplesFromEnvelopeWithResult() error = %v", err) + } + if result.SamplesAdjustedFutureEventTime != 1 { + t.Fatalf("future event adjustment count = %d, want 1", result.SamplesAdjustedFutureEventTime) + } +} + +func TestSamplesFromEnvelopeKeepsSmallFutureClockSkew(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + received := time.Date(2026, 7, 8, 23, 59, 0, 0, loc) + eventTime := received.Add(2 * time.Minute) + samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + EventTimeMS: eventTime.UnixMilli(), + ReceivedAtMS: received.UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4136.8, + }, + }, loc) + if err != nil { + t.Fatalf("SamplesFromEnvelope() error = %v", err) + } + if len(samples) != 1 { + t.Fatalf("sample count = %d", len(samples)) + } + if samples[0].StatDate != "2026-07-09" { + t.Fatalf("stat date = %q, want event date within skew", samples[0].StatDate) + } + if !samples[0].EventTime.Equal(eventTime) { + t.Fatalf("event time = %s, want event time %s", samples[0].EventTime, eventTime) + } +} + +func TestWriterSourceSeenAtUsesReceivedTimeOverDeviceEventTime(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(&recordingExec{}, loc) + received := time.Date(2026, 7, 8, 13, 20, 0, 0, loc) + staleDeviceEvent := received.Add(-72 * time.Hour) + + got := writer.sourceSeenAt(envelope.FrameEnvelope{ + EventTimeMS: staleDeviceEvent.UnixMilli(), + ReceivedAtMS: received.UnixMilli(), + }) + if !got.Equal(received) { + t.Fatalf("sourceSeenAt = %s, want received time %s", got, received) + } +} + func TestSamplesFromEnvelopeSkipsMissingVIN(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{ @@ -104,7 +209,7 @@ func TestSamplesFromEnvelopeSkipsMissingVIN(t *testing.T) { Phone: "13307811254", EventTimeMS: time.Date(2026, 7, 2, 0, 3, 0, 0, loc).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: 10985.7, + "jt808.location.total_mileage_km": 10985.7, }, }, loc) if err != nil { @@ -118,7 +223,7 @@ func TestSamplesFromEnvelopeSkipsMissingVIN(t *testing.T) { func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) { samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, - Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2}, + Fields: map[string]any{"jt808.location.total_mileage_km": 1.2}, }, nil) if err != nil { t.Fatalf("SamplesFromEnvelope() error = %v", err) @@ -147,7 +252,7 @@ func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) { VIN: "LNBVIN00000000001", EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: value, + "jt808.location.total_mileage_km": value, }, }, nil) if err != nil { @@ -167,6 +272,9 @@ func TestWriterAppendWritesSourceCandidateAndProjection(t *testing.T) { VIN: "LA9GG64L7PBAF4001", Phone: "13307765812", SourceEndpoint: "115.231.168.135:20215", + SourceCode: "g7s", + PlatformName: "G7s", + SourceKind: "PLATFORM", EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), Fields: map[string]any{ "jt808.location.total_mileage_km": 4123.9, @@ -176,17 +284,68 @@ func TestWriterAppendWritesSourceCandidateAndProjection(t *testing.T) { if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) } - if len(exec.calls) != 6 { - t.Fatalf("exec calls = %d, want source + candidate + clear + project + mark + cleanup", len(exec.calls)) + if len(exec.calls) != 7 { + t.Fatalf("exec calls = %d, want source + candidate + normalize insert/delete + project + reconcile + cleanup", len(exec.calls)) } if !strings.Contains(exec.calls[0].query, "INSERT INTO vehicle_data_source") { t.Fatalf("first call should upsert source: %s", exec.calls[0].query) } + if exec.calls[0].args[3] != "G7s" || exec.calls[0].args[4] != "g7s" || exec.calls[0].args[5] != "PLATFORM" { + t.Fatalf("source metadata args = %#v", exec.calls[0].args) + } if !strings.Contains(exec.calls[1].query, "INSERT INTO vehicle_daily_mileage_source") { t.Fatalf("second call should upsert candidate: %s", exec.calls[1].query) } - if !strings.Contains(exec.calls[3].query, "INSERT INTO vehicle_daily_mileage") { - t.Fatalf("fourth call should project final: %s", exec.calls[3].query) + if exec.calls[1].args[8] != "G7s" { + t.Fatalf("source mileage platform arg = %#v", exec.calls[1].args) + } + if !strings.Contains(exec.calls[2].query, "INSERT INTO vehicle_daily_mileage_source") || + !strings.Contains(exec.calls[2].query, "SELECT") || + !strings.Contains(exec.calls[2].query, "@PLATFORM:") { + t.Fatalf("third call should normalize legacy platform source rows: %s", exec.calls[2].query) + } + if !strings.Contains(exec.calls[3].query, "DELETE s") || + !strings.Contains(exec.calls[3].query, "@PLATFORM:") { + t.Fatalf("fourth call should delete normalized legacy platform rows: %s", exec.calls[3].query) + } + if !strings.Contains(exec.calls[4].query, "INSERT INTO vehicle_daily_mileage") { + t.Fatalf("fifth call should project final: %s", exec.calls[4].query) + } +} + +func TestWriterAppendWithResultReportsWrittenSample(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 1 || result.SamplesWritten != 1 { + t.Fatalf("result = %+v, want one found and written sample", result) + } + if result.ProjectionsAttempted != 1 || result.ProjectionsWritten != 1 || result.ProjectionsSkippedThrottled != 0 { + t.Fatalf("projection result = %+v, want one final projection", result) + } + if result.SourceTouchesSkippedUnmanaged != 1 || result.SourceTouchesAttempted != 0 || result.SourceTouchesWritten != 0 || result.SourceTouchesSkippedMissing != 0 { + t.Fatalf("source touch result = %+v, want unmanaged source skipped", result) + } + if result.SamplesSkippedMissingVIN != 0 || + result.SamplesSkippedMissingMileage != 0 || + result.SamplesSkippedNonPositiveMileage != 0 || + result.SamplesSkippedSameMileage != 0 || + result.SamplesSkippedMissingSource != 0 { + t.Fatalf("unexpected skipped result counters: %+v", result) } } @@ -211,10 +370,41 @@ func TestWriterAppendSkipsMileageWithoutSourceIdentity(t *testing.T) { } } -func TestWriterAppendTracksSourceWithoutMileageSample(t *testing.T) { +func TestWriterAppendWithResultReportsMissingSource(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 1 || result.SamplesSkippedMissingSource != 1 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want found sample skipped by missing source", result) + } + if result.SourceTouchesSkippedMissing != 1 || result.SourceTouchesAttempted != 0 || result.SourceTouchesWritten != 0 { + t.Fatalf("source touch result = %+v, want missing source endpoint", result) + } + if result.ProjectionsAttempted != 0 || result.ProjectionsWritten != 0 || result.ProjectionsSkippedThrottled != 0 { + t.Fatalf("projection result = %+v, want no projection without source", result) + } + if len(exec.calls) != 0 { + t.Fatalf("exec calls = %d, want 0 without source identity", len(exec.calls)) + } +} + +func TestWriterAppendTracksSourceWithoutMileageSample(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, Phone: "13307795518", SourceEndpoint: "222.66.200.68:43614", @@ -222,7 +412,7 @@ func TestWriterAppendTracksSourceWithoutMileageSample(t *testing.T) { Fields: map[string]any{ "jt808.location.speed_kmh": 38.5, }, - } + }) if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) @@ -238,10 +428,209 @@ func TestWriterAppendTracksSourceWithoutMileageSample(t *testing.T) { } } +func TestWriterAppendWithResultReportsMissingMileage(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.speed_kmh": 38.5, + }, + }) + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedMissingMileage != 1 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want missing mileage", result) + } + if result.SourceTouchesAttempted != 1 || result.SourceTouchesWritten != 1 || result.SourceTouchesSkippedThrottled != 0 { + t.Fatalf("source touch result = %+v, want source touch before missing mileage skip", result) + } + if result.ProjectionsAttempted != 0 || result.ProjectionsWritten != 0 || result.ProjectionsSkippedThrottled != 0 { + t.Fatalf("projection result = %+v, want no projection for missing mileage", result) + } +} + +func TestWriterAppendWithResultClassifiesGB32960VendorFrameAsNonMileage(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "LB9A32A24R0LS1426", + SourceEndpoint: "115.29.187.205:32960", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "gb32960.gd_fc_stack.stack_water_outlet_temp_c": 63, + "gb32960.gd_fc_stack.hydrogen_inlet_pressure_kpa": 130, + }, + }) + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedNonMileageFrame != 1 || result.SamplesSkippedMissingMileage != 0 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want non-mileage frame", result) + } + if result.SourceTouchesAttempted != 1 || result.SourceTouchesWritten != 1 { + t.Fatalf("source touch result = %+v, want source touch before non-mileage skip", result) + } +} + +func TestWriterAppendWithResultKeepsGB32960VehicleFrameMissingMileageActionable(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + VIN: "LB9A32A24R0LS1426", + SourceEndpoint: "115.29.187.205:32960", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "gb32960.vehicle.speed_kmh": 38.5, + "gb32960.vehicle.soc_percent": 81.2, + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedMissingMileage != 1 || result.SamplesSkippedNonMileageFrame != 0 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want actionable missing mileage", result) + } +} + +func TestWriterAppendWithResultClassifiesYutongSparseFrameAsMissingMileage(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC2R1004087", + SourceEndpoint: "mqtt://yutong/ytforward/shln/3", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "yutong_mqtt.data.latitude": 30.590921, + "yutong_mqtt.data.longitude": 121.075044, + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedMissingMileage != 1 || result.SamplesSkippedNonMileageFrame != 0 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want missing mileage", result) + } +} + +func TestWriterAppendWithResultClassifiesYutongMetadataOnlyFrameAsNonMileage(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + VIN: "LMRKH9AC2R1004087", + SourceEndpoint: "mqtt://yutong/ytforward/shln/3", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "yutong_mqtt.metadata.topic": "/ytforward/shln/3", + "yutong_mqtt.root.device": "LMRKH9AC2R1004087", + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedNonMileageFrame != 1 || result.SamplesSkippedMissingMileage != 0 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want non-mileage frame", result) + } +} + +func TestWriterAppendWithResultReportsMissingTime(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesFound != 0 || result.SamplesSkippedMissingTime != 1 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want missing time skip", result) + } + if result.SourceTouchesAttempted != 1 || result.SourceTouchesWritten != 1 || result.SourceTouchesSkippedThrottled != 0 { + t.Fatalf("source touch result = %+v, want source touch before missing time skip", result) + } + if result.ProjectionsAttempted != 0 || result.ProjectionsWritten != 0 || result.ProjectionsSkippedThrottled != 0 { + t.Fatalf("projection result = %+v, want no projection for missing time", result) + } +} + +func TestWriterAppendWithResultReportsMissingFieldsAndDoesNotTouchSource(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Parsed: map[string]any{ + "location": map[string]any{"total_mileage_km": 4123.9}, + }, + } + + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("AppendWithResult() error = %v", err) + } + if result.SamplesSkippedMissingFields != 1 || result.SamplesSkippedMissingMileage != 0 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want missing fields only", result) + } + if result.SourceTouchesAttempted != 0 || result.SourceTouchesWritten != 0 || result.SourceTouchesSkippedMissing != 0 { + t.Fatalf("source touch result = %+v, want no source touch without fields", result) + } + if len(exec.calls) != 0 { + t.Fatalf("exec calls = %d, want 0 for missing fields", len(exec.calls)) + } +} + +func TestSamplesFromEnvelopeUsesFieldsOnly(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + _, result, err := samplesFromEnvelopeWithResult(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, loc).UnixMilli(), + Parsed: map[string]any{ + "location": map[string]any{"total_mileage_km": 4123.9}, + }, + }, loc) + if err != nil { + t.Fatalf("samplesFromEnvelopeWithResult() error = %v", err) + } + if result.SamplesSkippedMissingFields != 1 || result.SamplesFound != 0 { + t.Fatalf("result = %+v, want missing fields and no extracted samples", result) + } +} + func TestWriterAppendDedupesRealtimeMileagePerSource(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) - base := envelope.FrameEnvelope{ + base := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LA9GG64L7PBAF4001", Phone: "13307765812", @@ -249,7 +638,7 @@ func TestWriterAppendDedupesRealtimeMileagePerSource(t *testing.T) { Fields: map[string]any{ "jt808.location.total_mileage_km": 4123.9, }, - } + }) sourceA := base sourceA.SourceEndpoint = "115.231.168.135:20215" @@ -278,6 +667,196 @@ func TestWriterAppendDedupesRealtimeMileagePerSource(t *testing.T) { } } +func TestWriterAppendCollapsesDirectSourceAcrossChangingIPs(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + base := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceKind: "DIRECT", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + } + + first := base + first.SourceEndpoint = "115.231.168.135:20215" + if err := writer.Append(context.Background(), first); err != nil { + t.Fatalf("Append(first) error = %v", err) + } + + second := base + second.SourceEndpoint = "39.144.3.22:60177" + second.ReceivedAtMS = second.EventTimeMS + 1000 + if err := writer.Append(context.Background(), second); err != nil { + t.Fatalf("Append(second) error = %v", err) + } + + var sourceKeys []string + for _, call := range exec.calls { + if strings.Contains(call.query, "INSERT INTO vehicle_daily_mileage_source") && len(call.args) > 3 { + sourceKeys = append(sourceKeys, call.args[3].(string)) + } + } + if len(sourceKeys) != 1 { + t.Fatalf("source mileage upserts = %d, want one because identical mileage from same direct device should be deduplicated: %#v", len(sourceKeys), sourceKeys) + } + if sourceKeys[0] != "JT808:13307765812@DIRECT" { + t.Fatalf("source key = %q", sourceKeys[0]) + } +} + +func TestWriterAppendCollapsesPlatformSourceAcrossChangingIPs(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + base := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LNXNEGRR0SR321372", + Phone: "41456413943", + SourceCode: "guangan_beidou", + PlatformName: "广安北斗", + SourceKind: "PLATFORM", + EventTimeMS: time.Date(2026, 7, 12, 8, 40, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 46484.2, + }, + } + + first := base + first.SourceEndpoint = "117.132.194.167:9806" + if err := writer.Append(context.Background(), first); err != nil { + t.Fatalf("Append(first) error = %v", err) + } + + second := base + second.SourceEndpoint = "117.132.198.90:12084" + second.EventTimeMS = second.EventTimeMS + 1000 + second.Fields = map[string]any{"jt808.location.total_mileage_km": 46484.3} + if err := writer.Append(context.Background(), second); err != nil { + t.Fatalf("Append(second) error = %v", err) + } + + var sourceKeys []string + for _, call := range exec.calls { + if strings.Contains(call.query, "INSERT INTO vehicle_daily_mileage_source") && len(call.args) > 3 { + sourceKeys = append(sourceKeys, call.args[3].(string)) + } + } + if len(sourceKeys) != 2 { + t.Fatalf("source mileage upserts = %d, want both changing mileage samples written: %#v", len(sourceKeys), sourceKeys) + } + for _, sourceKey := range sourceKeys { + if sourceKey != "JT808:41456413943@PLATFORM:guangan_beidou" { + t.Fatalf("source key = %q", sourceKey) + } + } + if got := countExecQueries(exec.calls, "INSERT INTO vehicle_data_source"); got != 2 { + t.Fatalf("source touch upserts = %d, want both platform IPs still observed", got) + } +} + +func TestWriterAppendThrottlesDailyProjectionPerVehicleProtocolDate(t *testing.T) { + exec := &recordingExec{} + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(exec, loc) + base := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + + first := base + first.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 0, 0, loc).UnixMilli() + if err := writer.Append(context.Background(), first); err != nil { + t.Fatalf("first Append() error = %v", err) + } + second := base + second.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 5, 0, loc).UnixMilli() + second.Fields = map[string]any{"jt808.location.total_mileage_km": 4124.2} + if err := writer.Append(context.Background(), second); err != nil { + t.Fatalf("second Append() error = %v", err) + } + third := base + third.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 16, 0, loc).UnixMilli() + third.Fields = map[string]any{"jt808.location.total_mileage_km": 4125.0} + if err := writer.Append(context.Background(), third); err != nil { + t.Fatalf("third Append() error = %v", err) + } + + if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage_source"); got != 3 { + t.Fatalf("source candidate upserts = %d, want 3", got) + } + if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage\n"); got != 2 { + t.Fatalf("daily projections = %d, want 2", got) + } +} + +func TestWriterAppendProjectsImmediatelyForNewSource(t *testing.T) { + exec := &recordingExec{} + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(exec, loc) + base := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + EventTimeMS: time.Date(2026, 7, 8, 13, 20, 0, 0, loc).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + + sourceA := base + sourceA.SourceEndpoint = "115.231.168.135:20215" + if err := writer.Append(context.Background(), sourceA); err != nil { + t.Fatalf("Append(sourceA) error = %v", err) + } + sourceB := base + sourceB.SourceEndpoint = "115.159.85.149:28316" + sourceB.Fields = map[string]any{"jt808.location.total_mileage_km": 4124.0} + if err := writer.Append(context.Background(), sourceB); err != nil { + t.Fatalf("Append(sourceB) error = %v", err) + } + + if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage\n"); got != 2 { + t.Fatalf("daily projections = %d, want immediate projection for each new source", got) + } +} + +func TestWriterProjectionIntervalZeroKeepsLegacyPerSampleProjection(t *testing.T) { + exec := &recordingExec{} + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(exec, loc) + writer.SetProjectionInterval(0) + base := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + + for i, mileage := range []float64{4123.9, 4124.0} { + event := base + event.EventTimeMS = time.Date(2026, 7, 8, 13, 20, i, 0, loc).UnixMilli() + event.Fields = map[string]any{"jt808.location.total_mileage_km": mileage} + if err := writer.Append(context.Background(), event); err != nil { + t.Fatalf("Append(%d) error = %v", i, err) + } + } + + if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage\n"); got != 2 { + t.Fatalf("daily projections = %d, want 2", got) + } +} + func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -288,7 +867,7 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) writer := NewWriter(db, loc) eventTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LA9GG64L7PBAF4001", Phone: "13307765812", @@ -297,17 +876,15 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { Fields: map[string]any{ "jt808.location.total_mileage_km": 4123.9, }, - } + }) mock.ExpectExec(`INSERT INTO vehicle_data_source`). - WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"})) - mock.ExpectQuery(`SELECT first_total_mileage_km, first_event_time FROM vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). - WillReturnRows(sqlmock.NewRows([]string{"first_total_mileage_km", "first_event_time"})) + mock.ExpectBegin() mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). WithArgs( "LA9GG64L7PBAF4001", @@ -326,21 +903,18 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { eventTime, eventTime, QualityOK, - "current_day_first_sample", + QualityReasonCurrentDayFirst, ). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808"). - WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(maxSelectedDailyMileageKM)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). WithArgs( "LA9GG64L7PBAF4001", "2026-07-08", "JT808", - int64(1000), + int64(maxSelectedDailyMileageKM), "LA9GG64L7PBAF4001", "2026-07-08", "JT808", @@ -356,6 +930,7 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { "JT808", ). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) @@ -365,6 +940,157 @@ func TestWriterAppendUsesCurrentSampleWhenNoHistoricalBaseline(t *testing.T) { } } +func TestWriterCachesMissingBaselineAfterSuccessfulFirstSample(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + firstTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc) + secondTime := firstTime.Add(5 * time.Second) + base := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + }) + + mock.ExpectExec(`INSERT INTO vehicle_data_source`). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"})) + mock.ExpectBegin() + mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). + WithArgs( + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + "JT808:13307765812@115.231.168.135", + "115.231.168.135", + "115.231.168.135:20215", + "13307765812", + "", + "", + float64(4123.9), + float64(4123.9), + float64(0), + int64(1), + firstTime, + firstTime, + QualityOK, + QualityReasonCurrentDayFirst, + ). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(maxSelectedDailyMileageKM)). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). + WithArgs( + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + int64(maxSelectedDailyMileageKM), + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + ). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DELETE FROM vehicle_daily_mileage`). + WithArgs( + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + ). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() + mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). + WithArgs( + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + "JT808:13307765812@115.231.168.135", + "115.231.168.135", + "115.231.168.135:20215", + "13307765812", + "", + "", + float64(4123.9), + float64(4124.4), + approxFloat64{want: 0.5, tolerance: 0.000001}, + int64(1), + firstTime, + secondTime, + QualityOK, + QualityReasonCurrentDayFirst, + ). + WillReturnResult(sqlmock.NewResult(0, 1)) + + first := base + first.EventTimeMS = firstTime.UnixMilli() + first.Fields = map[string]any{"jt808.location.total_mileage_km": 4123.9} + if err := writer.Append(context.Background(), first); err != nil { + t.Fatalf("first Append() error = %v", err) + } + second := base + second.EventTimeMS = secondTime.UnixMilli() + second.Fields = map[string]any{"jt808.location.total_mileage_km": 4124.4} + if err := writer.Append(context.Background(), second); err != nil { + t.Fatalf("second Append() error = %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterRetriesExpiredMissingPreviousDayBaseline(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + writer.SetBaselineMissTTL(0) + candidate := SourceMileageSample{ + VIN: "LA9GG64L7PBAF4001", + StatDate: "2026-07-13", + Protocol: envelope.ProtocolJT808, + SourceKey: "JT808:64013036430@PLATFORM:g7s", + } + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs(candidate.VIN, candidate.StatDate, "JT808", candidate.SourceKey). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"})) + + if _, found, err := writer.previousBaseline(context.Background(), candidate); err != nil || found { + t.Fatalf("first previousBaseline() found=%v error=%v, want cacheable miss", found, err) + } + + previousTime := time.Date(2026, 7, 12, 23, 58, 0, 0, loc) + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs(candidate.VIN, candidate.StatDate, "JT808", candidate.SourceKey). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).AddRow(42567.9, previousTime)) + + baseline, found, err := writer.previousBaseline(context.Background(), candidate) + if err != nil || !found { + t.Fatalf("second previousBaseline() found=%v error=%v, want recovered baseline", found, err) + } + if baseline.LatestTotalKM != 42567.9 || !baseline.LatestEventTime.Equal(previousTime) { + t.Fatalf("baseline = %+v, want previous natural-day last sample", baseline) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -376,7 +1102,7 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T writer := NewWriter(db, loc) previousTime := time.Date(2026, 7, 7, 23, 58, 0, 0, loc) currentTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LA9GG64L7PBAF4001", Phone: "13307765812", @@ -385,15 +1111,16 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T Fields: map[string]any{ "jt808.location.total_mileage_km": 4123.9, }, - } + }) mock.ExpectExec(`INSERT INTO vehicle_data_source`). - WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). AddRow(4100.8, previousTime)) + mock.ExpectBegin() mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). WithArgs( "LA9GG64L7PBAF4001", @@ -412,21 +1139,18 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T previousTime, currentTime, QualityOK, - "historical_source_baseline", + QualityReasonHistorical, ). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808"). - WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(maxSelectedDailyMileageKM)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). WithArgs( "LA9GG64L7PBAF4001", "2026-07-08", "JT808", - int64(1000), + int64(maxSelectedDailyMileageKM), "LA9GG64L7PBAF4001", "2026-07-08", "JT808", @@ -442,6 +1166,7 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T "JT808", ). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) @@ -462,7 +1187,7 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) { writer := NewWriter(db, loc) historicalTime := time.Date(2026, 7, 4, 23, 58, 0, 0, loc) currentTime := time.Date(2026, 7, 8, 13, 20, 0, 0, loc) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolYutongMQTT, VIN: "LMRKH9AC2R1004087", DeviceID: "LMRKH9AC2R1004087", @@ -471,15 +1196,16 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) { Fields: map[string]any{ "yutong_mqtt.data.total_mileage": 120788000, }, - } + }) mock.ExpectExec(`INSERT INTO vehicle_data_source`). - WithArgs("YUTONG_MQTT", "mqtt", "mqtt://yutong/ytforward/shln/3", sqlmock.AnyArg(), sqlmock.AnyArg()). + WithArgs("YUTONG_MQTT", "mqtt", "mqtt://yutong/ytforward/shln/3", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). WithArgs("LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT", "YUTONG_MQTT:LMRKH9AC2R1004087@mqtt"). WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). AddRow(120672.0, historicalTime)) + mock.ExpectBegin() mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). WithArgs( "LMRKH9AC2R1004087", @@ -498,21 +1224,18 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) { historicalTime, currentTime, QualityOK, - "historical_source_baseline", + QualityReasonHistorical, ). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`). - WithArgs("LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT"). - WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). - WithArgs("LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT", int64(1000)). + WithArgs("LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT", int64(maxSelectedDailyMileageKM)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). WithArgs( "LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT", - int64(1000), + int64(maxSelectedDailyMileageKM), "LMRKH9AC2R1004087", "2026-07-08", "YUTONG_MQTT", @@ -528,6 +1251,7 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) { "YUTONG_MQTT", ). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) @@ -537,7 +1261,259 @@ func TestWriterAppendUsesOlderHistoricalSourceBaseline(t *testing.T) { } } -func TestWriterAppendUsesCurrentOKCandidateWhenPreviousBaselineMissing(t *testing.T) { +func TestWriterApplyRealtimeBaselineClampsSmallNegativeMileageJitter(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + baselineTime := time.Date(2026, 7, 11, 19, 32, 20, 0, loc) + currentTime := time.Date(2026, 7, 12, 2, 38, 23, 0, loc) + candidate := SourceMileageSample{ + VIN: "LB9A32A28R0LS1574", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + SourceKey: "JT808:64115156034@115.159.85.149", + SourceIP: "115.159.85.149", + LatestTotalKM: 15355.3, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LB9A32A28R0LS1574", "2026-07-12", "JT808", "JT808:64115156034@115.159.85.149"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(15355.4, baselineTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityOK || candidate.QualityReason != "negative_jitter_clamped" { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM != 0 { + t.Fatalf("daily km = %v, want 0", candidate.DailyKM) + } + if candidate.FirstTotalKM != 15355.4 || !candidate.FirstEventTime.Equal(baselineTime) { + t.Fatalf("baseline not preserved: %#v", candidate) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterApplyRealtimeBaselineKeepsNegativeHistoricalDeltaInvalid(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + baselineTime := time.Date(2026, 7, 11, 19, 32, 20, 0, loc) + currentTime := time.Date(2026, 7, 12, 2, 38, 23, 0, loc) + candidate := SourceMileageSample{ + VIN: "LB9A32A28R0LS1574", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + SourceKey: "JT808:64115156034@115.159.85.149", + SourceIP: "115.159.85.149", + LatestTotalKM: 15300.0, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LB9A32A28R0LS1574", "2026-07-12", "JT808", "JT808:64115156034@115.159.85.149"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(15355.4, baselineTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityInvalidDelta || candidate.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM > -55 || candidate.DailyKM < -56 { + t.Fatalf("daily km = %v, want large negative delta", candidate.DailyKM) + } + if candidate.FirstTotalKM != 15355.4 || !candidate.FirstEventTime.Equal(baselineTime) { + t.Fatalf("previous-day baseline not preserved: %#v", candidate) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterApplyRealtimeBaselineKeepsInvalidWhenCurrentDayBaselineIsStillHigher(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + previousTime := time.Date(2026, 7, 11, 19, 32, 20, 0, loc) + currentTime := time.Date(2026, 7, 12, 2, 38, 23, 0, loc) + candidate := SourceMileageSample{ + VIN: "LB9A32A28R0LS1574", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + SourceKey: "JT808:64115156034@115.159.85.149", + SourceIP: "115.159.85.149", + LatestTotalKM: 15300.0, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LB9A32A28R0LS1574", "2026-07-12", "JT808", "JT808:64115156034@115.159.85.149"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(15355.4, previousTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityInvalidDelta || candidate.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM > -55 || candidate.DailyKM < -56 { + t.Fatalf("daily km = %v, want large negative delta", candidate.DailyKM) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterApplyRealtimeBaselineRejectsNegativeDeltaEvenWithLowerCurrentDaySample(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + previousTime := time.Date(2026, 7, 11, 19, 32, 20, 0, loc) + currentTime := time.Date(2026, 7, 12, 2, 38, 23, 0, loc) + candidate := SourceMileageSample{ + VIN: "LB9A32A28R0LS1574", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + SourceKey: "JT808:64115156034@115.159.85.149", + SourceIP: "115.159.85.149", + LatestTotalKM: 15300.0, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LB9A32A28R0LS1574", "2026-07-12", "JT808", "JT808:64115156034@115.159.85.149"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(15355.4, previousTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityInvalidDelta || candidate.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM > -55 || candidate.DailyKM < -56 { + t.Fatalf("daily km = %v, want large negative delta", candidate.DailyKM) + } + if candidate.FirstTotalKM != 15355.4 || !candidate.FirstEventTime.Equal(previousTime) { + t.Fatalf("previous-day baseline not preserved: %#v", candidate) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterApplyRealtimeBaselineAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + baselineTime := time.Date(2026, 7, 4, 11, 7, 58, 0, loc) + currentTime := time.Date(2026, 7, 12, 2, 52, 47, 0, loc) + candidate := SourceMileageSample{ + VIN: "LNXNEGRR1SR319498", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolGB32960, + SourceKey: "GB32960:unknown@8.134.95.166", + SourceIP: "8.134.95.166", + LatestTotalKM: 16665.6, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LNXNEGRR1SR319498", "2026-07-12", "GB32960", "GB32960:unknown@8.134.95.166"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(8832.1, baselineTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityOK || candidate.QualityReason != QualityReasonHistorical { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM < 7833.4 || candidate.DailyKM > 7833.6 { + t.Fatalf("daily km = %v, want multi-day gap delta", candidate.DailyKM) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterApplyRealtimeBaselineRejectsHistoricalBaselineJump(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + baselineTime := time.Date(2026, 7, 3, 19, 0, 39, 0, loc) + currentTime := time.Date(2026, 7, 12, 10, 44, 56, 0, loc) + candidate := SourceMileageSample{ + VIN: "LNXNEGRR6SR319464", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolGB32960, + SourceKey: "GB32960:unknown@8.134.95.166", + SourceIP: "8.134.95.166", + LatestTotalKM: 40009.7, + LatestEventTime: currentTime, + } + + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LNXNEGRR6SR319464", "2026-07-12", "GB32960", "GB32960:unknown@8.134.95.166"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}). + AddRow(10009.7, baselineTime)) + + if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { + t.Fatalf("applyRealtimeBaseline() error = %v", err) + } + if candidate.QualityStatus != QualityInvalidDelta || candidate.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) + } + if candidate.DailyKM < 29999.9 || candidate.DailyKM > 30000.1 { + t.Fatalf("daily km = %v, want historical jump delta", candidate.DailyKM) + } + if candidate.FirstTotalKM != 10009.7 || !candidate.FirstEventTime.Equal(baselineTime) { + t.Fatalf("previous-day baseline not preserved: %#v", candidate) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestWriterAppendMarksCandidateNoPreviousBaselineWhenPreviousBaselineMissing(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) @@ -546,9 +1522,8 @@ func TestWriterAppendUsesCurrentOKCandidateWhenPreviousBaselineMissing(t *testin loc := time.FixedZone("Asia/Shanghai", 8*3600) writer := NewWriter(db, loc) - firstTime := time.Date(2026, 7, 7, 23, 58, 0, 0, loc) currentTime := time.Date(2026, 7, 8, 15, 56, 0, 0, loc) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LA9GG64L7PBAF4001", Phone: "13307765812", @@ -557,18 +1532,15 @@ func TestWriterAppendUsesCurrentOKCandidateWhenPreviousBaselineMissing(t *testin Fields: map[string]any{ "jt808.location.total_mileage_km": 4136.8, }, - } + }) mock.ExpectExec(`INSERT INTO vehicle_data_source`). - WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg()). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"})) - mock.ExpectQuery(`SELECT first_total_mileage_km, first_event_time FROM vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). - WillReturnRows(sqlmock.NewRows([]string{"first_total_mileage_km", "first_event_time"}). - AddRow(4100.8, firstTime)) + mock.ExpectBegin() mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). WithArgs( "LA9GG64L7PBAF4001", @@ -580,28 +1552,25 @@ func TestWriterAppendUsesCurrentOKCandidateWhenPreviousBaselineMissing(t *testin "13307765812", "", "", - float64(4100.8), float64(4136.8), - approxFloat64{want: 36.0, tolerance: 0.000001}, + float64(4136.8), + float64(0), int64(1), - firstTime, + currentTime, currentTime, QualityOK, - "current_day_first_sample", + QualityReasonCurrentDayFirst, ). WillReturnResult(sqlmock.NewResult(0, 1)) - mock.ExpectExec(`UPDATE vehicle_daily_mileage_source`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808"). - WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). - WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(1000)). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(maxSelectedDailyMileageKM)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`UPDATE vehicle_daily_mileage_source s`). WithArgs( "LA9GG64L7PBAF4001", "2026-07-08", "JT808", - int64(1000), + int64(maxSelectedDailyMileageKM), "LA9GG64L7PBAF4001", "2026-07-08", "JT808", @@ -617,6 +1586,7 @@ func TestWriterAppendUsesCurrentOKCandidateWhenPreviousBaselineMissing(t *testin "JT808", ). WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("Append() error = %v", err) @@ -632,20 +1602,21 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) { if err := writer.EnsureSchema(context.Background()); err != nil { t.Fatalf("EnsureSchema() error = %v", err) } - if err := writer.Append(context.Background(), envelope.FrameEnvelope{ + if err := writer.Append(context.Background(), managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000002", Phone: "13307765812", SourceEndpoint: "115.231.168.135:20215", EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: "10000.0", + "gb32960.vehicle.total_mileage_km": "10000.0", }, - }); err != nil { + })); err != nil { t.Fatalf("Append() error = %v", err) } - if len(exec.calls) != 16 { + schemaCalls := 3 + len(DailyMileageAlterSQL) + if len(exec.calls) != schemaCalls+5 { t.Fatalf("exec calls = %d", len(exec.calls)) } for i, want := range []string{ @@ -673,7 +1644,7 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) { if strings.Contains(exec.calls[2].query, "KEY idx_vin (vin)") { t.Fatalf("daily mileage table should not keep redundant vin index covered by the primary key: %s", exec.calls[2].query) } - projectCall := exec.calls[13] + projectCall := exec.calls[schemaCalls+2] if !strings.Contains(projectCall.query, "INSERT INTO vehicle_daily_mileage") { t.Fatalf("unexpected project sql: %s", projectCall.query) } @@ -698,26 +1669,76 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) { t.Fatalf("first upsert arg should be vin, got %#v", got) } - if !strings.Contains(exec.calls[10].query, "INSERT INTO vehicle_data_source") { - t.Fatalf("source upsert query missing: %s", exec.calls[10].query) + if !strings.Contains(exec.calls[schemaCalls].query, "INSERT INTO vehicle_data_source") { + t.Fatalf("source upsert query missing: %s", exec.calls[schemaCalls].query) } - if !strings.Contains(exec.calls[11].query, "INSERT INTO vehicle_daily_mileage_source") { - t.Fatalf("candidate upsert query missing: %s", exec.calls[11].query) + if !strings.Contains(exec.calls[schemaCalls+1].query, "INSERT INTO vehicle_daily_mileage_source") { + t.Fatalf("candidate upsert query missing: %s", exec.calls[schemaCalls+1].query) + } +} + +func TestWriterAppendWithResultReportsProjectionThrottle(t *testing.T) { + exec := &recordingExec{} + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(exec, loc) + base := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + + first := base + first.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 0, 0, loc).UnixMilli() + firstResult, err := writer.AppendWithResult(context.Background(), first) + if err != nil { + t.Fatalf("first AppendWithResult() error = %v", err) + } + if firstResult.ProjectionsAttempted != 1 || firstResult.ProjectionsWritten != 1 || firstResult.ProjectionsSkippedThrottled != 0 { + t.Fatalf("first projection result = %+v", firstResult) + } + + second := base + second.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 5, 0, loc).UnixMilli() + second.Fields = map[string]any{"jt808.location.total_mileage_km": 4124.2} + secondResult, err := writer.AppendWithResult(context.Background(), second) + if err != nil { + t.Fatalf("second AppendWithResult() error = %v", err) + } + if secondResult.SamplesWritten != 1 || secondResult.ProjectionsAttempted != 0 || secondResult.ProjectionsWritten != 0 || secondResult.ProjectionsSkippedThrottled != 1 { + t.Fatalf("second projection result = %+v, want source write with throttled final projection", secondResult) + } + if secondResult.SourceTouchesSkippedThrottled != 1 || secondResult.SourceTouchesWritten != 0 { + t.Fatalf("second source result = %+v, want source touch throttled", secondResult) + } + + third := base + third.EventTimeMS = time.Date(2026, 7, 8, 13, 20, 16, 0, loc).UnixMilli() + third.Fields = map[string]any{"jt808.location.total_mileage_km": 4125.0} + thirdResult, err := writer.AppendWithResult(context.Background(), third) + if err != nil { + t.Fatalf("third AppendWithResult() error = %v", err) + } + if thirdResult.ProjectionsAttempted != 1 || thirdResult.ProjectionsWritten != 1 || thirdResult.ProjectionsSkippedThrottled != 0 { + t.Fatalf("third projection result = %+v", thirdResult) } } func TestWriterSkipsConsecutiveDuplicateMileageSamples(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LNBVIN00000000001", SourceEndpoint: "115.231.168.135:20215", EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: 10241.2, + "jt808.location.total_mileage_km": 10241.2, }, - } + }) if err := writer.Append(context.Background(), event); err != nil { t.Fatalf("first Append() error = %v", err) @@ -727,23 +1748,56 @@ func TestWriterSkipsConsecutiveDuplicateMileageSamples(t *testing.T) { t.Fatalf("duplicate Append() error = %v", err) } - if len(exec.calls) != 6 { - t.Fatalf("exec calls = %d, want 6", len(exec.calls)) + if len(exec.calls) != 5 { + t.Fatalf("exec calls = %d, want 5", len(exec.calls)) + } +} + +func TestWriterAppendWithResultReportsDuplicateMileage(t *testing.T) { + exec := &recordingExec{} + writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) + event := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LNBVIN00000000001", + SourceEndpoint: "115.231.168.135:20215", + EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 10241.2, + }, + }) + + if _, err := writer.AppendWithResult(context.Background(), event); err != nil { + t.Fatalf("first AppendWithResult() error = %v", err) + } + event.ReceivedAtMS += 1000 + result, err := writer.AppendWithResult(context.Background(), event) + if err != nil { + t.Fatalf("duplicate AppendWithResult() error = %v", err) + } + + if result.SamplesFound != 1 || result.SamplesSkippedSameMileage != 1 || result.SamplesWritten != 0 { + t.Fatalf("result = %+v, want duplicate mileage skipped", result) + } + if result.SourceTouchesSkippedThrottled != 1 || result.SourceTouchesWritten != 0 { + t.Fatalf("source touch result = %+v, want duplicate to throttle source touch", result) + } + if len(exec.calls) != 5 { + t.Fatalf("exec calls = %d, want duplicate to avoid mysql writes", len(exec.calls)) } } func TestWriterDoesNotCacheMileageWhenUpsertFails(t *testing.T) { exec := &recordingExec{errs: []error{errors.New("mysql unavailable"), nil}} writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LNBVIN00000000001", SourceEndpoint: "115.231.168.135:20215", EventTimeMS: time.Date(2026, 7, 1, 9, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli(), Fields: map[string]any{ - envelope.FieldTotalMileageKM: 10241.2, + "jt808.location.total_mileage_km": 10241.2, }, - } + }) if err := writer.Append(context.Background(), event); err == nil { t.Fatalf("first Append() error = nil, want mysql error") @@ -752,22 +1806,91 @@ func TestWriterDoesNotCacheMileageWhenUpsertFails(t *testing.T) { t.Fatalf("retry Append() error = %v", err) } - if len(exec.calls) != 7 { - t.Fatalf("exec calls = %d, want 7", len(exec.calls)) + if len(exec.calls) != 6 { + t.Fatalf("exec calls = %d, want 6", len(exec.calls)) + } +} + +func TestWriterRollsBackMileageTransactionWhenProjectionFails(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(db, loc) + eventTime := time.Date(2026, 7, 8, 9, 30, 0, 0, loc) + event := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: "LA9GG64L7PBAF4001", + Phone: "13307765812", + SourceEndpoint: "115.231.168.135:20215", + EventTimeMS: eventTime.UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 4123.9, + }, + }) + wantErr := errors.New("project final mileage failed") + + mock.ExpectExec(`INSERT INTO vehicle_data_source`). + WithArgs("JT808", "115.231.168.135", "115.231.168.135:20215", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307765812@115.231.168.135"). + WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"})) + mock.ExpectBegin() + mock.ExpectExec(`INSERT INTO vehicle_daily_mileage_source`). + WithArgs( + "LA9GG64L7PBAF4001", + "2026-07-08", + "JT808", + "JT808:13307765812@115.231.168.135", + "115.231.168.135", + "115.231.168.135:20215", + "13307765812", + "", + "", + float64(4123.9), + float64(4123.9), + float64(0), + int64(1), + eventTime, + eventTime, + QualityOK, + QualityReasonCurrentDayFirst, + ). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). + WithArgs("LA9GG64L7PBAF4001", "2026-07-08", "JT808", int64(maxSelectedDailyMileageKM)). + WillReturnError(wantErr) + mock.ExpectRollback() + + err = writer.Append(context.Background(), event) + if !errors.Is(err, wantErr) { + t.Fatalf("Append() error = %v, want %v", err, wantErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } + writer.mu.Lock() + defer writer.mu.Unlock() + if len(writer.lastTotalMileage) != 0 || len(writer.lastProjection) != 0 { + t.Fatalf("writer cache should not mark failed sample, mileage=%d projection=%d", len(writer.lastTotalMileage), len(writer.lastProjection)) } } func TestWriterKeepsOnlyLatestDateInMileageCache(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) - event := envelope.FrameEnvelope{ + event := managedTestSource(envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "LNBVIN00000000001", SourceEndpoint: "115.231.168.135:20215", Fields: map[string]any{ - envelope.FieldTotalMileageKM: 10241.2, + "jt808.location.total_mileage_km": 10241.2, }, - } + }) event.EventTimeMS = time.Date(2026, 7, 1, 23, 59, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)).UnixMilli() if err := writer.Append(context.Background(), event); err != nil { @@ -778,14 +1901,174 @@ func TestWriterKeepsOnlyLatestDateInMileageCache(t *testing.T) { t.Fatalf("next-day Append() error = %v", err) } - if len(exec.calls) != 12 { - t.Fatalf("exec calls = %d, want 12", len(exec.calls)) + if len(exec.calls) != 10 { + t.Fatalf("exec calls = %d, want 10", len(exec.calls)) } if len(writer.lastTotalMileage) != 1 { t.Fatalf("cache entries = %d, want 1", len(writer.lastTotalMileage)) } } +func TestWriterCacheLimitEvictsOldestEntries(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(&recordingExec{}, loc) + writer.SetMaxCacheEntries(2) + samples := []struct { + sample MetricSample + sourceIP string + }{ + {sample: MetricSample{VIN: "VIN00000000000001", Protocol: envelope.ProtocolJT808, StatDate: "2026-07-01", SourceKey: "JT808:a@115.231.168.135", EventTime: time.Date(2026, 7, 1, 8, 0, 0, 0, loc)}, sourceIP: "115.231.168.135"}, + {sample: MetricSample{VIN: "VIN00000000000002", Protocol: envelope.ProtocolJT808, StatDate: "2026-07-02", SourceKey: "JT808:b@115.231.168.136", EventTime: time.Date(2026, 7, 2, 8, 0, 0, 0, loc)}, sourceIP: "115.231.168.136"}, + {sample: MetricSample{VIN: "VIN00000000000003", Protocol: envelope.ProtocolJT808, StatDate: "2026-07-03", SourceKey: "JT808:c@115.231.168.137", EventTime: time.Date(2026, 7, 3, 8, 0, 0, 0, loc)}, sourceIP: "115.231.168.137"}, + } + for index, item := range samples { + sample := item.sample + sample.TotalMileageKM = float64(100 + index) + writer.markMileageWritten(sample) + writer.markProjected(sample) + writer.markSourceTouched(SourceIdentity{ + Protocol: sample.Protocol, + SourceIP: item.sourceIP, + SourceEndpoint: item.sourceIP + ":20215", + }, sample.EventTime) + writer.mu.Lock() + writer.baselineCache[mileageCacheKey(sample)] = sourceBaselineCacheEntry{found: true, baseline: sourceBaseline{LatestTotalKM: sample.TotalMileageKM, LatestEventTime: sample.EventTime}} + writer.addCacheKeyLocked(writer.baselineKeysByPrefix, mileageCachePrefix(sample), mileageCacheKey(sample)) + writer.enforceCacheLimitsLocked() + writer.mu.Unlock() + } + + stats := writer.CacheStats() + if stats.MaxEntries != 2 { + t.Fatalf("max entries = %d, want 2", stats.MaxEntries) + } + if stats.LastTotalMileageEntries != 2 || + stats.LastProjectionEntries != 2 || + stats.LastSourceSeenEntries != 2 || + stats.BaselineEntries != 2 { + t.Fatalf("cache entries = %+v, want each cache capped at 2", stats) + } + if stats.TotalMileageEvictions == 0 || + stats.ProjectionEvictions == 0 || + stats.SourceSeenEvictions == 0 || + stats.BaselineEvictions == 0 { + t.Fatalf("cache evictions = %+v, want all caches to evict", stats) + } + if _, ok := writer.lastTotalMileage[mileageCacheKey(samples[0].sample)]; ok { + t.Fatalf("oldest mileage cache key was not evicted") + } + if _, ok := writer.lastProjection[projectionCacheKey(samples[0].sample)]; ok { + t.Fatalf("oldest projection cache key was not evicted") + } + if _, ok := writer.baselineCache[mileageCacheKey(samples[0].sample)]; ok { + t.Fatalf("oldest baseline cache key was not evicted") + } + if _, ok := writer.lastSourceSeen["JT808|115.231.168.135"]; ok { + t.Fatalf("oldest source touch cache key was not evicted") + } +} + +func TestWriterCleanupCachesDropsExpiredVehicleState(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(&recordingExec{}, loc) + writer.SetCacheRetention(48 * time.Hour) + writer.SetCacheCleanupInterval(0) + oldSample := MetricSample{ + VIN: "OLDVIN00000000001", + Protocol: envelope.ProtocolJT808, + StatDate: "2026-07-01", + SourceKey: "JT808:old@115.231.168.135", + EventTime: time.Date(2026, 7, 1, 12, 0, 0, 0, loc), + } + currentSample := MetricSample{ + VIN: "NEWVIN00000000001", + Protocol: envelope.ProtocolJT808, + StatDate: "2026-07-04", + SourceKey: "JT808:new@115.231.168.136", + EventTime: time.Date(2026, 7, 4, 12, 0, 0, 0, loc), + } + oldMileageKey := mileageCacheKey(oldSample) + currentMileageKey := mileageCacheKey(currentSample) + oldProjectionKey := projectionCacheKey(oldSample) + currentProjectionKey := projectionCacheKey(currentSample) + writer.lastTotalMileage[oldMileageKey] = 100 + writer.lastTotalMileage[currentMileageKey] = 200 + writer.baselineCache[oldMileageKey] = sourceBaselineCacheEntry{found: true} + writer.baselineCache[currentMileageKey] = sourceBaselineCacheEntry{found: true} + writer.lastProjection[oldProjectionKey] = projectionCacheEntry{projectedAt: oldSample.EventTime, sourceKeys: map[string]struct{}{oldSample.SourceKey: {}}} + writer.lastProjection[currentProjectionKey] = projectionCacheEntry{projectedAt: currentSample.EventTime, sourceKeys: map[string]struct{}{currentSample.SourceKey: {}}} + writer.lastSourceSeen["JT808|115.231.168.135"] = oldSample.EventTime + writer.lastSourceSeen["JT808|115.231.168.136"] = currentSample.EventTime + + writer.maybeCleanupCaches(time.Date(2026, 7, 4, 12, 1, 0, 0, loc)) + + if _, ok := writer.lastTotalMileage[oldMileageKey]; ok { + t.Fatalf("expired mileage cache key was not removed") + } + if _, ok := writer.baselineCache[oldMileageKey]; ok { + t.Fatalf("expired baseline cache key was not removed") + } + if _, ok := writer.lastProjection[oldProjectionKey]; ok { + t.Fatalf("expired projection cache key was not removed") + } + if _, ok := writer.lastSourceSeen["JT808|115.231.168.135"]; ok { + t.Fatalf("expired source touch cache key was not removed") + } + if _, ok := writer.lastTotalMileage[currentMileageKey]; !ok { + t.Fatalf("current mileage cache key was removed") + } + if _, ok := writer.baselineCache[currentMileageKey]; !ok { + t.Fatalf("current baseline cache key was removed") + } + if _, ok := writer.lastProjection[currentProjectionKey]; !ok { + t.Fatalf("current projection cache key was removed") + } + if _, ok := writer.lastSourceSeen["JT808|115.231.168.136"]; !ok { + t.Fatalf("current source touch cache key was removed") + } + stats := writer.CacheStats() + if stats.LastTotalMileageEntries != 1 || + stats.BaselineEntries != 1 || + stats.LastProjectionEntries != 1 || + stats.LastSourceSeenEntries != 1 { + t.Fatalf("cache stats entries = %+v, want one current entry in each cache", stats) + } + if stats.LastCleanupTotalMileage != 1 || + stats.LastCleanupBaseline != 1 || + stats.LastCleanupProjection != 1 || + stats.LastCleanupSourceSeen != 1 { + t.Fatalf("cache cleanup stats = %+v, want one deleted from each cache", stats) + } + if !stats.LastCleanupAt.Equal(time.Date(2026, 7, 4, 12, 1, 0, 0, loc)) { + t.Fatalf("last cleanup at = %s", stats.LastCleanupAt) + } +} + +func TestWriterCleanupCachesCanBeDisabled(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + writer := NewWriter(&recordingExec{}, loc) + writer.SetCacheRetention(0) + writer.SetCacheCleanupInterval(0) + oldSample := MetricSample{ + VIN: "OLDVIN00000000001", + Protocol: envelope.ProtocolJT808, + StatDate: "2026-07-01", + SourceKey: "JT808:old@115.231.168.135", + EventTime: time.Date(2026, 7, 1, 12, 0, 0, 0, loc), + } + key := mileageCacheKey(oldSample) + writer.lastTotalMileage[key] = 100 + writer.baselineCache[key] = sourceBaselineCacheEntry{found: true} + writer.lastProjection[projectionCacheKey(oldSample)] = projectionCacheEntry{projectedAt: oldSample.EventTime} + writer.lastSourceSeen["JT808|115.231.168.135"] = oldSample.EventTime + + writer.maybeCleanupCaches(time.Date(2026, 7, 10, 12, 0, 0, 0, loc)) + + if len(writer.lastTotalMileage) != 1 || len(writer.baselineCache) != 1 || len(writer.lastProjection) != 1 || len(writer.lastSourceSeen) != 1 { + t.Fatalf("cache cleanup should be disabled, got mileage=%d baseline=%d projection=%d source=%d", len(writer.lastTotalMileage), len(writer.baselineCache), len(writer.lastProjection), len(writer.lastSourceSeen)) + } +} + func countExecQueries(calls []execCall, fragment string) int { count := 0 for _, call := range calls { @@ -796,6 +2079,52 @@ func countExecQueries(calls []execCall, fragment string) int { return count } +func TestWriterSupportsConcurrentPartitionConsumers(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + exec := &concurrentExec{} + writer := NewWriter(exec, loc) + writer.SetSourceTouchInterval(0) + writer.SetProjectionInterval(0) + + const workers = 32 + var wait sync.WaitGroup + errorsCh := make(chan error, workers) + for index := 0; index < workers; index++ { + wait.Add(1) + go func(index int) { + defer wait.Done() + env := managedTestSource(envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + VIN: fmt.Sprintf("LNBVIN%011d", index), + Phone: fmt.Sprintf("13%09d", index), + MessageID: "0x0200", + SourceEndpoint: fmt.Sprintf("10.0.0.%d:808", index+1), + EventTimeMS: time.Date(2026, 7, 13, 8, 0, index, 0, loc).UnixMilli(), + ReceivedAtMS: time.Date(2026, 7, 13, 8, 0, index, 0, loc).UnixMilli(), + Fields: map[string]any{ + "jt808.location.total_mileage_km": 1000 + float64(index), + }, + }) + result, err := writer.AppendWithResult(context.Background(), env) + if err != nil { + errorsCh <- err + return + } + if result.SamplesWritten != 1 { + errorsCh <- fmt.Errorf("worker %d samples written = %d", index, result.SamplesWritten) + } + }(index) + } + wait.Wait() + close(errorsCh) + for err := range errorsCh { + t.Fatal(err) + } + if exec.CallCount() == 0 { + t.Fatal("concurrent writers did not execute any MySQL statements") + } +} + type execCall struct { query string args []any @@ -806,6 +2135,24 @@ type recordingExec struct { errs []error } +type concurrentExec struct { + mu sync.Mutex + calls int +} + +func (e *concurrentExec) ExecContext(_ context.Context, _ string, _ ...any) (sql.Result, error) { + e.mu.Lock() + e.calls++ + e.mu.Unlock() + return nil, nil +} + +func (e *concurrentExec) CallCount() int { + e.mu.Lock() + defer e.mu.Unlock() + return e.calls +} + func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) { e.calls = append(e.calls, execCall{query: query, args: args}) if len(e.errs) > 0 { diff --git a/go/vehicle-gateway/internal/stats/data_source_query.go b/go/vehicle-gateway/internal/stats/data_source_query.go new file mode 100644 index 00000000..2a884c28 --- /dev/null +++ b/go/vehicle-gateway/internal/stats/data_source_query.go @@ -0,0 +1,1733 @@ +package stats + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +type DataSourceQuery struct { + Protocol string + SourceIP string + SourceCode string + SourceKind string + SourceCodeMissing *bool + Enabled *bool + IncludeTotal bool + Limit int + Offset int +} + +type DataSourceDiagnosticsQuery struct { + Protocol string + SourceIP string + SourceKind string + SourceCodeMissing *bool + MappingIssueOnly bool + Enabled *bool + IncludeTotal bool + Limit int + Offset int +} + +type JT808IdentityGapQuery struct { + SourceIP string + SourceCode string + RecentSeconds int64 + IncludeTotal bool + Limit int + Offset int +} + +type JT808MappingGapQuery struct { + SourceIP string + SourceCode string + RecentSeconds int64 + IncludeTotal bool + Limit int + Offset int +} + +type DataSourceRow struct { + ID int64 `json:"id"` + Protocol string `json:"protocol"` + SourceIP string `json:"source_ip"` + LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind"` + TrustPriority int `json:"trust_priority"` + Enabled bool `json:"enabled"` + FirstSeenAt string `json:"first_seen_at,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + Remark string `json:"remark,omitempty"` + UpdatedAt string `json:"updated_at"` +} + +type DataSourceUpdate struct { + PlatformName *string `json:"platform_name"` + SourceCode *string `json:"source_code"` + SourceKind *string `json:"source_kind"` + TrustPriority *int `json:"trust_priority"` + Enabled *bool `json:"enabled"` + Remark *string `json:"remark"` +} + +type DataSourceDiagnosticRow struct { + ID int64 `json:"id"` + Protocol string `json:"protocol"` + SourceIP string `json:"source_ip"` + LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind"` + FirstSeenAt string `json:"first_seen_at,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + ActiveSpanSeconds int64 `json:"active_span_seconds"` + LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` + RegistrationRows int64 `json:"registration_rows"` + PhoneCount int64 `json:"phone_count"` + UnknownVINRows int64 `json:"unknown_vin_rows"` + IdentifierMatchedPhones int64 `json:"identifier_matched_phones"` + UnmappedPhoneCount int64 `json:"unmapped_phone_count"` + IdentifierMatchRatio float64 `json:"identifier_match_ratio"` + ConfiguredSourceCodeMatchedPhones int64 `json:"configured_source_code_matched_phones"` + ConfiguredSourceCodePlatformName string `json:"configured_source_code_platform_name,omitempty"` + ConfiguredSourceCodeConflict bool `json:"configured_source_code_conflict"` + SourcePlatformNameMismatch bool `json:"source_platform_name_mismatch"` + MatchedSourceCodeCount int64 `json:"matched_source_code_count"` + CandidateSourceCode string `json:"candidate_source_code,omitempty"` + CandidatePlatformName string `json:"candidate_platform_name,omitempty"` + MatchedSourceCodes []string `json:"matched_source_codes,omitempty"` + MatchedPlatformNames []string `json:"matched_platform_names,omitempty"` + SamplePhones []string `json:"sample_phones,omitempty"` + Reason string `json:"reason"` + RecommendedOperatorAction string `json:"recommended_operator_action"` + SuggestedSourceKind string `json:"suggested_source_kind"` + SuggestionConfidence string `json:"suggestion_confidence"` + SuggestionReason string `json:"suggestion_reason"` +} + +type JT808IdentityGapRow struct { + Phone string `json:"phone"` + DeviceID string `json:"device_id,omitempty"` + Plate string `json:"plate,omitempty"` + VIN string `json:"vin,omitempty"` + SourceIP string `json:"source_ip,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + SourceCode string `json:"source_code,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + FirstRegisteredAt string `json:"first_registered_at,omitempty"` + LatestRegisteredAt string `json:"latest_registered_at,omitempty"` + LatestAuthenticatedAt string `json:"latest_authenticated_at,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` + Reason string `json:"reason"` + RecommendedOperatorAction string `json:"recommended_operator_action"` + RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` + DataSourceQueryPath string `json:"data_source_query_path,omitempty"` +} + +type JT808MappingGapRow struct { + Phone string `json:"phone"` + DeviceID string `json:"device_id,omitempty"` + Plate string `json:"plate,omitempty"` + VIN string `json:"vin,omitempty"` + SourceIP string `json:"source_ip,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + SourceCode string `json:"source_code,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + IdentifierVIN string `json:"identifier_vin,omitempty"` + IdentifierPlate string `json:"identifier_plate,omitempty"` + MatchedSourceCodes []string `json:"matched_source_codes,omitempty"` + MatchedPlatformNames []string `json:"matched_platform_names,omitempty"` + LatestSeenAt string `json:"latest_seen_at,omitempty"` + LatestSeenAgeSeconds int64 `json:"latest_seen_age_seconds"` + Reason string `json:"reason"` + RecommendedAction string `json:"recommended_action"` + SuggestedSourceCode string `json:"suggested_source_code,omitempty"` + SuggestedPlatformName string `json:"suggested_platform_name,omitempty"` + SuggestedIdentifierType string `json:"suggested_identifier_type,omitempty"` + SuggestedIdentifierValue string `json:"suggested_identifier_value,omitempty"` + SuggestedVIN string `json:"suggested_vin,omitempty"` + SuggestedPlate string `json:"suggested_plate,omitempty"` + RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` + DataSourceQueryPath string `json:"data_source_query_path,omitempty"` + VehicleIdentifierExample string `json:"vehicle_identifier_example,omitempty"` +} + +type dataSourceDB interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +type DataSourceRepository struct { + db dataSourceDB +} + +func NewDataSourceRepository(db dataSourceDB) *DataSourceRepository { + if db == nil { + panic("data source db must not be nil") + } + return &DataSourceRepository{db: db} +} + +func (r *DataSourceRepository) Query(ctx context.Context, query DataSourceQuery) ([]DataSourceRow, error) { + query = normalizeDataSourceQuery(query) + sqlText, args := buildDataSourceSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]DataSourceRow, 0) + for rows.Next() { + var row DataSourceRow + var endpoint, platform, sourceCode, sourceKind, remark sql.NullString + var firstSeen, latestSeen, updatedAt scanDateTime + var enabled int + if err := rows.Scan( + &row.ID, + &row.Protocol, + &row.SourceIP, + &endpoint, + &platform, + &sourceCode, + &sourceKind, + &row.TrustPriority, + &enabled, + &firstSeen, + &latestSeen, + &remark, + &updatedAt, + ); err != nil { + return nil, err + } + row.LatestSourceEndpoint = endpoint.String + row.PlatformName = platform.String + row.SourceCode = sourceCode.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + row.Enabled = enabled != 0 + row.FirstSeenAt = firstSeen.String + row.LatestSeenAt = latestSeen.String + row.Remark = remark.String + row.UpdatedAt = updatedAt.String + out = append(out, row) + } + return out, rows.Err() +} + +func (r *DataSourceRepository) Count(ctx context.Context, query DataSourceQuery) (int64, error) { + query = normalizeDataSourceQuery(query) + sqlText, args := buildDataSourceCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *DataSourceRepository) QueryDiagnostics(ctx context.Context, query DataSourceDiagnosticsQuery) ([]DataSourceDiagnosticRow, error) { + query = normalizeDataSourceDiagnosticsQuery(query) + sqlText, args := buildDataSourceDiagnosticsSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]DataSourceDiagnosticRow, 0) + for rows.Next() { + var row DataSourceDiagnosticRow + var endpoint, platform, sourceCode, sourceKind sql.NullString + var firstSeen, latestSeen scanDateTime + var configuredSourceCodePlatformName, candidateSourceCode, candidatePlatformName, matchedSourceCodes, matchedPlatformNames, samplePhones sql.NullString + if err := rows.Scan( + &row.ID, + &row.Protocol, + &row.SourceIP, + &endpoint, + &platform, + &sourceCode, + &sourceKind, + &firstSeen, + &latestSeen, + &row.ActiveSpanSeconds, + &row.LatestSeenAgeSeconds, + &row.RegistrationRows, + &row.PhoneCount, + &row.UnknownVINRows, + &row.IdentifierMatchedPhones, + &row.UnmappedPhoneCount, + &row.IdentifierMatchRatio, + &row.ConfiguredSourceCodeMatchedPhones, + &configuredSourceCodePlatformName, + &row.MatchedSourceCodeCount, + &candidateSourceCode, + &candidatePlatformName, + &matchedSourceCodes, + &matchedPlatformNames, + &samplePhones, + ); err != nil { + return nil, err + } + row.LatestSourceEndpoint = endpoint.String + row.PlatformName = platform.String + row.SourceCode = sourceCode.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + row.FirstSeenAt = firstSeen.String + row.LatestSeenAt = latestSeen.String + row.CandidateSourceCode = candidateSourceCode.String + row.CandidatePlatformName = candidatePlatformName.String + row.MatchedSourceCodes = splitCommaList(matchedSourceCodes.String) + row.MatchedPlatformNames = splitCommaList(matchedPlatformNames.String) + row.SamplePhones = splitCommaList(samplePhones.String) + row.ConfiguredSourceCodeConflict = configuredSourceCodeConflict(row) + row.ConfiguredSourceCodePlatformName = configuredSourceCodePlatformName.String + row.SourcePlatformNameMismatch = sourcePlatformNameMismatch(row) + row.Reason, row.RecommendedOperatorAction = diagnoseDataSource(row) + row.SuggestedSourceKind, row.SuggestionConfidence, row.SuggestionReason = suggestSourceKind(row) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *DataSourceRepository) CountDiagnostics(ctx context.Context, query DataSourceDiagnosticsQuery) (int64, error) { + query = normalizeDataSourceDiagnosticsQuery(query) + sqlText, args := buildDataSourceDiagnosticsCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *DataSourceRepository) QueryJT808IdentityGaps(ctx context.Context, query JT808IdentityGapQuery) ([]JT808IdentityGapRow, error) { + query = normalizeJT808IdentityGapQuery(query) + sqlText, args := buildJT808IdentityGapSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]JT808IdentityGapRow, 0) + for rows.Next() { + var row JT808IdentityGapRow + var deviceID, plate, vin, sourceIP, endpoint, sourceCode, platform, sourceKind sql.NullString + var firstRegistered, latestRegistered, latestAuthenticated, latestSeen scanDateTime + if err := rows.Scan( + &row.Phone, + &deviceID, + &plate, + &vin, + &sourceIP, + &endpoint, + &sourceCode, + &platform, + &sourceKind, + &firstRegistered, + &latestRegistered, + &latestAuthenticated, + &latestSeen, + &row.LatestSeenAgeSeconds, + ); err != nil { + return nil, err + } + row.DeviceID = deviceID.String + row.Plate = plate.String + row.VIN = vin.String + row.SourceIP = sourceIP.String + row.SourceEndpoint = endpoint.String + row.SourceCode = sourceCode.String + row.PlatformName = platform.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + row.FirstRegisteredAt = firstRegistered.String + row.LatestRegisteredAt = latestRegistered.String + row.LatestAuthenticatedAt = latestAuthenticated.String + row.LatestSeenAt = latestSeen.String + row.Reason, row.RecommendedOperatorAction = diagnoseJT808IdentityGap(row) + row.RawFrameQueryPath = jt808IdentityGapRawFrameQueryPath(row) + row.DataSourceQueryPath = jt808IdentityGapDataSourceQueryPath(row) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *DataSourceRepository) CountJT808IdentityGaps(ctx context.Context, query JT808IdentityGapQuery) (int64, error) { + query = normalizeJT808IdentityGapQuery(query) + sqlText, args := buildJT808IdentityGapCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *DataSourceRepository) QueryJT808MappingGaps(ctx context.Context, query JT808MappingGapQuery) ([]JT808MappingGapRow, error) { + query = normalizeJT808MappingGapQuery(query) + sqlText, args := buildJT808MappingGapSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]JT808MappingGapRow, 0) + for rows.Next() { + var row JT808MappingGapRow + var deviceID, plate, vin, sourceIP, endpoint, sourceCode, platform, sourceKind sql.NullString + var identifierVIN, identifierPlate, matchedSourceCodes, matchedPlatformNames sql.NullString + var latestSeen scanDateTime + if err := rows.Scan( + &row.Phone, + &deviceID, + &plate, + &vin, + &sourceIP, + &endpoint, + &sourceCode, + &platform, + &sourceKind, + &identifierVIN, + &identifierPlate, + &matchedSourceCodes, + &matchedPlatformNames, + &latestSeen, + &row.LatestSeenAgeSeconds, + ); err != nil { + return nil, err + } + row.DeviceID = deviceID.String + row.Plate = plate.String + row.VIN = vin.String + row.SourceIP = sourceIP.String + row.SourceEndpoint = endpoint.String + row.SourceCode = sourceCode.String + row.PlatformName = platform.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + row.IdentifierVIN = identifierVIN.String + row.IdentifierPlate = identifierPlate.String + row.MatchedSourceCodes = splitCommaList(matchedSourceCodes.String) + row.MatchedPlatformNames = splitCommaList(matchedPlatformNames.String) + row.LatestSeenAt = latestSeen.String + annotateJT808MappingGap(&row) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *DataSourceRepository) CountJT808MappingGaps(ctx context.Context, query JT808MappingGapQuery) (int64, error) { + query = normalizeJT808MappingGapQuery(query) + sqlText, args := buildJT808MappingGapCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *DataSourceRepository) Update(ctx context.Context, id int64, update DataSourceUpdate) (bool, error) { + if id <= 0 { + return false, errors.New("id must be positive") + } + if err := validateDataSourceUpdate(update); err != nil { + return false, err + } + var sets []string + var args []any + if update.PlatformName != nil { + sets = append(sets, "platform_name = ?") + args = append(args, nullableTrimmedString(*update.PlatformName)) + } + if update.SourceCode != nil { + sets = append(sets, "source_code = ?") + args = append(args, nullableTrimmedString(*update.SourceCode)) + } + if update.SourceKind != nil { + sets = append(sets, "source_kind = ?") + args = append(args, normalizeSourceKindForWrite(*update.SourceKind)) + } + if update.TrustPriority != nil { + sets = append(sets, "trust_priority = ?") + args = append(args, *update.TrustPriority) + } + if update.Enabled != nil { + sets = append(sets, "enabled = ?") + if *update.Enabled { + args = append(args, 1) + } else { + args = append(args, 0) + } + } + if update.Remark != nil { + sets = append(sets, "remark = ?") + args = append(args, nullableTrimmedString(*update.Remark)) + } + if len(sets) == 0 { + return false, errors.New("no mutable fields provided") + } + exists, err := r.exists(ctx, id) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + args = append(args, id) + _, err = r.db.ExecContext(ctx, "UPDATE vehicle_data_source SET "+strings.Join(sets, ", ")+", updated_at = CURRENT_TIMESTAMP WHERE id = ?", args...) + return err == nil, err +} + +func (r *DataSourceRepository) exists(ctx context.Context, id int64) (bool, error) { + rows, err := r.db.QueryContext(ctx, "SELECT 1 FROM vehicle_data_source WHERE id = ? LIMIT 1", id) + if err != nil { + return false, err + } + defer rows.Close() + return rows.Next(), rows.Err() +} + +func normalizeDataSourceQuery(query DataSourceQuery) DataSourceQuery { + query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) + query.SourceIP = strings.TrimSpace(query.SourceIP) + query.SourceCode = strings.TrimSpace(query.SourceCode) + query.SourceKind = normalizeSourceKindForQuery(query.SourceKind) + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + +func normalizeDataSourceDiagnosticsQuery(query DataSourceDiagnosticsQuery) DataSourceDiagnosticsQuery { + query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) + if query.Protocol == "" { + query.Protocol = "JT808" + } + query.SourceIP = strings.TrimSpace(query.SourceIP) + query.SourceKind = normalizeSourceKindForQuery(query.SourceKind) + if query.Enabled == nil { + enabled := true + query.Enabled = &enabled + } + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + +func normalizeJT808IdentityGapQuery(query JT808IdentityGapQuery) JT808IdentityGapQuery { + query.SourceIP = strings.TrimSpace(query.SourceIP) + query.SourceCode = strings.TrimSpace(query.SourceCode) + if query.RecentSeconds < 0 { + query.RecentSeconds = 0 + } + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + +func normalizeJT808MappingGapQuery(query JT808MappingGapQuery) JT808MappingGapQuery { + query.SourceIP = strings.TrimSpace(query.SourceIP) + query.SourceCode = strings.TrimSpace(query.SourceCode) + if query.RecentSeconds < 0 { + query.RecentSeconds = 0 + } + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + +func buildDataSourceSQL(query DataSourceQuery) (string, []any) { + where, args := buildDataSourceWhere(query) + sqlText := `SELECT id, protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, + trust_priority, enabled, first_seen_at, latest_seen_at, remark, updated_at +FROM vehicle_data_source` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += " ORDER BY latest_seen_at DESC, id ASC LIMIT ? OFFSET ?" + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildDataSourceCountSQL(query DataSourceQuery) (string, []any) { + where, args := buildDataSourceWhere(query) + sqlText := `SELECT COUNT(*) FROM vehicle_data_source` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func buildDataSourceDiagnosticsSQL(query DataSourceDiagnosticsQuery) (string, []any) { + where, args := buildDataSourceDiagnosticsWhere(query) + sqlText := `SELECT + ds.id, + ds.protocol, + ds.source_ip, + ds.latest_source_endpoint, + ds.platform_name, + ds.source_code, + ds.source_kind, + ds.first_seen_at, + ds.latest_seen_at, + COALESCE(TIMESTAMPDIFF(SECOND, ds.first_seen_at, ds.latest_seen_at), 0) AS active_span_seconds, + COALESCE(TIMESTAMPDIFF(SECOND, ds.latest_seen_at, CURRENT_TIMESTAMP), 0) AS latest_seen_age_seconds, + COUNT(r.phone) AS registration_rows, + COUNT(DISTINCT r.phone) AS phone_count, + SUM(CASE WHEN r.phone IS NOT NULL AND (r.vin IS NULL OR TRIM(r.vin) = '' OR LOWER(TRIM(r.vin)) = 'unknown') THEN 1 ELSE 0 END) AS unknown_vin_rows, + COUNT(DISTINCT CASE WHEN vi.identifier_value IS NOT NULL THEN r.phone END) AS identifier_matched_phones, + COUNT(DISTINCT CASE WHEN r.phone IS NOT NULL AND vi.identifier_value IS NULL THEN r.phone END) AS unmapped_phone_count, + COALESCE(COUNT(DISTINCT CASE WHEN vi.identifier_value IS NOT NULL THEN r.phone END) / NULLIF(COUNT(DISTINCT r.phone), 0), 0) AS identifier_match_ratio, + COUNT(DISTINCT CASE + WHEN ds.source_code IS NOT NULL AND TRIM(ds.source_code) <> '' + AND vi.source_code IS NOT NULL AND TRIM(vi.source_code) <> '' + AND TRIM(vi.source_code) = TRIM(ds.source_code) + THEN r.phone + END) AS configured_source_code_matched_phones, + MAX(configured_catalog.platform_name) AS configured_source_code_platform_name, + COUNT(DISTINCT NULLIF(TRIM(vi.source_code), '')) AS matched_source_code_count, + MIN(NULLIF(TRIM(vi.source_code), '')) AS candidate_source_code, + MIN(COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), ''))) AS candidate_platform_name, + GROUP_CONCAT(DISTINCT NULLIF(TRIM(vi.source_code), '') ORDER BY NULLIF(TRIM(vi.source_code), '') SEPARATOR ',') AS matched_source_codes, + GROUP_CONCAT(DISTINCT COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), '')) ORDER BY COALESCE(NULLIF(TRIM(vi.oem), ''), NULLIF(TRIM(vi.source_code), '')) SEPARATOR ',') AS matched_platform_names, + SUBSTRING_INDEX(GROUP_CONCAT(DISTINCT r.phone ORDER BY r.latest_seen_at DESC SEPARATOR ','), ',', 5) AS sample_phones +FROM vehicle_data_source ds +LEFT JOIN jt808_registration r + ON ds.protocol = 'JT808' + AND r.source_ip = ds.source_ip +LEFT JOIN vehicle_identifier vi + ON vi.protocol = 'JT808' + AND vi.identifier_type = 'JT808_PHONE' + AND vi.identifier_value = r.phone + AND vi.enabled = 1 +LEFT JOIN ( + SELECT + source_code, + MIN(COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), ''))) AS platform_name + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND enabled = 1 + AND source_code IS NOT NULL + AND TRIM(source_code) <> '' + GROUP BY source_code + HAVING COUNT(DISTINCT COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), ''))) = 1 +) configured_catalog + ON configured_catalog.source_code = ds.source_code` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += ` GROUP BY ds.id, ds.protocol, ds.source_ip, ds.latest_source_endpoint, ds.platform_name, ds.source_code, ds.source_kind, ds.first_seen_at, ds.latest_seen_at +` + if having := buildDataSourceDiagnosticsHaving(query); len(having) > 0 { + sqlText += "HAVING " + strings.Join(having, " AND ") + "\n" + } + sqlText += ` +ORDER BY ds.latest_seen_at DESC, ds.id ASC LIMIT ? OFFSET ?` + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildDataSourceDiagnosticsCountSQL(query DataSourceDiagnosticsQuery) (string, []any) { + where, args := buildDataSourceDiagnosticsWhere(query) + having := buildDataSourceDiagnosticsHaving(query) + if len(having) > 0 { + sqlText := `SELECT COUNT(*) +FROM ( + SELECT ds.id, + MAX(ds.platform_name) AS platform_name, + ds.source_code AS source_code, + COUNT(DISTINCT r.phone) AS phone_count, + COUNT(DISTINCT CASE WHEN vi.identifier_value IS NOT NULL THEN r.phone END) AS identifier_matched_phones, + COUNT(DISTINCT CASE WHEN r.phone IS NOT NULL AND vi.identifier_value IS NULL THEN r.phone END) AS unmapped_phone_count, + COALESCE(COUNT(DISTINCT CASE WHEN vi.identifier_value IS NOT NULL THEN r.phone END) / NULLIF(COUNT(DISTINCT r.phone), 0), 0) AS identifier_match_ratio, + COUNT(DISTINCT NULLIF(TRIM(vi.source_code), '')) AS matched_source_code_count, + COUNT(DISTINCT CASE + WHEN ds.source_code IS NOT NULL AND TRIM(ds.source_code) <> '' + AND vi.source_code IS NOT NULL AND TRIM(vi.source_code) <> '' + AND TRIM(vi.source_code) = TRIM(ds.source_code) + THEN r.phone + END) AS configured_source_code_matched_phones, + MAX(configured_catalog.platform_name) AS configured_source_code_platform_name + FROM vehicle_data_source ds + LEFT JOIN jt808_registration r + ON ds.protocol = 'JT808' + AND r.source_ip = ds.source_ip + LEFT JOIN vehicle_identifier vi + ON vi.protocol = 'JT808' + AND vi.identifier_type = 'JT808_PHONE' + AND vi.identifier_value = r.phone + AND vi.enabled = 1 + LEFT JOIN ( + SELECT + source_code, + MIN(COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), ''))) AS platform_name + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND enabled = 1 + AND source_code IS NOT NULL + AND TRIM(source_code) <> '' + GROUP BY source_code + HAVING COUNT(DISTINCT COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), ''))) = 1 + ) configured_catalog + ON configured_catalog.source_code = ds.source_code` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += ` GROUP BY ds.id, ds.source_code + HAVING ` + strings.Join(having, " AND ") + ` +) diagnostics` + return sqlText, args + } + sqlText := `SELECT COUNT(*) FROM vehicle_data_source ds` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func buildJT808IdentityGapSQL(query JT808IdentityGapQuery) (string, []any) { + fromSQL, args := buildJT808IdentityGapFromSQL(query) + sqlText := `SELECT + r.phone, + r.device_id, + r.plate, + r.vin, + r.source_ip, + r.source_endpoint, + ds.source_code, + ds.platform_name, + ds.source_kind, + r.first_registered_at, + r.latest_registered_at, + r.latest_authenticated_at, + r.latest_seen_at, + COALESCE(TIMESTAMPDIFF(SECOND, r.latest_seen_at, CURRENT_TIMESTAMP), 0) AS latest_seen_age_seconds +` + fromSQL + ` +ORDER BY r.latest_seen_at DESC, r.phone ASC +LIMIT ? OFFSET ?` + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildJT808IdentityGapCountSQL(query JT808IdentityGapQuery) (string, []any) { + fromSQL, args := buildJT808IdentityGapFromSQL(query) + return "SELECT COUNT(*) " + fromSQL, args +} + +func buildJT808MappingGapSQL(query JT808MappingGapQuery) (string, []any) { + fromSQL, args := buildJT808MappingGapFromSQL(query) + sqlText := `SELECT + r.phone, + r.device_id, + r.plate, + r.vin, + r.source_ip, + r.source_endpoint, + ds.source_code, + ds.platform_name, + ds.source_kind, + source_vi.vin AS identifier_vin, + source_vi.plate AS identifier_plate, + all_vi.matched_source_codes, + all_vi.matched_platform_names, + r.latest_seen_at, + COALESCE(TIMESTAMPDIFF(SECOND, r.latest_seen_at, CURRENT_TIMESTAMP), 0) AS latest_seen_age_seconds +` + fromSQL + ` +ORDER BY r.latest_seen_at DESC, r.phone ASC +LIMIT ? OFFSET ?` + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildJT808MappingGapCountSQL(query JT808MappingGapQuery) (string, []any) { + fromSQL, args := buildJT808MappingGapFromSQL(query) + return "SELECT COUNT(*) " + fromSQL, args +} + +func buildJT808IdentityGapFromSQL(query JT808IdentityGapQuery) (string, []any) { + where, args := buildJT808IdentityGapWhere(query) + sqlText := `FROM jt808_registration r +LEFT JOIN vehicle_data_source ds + ON ds.protocol = 'JT808' AND ds.source_ip = r.source_ip +LEFT JOIN ( + SELECT identifier_value, COUNT(*) AS match_count + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'JT808_PHONE' + AND enabled = 1 + AND vin IS NOT NULL AND TRIM(vin) <> '' + GROUP BY identifier_value +) phone_vi ON phone_vi.identifier_value = r.phone +LEFT JOIN ( + SELECT identifier_value, COUNT(*) AS match_count + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'PLATE' + AND enabled = 1 + AND vin IS NOT NULL AND TRIM(vin) <> '' + GROUP BY identifier_value +) plate_vi ON plate_vi.identifier_value = r.plate +LEFT JOIN vehicle_identity_binding phone_binding + ON phone_binding.phone = r.phone + AND phone_binding.vin IS NOT NULL AND TRIM(phone_binding.vin) <> '' +LEFT JOIN vehicle_identity_binding plate_binding + ON plate_binding.plate = r.plate + AND plate_binding.vin IS NOT NULL AND TRIM(plate_binding.vin) <> ''` + if len(where) > 0 { + sqlText += "\nWHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func buildJT808MappingGapFromSQL(query JT808MappingGapQuery) (string, []any) { + where, args := buildJT808MappingGapWhere(query) + sqlText := `FROM jt808_registration r +JOIN vehicle_data_source ds + ON ds.protocol = 'JT808' + AND ds.source_ip = r.source_ip + AND ds.enabled = 1 + AND ds.source_code IS NOT NULL + AND TRIM(ds.source_code) <> '' +LEFT JOIN vehicle_identifier source_vi + ON source_vi.protocol = 'JT808' + AND source_vi.source_code = ds.source_code + AND source_vi.identifier_type = 'JT808_PHONE' + AND source_vi.identifier_value = r.phone + AND source_vi.enabled = 1 +LEFT JOIN ( + SELECT + identifier_value, + GROUP_CONCAT(DISTINCT NULLIF(TRIM(source_code), '') ORDER BY NULLIF(TRIM(source_code), '') SEPARATOR ',') AS matched_source_codes, + GROUP_CONCAT(DISTINCT COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), '')) ORDER BY COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), '')) SEPARATOR ',') AS matched_platform_names + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'JT808_PHONE' + AND enabled = 1 + GROUP BY identifier_value +) all_vi + ON all_vi.identifier_value = r.phone` + if len(where) > 0 { + sqlText += "\nWHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func buildJT808IdentityGapWhere(query JT808IdentityGapQuery) ([]string, []any) { + where := []string{ + "r.phone IS NOT NULL AND TRIM(r.phone) <> ''", + "(r.vin IS NULL OR TRIM(r.vin) = '' OR LOWER(TRIM(r.vin)) = 'unknown')", + "COALESCE(phone_vi.match_count, 0) = 0", + "COALESCE(plate_vi.match_count, 0) = 0", + "phone_binding.vin IS NULL", + "plate_binding.vin IS NULL", + } + var args []any + if query.SourceIP != "" { + where = append(where, "r.source_ip = ?") + args = append(args, query.SourceIP) + } + if query.SourceCode != "" { + where = append(where, "ds.source_code = ?") + args = append(args, query.SourceCode) + } + if query.RecentSeconds > 0 { + where = append(where, "r.latest_seen_at >= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL ? SECOND)") + args = append(args, query.RecentSeconds) + } + return where, args +} + +func buildJT808MappingGapWhere(query JT808MappingGapQuery) ([]string, []any) { + where := []string{ + "r.phone IS NOT NULL AND TRIM(r.phone) <> ''", + "(source_vi.identifier_value IS NULL OR source_vi.vin IS NULL OR TRIM(source_vi.vin) = '')", + } + var args []any + if query.SourceIP != "" { + where = append(where, "r.source_ip = ?") + args = append(args, query.SourceIP) + } + if query.SourceCode != "" { + where = append(where, "ds.source_code = ?") + args = append(args, query.SourceCode) + } + if query.RecentSeconds > 0 { + where = append(where, "r.latest_seen_at >= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL ? SECOND)") + args = append(args, query.RecentSeconds) + } + return where, args +} + +func buildDataSourceWhere(query DataSourceQuery) ([]string, []any) { + var where []string + var args []any + add := func(clause string, value any) { + where = append(where, clause) + args = append(args, value) + } + if query.Protocol != "" { + add("protocol = ?", query.Protocol) + } + if query.SourceIP != "" { + add("source_ip = ?", query.SourceIP) + } + if query.SourceCode != "" { + add("source_code = ?", query.SourceCode) + } + if query.SourceKind != "" { + add("source_kind = ?", query.SourceKind) + } + if query.SourceCodeMissing != nil { + if *query.SourceCodeMissing { + where = append(where, "(source_code IS NULL OR TRIM(source_code) = '')") + } else { + where = append(where, "(source_code IS NOT NULL AND TRIM(source_code) <> '')") + } + } + if query.Enabled != nil { + if *query.Enabled { + add("enabled = ?", 1) + } else { + add("enabled = ?", 0) + } + } + return where, args +} + +func buildDataSourceDiagnosticsWhere(query DataSourceDiagnosticsQuery) ([]string, []any) { + var where []string + var args []any + add := func(clause string, value any) { + where = append(where, clause) + args = append(args, value) + } + if query.Protocol != "" { + add("ds.protocol = ?", query.Protocol) + } + if query.SourceIP != "" { + add("ds.source_ip = ?", query.SourceIP) + } + if query.SourceKind != "" { + add("ds.source_kind = ?", query.SourceKind) + } + if query.SourceCodeMissing != nil { + if *query.SourceCodeMissing { + where = append(where, "(ds.source_code IS NULL OR TRIM(ds.source_code) = '')") + } else { + where = append(where, "(ds.source_code IS NOT NULL AND TRIM(ds.source_code) <> '')") + } + } + if query.Enabled != nil { + if *query.Enabled { + add("ds.enabled = ?", 1) + } else { + add("ds.enabled = ?", 0) + } + } + return where, args +} + +func buildDataSourceDiagnosticsHaving(query DataSourceDiagnosticsQuery) []string { + if !query.MappingIssueOnly { + return nil + } + return []string{`( + ( + platform_name IS NOT NULL AND TRIM(platform_name) <> '' + AND configured_source_code_platform_name IS NOT NULL AND TRIM(configured_source_code_platform_name) <> '' + AND TRIM(platform_name) <> TRIM(configured_source_code_platform_name) + ) + OR ( + source_code IS NOT NULL AND TRIM(source_code) <> '' + AND matched_source_code_count > 0 + AND configured_source_code_matched_phones = 0 + ) + OR ( + source_code IS NOT NULL AND TRIM(source_code) <> '' + AND phone_count >= 10 + AND unmapped_phone_count > 0 + AND identifier_match_ratio < 0.8 + ) + )`} +} + +type DataSourceHandler struct { + repository *DataSourceRepository +} + +func NewDataSourceHandler(repository *DataSourceRepository) *DataSourceHandler { + if repository == nil { + panic("data source repository must not be nil") + } + return &DataSourceHandler{repository: repository} +} + +func (h *DataSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + path := strings.Trim(r.URL.Path, "/") + switch { + case r.Method == http.MethodGet && path == "api/stats/data-sources/diagnostics": + h.handleDiagnostics(w, r) + case r.Method == http.MethodGet && path == "api/stats/data-sources/kind-suggestions": + h.handleKindSuggestions(w, r) + case r.Method == http.MethodGet && path == "api/stats/data-sources/jt808-identity-gaps": + h.handleJT808IdentityGaps(w, r) + case r.Method == http.MethodGet && path == "api/stats/data-sources/jt808-mapping-gaps": + h.handleJT808MappingGaps(w, r) + case r.Method == http.MethodGet && path == "api/stats/data-sources": + h.handleQuery(w, r) + case r.Method == http.MethodPatch && strings.HasPrefix(path, "api/stats/data-sources/"): + h.handlePatch(w, r) + default: + writeMetricError(w, http.StatusNotFound, "route not found") + } +} + +func (h *DataSourceHandler) handleKindSuggestions(w http.ResponseWriter, r *http.Request) { + query, err := parseDataSourceKindSuggestionsQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QueryDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *DataSourceHandler) handleDiagnostics(w http.ResponseWriter, r *http.Request) { + query, err := parseDataSourceDiagnosticsQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QueryDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *DataSourceHandler) handleJT808IdentityGaps(w http.ResponseWriter, r *http.Request) { + query, err := parseJT808IdentityGapQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountJT808IdentityGaps(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QueryJT808IdentityGaps(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + "recentSeconds": query.RecentSeconds, + }) +} + +func (h *DataSourceHandler) handleJT808MappingGaps(w http.ResponseWriter, r *http.Request) { + query, err := parseJT808MappingGapQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountJT808MappingGaps(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QueryJT808MappingGaps(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + "recentSeconds": query.RecentSeconds, + }) +} + +func (h *DataSourceHandler) handleQuery(w http.ResponseWriter, r *http.Request) { + query, err := parseDataSourceQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.Count(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.Query(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *DataSourceHandler) handlePatch(w http.ResponseWriter, r *http.Request) { + id, err := parseDataSourceID(strings.TrimPrefix(strings.Trim(r.URL.Path, "/"), "api/stats/data-sources/")) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + defer r.Body.Close() + var update DataSourceUpdate + if err := json.NewDecoder(r.Body).Decode(&update); err != nil { + writeMetricError(w, http.StatusBadRequest, "invalid json body") + return + } + updated, err := h.repository.Update(r.Context(), id, update) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + if !updated { + writeMetricError(w, http.StatusNotFound, "data source not found") + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"updated": true, "id": id}) +} + +func parseDataSourceQuery(r *http.Request) (DataSourceQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return DataSourceQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return DataSourceQuery{}, err + } + enabled, err := parseOptionalBool(values.Get("enabled")) + if err != nil { + return DataSourceQuery{}, err + } + sourceCodeMissing, err := parseOptionalBool(values.Get("sourceCodeMissing")) + if err != nil { + return DataSourceQuery{}, errors.New("sourceCodeMissing must be true or false") + } + if strings.TrimSpace(values.Get("sourceCode")) != "" && sourceCodeMissing != nil { + return DataSourceQuery{}, errors.New("sourceCode and sourceCodeMissing cannot be used together") + } + if strings.TrimSpace(values.Get("sourceKind")) != "" { + if _, err := parseSourceKind(values.Get("sourceKind")); err != nil { + return DataSourceQuery{}, err + } + } + return normalizeDataSourceQuery(DataSourceQuery{ + Protocol: values.Get("protocol"), + SourceIP: values.Get("sourceIP"), + SourceCode: values.Get("sourceCode"), + SourceKind: values.Get("sourceKind"), + SourceCodeMissing: sourceCodeMissing, + Enabled: enabled, + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + }), nil +} + +func parseDataSourceDiagnosticsQuery(r *http.Request) (DataSourceDiagnosticsQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + sourceCodeMissing, err := parseOptionalBool(values.Get("sourceCodeMissing")) + if err != nil { + return DataSourceDiagnosticsQuery{}, errors.New("sourceCodeMissing must be true or false") + } + mappingIssueOnly, err := parseOptionalBool(values.Get("mappingIssueOnly")) + if err != nil { + return DataSourceDiagnosticsQuery{}, errors.New("mappingIssueOnly must be true or false") + } + enabled, err := parseOptionalBool(values.Get("enabled")) + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + if sourceCodeMissing == nil { + missing := true + sourceCodeMissing = &missing + } + query := normalizeDataSourceDiagnosticsQuery(DataSourceDiagnosticsQuery{ + Protocol: values.Get("protocol"), + SourceIP: values.Get("sourceIP"), + SourceCodeMissing: sourceCodeMissing, + MappingIssueOnly: mappingIssueOnly != nil && *mappingIssueOnly, + Enabled: enabled, + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + }) + return query, nil +} + +func parseDataSourceKindSuggestionsQuery(r *http.Request) (DataSourceDiagnosticsQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + sourceKind := strings.TrimSpace(values.Get("sourceKind")) + if sourceKind == "" { + sourceKind = "UNKNOWN" + } + if _, err := parseSourceKind(sourceKind); err != nil { + return DataSourceDiagnosticsQuery{}, err + } + sourceCodeMissing, err := parseOptionalBool(values.Get("sourceCodeMissing")) + if err != nil { + return DataSourceDiagnosticsQuery{}, errors.New("sourceCodeMissing must be true or false") + } + enabled, err := parseOptionalBool(values.Get("enabled")) + if err != nil { + return DataSourceDiagnosticsQuery{}, err + } + query := normalizeDataSourceDiagnosticsQuery(DataSourceDiagnosticsQuery{ + Protocol: values.Get("protocol"), + SourceIP: values.Get("sourceIP"), + SourceKind: sourceKind, + SourceCodeMissing: sourceCodeMissing, + Enabled: enabled, + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + }) + return query, nil +} + +func parseJT808IdentityGapQuery(r *http.Request) (JT808IdentityGapQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return JT808IdentityGapQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return JT808IdentityGapQuery{}, err + } + recentSeconds, err := parseBoundedInt(values.Get("recentSeconds"), 86400, 0, 30*86400, "recentSeconds") + if err != nil { + return JT808IdentityGapQuery{}, err + } + sourceCode := strings.TrimSpace(values.Get("sourceCode")) + if len(sourceCode) > 64 || !sourceCodePattern.MatchString(sourceCode) { + return JT808IdentityGapQuery{}, errors.New("sourceCode may contain only letters, digits, underscore, dot and dash") + } + return normalizeJT808IdentityGapQuery(JT808IdentityGapQuery{ + SourceIP: values.Get("sourceIP"), + SourceCode: sourceCode, + RecentSeconds: int64(recentSeconds), + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + }), nil +} + +func parseJT808MappingGapQuery(r *http.Request) (JT808MappingGapQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return JT808MappingGapQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return JT808MappingGapQuery{}, err + } + recentSeconds, err := parseBoundedInt(values.Get("recentSeconds"), 86400, 0, 30*86400, "recentSeconds") + if err != nil { + return JT808MappingGapQuery{}, err + } + sourceCode := strings.TrimSpace(values.Get("sourceCode")) + if len(sourceCode) > 64 || !sourceCodePattern.MatchString(sourceCode) { + return JT808MappingGapQuery{}, errors.New("sourceCode may contain only letters, digits, underscore, dot and dash") + } + return normalizeJT808MappingGapQuery(JT808MappingGapQuery{ + SourceIP: values.Get("sourceIP"), + SourceCode: sourceCode, + RecentSeconds: int64(recentSeconds), + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + }), nil +} + +func parseOptionalBool(value string) (*bool, error) { + value = strings.TrimSpace(value) + if value == "" { + return nil, nil + } + switch strings.ToLower(value) { + case "true", "1", "yes": + enabled := true + return &enabled, nil + case "false", "0", "no": + enabled := false + return &enabled, nil + default: + return nil, errors.New("enabled must be true or false") + } +} + +func parseDataSourceID(value string) (int64, error) { + value = strings.Trim(strings.TrimSpace(value), "/") + if value == "" || strings.Contains(value, "/") { + return 0, errors.New("data source id is required") + } + id, err := strconv.ParseInt(value, 10, 64) + if err != nil || id <= 0 { + return 0, errors.New("data source id must be positive") + } + return id, nil +} + +var sourceCodePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]*$`) + +func validateDataSourceUpdate(update DataSourceUpdate) error { + if update.PlatformName != nil && len([]rune(strings.TrimSpace(*update.PlatformName))) > 128 { + return errors.New("platform_name length exceeds 128") + } + if update.SourceCode != nil { + sourceCode := strings.TrimSpace(*update.SourceCode) + if len(sourceCode) > 64 { + return errors.New("source_code length exceeds 64") + } + if !sourceCodePattern.MatchString(sourceCode) { + return errors.New("source_code may only contain letters, numbers, dot, underscore or dash") + } + } + if update.SourceKind != nil { + if _, err := parseSourceKind(*update.SourceKind); err != nil { + return err + } + } + if update.TrustPriority != nil && (*update.TrustPriority < 0 || *update.TrustPriority > 100000) { + return errors.New("trust_priority must be between 0 and 100000") + } + if update.Remark != nil && len([]rune(strings.TrimSpace(*update.Remark))) > 512 { + return errors.New("remark length exceeds 512") + } + return nil +} + +func parseSourceKind(value string) (string, error) { + kind := normalizeSourceKindForRead(value) + switch kind { + case "UNKNOWN", "PLATFORM", "DIRECT": + return kind, nil + default: + return "", errors.New("source_kind must be UNKNOWN, PLATFORM or DIRECT") + } +} + +func normalizeSourceKindForRead(value string) string { + value = strings.ToUpper(strings.TrimSpace(value)) + if value == "" { + return "UNKNOWN" + } + return value +} + +func normalizeSourceKindForQuery(value string) string { + return strings.ToUpper(strings.TrimSpace(value)) +} + +func normalizeSourceKindForWrite(value string) string { + kind, err := parseSourceKind(value) + if err != nil { + return "UNKNOWN" + } + return kind +} + +func nullableTrimmedString(value string) any { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + return value +} + +func splitCommaList(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +func diagnoseDataSource(row DataSourceDiagnosticRow) (string, string) { + if strings.ToUpper(strings.TrimSpace(row.Protocol)) != "JT808" { + if strings.TrimSpace(row.SourceCode) == "" { + return "source_code_missing", "补充来源平台名称和 source_code，用于多源统计选举、断链告警和大屏展示" + } + return "source_configured", "该来源已具备基础平台标识；如参与统计选举，请确认 source_kind、trust_priority 和 enabled" + } + switch { + case row.RegistrationRows == 0: + return "no_registration", "等待该来源产生 JT808 注册/鉴权/位置记录，或核对 source_endpoint 是否被正确写入 jt808_registration" + case lowIdentifierCoverage(row): + return "low_identifier_coverage", "该来源已配置平台，但大量注册手机号未维护到 vehicle_identifier，会影响 VIN 解析、在线统计和里程来源选举" + case row.SourcePlatformNameMismatch: + return "source_platform_name_mismatch", "当前 source_code 在 vehicle_identifier 中对应的平台名与 vehicle_data_source.platform_name 不一致；先确认并修正来源配置，再导入手机号映射" + case row.IdentifierMatchedPhones == 0: + return "no_identifier_match", "补充该来源手机号到 vehicle_identifier，或先导入对应平台的 808 车牌/手机号映射文件" + case row.ConfiguredSourceCodeConflict: + return "source_code_conflict", "已配置的 source_code 与该来源手机号绑定推断出的 source_code 不一致，需确认来源平台或修正 vehicle_identifier" + case strings.TrimSpace(row.SourceCode) != "": + return "source_configured", "该来源已具备基础平台标识；继续维护缺失手机号可提升 VIN 解析和统计覆盖率" + case row.MatchedSourceCodeCount > 1: + return "ambiguous_source_code", "同一来源 IP 匹配到多个 source_code，需人工确认平台并在 vehicle_data_source 固定 source_code" + case row.CandidateSourceCode != "": + return "candidate_available", "可用候选 source_code，执行 identity-import -sync-data-sources -apply 或在来源管理中确认" + default: + return "no_source_code", "已匹配到标识但没有 source_code，补齐 vehicle_identifier.source_code 后重新同步" + } +} + +func diagnoseJT808IdentityGap(row JT808IdentityGapRow) (string, string) { + if strings.TrimSpace(row.Plate) != "" { + return "missing_phone_and_plate_binding", "将该 phone 或 plate 维护到 vehicle_identifier，并确认 VIN、source_code、oem；若仍使用旧表，也同步 vehicle_identity_binding" + } + return "missing_phone_binding", "将该 phone 维护到 vehicle_identifier，并确认 VIN、source_code、oem；如终端会补发注册帧，可等待 plate 后再复核" +} + +func jt808IdentityGapRawFrameQueryPath(row JT808IdentityGapRow) string { + if strings.TrimSpace(row.Phone) == "" { + return "" + } + dateFrom, dateTo := jt808IdentityGapDateWindow(row.LatestSeenAt) + values := url.Values{} + values.Set("protocol", "JT808") + values.Set("phone", row.Phone) + values.Set("dateFrom", dateFrom) + values.Set("dateTo", dateTo) + values.Set("orderBy", "eventTime") + values.Set("includeFields", "true") + values.Set("limit", "20") + return "/api/history/raw-frames?" + values.Encode() +} + +func jt808IdentityGapDataSourceQueryPath(row JT808IdentityGapRow) string { + if strings.TrimSpace(row.SourceIP) == "" { + return "" + } + values := url.Values{} + values.Set("protocol", "JT808") + values.Set("sourceIP", row.SourceIP) + values.Set("includeTotal", "true") + values.Set("limit", "20") + return "/api/stats/data-sources?" + values.Encode() +} + +func annotateJT808MappingGap(row *JT808MappingGapRow) { + if row == nil { + return + } + row.SuggestedSourceCode = strings.TrimSpace(row.SourceCode) + row.SuggestedPlatformName = strings.TrimSpace(firstNonEmpty(row.PlatformName, row.SourceCode)) + row.SuggestedIdentifierType = "JT808_PHONE" + row.SuggestedIdentifierValue = strings.TrimSpace(row.Phone) + row.SuggestedVIN = suggestedMappingVIN(*row) + row.SuggestedPlate = strings.TrimSpace(firstNonEmpty(row.Plate, row.IdentifierPlate)) + row.Reason, row.RecommendedAction = diagnoseJT808MappingGap(*row) + row.RawFrameQueryPath = jt808MappingGapRawFrameQueryPath(*row) + row.DataSourceQueryPath = jt808MappingGapDataSourceQueryPath(*row) + row.VehicleIdentifierExample = vehicleIdentifierExample(*row) +} + +func diagnoseJT808MappingGap(row JT808MappingGapRow) (string, string) { + if strings.TrimSpace(row.SourceCode) == "" { + return "source_code_missing", "先维护 vehicle_data_source.source_code，再按来源平台导入 phone 到 VIN 的 vehicle_identifier 映射" + } + if strings.TrimSpace(row.IdentifierVIN) == "" && strings.TrimSpace(row.VIN) != "" && !strings.EqualFold(strings.TrimSpace(row.VIN), "unknown") { + return "missing_source_phone_identifier", "按当前来源 source_code 维护 JT808_PHONE 到 VIN 的映射；如平台名与 source_code 不一致，先修正来源配置" + } + if len(row.MatchedSourceCodes) > 0 { + return "source_identifier_mismatch", "该 phone 存在其他 source_code 映射，但当前来源缺少映射；确认平台归属后补当前 source_code 或修正来源配置" + } + return "missing_source_phone_identifier", "补充该 phone 在当前来源 source_code 下的 vehicle_identifier 映射" +} + +func suggestedMappingVIN(row JT808MappingGapRow) string { + vin := strings.TrimSpace(row.VIN) + if vin != "" && !strings.EqualFold(vin, "unknown") { + return vin + } + return strings.TrimSpace(row.IdentifierVIN) +} + +func jt808MappingGapRawFrameQueryPath(row JT808MappingGapRow) string { + if strings.TrimSpace(row.Phone) == "" { + return "" + } + dateFrom, dateTo := jt808IdentityGapDateWindow(row.LatestSeenAt) + values := url.Values{} + values.Set("protocol", "JT808") + values.Set("phone", row.Phone) + values.Set("dateFrom", dateFrom) + values.Set("dateTo", dateTo) + values.Set("orderBy", "eventTime") + values.Set("includeFields", "true") + values.Set("limit", "20") + return "/api/history/raw-frames?" + values.Encode() +} + +func jt808MappingGapDataSourceQueryPath(row JT808MappingGapRow) string { + if strings.TrimSpace(row.SourceIP) == "" { + return "" + } + values := url.Values{} + values.Set("protocol", "JT808") + values.Set("sourceIP", row.SourceIP) + values.Set("includeTotal", "true") + values.Set("limit", "20") + return "/api/stats/data-sources?" + values.Encode() +} + +func vehicleIdentifierExample(row JT808MappingGapRow) string { + sourceCode := strings.TrimSpace(row.SuggestedSourceCode) + phone := strings.TrimSpace(row.SuggestedIdentifierValue) + vin := strings.TrimSpace(row.SuggestedVIN) + if sourceCode == "" || phone == "" || vin == "" { + return "" + } + plate := strings.ReplaceAll(strings.TrimSpace(row.SuggestedPlate), "'", "''") + oem := strings.ReplaceAll(strings.TrimSpace(row.SuggestedPlatformName), "'", "''") + sourceCode = strings.ReplaceAll(sourceCode, "'", "''") + phone = strings.ReplaceAll(phone, "'", "''") + vin = strings.ReplaceAll(vin, "'", "''") + return fmt.Sprintf("protocol=JT808, source_code=%s, identifier_type=JT808_PHONE, identifier_value=%s, vin=%s, plate=%s, oem=%s", sourceCode, phone, vin, plate, oem) +} + +func jt808IdentityGapDateWindow(value string) (string, string) { + value = strings.TrimSpace(value) + for _, layout := range []string{"2006-01-02 15:04:05", time.RFC3339} { + seenAt, err := time.ParseInLocation(layout, value, time.FixedZone("Asia/Shanghai", 8*3600)) + if err == nil { + start := time.Date(seenAt.Year(), seenAt.Month(), seenAt.Day(), 0, 0, 0, 0, seenAt.Location()) + return start.Format("2006-01-02 15:04:05"), start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05") + } + } + now := time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600)) + start := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + return start.Format("2006-01-02 15:04:05"), start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05") +} + +func suggestSourceKind(row DataSourceDiagnosticRow) (kind string, confidence string, reason string) { + current := normalizeSourceKindForRead(row.SourceKind) + if current != "UNKNOWN" { + return current, "HIGH", "already_classified" + } + if strings.ToUpper(strings.TrimSpace(row.Protocol)) != "JT808" && + (strings.TrimSpace(row.SourceCode) != "" || strings.TrimSpace(row.PlatformName) != "") { + return "PLATFORM", "MEDIUM", "non_jt808_configured_source" + } + if row.Reason == "ambiguous_source_code" || row.MatchedSourceCodeCount > 1 { + return "UNKNOWN", "HIGH", "ambiguous_source_code_requires_manual_review" + } + if row.Reason == "no_identifier_match" { + return "UNKNOWN", "MEDIUM", "missing_vehicle_identifier_mapping" + } + if row.CandidateSourceCode != "" && row.MatchedSourceCodeCount == 1 { + if row.PhoneCount >= 5 || row.ActiveSpanSeconds >= 6*3600 { + return "PLATFORM", "HIGH", "single_source_code_with_many_phones_or_long_activity" + } + if row.PhoneCount <= 1 && row.ActiveSpanSeconds < 2*3600 && row.LatestSeenAgeSeconds >= 3600 { + return "DIRECT", "MEDIUM", "single_phone_short_lived_stale_source" + } + return "PLATFORM", "MEDIUM", "single_source_code_candidate" + } + if row.Reason == "no_registration" { + if strings.TrimSpace(row.SourceCode) != "" || strings.TrimSpace(row.PlatformName) != "" { + return "UNKNOWN", "MEDIUM", "manual_source_without_registration_evidence" + } + if row.LatestSeenAgeSeconds >= 24*3600 { + return "DIRECT", "LOW", "stale_unclassified_source_without_registration" + } + } + return "UNKNOWN", "LOW", "insufficient_evidence" +} + +func configuredSourceCodeConflict(row DataSourceDiagnosticRow) bool { + sourceCode := strings.TrimSpace(row.SourceCode) + if sourceCode == "" || row.MatchedSourceCodeCount == 0 { + return false + } + for _, matched := range row.MatchedSourceCodes { + if strings.EqualFold(strings.TrimSpace(matched), sourceCode) { + return false + } + } + return true +} + +func sourcePlatformNameMismatch(row DataSourceDiagnosticRow) bool { + platformName := strings.TrimSpace(row.PlatformName) + configuredPlatformName := strings.TrimSpace(row.ConfiguredSourceCodePlatformName) + if platformName == "" || configuredPlatformName == "" { + return false + } + return platformName != configuredPlatformName +} + +func lowIdentifierCoverage(row DataSourceDiagnosticRow) bool { + if strings.ToUpper(strings.TrimSpace(row.Protocol)) != "JT808" { + return false + } + if strings.TrimSpace(row.SourceCode) == "" || row.PhoneCount < 10 { + return false + } + return row.UnmappedPhoneCount > 0 && row.IdentifierMatchRatio < 0.8 +} + +func (q DataSourceQuery) String() string { + enabled := "any" + if q.Enabled != nil { + enabled = fmt.Sprint(*q.Enabled) + } + sourceCodeMissing := "any" + if q.SourceCodeMissing != nil { + sourceCodeMissing = fmt.Sprint(*q.SourceCodeMissing) + } + return fmt.Sprintf("protocol=%s sourceIP=%s sourceCode=%s sourceKind=%s sourceCodeMissing=%s enabled=%s limit=%d offset=%d", q.Protocol, q.SourceIP, q.SourceCode, q.SourceKind, sourceCodeMissing, enabled, q.Limit, q.Offset) +} diff --git a/go/vehicle-gateway/internal/stats/data_source_query_test.go b/go/vehicle-gateway/internal/stats/data_source_query_test.go new file mode 100644 index 00000000..c6a4073c --- /dev/null +++ b/go/vehicle-gateway/internal/stats/data_source_query_test.go @@ -0,0 +1,700 @@ +package stats + +import ( + "context" + "database/sql" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +const dataSourceSelectPattern = "SELECT id, protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, trust_priority, enabled, first_seen_at, latest_seen_at, remark, updated_at FROM vehicle_data_source" + +func TestDataSourceRepositoryQueriesWithOperationalFilters(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dataSourceSelectPattern). + WithArgs("JT808", "115.231.168.135", "G7S", "PLATFORM", 1, 20, 10). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", + "trust_priority", "enabled", "first_seen_at", "latest_seen_at", "remark", "updated_at", + }).AddRow( + 3, "JT808", "115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", + 10, 1, + time.Date(2026, 7, 8, 10, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), + time.Date(2026, 7, 12, 1, 16, 4, 0, time.FixedZone("Asia/Shanghai", 8*3600)), + "trusted source", + time.Date(2026, 7, 12, 1, 16, 5, 0, time.FixedZone("Asia/Shanghai", 8*3600)), + )) + + enabled := true + rows, err := NewDataSourceRepository(db).Query(context.Background(), DataSourceQuery{ + Protocol: "jt808", + SourceIP: "115.231.168.135", + SourceCode: "G7S", + SourceKind: "platform", + Enabled: &enabled, + Limit: 20, + Offset: 10, + }) + if err != nil { + t.Fatalf("Query() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.ID != 3 || row.Protocol != "JT808" || row.PlatformName != "G7 平台" || row.SourceCode != "G7S" || row.SourceKind != "PLATFORM" || !row.Enabled { + t.Fatalf("unexpected source row: %#v", row) + } + if row.LatestSeenAt != "2026-07-12 01:16:04" { + t.Fatalf("latest seen = %q", row.LatestSeenAt) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceRepositoryFiltersMissingSourceCode(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dataSourceSelectPattern). + WithArgs("JT808", 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", + "trust_priority", "enabled", "first_seen_at", "latest_seen_at", "remark", "updated_at", + })) + + missing := true + _, err = NewDataSourceRepository(db).Query(context.Background(), DataSourceQuery{ + Protocol: "JT808", + SourceCodeMissing: &missing, + }) + if err != nil { + t.Fatalf("Query() error = %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceRepositoryDiagnosesJT808SourceMapping(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("JT808", "117.132.194.31", 1, 10, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + }).AddRow( + 242190, "JT808", "117.132.194.31", "117.132.194.31:20471", nil, nil, + "UNKNOWN", "2026-07-11 23:00:00", "2026-07-12 01:36:26", 9386, 120, + 5, 4, 2, + 3, 1, 0.75, 0, nil, 2, "g7s", "G7s", "g7s,xinda", "G7s,信达", "13307795425,14400000000", + )) + + missing := true + rows, err := NewDataSourceRepository(db).QueryDiagnostics(context.Background(), DataSourceDiagnosticsQuery{ + Protocol: "jt808", + SourceIP: "117.132.194.31", + SourceCodeMissing: &missing, + Limit: 10, + }) + if err != nil { + t.Fatalf("QueryDiagnostics() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.Reason != "ambiguous_source_code" || row.IdentifierMatchedPhones != 3 || len(row.MatchedSourceCodes) != 2 || len(row.SamplePhones) != 2 { + t.Fatalf("unexpected diagnostic row: %#v", row) + } + if row.SuggestedSourceKind != "UNKNOWN" || row.SuggestionConfidence != "HIGH" { + t.Fatalf("unexpected kind suggestion: %#v", row) + } + sqlText, _ := buildDataSourceDiagnosticsSQL(DataSourceDiagnosticsQuery{Protocol: "JT808", Limit: 10}) + if !strings.Contains(sqlText, "ON ds.protocol = 'JT808'") || !strings.Contains(sqlText, "AND r.source_ip = ds.source_ip") { + t.Fatalf("diagnostics should join by indexed source_ip:\n%s", sqlText) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceDiagnosticsMappingIssueFilterUsesHaving(t *testing.T) { + missing := false + query := normalizeDataSourceDiagnosticsQuery(DataSourceDiagnosticsQuery{ + Protocol: "JT808", + SourceCodeMissing: &missing, + MappingIssueOnly: true, + Limit: 10, + }) + sqlText, args := buildDataSourceDiagnosticsSQL(query) + if !strings.Contains(sqlText, "HAVING") || !strings.Contains(sqlText, "identifier_match_ratio < 0.8") || !strings.Contains(sqlText, "configured_source_code_platform_name") { + t.Fatalf("mapping issue query should include aggregate HAVING:\n%s", sqlText) + } + if strings.Contains(sqlText, "ds.platform_name IS NOT NULL") { + t.Fatalf("mapping issue HAVING should use aggregate aliases, not table-qualified platform_name:\n%s", sqlText) + } + if len(args) != 4 { + t.Fatalf("mapping issue query args = %#v, want protocol/enabled/limit/offset", args) + } + countSQL, _ := buildDataSourceDiagnosticsCountSQL(query) + if !strings.Contains(countSQL, "FROM (") || !strings.Contains(countSQL, "HAVING") { + t.Fatalf("mapping issue count should wrap grouped diagnostics:\n%s", countSQL) + } + if strings.Contains(countSQL, "ds.platform_name IS NOT NULL") { + t.Fatalf("mapping issue count HAVING should use aggregate aliases, not table-qualified platform_name:\n%s", countSQL) + } +} + +func TestDiagnoseDataSourceHighlightsConfiguredMappingIssues(t *testing.T) { + lowCoverage := DataSourceDiagnosticRow{ + Protocol: "JT808", + PlatformName: "东方北斗", + SourceCode: "dongfang_beidou", + RegistrationRows: 388, + PhoneCount: 388, + IdentifierMatchedPhones: 8, + UnmappedPhoneCount: 380, + IdentifierMatchRatio: 0.0206, + ConfiguredSourceCodeMatchedPhones: 8, + ConfiguredSourceCodePlatformName: "东方北斗", + MatchedSourceCodeCount: 1, + MatchedSourceCodes: []string{"dongfang_beidou"}, + } + lowCoverage.ConfiguredSourceCodeConflict = configuredSourceCodeConflict(lowCoverage) + lowCoverage.SourcePlatformNameMismatch = sourcePlatformNameMismatch(lowCoverage) + reason, _ := diagnoseDataSource(lowCoverage) + if reason != "low_identifier_coverage" { + t.Fatalf("reason = %q, want low_identifier_coverage", reason) + } + + lowCoverageMismatch := lowCoverage + lowCoverageMismatch.PlatformName = "G7易流" + lowCoverageMismatch.SourcePlatformNameMismatch = sourcePlatformNameMismatch(lowCoverageMismatch) + reason, _ = diagnoseDataSource(lowCoverageMismatch) + if reason != "low_identifier_coverage" { + t.Fatalf("reason = %q, want low_identifier_coverage when coverage is weak even if platform name differs", reason) + } + + nameMismatch := DataSourceDiagnosticRow{ + Protocol: "JT808", + PlatformName: "G7易流", + SourceCode: "dongfang_beidou", + RegistrationRows: 388, + PhoneCount: 388, + IdentifierMatchedPhones: 8, + ConfiguredSourceCodePlatformName: "东方北斗", + MatchedSourceCodeCount: 1, + MatchedSourceCodes: []string{"dongfang_beidou"}, + } + nameMismatch.SourcePlatformNameMismatch = sourcePlatformNameMismatch(nameMismatch) + reason, _ = diagnoseDataSource(nameMismatch) + if reason != "source_platform_name_mismatch" { + t.Fatalf("reason = %q, want source_platform_name_mismatch", reason) + } + + conflict := DataSourceDiagnosticRow{ + Protocol: "JT808", + SourceCode: "g7s", + RegistrationRows: 20, + PhoneCount: 20, + IdentifierMatchedPhones: 20, + IdentifierMatchRatio: 1, + ConfiguredSourceCodeMatchedPhones: 0, + MatchedSourceCodeCount: 1, + MatchedSourceCodes: []string{"dongfang_beidou"}, + } + conflict.ConfiguredSourceCodeConflict = configuredSourceCodeConflict(conflict) + reason, _ = diagnoseDataSource(conflict) + if reason != "source_code_conflict" { + t.Fatalf("reason = %q, want source_code_conflict", reason) + } +} + +func TestDataSourceHandlerReturnsSourcePage(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_data_source"). + WithArgs("GB32960", 1). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(3)) + mock.ExpectQuery(dataSourceSelectPattern). + WithArgs("GB32960", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", + "trust_priority", "enabled", "first_seen_at", "latest_seen_at", "remark", "updated_at", + }).AddRow( + 7, "GB32960", "8.134.95.166", "8.134.95.166:56432", "现代 HTWO", "HYUNDAI", "PLATFORM", + 5, 1, "2026-07-11 18:27:10", "2026-07-12 01:15:52", "", "2026-07-12 01:15:52", + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources?protocol=gb32960&enabled=true&includeTotal=true", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":3`, `"source_ip":"8.134.95.166"`, `"platform_name":"现代 HTWO"`, `"source_code":"HYUNDAI"`, `"source_kind":"PLATFORM"`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsDiagnosticsPage(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_data_source ds"). + WithArgs("JT808", 1). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("JT808", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + }).AddRow( + 242190, "JT808", "117.132.194.31", "117.132.194.31:20471", nil, nil, + "UNKNOWN", "2026-07-12 01:00:00", "2026-07-12 01:36:26", 2186, 1800, + 5, 4, 2, + 0, 4, 0.0, 0, nil, 0, nil, nil, nil, nil, "13307795425", + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/diagnostics?protocol=jt808&includeTotal=true", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":1`, `"reason":"no_identifier_match"`, `"sample_phones":["13307795425"]`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsJT808IdentityGaps(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM jt808_registration r"). + WithArgs("117.132.194.31", "guangan_beidou", 3600). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) + mock.ExpectQuery("FROM jt808_registration r"). + WithArgs("117.132.194.31", "guangan_beidou", 3600, 20, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "phone", "device_id", "plate", "vin", "source_ip", "source_endpoint", + "source_code", "platform_name", "source_kind", + "first_registered_at", "latest_registered_at", "latest_authenticated_at", "latest_seen_at", "latest_seen_age_seconds", + }).AddRow( + "13307795425", "", "沪A63305F", "unknown", "117.132.194.31", "117.132.194.31:20471", + "guangan_beidou", "广安北斗", "PLATFORM", + nil, nil, nil, "2026-07-12 22:24:53", 32, + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/jt808-identity-gaps?sourceIP=117.132.194.31&sourceCode=guangan_beidou&recentSeconds=3600&includeTotal=true&limit=20", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{ + `"total":1`, + `"phone":"13307795425"`, + `"reason":"missing_phone_and_plate_binding"`, + `"raw_frame_query_path":"/api/history/raw-frames?`, + `phone=13307795425`, + `dateFrom=2026-07-12+00%3A00%3A00`, + `"data_source_query_path":"/api/stats/data-sources?`, + `sourceIP=117.132.194.31`, + } { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsJT808MappingGaps(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM jt808_registration r"). + WithArgs("115.159.85.149", "dongfang_beidou", 3600). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) + mock.ExpectQuery("FROM jt808_registration r"). + WithArgs("115.159.85.149", "dongfang_beidou", 3600, 20, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "phone", "device_id", "plate", "vin", "source_ip", "source_endpoint", + "source_code", "platform_name", "source_kind", "identifier_vin", "identifier_plate", + "matched_source_codes", "matched_platform_names", "latest_seen_at", "latest_seen_age_seconds", + }).AddRow( + "64646848246", "", "粤AG18312", "LKLG7C4E8NA774778", "115.159.85.149", "115.159.85.149:16885", + "dongfang_beidou", "G7易流", "PLATFORM", nil, nil, + "g7s", "G7s", "2026-07-12 23:38:22", 32, + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/jt808-mapping-gaps?sourceIP=115.159.85.149&sourceCode=dongfang_beidou&recentSeconds=3600&includeTotal=true&limit=20", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{ + `"total":1`, + `"phone":"64646848246"`, + `"suggested_source_code":"dongfang_beidou"`, + `"suggested_identifier_type":"JT808_PHONE"`, + `"suggested_vin":"LKLG7C4E8NA774778"`, + `"reason":"missing_source_phone_identifier"`, + `"matched_source_codes":["g7s"]`, + `"raw_frame_query_path":"/api/history/raw-frames?`, + `phone=64646848246`, + `"data_source_query_path":"/api/stats/data-sources?`, + `sourceIP=115.159.85.149`, + `"vehicle_identifier_example":"protocol=JT808, source_code=dongfang_beidou, identifier_type=JT808_PHONE, identifier_value=64646848246, vin=LKLG7C4E8NA774778, plate=粤AG18312, oem=G7易流"`, + } { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestJT808MappingGapSQLUsesConfiguredSourceIdentifier(t *testing.T) { + sqlText, args := buildJT808MappingGapSQL(normalizeJT808MappingGapQuery(JT808MappingGapQuery{ + SourceIP: "115.159.85.149", + SourceCode: "dongfang_beidou", + RecentSeconds: 3600, + Limit: 20, + })) + for _, want := range []string{ + "JOIN vehicle_data_source ds", + "source_vi.source_code = ds.source_code", + "source_vi.identifier_type = 'JT808_PHONE'", + "source_vi.identifier_value = r.phone", + "source_vi.identifier_value IS NULL OR source_vi.vin IS NULL", + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("mapping gap SQL missing %q:\n%s", want, sqlText) + } + } + if len(args) != 5 { + t.Fatalf("args = %#v, want sourceIP/sourceCode/recent/limit/offset", args) + } +} + +func TestDataSourceDiagnosticsCanQueryRetiredSources(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("JT808", 0, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + })) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/diagnostics?protocol=jt808&enabled=false", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + sqlText, args := buildDataSourceDiagnosticsSQL(normalizeDataSourceDiagnosticsQuery(DataSourceDiagnosticsQuery{Protocol: "JT808"})) + if !strings.Contains(sqlText, "ds.enabled = ?") { + t.Fatalf("diagnostics should default to enabled sources:\n%s", sqlText) + } + if len(args) < 2 || args[1] != 1 { + t.Fatalf("diagnostics enabled default args = %#v, want enabled=1 before limit", args) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsGenericDiagnosticsForNonJT808(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("GB32960", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + }).AddRow( + 7, "GB32960", "8.134.95.166", "8.134.95.166:32960", "现代 HTWO", nil, + "UNKNOWN", "2026-07-12 01:00:00", "2026-07-12 01:36:26", 2186, 1800, + 0, 0, 0, + 0, 0, 0.0, 0, nil, 0, nil, nil, nil, nil, nil, + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/diagnostics?protocol=GB32960", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"protocol":"GB32960"`, `"reason":"source_code_missing"`, `"suggested_source_kind":"PLATFORM"`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsKindSuggestionsPage(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_data_source ds"). + WithArgs("JT808", "UNKNOWN", 1). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("JT808", "UNKNOWN", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + }).AddRow( + 242190, "JT808", "117.132.194.31", "117.132.194.31:20471", "广安北斗", "guangan_beidou", + "UNKNOWN", "2026-07-12 00:00:00", "2026-07-12 00:05:00", 300, 7200, + 0, 0, 0, + 0, 0, 0.0, 0, nil, 0, nil, nil, nil, nil, nil, + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/kind-suggestions?protocol=jt808&includeTotal=true", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":1`, `"suggested_source_kind":"UNKNOWN"`, `"suggestion_confidence":"MEDIUM"`, `"suggestion_reason":"manual_source_without_registration_evidence"`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerReturnsKindSuggestionsForNonJT808(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("FROM vehicle_data_source ds"). + WithArgs("GB32960", "UNKNOWN", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "source_ip", "latest_source_endpoint", "platform_name", "source_code", + "source_kind", "first_seen_at", "latest_seen_at", "active_span_seconds", "latest_seen_age_seconds", "registration_rows", "phone_count", "unknown_vin_rows", + "identifier_matched_phones", "unmapped_phone_count", "identifier_match_ratio", "configured_source_code_matched_phones", "configured_source_code_platform_name", "matched_source_code_count", "candidate_source_code", + "candidate_platform_name", "matched_source_codes", "matched_platform_names", "sample_phones", + }).AddRow( + 7, "GB32960", "8.134.95.166", "8.134.95.166:32960", "现代 HTWO", "HYUNDAI", + "UNKNOWN", "2026-07-12 01:00:00", "2026-07-12 01:36:26", 2186, 1800, + 0, 0, 0, + 0, 0, 0.0, 0, nil, 0, nil, nil, nil, nil, nil, + )) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources/kind-suggestions?protocol=GB32960", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"reason":"source_configured"`, `"suggested_source_kind":"PLATFORM"`, `"suggestion_reason":"non_jt808_configured_source"`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestSuggestSourceKindMarksStaleUnclassifiedSourceAsDirect(t *testing.T) { + kind, confidence, reason := suggestSourceKind(DataSourceDiagnosticRow{ + SourceKind: "UNKNOWN", + Reason: "no_registration", + LatestSeenAgeSeconds: 25 * 3600, + }) + if kind != "DIRECT" || confidence != "LOW" || reason != "stale_unclassified_source_without_registration" { + t.Fatalf("unexpected suggestion: kind=%s confidence=%s reason=%s", kind, confidence, reason) + } +} + +func TestDataSourceHandlerPatchesOnlyManualFields(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT 1 FROM vehicle_data_source WHERE id = \\? LIMIT 1"). + WithArgs(int64(3)). + WillReturnRows(sqlmock.NewRows([]string{"one"}).AddRow(1)) + mock.ExpectExec("UPDATE vehicle_data_source SET platform_name = \\?, source_code = \\?, source_kind = \\?, trust_priority = \\?, enabled = \\?, remark = \\?, updated_at = CURRENT_TIMESTAMP WHERE id = \\?"). + WithArgs("G7 平台", "G7S", "PLATFORM", 10, 0, "可信 808 来源", int64(3)). + WillReturnResult(sqlmock.NewResult(0, 0)) + + handler := NewDataSourceHandler(NewDataSourceRepository(db)) + body := strings.NewReader(`{"platform_name":"G7 平台","source_code":"G7S","source_kind":"platform","trust_priority":10,"enabled":false,"remark":"可信 808 来源","latest_seen_at":"2099-01-01 00:00:00"}`) + request := httptest.NewRequest(http.MethodPatch, "/api/stats/data-sources/3", body) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"updated":true`) { + t.Fatalf("patch response should confirm update: %s", response.Body.String()) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestDataSourceHandlerRejectsConflictingSourceCodeFilters(t *testing.T) { + handler := NewDataSourceHandler(NewDataSourceRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodGet, "/api/stats/data-sources?sourceCode=g7s&sourceCodeMissing=true", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "sourceCode") { + t.Fatalf("response should mention sourceCode: %s", response.Body.String()) + } +} + +func TestDataSourceHandlerRejectsInvalidSourceCode(t *testing.T) { + handler := NewDataSourceHandler(NewDataSourceRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodPatch, "/api/stats/data-sources/1", strings.NewReader(`{"source_code":"G7 中文"}`)) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "source_code") { + t.Fatalf("response should mention source_code: %s", response.Body.String()) + } +} + +func TestDataSourceHandlerRejectsInvalidSourceKind(t *testing.T) { + handler := NewDataSourceHandler(NewDataSourceRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodPatch, "/api/stats/data-sources/1", strings.NewReader(`{"source_kind":"temporary"}`)) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "source_kind") { + t.Fatalf("response should mention source_kind: %s", response.Body.String()) + } +} + +func TestDataSourceHandlerRejectsEmptyPatch(t *testing.T) { + handler := NewDataSourceHandler(NewDataSourceRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodPatch, "/api/stats/data-sources/1", strings.NewReader(`{}`)) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "no mutable fields") { + t.Fatalf("response should mention no mutable fields: %s", response.Body.String()) + } +} diff --git a/go/vehicle-gateway/internal/stats/query.go b/go/vehicle-gateway/internal/stats/query.go index 000f2499..5ddabc30 100644 --- a/go/vehicle-gateway/internal/stats/query.go +++ b/go/vehicle-gateway/internal/stats/query.go @@ -7,9 +7,13 @@ import ( "errors" "fmt" "net/http" + "net/url" + "sort" "strconv" "strings" "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) type Queryer interface { @@ -26,20 +30,182 @@ type MetricQuery struct { Offset int } +type MetricSourceQuery struct { + VIN string + Protocol string + DateFrom string + DateTo string + SourceIP string + QualityStatus string + QualityReason string + Selected *bool + OrderBy string + IncludeTotal bool + Limit int + Offset int +} + +type MetricDiagnosisQuery struct { + VIN string + Protocol string + Date string + Diagnosis string + Reason string + Severity string + IncludeTotal bool + Limit int + Offset int +} + type MetricRow struct { VIN string `json:"vin"` StatDate string `json:"stat_date"` Protocol string `json:"protocol"` SourceID *int64 `json:"source_id,omitempty"` + SourceIP string `json:"source_ip,omitempty"` + LatestSourceEndpoint string `json:"latest_source_endpoint,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind,omitempty"` DailyMileageKM float64 `json:"daily_mileage_km"` LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` UpdatedAt string `json:"updated_at"` } +type MetricSourceRow struct { + VIN string `json:"vin"` + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + SourceKey string `json:"source_key"` + SourceIP string `json:"source_ip"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + Phone string `json:"phone,omitempty"` + DeviceID string `json:"device_id,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + SourceID *int64 `json:"source_id,omitempty"` + SourceCode string `json:"source_code,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + SourceEnabled *bool `json:"source_enabled,omitempty"` + TrustPriority *int `json:"trust_priority,omitempty"` + FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"` + LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` + DailyMileageKM float64 `json:"daily_mileage_km"` + SampleCount int64 `json:"sample_count"` + FirstEventTime string `json:"first_event_time,omitempty"` + LatestEventTime string `json:"latest_event_time,omitempty"` + QualityStatus string `json:"quality_status"` + QualityReason string `json:"quality_reason,omitempty"` + IsSelected bool `json:"is_selected"` + SelectionStatus string `json:"selection_status,omitempty"` + SelectionReason string `json:"selection_reason,omitempty"` + SelectionAction string `json:"selection_action,omitempty"` + UpdatedAt string `json:"updated_at"` +} + +type MetricDiagnosisRow struct { + VIN string `json:"vin"` + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Plate string `json:"plate,omitempty"` + PlatformName string `json:"platform_name,omitempty"` + Peer string `json:"peer,omitempty"` + SnapshotEventTime string `json:"snapshot_event_time,omitempty"` + LocationEventTime string `json:"location_event_time,omitempty"` + SnapshotUpdatedAt string `json:"snapshot_updated_at,omitempty"` + LocationUpdatedAt string `json:"location_updated_at,omitempty"` + RealtimeTotalMileageKM *float64 `json:"realtime_total_mileage_km,omitempty"` + RealtimeTotalMileageAt string `json:"realtime_total_mileage_event_time,omitempty"` + DailyMileageKM *float64 `json:"daily_mileage_km,omitempty"` + DailyLatestTotalMileageKM *float64 `json:"daily_latest_total_mileage_km,omitempty"` + SourceSampleCount int64 `json:"source_sample_count"` + OKSourceCount int64 `json:"ok_source_count"` + SelectableSourceCount int64 `json:"selectable_source_count"` + LatestStatEventTime string `json:"latest_stat_event_time,omitempty"` + QualityStatuses []string `json:"quality_statuses,omitempty"` + RealtimeFieldCount int `json:"realtime_field_count,omitempty"` + RealtimeSampleFields []string `json:"realtime_sample_fields,omitempty"` + RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status,omitempty"` + RealtimeMileageCandidateFields []string `json:"realtime_mileage_candidate_fields,omitempty"` + RealtimeMileageEvidence string `json:"realtime_mileage_evidence,omitempty"` + RawFrameQueryPath string `json:"raw_frame_query_path,omitempty"` + SourceSampleQueryPath string `json:"source_sample_query_path,omitempty"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + Severity string `json:"severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` +} + +type MetricDiagnosisSummaryRow struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + ActiveCount int64 `json:"active_count"` + OKCount int64 `json:"ok_count"` + MissingDailyCount int64 `json:"missing_daily_count"` + NoSourceSampleCount int64 `json:"no_source_sample_count"` + NoTotalMileageCount int64 `json:"no_total_mileage_count"` + ActionableIssueCount int64 `json:"actionable_issue_count"` +} + +type MetricDiagnosisReasonSummaryRow struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + Count int64 `json:"count"` + Severity string `json:"severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` +} + +type MetricDiagnosisFieldStatusSummaryRow struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + Diagnosis string `json:"diagnosis"` + Reason string `json:"reason"` + RealtimeMileageFieldStatus string `json:"realtime_mileage_field_status"` + Count int64 `json:"count"` + Severity string `json:"severity,omitempty"` + FieldStatusSeverity string `json:"field_status_severity,omitempty"` + RecommendedOperatorAction string `json:"recommended_operator_action,omitempty"` + FieldStatusAction string `json:"field_status_action,omitempty"` +} + +type MetricSourceQualitySummaryRow struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + QualityStatus string `json:"quality_status"` + QualityReason string `json:"quality_reason,omitempty"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SelectedCount int64 `json:"selected_count"` + SampleCount int64 `json:"sample_count"` + DailyMileageKM float64 `json:"daily_mileage_km"` +} + +type MetricSourceSelectionSummaryRow struct { + StatDate string `json:"stat_date"` + Protocol string `json:"protocol"` + SelectionStatus string `json:"selection_status"` + SelectionReason string `json:"selection_reason"` + SelectionAction string `json:"selection_action,omitempty"` + SourceCount int64 `json:"source_count"` + VehicleCount int64 `json:"vehicle_count"` + SelectedCount int64 `json:"selected_count"` + SampleCount int64 `json:"sample_count"` + DailyMileageKM float64 `json:"daily_mileage_km"` +} + type MetricRepository struct { db Queryer } +const ( + metricDiagnosisSeverityOK = "ok" + metricDiagnosisSeverityPipeline = "pipeline" + metricDiagnosisSeveritySourceData = "source_data" + metricDiagnosisRealtimeFieldLimit = 80 + metricDiagnosisMileageCandidateLimit = 20 +) + func NewMetricRepository(db Queryer) *MetricRepository { if db == nil { panic("metric query db must not be nil") @@ -62,12 +228,18 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr var statDate scanDate var updatedAt scanDateTime var sourceID sql.NullInt64 + var sourceIP, endpoint, platform, sourceCode, sourceKind sql.NullString var latest sql.NullFloat64 if err := rows.Scan( &row.VIN, &statDate, &row.Protocol, &sourceID, + &sourceIP, + &endpoint, + &platform, + &sourceCode, + &sourceKind, &row.DailyMileageKM, &latest, &updatedAt, @@ -79,6 +251,14 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr if sourceID.Valid { row.SourceID = &sourceID.Int64 } + row.SourceIP = sourceIP.String + row.LatestSourceEndpoint = endpoint.String + row.PlatformName = platform.String + row.SourceCode = sourceCode.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + if !sourceKind.Valid { + row.SourceKind = "" + } if latest.Valid { row.LatestTotalMileageKM = &latest.Float64 } @@ -87,6 +267,334 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr return out, rows.Err() } +func (r *MetricRepository) QuerySources(ctx context.Context, query MetricSourceQuery) ([]MetricSourceRow, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricSourceRow, 0) + for rows.Next() { + var row MetricSourceRow + var statDate scanDate + var endpoint, phone, deviceID, platform, sourceCode, sourceKind, qualityReason sql.NullString + var sourceID, enabled, trustPriority sql.NullInt64 + var firstTotal, latestTotal sql.NullFloat64 + var firstEvent, latestEvent, updatedAt scanDateTime + var selected int + if err := rows.Scan( + &row.VIN, + &statDate, + &row.Protocol, + &row.SourceKey, + &row.SourceIP, + &endpoint, + &phone, + &deviceID, + &platform, + &sourceID, + &sourceCode, + &sourceKind, + &enabled, + &trustPriority, + &firstTotal, + &latestTotal, + &row.DailyMileageKM, + &row.SampleCount, + &firstEvent, + &latestEvent, + &row.QualityStatus, + &qualityReason, + &selected, + &updatedAt, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.SourceEndpoint = endpoint.String + row.Phone = phone.String + row.DeviceID = deviceID.String + row.PlatformName = platform.String + if sourceID.Valid { + row.SourceID = &sourceID.Int64 + } + row.SourceCode = sourceCode.String + row.SourceKind = normalizeSourceKindForRead(sourceKind.String) + if !sourceKind.Valid { + row.SourceKind = "" + } + if enabled.Valid { + value := enabled.Int64 != 0 + row.SourceEnabled = &value + } + if trustPriority.Valid { + value := int(trustPriority.Int64) + row.TrustPriority = &value + } + if firstTotal.Valid { + row.FirstTotalMileageKM = &firstTotal.Float64 + } + if latestTotal.Valid { + row.LatestTotalMileageKM = &latestTotal.Float64 + } + row.FirstEventTime = firstEvent.String + row.LatestEventTime = latestEvent.String + row.QualityReason = qualityReason.String + row.IsSelected = selected != 0 + row.SelectionStatus, row.SelectionReason, row.SelectionAction = metricSourceSelectionEvidence(row) + row.UpdatedAt = updatedAt.String + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QueryDiagnostics(ctx context.Context, query MetricDiagnosisQuery) ([]MetricDiagnosisRow, error) { + query = normalizeMetricDiagnosisQuery(query) + sqlText, args := buildMetricDiagnosisSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricDiagnosisRow, 0) + for rows.Next() { + var row MetricDiagnosisRow + var statDate scanDate + var snapshotEvent, locationEvent, snapshotUpdated, locationUpdated, mileageAt, latestStatEvent scanDateTime + var plate, platform, peer, qualityStatuses, snapshotParsedJSON sql.NullString + var realtimeTotal, dailyMileage, dailyLatestTotal sql.NullFloat64 + if err := rows.Scan( + &row.VIN, + &statDate, + &row.Protocol, + &plate, + &platform, + &peer, + &snapshotEvent, + &locationEvent, + &snapshotUpdated, + &locationUpdated, + &realtimeTotal, + &mileageAt, + &dailyMileage, + &dailyLatestTotal, + &row.SourceSampleCount, + &row.OKSourceCount, + &row.SelectableSourceCount, + &latestStatEvent, + &qualityStatuses, + &snapshotParsedJSON, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.Plate = plate.String + row.PlatformName = platform.String + row.Peer = peer.String + row.SnapshotEventTime = snapshotEvent.String + row.LocationEventTime = locationEvent.String + row.SnapshotUpdatedAt = snapshotUpdated.String + row.LocationUpdatedAt = locationUpdated.String + row.RealtimeTotalMileageAt = mileageAt.String + row.LatestStatEventTime = latestStatEvent.String + if realtimeTotal.Valid { + row.RealtimeTotalMileageKM = &realtimeTotal.Float64 + } + if dailyMileage.Valid { + row.DailyMileageKM = &dailyMileage.Float64 + } + if dailyLatestTotal.Valid { + row.DailyLatestTotalMileageKM = &dailyLatestTotal.Float64 + } + row.QualityStatuses = splitMetricStatuses(qualityStatuses.String) + row.RealtimeFieldCount, row.RealtimeSampleFields = metricDiagnosisRealtimeFields(snapshotParsedJSON.String, metricDiagnosisRealtimeFieldLimit) + row.Diagnosis, row.Reason = classifyMetricDiagnosis(dailyMileage.Valid, row.SourceSampleCount, row.OKSourceCount, row.SelectableSourceCount, realtimeTotal, row.RealtimeTotalMileageAt, row.StatDate) + row.Severity = metricDiagnosisIssueSeverity(row.Diagnosis, row.Reason) + row.RealtimeMileageFieldStatus, row.RealtimeMileageCandidateFields, row.RealtimeMileageEvidence = metricDiagnosisRealtimeMileageEvidence( + row.Protocol, + row.Diagnosis, + row.Reason, + row.RealtimeFieldCount, + row.RealtimeSampleFields, + snapshotParsedJSON.String, + row.RealtimeTotalMileageKM, + row.RealtimeTotalMileageAt, + row.StatDate, + ) + row.RawFrameQueryPath = metricDiagnosisRawFrameQueryPath(row) + row.SourceSampleQueryPath = metricDiagnosisSourceSampleQueryPath(row) + row.RecommendedOperatorAction = metricDiagnosisRecommendedOperatorAction(row.Diagnosis, row.Reason) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QueryDiagnosisSummary(ctx context.Context, query MetricDiagnosisQuery) ([]MetricDiagnosisSummaryRow, error) { + query = normalizeMetricDiagnosisQuery(query) + query.Diagnosis = "" + sqlText, args := buildMetricDiagnosisSummarySQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricDiagnosisSummaryRow, 0) + for rows.Next() { + var row MetricDiagnosisSummaryRow + var statDate scanDate + if err := rows.Scan( + &statDate, + &row.Protocol, + &row.ActiveCount, + &row.OKCount, + &row.MissingDailyCount, + &row.NoSourceSampleCount, + &row.NoTotalMileageCount, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.ActionableIssueCount = row.MissingDailyCount + row.NoSourceSampleCount + row.NoTotalMileageCount + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QueryDiagnosisReasonSummary(ctx context.Context, query MetricDiagnosisQuery) ([]MetricDiagnosisReasonSummaryRow, error) { + query = normalizeMetricDiagnosisQuery(query) + sqlText, args := buildMetricDiagnosisReasonSummarySQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricDiagnosisReasonSummaryRow, 0) + for rows.Next() { + var row MetricDiagnosisReasonSummaryRow + var statDate scanDate + if err := rows.Scan( + &statDate, + &row.Protocol, + &row.Diagnosis, + &row.Reason, + &row.Count, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.Severity = metricDiagnosisIssueSeverity(row.Diagnosis, row.Reason) + row.RecommendedOperatorAction = metricDiagnosisRecommendedOperatorAction(row.Diagnosis, row.Reason) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QueryDiagnosisFieldStatusSummary(ctx context.Context, query MetricDiagnosisQuery) ([]MetricDiagnosisFieldStatusSummaryRow, error) { + query = normalizeMetricDiagnosisQuery(query) + sqlText, args := buildMetricDiagnosisFieldStatusSummarySQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricDiagnosisFieldStatusSummaryRow, 0) + for rows.Next() { + var row MetricDiagnosisFieldStatusSummaryRow + var statDate scanDate + if err := rows.Scan( + &statDate, + &row.Protocol, + &row.Diagnosis, + &row.Reason, + &row.RealtimeMileageFieldStatus, + &row.Count, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.Severity = metricDiagnosisIssueSeverity(row.Diagnosis, row.Reason) + row.FieldStatusSeverity = metricDiagnosisMileageFieldStatusSeverity(row.RealtimeMileageFieldStatus) + row.RecommendedOperatorAction = metricDiagnosisRecommendedOperatorAction(row.Diagnosis, row.Reason) + row.FieldStatusAction = metricDiagnosisMileageFieldStatusAction(row.RealtimeMileageFieldStatus) + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QuerySourceQualitySummary(ctx context.Context, query MetricSourceQuery) ([]MetricSourceQualitySummaryRow, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceQualitySummarySQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricSourceQualitySummaryRow, 0) + for rows.Next() { + var row MetricSourceQualitySummaryRow + var statDate scanDate + var qualityReason sql.NullString + if err := rows.Scan( + &statDate, + &row.Protocol, + &row.QualityStatus, + &qualityReason, + &row.SourceCount, + &row.VehicleCount, + &row.SelectedCount, + &row.SampleCount, + &row.DailyMileageKM, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.QualityReason = qualityReason.String + out = append(out, row) + } + return out, rows.Err() +} + +func (r *MetricRepository) QuerySourceSelectionSummary(ctx context.Context, query MetricSourceQuery) ([]MetricSourceSelectionSummaryRow, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceSelectionSummarySQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + out := make([]MetricSourceSelectionSummaryRow, 0) + for rows.Next() { + var row MetricSourceSelectionSummaryRow + var statDate scanDate + if err := rows.Scan( + &statDate, + &row.Protocol, + &row.SelectionStatus, + &row.SelectionReason, + &row.SourceCount, + &row.VehicleCount, + &row.SelectedCount, + &row.SampleCount, + &row.DailyMileageKM, + ); err != nil { + return nil, err + } + row.StatDate = statDate.String + row.SelectionAction = metricSourceSelectionAction(row.SelectionStatus, row.SelectionReason) + out = append(out, row) + } + return out, rows.Err() +} + func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, error) { query = normalizeMetricQuery(query) sqlText, args := buildMetricCountSQL(query) @@ -105,6 +613,445 @@ func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64, return total, rows.Err() } +func (r *MetricRepository) CountSources(ctx context.Context, query MetricSourceQuery) (int64, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *MetricRepository) CountSourceQualitySummary(ctx context.Context, query MetricSourceQuery) (int64, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceQualitySummaryCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *MetricRepository) CountSourceSelectionSummary(ctx context.Context, query MetricSourceQuery) (int64, error) { + query = normalizeMetricSourceQuery(query) + sqlText, args := buildMetricSourceSelectionSummaryCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func (r *MetricRepository) CountDiagnostics(ctx context.Context, query MetricDiagnosisQuery) (int64, error) { + query = normalizeMetricDiagnosisQuery(query) + sqlText, args := buildMetricDiagnosisCountSQL(query) + rows, err := r.db.QueryContext(ctx, sqlText, args...) + if err != nil { + return 0, err + } + defer rows.Close() + + var total int64 + if rows.Next() { + if err := rows.Scan(&total); err != nil { + return 0, err + } + } + return total, rows.Err() +} + +func metricDiagnosisIssueSeverity(diagnosis, reason string) string { + switch strings.ToUpper(strings.TrimSpace(diagnosis)) { + case "", "OK": + return metricDiagnosisSeverityOK + case "MISSING_DAILY": + switch strings.ToLower(strings.TrimSpace(reason)) { + case "source_samples_all_invalid", "source_samples_all_excluded": + return metricDiagnosisSeveritySourceData + } + return metricDiagnosisSeverityPipeline + case "NO_SOURCE_SAMPLE": + return metricDiagnosisSeverityPipeline + case "NO_TOTAL_MILEAGE": + switch strings.ToLower(strings.TrimSpace(reason)) { + case "realtime_total_mileage_time_missing": + return metricDiagnosisSeverityPipeline + default: + return metricDiagnosisSeveritySourceData + } + default: + return metricDiagnosisSeverityPipeline + } +} + +func metricDiagnosisRecommendedOperatorAction(diagnosis, reason string) string { + diagnosis = strings.ToUpper(strings.TrimSpace(diagnosis)) + reason = strings.ToLower(strings.TrimSpace(reason)) + switch diagnosis { + case "", "OK": + return "无需处理，车辆当日里程已生成" + case "MISSING_DAILY": + switch reason { + case "source_samples_all_invalid": + return "统计候选样本已生成，但全部被质量规则排除，优先查看 source quality/selection 的 INVALID_DELTA、里程回退或异常跳变原因" + case "source_samples_all_excluded": + return "统计候选样本质量可用，但没有可参与选举的来源，优先检查 vehicle_data_source 是否被禁用或来源配置是否缺失" + } + return "统计源样本已存在但最终日表缺失，优先排查 stat-writer 投影任务、MySQL 写入和 source 选举" + case "NO_SOURCE_SAMPLE": + return "实时位置已有有效总里程但统计字段样本缺失，优先排查字段流 topic、stat-writer 消费和协议字段映射" + case "NO_TOTAL_MILEAGE": + switch reason { + case "realtime_total_mileage_time_missing": + return "总里程值存在但事件时间缺失，优先排查解析器和 history/realtime 写入的时间字段" + case "realtime_total_mileage_missing": + return "当日实时数据缺少总里程字段，核对平台是否上报该字段以及协议字段映射是否覆盖" + case "realtime_total_mileage_non_positive": + return "终端上报总里程小于等于 0，不参与差值统计，需核对设备或平台里程字段来源" + case "realtime_total_mileage_not_reported_on_stat_date": + return "最新总里程不是统计日期内上报；snapshot 里的里程可能是上一条完整帧保留值，需核对同日 raw_frames 是否真实上报总里程" + default: + return "车辆当日在线但没有可用于里程统计的总里程，优先核对源平台报文字段和解析映射" + } + default: + return "诊断状态未知，检查统计诊断 SQL 和上游字段流" + } +} + +func metricDiagnosisMileageFieldStatusSeverity(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "daily_metric_exists": + return metricDiagnosisSeverityOK + case "source_sample_exists", "standard_field_not_consumed", "standard_field_time_missing", "candidate_field_unmapped", "no_realtime_fields": + return metricDiagnosisSeverityPipeline + case "standard_field_non_positive", "standard_field_stale", "mapped_protocol_field_without_fresh_evidence", "no_candidate_mileage_field": + return metricDiagnosisSeveritySourceData + default: + return metricDiagnosisSeverityPipeline + } +} + +func metricDiagnosisMileageFieldStatusAction(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "daily_metric_exists": + return "最终日里程已生成，无需处理" + case "source_sample_exists": + return "统计候选样本已存在但最终表缺失，优先排查 source 选举和日表 upsert" + case "standard_field_not_consumed": + return "实时位置已有当日有效总里程但 stat-writer 未形成候选样本，优先排查字段流 topic 和消费位点" + case "standard_field_non_positive": + return "源头总里程小于等于 0，当前不参与差值统计，需要核对终端或平台字段来源" + case "standard_field_time_missing": + return "总里程值存在但缺少事件时间，优先修复协议解析或实时写入的时间字段" + case "standard_field_stale": + return "标准总里程仍停留在非统计日期；snapshot 字段可能是合并保留值，以同日 raw_frames 是否包含总里程为准" + case "candidate_field_unmapped": + return "实时字段中存在疑似里程字段但未进入标准 total_mileage_km，优先补字段映射和单位换算" + case "mapped_protocol_field_without_fresh_evidence": + return "合并快照保留了协议里程字段，但没有对应的新鲜里程时间；不能据此判定当前帧已上报，请按 raw_frame_query_path 核对统计日原始帧" + case "no_candidate_mileage_field": + return "实时字段中没有疑似里程字段，优先向源平台核对是否上报总里程" + case "no_realtime_fields": + return "实时快照没有解析字段，优先排查协议解析、实时写入和 snapshot 表更新" + default: + return "字段状态未知，检查诊断 SQL 和实时快照数据" + } +} + +func metricSourceSelectionEvidence(row MetricSourceRow) (status string, reason string, action string) { + if row.IsSelected { + status, reason = "selected", "selected_current_projection" + return status, reason, metricSourceSelectionAction(status, reason) + } + qualityStatus := strings.ToUpper(strings.TrimSpace(row.QualityStatus)) + qualityReason := strings.ToLower(strings.TrimSpace(row.QualityReason)) + if qualityStatus != "" && qualityStatus != QualityOK { + if qualityReason == "" { + qualityReason = strings.ToLower(qualityStatus) + } + status, reason = "excluded", "quality_"+qualityReason + return status, reason, metricSourceSelectionAction(status, reason) + } + if row.SourceEnabled != nil && !*row.SourceEnabled { + status, reason = "excluded", "source_disabled" + return status, reason, metricSourceSelectionAction(status, reason) + } + if row.SourceID == nil { + if normalizeSourceKindForRead(row.SourceKind) == "DIRECT" { + status, reason = "not_selected", "lower_trust_priority_or_sample_count" + return status, reason, metricSourceSelectionAction(status, reason) + } + status, reason = "not_selected", "unmanaged_source_lower_priority" + return status, reason, metricSourceSelectionAction(status, reason) + } + switch normalizeSourceKindForRead(row.SourceKind) { + case "UNKNOWN": + status, reason = "not_selected", "unknown_source_kind_lower_priority" + default: + status, reason = "not_selected", "lower_trust_priority_or_sample_count" + } + return status, reason, metricSourceSelectionAction(status, reason) +} + +func metricSourceSelectionAction(status string, reason string) string { + switch strings.ToLower(strings.TrimSpace(reason)) { + case "selected_current_projection": + return "当前来源已被投影到最终日里程表" + case "source_disabled": + return "来源已在 vehicle_data_source 中禁用，不参与最终日里程选举" + case "unmanaged_source_lower_priority": + return "来源尚未纳入 vehicle_data_source 管理；质量可用但未被本次选中" + case "unknown_source_kind_lower_priority": + return "来源类型仍是 UNKNOWN；质量可用但优先级低于已配置来源" + case "lower_trust_priority_or_sample_count": + return "来源质量可用，但被更高可信优先级、更多样本或更新时间更新的来源覆盖" + default: + if strings.EqualFold(strings.TrimSpace(status), "excluded") && strings.HasPrefix(strings.ToLower(strings.TrimSpace(reason)), "quality_") { + return "来源质量未通过，不参与最终日里程选举" + } + return "来源选举状态未知，检查 vehicle_daily_mileage_source 与 vehicle_data_source" + } +} + +func metricDiagnosisRealtimeFields(raw string, limit int) (int, []string) { + keys := metricDiagnosisRealtimeFieldKeys(raw) + if limit <= 0 || len(keys) <= limit { + return len(keys), keys + } + return len(keys), keys[:limit] +} + +func metricDiagnosisRealtimeFieldKeys(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + var values map[string]any + if err := json.Unmarshal([]byte(raw), &values); err != nil || len(values) == 0 { + return nil + } + keys := make([]string, 0, len(values)) + for key := range values { + key = strings.TrimSpace(key) + if key != "" { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +func metricDiagnosisRealtimeMileageEvidence(protocol string, diagnosis string, reason string, fieldCount int, sampleFields []string, raw string, realtimeTotal *float64, realtimeTotalAt string, statDate string) (string, []string, string) { + if strings.ToUpper(strings.TrimSpace(diagnosis)) != "NO_TOTAL_MILEAGE" { + return "", nil, "" + } + reason = strings.ToLower(strings.TrimSpace(reason)) + if reason != "realtime_total_mileage_missing" && reason != "realtime_total_mileage_not_reported_on_stat_date" { + return "", nil, "" + } + if realtimeTotal != nil { + switch { + case *realtimeTotal <= 0: + return "standard_field_non_positive", nil, "实时位置标准总里程小于等于 0，当前不参与差值统计" + case strings.TrimSpace(realtimeTotalAt) == "": + return "standard_field_time_missing", nil, "实时位置标准总里程已存在，但缺少该字段对应的事件时间" + case reason == "realtime_total_mileage_not_reported_on_stat_date" || !dateTimeInStatDate(realtimeTotalAt, statDate): + return "standard_field_stale", nil, "实时位置标准总里程已存在，但该里程时间不在统计日内；snapshot 可能保留上一条完整帧字段，请用 raw_frame_query_path 核对同日原始帧是否真实上报总里程" + } + } + if fieldCount == 0 { + return "no_realtime_fields", nil, "实时快照没有解析字段，优先检查协议解析、实时写入和 snapshot 表更新" + } + keys := metricDiagnosisRealtimeFieldKeys(raw) + if len(keys) == 0 { + keys = sampleFields + } + candidates := metricDiagnosisMileageCandidateFields(keys, metricDiagnosisMileageCandidateLimit) + if len(candidates) == 0 { + return "no_candidate_mileage_field", nil, "实时快照已有字段，但没有发现 mileage/odometer/odo 等疑似总里程字段；大概率源头未上报可用总里程，必要时查询 raw_frames 核对是否上报了 0 或异常值" + } + if known := knownMappedMileageFields(protocol, candidates); len(known) > 0 { + return "mapped_protocol_field_without_fresh_evidence", known, "合并快照保留了已知协议总里程字段，但没有对应的新鲜 total_mileage_event_time；该字段可能来自历史完整帧，不能作为当前帧上报证据" + } + return "candidate_field_unmapped", candidates, "实时快照存在疑似里程字段但未进入标准 total_mileage_km，优先检查协议字段映射、单位换算和实时 location 写入" +} + +func dateTimeInStatDate(value string, statDate string) bool { + value = strings.TrimSpace(value) + statDate = strings.TrimSpace(statDate) + if value == "" || statDate == "" { + return false + } + return strings.HasPrefix(value, statDate+" ") +} + +func knownMappedMileageFields(protocol string, candidates []string) []string { + if len(candidates) == 0 { + return nil + } + known := map[string]struct{}{} + for _, mapping := range mileageMappingsByProtocol(envelope.Protocol(strings.TrimSpace(protocol))) { + known[strings.ToLower(strings.TrimSpace(mapping.Key))] = struct{}{} + } + out := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + normalized := strings.ToLower(strings.TrimSpace(candidate)) + if _, ok := known[normalized]; !ok { + continue + } + out = append(out, candidate) + } + return out +} + +func metricDiagnosisRawFrameQueryPath(row MetricDiagnosisRow) string { + if strings.TrimSpace(row.VIN) == "" || strings.TrimSpace(row.Protocol) == "" || strings.TrimSpace(row.StatDate) == "" { + return "" + } + dateFrom, dateTo := metricDiagnosisDateWindow(row.StatDate) + values := url.Values{} + values.Set("protocol", row.Protocol) + values.Set("vin", row.VIN) + values.Set("dateFrom", dateFrom) + values.Set("dateTo", dateTo) + values.Set("orderBy", "eventTime") + values.Set("includeFields", "true") + values.Set("limit", "20") + return "/api/history/raw-frames?" + values.Encode() +} + +func metricDiagnosisSourceSampleQueryPath(row MetricDiagnosisRow) string { + if strings.TrimSpace(row.VIN) == "" || strings.TrimSpace(row.Protocol) == "" || strings.TrimSpace(row.StatDate) == "" { + return "" + } + values := url.Values{} + values.Set("vin", row.VIN) + values.Set("protocol", row.Protocol) + values.Set("dateFrom", row.StatDate) + values.Set("dateTo", row.StatDate) + values.Set("includeTotal", "true") + values.Set("limit", "50") + return "/api/stats/daily-metrics/sources?" + values.Encode() +} + +func metricDiagnosisDateWindow(statDate string) (string, string) { + date := strings.TrimSpace(statDate) + start, err := time.Parse("2006-01-02", date) + if err != nil { + return date, date + } + return start.Format("2006-01-02 15:04:05"), start.AddDate(0, 0, 1).Format("2006-01-02 15:04:05") +} + +func metricDiagnosisMileageCandidateFields(fields []string, limit int) []string { + out := make([]string, 0) + seen := map[string]struct{}{} + for _, field := range fields { + field = strings.TrimSpace(field) + if field == "" || !looksLikeMileageField(field) { + continue + } + if _, ok := seen[field]; ok { + continue + } + seen[field] = struct{}{} + out = append(out, field) + if limit > 0 && len(out) >= limit { + return out + } + } + return out +} + +func looksLikeMileageField(field string) bool { + normalized := strings.ToLower(strings.TrimSpace(field)) + normalized = strings.ReplaceAll(normalized, "-", "_") + normalized = strings.ReplaceAll(normalized, "/", ".") + switch { + case strings.Contains(normalized, "mileage"): + return true + case strings.Contains(normalized, "odometer"): + return true + case strings.Contains(normalized, ".odo"): + return true + case strings.Contains(normalized, "_odo"): + return true + case strings.HasSuffix(normalized, ".odo"): + return true + case strings.HasSuffix(normalized, "_odo"): + return true + default: + return false + } +} + +func metricDiagnosisReasonSummaryIssueTotals(rows []MetricDiagnosisReasonSummaryRow) (vehicleTotal, actionableTotal, pipelineTotal, sourceDataTotal int64) { + for _, row := range rows { + vehicleTotal += row.Count + switch row.Severity { + case metricDiagnosisSeverityPipeline: + actionableTotal += row.Count + pipelineTotal += row.Count + case metricDiagnosisSeveritySourceData: + actionableTotal += row.Count + sourceDataTotal += row.Count + default: + if strings.ToUpper(strings.TrimSpace(row.Diagnosis)) != "OK" { + actionableTotal += row.Count + } + } + } + return vehicleTotal, actionableTotal, pipelineTotal, sourceDataTotal +} + +func metricDiagnosisFieldStatusSummaryIssueTotals(rows []MetricDiagnosisFieldStatusSummaryRow) (vehicleTotal, actionableTotal, pipelineTotal, sourceDataTotal int64) { + for _, row := range rows { + vehicleTotal += row.Count + severity := strings.ToLower(strings.TrimSpace(row.Severity)) + if severity == "" { + severity = strings.ToLower(strings.TrimSpace(row.FieldStatusSeverity)) + } + switch severity { + case metricDiagnosisSeverityPipeline: + actionableTotal += row.Count + pipelineTotal += row.Count + case metricDiagnosisSeveritySourceData: + actionableTotal += row.Count + sourceDataTotal += row.Count + default: + if strings.ToUpper(strings.TrimSpace(row.Diagnosis)) != "OK" { + actionableTotal += row.Count + } + } + } + return vehicleTotal, actionableTotal, pipelineTotal, sourceDataTotal +} + func normalizeMetricQuery(query MetricQuery) MetricQuery { query.VIN = strings.TrimSpace(query.VIN) query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) @@ -116,28 +1063,559 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery { return query } +func normalizeMetricSourceQuery(query MetricSourceQuery) MetricSourceQuery { + query.VIN = strings.TrimSpace(query.VIN) + query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) + query.DateFrom = strings.TrimSpace(query.DateFrom) + query.DateTo = strings.TrimSpace(query.DateTo) + query.SourceIP = strings.TrimSpace(query.SourceIP) + query.QualityStatus = strings.ToUpper(strings.TrimSpace(query.QualityStatus)) + query.QualityReason = strings.ToLower(strings.TrimSpace(query.QualityReason)) + query.OrderBy = normalizeMetricSourceOrderBy(query.OrderBy) + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + +func normalizeMetricSourceOrderBy(value string) string { + normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", "")) + switch normalized { + case "dailymileagedesc", "dailymileage": + return "dailyMileageDesc" + default: + return "" + } +} + +func normalizeMetricDiagnosisQuery(query MetricDiagnosisQuery) MetricDiagnosisQuery { + query.VIN = strings.TrimSpace(query.VIN) + query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) + query.Date = strings.TrimSpace(query.Date) + if query.Date == "" { + query.Date = time.Now().In(time.FixedZone("Asia/Shanghai", 8*3600)).Format("2006-01-02") + } + query.Diagnosis = strings.ToUpper(strings.TrimSpace(query.Diagnosis)) + query.Reason = strings.ToLower(strings.TrimSpace(query.Reason)) + query.Severity = strings.ToLower(strings.TrimSpace(query.Severity)) + if query.Limit <= 0 { + query.Limit = 50 + } + return query +} + func buildMetricSQL(query MetricQuery) (string, []any) { where, args := buildMetricWhere(query) - sqlText := `SELECT vin, stat_date, protocol, source_id, - daily_mileage_km, latest_total_mileage_km, updated_at -FROM vehicle_daily_mileage` + sqlText := `SELECT m.vin, m.stat_date, m.protocol, m.source_id, + ds.source_ip, ds.latest_source_endpoint, ds.platform_name, ds.source_code, ds.source_kind, + m.daily_mileage_km, m.latest_total_mileage_km, m.updated_at +FROM vehicle_daily_mileage m +LEFT JOIN vehicle_data_source ds ON ds.id = m.source_id` if len(where) > 0 { sqlText += " WHERE " + strings.Join(where, " AND ") } - sqlText += " ORDER BY stat_date DESC, vin ASC, protocol ASC LIMIT ? OFFSET ?" + sqlText += " ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?" args = append(args, query.Limit, query.Offset) return sqlText, args } +func buildMetricSourceSQL(query MetricSourceQuery) (string, []any) { + where, args := buildMetricSourceWhere(query) + sqlText := `SELECT s.vin, s.stat_date, s.protocol, s.source_key, s.source_ip, s.source_endpoint, s.phone, s.device_id, + COALESCE(NULLIF(TRIM(s.platform_name), ''), ds.platform_name) AS platform_name, + ds.id, ds.source_code, ` + metricSourceKindSQL("s") + ` AS source_kind, ds.enabled, ds.trust_priority, + s.first_total_mileage_km, s.latest_total_mileage_km, s.daily_mileage_km, s.sample_count, + s.first_event_time, s.latest_event_time, s.quality_status, s.quality_reason, s.is_selected, s.updated_at +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += metricSourceOrderBySQL(query) + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func metricSourceOrderBySQL(query MetricSourceQuery) string { + if query.OrderBy == "dailyMileageDesc" { + return ` ORDER BY s.daily_mileage_km DESC, s.sample_count DESC, s.latest_event_time DESC, + s.stat_date DESC, s.vin ASC, s.protocol ASC, s.is_selected DESC, + CASE ` + metricSourceKindSQL("s") + ` + WHEN 'PLATFORM' THEN 0 + WHEN 'DIRECT' THEN 1 + WHEN 'UNKNOWN' THEN 2 + ELSE 3 + END, + ds.trust_priority, + s.source_key ASC +LIMIT ? OFFSET ?` + } + return ` ORDER BY s.stat_date DESC, s.vin ASC, s.protocol ASC, s.is_selected DESC, + CASE ` + metricSourceKindSQL("s") + ` + WHEN 'PLATFORM' THEN 0 + WHEN 'DIRECT' THEN 1 + WHEN 'UNKNOWN' THEN 2 + ELSE 3 + END, + ds.trust_priority, + s.sample_count DESC, + s.latest_event_time DESC, + s.source_key ASC +LIMIT ? OFFSET ?` +} + +func buildMetricSourceQualitySummarySQL(query MetricSourceQuery) (string, []any) { + where, args := buildMetricSourceWhere(query) + sqlText := `SELECT s.stat_date, s.protocol, s.quality_status, s.quality_reason, + COUNT(*) AS source_count, + COUNT(DISTINCT s.vin) AS vehicle_count, + SUM(CASE WHEN s.is_selected = 1 THEN 1 ELSE 0 END) AS selected_count, + COALESCE(SUM(s.sample_count), 0) AS sample_count, + COALESCE(SUM(s.daily_mileage_km), 0) AS daily_mileage_km +FROM vehicle_daily_mileage_source s` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += ` GROUP BY s.stat_date, s.protocol, s.quality_status, s.quality_reason +ORDER BY s.stat_date DESC, + s.protocol ASC, + CASE s.quality_status + WHEN 'OK' THEN 0 + WHEN 'NO_PREVIOUS_BASELINE' THEN 1 + WHEN 'INVALID_DELTA' THEN 2 + ELSE 3 + END, + source_count DESC, + s.quality_reason ASC +LIMIT ? OFFSET ?` + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildMetricSourceSelectionSummarySQL(query MetricSourceQuery) (string, []any) { + baseSQL, args := buildMetricSourceSelectionSummaryBaseSQL(query) + sqlText := `SELECT stat_date, protocol, selection_status, selection_reason, + COUNT(*) AS source_count, + COUNT(DISTINCT vin) AS vehicle_count, + SUM(CASE WHEN is_selected = 1 THEN 1 ELSE 0 END) AS selected_count, + COALESCE(SUM(sample_count), 0) AS sample_count, + COALESCE(SUM(daily_mileage_km), 0) AS daily_mileage_km +FROM (` + baseSQL + ` +) source_selection +GROUP BY stat_date, protocol, selection_status, selection_reason +ORDER BY stat_date DESC, + protocol ASC, + CASE selection_status + WHEN 'selected' THEN 0 + WHEN 'excluded' THEN 1 + WHEN 'not_selected' THEN 2 + ELSE 3 + END, + source_count DESC, + selection_reason ASC +LIMIT ? OFFSET ?` + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildMetricDiagnosisSQL(query MetricDiagnosisQuery) (string, []any) { + fromSQL, args := buildMetricDiagnosisFromSQL(query) + sqlText := `SELECT rt.vin, ? AS stat_date, rt.protocol, + COALESCE(NULLIF(s.plate, ''), NULLIF(l.plate, ''), '') AS plate, + COALESCE(NULLIF(s.platform_name, ''), '') AS platform_name, + COALESCE(NULLIF(s.peer, ''), '') AS peer, + s.event_time AS snapshot_event_time, + l.event_time AS location_event_time, + s.updated_at AS snapshot_updated_at, + l.updated_at AS location_updated_at, + l.total_mileage_km AS realtime_total_mileage_km, + l.total_mileage_event_time AS realtime_total_mileage_event_time, + m.daily_mileage_km, + m.latest_total_mileage_km AS daily_latest_total_mileage_km, + COALESCE(ms.sample_count, 0) AS source_sample_count, + COALESCE(ms.ok_source_count, 0) AS ok_source_count, + COALESCE(ms.selectable_source_count, 0) AS selectable_source_count, + ms.latest_event_time AS latest_stat_event_time, + COALESCE(ms.quality_statuses, '') AS quality_statuses, + s.parsed_json AS snapshot_parsed_json +` + fromSQL + ` +ORDER BY CASE WHEN m.vin IS NULL THEN 0 ELSE 1 END, rt.protocol ASC, rt.vin ASC +LIMIT ? OFFSET ?` + args = append([]any{query.Date}, args...) + args = append(args, query.Limit, query.Offset) + return sqlText, args +} + +func buildMetricDiagnosisCountSQL(query MetricDiagnosisQuery) (string, []any) { + fromSQL, args := buildMetricDiagnosisFromSQL(query) + return "SELECT COUNT(*) " + fromSQL, args +} + +func buildMetricDiagnosisSummarySQL(query MetricDiagnosisQuery) (string, []any) { + query.Diagnosis = "" + fromSQL, fromArgs := buildMetricDiagnosisFromSQL(query) + sqlText := `SELECT ? AS stat_date, + rt.protocol, + COUNT(*) AS active_count, + SUM(CASE WHEN m.vin IS NOT NULL THEN 1 ELSE 0 END) AS ok_count, + SUM(CASE WHEN m.vin IS NULL AND COALESCE(ms.sample_count, 0) > 0 THEN 1 ELSE 0 END) AS missing_daily_count, + SUM(CASE WHEN m.vin IS NULL AND COALESCE(ms.sample_count, 0) = 0 + AND l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY) + THEN 1 ELSE 0 END) AS no_source_sample_count, + SUM(CASE WHEN m.vin IS NULL AND COALESCE(ms.sample_count, 0) = 0 + AND (l.total_mileage_km IS NULL OR l.total_mileage_km <= 0 OR l.total_mileage_event_time IS NULL OR l.total_mileage_event_time < ? OR l.total_mileage_event_time >= DATE_ADD(?, INTERVAL 1 DAY)) + THEN 1 ELSE 0 END) AS no_total_mileage_count +` + fromSQL + ` +GROUP BY rt.protocol +ORDER BY rt.protocol ASC` + args := []any{query.Date, query.Date, query.Date, query.Date, query.Date} + args = append(args, fromArgs...) + return sqlText, args +} + +func buildMetricDiagnosisReasonSummarySQL(query MetricDiagnosisQuery) (string, []any) { + baseQuery := query + baseQuery.Diagnosis = "" + baseQuery.Reason = "" + baseQuery.Severity = "" + fromSQL, fromArgs := buildMetricDiagnosisFromSQL(baseQuery) + diagnosisSQL, diagnosisArgs := metricDiagnosisCaseSQL(query.Date) + reasonSQL, reasonArgs := metricDiagnosisReasonCaseSQL(query.Date) + severitySQL, severityArgs := metricDiagnosisSeverityCaseSQL(query.Date) + sqlText := `SELECT stat_date, protocol, diagnosis, reason, COUNT(*) AS count +FROM ( + SELECT ? AS stat_date, + rt.protocol, + ` + diagnosisSQL + ` AS diagnosis, + ` + reasonSQL + ` AS reason, + ` + severitySQL + ` AS severity +` + fromSQL + ` +) diagnostic_rows` + args := []any{query.Date} + args = append(args, diagnosisArgs...) + args = append(args, reasonArgs...) + args = append(args, severityArgs...) + args = append(args, fromArgs...) + var outerWhere []string + if query.Diagnosis != "" { + outerWhere = append(outerWhere, "diagnosis = ?") + args = append(args, query.Diagnosis) + } + if query.Reason != "" { + outerWhere = append(outerWhere, "reason = ?") + args = append(args, query.Reason) + } + if query.Severity != "" { + outerWhere = append(outerWhere, "severity = ?") + args = append(args, query.Severity) + } + if len(outerWhere) > 0 { + sqlText += " WHERE " + strings.Join(outerWhere, " AND ") + } + sqlText += ` +GROUP BY stat_date, protocol, diagnosis, reason +ORDER BY protocol ASC, + CASE diagnosis + WHEN 'OK' THEN 0 + WHEN 'MISSING_DAILY' THEN 1 + WHEN 'NO_SOURCE_SAMPLE' THEN 2 + WHEN 'NO_TOTAL_MILEAGE' THEN 3 + ELSE 4 + END, + reason ASC` + return sqlText, args +} + +func buildMetricDiagnosisFieldStatusSummarySQL(query MetricDiagnosisQuery) (string, []any) { + baseQuery := query + baseQuery.Diagnosis = "" + baseQuery.Reason = "" + baseQuery.Severity = "" + fromSQL, fromArgs := buildMetricDiagnosisFromSQL(baseQuery) + diagnosisSQL, diagnosisArgs := metricDiagnosisCaseSQL(query.Date) + reasonSQL, reasonArgs := metricDiagnosisReasonCaseSQL(query.Date) + severitySQL, severityArgs := metricDiagnosisSeverityCaseSQL(query.Date) + fieldStatusSQL, fieldStatusArgs := metricDiagnosisMileageFieldStatusCaseSQL(query.Date) + sqlText := `SELECT stat_date, protocol, diagnosis, reason, realtime_mileage_field_status, COUNT(*) AS count +FROM ( + SELECT ? AS stat_date, + rt.protocol, + ` + diagnosisSQL + ` AS diagnosis, + ` + reasonSQL + ` AS reason, + ` + severitySQL + ` AS severity, + ` + fieldStatusSQL + ` AS realtime_mileage_field_status +` + fromSQL + ` +) diagnostic_rows` + args := []any{query.Date} + args = append(args, diagnosisArgs...) + args = append(args, reasonArgs...) + args = append(args, severityArgs...) + args = append(args, fieldStatusArgs...) + args = append(args, fromArgs...) + var outerWhere []string + if query.Diagnosis != "" { + outerWhere = append(outerWhere, "diagnosis = ?") + args = append(args, query.Diagnosis) + } + if query.Reason != "" { + outerWhere = append(outerWhere, "reason = ?") + args = append(args, query.Reason) + } + if query.Severity != "" { + outerWhere = append(outerWhere, "severity = ?") + args = append(args, query.Severity) + } + if len(outerWhere) > 0 { + sqlText += " WHERE " + strings.Join(outerWhere, " AND ") + } + sqlText += ` +GROUP BY stat_date, protocol, diagnosis, reason, realtime_mileage_field_status +ORDER BY protocol ASC, + CASE diagnosis + WHEN 'OK' THEN 0 + WHEN 'MISSING_DAILY' THEN 1 + WHEN 'NO_SOURCE_SAMPLE' THEN 2 + WHEN 'NO_TOTAL_MILEAGE' THEN 3 + ELSE 4 + END, + reason ASC, + count DESC, + realtime_mileage_field_status ASC` + return sqlText, args +} + +func metricDiagnosisCaseSQL(statDate string) (string, []any) { + return `CASE + WHEN m.vin IS NOT NULL THEN 'OK' + WHEN COALESCE(ms.sample_count, 0) > 0 THEN 'MISSING_DAILY' + WHEN l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY) + THEN 'NO_SOURCE_SAMPLE' + ELSE 'NO_TOTAL_MILEAGE' + END`, []any{statDate, statDate} +} + +func metricDiagnosisReasonCaseSQL(statDate string) (string, []any) { + return `CASE + WHEN m.vin IS NOT NULL THEN 'daily_metric_exists' + WHEN COALESCE(ms.sample_count, 0) > 0 AND COALESCE(ms.ok_source_count, 0) = 0 THEN 'source_samples_all_invalid' + WHEN COALESCE(ms.sample_count, 0) > 0 AND COALESCE(ms.selectable_source_count, 0) = 0 THEN 'source_samples_all_excluded' + WHEN COALESCE(ms.sample_count, 0) > 0 THEN 'source_samples_exist_but_final_metric_missing' + WHEN l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY) + THEN 'realtime_location_has_total_mileage_but_no_stat_sample' + WHEN l.total_mileage_km IS NULL THEN 'realtime_total_mileage_missing' + WHEN l.total_mileage_km <= 0 THEN 'realtime_total_mileage_non_positive' + WHEN l.total_mileage_event_time IS NULL THEN 'realtime_total_mileage_time_missing' + WHEN l.total_mileage_event_time < ? OR l.total_mileage_event_time >= DATE_ADD(?, INTERVAL 1 DAY) + THEN 'realtime_total_mileage_not_reported_on_stat_date' + ELSE 'realtime_active_without_total_mileage' + END`, []any{statDate, statDate, statDate, statDate} +} + +func metricDiagnosisSeverityCaseSQL(statDate string) (string, []any) { + return `CASE + WHEN m.vin IS NOT NULL THEN 'ok' + WHEN COALESCE(ms.sample_count, 0) > 0 AND COALESCE(ms.ok_source_count, 0) = 0 THEN 'source_data' + WHEN COALESCE(ms.sample_count, 0) > 0 AND COALESCE(ms.selectable_source_count, 0) = 0 THEN 'source_data' + WHEN COALESCE(ms.sample_count, 0) > 0 THEN 'pipeline' + WHEN l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY) + THEN 'pipeline' + WHEN l.total_mileage_km IS NULL THEN 'source_data' + WHEN l.total_mileage_km <= 0 THEN 'source_data' + WHEN l.total_mileage_event_time IS NULL THEN 'pipeline' + ELSE 'source_data' + END`, []any{statDate, statDate} +} + +func metricDiagnosisMileageFieldStatusCaseSQL(statDate string) (string, []any) { + return `CASE + WHEN m.vin IS NOT NULL THEN 'daily_metric_exists' + WHEN COALESCE(ms.sample_count, 0) > 0 THEN 'source_sample_exists' + WHEN l.total_mileage_km > 0 + AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY) + THEN 'standard_field_not_consumed' + WHEN l.total_mileage_km <= 0 THEN 'standard_field_non_positive' + WHEN l.total_mileage_km > 0 AND l.total_mileage_event_time IS NULL THEN 'standard_field_time_missing' + WHEN l.total_mileage_km > 0 + AND (l.total_mileage_event_time < ? OR l.total_mileage_event_time >= DATE_ADD(?, INTERVAL 1 DAY)) + THEN 'standard_field_stale' + WHEN s.parsed_json IS NULL OR TRIM(s.parsed_json) = '' OR TRIM(s.parsed_json) = '{}' THEN 'no_realtime_fields' + WHEN rt.protocol = 'YUTONG_MQTT' + AND ( + LOWER(s.parsed_json) LIKE '%yutong_mqtt.data.total_mileage%' + OR LOWER(s.parsed_json) LIKE '%yutong_mqtt.root.data.total_mileage%' + ) + THEN 'mapped_protocol_field_without_fresh_evidence' + WHEN LOWER(s.parsed_json) LIKE '%mileage%' + OR LOWER(s.parsed_json) LIKE '%odometer%' + OR LOWER(s.parsed_json) LIKE '%.odo%' + OR LOWER(s.parsed_json) LIKE '%_odo%' + THEN 'candidate_field_unmapped' + ELSE 'no_candidate_mileage_field' + END`, []any{statDate, statDate, statDate, statDate} +} + func buildMetricCountSQL(query MetricQuery) (string, []any) { where, args := buildMetricWhere(query) - sqlText := `SELECT COUNT(*) FROM vehicle_daily_mileage` + sqlText := `SELECT COUNT(*) FROM vehicle_daily_mileage m` if len(where) > 0 { sqlText += " WHERE " + strings.Join(where, " AND ") } return sqlText, args } +func buildMetricSourceCountSQL(query MetricSourceQuery) (string, []any) { + where, args := buildMetricSourceWhere(query) + sqlText := `SELECT COUNT(*) FROM vehicle_daily_mileage_source s` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func buildMetricSourceQualitySummaryCountSQL(query MetricSourceQuery) (string, []any) { + where, args := buildMetricSourceWhere(query) + sqlText := `SELECT COUNT(*) FROM ( + SELECT 1 + FROM vehicle_daily_mileage_source s` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + sqlText += ` + GROUP BY s.stat_date, s.protocol, s.quality_status, s.quality_reason +) q` + return sqlText, args +} + +func buildMetricSourceSelectionSummaryCountSQL(query MetricSourceQuery) (string, []any) { + baseSQL, args := buildMetricSourceSelectionSummaryBaseSQL(query) + sqlText := `SELECT COUNT(*) FROM ( + SELECT 1 + FROM (` + baseSQL + ` + ) source_selection + GROUP BY stat_date, protocol, selection_status, selection_reason +) q` + return sqlText, args +} + +func buildMetricSourceSelectionSummaryBaseSQL(query MetricSourceQuery) (string, []any) { + where, args := buildMetricSourceWhere(query) + sqlText := `SELECT s.stat_date, s.protocol, s.vin, s.is_selected, s.sample_count, s.daily_mileage_km, + ` + metricSourceSelectionStatusSQL() + ` AS selection_status, + ` + metricSourceSelectionReasonSQL() + ` AS selection_reason +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip` + if len(where) > 0 { + sqlText += " WHERE " + strings.Join(where, " AND ") + } + return sqlText, args +} + +func metricSourceKindSQL(sourceAlias string) string { + alias := strings.TrimSpace(sourceAlias) + if alias == "" { + alias = "s" + } + return `COALESCE(NULLIF(TRIM(ds.source_kind), ''), CASE WHEN ` + alias + `.source_key LIKE '%` + platformSourceKeyPrefix + `%' THEN 'PLATFORM' WHEN ` + alias + `.source_key LIKE '%` + directSourceKeySuffix + `' THEN 'DIRECT' ELSE 'UNKNOWN' END)` +} + +func metricSourceSelectionStatusSQL() string { + return `CASE + WHEN s.is_selected = 1 THEN 'selected' + WHEN UPPER(COALESCE(NULLIF(TRIM(s.quality_status), ''), '')) <> '' + AND UPPER(COALESCE(NULLIF(TRIM(s.quality_status), ''), '')) <> '` + QualityOK + `' THEN 'excluded' + WHEN ds.enabled = 0 THEN 'excluded' + ELSE 'not_selected' + END` +} + +func metricSourceSelectionReasonSQL() string { + return `CASE + WHEN s.is_selected = 1 THEN 'selected_current_projection' + WHEN UPPER(COALESCE(NULLIF(TRIM(s.quality_status), ''), '')) <> '' + AND UPPER(COALESCE(NULLIF(TRIM(s.quality_status), ''), '')) <> '` + QualityOK + `' + THEN CONCAT('quality_', LOWER(COALESCE(NULLIF(TRIM(s.quality_reason), ''), s.quality_status))) + WHEN ds.enabled = 0 THEN 'source_disabled' + WHEN ds.id IS NULL AND ` + metricSourceKindSQL("s") + ` <> 'DIRECT' THEN 'unmanaged_source_lower_priority' + WHEN ` + metricSourceKindSQL("s") + ` = 'UNKNOWN' THEN 'unknown_source_kind_lower_priority' + ELSE 'lower_trust_priority_or_sample_count' + END` +} + +func buildMetricDiagnosisFromSQL(query MetricDiagnosisQuery) (string, []any) { + realtimeSQL, args := buildActiveRealtimeMetricSQL(query) + sourceSummarySQL := `LEFT JOIN ( + SELECT s.vin, s.protocol, s.stat_date, + SUM(s.sample_count) AS sample_count, + SUM(CASE WHEN s.quality_status = 'OK' THEN 1 ELSE 0 END) AS ok_source_count, + SUM(CASE + WHEN s.quality_status = 'OK' + AND (ds.id IS NULL OR ds.enabled = 1 OR ( + COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN' + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + )) + THEN 1 ELSE 0 + END) AS selectable_source_count, + MAX(s.latest_event_time) AS latest_event_time, + GROUP_CONCAT(DISTINCT s.quality_status ORDER BY s.quality_status SEPARATOR ',') AS quality_statuses + FROM vehicle_daily_mileage_source s + LEFT JOIN vehicle_data_source ds ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip + WHERE s.stat_date = ? + GROUP BY s.vin, s.protocol, s.stat_date +) ms ON ms.vin = rt.vin AND ms.protocol = rt.protocol` + args = append(args, query.Date) + fromSQL := `FROM (` + realtimeSQL + `) rt +LEFT JOIN vehicle_realtime_snapshot s ON s.protocol = rt.protocol AND s.vin = rt.vin +LEFT JOIN vehicle_realtime_location l ON l.protocol = rt.protocol AND l.vin = rt.vin +` + sourceSummarySQL + ` +LEFT JOIN vehicle_daily_mileage m ON m.vin = rt.vin AND m.protocol = rt.protocol AND m.stat_date = ?` + args = append(args, query.Date) + where, whereArgs := buildMetricDiagnosisWhere(query) + if len(where) > 0 { + fromSQL += "\nWHERE " + strings.Join(where, " AND ") + args = append(args, whereArgs...) + } + return fromSQL, args +} + +func buildActiveRealtimeMetricSQL(query MetricDiagnosisQuery) (string, []any) { + snapshotWhere, snapshotArgs := buildActiveRealtimeMetricWhere(query) + locationWhere, locationArgs := buildActiveRealtimeMetricWhere(query) + sqlText := `SELECT protocol, vin FROM vehicle_realtime_snapshot + WHERE ` + snapshotWhere + ` +UNION +SELECT protocol, vin FROM vehicle_realtime_location + WHERE ` + locationWhere + args := append(snapshotArgs, locationArgs...) + return sqlText, args +} + +func buildActiveRealtimeMetricWhere(query MetricDiagnosisQuery) (string, []any) { + where := []string{"vin <> ''", realtimeActiveDatePredicate()} + args := []any{ + query.Date, query.Date, + query.Date, query.Date, + } + if query.Protocol != "" { + where = append(where, "protocol = ?") + args = append(args, query.Protocol) + } + if query.VIN != "" { + where = append(where, "vin = ?") + args = append(args, query.VIN) + } + return strings.Join(where, " AND "), args +} + +func realtimeActiveDatePredicate() string { + return `((event_time >= ? AND event_time < DATE_ADD(?, INTERVAL 1 DAY)) + OR (received_at >= ? AND received_at < DATE_ADD(?, INTERVAL 1 DAY)))` +} + func buildMetricWhere(query MetricQuery) ([]string, []any) { var where []string var args []any @@ -146,16 +1624,94 @@ func buildMetricWhere(query MetricQuery) ([]string, []any) { args = append(args, value) } if query.VIN != "" { - add("vin = ?", query.VIN) + add("m.vin = ?", query.VIN) } if query.Protocol != "" { - add("protocol = ?", query.Protocol) + add("m.protocol = ?", query.Protocol) } if query.DateFrom != "" { - add("stat_date >= ?", query.DateFrom) + add("m.stat_date >= ?", query.DateFrom) } if query.DateTo != "" { - add("stat_date <= ?", query.DateTo) + add("m.stat_date <= ?", query.DateTo) + } + return where, args +} + +func buildMetricSourceWhere(query MetricSourceQuery) ([]string, []any) { + var where []string + var args []any + add := func(clause string, value any) { + where = append(where, clause) + args = append(args, value) + } + if query.VIN != "" { + add("s.vin = ?", query.VIN) + } + if query.Protocol != "" { + add("s.protocol = ?", query.Protocol) + } + if query.DateFrom != "" { + add("s.stat_date >= ?", query.DateFrom) + } + if query.DateTo != "" { + add("s.stat_date <= ?", query.DateTo) + } + if query.SourceIP != "" { + add("s.source_ip = ?", query.SourceIP) + } + if query.QualityStatus != "" { + add("s.quality_status = ?", query.QualityStatus) + } + if query.QualityReason != "" { + add("s.quality_reason = ?", query.QualityReason) + } + if query.Selected != nil { + if *query.Selected { + add("s.is_selected = ?", 1) + } else { + add("s.is_selected = ?", 0) + } + } + return where, args +} + +func buildMetricDiagnosisWhere(query MetricDiagnosisQuery) ([]string, []any) { + var where []string + var args []any + add := func(clause string, value any) { + where = append(where, clause) + args = append(args, value) + } + if query.VIN != "" { + add("rt.vin = ?", query.VIN) + } + if query.Protocol != "" { + add("rt.protocol = ?", query.Protocol) + } + switch query.Diagnosis { + case "OK": + where = append(where, "m.vin IS NOT NULL") + case "MISSING_DAILY": + where = append(where, "m.vin IS NULL AND COALESCE(ms.sample_count, 0) > 0") + case "NO_SOURCE_SAMPLE": + where = append(where, "m.vin IS NULL AND COALESCE(ms.sample_count, 0) = 0 AND l.total_mileage_km > 0 AND l.total_mileage_event_time >= ? AND l.total_mileage_event_time < DATE_ADD(?, INTERVAL 1 DAY)") + args = append(args, query.Date, query.Date) + case "NO_TOTAL_MILEAGE": + where = append(where, "m.vin IS NULL AND COALESCE(ms.sample_count, 0) = 0 AND (l.total_mileage_km IS NULL OR l.total_mileage_km <= 0 OR l.total_mileage_event_time IS NULL OR l.total_mileage_event_time < ? OR l.total_mileage_event_time >= DATE_ADD(?, INTERVAL 1 DAY))") + args = append(args, query.Date, query.Date) + } + if query.Reason != "" { + reasonSQL, reasonArgs := metricDiagnosisReasonCaseSQL(query.Date) + where = append(where, "("+reasonSQL+") = ?") + args = append(args, reasonArgs...) + args = append(args, query.Reason) + } + if query.Severity != "" { + severitySQL, severityArgs := metricDiagnosisSeverityCaseSQL(query.Date) + where = append(where, "("+severitySQL+") = ?") + args = append(args, severityArgs...) + args = append(args, query.Severity) } return where, args } @@ -176,10 +1732,29 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeMetricError(w, http.StatusMethodNotAllowed, "method not allowed") return } - if strings.Trim(r.URL.Path, "/") != "api/stats/daily-metrics" { + switch strings.Trim(r.URL.Path, "/") { + case "api/stats/daily-metrics": + h.handleDailyMetrics(w, r) + case "api/stats/daily-metrics/sources": + h.handleDailyMetricSources(w, r) + case "api/stats/daily-metrics/sources/quality": + h.handleDailyMetricSourceQuality(w, r) + case "api/stats/daily-metrics/sources/selection": + h.handleDailyMetricSourceSelection(w, r) + case "api/stats/daily-metrics/diagnostics": + h.handleDailyMetricDiagnostics(w, r) + case "api/stats/daily-metrics/diagnostics/summary": + h.handleDailyMetricDiagnosisSummary(w, r) + case "api/stats/daily-metrics/diagnostics/reasons": + h.handleDailyMetricDiagnosisReasonSummary(w, r) + case "api/stats/daily-metrics/diagnostics/field-status": + h.handleDailyMetricDiagnosisFieldStatusSummary(w, r) + default: writeMetricError(w, http.StatusNotFound, "route not found") - return } +} + +func (h *MetricHandler) handleDailyMetrics(w http.ResponseWriter, r *http.Request) { query, err := parseMetricQuery(r) if err != nil { writeMetricError(w, http.StatusBadRequest, err.Error()) @@ -210,6 +1785,202 @@ func (h *MetricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }) } +func (h *MetricHandler) handleDailyMetricSources(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricSourceQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountSources(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QuerySources(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *MetricHandler) handleDailyMetricSourceQuality(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricSourceQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountSourceQualitySummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QuerySourceQualitySummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *MetricHandler) handleDailyMetricSourceSelection(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricSourceQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountSourceSelectionSummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QuerySourceSelectionSummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *MetricHandler) handleDailyMetricDiagnostics(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricDiagnosisQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + var total int64 + if query.IncludeTotal { + total, err = h.repository.CountDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + } + rows, err := h.repository.QueryDiagnostics(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + if !query.IncludeTotal { + total = int64(len(rows)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": total, + "limit": query.Limit, + "offset": query.Offset, + }) +} + +func (h *MetricHandler) handleDailyMetricDiagnosisSummary(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricDiagnosisSummaryQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + rows, err := h.repository.QueryDiagnosisSummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + var totalActive int64 + var totalActionable int64 + for _, row := range rows { + totalActive += row.ActiveCount + totalActionable += row.ActionableIssueCount + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": len(rows), + "active_total": totalActive, + "actionable_issue_total": totalActionable, + }) +} + +func (h *MetricHandler) handleDailyMetricDiagnosisReasonSummary(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricDiagnosisReasonSummaryQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + rows, err := h.repository.QueryDiagnosisReasonSummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + totalVehicles, totalActionable, pipelineIssues, sourceDataIssues := metricDiagnosisReasonSummaryIssueTotals(rows) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": len(rows), + "vehicle_total": totalVehicles, + "actionable_issue_total": totalActionable, + "pipeline_issue_total": pipelineIssues, + "source_data_issue_total": sourceDataIssues, + }) +} + +func (h *MetricHandler) handleDailyMetricDiagnosisFieldStatusSummary(w http.ResponseWriter, r *http.Request) { + query, err := parseMetricDiagnosisFieldStatusSummaryQuery(r) + if err != nil { + writeMetricError(w, http.StatusBadRequest, err.Error()) + return + } + rows, err := h.repository.QueryDiagnosisFieldStatusSummary(r.Context(), query) + if err != nil { + writeMetricError(w, http.StatusInternalServerError, err.Error()) + return + } + totalVehicles, totalActionable, pipelineIssues, sourceDataIssues := metricDiagnosisFieldStatusSummaryIssueTotals(rows) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": rows, + "total": len(rows), + "vehicle_total": totalVehicles, + "actionable_issue_total": totalActionable, + "pipeline_issue_total": pipelineIssues, + "source_data_issue_total": sourceDataIssues, + }) +} + func parseMetricQuery(r *http.Request) (MetricQuery, error) { values := r.URL.Query() limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") @@ -235,6 +2006,140 @@ func parseMetricQuery(r *http.Request) (MetricQuery, error) { return normalizeMetricQuery(query), nil } +func parseMetricSourceQuery(r *http.Request) (MetricSourceQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return MetricSourceQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return MetricSourceQuery{}, err + } + selected, err := parseOptionalBool(values.Get("selected")) + if err != nil { + return MetricSourceQuery{}, errors.New("selected must be true or false") + } + query := MetricSourceQuery{ + VIN: values.Get("vin"), + Protocol: values.Get("protocol"), + DateFrom: values.Get("dateFrom"), + DateTo: values.Get("dateTo"), + SourceIP: values.Get("sourceIP"), + QualityStatus: values.Get("qualityStatus"), + QualityReason: values.Get("qualityReason"), + Selected: selected, + OrderBy: values.Get("orderBy"), + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + } + if !validDate(query.DateFrom) || !validDate(query.DateTo) { + return MetricSourceQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD") + } + return normalizeMetricSourceQuery(query), nil +} + +func parseMetricDiagnosisQuery(r *http.Request) (MetricDiagnosisQuery, error) { + values := r.URL.Query() + limit, err := parseBoundedInt(values.Get("limit"), 50, 1, 1000, "limit") + if err != nil { + return MetricDiagnosisQuery{}, err + } + offset, err := parseBoundedInt(values.Get("offset"), 0, 0, 1_000_000, "offset") + if err != nil { + return MetricDiagnosisQuery{}, err + } + query := MetricDiagnosisQuery{ + VIN: values.Get("vin"), + Protocol: values.Get("protocol"), + Date: values.Get("date"), + Diagnosis: values.Get("diagnosis"), + Reason: values.Get("reason"), + Severity: values.Get("severity"), + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + Limit: limit, + Offset: offset, + } + query = normalizeMetricDiagnosisQuery(query) + if !validDate(query.Date) { + return MetricDiagnosisQuery{}, errors.New("date must use YYYY-MM-DD") + } + if !validMetricDiagnosis(query.Diagnosis) { + return MetricDiagnosisQuery{}, errors.New("diagnosis must be OK, MISSING_DAILY, NO_SOURCE_SAMPLE, or NO_TOTAL_MILEAGE") + } + if !validMetricDiagnosisReason(query.Reason) { + return MetricDiagnosisQuery{}, errors.New("reason is not a supported daily mileage diagnosis reason") + } + if !validMetricDiagnosisSeverity(query.Severity) { + return MetricDiagnosisQuery{}, errors.New("severity must be ok, pipeline, or source_data") + } + return query, nil +} + +func parseMetricDiagnosisSummaryQuery(r *http.Request) (MetricDiagnosisQuery, error) { + values := r.URL.Query() + query := normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + VIN: values.Get("vin"), + Protocol: values.Get("protocol"), + Date: values.Get("date"), + }) + if !validDate(query.Date) { + return MetricDiagnosisQuery{}, errors.New("date must use YYYY-MM-DD") + } + return query, nil +} + +func parseMetricDiagnosisReasonSummaryQuery(r *http.Request) (MetricDiagnosisQuery, error) { + values := r.URL.Query() + query := normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + VIN: values.Get("vin"), + Protocol: values.Get("protocol"), + Date: values.Get("date"), + Diagnosis: values.Get("diagnosis"), + Reason: values.Get("reason"), + Severity: values.Get("severity"), + }) + if !validDate(query.Date) { + return MetricDiagnosisQuery{}, errors.New("date must use YYYY-MM-DD") + } + if !validMetricDiagnosis(query.Diagnosis) { + return MetricDiagnosisQuery{}, errors.New("diagnosis must be OK, MISSING_DAILY, NO_SOURCE_SAMPLE, or NO_TOTAL_MILEAGE") + } + if !validMetricDiagnosisReason(query.Reason) { + return MetricDiagnosisQuery{}, errors.New("reason is not a supported daily mileage diagnosis reason") + } + if !validMetricDiagnosisSeverity(query.Severity) { + return MetricDiagnosisQuery{}, errors.New("severity must be ok, pipeline, or source_data") + } + return query, nil +} + +func parseMetricDiagnosisFieldStatusSummaryQuery(r *http.Request) (MetricDiagnosisQuery, error) { + values := r.URL.Query() + query := normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + VIN: values.Get("vin"), + Protocol: values.Get("protocol"), + Date: values.Get("date"), + Diagnosis: values.Get("diagnosis"), + Reason: values.Get("reason"), + Severity: values.Get("severity"), + }) + if !validDate(query.Date) { + return MetricDiagnosisQuery{}, errors.New("date must use YYYY-MM-DD") + } + if !validMetricDiagnosis(query.Diagnosis) { + return MetricDiagnosisQuery{}, errors.New("diagnosis must be OK, MISSING_DAILY, NO_SOURCE_SAMPLE, or NO_TOTAL_MILEAGE") + } + if !validMetricDiagnosisReason(query.Reason) { + return MetricDiagnosisQuery{}, errors.New("reason is not a supported daily mileage diagnosis reason") + } + if !validMetricDiagnosisSeverity(query.Severity) { + return MetricDiagnosisQuery{}, errors.New("severity must be ok, pipeline, or source_data") + } + return query, nil +} + func parseBoundedInt(raw string, fallback int, min int, max int, name string) (int, error) { raw = strings.TrimSpace(raw) if raw == "" { @@ -270,6 +2175,94 @@ func validDate(value string) bool { return true } +func validMetricDiagnosis(value string) bool { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "", "OK", "MISSING_DAILY", "NO_SOURCE_SAMPLE", "NO_TOTAL_MILEAGE": + return true + default: + return false + } +} + +func validMetricDiagnosisReason(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", + "daily_metric_exists", + "source_samples_all_invalid", + "source_samples_all_excluded", + "source_samples_exist_but_final_metric_missing", + "realtime_location_has_total_mileage_but_no_stat_sample", + "realtime_total_mileage_missing", + "realtime_total_mileage_non_positive", + "realtime_total_mileage_time_missing", + "realtime_total_mileage_not_reported_on_stat_date", + "realtime_active_without_total_mileage": + return true + default: + return false + } +} + +func validMetricDiagnosisSeverity(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", metricDiagnosisSeverityOK, metricDiagnosisSeverityPipeline, metricDiagnosisSeveritySourceData: + return true + default: + return false + } +} + +func classifyMetricDiagnosis(hasDailyMetric bool, sourceSampleCount int64, okSourceCount int64, selectableSourceCount int64, realtimeTotal sql.NullFloat64, totalMileageAt string, statDate string) (string, string) { + if hasDailyMetric { + return "OK", "daily_metric_exists" + } + if sourceSampleCount > 0 { + if okSourceCount == 0 { + return "MISSING_DAILY", "source_samples_all_invalid" + } + if selectableSourceCount == 0 { + return "MISSING_DAILY", "source_samples_all_excluded" + } + return "MISSING_DAILY", "source_samples_exist_but_final_metric_missing" + } + if hasUsableRealtimeTotalMileage(realtimeTotal, totalMileageAt, statDate) { + return "NO_SOURCE_SAMPLE", "realtime_location_has_total_mileage_but_no_stat_sample" + } + if !realtimeTotal.Valid { + return "NO_TOTAL_MILEAGE", "realtime_total_mileage_missing" + } + if realtimeTotal.Float64 <= 0 { + return "NO_TOTAL_MILEAGE", "realtime_total_mileage_non_positive" + } + if strings.TrimSpace(totalMileageAt) == "" { + return "NO_TOTAL_MILEAGE", "realtime_total_mileage_time_missing" + } + if !strings.HasPrefix(totalMileageAt, statDate) { + return "NO_TOTAL_MILEAGE", "realtime_total_mileage_not_reported_on_stat_date" + } + return "NO_TOTAL_MILEAGE", "realtime_active_without_total_mileage" +} + +func hasUsableRealtimeTotalMileage(total sql.NullFloat64, totalMileageAt string, statDate string) bool { + return total.Valid && total.Float64 > 0 && strings.HasPrefix(totalMileageAt, statDate) +} + +func splitMetricStatuses(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + func writeMetricError(w http.ResponseWriter, status int, message string) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) diff --git a/go/vehicle-gateway/internal/stats/query_test.go b/go/vehicle-gateway/internal/stats/query_test.go index 236b7724..8e305c6e 100644 --- a/go/vehicle-gateway/internal/stats/query_test.go +++ b/go/vehicle-gateway/internal/stats/query_test.go @@ -12,7 +12,14 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -const dailyMetricSelectPattern = "SELECT vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km, updated_at FROM vehicle_daily_mileage" +const dailyMetricSelectPattern = "SELECT m.vin, m.stat_date, m.protocol, m.source_id" +const dailyMetricSourceSelectPattern = "SELECT s.vin, s.stat_date, s.protocol, s.source_key, s.source_ip" +const dailyMetricSourceQualitySummarySelectPattern = "SELECT s.stat_date, s.protocol, s.quality_status, s.quality_reason" +const dailyMetricSourceSelectionSummarySelectPattern = "SELECT stat_date, protocol, selection_status, selection_reason" +const dailyMetricDiagnosisSelectPattern = "SELECT rt.vin, \\? AS stat_date, rt.protocol" +const dailyMetricDiagnosisSummarySelectPattern = "SELECT \\? AS stat_date,\\s+rt.protocol" +const dailyMetricDiagnosisReasonSummarySelectPattern = "SELECT stat_date, protocol, diagnosis, reason, COUNT\\(\\*\\) AS count" +const dailyMetricDiagnosisFieldStatusSummarySelectPattern = "SELECT stat_date, protocol, diagnosis, reason, realtime_mileage_field_status, COUNT\\(\\*\\) AS count" func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) { db, mock, err := sqlmock.New() @@ -23,10 +30,12 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) { mock.ExpectQuery(dailyMetricSelectPattern). WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "stat_date", "protocol", "source_id", "daily_mileage_km", + "vin", "stat_date", "protocol", "source_id", + "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km", "latest_total_mileage_km", "updated_at", }).AddRow( - "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 3, 12.3, + "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 3, + "115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3, 12357.9, time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)), )) @@ -50,6 +59,9 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) { if rows[0].SourceID == nil || *rows[0].SourceID != 3 { t.Fatalf("source id = %#v", rows[0].SourceID) } + if rows[0].SourceIP != "115.231.168.135" || rows[0].PlatformName != "G7 平台" || rows[0].SourceCode != "G7S" || rows[0].SourceKind != "PLATFORM" { + t.Fatalf("unexpected source evidence: %#v", rows[0]) + } if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" { t.Fatalf("unexpected time formatting: %#v", rows[0]) } @@ -66,10 +78,12 @@ func TestMetricQueryReturnsSelectedSourceID(t *testing.T) { defer db.Close() mock.ExpectQuery(dailyMetricSelectPattern). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "stat_date", "protocol", "source_id", "daily_mileage_km", + "vin", "stat_date", "protocol", "source_id", + "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km", "latest_total_mileage_km", "updated_at", }).AddRow( - "LA9GG64L7PBAF4001", "2026-07-08", "JT808", 2, 23.1, + "LA9GG64L7PBAF4001", "2026-07-08", "JT808", 2, + "117.132.194.31", "117.132.194.31:20471", "广安北斗", "guangan_beidou", "PLATFORM", 23.1, 4123.9, "2026-07-08 13:30:57", )) @@ -85,6 +99,747 @@ func TestMetricQueryReturnsSelectedSourceID(t *testing.T) { if got[0].SourceID == nil || *got[0].SourceID != 2 { t.Fatalf("source id = %#v", got[0].SourceID) } + if got[0].SourceKind != "PLATFORM" || got[0].PlatformName != "广安北斗" { + t.Fatalf("source evidence = %#v", got[0]) + } +} + +func TestMetricRepositoryQueriesDailyMetricSourcesWithEvidence(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricSourceSelectPattern). + WithArgs("LA9GG64L7PBAF4001", "JT808", "2026-07-08", "2026-07-08", "115.231.168.135", "OK", "historical_source_baseline", 1, 20, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "source_key", "source_ip", "source_endpoint", "phone", "device_id", + "platform_name", "source_id", "source_code", "source_kind", "enabled", "trust_priority", + "first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "sample_count", + "first_event_time", "latest_event_time", "quality_status", "quality_reason", "is_selected", "updated_at", + }).AddRow( + "LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307795425@115.231.168.135", "115.231.168.135", + "115.231.168.135:41561", "13307795425", nil, + "信达", 5, "xinda", "PLATFORM", 1, 10, + 4100.8, 4123.9, 23.1, 128, + "2026-07-08 00:01:00", "2026-07-08 23:59:00", "OK", "historical_source_baseline", 1, "2026-07-08 23:59:10", + )) + + selected := true + repository := NewMetricRepository(db) + rows, err := repository.QuerySources(context.Background(), MetricSourceQuery{ + VIN: "LA9GG64L7PBAF4001", + Protocol: "jt808", + DateFrom: "2026-07-08", + DateTo: "2026-07-08", + SourceIP: "115.231.168.135", + QualityStatus: "ok", + QualityReason: "Historical_Source_Baseline", + Selected: &selected, + Limit: 20, + }) + if err != nil { + t.Fatalf("QuerySources() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.PlatformName != "信达" || row.SourceCode != "xinda" || row.SourceKind != "PLATFORM" || !row.IsSelected { + t.Fatalf("unexpected source row: %#v", row) + } + if row.SourceEnabled == nil || !*row.SourceEnabled { + t.Fatalf("source enabled = %#v", row.SourceEnabled) + } + if row.TrustPriority == nil || *row.TrustPriority != 10 { + t.Fatalf("trust priority = %#v", row.TrustPriority) + } + if row.FirstTotalMileageKM == nil || *row.FirstTotalMileageKM != 4100.8 { + t.Fatalf("first total mileage = %#v", row.FirstTotalMileageKM) + } + if row.LatestEventTime != "2026-07-08 23:59:00" || row.QualityReason != "historical_source_baseline" { + t.Fatalf("unexpected event/quality fields: %#v", row) + } + if row.SelectionStatus != "selected" || row.SelectionReason != "selected_current_projection" { + t.Fatalf("selection evidence = %#v", row) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricSourceSelectionEvidence(t *testing.T) { + disabled := false + sourceID := int64(3) + tests := []struct { + name string + row MetricSourceRow + wantStatus string + wantReason string + }{ + { + name: "selected", + row: MetricSourceRow{ + IsSelected: true, + QualityStatus: QualityOK, + }, + wantStatus: "selected", + wantReason: "selected_current_projection", + }, + { + name: "invalid quality", + row: MetricSourceRow{ + QualityStatus: QualityInvalidDelta, + QualityReason: "outside_daily_range", + }, + wantStatus: "excluded", + wantReason: "quality_outside_daily_range", + }, + { + name: "disabled source", + row: MetricSourceRow{ + QualityStatus: QualityOK, + SourceEnabled: &disabled, + SourceID: &sourceID, + }, + wantStatus: "excluded", + wantReason: "source_disabled", + }, + { + name: "unmanaged source", + row: MetricSourceRow{ + QualityStatus: QualityOK, + }, + wantStatus: "not_selected", + wantReason: "unmanaged_source_lower_priority", + }, + { + name: "direct source without source table row", + row: MetricSourceRow{ + QualityStatus: QualityOK, + SourceKind: "DIRECT", + }, + wantStatus: "not_selected", + wantReason: "lower_trust_priority_or_sample_count", + }, + { + name: "unknown source kind", + row: MetricSourceRow{ + QualityStatus: QualityOK, + SourceID: &sourceID, + SourceKind: "UNKNOWN", + }, + wantStatus: "not_selected", + wantReason: "unknown_source_kind_lower_priority", + }, + { + name: "lower priority", + row: MetricSourceRow{ + QualityStatus: QualityOK, + SourceID: &sourceID, + SourceKind: "PLATFORM", + }, + wantStatus: "not_selected", + wantReason: "lower_trust_priority_or_sample_count", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status, reason, action := metricSourceSelectionEvidence(tt.row) + if status != tt.wantStatus || reason != tt.wantReason || action == "" { + t.Fatalf("metricSourceSelectionEvidence() = %q, %q, %q", status, reason, action) + } + }) + } +} + +func TestMetricRepositoryQueriesDailyMetricSourceQualitySummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricSourceQualitySummarySelectPattern). + WithArgs("JT808", "2026-07-12", "2026-07-12", "INVALID_DELTA", "outside_daily_range", 10, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "quality_status", "quality_reason", + "source_count", "vehicle_count", "selected_count", "sample_count", "daily_mileage_km", + }).AddRow( + "2026-07-12", "JT808", "INVALID_DELTA", "outside_daily_range", + int64(3), int64(3), int64(0), int64(128), 12435.2, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QuerySourceQualitySummary(context.Background(), MetricSourceQuery{ + Protocol: "jt808", + DateFrom: "2026-07-12", + DateTo: "2026-07-12", + QualityStatus: "invalid_delta", + QualityReason: "Outside_Daily_Range", + Limit: 10, + }) + if err != nil { + t.Fatalf("QuerySourceQualitySummary() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.StatDate != "2026-07-12" || row.Protocol != "JT808" || row.QualityStatus != "INVALID_DELTA" || row.QualityReason != "outside_daily_range" { + t.Fatalf("unexpected summary key: %#v", row) + } + if row.SourceCount != 3 || row.VehicleCount != 3 || row.SelectedCount != 0 || row.SampleCount != 128 || row.DailyMileageKM != 12435.2 { + t.Fatalf("unexpected summary values: %#v", row) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricRepositoryQueriesDailyMetricSourceSelectionSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricSourceSelectionSummarySelectPattern). + WithArgs("JT808", "2026-07-12", "2026-07-12", 10, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "selection_status", "selection_reason", + "source_count", "vehicle_count", "selected_count", "sample_count", "daily_mileage_km", + }).AddRow( + "2026-07-12", "JT808", "not_selected", "lower_trust_priority_or_sample_count", + int64(36), int64(31), int64(0), int64(3200), 812.5, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QuerySourceSelectionSummary(context.Background(), MetricSourceQuery{ + Protocol: "jt808", + DateFrom: "2026-07-12", + DateTo: "2026-07-12", + Limit: 10, + }) + if err != nil { + t.Fatalf("QuerySourceSelectionSummary() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.StatDate != "2026-07-12" || row.Protocol != "JT808" || row.SelectionStatus != "not_selected" || row.SelectionReason != "lower_trust_priority_or_sample_count" { + t.Fatalf("unexpected summary key: %#v", row) + } + if row.SourceCount != 36 || row.VehicleCount != 31 || row.SelectedCount != 0 || row.SampleCount != 3200 || row.DailyMileageKM != 812.5 { + t.Fatalf("unexpected summary values: %#v", row) + } + if row.SelectionAction == "" || !strings.Contains(row.SelectionAction, "更高可信优先级") { + t.Fatalf("selection action = %q", row.SelectionAction) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestBuildMetricSourceSelectionSummarySQLGroupsByComputedSelectionEvidence(t *testing.T) { + sqlText, args := buildMetricSourceSelectionSummarySQL(normalizeMetricSourceQuery(MetricSourceQuery{ + Protocol: "jt808", + DateFrom: "2026-07-12", + DateTo: "2026-07-12", + Limit: 20, + })) + for _, want := range []string{ + "selection_status", + "selection_reason", + "selected_current_projection", + "source_disabled", + "source_key LIKE '%@PLATFORM:%'", + "source_key LIKE '%@DIRECT'", + "ds.id IS NULL AND COALESCE(NULLIF(TRIM(ds.source_kind), ''), CASE WHEN s.source_key LIKE '%@PLATFORM:%' THEN 'PLATFORM' WHEN s.source_key LIKE '%@DIRECT' THEN 'DIRECT' ELSE 'UNKNOWN' END) <> 'DIRECT'", + "unmanaged_source_lower_priority", + "unknown_source_kind_lower_priority", + "lower_trust_priority_or_sample_count", + "GROUP BY stat_date, protocol, selection_status, selection_reason", + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("selection summary SQL missing %q:\n%s", want, sqlText) + } + } + if len(args) != 5 || args[0] != "JT808" || args[1] != "2026-07-12" || args[2] != "2026-07-12" || args[3] != 20 || args[4] != 0 { + t.Fatalf("args = %#v", args) + } +} + +func TestBuildMetricSourceSQLCanOrderByDailyMileageDesc(t *testing.T) { + selected := false + sqlText, args := buildMetricSourceSQL(normalizeMetricSourceQuery(MetricSourceQuery{ + Protocol: "jt808", + DateFrom: "2026-07-12", + DateTo: "2026-07-12", + Selected: &selected, + OrderBy: "daily_mileage_desc", + Limit: 20, + })) + + for _, want := range []string{ + "ORDER BY s.daily_mileage_km DESC", + "s.sample_count DESC", + "s.latest_event_time DESC", + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("sql missing %q:\n%s", want, sqlText) + } + } + if strings.Contains(sqlText, "ORDER BY s.stat_date DESC, s.vin ASC") { + t.Fatalf("sql should use mileage ordering:\n%s", sqlText) + } + if len(args) != 6 || args[0] != "JT808" || args[1] != "2026-07-12" || args[2] != "2026-07-12" || args[3] != 0 || args[4] != 20 || args[5] != 0 { + t.Fatalf("args = %#v", args) + } +} + +func TestMetricRepositoryQueriesDailyMetricDiagnostics(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "plate", "platform_name", "peer", + "snapshot_event_time", "location_event_time", "snapshot_updated_at", "location_updated_at", + "realtime_total_mileage_km", "realtime_total_mileage_event_time", "daily_mileage_km", "daily_latest_total_mileage_km", + "source_sample_count", "ok_source_count", "selectable_source_count", "latest_stat_event_time", "quality_statuses", "snapshot_parsed_json", + }).AddRow( + "LA9HE60A1PBAF4008", "2026-07-12", "JT808", "粤A12345", "信达", "115.231.168.135:47849", + "2026-07-12 02:23:27", "2026-07-12 02:23:27", "2026-07-12 02:23:29", "2026-07-12 02:23:29", + 25246.6, "2026-07-12 02:23:27", nil, nil, + 0, 0, 0, nil, "", `{"jt808.location.latitude":"30.1","jt808.location.total_mileage_km":"25246.6"}`, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnostics(context.Background(), MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + Diagnosis: "NO_SOURCE_SAMPLE", + Limit: 20, + }) + if err != nil { + t.Fatalf("QueryDiagnostics() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.Diagnosis != "NO_SOURCE_SAMPLE" || row.Reason != "realtime_location_has_total_mileage_but_no_stat_sample" { + t.Fatalf("unexpected diagnosis: %#v", row) + } + if row.Severity != "pipeline" { + t.Fatalf("severity = %q, want pipeline", row.Severity) + } + if !strings.Contains(row.RecommendedOperatorAction, "字段流 topic") { + t.Fatalf("recommended action = %q", row.RecommendedOperatorAction) + } + if row.RealtimeTotalMileageKM == nil || *row.RealtimeTotalMileageKM != 25246.6 { + t.Fatalf("realtime total mileage = %#v", row.RealtimeTotalMileageKM) + } + if row.DailyMileageKM != nil { + t.Fatalf("daily mileage should be nil: %#v", row.DailyMileageKM) + } + if row.PlatformName != "信达" || row.Peer != "115.231.168.135:47849" { + t.Fatalf("unexpected realtime evidence: %#v", row) + } + if row.RealtimeFieldCount != 2 || len(row.RealtimeSampleFields) != 2 { + t.Fatalf("realtime field evidence = count:%d fields:%#v", row.RealtimeFieldCount, row.RealtimeSampleFields) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricDiagnosisWhereSupportsReasonAndSeverityFilters(t *testing.T) { + query := normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Date: "2026-07-12", + Reason: "REALTIME_TOTAL_MILEAGE_MISSING", + Severity: "SOURCE_DATA", + }) + + where, args := buildMetricDiagnosisWhere(query) + + joined := strings.Join(where, "\n") + for _, want := range []string{ + "CASE", + "l.total_mileage_km IS NULL", + "source_data", + } { + if !strings.Contains(joined, want) { + t.Fatalf("where missing %q:\n%s", want, joined) + } + } + if len(args) < 4 { + t.Fatalf("args too short: %#v", args) + } + var hasReason, hasSeverity bool + for _, arg := range args { + switch arg { + case "realtime_total_mileage_missing": + hasReason = true + case "source_data": + hasSeverity = true + } + } + if !hasReason || !hasSeverity { + t.Fatalf("args = %#v, want reason/severity filters", args) + } +} + +func TestMetricRepositoryQueriesDailyMetricDiagnosisSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "active_count", "ok_count", "missing_daily_count", "no_source_sample_count", "no_total_mileage_count", + }).AddRow( + "2026-07-12", "JT808", int64(244), int64(244), int64(0), int64(0), int64(0), + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnosisSummary(context.Background(), MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + }) + if err != nil { + t.Fatalf("QueryDiagnosisSummary() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.StatDate != "2026-07-12" || row.Protocol != "JT808" || row.ActiveCount != 244 || row.OKCount != 244 || row.ActionableIssueCount != 0 { + t.Fatalf("unexpected summary row: %#v", row) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricRepositoryQueriesDailyMetricDiagnosisReasonSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisReasonSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "diagnosis", "reason", "count", + }).AddRow( + "2026-07-12", "YUTONG_MQTT", "NO_TOTAL_MILEAGE", "realtime_total_mileage_not_reported_on_stat_date", int64(7), + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnosisReasonSummary(context.Background(), MetricDiagnosisQuery{ + Protocol: "yutong_mqtt", + Date: "2026-07-12", + Diagnosis: "no_total_mileage", + }) + if err != nil { + t.Fatalf("QueryDiagnosisReasonSummary() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.StatDate != "2026-07-12" || row.Protocol != "YUTONG_MQTT" || row.Diagnosis != "NO_TOTAL_MILEAGE" || row.Reason != "realtime_total_mileage_not_reported_on_stat_date" || row.Count != 7 || row.Severity != metricDiagnosisSeveritySourceData { + t.Fatalf("unexpected reason summary row: %#v", row) + } + if !strings.Contains(row.RecommendedOperatorAction, "不是统计日期内上报") { + t.Fatalf("recommended action = %q", row.RecommendedOperatorAction) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricRepositoryQueriesDailyMetricDiagnosisFieldStatusSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisFieldStatusSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "diagnosis", "reason", "realtime_mileage_field_status", "count", + }).AddRow( + "2026-07-12", "YUTONG_MQTT", "NO_TOTAL_MILEAGE", "realtime_total_mileage_missing", "no_candidate_mileage_field", int64(9), + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnosisFieldStatusSummary(context.Background(), MetricDiagnosisQuery{ + Protocol: "yutong_mqtt", + Date: "2026-07-12", + Diagnosis: "no_total_mileage", + Reason: "realtime_total_mileage_missing", + }) + if err != nil { + t.Fatalf("QueryDiagnosisFieldStatusSummary() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.StatDate != "2026-07-12" || row.Protocol != "YUTONG_MQTT" || row.Diagnosis != "NO_TOTAL_MILEAGE" || row.RealtimeMileageFieldStatus != "no_candidate_mileage_field" || row.Count != 9 { + t.Fatalf("unexpected field status summary row: %#v", row) + } + if row.Severity != metricDiagnosisSeveritySourceData { + t.Fatalf("severity = %q", row.Severity) + } + if row.FieldStatusSeverity != metricDiagnosisSeveritySourceData { + t.Fatalf("field status severity = %q", row.FieldStatusSeverity) + } + if !strings.Contains(row.FieldStatusAction, "源平台") { + t.Fatalf("field status action = %q", row.FieldStatusAction) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricDiagnosisFieldStatusSeverityDistinguishesFreshProjectionEvidence(t *testing.T) { + tests := []struct { + status string + want string + }{ + {status: "daily_metric_exists", want: metricDiagnosisSeverityOK}, + {status: "candidate_field_unmapped", want: metricDiagnosisSeverityPipeline}, + {status: "mapped_protocol_field_without_fresh_evidence", want: metricDiagnosisSeveritySourceData}, + {status: "standard_field_not_consumed", want: metricDiagnosisSeverityPipeline}, + {status: "standard_field_time_missing", want: metricDiagnosisSeverityPipeline}, + {status: "no_realtime_fields", want: metricDiagnosisSeverityPipeline}, + {status: "standard_field_non_positive", want: metricDiagnosisSeveritySourceData}, + {status: "standard_field_stale", want: metricDiagnosisSeveritySourceData}, + {status: "no_candidate_mileage_field", want: metricDiagnosisSeveritySourceData}, + } + for _, tt := range tests { + t.Run(tt.status, func(t *testing.T) { + if got := metricDiagnosisMileageFieldStatusSeverity(tt.status); got != tt.want { + t.Fatalf("metricDiagnosisMileageFieldStatusSeverity(%q) = %q, want %q", tt.status, got, tt.want) + } + }) + } +} + +func TestMetricDiagnosisFieldStatusSummaryTotalsPreferDiagnosisSeverity(t *testing.T) { + _, actionable, pipeline, sourceData := metricDiagnosisFieldStatusSummaryIssueTotals([]MetricDiagnosisFieldStatusSummaryRow{ + { + Diagnosis: "MISSING_DAILY", + Reason: "source_samples_all_invalid", + Severity: metricDiagnosisSeveritySourceData, + RealtimeMileageFieldStatus: "source_sample_exists", + FieldStatusSeverity: metricDiagnosisSeverityPipeline, + Count: 2, + }, + }) + + if actionable != 2 || pipeline != 0 || sourceData != 2 { + t.Fatalf("totals actionable=%d pipeline=%d source=%d", actionable, pipeline, sourceData) + } +} + +func TestMetricDiagnosisIssueSeverity(t *testing.T) { + tests := []struct { + name string + diagnosis string + reason string + want string + }{ + {name: "ok", diagnosis: "OK", want: metricDiagnosisSeverityOK}, + {name: "missing daily", diagnosis: "MISSING_DAILY", want: metricDiagnosisSeverityPipeline}, + {name: "missing daily all invalid samples", diagnosis: "MISSING_DAILY", reason: "source_samples_all_invalid", want: metricDiagnosisSeveritySourceData}, + {name: "missing daily all excluded samples", diagnosis: "MISSING_DAILY", reason: "source_samples_all_excluded", want: metricDiagnosisSeveritySourceData}, + {name: "no source sample", diagnosis: "NO_SOURCE_SAMPLE", want: metricDiagnosisSeverityPipeline}, + {name: "time missing", diagnosis: "NO_TOTAL_MILEAGE", reason: "realtime_total_mileage_time_missing", want: metricDiagnosisSeverityPipeline}, + {name: "source no mileage", diagnosis: "NO_TOTAL_MILEAGE", reason: "realtime_total_mileage_non_positive", want: metricDiagnosisSeveritySourceData}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := metricDiagnosisIssueSeverity(tt.diagnosis, tt.reason); got != tt.want { + t.Fatalf("metricDiagnosisIssueSeverity() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestBuildActiveRealtimeMetricSQLUsesBusinessRealtimeTimestamp(t *testing.T) { + sqlText, args := buildActiveRealtimeMetricSQL(normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + })) + for _, want := range []string{ + "vehicle_realtime_snapshot", + "vehicle_realtime_location", + "event_time >= ?", + "received_at >= ?", + "protocol = ?", + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("active realtime SQL missing %q:\n%s", want, sqlText) + } + } + if strings.Contains(sqlText, "updated_at >= ?") { + t.Fatalf("active realtime SQL should not use row mutation time as vehicle activity evidence:\n%s", sqlText) + } + if strings.Contains(sqlText, "COALESCE(event_time") { + t.Fatalf("active realtime SQL should not let stale event_time hide received_at:\n%s", sqlText) + } + if strings.Count(sqlText, "protocol = ?") != 2 { + t.Fatalf("protocol filter should be pushed into both realtime tables:\n%s", sqlText) + } + if len(args) != 10 { + t.Fatalf("args = %d, want 10: %#v", len(args), args) + } +} + +func TestBuildMetricDiagnosisSQLRequiresPositiveRealtimeMileage(t *testing.T) { + summarySQL, _ := buildMetricDiagnosisSummarySQL(normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + })) + detailSQL, _ := buildMetricDiagnosisSQL(normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + Diagnosis: "NO_SOURCE_SAMPLE", + })) + noTotalSQL, _ := buildMetricDiagnosisSQL(normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Protocol: "jt808", + Date: "2026-07-12", + Diagnosis: "NO_TOTAL_MILEAGE", + })) + for _, want := range []string{ + "l.total_mileage_km > 0", + "l.total_mileage_km IS NULL OR l.total_mileage_km <= 0", + } { + combinedSQL := summarySQL + "\n" + detailSQL + "\n" + noTotalSQL + if !strings.Contains(combinedSQL, want) { + t.Fatalf("diagnosis SQL missing %q:\n%s", want, combinedSQL) + } + } +} + +func TestBuildMetricDiagnosisFieldStatusSummarySQLClassifiesMileageEvidence(t *testing.T) { + sqlText, args := buildMetricDiagnosisFieldStatusSummarySQL(normalizeMetricDiagnosisQuery(MetricDiagnosisQuery{ + Protocol: "yutong_mqtt", + Date: "2026-07-12", + Diagnosis: "NO_TOTAL_MILEAGE", + Reason: "realtime_total_mileage_missing", + Severity: "source_data", + })) + for _, want := range []string{ + "realtime_mileage_field_status", + "standard_field_non_positive", + "standard_field_stale", + "mapped_protocol_field_without_fresh_evidence", + "candidate_field_unmapped", + "no_candidate_mileage_field", + "LOWER(s.parsed_json) LIKE '%mileage%'", + "GROUP BY stat_date, protocol, diagnosis, reason, realtime_mileage_field_status", + } { + if !strings.Contains(sqlText, want) { + t.Fatalf("field status summary SQL missing %q:\n%s", want, sqlText) + } + } + for _, want := range []any{"NO_TOTAL_MILEAGE", "realtime_total_mileage_missing", "source_data"} { + var found bool + for _, arg := range args { + if arg == want { + found = true + break + } + } + if !found { + t.Fatalf("args missing %v: %#v", want, args) + } + } +} + +func TestMetricDiagnosisTreatsZeroMileageAsNoTotalMileage(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 0, 0, 0, + sql.NullFloat64{Float64: 0, Valid: true}, + "2026-07-12 02:51:04", + "2026-07-12", + ) + if diagnosis != "NO_TOTAL_MILEAGE" || reason != "realtime_total_mileage_non_positive" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } +} + +func TestMetricDiagnosisTreatsAllInvalidSourceSamplesAsSourceData(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 2, 0, 0, + sql.NullFloat64{Float64: 47724.2, Valid: true}, + "2026-07-13 00:07:45", + "2026-07-13", + ) + if diagnosis != "MISSING_DAILY" || reason != "source_samples_all_invalid" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } + if got := metricDiagnosisIssueSeverity(diagnosis, reason); got != metricDiagnosisSeveritySourceData { + t.Fatalf("severity=%s, want source_data", got) + } + if action := metricDiagnosisRecommendedOperatorAction(diagnosis, reason); !strings.Contains(action, "质量规则排除") { + t.Fatalf("action=%q", action) + } +} + +func TestMetricDiagnosisTreatsAllExcludedSourceSamplesAsSourceData(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 1, 1, 0, + sql.NullFloat64{Float64: 33574.6, Valid: true}, + "2026-07-13 00:11:56", + "2026-07-13", + ) + if diagnosis != "MISSING_DAILY" || reason != "source_samples_all_excluded" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } + if got := metricDiagnosisIssueSeverity(diagnosis, reason); got != metricDiagnosisSeveritySourceData { + t.Fatalf("severity=%s, want source_data", got) + } + if action := metricDiagnosisRecommendedOperatorAction(diagnosis, reason); !strings.Contains(action, "没有可参与选举的来源") { + t.Fatalf("action=%q", action) + } +} + +func TestMetricDiagnosisExplainsStaleRealtimeMileage(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 0, 0, 0, + sql.NullFloat64{Float64: 85344, Valid: true}, + "2026-07-11 23:59:59", + "2026-07-12", + ) + if diagnosis != "NO_TOTAL_MILEAGE" || reason != "realtime_total_mileage_not_reported_on_stat_date" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } +} + +func TestMetricDiagnosisExplainsMissingRealtimeMileageTime(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 0, 0, 0, + sql.NullFloat64{Float64: 85344, Valid: true}, + "", + "2026-07-12", + ) + if diagnosis != "NO_TOTAL_MILEAGE" || reason != "realtime_total_mileage_time_missing" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } +} + +func TestMetricDiagnosisExplainsMissingRealtimeMileage(t *testing.T) { + diagnosis, reason := classifyMetricDiagnosis(false, 0, 0, 0, + sql.NullFloat64{}, + "", + "2026-07-12", + ) + if diagnosis != "NO_TOTAL_MILEAGE" || reason != "realtime_total_mileage_missing" { + t.Fatalf("diagnosis=%s reason=%s", diagnosis, reason) + } } func TestMetricHandlerReturnsDailyMetrics(t *testing.T) { @@ -93,16 +848,18 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) { t.Fatalf("sqlmock.New() error = %v", err) } defer db.Close() - mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage"). + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage m"). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01"). WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42)) mock.ExpectQuery(dailyMetricSelectPattern). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "stat_date", "protocol", "source_id", "daily_mileage_km", + "vin", "stat_date", "protocol", "source_id", + "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km", "latest_total_mileage_km", "updated_at", }).AddRow( - "LB9A32A21R0LS1707", "2020-07-01", "GB32960", 1, 0.0, + "LB9A32A21R0LS1707", "2020-07-01", "GB32960", 1, + "8.134.95.166", "8.134.95.166:32960", "现代 HTWO", "HYUNDAI", "PLATFORM", 0.0, 53490.9, "2026-07-01 22:28:25", )) @@ -116,7 +873,7 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } body := response.Body.String() - for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"daily_mileage_km":0`, `"total":42`} { + for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"platform_name":"现代 HTWO"`, `"source_kind":"PLATFORM"`, `"daily_mileage_km":0`, `"total":42`} { if !strings.Contains(body, want) { t.Fatalf("response missing %s: %s", want, body) } @@ -129,6 +886,449 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) { } } +func TestMetricHandlerReturnsDailyMetricSources(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage_source s"). + WithArgs("LA9GG64L7PBAF4001", "JT808", "2026-07-08", "2026-07-08", "historical_source_baseline", 1). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(2)) + mock.ExpectQuery(dailyMetricSourceSelectPattern). + WithArgs("LA9GG64L7PBAF4001", "JT808", "2026-07-08", "2026-07-08", "historical_source_baseline", 1, 50, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "source_key", "source_ip", "source_endpoint", "phone", "device_id", + "platform_name", "source_id", "source_code", "source_kind", "enabled", "trust_priority", + "first_total_mileage_km", "latest_total_mileage_km", "daily_mileage_km", "sample_count", + "first_event_time", "latest_event_time", "quality_status", "quality_reason", "is_selected", "updated_at", + }).AddRow( + "LA9GG64L7PBAF4001", "2026-07-08", "JT808", "JT808:13307795425@115.231.168.135", "115.231.168.135", + "115.231.168.135:41561", "13307795425", nil, + "信达", 5, "xinda", "PLATFORM", 1, 10, + 4100.8, 4123.9, 23.1, 128, + "2026-07-08 00:01:00", "2026-07-08 23:59:00", "OK", "historical_source_baseline", 1, "2026-07-08 23:59:10", + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/sources?vin=LA9GG64L7PBAF4001&protocol=JT808&dateFrom=2026-07-08&dateTo=2026-07-08&qualityReason=Historical_Source_Baseline&selected=true&includeTotal=true", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":2`, `"source_key":"JT808:13307795425@115.231.168.135"`, `"platform_name":"信达"`, `"source_kind":"PLATFORM"`, `"is_selected":true`, `"quality_status":"OK"`, `"selection_status":"selected"`, `"selection_reason":"selected_current_projection"`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricSourceQualitySummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM \\("). + WithArgs("JT808", "2026-07-12", "2026-07-12", "INVALID_DELTA", "outside_daily_range"). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) + mock.ExpectQuery(dailyMetricSourceQualitySummarySelectPattern). + WithArgs("JT808", "2026-07-12", "2026-07-12", "INVALID_DELTA", "outside_daily_range", 5, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "quality_status", "quality_reason", + "source_count", "vehicle_count", "selected_count", "sample_count", "daily_mileage_km", + }).AddRow( + "2026-07-12", "JT808", "INVALID_DELTA", "outside_daily_range", + int64(3), int64(3), int64(0), int64(128), 12435.2, + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/sources/quality?protocol=JT808&dateFrom=2026-07-12&dateTo=2026-07-12&qualityStatus=invalid_delta&qualityReason=Outside_Daily_Range&includeTotal=true&limit=5", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":1`, `"protocol":"JT808"`, `"quality_status":"INVALID_DELTA"`, `"quality_reason":"outside_daily_range"`, `"source_count":3`, `"vehicle_count":3`, `"selected_count":0`, `"sample_count":128`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricSourceSelectionSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM \\("). + WithArgs("JT808", "2026-07-12", "2026-07-12"). + WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(2)) + mock.ExpectQuery(dailyMetricSourceSelectionSummarySelectPattern). + WithArgs("JT808", "2026-07-12", "2026-07-12", 10, 0). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "selection_status", "selection_reason", + "source_count", "vehicle_count", "selected_count", "sample_count", "daily_mileage_km", + }).AddRow( + "2026-07-12", "JT808", "excluded", "quality_outside_daily_range", + int64(3), int64(3), int64(0), int64(128), 12435.2, + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/sources/selection?protocol=JT808&dateFrom=2026-07-12&dateTo=2026-07-12&includeTotal=true&limit=10", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"total":2`, `"protocol":"JT808"`, `"selection_status":"excluded"`, `"selection_reason":"quality_outside_daily_range"`, `"selection_action":"来源质量未通过`, `"source_count":3`, `"vehicle_count":3`, `"sample_count":128`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricDiagnostics(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "plate", "platform_name", "peer", + "snapshot_event_time", "location_event_time", "snapshot_updated_at", "location_updated_at", + "realtime_total_mileage_km", "realtime_total_mileage_event_time", "daily_mileage_km", "daily_latest_total_mileage_km", + "source_sample_count", "ok_source_count", "selectable_source_count", "latest_stat_event_time", "quality_statuses", "snapshot_parsed_json", + }).AddRow( + "LMRKH9AC2R1004087", "2026-07-12", "YUTONG_MQTT", "粤A99999", "宇通 MQTT", "yutong", + "2026-07-12 10:01:00", nil, "2026-07-12 10:01:01", nil, + nil, nil, nil, nil, + 0, 0, 0, nil, "", `{"yutong_mqtt.data.latitude":"30.593478","yutong_mqtt.data.longitude":"121.073451","yutong_mqtt.data.meter_speed":"0"}`, + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics?protocol=YUTONG_MQTT&date=2026-07-12&diagnosis=NO_TOTAL_MILEAGE", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"vin":"LMRKH9AC2R1004087"`, `"diagnosis":"NO_TOTAL_MILEAGE"`, `"reason":"realtime_total_mileage_missing"`, `"severity":"source_data"`, `"recommended_operator_action":"当日实时数据缺少总里程字段`, `"realtime_field_count":3`, `"realtime_mileage_field_status":"no_candidate_mileage_field"`, `"realtime_mileage_evidence":"实时快照已有字段`, `"raw_frame_query_path":"/api/history/raw-frames?`, `"source_sample_query_path":"/api/stats/daily-metrics/sources?`, `dateFrom=2026-07-12+00%3A00%3A00`, `protocol=YUTONG_MQTT`, `vin=LMRKH9AC2R1004087`, `"yutong_mqtt.data.latitude"`, `"yutong_mqtt.data.meter_speed"`, `"total":1`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricDiagnosisTraceQueryPaths(t *testing.T) { + row := MetricDiagnosisRow{ + VIN: "LMRKH9AC2R1004087", + StatDate: "2026-07-12", + Protocol: "YUTONG_MQTT", + } + + rawPath := metricDiagnosisRawFrameQueryPath(row) + for _, want := range []string{ + "/api/history/raw-frames?", + "dateFrom=2026-07-12+00%3A00%3A00", + "dateTo=2026-07-13+00%3A00%3A00", + "includeFields=true", + "limit=20", + "orderBy=eventTime", + "protocol=YUTONG_MQTT", + "vin=LMRKH9AC2R1004087", + } { + if !strings.Contains(rawPath, want) { + t.Fatalf("raw path missing %q: %s", want, rawPath) + } + } + + sourcePath := metricDiagnosisSourceSampleQueryPath(row) + for _, want := range []string{ + "/api/stats/daily-metrics/sources?", + "dateFrom=2026-07-12", + "dateTo=2026-07-12", + "includeTotal=true", + "limit=50", + "protocol=YUTONG_MQTT", + "vin=LMRKH9AC2R1004087", + } { + if !strings.Contains(sourcePath, want) { + t.Fatalf("source path missing %q: %s", want, sourcePath) + } + } +} + +func TestMetricRepositoryFlagsCandidateMileageFieldsWhenStandardMileageMissing(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "plate", "platform_name", "peer", + "snapshot_event_time", "location_event_time", "snapshot_updated_at", "location_updated_at", + "realtime_total_mileage_km", "realtime_total_mileage_event_time", "daily_mileage_km", "daily_latest_total_mileage_km", + "source_sample_count", "ok_source_count", "selectable_source_count", "latest_stat_event_time", "quality_statuses", "snapshot_parsed_json", + }).AddRow( + "VIN001", "2026-07-12", "YUTONG_MQTT", "", "宇通 MQTT", "mqtt://yutong/ytforward/shln/3", + "2026-07-12 10:01:00", nil, "2026-07-12 10:01:01", nil, + nil, nil, nil, nil, + 0, 0, 0, nil, "", `{"yutong_mqtt.data.latitude":"30.593478","yutong_mqtt.data.odometer":"65423000","yutong_mqtt.data.meter_speed":"12"}`, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnostics(context.Background(), MetricDiagnosisQuery{ + Protocol: "YUTONG_MQTT", + Date: "2026-07-12", + Diagnosis: "NO_TOTAL_MILEAGE", + Limit: 20, + }) + if err != nil { + t.Fatalf("QueryDiagnostics() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.RealtimeMileageFieldStatus != "candidate_field_unmapped" { + t.Fatalf("mileage field status = %q, row=%#v", row.RealtimeMileageFieldStatus, row) + } + if len(row.RealtimeMileageCandidateFields) != 1 || row.RealtimeMileageCandidateFields[0] != "yutong_mqtt.data.odometer" { + t.Fatalf("candidate fields = %#v", row.RealtimeMileageCandidateFields) + } + if !strings.Contains(row.RealtimeMileageEvidence, "协议字段映射") { + t.Fatalf("evidence = %q", row.RealtimeMileageEvidence) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricRepositoryTreatsMergedMappedMileageFieldAsStaleEvidence(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "plate", "platform_name", "peer", + "snapshot_event_time", "location_event_time", "snapshot_updated_at", "location_updated_at", + "realtime_total_mileage_km", "realtime_total_mileage_event_time", "daily_mileage_km", "daily_latest_total_mileage_km", + "source_sample_count", "ok_source_count", "selectable_source_count", "latest_stat_event_time", "quality_statuses", "snapshot_parsed_json", + }).AddRow( + "VIN001", "2026-07-12", "YUTONG_MQTT", "", "宇通 MQTT", "mqtt://yutong/ytforward/shln/3", + "2026-07-12 10:01:00", nil, "2026-07-12 10:01:01", nil, + nil, nil, nil, nil, + 0, 0, 0, nil, "", `{"yutong_mqtt.data.latitude":"30.593478","yutong_mqtt.data.total_mileage":"65423000","yutong_mqtt.data.meter_speed":"12"}`, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnostics(context.Background(), MetricDiagnosisQuery{ + Protocol: "YUTONG_MQTT", + Date: "2026-07-12", + Diagnosis: "NO_TOTAL_MILEAGE", + Limit: 20, + }) + if err != nil { + t.Fatalf("QueryDiagnostics() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.RealtimeMileageFieldStatus != "mapped_protocol_field_without_fresh_evidence" { + t.Fatalf("mileage field status = %q, row=%#v", row.RealtimeMileageFieldStatus, row) + } + if len(row.RealtimeMileageCandidateFields) != 1 || row.RealtimeMileageCandidateFields[0] != "yutong_mqtt.data.total_mileage" { + t.Fatalf("candidate fields = %#v", row.RealtimeMileageCandidateFields) + } + if !strings.Contains(row.RealtimeMileageEvidence, "合并快照") { + t.Fatalf("evidence = %q", row.RealtimeMileageEvidence) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricRepositoryReportsStaleStandardMileageBeforeCandidateFields(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "stat_date", "protocol", "plate", "platform_name", "peer", + "snapshot_event_time", "location_event_time", "snapshot_updated_at", "location_updated_at", + "realtime_total_mileage_km", "realtime_total_mileage_event_time", "daily_mileage_km", "daily_latest_total_mileage_km", + "source_sample_count", "ok_source_count", "selectable_source_count", "latest_stat_event_time", "quality_statuses", "snapshot_parsed_json", + }).AddRow( + "VIN001", "2026-07-13", "YUTONG_MQTT", "", "宇通 MQTT", "mqtt://yutong/ytforward/shln/3", + "2026-07-13 10:01:00", "2026-07-13 10:01:00", "2026-07-13 10:01:01", "2026-07-13 10:01:01", + 65423.0, "2026-07-12 23:59:59", nil, nil, + 0, 0, 0, nil, "", `{"yutong_mqtt.data.total_mileage":"65423000"}`, + )) + + repository := NewMetricRepository(db) + rows, err := repository.QueryDiagnostics(context.Background(), MetricDiagnosisQuery{ + Protocol: "YUTONG_MQTT", + Date: "2026-07-13", + Diagnosis: "NO_TOTAL_MILEAGE", + Limit: 20, + }) + if err != nil { + t.Fatalf("QueryDiagnostics() error = %v", err) + } + if len(rows) != 1 { + t.Fatalf("row count = %d", len(rows)) + } + row := rows[0] + if row.Reason != "realtime_total_mileage_not_reported_on_stat_date" { + t.Fatalf("reason = %q", row.Reason) + } + if row.RealtimeMileageFieldStatus != "standard_field_stale" { + t.Fatalf("mileage field status = %q, row=%#v", row.RealtimeMileageFieldStatus, row) + } + if len(row.RealtimeMileageCandidateFields) != 0 { + t.Fatalf("candidate fields should be hidden for standard stale field: %#v", row.RealtimeMileageCandidateFields) + } + if !strings.Contains(row.RealtimeMileageEvidence, "不在统计日") { + t.Fatalf("evidence = %q", row.RealtimeMileageEvidence) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricDiagnosisSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "active_count", "ok_count", "missing_daily_count", "no_source_sample_count", "no_total_mileage_count", + }).AddRow( + "2026-07-12", "YUTONG_MQTT", int64(13), int64(7), int64(0), int64(0), int64(6), + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics/summary?protocol=YUTONG_MQTT&date=2026-07-12", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"protocol":"YUTONG_MQTT"`, `"active_count":13`, `"ok_count":7`, `"no_total_mileage_count":6`, `"active_total":13`, `"actionable_issue_total":6`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricDiagnosisReasonSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisReasonSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "diagnosis", "reason", "count", + }).AddRow( + "2026-07-12", "JT808", "NO_TOTAL_MILEAGE", "realtime_total_mileage_non_positive", int64(2), + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics/reasons?protocol=JT808&date=2026-07-12&diagnosis=NO_TOTAL_MILEAGE", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"protocol":"JT808"`, `"diagnosis":"NO_TOTAL_MILEAGE"`, `"reason":"realtime_total_mileage_non_positive"`, `"count":2`, `"severity":"source_data"`, `"recommended_operator_action":"终端上报总里程小于等于 0`, `"vehicle_total":2`, `"actionable_issue_total":2`, `"pipeline_issue_total":0`, `"source_data_issue_total":2`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + +func TestMetricHandlerReturnsDailyMetricDiagnosisFieldStatusSummary(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery(dailyMetricDiagnosisFieldStatusSummarySelectPattern). + WillReturnRows(sqlmock.NewRows([]string{ + "stat_date", "protocol", "diagnosis", "reason", "realtime_mileage_field_status", "count", + }).AddRow( + "2026-07-12", "JT808", "NO_TOTAL_MILEAGE", "realtime_total_mileage_non_positive", "standard_field_non_positive", int64(2), + )) + + handler := NewMetricHandler(NewMetricRepository(db)) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics/field-status?protocol=JT808&date=2026-07-12&diagnosis=NO_TOTAL_MILEAGE", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{`"protocol":"JT808"`, `"diagnosis":"NO_TOTAL_MILEAGE"`, `"reason":"realtime_total_mileage_non_positive"`, `"realtime_mileage_field_status":"standard_field_non_positive"`, `"field_status_severity":"source_data"`, `"field_status_action":"源头总里程小于等于 0`, `"vehicle_total":2`, `"source_data_issue_total":2`} { + if !strings.Contains(body, want) { + t.Fatalf("response missing %s: %s", want, body) + } + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -138,7 +1338,8 @@ func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) { mock.ExpectQuery(dailyMetricSelectPattern). WithArgs("YUTONG_MQTT", 50, 0). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "stat_date", "protocol", "source_id", "daily_mileage_km", + "vin", "stat_date", "protocol", "source_id", + "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km", "latest_total_mileage_km", "updated_at", })) @@ -169,10 +1370,12 @@ func TestMetricHandlerSkipsTotalCountByDefault(t *testing.T) { mock.ExpectQuery(dailyMetricSelectPattern). WithArgs("JT808", 1, 0). WillReturnRows(sqlmock.NewRows([]string{ - "vin", "stat_date", "protocol", "source_id", "daily_mileage_km", + "vin", "stat_date", "protocol", "source_id", + "source_ip", "latest_source_endpoint", "platform_name", "source_code", "source_kind", "daily_mileage_km", "latest_total_mileage_km", "updated_at", }).AddRow( - "LKLG7C4E3NA774736", "2026-07-02", "JT808", 3, 12.3, + "LKLG7C4E3NA774736", "2026-07-02", "JT808", 3, + "115.231.168.135", "115.231.168.135:41561", "G7 平台", "G7S", "PLATFORM", 12.3, 8805.1, "2026-07-02 23:59:59", )) @@ -205,3 +1408,39 @@ func TestMetricHandlerRejectsInvalidPagination(t *testing.T) { t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) } } + +func TestMetricHandlerRejectsInvalidDiagnosis(t *testing.T) { + handler := NewMetricHandler(NewMetricRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics?diagnosis=BAD", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } +} + +func TestMetricHandlerRejectsInvalidDiagnosisReason(t *testing.T) { + handler := NewMetricHandler(NewMetricRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics?reason=unknown_reason", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } +} + +func TestMetricHandlerRejectsInvalidDiagnosisSeverity(t *testing.T) { + handler := NewMetricHandler(NewMetricRepository(&sql.DB{})) + request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics/diagnostics?severity=critical", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } +} diff --git a/go/vehicle-gateway/internal/stats/schema.go b/go/vehicle-gateway/internal/stats/schema.go index 07257f6a..40e046c3 100644 --- a/go/vehicle-gateway/internal/stats/schema.go +++ b/go/vehicle-gateway/internal/stats/schema.go @@ -15,6 +15,10 @@ const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage ( )` var DailyMileageAlterSQL = []string{ + "ALTER TABLE vehicle_data_source ADD COLUMN source_code VARCHAR(64) NULL AFTER platform_name", + "ALTER TABLE vehicle_data_source ADD COLUMN source_kind VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN' AFTER source_code", + "ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_code (protocol, source_code)", + "ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_kind_seen (protocol, source_kind, latest_seen_at)", "ALTER TABLE vehicle_daily_mileage ADD COLUMN source_id BIGINT NULL AFTER protocol", "ALTER TABLE vehicle_daily_mileage ADD KEY idx_source_id (source_id)", "ALTER TABLE vehicle_daily_mileage DROP COLUMN first_total_mileage_km", @@ -22,6 +26,10 @@ var DailyMileageAlterSQL = []string{ "ALTER TABLE vehicle_daily_mileage DROP COLUMN trusted_phone", "ALTER TABLE vehicle_daily_mileage DROP COLUMN trusted_source_endpoint", "ALTER TABLE vehicle_daily_mileage DROP COLUMN sample_count", + "ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_protocol_date_selected (protocol, stat_date, is_selected, vin)", + "ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_protocol_date_quality (protocol, stat_date, quality_status, vin)", + "ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_protocol_date_quality_reason (protocol, stat_date, quality_status, quality_reason, vin)", + "ALTER TABLE vehicle_daily_mileage_source ADD KEY idx_source_ip_date (protocol, source_ip, stat_date)", } const DataSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_data_source ( @@ -30,6 +38,8 @@ const DataSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_data_source ( source_ip VARCHAR(64) NOT NULL, latest_source_endpoint VARCHAR(128) NULL, platform_name VARCHAR(128) NULL, + source_code VARCHAR(64) NULL, + source_kind VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', trust_priority INT NOT NULL DEFAULT 100, enabled TINYINT(1) NOT NULL DEFAULT 1, first_seen_at DATETIME NULL, @@ -39,6 +49,8 @@ const DataSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_data_source ( updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (id), UNIQUE KEY uk_protocol_source_ip (protocol, source_ip), + KEY idx_protocol_source_code (protocol, source_code), + KEY idx_protocol_source_kind_seen (protocol, source_kind, latest_seen_at), KEY idx_protocol_enabled_priority (protocol, enabled, trust_priority) )` @@ -66,5 +78,9 @@ const DailyMileageSourceTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mil PRIMARY KEY (vin, stat_date, protocol, source_key), KEY idx_protocol_date (protocol, stat_date), KEY idx_source_ip (protocol, source_ip), - KEY idx_selected (stat_date, protocol, is_selected) + KEY idx_selected (stat_date, protocol, is_selected), + KEY idx_protocol_date_selected (protocol, stat_date, is_selected, vin), + KEY idx_protocol_date_quality (protocol, stat_date, quality_status, vin), + KEY idx_protocol_date_quality_reason (protocol, stat_date, quality_status, quality_reason, vin), + KEY idx_source_ip_date (protocol, source_ip, stat_date) )` diff --git a/go/vehicle-gateway/internal/stats/source.go b/go/vehicle-gateway/internal/stats/source.go index 8dac95cd..fcfb0eab 100644 --- a/go/vehicle-gateway/internal/stats/source.go +++ b/go/vehicle-gateway/internal/stats/source.go @@ -12,6 +12,9 @@ type SourceIdentity struct { Protocol envelope.Protocol SourceIP string SourceEndpoint string + SourceCode string + PlatformName string + SourceKind string } func NewSourceIdentity(protocol envelope.Protocol, endpoint string) (SourceIdentity, bool) { @@ -26,15 +29,29 @@ func NewSourceIdentity(protocol envelope.Protocol, endpoint string) (SourceIdent }, true } +func NewSourceIdentityFromEnvelope(env envelope.FrameEnvelope) (SourceIdentity, bool) { + identity, ok := NewSourceIdentity(env.Protocol, env.SourceEndpoint) + if !ok { + return SourceIdentity{}, false + } + identity.SourceCode = strings.TrimSpace(env.SourceCode) + identity.PlatformName = strings.TrimSpace(env.PlatformName) + identity.SourceKind = normalizeSourceKindForRead(env.SourceKind) + return identity, true +} + +func ShouldManageDataSource(identity SourceIdentity) bool { + if strings.TrimSpace(identity.SourceIP) == "" { + return false + } + if strings.TrimSpace(identity.SourceCode) != "" || strings.TrimSpace(identity.PlatformName) != "" { + return true + } + return normalizeSourceKindForRead(identity.SourceKind) == "PLATFORM" +} + func NormalizeSourceIP(endpoint string) string { - endpoint = strings.TrimSpace(endpoint) - if endpoint == "" { - return "" - } - if host, _, ok := strings.Cut(endpoint, ":"); ok { - return strings.TrimSpace(host) - } - return endpoint + return envelope.NormalizeSourceEndpointKey(endpoint) } func UpsertDataSource(ctx context.Context, exec Execer, identity SourceIdentity, now time.Time) error { @@ -51,18 +68,69 @@ func UpsertDataSource(ctx context.Context, exec Execer, identity SourceIdentity, string(identity.Protocol), identity.SourceIP, identity.SourceEndpoint, + nullableTrimmedString(identity.PlatformName), + nullableTrimmedString(identity.SourceCode), + sourceKindForDataSourceWrite(identity), now, now, ) return err } +func sourceKindForDataSourceWrite(identity SourceIdentity) string { + kind := normalizeSourceKindForWrite(identity.SourceKind) + if kind != "UNKNOWN" { + return kind + } + if strings.TrimSpace(identity.SourceCode) != "" || strings.TrimSpace(identity.PlatformName) != "" { + return "PLATFORM" + } + return "UNKNOWN" +} + const upsertDataSourceSQL = ` INSERT INTO vehicle_data_source - (protocol, source_ip, latest_source_endpoint, first_seen_at, latest_seen_at) -VALUES (?, ?, ?, ?, ?) + (protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, first_seen_at, latest_seen_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE latest_source_endpoint = VALUES(latest_source_endpoint), - latest_seen_at = VALUES(latest_seen_at), + platform_name = CASE + WHEN vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = '' + THEN VALUES(platform_name) + ELSE vehicle_data_source.platform_name + END, + source_code = CASE + WHEN vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = '' + THEN VALUES(source_code) + ELSE vehicle_data_source.source_code + END, + source_kind = CASE + WHEN vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN' + THEN VALUES(source_kind) + ELSE vehicle_data_source.source_kind + END, + enabled = CASE + WHEN vehicle_data_source.enabled = 0 + AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored') + AND ( + (VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '') + OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '') + OR VALUES(source_kind) <> 'UNKNOWN' + ) + THEN 1 + ELSE vehicle_data_source.enabled + END, + remark = CASE + WHEN vehicle_data_source.enabled = 0 + AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored') + AND ( + (VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '') + OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '') + OR VALUES(source_kind) <> 'UNKNOWN' + ) + THEN 'auto-reenabled: source evidence restored' + ELSE vehicle_data_source.remark + END, + latest_seen_at = GREATEST(COALESCE(latest_seen_at, VALUES(latest_seen_at)), VALUES(latest_seen_at)), updated_at = CURRENT_TIMESTAMP ` diff --git a/go/vehicle-gateway/internal/stats/source_mileage.go b/go/vehicle-gateway/internal/stats/source_mileage.go index 4d433d3d..cb5ea063 100644 --- a/go/vehicle-gateway/internal/stats/source_mileage.go +++ b/go/vehicle-gateway/internal/stats/source_mileage.go @@ -10,10 +10,16 @@ import ( ) const ( - QualityOK = "OK" - QualityNoPreviousBaseline = "NO_PREVIOUS_BASELINE" - QualityInvalidDelta = "INVALID_DELTA" - maxSelectedDailyMileageKM = 1000 + QualityOK = "OK" + QualityNoPreviousBaseline = "NO_PREVIOUS_BASELINE" + QualityInvalidDelta = "INVALID_DELTA" + QualityReasonHistorical = "historical_source_baseline" + QualityReasonCurrentDayFirst = "current_day_first_baseline" + maxSelectedDailyMileageKM = 2500 + maxSelectedDailyMileageKMSQL = "2500" + maxNegativeMileageJitterKMSQL = "1" + directSourceKeySuffix = "@DIRECT" + platformSourceKeyPrefix = "@PLATFORM:" ) type SourceMileageSample struct { @@ -37,14 +43,37 @@ type SourceMileageSample struct { } func SourceKey(protocol envelope.Protocol, phone string, deviceID string, sourceIP string) string { + identity, _ := sourceKeyIdentity(phone, deviceID) + return string(protocol) + ":" + identity + "@" + strings.TrimSpace(sourceIP) +} + +func sourceKeyIdentity(phone string, deviceID string) (string, bool) { identity := strings.TrimSpace(phone) if identity == "" { identity = strings.TrimSpace(deviceID) } if identity == "" { - identity = "unknown" + return "unknown", false } - return string(protocol) + ":" + identity + "@" + strings.TrimSpace(sourceIP) + return identity, true +} + +func SourceKeyForKind(protocol envelope.Protocol, phone string, deviceID string, sourceIP string, sourceKind string) string { + return SourceKeyForSource(protocol, phone, deviceID, sourceIP, sourceKind, "") +} + +func SourceKeyForSource(protocol envelope.Protocol, phone string, deviceID string, sourceIP string, sourceKind string, sourceCode string) string { + identity, hasIdentity := sourceKeyIdentity(phone, deviceID) + if strings.EqualFold(strings.TrimSpace(sourceKind), "DIRECT") && hasIdentity { + return string(protocol) + ":" + identity + directSourceKeySuffix + } + if strings.EqualFold(strings.TrimSpace(sourceKind), "PLATFORM") && hasIdentity { + sourceCode = strings.TrimSpace(sourceCode) + if sourceCode != "" { + return string(protocol) + ":" + identity + platformSourceKeyPrefix + sourceCode + } + } + return SourceKey(protocol, phone, deviceID, sourceIP) } func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) SourceMileageSample { @@ -53,11 +82,12 @@ func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) VIN: sample.VIN, StatDate: sample.StatDate, Protocol: sample.Protocol, - SourceKey: SourceKey(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP), + SourceKey: SourceKeyForSource(sample.Protocol, sample.Phone, sample.DeviceID, identity.SourceIP, identity.SourceKind, identity.SourceCode), SourceIP: identity.SourceIP, SourceEndpoint: identity.SourceEndpoint, Phone: sample.Phone, DeviceID: sample.DeviceID, + PlatformName: firstNonEmpty(sample.PlatformName, identity.PlatformName), FirstTotalKM: sample.TotalMileageKM, LatestTotalKM: sample.TotalMileageKM, DailyKM: 0, @@ -69,6 +99,81 @@ func SourceMileageSampleFromMetric(sample MetricSample, identity SourceIdentity) } } +func firstNonEmpty(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} + +func NormalizeDailyMileageDelta(deltaKM float64) (float64, bool, string) { + return NormalizeDailyMileageDeltaForWindow(deltaKM, time.Time{}, time.Time{}) +} + +// DailyMileageFromDayBoundary keeps the business formula explicit: the +// current day's latest cumulative odometer minus the nearest earlier +// cumulative odometer from the same source. +func DailyMileageFromDayBoundary(previousBaselineKM float64, currentDayLatestKM float64) float64 { + return currentDayLatestKM - previousBaselineKM +} + +func NormalizeDailyMileageDeltaForWindow(deltaKM float64, firstEventTime time.Time, latestEventTime time.Time) (float64, bool, string) { + if deltaKM < 0 && deltaKM >= -maxNegativeMileageJitterKM { + return 0, true, "negative_jitter_clamped" + } + maxMileage := float64(MileageQualityLimitKM(firstEventTime, latestEventTime)) + if deltaKM < 0 || deltaKM > maxMileage { + return deltaKM, false, "outside_daily_range" + } + return deltaKM, true, "" +} + +func MileageQualityWindowDays(firstEventTime time.Time, latestEventTime time.Time) int { + if firstEventTime.IsZero() || latestEventTime.IsZero() || !latestEventTime.After(firstEventTime) { + return 1 + } + location := latestEventTime.Location() + firstYear, firstMonth, firstDay := firstEventTime.In(location).Date() + latestYear, latestMonth, latestDay := latestEventTime.Date() + firstDate := time.Date(firstYear, firstMonth, firstDay, 0, 0, 0, 0, time.UTC) + latestDate := time.Date(latestYear, latestMonth, latestDay, 0, 0, 0, 0, time.UTC) + days := int(latestDate.Sub(firstDate) / (24 * time.Hour)) + if days < 1 { + return 1 + } + return days +} + +func MileageQualityLimitKM(firstEventTime time.Time, latestEventTime time.Time) int { + // When the previous natural day is missing, the business baseline walks + // farther back. Scale the plausibility guard by that calendar-day gap while + // preserving the exact cumulative-odometer difference in the current row. + return maxSelectedDailyMileageKM * MileageQualityWindowDays(firstEventTime, latestEventTime) +} + +func ApplyMileageQualityRules(sample *SourceMileageSample) { + if sample == nil { + return + } + if sample.QualityStatus == "" { + sample.QualityStatus = QualityOK + } + if sample.QualityStatus != QualityOK { + return + } + normalized, ok, reason := NormalizeDailyMileageDeltaForWindow(sample.DailyKM, sample.FirstEventTime, sample.LatestEventTime) + sample.DailyKM = normalized + if reason != "" { + sample.QualityReason = reason + } + if !ok { + sample.QualityStatus = QualityInvalidDelta + } +} + func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageSample) error { if exec == nil { panic("stats execer must not be nil") @@ -79,6 +184,11 @@ func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageS if sample.QualityStatus == "" { sample.QualityStatus = QualityOK } + // MySQL DATETIME(0) may round 23:59:59.xxx into the next natural day. + // Truncate protocol event timestamps before persistence so day boundaries + // remain stable and match the TDengine event_time predicate. + sample.FirstEventTime = truncateMileageEventTime(sample.FirstEventTime) + sample.LatestEventTime = truncateMileageEventTime(sample.LatestEventTime) _, err := exec.ExecContext(ctx, upsertSourceMileageSQL, sample.VIN, sample.StatDate, @@ -101,6 +211,75 @@ func UpsertSourceMileage(ctx context.Context, exec Execer, sample SourceMileageS return err } +func truncateMileageEventTime(value time.Time) time.Time { + if value.IsZero() { + return value + } + return value.Truncate(time.Second) +} + +func NormalizePlatformSourceMileage(ctx context.Context, exec Execer, vin string, statDate string, protocol envelope.Protocol) error { + if exec == nil { + panic("stats execer must not be nil") + } + if strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" { + return nil + } + _, err := exec.ExecContext(ctx, normalizePlatformSourceMileageInsertSQL, vin, statDate, string(protocol)) + if err != nil { + return err + } + _, err = exec.ExecContext(ctx, normalizePlatformSourceMileageDeleteSQL, vin, statDate, string(protocol)) + return err +} + +func NormalizePlatformSourceMileageForDate(ctx context.Context, db interface { + Execer + Queryer +}, statDate string, protocol envelope.Protocol) (int, error) { + if db == nil { + panic("stats db must not be nil") + } + if strings.TrimSpace(statDate) == "" { + return 0, nil + } + rows, err := db.QueryContext(ctx, selectPlatformSourceMileageLegacyVINSQL, statDate, string(protocol)) + if err != nil { + return 0, err + } + defer rows.Close() + + var vins []string + for rows.Next() { + var vin string + if err := rows.Scan(&vin); err != nil { + return 0, err + } + if strings.TrimSpace(vin) != "" { + vins = append(vins, vin) + } + } + if err := rows.Err(); err != nil { + return 0, err + } + + normalized := 0 + for _, vin := range vins { + if err := NormalizePlatformSourceMileage(ctx, db, vin, statDate, protocol); err != nil { + return normalized, err + } + if err := ProjectDailyMileage(ctx, db, vin, statDate, protocol); err != nil { + return normalized, err + } + normalized++ + } + return normalized, nil +} + +func ShouldNormalizePlatformSourceMileage(sample SourceMileageSample) bool { + return strings.Contains(sample.SourceKey, platformSourceKeyPrefix) +} + func ProjectDailyMileage(ctx context.Context, exec Execer, vin string, statDate string, protocol envelope.Protocol) error { if exec == nil { panic("stats execer must not be nil") @@ -108,9 +287,25 @@ func ProjectDailyMileage(ctx context.Context, exec Execer, vin string, statDate if strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" { return nil } - if _, err := exec.ExecContext(ctx, clearSelectedSourceSQL, vin, statDate, string(protocol)); err != nil { - return err + if beginner, ok := exec.(txBeginner); ok { + tx, err := beginner.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := projectDailyMileageWithExec(ctx, tx, vin, statDate, protocol); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() } + return projectDailyMileageWithExec(ctx, exec, vin, statDate, protocol) +} + +type txBeginner interface { + BeginTx(context.Context, *sql.TxOptions) (*sql.Tx, error) +} + +func projectDailyMileageWithExec(ctx context.Context, exec Execer, vin string, statDate string, protocol envelope.Protocol) error { if _, err := exec.ExecContext(ctx, projectDailyMileageSQL, vin, statDate, @@ -142,6 +337,70 @@ func ProjectDailyMileage(ctx context.Context, exec Execer, vin string, statDate return err } +const upsertSourceMergedFirstTotalSQL = `CASE + WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0 + THEN VALUES(first_total_mileage_km) + WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time + THEN VALUES(first_total_mileage_km) + ELSE first_total_mileage_km + END` + +const upsertSourceMergedLatestTotalSQL = `CASE + WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0 + THEN VALUES(latest_total_mileage_km) + WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time + THEN VALUES(latest_total_mileage_km) + ELSE latest_total_mileage_km + END` + +const upsertSourceMergedFirstEventSQL = `CASE + WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time + THEN VALUES(first_event_time) + ELSE first_event_time + END` + +const upsertSourceMergedLatestEventSQL = `CASE + WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time + THEN VALUES(latest_event_time) + ELSE latest_event_time + END` + +const upsertSourceMergedDeltaSQL = `(` + upsertSourceMergedLatestTotalSQL + ` - ` + upsertSourceMergedFirstTotalSQL + `)` + +const upsertSourceMergedDailySQL = `CASE + WHEN ` + upsertSourceMergedDeltaSQL + ` < 0 + AND ` + upsertSourceMergedDeltaSQL + ` >= -` + maxNegativeMileageJitterKMSQL + ` + THEN 0 + ELSE ` + upsertSourceMergedDeltaSQL + ` + END` + +const upsertSourceMergedOutsideDailyRangeSQL = `(` + upsertSourceMergedDailySQL + ` < 0 OR ` + upsertSourceMergedDailySQL + ` > ` + upsertSourceMergedMaxMileageSQL + `)` + +const upsertSourceMergedMissingPreviousBaselineSQL = `(` + upsertSourceMergedFirstEventSQL + ` >= CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME))` + +const upsertSourceMergedQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(` + upsertSourceMergedLatestEventSQL + `), DATE(` + upsertSourceMergedFirstEventSQL + `)))` + +const upsertSourceMergedMaxMileageSQL = `(` + upsertSourceMergedQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)` + +const normalizePlatformFirstTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.first_total_mileage_km ORDER BY s.first_event_time ASC, s.updated_at ASC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))` + +const normalizePlatformLatestTotalSQL = `CAST(SUBSTRING_INDEX(GROUP_CONCAT(s.latest_total_mileage_km ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS DECIMAL(18,3))` + +const normalizePlatformDeltaSQL = `(` + normalizePlatformLatestTotalSQL + ` - ` + normalizePlatformFirstTotalSQL + `)` + +const normalizePlatformDailySQL = `CASE + WHEN ` + normalizePlatformDeltaSQL + ` < 0 + AND ` + normalizePlatformDeltaSQL + ` >= -` + maxNegativeMileageJitterKMSQL + ` + THEN 0 + ELSE ` + normalizePlatformDeltaSQL + ` + END` + +const normalizePlatformQualityWindowDaysSQL = `GREATEST(1, DATEDIFF(DATE(MAX(s.latest_event_time)), DATE(MIN(s.first_event_time))))` + +const normalizePlatformMaxMileageSQL = `(` + normalizePlatformQualityWindowDaysSQL + ` * ` + maxSelectedDailyMileageKMSQL + `)` + +const normalizePlatformOutsideDailyRangeSQL = `(` + normalizePlatformDailySQL + ` < 0 OR ` + normalizePlatformDailySQL + ` > ` + normalizePlatformMaxMileageSQL + `)` + const upsertSourceMileageSQL = ` INSERT INTO vehicle_daily_mileage_source (vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name, @@ -154,48 +413,186 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) phone = VALUES(phone), device_id = VALUES(device_id), platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name), - first_total_mileage_km = CASE - WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0 - THEN VALUES(first_total_mileage_km) - ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)) - END, - latest_total_mileage_km = CASE - WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0 - THEN VALUES(latest_total_mileage_km) - ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)) - END, - daily_mileage_km = GREATEST( - CASE - WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0 - THEN VALUES(latest_total_mileage_km) - ELSE latest_total_mileage_km - END, - VALUES(latest_total_mileage_km) - ) - CASE - WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0 - THEN VALUES(first_total_mileage_km) - ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)) - END, + first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `, + latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `, + daily_mileage_km = ` + upsertSourceMergedDailySQL + `, sample_count = sample_count + VALUES(sample_count), - first_event_time = CASE - WHEN first_event_time IS NULL OR VALUES(first_event_time) < first_event_time - THEN VALUES(first_event_time) - ELSE first_event_time + first_event_time = ` + upsertSourceMergedFirstEventSQL + `, + latest_event_time = ` + upsertSourceMergedLatestEventSQL + `, + quality_status = CASE + WHEN ` + upsertSourceMergedOutsideDailyRangeSQL + ` THEN '` + QualityInvalidDelta + `' + ELSE '` + QualityOK + `' END, - latest_event_time = CASE - WHEN latest_event_time IS NULL OR VALUES(latest_event_time) > latest_event_time - THEN VALUES(latest_event_time) - ELSE latest_event_time + quality_reason = CASE + WHEN ` + upsertSourceMergedOutsideDailyRangeSQL + ` THEN 'outside_daily_range' + WHEN ` + upsertSourceMergedDailySQL + ` = 0 AND ` + upsertSourceMergedDeltaSQL + ` < 0 THEN 'negative_jitter_clamped' + WHEN ` + upsertSourceMergedMissingPreviousBaselineSQL + ` THEN '` + QualityReasonCurrentDayFirst + `' + ELSE '` + QualityReasonHistorical + `' END, - quality_status = VALUES(quality_status), - quality_reason = VALUES(quality_reason), updated_at = CURRENT_TIMESTAMP ` -const clearSelectedSourceSQL = ` -UPDATE vehicle_daily_mileage_source -SET is_selected = 0 -WHERE vin = ? AND stat_date = ? AND protocol = ? +const normalizePlatformSourceMileageInsertSQL = ` +INSERT INTO vehicle_daily_mileage_source + (vin, stat_date, protocol, source_key, source_ip, source_endpoint, phone, device_id, platform_name, + first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count, + first_event_time, latest_event_time, quality_status, quality_reason) +SELECT + s.vin, + s.stat_date, + s.protocol, + CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code)) AS stable_source_key, + SUBSTRING_INDEX(GROUP_CONCAT(s.source_ip ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS source_ip, + SUBSTRING_INDEX(GROUP_CONCAT(COALESCE(s.source_endpoint, '') ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS source_endpoint, + SUBSTRING_INDEX(GROUP_CONCAT(COALESCE(s.phone, '') ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS phone, + SUBSTRING_INDEX(GROUP_CONCAT(COALESCE(s.device_id, '') ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1) AS device_id, + COALESCE(NULLIF(TRIM(SUBSTRING_INDEX(GROUP_CONCAT(COALESCE(NULLIF(TRIM(s.platform_name), ''), ds.platform_name, vi.platform_name, '') ORDER BY s.latest_event_time DESC, s.updated_at DESC SEPARATOR ','), ',', 1)), ''), MAX(TRIM(COALESCE(ds.platform_name, vi.platform_name)))) AS platform_name, + ` + normalizePlatformFirstTotalSQL + ` AS first_total_mileage_km, + ` + normalizePlatformLatestTotalSQL + ` AS latest_total_mileage_km, + ` + normalizePlatformDailySQL + ` AS daily_mileage_km, + SUM(s.sample_count) AS sample_count, + MIN(s.first_event_time) AS first_event_time, + MAX(s.latest_event_time) AS latest_event_time, + CASE + WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN '` + QualityInvalidDelta + `' + ELSE '` + QualityOK + `' + END AS quality_status, + CASE + WHEN ` + normalizePlatformOutsideDailyRangeSQL + ` THEN 'outside_daily_range' + WHEN ` + normalizePlatformDailySQL + ` = 0 AND ` + normalizePlatformDeltaSQL + ` < 0 THEN 'negative_jitter_clamped' + WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME) + THEN '` + QualityReasonHistorical + `' + ELSE '` + QualityReasonCurrentDayFirst + `' + END AS quality_reason +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds + ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip + AND ds.enabled = 1 + AND ds.source_kind = 'PLATFORM' + AND ds.source_code IS NOT NULL + AND TRIM(ds.source_code) <> '' +LEFT JOIN ( + SELECT + identifier_value AS phone, + MIN(TRIM(source_code)) AS source_code, + MIN(COALESCE(NULLIF(TRIM(oem), ''), NULLIF(TRIM(source_code), ''))) AS platform_name + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'JT808_PHONE' + AND enabled = 1 + AND source_code IS NOT NULL + AND TRIM(source_code) <> '' + GROUP BY identifier_value + HAVING COUNT(DISTINCT TRIM(source_code)) = 1 +) vi + ON s.protocol = 'JT808' AND vi.phone = TRIM(s.phone) +WHERE s.vin = ? + AND s.stat_date = ? + AND s.protocol = ? + AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `') + AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL + AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL + AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code)) +GROUP BY s.vin, s.stat_date, s.protocol, stable_source_key +ON DUPLICATE KEY UPDATE + source_ip = CASE + WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(source_ip) + ELSE source_ip + END, + source_endpoint = CASE + WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(source_endpoint) + ELSE source_endpoint + END, + phone = COALESCE(NULLIF(TRIM(VALUES(phone)), ''), phone), + device_id = COALESCE(NULLIF(TRIM(VALUES(device_id)), ''), device_id), + platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name), + first_total_mileage_km = ` + upsertSourceMergedFirstTotalSQL + `, + latest_total_mileage_km = ` + upsertSourceMergedLatestTotalSQL + `, + daily_mileage_km = ` + upsertSourceMergedDailySQL + `, + sample_count = sample_count + VALUES(sample_count), + first_event_time = CASE + WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time THEN VALUES(first_event_time) + ELSE first_event_time + END, + latest_event_time = CASE + WHEN latest_event_time IS NULL OR VALUES(latest_event_time) >= latest_event_time THEN VALUES(latest_event_time) + ELSE latest_event_time + END, + quality_status = CASE + WHEN ` + upsertSourceMergedOutsideDailyRangeSQL + ` THEN '` + QualityInvalidDelta + `' + ELSE '` + QualityOK + `' + END, + quality_reason = CASE + WHEN ` + upsertSourceMergedOutsideDailyRangeSQL + ` THEN 'outside_daily_range' + WHEN ` + upsertSourceMergedDailySQL + ` = 0 AND ` + upsertSourceMergedDeltaSQL + ` < 0 THEN 'negative_jitter_clamped' + WHEN ` + upsertSourceMergedMissingPreviousBaselineSQL + ` THEN '` + QualityReasonCurrentDayFirst + `' + ELSE '` + QualityReasonHistorical + `' + END, + updated_at = CURRENT_TIMESTAMP +` + +const normalizePlatformSourceMileageDeleteSQL = ` +DELETE s +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds + ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip + AND ds.enabled = 1 + AND ds.source_kind = 'PLATFORM' + AND ds.source_code IS NOT NULL + AND TRIM(ds.source_code) <> '' +LEFT JOIN ( + SELECT + identifier_value AS phone, + MIN(TRIM(source_code)) AS source_code + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'JT808_PHONE' + AND enabled = 1 + AND source_code IS NOT NULL + AND TRIM(source_code) <> '' + GROUP BY identifier_value + HAVING COUNT(DISTINCT TRIM(source_code)) = 1 +) vi + ON s.protocol = 'JT808' AND vi.phone = TRIM(s.phone) +WHERE s.vin = ? + AND s.stat_date = ? + AND s.protocol = ? + AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `') + AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL + AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL + AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code)) +` + +const selectPlatformSourceMileageLegacyVINSQL = ` +SELECT DISTINCT s.vin +FROM vehicle_daily_mileage_source s +LEFT JOIN vehicle_data_source ds + ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip + AND ds.enabled = 1 + AND ds.source_kind = 'PLATFORM' + AND ds.source_code IS NOT NULL + AND TRIM(ds.source_code) <> '' +LEFT JOIN ( + SELECT + identifier_value AS phone, + MIN(TRIM(source_code)) AS source_code + FROM vehicle_identifier + WHERE protocol = 'JT808' + AND identifier_type = 'JT808_PHONE' + AND enabled = 1 + AND source_code IS NOT NULL + AND TRIM(source_code) <> '' + GROUP BY identifier_value + HAVING COUNT(DISTINCT TRIM(source_code)) = 1 +) vi + ON s.protocol = 'JT808' AND vi.phone = TRIM(s.phone) +WHERE s.stat_date = ? + AND s.protocol = ? + AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `') + AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL + AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL + AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code)) +ORDER BY s.vin ` const projectDailyMileageSQL = ` @@ -209,15 +606,25 @@ SELECT s.daily_mileage_km, s.latest_total_mileage_km FROM vehicle_daily_mileage_source s -JOIN vehicle_data_source ds +LEFT JOIN vehicle_data_source ds ON ds.protocol = s.protocol AND ds.source_ip = s.source_ip WHERE s.vin = ? AND s.stat_date = ? AND s.protocol = ? AND s.quality_status = '` + QualityOK + `' - AND s.daily_mileage_km BETWEEN 0 AND ? - AND ds.enabled = 1 -ORDER BY ds.trust_priority, + AND s.daily_mileage_km BETWEEN 0 AND (? * GREATEST(1, DATEDIFF(DATE(s.latest_event_time), DATE(s.first_event_time)))) + AND (ds.id IS NULL OR ds.enabled = 1 OR ( + COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN' + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + )) +ORDER BY CASE COALESCE(NULLIF(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN'), CASE WHEN s.source_key LIKE '%` + platformSourceKeyPrefix + `%' THEN 'PLATFORM' WHEN s.source_key LIKE '%` + directSourceKeySuffix + `' THEN 'DIRECT' ELSE 'UNKNOWN' END) + WHEN 'PLATFORM' THEN 0 + WHEN 'DIRECT' THEN 1 + WHEN 'UNKNOWN' THEN 2 + ELSE 3 + END, + COALESCE(ds.trust_priority, 100000), s.sample_count DESC, s.latest_event_time DESC, s.source_key ASC @@ -231,22 +638,32 @@ ON DUPLICATE KEY UPDATE const markSelectedSourceSQL = ` UPDATE vehicle_daily_mileage_source s -JOIN ( +LEFT JOIN ( SELECT s2.source_key, s2.vin, s2.stat_date, s2.protocol FROM vehicle_daily_mileage_source s2 - JOIN vehicle_data_source ds + LEFT JOIN vehicle_data_source ds ON ds.protocol = s2.protocol AND ds.source_ip = s2.source_ip WHERE s2.vin = ? AND s2.stat_date = ? AND s2.protocol = ? AND s2.quality_status = '` + QualityOK + `' - AND s2.daily_mileage_km BETWEEN 0 AND ? - AND ds.enabled = 1 - ORDER BY ds.trust_priority, + AND s2.daily_mileage_km BETWEEN 0 AND (? * GREATEST(1, DATEDIFF(DATE(s2.latest_event_time), DATE(s2.first_event_time)))) + AND (ds.id IS NULL OR ds.enabled = 1 OR ( + COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN' + AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '') + AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '') + )) + ORDER BY CASE COALESCE(NULLIF(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN'), CASE WHEN s2.source_key LIKE '%` + platformSourceKeyPrefix + `%' THEN 'PLATFORM' WHEN s2.source_key LIKE '%` + directSourceKeySuffix + `' THEN 'DIRECT' ELSE 'UNKNOWN' END) + WHEN 'PLATFORM' THEN 0 + WHEN 'DIRECT' THEN 1 + WHEN 'UNKNOWN' THEN 2 + ELSE 3 + END, + COALESCE(ds.trust_priority, 100000), s2.sample_count DESC, s2.latest_event_time DESC, s2.source_key ASC @@ -256,7 +673,7 @@ JOIN ( AND selected_source.vin = s.vin AND selected_source.stat_date = s.stat_date AND selected_source.protocol = s.protocol -SET s.is_selected = 1 +SET s.is_selected = CASE WHEN selected_source.source_key IS NULL THEN 0 ELSE 1 END WHERE s.vin = ? AND s.stat_date = ? AND s.protocol = ? ` @@ -302,60 +719,30 @@ func lookupPreviousSourceBaseline(ctx context.Context, query Queryer, vin string return sourceBaseline{ LatestTotalKM: latestTotal.Float64, LatestEventTime: latestEvent.Time, - QualityReason: "historical_source_baseline", + QualityReason: QualityReasonHistorical, }, latestTotal.Valid, nil } -func lookupCurrentSourceBaseline(ctx context.Context, query Queryer, vin string, statDate string, protocol envelope.Protocol, sourceKey string) (sourceBaseline, bool, error) { - if query == nil || strings.TrimSpace(vin) == "" || strings.TrimSpace(statDate) == "" || strings.TrimSpace(sourceKey) == "" { - return sourceBaseline{}, false, nil - } - rows, err := query.QueryContext(ctx, currentSourceBaselineSQL, vin, statDate, string(protocol), sourceKey) - if err != nil { - return sourceBaseline{}, false, err - } - defer rows.Close() - if !rows.Next() { - if err := rows.Err(); err != nil { - return sourceBaseline{}, false, err - } - return sourceBaseline{}, false, nil - } - var firstTotal sql.NullFloat64 - var firstEvent sql.NullTime - if err := rows.Scan(&firstTotal, &firstEvent); err != nil { - return sourceBaseline{}, false, err - } - if err := rows.Err(); err != nil { - return sourceBaseline{}, false, err - } - return sourceBaseline{ - LatestTotalKM: firstTotal.Float64, - LatestEventTime: firstEvent.Time, - QualityReason: "current_day_first_sample", - }, firstTotal.Valid, nil +// LookupLatestSourceBaselineBefore returns the nearest durable odometer for the +// same VIN, protocol and source before statDate. Missing calendar days are +// skipped automatically. +func LookupLatestSourceBaselineBefore(ctx context.Context, query Queryer, vin string, statDate string, protocol envelope.Protocol, sourceKey string) (float64, time.Time, bool, error) { + baseline, found, err := lookupPreviousSourceBaseline(ctx, query, vin, statDate, protocol, sourceKey) + return baseline.LatestTotalKM, baseline.LatestEventTime, found, err } const previousSourceBaselineSQL = ` SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source WHERE vin = ? - AND stat_date < ? + AND stat_date < ? AND protocol = ? AND source_key = ? - AND quality_status = '` + QualityOK + `' + AND latest_total_mileage_km IS NOT NULL + AND latest_total_mileage_km > 0 + AND latest_event_time IS NOT NULL + AND latest_event_time >= CAST(CONCAT(stat_date, ' 00:00:00') AS DATETIME) + AND latest_event_time < DATE_ADD(CAST(CONCAT(stat_date, ' 00:00:00') AS DATETIME), INTERVAL 1 DAY) ORDER BY stat_date DESC, latest_event_time DESC LIMIT 1 ` - -const currentSourceBaselineSQL = ` -SELECT first_total_mileage_km, first_event_time -FROM vehicle_daily_mileage_source -WHERE vin = ? - AND stat_date = ? - AND protocol = ? - AND source_key = ? - AND quality_status = '` + QualityOK + `' -ORDER BY latest_event_time DESC -LIMIT 1 -` diff --git a/go/vehicle-gateway/internal/stats/source_mileage_test.go b/go/vehicle-gateway/internal/stats/source_mileage_test.go index 54747e9d..db1882f1 100644 --- a/go/vehicle-gateway/internal/stats/source_mileage_test.go +++ b/go/vehicle-gateway/internal/stats/source_mileage_test.go @@ -2,10 +2,13 @@ package stats import ( "context" + "errors" "strings" "testing" "time" + "github.com/DATA-DOG/go-sqlmock" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) @@ -21,6 +24,85 @@ func TestSourceKeyUsesProtocolDeviceAndSourceIP(t *testing.T) { } } +func TestSourceKeyForDirectUsesStableDeviceIdentity(t *testing.T) { + first := SourceKeyForKind(envelope.ProtocolJT808, "13307765812", "", "115.231.168.135", "DIRECT") + second := SourceKeyForKind(envelope.ProtocolJT808, "13307765812", "", "39.144.3.22", "DIRECT") + if first != "JT808:13307765812@DIRECT" || second != first { + t.Fatalf("direct source keys = %q/%q, want stable phone key", first, second) + } + + fallback := SourceKeyForKind(envelope.ProtocolJT808, "", "", "39.144.3.22", "DIRECT") + if fallback != "JT808:unknown@39.144.3.22" { + t.Fatalf("direct fallback source key = %q", fallback) + } +} + +func TestSourceKeyForPlatformUsesStableSourceCode(t *testing.T) { + first := SourceKeyForSource(envelope.ProtocolJT808, "41456413943", "", "117.132.194.167", "PLATFORM", "guangan_beidou") + second := SourceKeyForSource(envelope.ProtocolJT808, "41456413943", "", "117.132.198.90", "PLATFORM", "guangan_beidou") + if first != "JT808:41456413943@PLATFORM:guangan_beidou" || second != first { + t.Fatalf("platform source keys = %q/%q, want stable source_code key", first, second) + } + + withoutSourceCode := SourceKeyForSource(envelope.ProtocolJT808, "41456413943", "", "117.132.198.90", "PLATFORM", "") + if withoutSourceCode != "JT808:41456413943@117.132.198.90" { + t.Fatalf("platform fallback source key = %q", withoutSourceCode) + } + + withoutDeviceIdentity := SourceKeyForSource(envelope.ProtocolJT808, "", "", "117.132.198.90", "PLATFORM", "guangan_beidou") + if withoutDeviceIdentity != "JT808:unknown@117.132.198.90" { + t.Fatalf("unknown platform source key = %q", withoutDeviceIdentity) + } +} + +func TestSourceMileageSampleFromMetricUsesDirectSourceKey(t *testing.T) { + sample := MetricSample{ + VIN: "LA9GG64L7PBAF4001", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + Phone: "13307765812", + TotalMileageKM: 4123.9, + EventTime: time.Date(2026, 7, 12, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), + } + candidate := SourceMileageSampleFromMetric(sample, SourceIdentity{ + Protocol: envelope.ProtocolJT808, + SourceIP: "115.231.168.135", + SourceEndpoint: "115.231.168.135:43625", + SourceKind: "DIRECT", + }) + if candidate.SourceKey != "JT808:13307765812@DIRECT" { + t.Fatalf("source key = %q", candidate.SourceKey) + } + if candidate.SourceIP != "115.231.168.135" { + t.Fatalf("source ip = %q", candidate.SourceIP) + } +} + +func TestSourceMileageSampleFromMetricUsesPlatformSourceCode(t *testing.T) { + sample := MetricSample{ + VIN: "LNXNEGRR0SR321372", + StatDate: "2026-07-12", + Protocol: envelope.ProtocolJT808, + Phone: "41456413943", + TotalMileageKM: 46484.2, + EventTime: time.Date(2026, 7, 12, 8, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), + } + candidate := SourceMileageSampleFromMetric(sample, SourceIdentity{ + Protocol: envelope.ProtocolJT808, + SourceIP: "117.132.194.167", + SourceEndpoint: "117.132.194.167:9806", + SourceCode: "guangan_beidou", + PlatformName: "广安北斗", + SourceKind: "PLATFORM", + }) + if candidate.SourceKey != "JT808:41456413943@PLATFORM:guangan_beidou" { + t.Fatalf("source key = %q", candidate.SourceKey) + } + if candidate.PlatformName != "广安北斗" { + t.Fatalf("platform name = %q", candidate.PlatformName) + } +} + func TestUpsertSourceMileageWritesCandidateRow(t *testing.T) { exec := &recordingExec{} sample := SourceMileageSample{ @@ -50,8 +132,13 @@ func TestUpsertSourceMileageWritesCandidateRow(t *testing.T) { for _, want := range []string{ "INSERT INTO vehicle_daily_mileage_source", "ON DUPLICATE KEY UPDATE", - "daily_mileage_km = GREATEST(", - "quality_status = VALUES(quality_status)", + "daily_mileage_km = CASE", + "VALUES(latest_event_time) >= latest_event_time", + "VALUES(first_event_time) <= first_event_time", + "quality_status = CASE", + "THEN '" + QualityInvalidDelta + "'", + "quality_reason = CASE", + "THEN 'outside_daily_range'", "platform_name = COALESCE(NULLIF(TRIM(VALUES(platform_name)), ''), platform_name)", } { if !strings.Contains(sql, want) { @@ -63,6 +150,114 @@ func TestUpsertSourceMileageWritesCandidateRow(t *testing.T) { } } +func TestUpsertSourceMileageTruncatesSubsecondEventTimesAtDayBoundary(t *testing.T) { + exec := &recordingExec{} + loc := time.FixedZone("Asia/Shanghai", 8*3600) + previous := time.Date(2026, 7, 12, 23, 59, 59, 999_000_000, loc) + current := time.Date(2026, 7, 13, 0, 0, 1, 999_000_000, loc) + sample := SourceMileageSample{ + VIN: "LMRKH9AC7R1004098", + StatDate: "2026-07-13", + Protocol: envelope.ProtocolYutongMQTT, + SourceKey: "YUTONG_MQTT:LMRKH9AC7R1004098@PLATFORM:yutong", + SourceIP: "mqtt", + FirstTotalKM: 41249, + LatestTotalKM: 41250, + DailyKM: 1, + SampleCount: 1, + FirstEventTime: previous, + LatestEventTime: current, + QualityStatus: QualityOK, + QualityReason: "historical_source_baseline", + } + if err := UpsertSourceMileage(context.Background(), exec, sample); err != nil { + t.Fatalf("UpsertSourceMileage() error = %v", err) + } + if len(exec.calls) != 1 { + t.Fatalf("exec calls = %d, want 1", len(exec.calls)) + } + if got := exec.calls[0].args[13]; got != previous.Truncate(time.Second) { + t.Fatalf("first event arg = %v, want %v", got, previous.Truncate(time.Second)) + } + if got := exec.calls[0].args[14]; got != current.Truncate(time.Second) { + t.Fatalf("latest event arg = %v, want %v", got, current.Truncate(time.Second)) + } +} + +func TestUpsertSourceMileageUsesEventTimeBoundaries(t *testing.T) { + for _, want := range []string{ + "first_total_mileage_km = CASE", + "VALUES(first_event_time) <= first_event_time", + "latest_total_mileage_km = CASE", + "VALUES(latest_event_time) >= latest_event_time", + "daily_mileage_km = CASE", + "VALUES(latest_total_mileage_km)", + "VALUES(first_total_mileage_km)", + } { + if !strings.Contains(upsertSourceMileageSQL, want) { + t.Fatalf("event-time boundary upsert SQL missing %q:\n%s", want, upsertSourceMileageSQL) + } + } + for _, forbidden := range []string{ + "GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))", + "LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))", + "current_day_fallback_after_invalid_baseline", + } { + if strings.Contains(upsertSourceMileageSQL, forbidden) { + t.Fatalf("event-time boundary upsert must not contain %q:\n%s", forbidden, upsertSourceMileageSQL) + } + } +} + +func TestUpsertSourceMileageRechecksMergedDailyRange(t *testing.T) { + if maxSelectedDailyMileageKMSQL != "2500" || maxNegativeMileageJitterKMSQL != "1" { + t.Fatalf("SQL mileage limits drifted from Go quality constants: selected=%s jitter=%s", maxSelectedDailyMileageKMSQL, maxNegativeMileageJitterKMSQL) + } + for _, want := range []string{ + "quality_status = CASE", + ">= -" + maxNegativeMileageJitterKMSQL, + "DATEDIFF(DATE(", + "* " + maxSelectedDailyMileageKMSQL, + "THEN '" + QualityInvalidDelta + "'", + "quality_reason = CASE", + "THEN 'outside_daily_range'", + "THEN '" + QualityReasonCurrentDayFirst + "'", + "ELSE '" + QualityReasonHistorical + "'", + } { + if !strings.Contains(upsertSourceMileageSQL, want) { + t.Fatalf("merged range guard missing %q:\n%s", want, upsertSourceMileageSQL) + } + } +} + +func TestPreviousSourceBaselineUsesLatestEarlierCalendarDay(t *testing.T) { + if !strings.Contains(previousSourceBaselineSQL, "stat_date < ?") { + t.Fatalf("previous baseline should search earlier calendar days:\n%s", previousSourceBaselineSQL) + } + if !strings.Contains(previousSourceBaselineSQL, "ORDER BY stat_date DESC, latest_event_time DESC") { + t.Fatalf("previous baseline should prefer the nearest earlier sample:\n%s", previousSourceBaselineSQL) + } + if strings.Contains(previousSourceBaselineSQL, "quality_status =") { + t.Fatalf("previous day's last odometer must remain usable even when that day's delta has no baseline:\n%s", previousSourceBaselineSQL) + } + for _, want := range []string{ + "latest_total_mileage_km > 0", + "latest_event_time >= CAST(CONCAT(stat_date, ' 00:00:00') AS DATETIME)", + "latest_event_time < DATE_ADD(CAST(CONCAT(stat_date, ' 00:00:00') AS DATETIME), INTERVAL 1 DAY)", + } { + if !strings.Contains(previousSourceBaselineSQL, want) { + t.Fatalf("previous baseline SQL missing %q:\n%s", want, previousSourceBaselineSQL) + } + } +} + +func TestDailyMileageFromDayBoundaryUsesCurrentMinusPrevious(t *testing.T) { + got := DailyMileageFromDayBoundary(14989.0, 15369.0) + if got != 380.0 { + t.Fatalf("daily mileage = %v, want current latest minus previous last = 380", got) + } +} + func TestUpsertSourceMileageSkipsBlankSourceIP(t *testing.T) { exec := &recordingExec{} sample := SourceMileageSample{ @@ -82,30 +277,227 @@ func TestUpsertSourceMileageSkipsBlankSourceIP(t *testing.T) { } } +func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) { + exec := &recordingExec{} + err := NormalizePlatformSourceMileage(context.Background(), exec, "LNXNEGRR0SR321372", "2026-07-12", envelope.ProtocolJT808) + if err != nil { + t.Fatalf("NormalizePlatformSourceMileage() error = %v", err) + } + if len(exec.calls) != 2 { + t.Fatalf("exec calls = %d, want insert merge + delete legacy", len(exec.calls)) + } + insertSQL := exec.calls[0].query + for _, want := range []string{ + "INSERT INTO vehicle_daily_mileage_source", + "CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '@PLATFORM:', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))", + "LEFT JOIN vehicle_data_source ds", + "LEFT JOIN (", + "FROM vehicle_identifier", + "identifier_type = 'JT808_PHONE'", + "HAVING COUNT(DISTINCT TRIM(source_code)) = 1", + "ds.source_kind = 'PLATFORM'", + "MAX(TRIM(COALESCE(ds.platform_name, vi.platform_name)))", + "s.source_key <> CONCAT", + "GROUP BY s.vin, s.stat_date, s.protocol, stable_source_key", + "ON DUPLICATE KEY UPDATE", + "sample_count = sample_count + VALUES(sample_count)", + "s.quality_status IN ('" + QualityOK + "', '" + QualityNoPreviousBaseline + "')", + "WHEN MIN(s.first_event_time) < CAST(CONCAT(s.stat_date, ' 00:00:00') AS DATETIME)", + "THEN '" + QualityInvalidDelta + "'", + "ELSE '" + QualityOK + "'", + "THEN 'negative_jitter_clamped'", + "THEN '" + QualityReasonHistorical + "'", + "ELSE '" + QualityReasonCurrentDayFirst + "'", + upsertSourceMergedMissingPreviousBaselineSQL, + } { + if !strings.Contains(insertSQL, want) { + t.Fatalf("normalize insert SQL missing %q:\n%s", want, insertSQL) + } + } + deleteSQL := exec.calls[1].query + for _, want := range []string{ + "DELETE s", + "FROM vehicle_daily_mileage_source s", + "LEFT JOIN vehicle_data_source ds", + "FROM vehicle_identifier", + "ds.source_kind = 'PLATFORM'", + "s.source_key <> CONCAT", + } { + if !strings.Contains(deleteSQL, want) { + t.Fatalf("normalize delete SQL missing %q:\n%s", want, deleteSQL) + } + } + for i, call := range exec.calls { + if len(call.args) != 3 || call.args[0] != "LNXNEGRR0SR321372" || call.args[1] != "2026-07-12" || call.args[2] != "JT808" { + t.Fatalf("call %d args = %#v", i, call.args) + } + } +} + +func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + statDate := "2026-07-12" + protocol := envelope.ProtocolJT808 + rows := sqlmock.NewRows([]string{"vin"}). + AddRow("LA9HE60A0PBAF4002"). + AddRow("LA9HE60A1PBAF4008") + mock.ExpectQuery(selectPlatformSourceMileageLegacyVINSQL). + WithArgs(statDate, string(protocol)). + WillReturnRows(rows) + for _, vin := range []string{"LA9HE60A0PBAF4002", "LA9HE60A1PBAF4008"} { + mock.ExpectExec(normalizePlatformSourceMileageInsertSQL). + WithArgs(vin, statDate, string(protocol)). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(normalizePlatformSourceMileageDeleteSQL). + WithArgs(vin, statDate, string(protocol)). + WillReturnResult(sqlmock.NewResult(0, 1)) + expectProjectDailyMileageSQL(mock, vin, statDate, protocol) + mock.ExpectCommit() + } + + normalized, err := NormalizePlatformSourceMileageForDate(context.Background(), db, statDate, protocol) + if err != nil { + t.Fatalf("NormalizePlatformSourceMileageForDate() error = %v", err) + } + if normalized != 2 { + t.Fatalf("normalized = %d, want 2", normalized) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations were not met: %v", err) + } +} + +func TestShouldNormalizePlatformSourceMileage(t *testing.T) { + if !ShouldNormalizePlatformSourceMileage(SourceMileageSample{SourceKey: "JT808:41456413943@PLATFORM:guangan_beidou"}) { + t.Fatal("platform source key should trigger legacy key normalization") + } + for _, sourceKey := range []string{"JT808:41456413943@DIRECT", "JT808:41456413943@117.132.194.167", ""} { + if ShouldNormalizePlatformSourceMileage(SourceMileageSample{SourceKey: sourceKey}) { + t.Fatalf("source key %q should not trigger platform normalization", sourceKey) + } + } +} + +func TestApplyMileageQualityRulesClampsSmallNegativeJitter(t *testing.T) { + sample := SourceMileageSample{ + DailyKM: -0.1, + QualityStatus: QualityOK, + QualityReason: "historical_source_baseline", + } + ApplyMileageQualityRules(&sample) + if sample.DailyKM != 0 { + t.Fatalf("daily km = %v, want 0", sample.DailyKM) + } + if sample.QualityStatus != QualityOK || sample.QualityReason != "negative_jitter_clamped" { + t.Fatalf("quality = %s/%s", sample.QualityStatus, sample.QualityReason) + } +} + +func TestApplyMileageQualityRulesRejectsLargeNegativeDelta(t *testing.T) { + sample := SourceMileageSample{ + DailyKM: -55, + QualityStatus: QualityOK, + QualityReason: "historical_source_baseline", + } + ApplyMileageQualityRules(&sample) + if sample.DailyKM != -55 { + t.Fatalf("daily km = %v, want original invalid delta", sample.DailyKM) + } + if sample.QualityStatus != QualityInvalidDelta || sample.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", sample.QualityStatus, sample.QualityReason) + } +} + +func TestApplyMileageQualityRulesAcceptsPlausibleMultiDayFallbackDelta(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + sample := SourceMileageSample{ + DailyKM: 7833.5, + FirstEventTime: time.Date(2026, 7, 4, 11, 7, 58, 0, loc), + LatestEventTime: time.Date(2026, 7, 12, 2, 52, 47, 0, loc), + QualityStatus: QualityOK, + QualityReason: "historical_source_baseline", + } + ApplyMileageQualityRules(&sample) + if sample.QualityStatus != QualityOK || sample.QualityReason != "historical_source_baseline" { + t.Fatalf("quality = %s/%s", sample.QualityStatus, sample.QualityReason) + } + if sample.DailyKM != 7833.5 { + t.Fatalf("daily km = %v", sample.DailyKM) + } + if got := MileageQualityWindowDays(sample.FirstEventTime, sample.LatestEventTime); got != 8 { + t.Fatalf("window days = %d, want 8", got) + } + if got := MileageQualityLimitKM(sample.FirstEventTime, sample.LatestEventTime); got != 8*maxSelectedDailyMileageKM { + t.Fatalf("quality limit = %d, want 8-day limit %d", got, 8*maxSelectedDailyMileageKM) + } +} + +func TestApplyMileageQualityRulesAcceptsContinuousHighUtilizationDay(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + sample := SourceMileageSample{ + DailyKM: 1895.7, + FirstEventTime: time.Date(2026, 7, 12, 23, 59, 37, 0, loc), + LatestEventTime: time.Date(2026, 7, 13, 20, 32, 42, 0, loc), + QualityStatus: QualityOK, + QualityReason: QualityReasonHistorical, + } + ApplyMileageQualityRules(&sample) + if sample.QualityStatus != QualityOK || sample.QualityReason != QualityReasonHistorical { + t.Fatalf("quality = %s/%s", sample.QualityStatus, sample.QualityReason) + } + if got := MileageQualityLimitKM(sample.FirstEventTime, sample.LatestEventTime); got != 2500 { + t.Fatalf("quality limit = %d, want 2500", got) + } +} + +func TestApplyMileageQualityRulesRejectsImplausibleSingleDayJump(t *testing.T) { + loc := time.FixedZone("Asia/Shanghai", 8*3600) + sample := SourceMileageSample{ + DailyKM: 4000, + FirstEventTime: time.Date(2026, 7, 12, 23, 59, 37, 0, loc), + LatestEventTime: time.Date(2026, 7, 13, 20, 32, 42, 0, loc), + QualityStatus: QualityOK, + QualityReason: QualityReasonHistorical, + } + ApplyMileageQualityRules(&sample) + if sample.QualityStatus != QualityInvalidDelta || sample.QualityReason != "outside_daily_range" { + t.Fatalf("quality = %s/%s", sample.QualityStatus, sample.QualityReason) + } +} + func TestProjectDailyMileageSelectsCandidateAndMarksSource(t *testing.T) { exec := &recordingExec{} err := ProjectDailyMileage(context.Background(), exec, "LA9GG64L7PBAF4001", "2026-07-08", envelope.ProtocolJT808) if err != nil { t.Fatalf("ProjectDailyMileage() error = %v", err) } - if len(exec.calls) != 4 { - t.Fatalf("exec calls = %d, want 4", len(exec.calls)) - } - updateSources := exec.calls[0].query - projectFinal := exec.calls[1].query - markSelected := exec.calls[2].query - cleanupFinal := exec.calls[3].query - if !strings.Contains(updateSources, "UPDATE vehicle_daily_mileage_source") || !strings.Contains(updateSources, "is_selected = 0") { - t.Fatalf("first query should clear selected candidates: %s", updateSources) + if len(exec.calls) != 3 { + t.Fatalf("exec calls = %d, want 3", len(exec.calls)) } + projectFinal := exec.calls[0].query + markSelected := exec.calls[1].query + cleanupFinal := exec.calls[2].query for _, want := range []string{ "INSERT INTO vehicle_daily_mileage", "FROM vehicle_daily_mileage_source s", - "JOIN vehicle_data_source ds", + "LEFT JOIN vehicle_data_source ds", "ds.id", - "ORDER BY ds.trust_priority", + "CASE COALESCE(NULLIF(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN'), CASE WHEN s.source_key LIKE '%@PLATFORM:%' THEN 'PLATFORM' WHEN s.source_key LIKE '%@DIRECT' THEN 'DIRECT' ELSE 'UNKNOWN' END)", + "WHEN 'PLATFORM' THEN 0", + "WHEN 'DIRECT' THEN 1", + "WHEN 'UNKNOWN' THEN 2", + "ds.trust_priority", "s.quality_status = '" + QualityOK + "'", + "COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'", + "AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')", + "AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')", "s.daily_mileage_km BETWEEN 0 AND", + "DATEDIFF(DATE(s.latest_event_time), DATE(s.first_event_time))", } { if !strings.Contains(projectFinal, want) { t.Fatalf("project query missing %q: %s", want, projectFinal) @@ -123,8 +515,10 @@ func TestProjectDailyMileageSelectsCandidateAndMarksSource(t *testing.T) { t.Fatalf("project query should not use final-table field %q: %s", forbidden, projectFinal) } } - if !strings.Contains(markSelected, "UPDATE vehicle_daily_mileage_source s") || !strings.Contains(markSelected, "SET s.is_selected = 1") { - t.Fatalf("mark query should flag the elected source: %s", markSelected) + if !strings.Contains(markSelected, "UPDATE vehicle_daily_mileage_source s") || + !strings.Contains(markSelected, "LEFT JOIN (") || + !strings.Contains(markSelected, "SET s.is_selected = CASE WHEN selected_source.source_key IS NULL THEN 0 ELSE 1 END") { + t.Fatalf("mark query should reconcile the elected source: %s", markSelected) } if strings.Contains(markSelected, "JOIN vehicle_daily_mileage m") { t.Fatalf("mark query should not rejoin final mileage for candidate selection: %s", markSelected) @@ -135,18 +529,33 @@ func TestProjectDailyMileageSelectsCandidateAndMarksSource(t *testing.T) { "s2.source_key", "s2.quality_status = '" + QualityOK + "'", "s2.daily_mileage_km BETWEEN 0 AND", - "ds.enabled = 1", - "ORDER BY ds.trust_priority", + "DATEDIFF(DATE(s2.latest_event_time), DATE(s2.first_event_time))", + "COALESCE(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN') = 'UNKNOWN'", + "AND (ds.source_code IS NULL OR TRIM(ds.source_code) = '')", + "AND (ds.platform_name IS NULL OR TRIM(ds.platform_name) = '')", + "CASE COALESCE(NULLIF(NULLIF(TRIM(ds.source_kind), ''), 'UNKNOWN'), CASE WHEN s2.source_key LIKE '%@PLATFORM:%' THEN 'PLATFORM' WHEN s2.source_key LIKE '%@DIRECT' THEN 'DIRECT' ELSE 'UNKNOWN' END)", + "WHEN 'PLATFORM' THEN 0", + "WHEN 'DIRECT' THEN 1", + "WHEN 'UNKNOWN' THEN 2", + "ds.trust_priority", "LIMIT 1", } { if !strings.Contains(markSelected, want) { t.Fatalf("mark query missing %q: %s", want, markSelected) } } - if len(exec.calls[2].args) != 7 { - t.Fatalf("mark query args = %d, want 7", len(exec.calls[2].args)) + if len(exec.calls[1].args) != 7 { + t.Fatalf("mark query args = %d, want 7", len(exec.calls[1].args)) } - if got := exec.calls[2].args[3]; got != maxSelectedDailyMileageKM { + for _, forbidden := range []string{ + "WHEN 'UNKNOWN' THEN 1", + "WHEN 'DIRECT' THEN 2", + } { + if strings.Contains(projectFinal, forbidden) || strings.Contains(markSelected, forbidden) { + t.Fatalf("source selection should prefer classified DIRECT before UNKNOWN, found %q", forbidden) + } + } + if got := exec.calls[1].args[3]; got != maxSelectedDailyMileageKM { t.Fatalf("mark query max mileage arg = %#v, want %d", got, maxSelectedDailyMileageKM) } for _, want := range []string{ @@ -159,7 +568,70 @@ func TestProjectDailyMileageSelectsCandidateAndMarksSource(t *testing.T) { t.Fatalf("cleanup query missing %q: %s", want, cleanupFinal) } } - if len(exec.calls[3].args) != 6 { - t.Fatalf("cleanup query args = %d, want 6", len(exec.calls[3].args)) + if len(exec.calls[2].args) != 6 { + t.Fatalf("cleanup query args = %d, want 6", len(exec.calls[2].args)) } } + +func TestProjectDailyMileageCommitsTransactionForSQLDB(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + vin := "LA9GG64L7PBAF4001" + statDate := "2026-07-08" + protocol := envelope.ProtocolJT808 + expectProjectDailyMileageSQL(mock, vin, statDate, protocol) + mock.ExpectCommit() + + if err := ProjectDailyMileage(context.Background(), db, vin, statDate, protocol); err != nil { + t.Fatalf("ProjectDailyMileage() error = %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations were not met: %v", err) + } +} + +func TestProjectDailyMileageRollsBackTransactionOnFailure(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + vin := "LA9GG64L7PBAF4001" + statDate := "2026-07-08" + protocol := envelope.ProtocolJT808 + wantErr := errors.New("mark selected failed") + mock.ExpectBegin() + mock.ExpectExec(projectDailyMileageSQL). + WithArgs(vin, statDate, string(protocol), maxSelectedDailyMileageKM). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(markSelectedSourceSQL). + WithArgs(vin, statDate, string(protocol), maxSelectedDailyMileageKM, vin, statDate, string(protocol)). + WillReturnError(wantErr) + mock.ExpectRollback() + + err = ProjectDailyMileage(context.Background(), db, vin, statDate, protocol) + if !errors.Is(err, wantErr) { + t.Fatalf("ProjectDailyMileage() error = %v, want %v", err, wantErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations were not met: %v", err) + } +} + +func expectProjectDailyMileageSQL(mock sqlmock.Sqlmock, vin string, statDate string, protocol envelope.Protocol) { + mock.ExpectBegin() + mock.ExpectExec(projectDailyMileageSQL). + WithArgs(vin, statDate, string(protocol), maxSelectedDailyMileageKM). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(markSelectedSourceSQL). + WithArgs(vin, statDate, string(protocol), maxSelectedDailyMileageKM, vin, statDate, string(protocol)). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(cleanupProjectedDailyMileageSQL). + WithArgs(vin, statDate, string(protocol), vin, statDate, string(protocol)). + WillReturnResult(sqlmock.NewResult(0, 1)) +} diff --git a/go/vehicle-gateway/internal/stats/source_test.go b/go/vehicle-gateway/internal/stats/source_test.go index 1a06742f..72ac14fb 100644 --- a/go/vehicle-gateway/internal/stats/source_test.go +++ b/go/vehicle-gateway/internal/stats/source_test.go @@ -11,10 +11,12 @@ import ( func TestNormalizeSourceIPDropsPort(t *testing.T) { tests := map[string]string{ - "115.231.168.135:20215": "115.231.168.135", - "115.231.168.135": "115.231.168.135", - " 115.159.85.149:28316 ": "115.159.85.149", - "": "", + "115.231.168.135:20215": "115.231.168.135", + "115.231.168.135": "115.231.168.135", + " 115.159.85.149:28316 ": "115.159.85.149", + "mqtt://yutong/ytforward/shln/3": "mqtt", + "MQTT://YUTONG/topic": "mqtt", + "": "", } for input, want := range tests { if got := NormalizeSourceIP(input); got != want { @@ -43,6 +45,60 @@ func TestNewSourceIdentityRequiresSourceIP(t *testing.T) { } } +func TestShouldManageDataSourceRequiresPlatformEvidence(t *testing.T) { + if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135"}) { + t.Fatal("unclassified source should not be managed") + } + if !ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "g7s"}) { + t.Fatal("source_code should make source manageable") + } + if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceKind: "DIRECT"}) { + t.Fatal("direct source without platform evidence should not be auto-managed by source IP") + } + if !ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceKind: "PLATFORM"}) { + t.Fatal("explicit platform source_kind should make source manageable") + } + if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceCode: "g7s"}) { + t.Fatal("empty source ip should not be managed") + } +} + +func TestSourceKindForDataSourceWriteInfersPlatformEvidence(t *testing.T) { + tests := []struct { + name string + identity SourceIdentity + want string + }{ + { + name: "empty evidence", + identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135"}, + want: "UNKNOWN", + }, + { + name: "source code", + identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "g7s"}, + want: "PLATFORM", + }, + { + name: "platform name", + identity: SourceIdentity{Protocol: envelope.ProtocolGB32960, SourceIP: "8.134.95.166", PlatformName: "Hyundai"}, + want: "PLATFORM", + }, + { + name: "explicit direct wins", + identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "direct-import", SourceKind: "DIRECT"}, + want: "DIRECT", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sourceKindForDataSourceWrite(tt.identity); got != tt.want { + t.Fatalf("sourceKindForDataSourceWrite() = %q, want %q", got, tt.want) + } + }) + } +} + func TestUpsertDataSourcePreservesManualFields(t *testing.T) { exec := &recordingExec{} identity := SourceIdentity{ @@ -60,7 +116,7 @@ func TestUpsertDataSourcePreservesManualFields(t *testing.T) { for _, want := range []string{ "INSERT INTO vehicle_data_source", "latest_source_endpoint = VALUES(latest_source_endpoint)", - "latest_seen_at = VALUES(latest_seen_at)", + "latest_seen_at = GREATEST", } { if !strings.Contains(sql, want) { t.Fatalf("source upsert missing %q: %s", want, sql) @@ -77,3 +133,103 @@ func TestUpsertDataSourcePreservesManualFields(t *testing.T) { } } } + +func TestUpsertDataSourceWritesPlatformKindWhenEvidenceExists(t *testing.T) { + exec := &recordingExec{} + identity := SourceIdentity{ + Protocol: envelope.ProtocolGB32960, + SourceIP: "8.134.95.166", + SourceEndpoint: "8.134.95.166:32960", + SourceCode: "Hyundai", + PlatformName: "现代 HTWO", + } + if err := UpsertDataSource(context.Background(), exec, identity, time.Date(2026, 7, 12, 18, 0, 0, 0, time.UTC)); err != nil { + t.Fatalf("UpsertDataSource() error = %v", err) + } + if len(exec.calls) != 1 { + t.Fatalf("exec calls = %d", len(exec.calls)) + } + if got, want := exec.calls[0].args[5], "PLATFORM"; got != want { + t.Fatalf("source_kind arg = %#v, want %q", got, want) + } +} + +func TestUpsertDataSourceCanReenableAutoRetiredSourceWithEvidence(t *testing.T) { + exec := &recordingExec{} + identity := SourceIdentity{ + Protocol: envelope.ProtocolJT808, + SourceIP: "115.231.168.135", + SourceEndpoint: "115.231.168.135:20215", + SourceCode: "g7s", + PlatformName: "G7s", + } + if err := UpsertDataSource(context.Background(), exec, identity, time.Date(2026, 7, 12, 18, 0, 0, 0, time.UTC)); err != nil { + t.Fatalf("UpsertDataSource() error = %v", err) + } + sql := exec.calls[0].query + for _, want := range []string{ + "vehicle_data_source.enabled = 0", + "vehicle_data_source.remark LIKE 'auto-retired:%'", + "vehicle_data_source.remark = 'auto-reenabled: source evidence restored'", + "THEN 'auto-reenabled: source evidence restored'", + "THEN 1", + } { + if !strings.Contains(sql, want) { + t.Fatalf("source upsert should re-enable auto-retired source with evidence; missing %q:\n%s", want, sql) + } + } + if strings.Contains(sql, "enabled = VALUES(enabled)") { + t.Fatalf("source upsert should not blindly copy enabled from values:\n%s", sql) + } + if strings.Index(sql, "enabled = CASE") < 0 || strings.Index(sql, "remark = CASE") < 0 || strings.Index(sql, "enabled = CASE") > strings.Index(sql, "remark = CASE") { + t.Fatalf("source upsert must restore enabled before updating remark because MySQL evaluates assignments in order:\n%s", sql) + } +} + +func TestDataSourceSchemaIncludesStableSourceCode(t *testing.T) { + for _, want := range []string{ + "source_code VARCHAR(64) NULL", + "source_kind VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN'", + "KEY idx_protocol_source_code (protocol, source_code)", + "KEY idx_protocol_source_kind_seen (protocol, source_kind, latest_seen_at)", + } { + if !strings.Contains(DataSourceTableSQL, want) { + t.Fatalf("data source schema missing %q:\n%s", want, DataSourceTableSQL) + } + } + for _, want := range []string{ + "ALTER TABLE vehicle_data_source ADD COLUMN source_code", + "ALTER TABLE vehicle_data_source ADD COLUMN source_kind", + "ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_code", + "ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_kind_seen", + } { + if !containsStatement(DailyMileageAlterSQL, want) { + t.Fatalf("data source alter SQL missing %q: %#v", want, DailyMileageAlterSQL) + } + } +} + +func TestDailyMileageSourceSchemaIncludesQueryIndexes(t *testing.T) { + for _, want := range []string{ + "KEY idx_protocol_date_selected (protocol, stat_date, is_selected, vin)", + "KEY idx_protocol_date_quality (protocol, stat_date, quality_status, vin)", + "KEY idx_protocol_date_quality_reason (protocol, stat_date, quality_status, quality_reason, vin)", + "KEY idx_source_ip_date (protocol, source_ip, stat_date)", + } { + if !strings.Contains(DailyMileageSourceTableSQL, want) { + t.Fatalf("daily mileage source schema missing %q:\n%s", want, DailyMileageSourceTableSQL) + } + if !containsStatement(DailyMileageAlterSQL, "ALTER TABLE vehicle_daily_mileage_source ADD "+want) { + t.Fatalf("daily mileage source alter SQL missing %q: %#v", want, DailyMileageAlterSQL) + } + } +} + +func containsStatement(statements []string, fragment string) bool { + for _, statement := range statements { + if strings.Contains(statement, fragment) { + return true + } + } + return false +} diff --git a/go/vehicle-gateway/internal/telemetry/location.go b/go/vehicle-gateway/internal/telemetry/location.go new file mode 100644 index 00000000..1c14060f --- /dev/null +++ b/go/vehicle-gateway/internal/telemetry/location.go @@ -0,0 +1,170 @@ +package telemetry + +import ( + "strings" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +type LocationProjection struct { + Latitude float64 + Longitude float64 + SpeedKMH *float64 + SOCPercent *float64 + AltitudeM *float64 + DirectionDeg *float64 + AlarmFlag *int64 + StatusFlag *int64 +} + +type locationFieldMapping struct { + Latitude []string + Longitude []string + SpeedKMH []string + SOCPercent []string + AltitudeM []string + DirectionDeg []string + AlarmFlag []string + StatusFlag []string +} + +// HasRealtimeFields separates protocol telemetry from headers, identity +// annotations, and transport metadata before realtime/stat projections. +func HasRealtimeFields(protocol envelope.Protocol, fields map[string]any) bool { + switch protocol { + case envelope.ProtocolGB32960: + for field := range fields { + if !strings.HasPrefix(field, "gb32960.") { + continue + } + if strings.HasPrefix(field, "gb32960.header.") || + strings.HasPrefix(field, "gb32960.platform.") || + strings.HasPrefix(field, "gb32960.identity.") || + strings.HasPrefix(field, "gb32960.device_time.") { + continue + } + return true + } + return false + case envelope.ProtocolJT808: + return hasPrefix(fields, "jt808.location.") + case envelope.ProtocolYutongMQTT: + return hasPrefix(fields, "yutong_mqtt.data.") || + hasPrefix(fields, "yutong_mqtt.root.data.") + default: + return false + } +} + +// LocationProjectionForProtocol is the single protocol-field contract used by +// storage projections. It intentionally ignores bare standardized field names. +func LocationProjectionForProtocol(protocol envelope.Protocol, fields map[string]any) (LocationProjection, bool) { + mapping := locationMapping(protocol) + latitude, hasLatitude := firstNumber(fields, mapping.Latitude) + longitude, hasLongitude := firstNumber(fields, mapping.Longitude) + if !hasLatitude || !hasLongitude { + return LocationProjection{}, false + } + return LocationProjection{ + Latitude: latitude, + Longitude: longitude, + SpeedKMH: numberPointer(fields, mapping.SpeedKMH), + SOCPercent: numberPointer(fields, mapping.SOCPercent), + AltitudeM: numberPointer(fields, mapping.AltitudeM), + DirectionDeg: numberPointer(fields, mapping.DirectionDeg), + AlarmFlag: integerPointer(fields, mapping.AlarmFlag), + StatusFlag: integerPointer(fields, mapping.StatusFlag), + }, true +} + +func locationMapping(protocol envelope.Protocol) locationFieldMapping { + switch protocol { + case envelope.ProtocolGB32960: + return locationFieldMapping{ + Latitude: []string{"gb32960.position.latitude"}, + Longitude: []string{"gb32960.position.longitude"}, + SpeedKMH: []string{"gb32960.vehicle.speed_kmh"}, + SOCPercent: []string{"gb32960.vehicle.soc_percent"}, + } + case envelope.ProtocolJT808: + return locationFieldMapping{ + Latitude: []string{"jt808.location.latitude"}, + Longitude: []string{"jt808.location.longitude"}, + SpeedKMH: []string{"jt808.location.speed_kmh"}, + AltitudeM: []string{"jt808.location.altitude_m"}, + DirectionDeg: []string{"jt808.location.direction_deg"}, + AlarmFlag: []string{"jt808.location.alarm_flag"}, + StatusFlag: []string{"jt808.location.status_flag"}, + } + case envelope.ProtocolYutongMQTT: + return locationFieldMapping{ + Latitude: []string{ + "yutong_mqtt.data.latitude", + "yutong_mqtt.root.data.latitude", + }, + Longitude: []string{ + "yutong_mqtt.data.longitude", + "yutong_mqtt.root.data.longitude", + }, + SpeedKMH: []string{ + "yutong_mqtt.data.meter_speed", + "yutong_mqtt.root.data.meter_speed", + "yutong_mqtt.data.speed_kmh", + "yutong_mqtt.root.data.speed_kmh", + "yutong_mqtt.data.speed", + "yutong_mqtt.root.data.speed", + }, + SOCPercent: []string{ + "yutong_mqtt.data.battery_capacity_soc", + "yutong_mqtt.root.data.battery_capacity_soc", + "yutong_mqtt.data.soc_percent", + "yutong_mqtt.root.data.soc_percent", + }, + DirectionDeg: []string{ + "yutong_mqtt.data.gpsdirection", + "yutong_mqtt.root.data.gpsdirection", + "yutong_mqtt.data.direction_deg", + "yutong_mqtt.root.data.direction_deg", + "yutong_mqtt.data.direction", + "yutong_mqtt.root.data.direction", + }, + } + default: + return locationFieldMapping{} + } +} + +func firstNumber(fields map[string]any, keys []string) (float64, bool) { + for _, key := range keys { + if value, ok := Number(fields, key); ok { + return value, true + } + } + return 0, false +} + +func numberPointer(fields map[string]any, keys []string) *float64 { + value, ok := firstNumber(fields, keys) + if !ok { + return nil + } + return &value +} + +func integerPointer(fields map[string]any, keys []string) *int64 { + value, ok := firstNumber(fields, keys) + if !ok { + return nil + } + integer := int64(value) + return &integer +} + +func hasPrefix(fields map[string]any, prefix string) bool { + for field := range fields { + if strings.HasPrefix(field, prefix) { + return true + } + } + return false +} diff --git a/go/vehicle-gateway/internal/telemetry/location_test.go b/go/vehicle-gateway/internal/telemetry/location_test.go new file mode 100644 index 00000000..7c9998c1 --- /dev/null +++ b/go/vehicle-gateway/internal/telemetry/location_test.go @@ -0,0 +1,131 @@ +package telemetry + +import ( + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestLocationProjectionForProtocolUsesProtocolFields(t *testing.T) { + tests := []struct { + name string + protocol envelope.Protocol + fields map[string]any + assert func(*testing.T, LocationProjection) + }{ + { + name: "gb32960", + protocol: envelope.ProtocolGB32960, + fields: map[string]any{ + "gb32960.position.latitude": "30.56", + "gb32960.position.longitude": "121.0", + "gb32960.vehicle.speed_kmh": "54.3", + "gb32960.vehicle.soc_percent": "85", + "gb32960.vehicle.total_mileage": "123", + }, + assert: func(t *testing.T, location LocationProjection) { + assertFloatPointer(t, location.SpeedKMH, 54.3) + assertFloatPointer(t, location.SOCPercent, 85) + }, + }, + { + name: "jt808", + protocol: envelope.ProtocolJT808, + fields: map[string]any{ + "jt808.location.latitude": "30.590151", + "jt808.location.longitude": "121.069881", + "jt808.location.speed_kmh": "23", + "jt808.location.altitude_m": "5", + "jt808.location.direction_deg": "79", + "jt808.location.alarm_flag": "2", + "jt808.location.status_flag": "786435", + }, + assert: func(t *testing.T, location LocationProjection) { + assertFloatPointer(t, location.SpeedKMH, 23) + assertFloatPointer(t, location.AltitudeM, 5) + assertFloatPointer(t, location.DirectionDeg, 79) + assertIntPointer(t, location.AlarmFlag, 2) + assertIntPointer(t, location.StatusFlag, 786435) + }, + }, + { + name: "yutong root fallback", + protocol: envelope.ProtocolYutongMQTT, + fields: map[string]any{ + "yutong_mqtt.root.data.latitude": "30.590921", + "yutong_mqtt.root.data.longitude": "121.075044", + "yutong_mqtt.root.data.meter_speed": "27", + "yutong_mqtt.root.data.battery_capacity_soc": "78.4", + "yutong_mqtt.root.data.gpsdirection": "88", + }, + assert: func(t *testing.T, location LocationProjection) { + assertFloatPointer(t, location.SpeedKMH, 27) + assertFloatPointer(t, location.SOCPercent, 78.4) + assertFloatPointer(t, location.DirectionDeg, 88) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + location, ok := LocationProjectionForProtocol(tt.protocol, tt.fields) + if !ok { + t.Fatal("LocationProjectionForProtocol() did not find coordinates") + } + if location.Latitude == 0 || location.Longitude == 0 { + t.Fatalf("coordinates = %v,%v", location.Latitude, location.Longitude) + } + tt.assert(t, location) + }) + } +} + +func TestLocationProjectionForProtocolRejectsBareStandardizedFields(t *testing.T) { + for _, protocol := range []envelope.Protocol{ + envelope.ProtocolGB32960, + envelope.ProtocolJT808, + envelope.ProtocolYutongMQTT, + } { + if location, ok := LocationProjectionForProtocol(protocol, map[string]any{ + "latitude": 30.5, + "longitude": 121.0, + "speed_kmh": 20, + }); ok { + t.Fatalf("protocol %s unexpectedly accepted bare fields: %#v", protocol, location) + } + } +} + +func TestHasRealtimeFieldsExcludesMetadataOnlyPayloads(t *testing.T) { + tests := []struct { + protocol envelope.Protocol + fields map[string]any + want bool + }{ + {protocol: envelope.ProtocolGB32960, fields: map[string]any{"gb32960.header.command": "0x02"}, want: false}, + {protocol: envelope.ProtocolGB32960, fields: map[string]any{"gb32960.gd_fc_stack.stack_count": "1"}, want: true}, + {protocol: envelope.ProtocolJT808, fields: map[string]any{"jt808.registration.plate": "粤A00001"}, want: false}, + {protocol: envelope.ProtocolJT808, fields: map[string]any{"jt808.location.speed_kmh": "20"}, want: true}, + {protocol: envelope.ProtocolYutongMQTT, fields: map[string]any{"yutong_mqtt.metadata.topic": "/ytforward/shln/3"}, want: false}, + {protocol: envelope.ProtocolYutongMQTT, fields: map[string]any{"yutong_mqtt.data.meter_speed": "20"}, want: true}, + } + for _, tt := range tests { + if got := HasRealtimeFields(tt.protocol, tt.fields); got != tt.want { + t.Fatalf("HasRealtimeFields(%s, %#v) = %v, want %v", tt.protocol, tt.fields, got, tt.want) + } + } +} + +func assertFloatPointer(t *testing.T, value *float64, want float64) { + t.Helper() + if value == nil || *value != want { + t.Fatalf("float pointer = %v, want %v", value, want) + } +} + +func assertIntPointer(t *testing.T, value *int64, want int64) { + t.Helper() + if value == nil || *value != want { + t.Fatalf("int pointer = %v, want %v", value, want) + } +} diff --git a/go/vehicle-gateway/internal/telemetry/mileage.go b/go/vehicle-gateway/internal/telemetry/mileage.go new file mode 100644 index 00000000..43698b99 --- /dev/null +++ b/go/vehicle-gateway/internal/telemetry/mileage.go @@ -0,0 +1,89 @@ +package telemetry + +import ( + "encoding/json" + "strconv" + "strings" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +type MileageFieldMapping struct { + Key string + Scale float64 +} + +// MileageFieldMappings is the single protocol-to-unit contract used by every +// projection that exposes or calculates total mileage. +func MileageFieldMappings(protocol envelope.Protocol) []MileageFieldMapping { + switch protocol { + case envelope.ProtocolGB32960: + return []MileageFieldMapping{{Key: "gb32960.vehicle.total_mileage_km", Scale: 1}} + case envelope.ProtocolJT808: + return []MileageFieldMapping{{Key: "jt808.location.total_mileage_km", Scale: 1}} + case envelope.ProtocolYutongMQTT: + return []MileageFieldMapping{ + {Key: "yutong_mqtt.data.total_mileage_km", Scale: 1}, + {Key: "yutong_mqtt.root.data.total_mileage_km", Scale: 1}, + {Key: "yutong_mqtt.data.total_mileage", Scale: 0.001}, + {Key: "yutong_mqtt.root.data.total_mileage", Scale: 0.001}, + } + default: + return nil + } +} + +func TotalMileageKM(protocol envelope.Protocol, fields map[string]any) (float64, bool) { + for _, mapping := range MileageFieldMappings(protocol) { + value, ok := Number(fields, mapping.Key) + if !ok { + continue + } + return value * mapping.Scale, true + } + return 0, false +} + +func Number(fields map[string]any, key string) (float64, bool) { + if len(fields) == 0 { + return 0, false + } + value, ok := fields[key] + if !ok || value == nil { + return 0, false + } + switch typed := value.(type) { + case float64: + return typed, true + case float32: + return float64(typed), true + case int: + return float64(typed), true + case int8: + return float64(typed), true + case int16: + return float64(typed), true + case int32: + return float64(typed), true + case int64: + return float64(typed), true + case uint: + return float64(typed), true + case uint8: + return float64(typed), true + case uint16: + return float64(typed), true + case uint32: + return float64(typed), true + case uint64: + return float64(typed), true + case json.Number: + parsed, err := typed.Float64() + return parsed, err == nil + case string: + parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64) + return parsed, err == nil + default: + return 0, false + } +} diff --git a/go/vehicle-gateway/internal/topics/topics.go b/go/vehicle-gateway/internal/topics/topics.go index cf3297d8..dcf49bcd 100644 --- a/go/vehicle-gateway/internal/topics/topics.go +++ b/go/vehicle-gateway/internal/topics/topics.go @@ -1,5 +1,13 @@ package topics +import ( + "fmt" + "sort" + "strings" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + const ( RawGB32960 = "vehicle.raw.go.gb32960.v1" RawJT808 = "vehicle.raw.go.jt808.v1" @@ -9,3 +17,196 @@ const ( FieldsYutongMQTT = "vehicle.fields.go.yutong-mqtt.v1" Unified = "vehicle.event.go.unified.v1" ) + +const ( + RawPrefix = "vehicle.raw." + FieldsPrefix = "vehicle.fields." +) + +func ProtocolForKnownRawTopic(topic string) (string, bool) { + switch strings.TrimSpace(topic) { + case RawGB32960: + return "GB32960", true + case RawJT808: + return "JT808", true + case RawYutongMQTT: + return "YUTONG_MQTT", true + default: + return "", false + } +} + +func ProtocolForKnownFieldsTopic(topic string) (string, bool) { + switch strings.TrimSpace(topic) { + case FieldsGB32960: + return "GB32960", true + case FieldsJT808: + return "JT808", true + case FieldsYutongMQTT: + return "YUTONG_MQTT", true + default: + return "", false + } +} + +func ValidateKnownRawTopicProtocol(topic string, protocol string) error { + expectedProtocol, ok := ProtocolForKnownRawTopic(topic) + if !ok { + return nil + } + protocol = strings.TrimSpace(protocol) + if protocol == expectedProtocol { + return nil + } + if protocol == "" { + protocol = "UNKNOWN" + } + return fmt.Errorf("raw topic %q expects protocol %s, got %s", strings.TrimSpace(topic), expectedProtocol, protocol) +} + +func ValidateKnownFieldsTopicProtocol(topic string, protocol string) error { + expectedProtocol, ok := ProtocolForKnownFieldsTopic(topic) + if !ok { + return nil + } + protocol = strings.TrimSpace(protocol) + if protocol == expectedProtocol { + return nil + } + if protocol == "" { + protocol = "UNKNOWN" + } + return fmt.Errorf("fields topic %q expects protocol %s, got %s", strings.TrimSpace(topic), expectedProtocol, protocol) +} + +func ValidateRawEnvelope(topic string, env envelope.FrameEnvelope) (string, error) { + if err := ValidateKnownRawTopicProtocol(topic, string(env.Protocol)); err != nil { + return "protocol_topic_mismatch", err + } + if strings.HasPrefix(strings.TrimSpace(topic), RawPrefix) { + switch env.EventKind { + case "", envelope.EventKindRaw: + return "", nil + default: + return "event_kind_mismatch", fmt.Errorf("raw topic %q expects event kind %s, got %s", strings.TrimSpace(topic), envelope.EventKindRaw, env.EventKind) + } + } + return "", nil +} + +func ValidateFieldsEnvelope(topic string, env envelope.FrameEnvelope) (string, error) { + if err := ValidateKnownFieldsTopicProtocol(topic, string(env.Protocol)); err != nil { + return "protocol_topic_mismatch", err + } + if strings.HasPrefix(strings.TrimSpace(topic), FieldsPrefix) { + if env.EventKind != envelope.EventKindFields { + return "event_kind_mismatch", fmt.Errorf("fields topic %q expects event kind %s, got %s", strings.TrimSpace(topic), envelope.EventKindFields, env.EventKind) + } + if strings.TrimSpace(env.FieldMapping) == "" { + return "missing_field_mapping", fmt.Errorf("fields topic %q expects non-empty field mapping", strings.TrimSpace(topic)) + } + if len(env.Fields) == 0 { + return "missing_fields", fmt.Errorf("fields topic %q expects non-empty fields payload", strings.TrimSpace(topic)) + } + if err := ValidateProtocolFieldNames(env.Protocol, env.Fields); err != nil { + return "invalid_field_name", err + } + } + return "", nil +} + +func ValidateProtocolFieldNames(protocol envelope.Protocol, fields map[string]any) error { + prefix, ok := protocolFieldPrefix(protocol) + if !ok { + return nil + } + invalid := make([]string, 0) + for key := range fields { + trimmed := strings.TrimSpace(key) + if key == trimmed && len(trimmed) > len(prefix) && strings.HasPrefix(trimmed, prefix) { + continue + } + invalid = append(invalid, key) + } + if len(invalid) == 0 { + return nil + } + sort.Strings(invalid) + if len(invalid) > 3 { + invalid = invalid[:3] + } + return fmt.Errorf("fields protocol %s expects every field name under %q; invalid fields: %q", protocol, prefix, invalid) +} + +func protocolFieldPrefix(protocol envelope.Protocol) (string, bool) { + switch protocol { + case envelope.ProtocolGB32960: + return "gb32960.", true + case envelope.ProtocolJT808: + return "jt808.", true + case envelope.ProtocolYutongMQTT: + return "yutong_mqtt.", true + default: + return "", false + } +} + +func ValidateKnownRawFieldsProtocols(raw map[string]string, fields map[string]string, valueName string) error { + for protocol, value := range raw { + if err := ValidateKnownRawTopicProtocol(value, protocol); err != nil { + return fmt.Errorf("raw %s for %s must match protocol: %w", valueName, protocol, err) + } + } + for protocol, value := range fields { + if err := ValidateKnownFieldsTopicProtocol(value, protocol); err != nil { + return fmt.Errorf("fields %s for %s must match protocol: %w", valueName, protocol, err) + } + } + return nil +} + +func ValidateKafkaRawFields(raw map[string]string, fields map[string]string) error { + for name, topic := range raw { + topic = strings.TrimSpace(topic) + if topic == "" { + return fmt.Errorf("raw kafka topic for %s is empty", name) + } + if !strings.HasPrefix(topic, RawPrefix) { + return fmt.Errorf("raw kafka topic for %s must start with %q, got %q", name, RawPrefix, topic) + } + } + for name, topic := range fields { + topic = strings.TrimSpace(topic) + if topic == "" { + return fmt.Errorf("fields kafka topic for %s is empty", name) + } + if !strings.HasPrefix(topic, FieldsPrefix) { + return fmt.Errorf("fields kafka topic for %s must start with %q, got %q", name, FieldsPrefix, topic) + } + } + if err := ValidateKnownRawFieldsProtocols(raw, fields, "kafka topic"); err != nil { + return err + } + return ValidateRawFieldsDisjoint(raw, fields, "kafka topic") +} + +func ValidateRawFieldsDisjoint(raw map[string]string, fields map[string]string, valueName string) error { + rawByValue := map[string]string{} + for name, value := range raw { + value = strings.TrimSpace(value) + if value == "" { + continue + } + rawByValue[value] = name + } + for fieldsName, value := range fields { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if rawName, ok := rawByValue[value]; ok { + return fmt.Errorf("raw and fields %s must be different: raw %s and fields %s both use %q", valueName, rawName, fieldsName, value) + } + } + return nil +} diff --git a/go/vehicle-gateway/internal/topics/topics_test.go b/go/vehicle-gateway/internal/topics/topics_test.go new file mode 100644 index 00000000..430f194a --- /dev/null +++ b/go/vehicle-gateway/internal/topics/topics_test.go @@ -0,0 +1,249 @@ +package topics + +import ( + "strings" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestValidateKafkaRawFieldsRejectsWrongFamilyPrefix(t *testing.T) { + err := ValidateKafkaRawFields( + map[string]string{"JT808": FieldsJT808}, + map[string]string{"JT808": FieldsJT808}, + ) + if err == nil { + t.Fatal("ValidateKafkaRawFields() error = nil, want raw prefix rejection") + } + if !strings.Contains(err.Error(), "raw kafka topic") { + t.Fatalf("error = %q, want raw kafka topic hint", err) + } + + err = ValidateKafkaRawFields( + map[string]string{"JT808": RawJT808}, + map[string]string{"JT808": RawJT808}, + ) + if err == nil { + t.Fatal("ValidateKafkaRawFields() error = nil, want fields prefix rejection") + } + if !strings.Contains(err.Error(), "fields kafka topic") { + t.Fatalf("error = %q, want fields kafka topic hint", err) + } +} + +func TestValidateRawFieldsDisjointRejectsSharedValue(t *testing.T) { + err := ValidateRawFieldsDisjoint( + map[string]string{"raw-jt808": "vehicle.same.jt808"}, + map[string]string{"fields-jt808": "vehicle.same.jt808"}, + "nats subject", + ) + if err == nil { + t.Fatal("ValidateRawFieldsDisjoint() error = nil, want duplicate rejection") + } + if !strings.Contains(err.Error(), "must be different") { + t.Fatalf("error = %q, want disjoint hint", err) + } +} + +func TestValidateKnownFieldsTopicProtocolRejectsMismatchedProtocol(t *testing.T) { + err := ValidateKnownFieldsTopicProtocol(FieldsGB32960, "JT808") + if err == nil { + t.Fatal("ValidateKnownFieldsTopicProtocol() error = nil, want mismatch rejection") + } + if !strings.Contains(err.Error(), "expects protocol GB32960") { + t.Fatalf("error = %q, want expected protocol hint", err) + } +} + +func TestValidateKnownRawTopicProtocolRejectsMismatchedProtocol(t *testing.T) { + err := ValidateKnownRawTopicProtocol(RawJT808, "GB32960") + if err == nil { + t.Fatal("ValidateKnownRawTopicProtocol() error = nil, want mismatch rejection") + } + if !strings.Contains(err.Error(), "expects protocol JT808") { + t.Fatalf("error = %q, want expected protocol hint", err) + } +} + +func TestValidateKnownTopicProtocolAllowsMatchingAndCustomTopics(t *testing.T) { + if err := ValidateKnownRawTopicProtocol(RawGB32960, "GB32960"); err != nil { + t.Fatalf("ValidateKnownRawTopicProtocol() matching topic error = %v", err) + } + if err := ValidateKnownRawTopicProtocol("vehicle.raw.custom.g7", "JT808"); err != nil { + t.Fatalf("ValidateKnownRawTopicProtocol() custom topic error = %v", err) + } + if err := ValidateKnownFieldsTopicProtocol(FieldsJT808, "JT808"); err != nil { + t.Fatalf("ValidateKnownFieldsTopicProtocol() matching topic error = %v", err) + } + if err := ValidateKnownFieldsTopicProtocol("vehicle.fields.custom.g7", "JT808"); err != nil { + t.Fatalf("ValidateKnownFieldsTopicProtocol() custom topic error = %v", err) + } +} + +func TestValidateRawEnvelopeRejectsExplicitNonRawEventKind(t *testing.T) { + status, err := ValidateRawEnvelope(RawJT808, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + }) + if err == nil { + t.Fatal("ValidateRawEnvelope() error = nil, want event kind mismatch") + } + if status != "event_kind_mismatch" { + t.Fatalf("status = %q, want event_kind_mismatch", status) + } + if !strings.Contains(err.Error(), "expects event kind RAW") { + t.Fatalf("error = %q, want RAW hint", err) + } +} + +func TestValidateRawEnvelopeAllowsHistoricalEmptyAndRawEventKind(t *testing.T) { + for _, kind := range []envelope.EventKind{"", envelope.EventKindRaw} { + if status, err := ValidateRawEnvelope(RawGB32960, envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, EventKind: kind}); err != nil { + t.Fatalf("ValidateRawEnvelope(%q) status=%q error=%v", kind, status, err) + } + } +} + +func TestValidateFieldsEnvelopeRejectsExplicitNonFieldsEventKind(t *testing.T) { + status, err := ValidateFieldsEnvelope(FieldsYutongMQTT, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolYutongMQTT, + EventKind: envelope.EventKindRaw, + }) + if err == nil { + t.Fatal("ValidateFieldsEnvelope() error = nil, want event kind mismatch") + } + if status != "event_kind_mismatch" { + t.Fatalf("status = %q, want event_kind_mismatch", status) + } + if !strings.Contains(err.Error(), "expects event kind FIELDS") { + t.Fatalf("error = %q, want FIELDS hint", err) + } +} + +func TestValidateFieldsEnvelopeRequiresFieldsContract(t *testing.T) { + status, err := ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + Fields: map[string]any{"jt808.location.total_mileage_km": 10241.2}, + }) + if err == nil { + t.Fatal("ValidateFieldsEnvelope() error = nil, want missing field mapping") + } + if status != "missing_field_mapping" { + t.Fatalf("status = %q, want missing_field_mapping", status) + } + + status, err = ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + FieldMapping: "2026-07-03.v1", + }) + if err == nil { + t.Fatal("ValidateFieldsEnvelope() error = nil, want missing fields") + } + if status != "missing_fields" { + t.Fatalf("status = %q, want missing_fields", status) + } + + status, err = ValidateFieldsEnvelope(FieldsJT808, envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + EventKind: envelope.EventKindFields, + FieldMapping: "2026-07-03.v1", + Fields: map[string]any{"jt808.location.total_mileage_km": 10241.2}, + }) + if err != nil || status != "" { + t.Fatalf("ValidateFieldsEnvelope() status=%q error=%v, want ok", status, err) + } +} + +func TestValidateFieldsEnvelopeRejectsNonProtocolFieldNames(t *testing.T) { + for _, tt := range []struct { + name string + topic string + protocol envelope.Protocol + fields map[string]any + }{ + { + name: "normalized core field", + topic: FieldsJT808, + protocol: envelope.ProtocolJT808, + fields: map[string]any{"total_mileage_km": 10241.2}, + }, + { + name: "other protocol namespace", + topic: FieldsGB32960, + protocol: envelope.ProtocolGB32960, + fields: map[string]any{"jt808.location.latitude": 30.1}, + }, + { + name: "empty suffix", + topic: FieldsYutongMQTT, + protocol: envelope.ProtocolYutongMQTT, + fields: map[string]any{"yutong_mqtt.": 1}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + status, err := ValidateFieldsEnvelope(tt.topic, envelope.FrameEnvelope{ + Protocol: tt.protocol, + EventKind: envelope.EventKindFields, + FieldMapping: "2026-07-03.v1", + Fields: tt.fields, + }) + if err == nil { + t.Fatal("ValidateFieldsEnvelope() error = nil, want invalid field rejection") + } + if status != "invalid_field_name" { + t.Fatalf("status = %q, want invalid_field_name", status) + } + if !strings.Contains(err.Error(), "expects every field name") { + t.Fatalf("error = %q, want protocol namespace hint", err) + } + }) + } +} + +func TestValidateFieldsEnvelopeAcceptsProtocolNamespaces(t *testing.T) { + for _, tt := range []struct { + topic string + protocol envelope.Protocol + field string + }{ + {topic: FieldsGB32960, protocol: envelope.ProtocolGB32960, field: "gb32960.vehicle.total_mileage_km"}, + {topic: FieldsJT808, protocol: envelope.ProtocolJT808, field: "jt808.location.total_mileage_km"}, + {topic: FieldsYutongMQTT, protocol: envelope.ProtocolYutongMQTT, field: "yutong_mqtt.root.data.total_mileage"}, + } { + status, err := ValidateFieldsEnvelope(tt.topic, envelope.FrameEnvelope{ + Protocol: tt.protocol, + EventKind: envelope.EventKindFields, + FieldMapping: "2026-07-03.v1", + Fields: map[string]any{tt.field: 1}, + }) + if err != nil || status != "" { + t.Fatalf("ValidateFieldsEnvelope(%s) status=%q error=%v", tt.protocol, status, err) + } + } +} + +func TestValidateKafkaRawFieldsRejectsKnownProtocolMismatch(t *testing.T) { + err := ValidateKafkaRawFields( + map[string]string{"JT808": RawGB32960}, + map[string]string{"JT808": FieldsJT808}, + ) + if err == nil { + t.Fatal("ValidateKafkaRawFields() error = nil, want known raw protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } + + err = ValidateKafkaRawFields( + map[string]string{"JT808": RawJT808}, + map[string]string{"JT808": FieldsGB32960}, + ) + if err == nil { + t.Fatal("ValidateKafkaRawFields() error = nil, want known fields protocol mismatch") + } + if !strings.Contains(err.Error(), "must match protocol") { + t.Fatalf("error = %q, want protocol mismatch hint", err) + } +} diff --git a/go/vehicle-gateway/scripts/deploy-ecs-release.sh b/go/vehicle-gateway/scripts/deploy-ecs-release.sh new file mode 100755 index 00000000..96c6ff49 --- /dev/null +++ b/go/vehicle-gateway/scripts/deploy-ecs-release.sh @@ -0,0 +1,449 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RELEASE="go-native-$(date +%Y%m%d%H%M%S)" +OUT_DIR="" +HOST="" +REMOTE_ROOT="/opt/lingniu-go-native" +BUILD_ONLY=0 +STAGE_ONLY=0 +CAPACITY_RETRIES=10 +CAPACITY_WARMUP_SECONDS=60 +CAPACITY_RETRY_INTERVAL_SECONDS=15 +CAPACITY_TIMER="lingniu-go-capacity-check.timer" +CAPACITY_SERVICE="lingniu-go-capacity-check.service" +CAPACITY_ARGS="${CAPACITY_ARGS:-}" +RELEASE_RETAIN_COUNT="${RELEASE_RETAIN_COUNT:-8}" + +BINARIES=( + "gateway:./cmd/gateway" + "history-writer:./cmd/history-writer" + "stat-writer:./cmd/stat-writer" + "realtime-api:./cmd/realtime-api" + "nats-fast-writer:./cmd/nats-fast-writer" + "nats-kafka-bridge:./cmd/nats-kafka-bridge" + "fields-projector:./cmd/fields-projector" + "capacity-check:./cmd/capacity-check" + "load-sim:./cmd/load-sim" + "stats-backfill:./cmd/stats-backfill" + "identity-import:./cmd/identity-import" + "identity-writer:./cmd/identity-writer" +) + +SERVICES=( + "lingniu-go-nats-kafka-bridge.service" + "lingniu-go-fields-projector.service" + "lingniu-go-nats-fast-writer.service" + "lingniu-go-history-writer.service" + "lingniu-go-stat-writer.service" + "lingniu-go-realtime-writer.service" + "lingniu-go-identity-writer.service" + "lingniu-go-realtime-api.service" + "lingniu-go-gateway.service" +) + +READYZ_PORTS=(20214 20218 20215 20212 20213 20216 20217 20200 20211) + +usage() { + cat <<'USAGE' +Usage: + scripts/deploy-ecs-release.sh --build-only [--release NAME] [--out-dir DIR] + scripts/deploy-ecs-release.sh --host root@115.29.187.205 [--release NAME] [--out-dir DIR] + +Options: + --host HOST SSH target for ECS deployment. Omit with --build-only. + --release NAME Release directory name under /opt/lingniu-go-native/releases. + --out-dir DIR Local build output directory. Defaults to /tmp/lingniu-go-native-. + --remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native. + --build-only Build and verify Linux amd64 artifacts without uploading. + --stage-only Upload and verify release, but do not switch current or restart services. + --capacity-retries N + Retry capacity-check after restart. Defaults to 10. + --capacity-warmup-seconds N + Wait after readyz before capacity-check, so recent-window metrics are not dominated by restart catch-up. Defaults to 60. + --capacity-retry-interval-seconds N + Wait between capacity-check retries. Defaults to 15. + --capacity-timer UNIT + systemd timer paused during restart. Defaults to lingniu-go-capacity-check.timer. + --capacity-args ARGS + Extra arguments passed to capacity-check during deploy verification. + Example: --capacity-args "-daily-mileage-diagnostics-max-pipeline=0" + --release-retain-count N + Keep the newest N remote releases after a successful deployment. Defaults to 8. + -h, --help Show this help. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --host) + HOST="${2:-}" + shift 2 + ;; + --release) + RELEASE="${2:-}" + shift 2 + ;; + --out-dir) + OUT_DIR="${2:-}" + shift 2 + ;; + --remote-root) + REMOTE_ROOT="${2:-}" + shift 2 + ;; + --build-only) + BUILD_ONLY=1 + shift + ;; + --stage-only|--skip-restart) + STAGE_ONLY=1 + shift + ;; + --capacity-retries) + CAPACITY_RETRIES="${2:-}" + shift 2 + ;; + --capacity-warmup-seconds) + CAPACITY_WARMUP_SECONDS="${2:-}" + shift 2 + ;; + --capacity-retry-interval-seconds) + CAPACITY_RETRY_INTERVAL_SECONDS="${2:-}" + shift 2 + ;; + --capacity-timer) + CAPACITY_TIMER="${2:-}" + shift 2 + ;; + --capacity-args) + CAPACITY_ARGS="${2:-}" + shift 2 + ;; + --release-retain-count) + RELEASE_RETAIN_COUNT="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="/tmp/lingniu-go-native-${RELEASE}" +fi + +if [[ "$BUILD_ONLY" -eq 0 && -z "$HOST" ]]; then + echo "--host is required unless --build-only is set" >&2 + usage >&2 + exit 2 +fi + +case "$CAPACITY_RETRIES" in + ''|*[!0-9]*|0) + echo "--capacity-retries must be a positive integer" >&2 + exit 2 + ;; +esac +case "$CAPACITY_WARMUP_SECONDS" in + ''|*[!0-9]*) + echo "--capacity-warmup-seconds must be a non-negative integer" >&2 + exit 2 + ;; +esac +case "$CAPACITY_RETRY_INTERVAL_SECONDS" in + ''|*[!0-9]*) + echo "--capacity-retry-interval-seconds must be a non-negative integer" >&2 + exit 2 + ;; +esac +case "$RELEASE_RETAIN_COUNT" in + ''|*[!0-9]*|0) + echo "--release-retain-count must be a positive integer" >&2 + exit 2 + ;; +esac + +assert_linux_amd64() { + local binary="$1" + local info + info="$(file "$binary")" + case "$info" in + *"ELF 64-bit"*"x86-64"*) ;; + *) + echo "binary is not Linux amd64 ELF: $binary" >&2 + echo "$info" >&2 + exit 1 + ;; + esac +} + +hash_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$@" + else + shasum -a 256 "$@" + fi +} + +build_release() { + rm -rf "$OUT_DIR" + mkdir -p "$OUT_DIR" + pushd "$ROOT_DIR" >/dev/null + local names=() + for spec in "${BINARIES[@]}"; do + local name="${spec%%:*}" + local pkg="${spec#*:}" + names+=("$name") + echo "building $name from $pkg" + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ + go build -trimpath -ldflags="${GO_LDFLAGS:--s -w}" -o "$OUT_DIR/$name" "$pkg" + assert_linux_amd64 "$OUT_DIR/$name" + done + popd >/dev/null + + ( + cd "$OUT_DIR" + hash_file "${names[@]}" > SHA256SUMS + { + echo "release=${RELEASE}" + echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "git_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" + echo "go_version=$(go version)" + echo "target=linux/amd64" + } > RELEASE_NAME + ) + echo "build output: $OUT_DIR" +} + +preflight_remote_ssh() { + echo "checking ssh connectivity to ${HOST}" + if ssh -o ConnectTimeout=10 -o ConnectionAttempts=1 "$HOST" "true"; then + return + fi + cat >&2 </dev/null | grep -q "${CAPACITY_TIMER}"; then + timer_exists=1 + if systemctl is-active --quiet "${CAPACITY_TIMER}"; then + timer_was_active=1 + systemctl stop "${CAPACITY_TIMER}" + echo "paused ${CAPACITY_TIMER} during deploy" + fi +fi +if [[ -n "${CAPACITY_SERVICE}" ]] && systemctl list-unit-files "${CAPACITY_SERVICE}" --no-legend 2>/dev/null | grep -q "${CAPACITY_SERVICE}"; then + systemctl stop "${CAPACITY_SERVICE}" 2>/dev/null || true + systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true +fi +restore_capacity_timer() { + if [[ -n "${CAPACITY_SERVICE}" ]]; then + systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true + fi + if [[ "${timer_exists}" == "1" && "${timer_was_active}" == "1" ]]; then + if [[ -n "${CAPACITY_SERVICE}" ]]; then + if systemctl start "${CAPACITY_SERVICE}" 2>/dev/null; then + systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true + echo "verified ${CAPACITY_SERVICE} after deploy" + fi + fi + systemctl start "${CAPACITY_TIMER}" 2>/dev/null || true + echo "resumed ${CAPACITY_TIMER}" + if [[ -n "${CAPACITY_SERVICE}" ]]; then + for _ in 1 2 3 4 5 6 7 8 9 10; do + if ! systemctl is-active --quiet "${CAPACITY_SERVICE}"; then + break + fi + sleep 1 + done + if systemctl is-failed --quiet "${CAPACITY_SERVICE}" && run_capacity_check >/dev/null 2>&1; then + systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true + echo "cleared transient ${CAPACITY_SERVICE} failure after timer resume" + fi + fi + fi +} +cleanup_old_releases() { + local current_dir + current_dir="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)" + local kept=0 + while IFS= read -r release_path; do + [[ -n "${release_path}" ]] || continue + if [[ "${release_path}" == "${current_dir}" || "${kept}" -lt "${RELEASE_RETAIN_COUNT}" ]]; then + kept=$((kept + 1)) + continue + fi + rm -rf -- "${release_path}" + done < <(find "${REMOTE_ROOT}/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-) +} +previous_release="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)" +release_switched=0 +deploy_succeeded=0 +on_exit() { + local status=$? + trap - EXIT + set +e + if [[ "${release_switched}" == "1" && "${deploy_succeeded}" != "1" && -n "${previous_release}" && -d "${previous_release}" ]]; then + echo "deployment failed; rolling back to ${previous_release}" >&2 + ln -sfn "${previous_release}" "${REMOTE_ROOT}/current" + for service in ${SERVICES}; do + systemctl restart "${service}" + done + for service in ${SERVICES}; do + systemctl is-active "${service}" + done + fi + cleanup_remote_tmp + restore_capacity_timer + exit "${status}" +} +trap on_exit EXIT + +ln -sfn "${release_dir}" "${REMOTE_ROOT}/current" +release_switched=1 + +for service in ${SERVICES}; do + systemctl restart "${service}" +done + +for service in ${SERVICES}; do + systemctl is-active "${service}" +done + +for port in ${READYZ_PORTS}; do + ok=0 + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do + if curl -fsS "http://127.0.0.1:${port}/readyz"; then + echo + ok=1 + break + fi + sleep 2 + done + if [[ "${ok}" != "1" ]]; then + echo "readyz failed for port ${port}" >&2 + exit 1 + fi +done + +if [[ "${CAPACITY_WARMUP_SECONDS}" != "0" ]]; then + echo "waiting ${CAPACITY_WARMUP_SECONDS}s before capacity-check warmup" + sleep "${CAPACITY_WARMUP_SECONDS}" +fi + +capacity_ok=0 +for attempt in $(seq 1 "${CAPACITY_RETRIES}"); do + echo "capacity-check attempt ${attempt}/${CAPACITY_RETRIES}" + if run_capacity_check; then + capacity_ok=1 + break + fi + if [[ "${attempt}" != "${CAPACITY_RETRIES}" && "${CAPACITY_RETRY_INTERVAL_SECONDS}" != "0" ]]; then + sleep "${CAPACITY_RETRY_INTERVAL_SECONDS}" + fi +done +if [[ "${capacity_ok}" != "1" ]]; then + exit 1 +fi +deploy_succeeded=1 +cleanup_old_releases +REMOTE + rm -f "$archive" + if [[ "$deploy_status" -ne 0 ]]; then + return "$deploy_status" + fi + if [[ "$STAGE_ONLY" -eq 1 ]]; then + echo "staged release ${RELEASE} at ${HOST}:${REMOTE_ROOT}/releases/${RELEASE}" + else + echo "deployed release ${RELEASE} to ${HOST}:${REMOTE_ROOT}" + fi +} + +if [[ "$BUILD_ONLY" -eq 0 ]]; then + preflight_remote_ssh +fi +build_release +if [[ "$BUILD_ONLY" -eq 1 ]]; then + exit 0 +fi +deploy_release diff --git a/go/vehicle-gateway/scripts/install-fields-projector-service.sh b/go/vehicle-gateway/scripts/install-fields-projector-service.sh new file mode 100755 index 00000000..cc711c6a --- /dev/null +++ b/go/vehicle-gateway/scripts/install-fields-projector-service.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOST="" +REMOTE_ROOT="/opt/lingniu-go-native" + +usage() { + cat <<'USAGE' +Usage: + scripts/install-fields-projector-service.sh --host root@115.29.187.205 [--remote-root DIR] + +Prepares the Kafka RAW-to-fields projector service. It does not disable the +legacy bridge projection; switch BRIDGE_DERIVE_FIELDS_FROM_RAW_ENABLED only +after the new consumer has started and its offsets are established. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --host) + HOST="${2:-}" + shift 2 + ;; + --remote-root) + REMOTE_ROOT="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$HOST" ]]; then + echo "--host is required" >&2 + exit 2 +fi + +ssh "$HOST" "REMOTE_ROOT='$REMOTE_ROOT' bash -s" <<'REMOTE' +set -euo pipefail + +env_dir="${REMOTE_ROOT}/env" +projector_env="${env_dir}/fields-projector.env" +unit="/etc/systemd/system/lingniu-go-fields-projector.service" +mkdir -p "${env_dir}" + +cat >"${projector_env}" <<'ENV' +FIELDS_PROJECTOR_GROUP_PREFIX=vehicle-fields-projector-v1 +KAFKA_START_OFFSET=last +FIELDS_PROJECTOR_WORKERS_PER_PROTOCOL=1 +FIELDS_PROJECTOR_BATCH_SIZE=500 +FIELDS_PROJECTOR_BATCH_WAIT_MS=20 +FIELDS_PROJECTOR_OPERATION_TIMEOUT_MS=30000 +FIELDS_PROJECTOR_RETRY_DELAY_MS=500 +HEALTH_ADDR=127.0.0.1:20218 +ENV +chmod 0640 "${projector_env}" + +cat >"${unit}" <&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$HOST" ]]; then + echo "--host is required" >&2 + usage >&2 + exit 2 +fi + +if [[ -z "$ON_CALENDAR" ]]; then + echo "--on-calendar must not be empty" >&2 + exit 2 +fi +if [[ -z "$TIMEOUT" ]]; then + echo "--timeout must not be empty" >&2 + exit 2 +fi +if [[ -z "$PRUNE_MIN_AGE" ]]; then + echo "--prune-min-age must not be empty" >&2 + exit 2 +fi +if [[ -z "$RETIRE_MIN_AGE" ]]; then + echo "--retire-min-age must not be empty" >&2 + exit 2 +fi + +remote_cmd="" +printf -v remote_cmd \ + "REMOTE_ROOT=%q ON_CALENDAR=%q TIMEOUT=%q APPLY=%q PRUNE_UNMANAGED=%q PRUNE_MIN_AGE=%q RETIRE_STALE_UNMANAGED=%q RETIRE_MIN_AGE=%q bash -s" \ + "$REMOTE_ROOT" "$ON_CALENDAR" "$TIMEOUT" "$APPLY" "$PRUNE_UNMANAGED" "$PRUNE_MIN_AGE" "$RETIRE_STALE_UNMANAGED" "$RETIRE_MIN_AGE" +ssh "$HOST" "$remote_cmd" <<'REMOTE' +set -euo pipefail + +install -d /etc/systemd/system + +apply_arg="" +if [[ "${APPLY}" == "true" ]]; then + apply_arg=" -apply" +fi +prune_arg="" +if [[ "${PRUNE_UNMANAGED}" == "true" ]]; then + prune_arg=" -prune-unmanaged-data-sources -prune-data-source-min-age=\"${PRUNE_MIN_AGE}\"" +fi +retire_arg="" +if [[ "${RETIRE_STALE_UNMANAGED}" == "true" ]]; then + retire_arg=" -retire-stale-unmanaged-data-sources -retire-data-source-min-age=\"${RETIRE_MIN_AGE}\"" +fi + +cat >/etc/systemd/system/lingniu-go-identity-source-sync.service </etc/systemd/system/lingniu-go-identity-source-sync.timer </dev/null || true +systemctl enable --now lingniu-go-identity-source-sync.timer +systemctl start lingniu-go-identity-source-sync.service +systemctl reset-failed lingniu-go-identity-source-sync.service 2>/dev/null || true +systemctl list-timers lingniu-go-identity-source-sync.timer --no-pager +systemctl show lingniu-go-identity-source-sync.service -p Result -p ExecMainStatus -p ActiveState -p SubState --no-pager +REMOTE diff --git a/go/vehicle-gateway/scripts/install-identity-writer-service.sh b/go/vehicle-gateway/scripts/install-identity-writer-service.sh new file mode 100755 index 00000000..e660b5a9 --- /dev/null +++ b/go/vehicle-gateway/scripts/install-identity-writer-service.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOST="" +REMOTE_ROOT="/opt/lingniu-go-native" + +usage() { + cat <<'USAGE' +Usage: + scripts/install-identity-writer-service.sh --host root@115.29.187.205 [--remote-root DIR] + +Prepares the native identity-writer service and switches Gateway registration +persistence to Kafka delegation. The existing Gateway binary ignores the new +environment switch, so this is safe to run before staging the first release +that contains identity-writer. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --host) + HOST="${2:-}" + shift 2 + ;; + --remote-root) + REMOTE_ROOT="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$HOST" ]]; then + echo "--host is required" >&2 + exit 2 +fi + +ssh "$HOST" "REMOTE_ROOT='$REMOTE_ROOT' bash -s" <<'REMOTE' +set -euo pipefail + +env_dir="${REMOTE_ROOT}/env" +gateway_env="${env_dir}/gateway.env" +identity_env="${env_dir}/identity-writer.env" +unit="/etc/systemd/system/lingniu-go-identity-writer.service" +mkdir -p "${env_dir}" + +cat >"${identity_env}" <<'ENV' +KAFKA_GROUP=go-identity-writer +KAFKA_TOPIC=vehicle.raw.go.jt808.v1 +KAFKA_START_OFFSET=last +HEALTH_ADDR=127.0.0.1:20217 +MYSQL_ENSURE_SCHEMA=true +MYSQL_MAX_OPEN_CONNS=8 +MYSQL_MAX_IDLE_CONNS=4 +MYSQL_CONN_MAX_LIFETIME_SECONDS=300 +IDENTITY_WRITER_BATCH_SIZE=500 +IDENTITY_WRITER_BATCH_WAIT_MS=20 +IDENTITY_WRITER_RETRY_DELAY_MS=1000 +JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS=600 +LOCAL_TZ=Asia/Shanghai +ENV +chmod 0640 "${identity_env}" + +touch "${gateway_env}" +if grep -q '^JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=' "${gateway_env}"; then + sed -i 's/^JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=.*/JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false/' "${gateway_env}" +else + printf '\nJT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false\n' >>"${gateway_env}" +fi + +cat >"${unit}" <&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$HOST" ]]; then + echo "--host is required" >&2 + usage >&2 + exit 2 +fi + +case "$DAYS_BACK" in + ''|*[!0-9]*) + echo "--days-back must be a non-negative integer" >&2 + exit 2 + ;; +esac +case "$WINDOW_DAYS" in + ''|*[!0-9]*|0) + echo "--window-days must be a positive integer" >&2 + exit 2 + ;; +esac + +remote_cmd="" +printf -v remote_cmd \ + "REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \ + "$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN" +ssh "$HOST" "$remote_cmd" <<'REMOTE' +set -euo pipefail + +install -d /etc/systemd/system + +cat >/etc/systemd/system/lingniu-go-stats-backfill.service </etc/systemd/system/lingniu-go-stats-backfill.timer </dev/null || true +systemctl enable --now lingniu-go-stats-backfill.timer +systemctl list-timers lingniu-go-stats-backfill.timer --no-pager +REMOTE diff --git a/vehicle-data-platform/apps/api/cmd/alert-benchmark/main.go b/vehicle-data-platform/apps/api/cmd/alert-benchmark/main.go new file mode 100644 index 00000000..50b9ee0f --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/alert-benchmark/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "os" + "strconv" + "strings" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +const ( + temporaryMode = "temporary" + durableMode = "durable" + temporaryTable = "vehicle_alert_candidate_benchmark" + durableTable = "vehicle_alert_candidate_benchmark_durable" + durableLock = "lingniu.vehicle-alert-candidate-benchmark" +) + +type benchmarkConfig struct { + Rows int + BatchSize int + Mode string + ConfirmDurable bool +} + +type benchmarkResult struct { + Mode string `json:"mode"` + Rows int `json:"rows"` + BatchSize int `json:"batchSize"` + Batches int `json:"batches"` + Transactions int `json:"transactions"` + DurationMS int64 `json:"durationMs"` + RowsPerSec float64 `json:"rowsPerSec"` + VerifiedRows int `json:"verifiedRows"` + Temporary bool `json:"temporaryTable"` + DurableWrite bool `json:"durableWrite"` + CleanupMS int64 `json:"cleanupMs"` + CleanupVerified bool `json:"cleanupVerified"` + GlobalStatusDelta map[string]uint64 `json:"globalStatusDelta,omitempty"` + GlobalStatusAvailable bool `json:"globalStatusAvailable"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "alert benchmark failed:", err) + os.Exit(1) + } +} + +func run() error { + config := benchmarkConfig{} + flag.IntVar(&config.Rows, "rows", 10_000, "candidate rows to write") + flag.IntVar(&config.BatchSize, "batch-size", 500, "rows per insert") + flag.StringVar(&config.Mode, "mode", temporaryMode, "storage mode: temporary or durable") + flag.BoolVar(&config.ConfirmDurable, "confirm-durable-write", false, "required acknowledgement for durable writes and DDL") + flag.Parse() + if err := config.validate(); err != nil { + return err + } + dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")) + if dsn == "" { + return fmt.Errorf("MYSQL_DSN is required") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + result, err := executeBenchmark(ctx, conn, config) + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(result) +} + +func (config benchmarkConfig) validate() error { + config.Mode = strings.ToLower(strings.TrimSpace(config.Mode)) + if config.Rows < 1 || config.Rows > 100_000 || config.BatchSize < 1 || config.BatchSize > 1_000 { + return fmt.Errorf("rows must be 1..100000 and batch-size 1..1000") + } + if config.Mode != temporaryMode && config.Mode != durableMode { + return fmt.Errorf("mode must be temporary or durable") + } + if config.Mode == durableMode && !config.ConfirmDurable { + return fmt.Errorf("durable mode creates, commits to, and drops a dedicated physical table; pass --confirm-durable-write") + } + return nil +} + +func executeBenchmark(ctx context.Context, conn *sql.Conn, config benchmarkConfig) (result benchmarkResult, err error) { + mode := strings.ToLower(strings.TrimSpace(config.Mode)) + table := temporaryTable + createPrefix := "CREATE TEMPORARY TABLE" + dropPrefix := "DROP TEMPORARY TABLE IF EXISTS" + if mode == durableMode { + table = durableTable + createPrefix = "CREATE TABLE" + dropPrefix = "DROP TABLE IF EXISTS" + locked, lockErr := acquireBenchmarkLock(ctx, conn) + if lockErr != nil { + return result, fmt.Errorf("acquire durable benchmark lock: %w", lockErr) + } + if !locked { + return result, fmt.Errorf("another durable benchmark is already running") + } + defer releaseBenchmarkLock(conn) + if _, err = conn.ExecContext(ctx, dropPrefix+" "+table); err != nil { + return result, fmt.Errorf("remove stale durable benchmark table: %w", err) + } + } + if _, err = conn.ExecContext(ctx, createPrefix+" "+table+" LIKE vehicle_alert_candidate"); err != nil { + return result, fmt.Errorf("create %s benchmark table: %w", mode, err) + } + cleanupStarted := time.Time{} + cleanupComplete := false + defer func() { + cleanupStarted = time.Now() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, cleanupErr := conn.ExecContext(cleanupCtx, dropPrefix+" "+table) + if cleanupErr == nil { + cleanupComplete = true + if mode == durableMode { + var remaining int + cleanupErr = conn.QueryRowContext(cleanupCtx, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name=?`, table).Scan(&remaining) + cleanupComplete = cleanupErr == nil && remaining == 0 + } + } + result.CleanupMS = time.Since(cleanupStarted).Milliseconds() + result.CleanupVerified = cleanupComplete + if err == nil && cleanupErr != nil { + err = fmt.Errorf("cleanup benchmark table: %w", cleanupErr) + } else if err == nil && !cleanupComplete { + err = fmt.Errorf("benchmark table cleanup could not be verified") + } + }() + + statusBefore, statusBeforeErr := readGlobalStatus(ctx, conn) + started := time.Now() + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return result, err + } + now := time.Now().UTC() + batches := 0 + for start := 0; start < config.Rows; start += config.BatchSize { + end := min(start+config.BatchSize, config.Rows) + query, args := buildCandidateInsert(table, start, end, now) + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + tx.Rollback() + return result, fmt.Errorf("insert batch %d: %w", batches+1, err) + } + batches++ + } + if err = tx.Commit(); err != nil { + return result, err + } + duration := time.Since(started) + var verified int + if err = conn.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&verified); err != nil { + return result, err + } + if verified != config.Rows { + return result, fmt.Errorf("row verification failed: wrote %d, found %d", config.Rows, verified) + } + statusAfter, statusAfterErr := readGlobalStatus(ctx, conn) + statusDelta := map[string]uint64{} + statusAvailable := statusBeforeErr == nil && statusAfterErr == nil + if statusAvailable { + for key, after := range statusAfter { + if before, ok := statusBefore[key]; ok && after >= before { + statusDelta[key] = after - before + } + } + } + result = benchmarkResult{ + Mode: mode, + Rows: config.Rows, + BatchSize: config.BatchSize, + Batches: batches, + Transactions: 1, + DurationMS: duration.Milliseconds(), + RowsPerSec: float64(config.Rows) / duration.Seconds(), + VerifiedRows: verified, + Temporary: mode == temporaryMode, + DurableWrite: mode == durableMode, + GlobalStatusDelta: statusDelta, + GlobalStatusAvailable: statusAvailable, + } + return result, nil +} + +func acquireBenchmarkLock(ctx context.Context, conn *sql.Conn) (bool, error) { + var acquired sql.NullInt64 + if err := conn.QueryRowContext(ctx, `SELECT GET_LOCK(?,0)`, durableLock).Scan(&acquired); err != nil { + return false, err + } + return acquired.Valid && acquired.Int64 == 1, nil +} + +func releaseBenchmarkLock(conn *sql.Conn) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, _ = conn.ExecContext(ctx, `SELECT RELEASE_LOCK(?)`, durableLock) +} + +func readGlobalStatus(ctx context.Context, conn *sql.Conn) (map[string]uint64, error) { + rows, err := conn.QueryContext(ctx, `SHOW GLOBAL STATUS WHERE Variable_name IN ('Innodb_os_log_written','Binlog_cache_use','Binlog_cache_disk_use','Com_commit')`) + if err != nil { + return nil, err + } + defer rows.Close() + values := map[string]uint64{} + for rows.Next() { + var key, raw string + if err := rows.Scan(&key, &raw); err != nil { + return nil, err + } + value, parseErr := strconv.ParseUint(raw, 10, 64) + if parseErr != nil { + return nil, parseErr + } + values[key] = value + } + return values, rows.Err() +} + +func buildCandidateInsert(table string, start, end int, now time.Time) (string, []any) { + if table != temporaryTable && table != durableTable { + panic("unsupported benchmark table") + } + values := make([]string, 0, end-start) + args := make([]any, 0, (end-start)*7) + for index := start; index < end; index++ { + values = append(values, "(?,?,?,?,?,?,?)") + args = append(args, "benchmark-rule", fmt.Sprintf("BENCH%017d", index), "JT808", now, now, float64(index%120), fmt.Sprintf("benchmark-event-%d", index)) + } + return `INSERT INTO ` + table + `(rule_id,vin,protocol,first_matched_at,last_matched_at,latest_value,source_event_id) VALUES ` + strings.Join(values, ","), args +} diff --git a/vehicle-data-platform/apps/api/cmd/alert-benchmark/main_test.go b/vehicle-data-platform/apps/api/cmd/alert-benchmark/main_test.go new file mode 100644 index 00000000..9d99f631 --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/alert-benchmark/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestBuildCandidateInsertMatchesBatchArguments(t *testing.T) { + query, args := buildCandidateInsert(temporaryTable, 10, 13, time.Now()) + if strings.Count(query, "(?,?,?,?,?,?,?)") != 3 || strings.Count(query, "?") != len(args) || len(args) != 21 { + t.Fatalf("query/args mismatch: placeholders=%d args=%d query=%s", strings.Count(query, "?"), len(args), query) + } +} + +func TestBenchmarkConfigRequiresExplicitDurableConfirmation(t *testing.T) { + valid := benchmarkConfig{Rows: 100_000, BatchSize: 500, Mode: durableMode, ConfirmDurable: true} + if err := valid.validate(); err != nil { + t.Fatalf("valid durable benchmark rejected: %v", err) + } + valid.ConfirmDurable = false + if err := valid.validate(); err == nil || !strings.Contains(err.Error(), "--confirm-durable-write") { + t.Fatalf("durable mode must require confirmation, got %v", err) + } +} + +func TestBenchmarkConfigBoundsAndModes(t *testing.T) { + for _, config := range []benchmarkConfig{ + {Rows: 0, BatchSize: 500, Mode: temporaryMode}, + {Rows: 1, BatchSize: 1001, Mode: temporaryMode}, + {Rows: 1, BatchSize: 1, Mode: "business-table"}, + } { + if err := config.validate(); err == nil { + t.Fatalf("invalid config accepted: %+v", config) + } + } +} diff --git a/vehicle-data-platform/apps/api/cmd/alert-evaluator/main.go b/vehicle-data-platform/apps/api/cmd/alert-evaluator/main.go new file mode 100644 index 00000000..8fd5f99a --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/alert-evaluator/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "log" + "os/signal" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +func main() { + cfg := config.Load() + if cfg.MySQLDSN == "" { + log.Fatal("MYSQL_DSN is required") + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN) + if err != nil { + log.Fatal(err) + } + defer db.Close() + store := platform.NewProductionStore(db, nil, "").WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup) + service := platform.NewService(store) + interval := cfg.AlertEvaluationInterval + if interval < time.Second { + interval = time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + log.Printf("alert evaluator started interval=%s stream_mode=%s", interval, cfg.AlertStreamMode) + for { + started := time.Now() + result, err := service.EvaluateAlerts(ctx) + if err != nil { + log.Printf("alert evaluation failed duration=%s error=%v", time.Since(started), err) + } else { + log.Printf("alert evaluation completed rules=%d vehicles=%d candidates_advanced=%d duplicate_observations=%d late_observations=%d stale_evidence_skipped=%d opened=%d recovered=%d duration=%s", result.RulesEvaluated, result.VehiclesScanned, result.CandidatesAdvanced, result.DuplicateObservations, result.LateObservations, result.StaleEvidenceSkipped, result.Opened, result.Recovered, time.Since(started)) + } + select { + case <-ctx.Done(): + log.Printf("alert evaluator stopped") + return + case <-ticker.C: + } + } +} diff --git a/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main.go b/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main.go new file mode 100644 index 00000000..cdb94fcd --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "fmt" + "log" + "os/signal" + "sort" + "strings" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/segmentio/kafka-go" + + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +const alertStreamOperationTimeout = 30 * time.Second + +type alertStreamStore interface { + RecordAlertStreamBatch(context.Context, string, []platform.AlertStreamRecord) (platform.AlertStreamBatchResult, error) +} + +type kafkaMessageFetcher interface { + FetchMessage(context.Context) (kafka.Message, error) +} + +type kafkaMessageCommitter interface { + CommitMessages(context.Context, ...kafka.Message) error +} + +func main() { + cfg := config.Load() + if err := validateAlertStreamConfig(cfg); err != nil { + log.Fatal(err) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN) + if err != nil { + log.Fatal(err) + } + defer db.Close() + store := platform.NewProductionStore(db, nil, "").WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup) + reader := kafka.NewReader(alertStreamReaderConfig(cfg)) + defer reader.Close() + log.Printf("alert stream evaluator started mode=%s group=%s topics=%s batch_size=%d batch_wait=%s lateness=%s", cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup, strings.Join(cfg.AlertStreamKafkaTopics, ","), cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait, cfg.AlertStreamLateness) + for { + message, err := reader.FetchMessage(ctx) + if err != nil { + if ctx.Err() != nil { + log.Printf("alert stream evaluator stopped") + return + } + log.Printf("alert stream fetch failed error=%v", err) + continue + } + messages := collectAlertStreamBatch(ctx, reader, message, cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait) + started := time.Now() + operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertStreamOperationTimeout) + records := make([]platform.AlertStreamRecord, 0, len(messages)) + for _, item := range messages { + records = append(records, platform.DecodeAlertStreamRecord(item.Topic, item.Partition, item.Offset, item.HighWaterMark, item.Value, cfg.AlertStreamLateness)) + } + invalidCodes := alertStreamInvalidCodes(records) + result, recordErr := store.RecordAlertStreamBatch(operationCtx, cfg.AlertStreamKafkaGroup, records) + if recordErr == nil { + recordErr = reader.CommitMessages(operationCtx, messages...) + } + cancel() + if recordErr != nil { + log.Printf("alert stream batch failed fetched=%d duration=%s error=%v", len(messages), time.Since(started), recordErr) + time.Sleep(time.Second) + continue + } + log.Printf("alert stream batch completed mode=%s fetched=%d processed=%d valid=%d invalid=%d invalid_codes=%s late=%d replay_skipped=%d partitions=%d rules=%d candidates_advanced=%d duplicate_observations=%d late_observations=%d opened=%d recovered=%d duration=%s", cfg.AlertStreamMode, result.Fetched, result.Processed, result.Valid, result.Invalid, invalidCodes, result.Late, result.ReplaySkipped, result.Partitions, result.RulesEvaluated, result.CandidatesAdvanced, result.DuplicateObservations, result.LateObservations, result.Opened, result.Recovered, time.Since(started)) + } +} + +func alertStreamInvalidCodes(records []platform.AlertStreamRecord) string { + counts := map[string]int{} + for _, record := range records { + if !record.Valid { + counts[record.ErrorCode]++ + } + } + if len(counts) == 0 { + return "none" + } + keys := make([]string, 0, len(counts)) + for key := range counts { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key])) + } + return strings.Join(parts, ",") +} + +func alertStreamReaderConfig(cfg config.Config) kafka.ReaderConfig { + return kafka.ReaderConfig{ + Brokers: cfg.AlertStreamKafkaBrokers, + GroupID: cfg.AlertStreamKafkaGroup, + GroupTopics: cfg.AlertStreamKafkaTopics, + MinBytes: 1, + MaxBytes: 10e6, + StartOffset: kafka.LastOffset, + } +} + +func validateAlertStreamConfig(cfg config.Config) error { + if cfg.AlertStreamMode != "shadow" && cfg.AlertStreamMode != "active" { + return fmt.Errorf("ALERT_STREAM_MODE must be shadow or active") + } + if cfg.MySQLDSN == "" { + return fmt.Errorf("MYSQL_DSN is required") + } + if len(cfg.AlertStreamKafkaBrokers) == 0 || len(cfg.AlertStreamKafkaTopics) == 0 { + return fmt.Errorf("ALERT_STREAM_KAFKA_BROKERS and ALERT_STREAM_KAFKA_TOPICS are required") + } + if strings.TrimSpace(cfg.AlertStreamKafkaGroup) == "" { + return fmt.Errorf("ALERT_STREAM_KAFKA_GROUP is required") + } + if cfg.AlertStreamBatchSize < 1 || cfg.AlertStreamBatchSize > 1000 { + return fmt.Errorf("ALERT_STREAM_BATCH_SIZE must be between 1 and 1000") + } + if cfg.AlertStreamBatchWait < time.Millisecond || cfg.AlertStreamBatchWait > 5*time.Second { + return fmt.Errorf("ALERT_STREAM_BATCH_WAIT_MS must be between 1 and 5000") + } + if cfg.AlertStreamLateness < 0 || cfg.AlertStreamLateness > 24*time.Hour { + return fmt.Errorf("ALERT_STREAM_LATENESS_SEC must be between 0 and 86400") + } + for _, topic := range cfg.AlertStreamKafkaTopics { + if _, known := alertStreamTopicProtocol(topic); !known { + return fmt.Errorf("unsupported alert stream topic %q", topic) + } + } + return nil +} + +func alertStreamTopicProtocol(topic string) (string, bool) { + switch strings.TrimSpace(topic) { + case "vehicle.fields.go.gb32960.v1": + return "GB32960", true + case "vehicle.fields.go.jt808.v1": + return "JT808", true + case "vehicle.fields.go.yutong-mqtt.v1": + return "YUTONG_MQTT", true + default: + return "", false + } +} + +func collectAlertStreamBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { + if maxSize <= 1 { + return []kafka.Message{first} + } + messages := []kafka.Message{first} + deadline := time.Now().Add(maxWait) + for len(messages) < maxSize { + remaining := time.Until(deadline) + if remaining <= 0 { + break + } + fetchCtx, cancel := context.WithTimeout(ctx, remaining) + message, err := fetcher.FetchMessage(fetchCtx) + cancel() + if err != nil { + break + } + messages = append(messages, message) + } + return messages +} diff --git a/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main_test.go b/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main_test.go new file mode 100644 index 00000000..97bc1f4c --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + "time" + + "github.com/segmentio/kafka-go" + + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +func validAlertStreamConfig() config.Config { + return config.Config{ + MySQLDSN: "user:pass@tcp(localhost:3306)/db?parseTime=true", + AlertStreamMode: "shadow", + AlertStreamKafkaBrokers: []string{"kafka-a:9092"}, + AlertStreamKafkaTopics: []string{"vehicle.fields.go.jt808.v1"}, + AlertStreamKafkaGroup: "vehicle-alert-stream-v1", + AlertStreamBatchSize: 200, + AlertStreamBatchWait: 100 * time.Millisecond, + AlertStreamLateness: 2 * time.Minute, + } +} + +func TestAlertStreamConfigAcceptsReleasedModesAndRefusesUnknownTopics(t *testing.T) { + cfg := validAlertStreamConfig() + if err := validateAlertStreamConfig(cfg); err != nil { + t.Fatalf("valid shadow config rejected: %v", err) + } + cfg.AlertStreamMode = "active" + if err := validateAlertStreamConfig(cfg); err != nil { + t.Fatalf("released active mode rejected: %v", err) + } + cfg.AlertStreamMode = "unsafe" + if err := validateAlertStreamConfig(cfg); err == nil { + t.Fatal("unknown mode must fail closed") + } + cfg = validAlertStreamConfig() + cfg.AlertStreamKafkaTopics = []string{"vehicle.raw.go.jt808.v1"} + if err := validateAlertStreamConfig(cfg); err == nil { + t.Fatal("raw topic must not be accepted as a fields alert stream") + } +} + +func TestNewAlertStreamGroupStartsAtLatestAndThenUsesCommittedOffsets(t *testing.T) { + cfg := validAlertStreamConfig() + reader := alertStreamReaderConfig(cfg) + if reader.StartOffset != kafka.LastOffset { + t.Fatalf("new production group would replay the full retained topic: start=%d", reader.StartOffset) + } + if reader.GroupID != cfg.AlertStreamKafkaGroup || len(reader.GroupTopics) != 1 || reader.GroupTopics[0] != cfg.AlertStreamKafkaTopics[0] { + t.Fatalf("reader group contract drifted: %+v", reader) + } +} + +func TestAlertStreamInvalidCodesAreAggregatedWithoutPayloads(t *testing.T) { + records := []platform.AlertStreamRecord{{ErrorCode: "missing_vin_jt808"}, {Valid: true}, {ErrorCode: "invalid_field_name"}, {ErrorCode: "missing_vin_jt808"}} + if got := alertStreamInvalidCodes(records); got != "invalid_field_name:1,missing_vin_jt808:2" { + t.Fatalf("invalid code summary=%q", got) + } +} diff --git a/vehicle-data-platform/apps/api/cmd/platform-migrate/main.go b/vehicle-data-platform/apps/api/cmd/platform-migrate/main.go new file mode 100644 index 00000000..fc58bc38 --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/platform-migrate/main.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-sql-driver/mysql" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "platform migration failed:", err) + os.Exit(1) + } +} + +func run() error { + dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")) + if dsn == "" { + return fmt.Errorf("MYSQL_DSN is required") + } + if len(os.Args) < 2 { + return fmt.Errorf("usage: platform-migrate migration.sql [migration.sql ...]") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("connect mysql: %w", err) + } + if len(os.Args) == 2 && os.Args[1] == "--server-version" { + var version string + if err := db.QueryRowContext(ctx, "SELECT VERSION()").Scan(&version); err != nil { + return err + } + fmt.Println(version) + return nil + } + if _, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS vehicle_platform_schema_migration ( +version VARCHAR(255) NOT NULL PRIMARY KEY, +checksum CHAR(64) NOT NULL, +applied_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) +)`); err != nil { + return fmt.Errorf("ensure migration journal: %w", err) + } + for _, path := range os.Args[1:] { + contents, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + version := filepath.Base(path) + sum := sha256.Sum256(contents) + checksum := hex.EncodeToString(sum[:]) + var recorded string + err = db.QueryRowContext(ctx, `SELECT checksum FROM vehicle_platform_schema_migration WHERE version=?`, version).Scan(&recorded) + if err == nil { + if recorded != checksum { + return fmt.Errorf("migration %s was changed after being applied", version) + } + fmt.Printf("skipped %s (already applied)\n", path) + continue + } + if err != sql.ErrNoRows { + return fmt.Errorf("check migration %s: %w", version, err) + } + statements := splitSQL(string(contents)) + for index, statement := range statements { + if _, err := db.ExecContext(ctx, statement); err != nil { + if isResumableMigrationDDL(err, statement) { + fmt.Printf("skipped %s statement %d (schema object already exists)\n", path, index+1) + continue + } + return fmt.Errorf("apply %s statement %d: %w", path, index+1, err) + } + } + if _, err := db.ExecContext(ctx, `INSERT INTO vehicle_platform_schema_migration(version,checksum) VALUES(?,?)`, version, checksum); err != nil { + return fmt.Errorf("record migration %s: %w", version, err) + } + fmt.Printf("applied %s (%d statements)\n", path, len(statements)) + } + return nil +} + +func isResumableMigrationDDL(err error, statement string) bool { + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) { + return false + } + upper := strings.ToUpper(strings.TrimSpace(statement)) + duplicateColumn := mysqlErr.Number == 1060 && strings.Contains(upper, "ALTER TABLE") && strings.Contains(upper, "ADD COLUMN") + duplicateIndex := mysqlErr.Number == 1061 && strings.HasPrefix(upper, "CREATE INDEX") + return duplicateColumn || duplicateIndex +} + +// splitSQL intentionally supports the platform's forward-only DDL files. Those +// files contain no procedures or quoted semicolons; rejecting empty fragments +// keeps deployment output deterministic without enabling multiStatements in DSN. +func splitSQL(contents string) []string { + lines := strings.Split(contents, "\n") + withoutComments := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + withoutComments = append(withoutComments, line) + } + contents = strings.Join(withoutComments, "\n") + fragments := strings.Split(contents, ";") + statements := make([]string, 0, len(fragments)) + for _, fragment := range fragments { + statement := strings.TrimSpace(fragment) + if statement != "" { + statements = append(statements, statement) + } + } + return statements +} diff --git a/vehicle-data-platform/apps/api/cmd/platform-migrate/main_test.go b/vehicle-data-platform/apps/api/cmd/platform-migrate/main_test.go new file mode 100644 index 00000000..f8e50e4a --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/platform-migrate/main_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "errors" + "testing" + + "github.com/go-sql-driver/mysql" +) + +func TestSplitSQLKeepsForwardMigrationStatements(t *testing.T) { + statements := splitSQL("-- migration\nCREATE TABLE x (id INT);\n\nINSERT INTO x VALUES (1);\n") + if len(statements) != 2 || statements[0] != "CREATE TABLE x (id INT)" || statements[1] != "INSERT INTO x VALUES (1)" { + t.Fatalf("unexpected statements: %#v", statements) + } +} + +func TestSplitSQLIgnoresSemicolonsInFullLineComments(t *testing.T) { + statements := splitSQL("-- first clause; second clause\nCREATE TABLE x (id INT);\n-- trailing; note\n") + if len(statements) != 1 || statements[0] != "CREATE TABLE x (id INT)" { + t.Fatalf("comment punctuation became executable SQL: %#v", statements) + } +} + +func TestOnlyDuplicateForwardDDLIsIgnoredForMigrationResume(t *testing.T) { + duplicate := &mysql.MySQLError{Number: 1060, Message: "Duplicate column"} + if !isResumableMigrationDDL(duplicate, "ALTER TABLE x ADD COLUMN y INT") { + t.Fatal("duplicate ADD COLUMN should be resumable") + } + if isResumableMigrationDDL(duplicate, "CREATE TABLE x (y INT)") || isResumableMigrationDDL(errors.New("duplicate"), "ALTER TABLE x ADD COLUMN y INT") { + t.Fatal("unrelated migration errors must not be ignored") + } + duplicateIndex := &mysql.MySQLError{Number: 1061, Message: "Duplicate key name"} + if !isResumableMigrationDDL(duplicateIndex, "CREATE INDEX idx_x ON x(y)") || isResumableMigrationDDL(duplicateIndex, "DROP INDEX idx_x ON x") { + t.Fatal("only duplicate CREATE INDEX should be resumable") + } +} diff --git a/vehicle-data-platform/apps/api/go.mod b/vehicle-data-platform/apps/api/go.mod index ae8bac84..23c3afa9 100644 --- a/vehicle-data-platform/apps/api/go.mod +++ b/vehicle-data-platform/apps/api/go.mod @@ -9,9 +9,13 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/segmentio/kafka-go v0.4.49 // indirect ) diff --git a/vehicle-data-platform/apps/api/go.sum b/vehicle-data-platform/apps/api/go.sum index c3ba046d..1718857b 100644 --- a/vehicle-data-platform/apps/api/go.sum +++ b/vehicle-data-platform/apps/api/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -12,12 +14,19 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk= +github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= diff --git a/vehicle-data-platform/apps/api/internal/app/auth.go b/vehicle-data-platform/apps/api/internal/app/auth.go new file mode 100644 index 00000000..dd5712ba --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/app/auth.go @@ -0,0 +1,176 @@ +package app + +import ( + "crypto/sha256" + "crypto/subtle" + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/httpx" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +type configuredPrincipal struct { + Token string `json:"token"` + Name string `json:"name"` + Role string `json:"role"` +} + +type tokenPrincipal struct { + hash [sha256.Size]byte + principal platform.Principal +} + +type apiAuthenticator struct { + mode string + tokens []tokenPrincipal +} + +func withAPIAuth(next http.Handler, cfg config.Config) http.Handler { + authenticator, err := newAPIAuthenticator(cfg) + if err != nil { + log.Printf("platform API authentication misconfigured: %v", err) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpx.WriteError(w, http.StatusServiceUnavailable, "AUTH_CONFIG_INVALID", "平台鉴权配置无效", err.Error(), requestTraceID(r)) + }) + } + return authenticator.middleware(next) +} + +func newAPIAuthenticator(cfg config.Config) (*apiAuthenticator, error) { + mode := strings.ToLower(strings.TrimSpace(cfg.AuthMode)) + if mode == "" { + mode = "disabled" + } + if mode != "disabled" && mode != "enforce" { + return nil, fmt.Errorf("AUTH_MODE must be disabled or enforce") + } + authenticator := &apiAuthenticator{mode: mode} + configured := []configuredPrincipal{} + if strings.TrimSpace(cfg.AuthTokensJSON) != "" { + if err := json.Unmarshal([]byte(cfg.AuthTokensJSON), &configured); err != nil { + return nil, fmt.Errorf("decode AUTH_TOKENS_JSON: %w", err) + } + } + if token := strings.TrimSpace(cfg.AuthToken); token != "" { + configured = append(configured, configuredPrincipal{Token: token, Name: "platform-admin", Role: "admin"}) + } + seen := map[[sha256.Size]byte]bool{} + for _, item := range configured { + item.Token = strings.TrimSpace(item.Token) + item.Name = strings.TrimSpace(item.Name) + item.Role = strings.ToLower(strings.TrimSpace(item.Role)) + if len(item.Token) < 16 { + return nil, fmt.Errorf("token for %q must contain at least 16 characters", item.Name) + } + if item.Name == "" || roleRank(item.Role) == 0 { + return nil, fmt.Errorf("token principal requires name and viewer/operator/admin role") + } + hash := sha256.Sum256([]byte(item.Token)) + if seen[hash] { + return nil, fmt.Errorf("duplicate authentication token") + } + seen[hash] = true + authenticator.tokens = append(authenticator.tokens, tokenPrincipal{hash: hash, principal: platform.Principal{Name: item.Name, Role: item.Role}}) + } + if mode == "enforce" && len(authenticator.tokens) == 0 { + return nil, fmt.Errorf("enforce mode requires AUTH_TOKEN or AUTH_TOKENS_JSON") + } + return authenticator, nil +} + +func (a *apiAuthenticator) middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + principal, ok := a.authenticate(r) + if !ok { + w.Header().Set("WWW-Authenticate", `Bearer realm="lingniu-vehicle-platform"`) + httpx.WriteError(w, http.StatusUnauthorized, "AUTH_REQUIRED", "需要有效的访问令牌", "", requestTraceID(r)) + return + } + required := requiredRole(r) + if roleRank(principal.Role) < roleRank(required) { + httpx.WriteError(w, http.StatusForbidden, "PERMISSION_DENIED", "当前角色无权执行该操作", "需要 "+required+" 角色", requestTraceID(r)) + return + } + ctx := platform.WithPrincipal(r.Context(), principal) + r = r.WithContext(ctx) + if r.URL.Path == "/api/v2/session" { + httpx.WriteOK(w, requestTraceID(r), struct { + Name string `json:"name"` + Role string `json:"role"` + AuthMode string `json:"authMode"` + }{principal.Name, principal.Role, a.mode}) + return + } + next.ServeHTTP(w, r) + }) +} + +func (a *apiAuthenticator) authenticate(r *http.Request) (platform.Principal, bool) { + if a.mode == "disabled" { + return platform.Principal{Name: "local-developer", Role: "admin"}, true + } + header := strings.TrimSpace(r.Header.Get("Authorization")) + if len(header) < 8 || !strings.EqualFold(header[:7], "Bearer ") { + return platform.Principal{}, false + } + token := strings.TrimSpace(header[7:]) + if token == "" { + return platform.Principal{}, false + } + hash := sha256.Sum256([]byte(token)) + var match platform.Principal + matched := 0 + for _, item := range a.tokens { + equal := subtle.ConstantTimeCompare(hash[:], item.hash[:]) + matched |= equal + if equal == 1 { + match = item.principal + } + } + return match, matched == 1 +} + +func requiredRole(r *http.Request) string { + if (r.Method == http.MethodGet || r.Method == http.MethodHead) && strings.HasPrefix(r.URL.Path, "/api/v2/exports") { + return "operator" + } + if r.Method == http.MethodGet || r.Method == http.MethodHead { + return "viewer" + } + path := r.URL.Path + if r.Method == http.MethodPost { + switch path { + case "/api/vehicle-service/overviews", "/api/history/raw-frames/query", "/api/v2/access/summary", "/api/v2/access/vehicles", "/api/v2/alerts/summary", "/api/v2/alerts/events": + return "viewer" + case "/api/v2/exports", "/api/v2/alerts/notifications/read": + return "operator" + } + if strings.HasPrefix(path, "/api/v2/alerts/events/") && strings.HasSuffix(path, "/actions") { + return "operator" + } + if path == "/api/v2/alerts/rules" { + return "admin" + } + } + if r.Method == http.MethodPut && (path == "/api/v2/access/thresholds" || strings.HasPrefix(path, "/api/v2/alerts/rules/")) { + return "admin" + } + return "admin" +} + +func roleRank(role string) int { + switch strings.ToLower(role) { + case "viewer": + return 1 + case "operator": + return 2 + case "admin": + return 3 + } + return 0 +} diff --git a/vehicle-data-platform/apps/api/internal/app/auth_test.go b/vehicle-data-platform/apps/api/internal/app/auth_test.go new file mode 100644 index 00000000..c4466643 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/app/auth_test.go @@ -0,0 +1,135 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +const ( + viewerToken = "viewer-token-at-least-16" + operatorToken = "operator-token-at-least-16" + adminToken = "admin-token-at-least-16" +) + +func testAuthConfig() config.Config { + return config.Config{ + AuthMode: "enforce", + AuthTokensJSON: `[ + {"token":"` + viewerToken + `","name":"viewer-a","role":"viewer"}, + {"token":"` + operatorToken + `","name":"operator-a","role":"operator"}, + {"token":"` + adminToken + `","name":"admin-a","role":"admin"} + ]`, + } +} + +func authRequest(t *testing.T, cfg config.Config, method, path, token string) *httptest.ResponseRecorder { + t.Helper() + next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + principal, ok := platform.PrincipalFromContext(r.Context()) + if !ok { + t.Fatal("authenticated request reached handler without principal") + } + w.Header().Set("X-Principal", principal.Name+":"+principal.Role) + w.WriteHeader(http.StatusNoContent) + }) + req := httptest.NewRequest(method, path, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + withAPIAuth(next, cfg).ServeHTTP(rec, req) + return rec +} + +func TestAPIAuthEnforcesTokensAndRoleBoundaries(t *testing.T) { + cfg := testAuthConfig() + + missing := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "") + if missing.Code != http.StatusUnauthorized || !strings.HasPrefix(missing.Header().Get("WWW-Authenticate"), "Bearer") { + t.Fatalf("missing token status=%d headers=%v body=%s", missing.Code, missing.Header(), missing.Body.String()) + } + invalid := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "wrong-token-at-least-16") + if invalid.Code != http.StatusUnauthorized { + t.Fatalf("invalid token status=%d body=%s", invalid.Code, invalid.Body.String()) + } + + viewerQuery := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events", viewerToken) + if viewerQuery.Code != http.StatusNoContent || viewerQuery.Header().Get("X-Principal") != "viewer-a:viewer" { + t.Fatalf("viewer query status=%d principal=%s", viewerQuery.Code, viewerQuery.Header().Get("X-Principal")) + } + viewerAction := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events/a/actions", viewerToken) + if viewerAction.Code != http.StatusForbidden { + t.Fatalf("viewer mutation should be forbidden, status=%d", viewerAction.Code) + } + viewerExports := authRequest(t, cfg, http.MethodGet, "/api/v2/exports", viewerToken) + if viewerExports.Code != http.StatusForbidden { + t.Fatalf("viewer export listing should be forbidden, status=%d", viewerExports.Code) + } + operatorExports := authRequest(t, cfg, http.MethodGet, "/api/v2/exports/exp_1/download", operatorToken) + if operatorExports.Code != http.StatusNoContent { + t.Fatalf("operator export download status=%d", operatorExports.Code) + } + operatorAction := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events/a/actions", operatorToken) + if operatorAction.Code != http.StatusNoContent { + t.Fatalf("operator action status=%d body=%s", operatorAction.Code, operatorAction.Body.String()) + } + operatorRule := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/rules", operatorToken) + if operatorRule.Code != http.StatusForbidden { + t.Fatalf("operator rule mutation should be forbidden, status=%d", operatorRule.Code) + } + operatorProfile := authRequest(t, cfg, http.MethodPut, "/api/v2/vehicles/VIN001/profile", operatorToken) + if operatorProfile.Code != http.StatusForbidden { + t.Fatalf("operator profile mutation should be forbidden, status=%d", operatorProfile.Code) + } + operatorProfileSync := authRequest(t, cfg, http.MethodPost, "/api/v2/vehicle-profiles/sync", operatorToken) + if operatorProfileSync.Code != http.StatusForbidden { + t.Fatalf("operator profile sync should be forbidden, status=%d", operatorProfileSync.Code) + } + adminProfile := authRequest(t, cfg, http.MethodPut, "/api/v2/vehicles/VIN001/profile", adminToken) + if adminProfile.Code != http.StatusNoContent || adminProfile.Header().Get("X-Principal") != "admin-a:admin" { + t.Fatalf("admin profile mutation status=%d principal=%s", adminProfile.Code, adminProfile.Header().Get("X-Principal")) + } + adminProfileSync := authRequest(t, cfg, http.MethodPost, "/api/v2/vehicle-profiles/sync", adminToken) + if adminProfileSync.Code != http.StatusNoContent || adminProfileSync.Header().Get("X-Principal") != "admin-a:admin" { + t.Fatalf("admin profile sync status=%d principal=%s", adminProfileSync.Code, adminProfileSync.Header().Get("X-Principal")) + } + adminThreshold := authRequest(t, cfg, http.MethodPut, "/api/v2/access/thresholds", adminToken) + if adminThreshold.Code != http.StatusNoContent { + t.Fatalf("admin threshold status=%d body=%s", adminThreshold.Code, adminThreshold.Body.String()) + } +} + +func TestAPIAuthSessionAndDisabledMode(t *testing.T) { + rec := authRequest(t, config.Config{AuthMode: "disabled"}, http.MethodGet, "/api/v2/alerts/rules", "") + if rec.Code != http.StatusNoContent || rec.Header().Get("X-Principal") != "local-developer:admin" { + t.Fatalf("disabled mode status=%d principal=%s", rec.Code, rec.Header().Get("X-Principal")) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v2/session", nil) + req.Header.Set("Authorization", "Bearer "+operatorToken) + session := httptest.NewRecorder() + withAPIAuth(http.NotFoundHandler(), testAuthConfig()).ServeHTTP(session, req) + if session.Code != http.StatusOK || !strings.Contains(session.Body.String(), `"name":"operator-a"`) || !strings.Contains(session.Body.String(), `"role":"operator"`) { + t.Fatalf("session status=%d body=%s", session.Code, session.Body.String()) + } +} + +func TestAPIAuthMisconfigurationFailsClosed(t *testing.T) { + cases := []config.Config{ + {AuthMode: "enforce"}, + {AuthMode: "unknown"}, + {AuthMode: "enforce", AuthTokensJSON: `not-json`}, + {AuthMode: "enforce", AuthToken: "short"}, + } + for _, cfg := range cases { + rec := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "") + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("misconfigured auth should fail closed: cfg=%+v status=%d", cfg, rec.Code) + } + } +} diff --git a/vehicle-data-platform/apps/api/internal/app/server.go b/vehicle-data-platform/apps/api/internal/app/server.go index 0a95c95d..22910a39 100644 --- a/vehicle-data-platform/apps/api/internal/app/server.go +++ b/vehicle-data-platform/apps/api/internal/app/server.go @@ -23,13 +23,27 @@ import ( ) func NewServer(cfg config.Config) http.Handler { + dataMode := strings.ToLower(strings.TrimSpace(cfg.DataMode)) + if dataMode == "" { + dataMode = "mock" + } var store platform.Store = platform.NewMockStore() - if cfg.MySQLDSN != "" { + var storeErr error + if dataMode != "mock" && dataMode != "production" { + storeErr = fmt.Errorf("DATA_MODE must be mock or production") + } + if dataMode == "production" && strings.TrimSpace(cfg.MySQLDSN) == "" { + storeErr = fmt.Errorf("production data mode requires MYSQL_DSN") + } + if cfg.MySQLDSN != "" && dataMode == "production" { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN) if err != nil { log.Printf("production mysql store disabled: %v", err) + if dataMode == "production" { + storeErr = fmt.Errorf("connect production mysql: %w", err) + } } else { var tdengine *sql.DB if cfg.TDengineDSN != "" { @@ -49,11 +63,15 @@ func NewServer(cfg config.Config) http.Handler { productionStore.WithCapacityChecker(platform.NewCapacityCheckCommand(cfg.CapacityCheckBin)) log.Printf("production capacity-check probe enabled") } + productionStore.WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup) store = productionStore + storeErr = nil log.Printf("production mysql store enabled") } } - api := platform.NewHandler(platform.NewServiceWithRuntime(store, platform.RuntimeInfo{ + var api http.Handler = platform.NewHandler(platform.NewServiceWithRuntime(store, platform.RuntimeInfo{ + DataMode: dataMode, + ExportDir: strings.TrimSpace(cfg.ExportDir), RequestTimeoutMs: int(cfg.RequestTimeout / time.Millisecond), AMapWebJSConfigured: strings.TrimSpace(cfg.AMapWebJSKey) != "", AMapAPIConfigured: strings.TrimSpace(cfg.AMapAPIKey) != "", @@ -61,8 +79,16 @@ func NewServer(cfg config.Config) http.Handler { AMapSecurityCodeExposed: exposedAMapSecurityCode(cfg) != "", AMapSecurityServiceHost: strings.TrimSpace(cfg.AMapServiceHost), PlatformRelease: strings.TrimSpace(cfg.PlatformRelease), + AlertStreamMode: strings.TrimSpace(cfg.AlertStreamMode), + AlertStreamConsumerGroup: strings.TrimSpace(cfg.AlertStreamKafkaGroup), })) - handler := static.Handler(cfg.StaticDir, api) + if storeErr != nil { + log.Printf("platform data store unavailable: %v", storeErr) + api = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpx.WriteError(w, http.StatusServiceUnavailable, "DATA_STORE_UNAVAILABLE", "生产数据源不可用", storeErr.Error(), requestTraceID(r)) + }) + } + handler := static.Handler(cfg.StaticDir, withAPIAuth(api, cfg)) handler = withAMapReverseGeocodeAPI(handler, cfg, "https://restapi.amap.com", http.DefaultClient) handler = withAppConfig(handler, cfg) handler = withAMapSecurityProxy(handler, cfg, defaultAMapProxyUpstreams(), http.DefaultClient) @@ -244,7 +270,8 @@ func withAMapReverseGeocodeAPI(next http.Handler, cfg config.Config, upstream st } query := target.Query() query.Set("key", apiKey) - query.Set("location", fmt.Sprintf("%.6f,%.6f", longitude, latitude)) + mapLongitude, mapLatitude := wgs84ToGCJ02(longitude, latitude) + query.Set("location", fmt.Sprintf("%.6f,%.6f", mapLongitude, mapLatitude)) query.Set("extensions", "base") query.Set("radius", "1000") query.Set("output", "JSON") @@ -321,6 +348,38 @@ func parseReverseGeocodeCoordinate(query url.Values) (float64, float64, error) { return longitude, latitude, nil } +func wgs84ToGCJ02(longitude float64, latitude float64) (float64, float64) { + if longitude < 72.004 || longitude > 137.8347 || latitude < 0.8293 || latitude > 55.8271 { + return longitude, latitude + } + const semiMajorAxis = 6378245.0 + const eccentricitySquared = 0.006693421622965943 + longitudeOffset := transformGCJLongitude(longitude-105, latitude-35) + latitudeOffset := transformGCJLatitude(longitude-105, latitude-35) + radianLatitude := latitude / 180 * math.Pi + magic := 1 - eccentricitySquared*math.Pow(math.Sin(radianLatitude), 2) + squareRootMagic := math.Sqrt(magic) + convertedLatitude := latitude + latitudeOffset*180/((semiMajorAxis*(1-eccentricitySquared))/(magic*squareRootMagic)*math.Pi) + convertedLongitude := longitude + longitudeOffset*180/(semiMajorAxis/squareRootMagic*math.Cos(radianLatitude)*math.Pi) + return convertedLongitude, convertedLatitude +} + +func transformGCJLatitude(longitude float64, latitude float64) float64 { + value := -100 + 2*longitude + 3*latitude + 0.2*latitude*latitude + 0.1*longitude*latitude + 0.2*math.Sqrt(math.Abs(longitude)) + value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3 + value += (20*math.Sin(latitude*math.Pi) + 40*math.Sin(latitude/3*math.Pi)) * 2 / 3 + value += (160*math.Sin(latitude/12*math.Pi) + 320*math.Sin(latitude*math.Pi/30)) * 2 / 3 + return value +} + +func transformGCJLongitude(longitude float64, latitude float64) float64 { + value := 300 + longitude + 2*latitude + 0.1*longitude*longitude + 0.1*longitude*latitude + 0.1*math.Sqrt(math.Abs(longitude)) + value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3 + value += (20*math.Sin(longitude*math.Pi) + 40*math.Sin(longitude/3*math.Pi)) * 2 / 3 + value += (150*math.Sin(longitude/12*math.Pi) + 300*math.Sin(longitude/30*math.Pi)) * 2 / 3 + return value +} + func isCoordinate(value float64, min float64, max float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= min && value <= max } diff --git a/vehicle-data-platform/apps/api/internal/app/server_test.go b/vehicle-data-platform/apps/api/internal/app/server_test.go index eadc380e..aab202d9 100644 --- a/vehicle-data-platform/apps/api/internal/app/server_test.go +++ b/vehicle-data-platform/apps/api/internal/app/server_test.go @@ -2,12 +2,14 @@ package app import ( "encoding/json" - "lingniu/vehicle-data-platform/apps/api/internal/config" + "fmt" "net/http" "net/http/httptest" "strings" "testing" "time" + + "lingniu/vehicle-data-platform/apps/api/internal/config" ) func TestWithRequestTimeoutAddsContextDeadline(t *testing.T) { @@ -31,6 +33,15 @@ func TestWithRequestTimeoutAddsContextDeadline(t *testing.T) { } } +func TestProductionDataModeFailsClosedWithoutMySQL(t *testing.T) { + handler := NewServer(config.Config{DataMode: "production", RequestTimeout: time.Second}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/ops/health", nil)) + if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "DATA_STORE_UNAVAILABLE") { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } +} + func TestWithRequestTimeoutReturnsEnvelopeWithTraceID(t *testing.T) { handler := withRequestTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() @@ -192,7 +203,8 @@ func TestAMapReverseGeocodeAPIUsesServerSideKey(t *testing.T) { if rec.Code != http.StatusOK { t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) } - if gotKey != "server-api-key" || gotLocation != "113.123457,23.765432" { + mapLongitude, mapLatitude := wgs84ToGCJ02(113.1234567, 23.7654321) + if gotKey != "server-api-key" || gotLocation != fmt.Sprintf("%.6f,%.6f", mapLongitude, mapLatitude) { t.Fatalf("key=%q location=%q", gotKey, gotLocation) } var body struct { diff --git a/vehicle-data-platform/apps/api/internal/config/config.go b/vehicle-data-platform/apps/api/internal/config/config.go index b385f453..a2e9768b 100644 --- a/vehicle-data-platform/apps/api/internal/config/config.go +++ b/vehicle-data-platform/apps/api/internal/config/config.go @@ -3,53 +3,121 @@ package config import ( "os" "strconv" + "strings" "time" ) type Config struct { - HTTPAddr string - StaticDir string - MySQLDSN string - RedisAddr string - RedisUsername string - RedisPassword string - RedisDB int - TDengineDriver string - TDengineDSN string - TDengineDatabase string - CapacityCheckBin string - AuthToken string - RequestTimeout time.Duration - AMapWebJSKey string - AMapAPIKey string - AMapSecurityCode string - AMapServiceHost string - PlatformRelease string + HTTPAddr string + StaticDir string + MySQLDSN string + RedisAddr string + RedisUsername string + RedisPassword string + RedisDB int + TDengineDriver string + TDengineDSN string + TDengineDatabase string + CapacityCheckBin string + AuthToken string + AuthMode string + AuthTokensJSON string + DataMode string + ExportDir string + RequestTimeout time.Duration + AMapWebJSKey string + AMapAPIKey string + AMapSecurityCode string + AMapServiceHost string + PlatformRelease string + AlertEvaluationInterval time.Duration + AlertStreamMode string + AlertStreamKafkaBrokers []string + AlertStreamKafkaTopics []string + AlertStreamKafkaGroup string + AlertStreamBatchSize int + AlertStreamBatchWait time.Duration + AlertStreamLateness time.Duration } func Load() Config { return Config{ - HTTPAddr: env("HTTP_ADDR", ":20300"), - StaticDir: env("STATIC_DIR", ""), - MySQLDSN: os.Getenv("MYSQL_DSN"), - RedisAddr: os.Getenv("REDIS_ADDR"), - RedisUsername: os.Getenv("REDIS_USERNAME"), - RedisPassword: os.Getenv("REDIS_PASSWORD"), - RedisDB: envInt("REDIS_DB", 50), - TDengineDriver: env("TDENGINE_DRIVER", "taosWS"), - TDengineDSN: os.Getenv("TDENGINE_DSN"), - TDengineDatabase: env("TDENGINE_DATABASE", "lingniu_vehicle_ts"), - CapacityCheckBin: os.Getenv("CAPACITY_CHECK_BIN"), - AuthToken: os.Getenv("AUTH_TOKEN"), - RequestTimeout: time.Duration(envInt("REQUEST_TIMEOUT_MS", 5000)) * time.Millisecond, - AMapWebJSKey: os.Getenv("AMAP_WEB_JS_KEY"), - AMapAPIKey: os.Getenv("AMAP_API_KEY"), - AMapSecurityCode: os.Getenv("AMAP_SECURITY_JS_CODE"), - AMapServiceHost: os.Getenv("AMAP_SECURITY_SERVICE_HOST"), - PlatformRelease: os.Getenv("PLATFORM_RELEASE"), + HTTPAddr: env("HTTP_ADDR", ":20300"), + StaticDir: env("STATIC_DIR", ""), + MySQLDSN: os.Getenv("MYSQL_DSN"), + RedisAddr: os.Getenv("REDIS_ADDR"), + RedisUsername: os.Getenv("REDIS_USERNAME"), + RedisPassword: os.Getenv("REDIS_PASSWORD"), + RedisDB: envInt("REDIS_DB", 50), + TDengineDriver: env("TDENGINE_DRIVER", "taosWS"), + TDengineDSN: os.Getenv("TDENGINE_DSN"), + TDengineDatabase: env("TDENGINE_DATABASE", "lingniu_vehicle_ts"), + CapacityCheckBin: os.Getenv("CAPACITY_CHECK_BIN"), + AuthToken: os.Getenv("AUTH_TOKEN"), + AuthMode: authMode(), + AuthTokensJSON: os.Getenv("AUTH_TOKENS_JSON"), + DataMode: dataMode(), + ExportDir: os.Getenv("EXPORT_DIR"), + RequestTimeout: time.Duration(envInt("REQUEST_TIMEOUT_MS", 5000)) * time.Millisecond, + AMapWebJSKey: os.Getenv("AMAP_WEB_JS_KEY"), + AMapAPIKey: os.Getenv("AMAP_API_KEY"), + AMapSecurityCode: os.Getenv("AMAP_SECURITY_JS_CODE"), + AMapServiceHost: os.Getenv("AMAP_SECURITY_SERVICE_HOST"), + PlatformRelease: os.Getenv("PLATFORM_RELEASE"), + AlertEvaluationInterval: time.Duration(envInt("ALERT_EVALUATION_INTERVAL_SEC", 10)) * time.Second, + AlertStreamMode: strings.ToLower(env("ALERT_STREAM_MODE", "disabled")), + AlertStreamKafkaBrokers: splitCSV(firstEnv("ALERT_STREAM_KAFKA_BROKERS", "KAFKA_BROKERS")), + AlertStreamKafkaTopics: splitCSV(env("ALERT_STREAM_KAFKA_TOPICS", "vehicle.fields.go.gb32960.v1,vehicle.fields.go.jt808.v1,vehicle.fields.go.yutong-mqtt.v1")), + AlertStreamKafkaGroup: env("ALERT_STREAM_KAFKA_GROUP", "vehicle-alert-stream-v1"), + AlertStreamBatchSize: envInt("ALERT_STREAM_BATCH_SIZE", 200), + AlertStreamBatchWait: time.Duration(envInt("ALERT_STREAM_BATCH_WAIT_MS", 100)) * time.Millisecond, + AlertStreamLateness: time.Duration(envInt("ALERT_STREAM_LATENESS_SEC", 120)) * time.Second, } } +func firstEnv(keys ...string) string { + for _, key := range keys { + if value := os.Getenv(key); value != "" { + return value + } + } + return "" +} + +func splitCSV(raw string) []string { + parts := strings.Split(raw, ",") + values := make([]string, 0, len(parts)) + seen := map[string]bool{} + for _, part := range parts { + part = strings.TrimSpace(part) + if part != "" && !seen[part] { + seen[part] = true + values = append(values, part) + } + } + return values +} + +func dataMode() string { + if value := os.Getenv("DATA_MODE"); value != "" { + return value + } + if os.Getenv("MYSQL_DSN") != "" { + return "production" + } + return "mock" +} + +func authMode() string { + if value := os.Getenv("AUTH_MODE"); value != "" { + return value + } + if os.Getenv("AUTH_TOKEN") != "" || os.Getenv("AUTH_TOKENS_JSON") != "" { + return "enforce" + } + return "disabled" +} + func env(key, fallback string) string { if value := os.Getenv(key); value != "" { return value diff --git a/vehicle-data-platform/apps/api/internal/config/config_test.go b/vehicle-data-platform/apps/api/internal/config/config_test.go index db03fcad..43bc75a0 100644 --- a/vehicle-data-platform/apps/api/internal/config/config_test.go +++ b/vehicle-data-platform/apps/api/internal/config/config_test.go @@ -34,3 +34,19 @@ func TestLoadReadsAMapServerAPIKey(t *testing.T) { t.Fatalf("AMapAPIKey = %q", cfg.AMapAPIKey) } } + +func TestLoadSelectsProductionDataModeWhenMySQLIsConfigured(t *testing.T) { + t.Setenv("DATA_MODE", "") + t.Setenv("MYSQL_DSN", "user:password@tcp(db:3306)/platform") + if cfg := Load(); cfg.DataMode != "production" { + t.Fatalf("DataMode = %q, want production", cfg.DataMode) + } +} + +func TestLoadKeepsExplicitMockDataMode(t *testing.T) { + t.Setenv("DATA_MODE", "mock") + t.Setenv("MYSQL_DSN", "user:password@tcp(db:3306)/platform") + if cfg := Load(); cfg.DataMode != "mock" { + t.Fatalf("DataMode = %q, want mock", cfg.DataMode) + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/access.go b/vehicle-data-platform/apps/api/internal/platform/access.go new file mode 100644 index 00000000..924d0915 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/access.go @@ -0,0 +1,480 @@ +package platform + +import ( + "context" + "fmt" + "sort" + "strings" + "time" +) + +type accessEvidenceStore interface { + AccessEvidence(context.Context) ([]AccessEvidenceRow, error) +} + +type accessThresholdStore interface { + AccessThresholds(context.Context) (AccessThresholdConfig, error) + SaveAccessThresholds(context.Context, AccessThresholdUpdate) (AccessThresholdConfig, error) +} + +type accessUnresolvedIdentityStore interface { + AccessUnresolvedIdentities(context.Context, AccessUnresolvedIdentityQuery) (Page[AccessUnresolvedIdentity], error) +} + +func defaultAccessThresholds(now time.Time) AccessThresholdConfig { + return AccessThresholdConfig{ + Version: 1, + DefaultThresholdSec: 300, + DelayThresholdSec: 30, + LongOfflineSec: 1800, + Protocols: []AccessProtocolThreshold{ + {Protocol: "GB32960", ThresholdSec: 300}, + {Protocol: "JT808", ThresholdSec: 300}, + {Protocol: "YUTONG_MQTT", ThresholdSec: 600}, + }, + UpdatedBy: "system", + UpdatedAt: now.Format(time.RFC3339), + } +} + +func (s *Service) AccessThresholds(ctx context.Context) (AccessThresholdConfig, error) { + store, ok := s.store.(accessThresholdStore) + if !ok { + return defaultAccessThresholds(time.Now()), nil + } + config, err := store.AccessThresholds(ctx) + if err != nil { + return AccessThresholdConfig{}, err + } + return normalizeAccessThresholdConfig(config, time.Now()), nil +} + +func (s *Service) UpdateAccessThresholds(ctx context.Context, update AccessThresholdUpdate) (AccessThresholdConfig, error) { + if err := validateAccessThresholdUpdate(update); err != nil { + return AccessThresholdConfig{}, err + } + store, ok := s.store.(accessThresholdStore) + if !ok { + return AccessThresholdConfig{}, clientError{Code: "ACCESS_THRESHOLD_READ_ONLY", Message: "当前存储不支持更新接入阈值"} + } + update.Actor = strings.TrimSpace(update.Actor) + if update.Actor == "" { + update.Actor = "platform-admin" + } + return store.SaveAccessThresholds(ctx, update) +} + +func (s *Service) AccessVehicles(ctx context.Context, query AccessQuery) (Page[AccessVehicleRow], error) { + rows, config, err := s.accessRows(ctx, query) + if err != nil { + return Page[AccessVehicleRow]{}, err + } + _ = config + limit := query.Limit + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + offset := query.Offset + if offset < 0 { + offset = 0 + } + total := len(rows) + if offset >= total { + return Page[AccessVehicleRow]{Items: []AccessVehicleRow{}, Total: total, Limit: limit, Offset: offset}, nil + } + end := offset + limit + if end > total { + end = total + } + return Page[AccessVehicleRow]{Items: append([]AccessVehicleRow(nil), rows[offset:end]...), Total: total, Limit: limit, Offset: offset}, nil +} + +func (s *Service) AccessUnresolvedIdentities(ctx context.Context, query AccessUnresolvedIdentityQuery) (Page[AccessUnresolvedIdentity], error) { + store, ok := s.store.(accessUnresolvedIdentityStore) + if !ok { + return Page[AccessUnresolvedIdentity]{}, fmt.Errorf("store does not provide unresolved identity evidence") + } + query.Keyword = strings.TrimSpace(query.Keyword) + query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) + if query.Protocol != "" && query.Protocol != "JT808" { + return Page[AccessUnresolvedIdentity]{Items: []AccessUnresolvedIdentity{}, Limit: 50}, nil + } + if query.Limit <= 0 { + query.Limit = 50 + } + if query.Limit > 200 { + query.Limit = 200 + } + if query.Offset < 0 { + query.Offset = 0 + } + return store.AccessUnresolvedIdentities(ctx, query) +} + +func (s *Service) AccessSummary(ctx context.Context, query AccessQuery) (AccessSummary, error) { + rows, config, err := s.accessRows(ctx, query) + if err != nil { + return AccessSummary{}, err + } + now := time.Now() + result := AccessSummary{TotalVehicles: len(rows), AsOf: now.Format(time.RFC3339), ThresholdVersion: config.Version} + protocols := map[string]*AccessDistribution{} + oems := map[string]*AccessDistribution{} + for _, row := range rows { + switch row.OnlineState { + case "online": + result.OnlineVehicles++ + case "offline": + result.OfflineVehicles++ + if row.FreshnessSec != nil && *row.FreshnessSec >= config.LongOfflineSec { + result.LongOfflineVehicles++ + } + case "never_reported": + result.NeverReported++ + default: + result.UnknownVehicles++ + } + if row.DelayAbnormal { + result.DelayAbnormal++ + } + if reportedOnDay(row.LatestReceivedAt, now) { + result.ReportedToday++ + } + addAccessDistribution(protocols, firstNonEmpty(row.Protocol, "未识别"), row.OnlineState == "online") + addAccessDistribution(oems, firstNonEmpty(row.OEM, "未维护"), row.OnlineState == "online") + } + if result.TotalVehicles > 0 { + result.OnlineRate = float64(result.OnlineVehicles) / float64(result.TotalVehicles) * 100 + } + result.Protocols = sortedAccessDistributions(protocols) + result.OEMs = sortedAccessDistributions(oems) + return result, nil +} + +func (s *Service) accessRows(ctx context.Context, query AccessQuery) ([]AccessVehicleRow, AccessThresholdConfig, error) { + if err := validateAccessQuery(query); err != nil { + return nil, AccessThresholdConfig{}, err + } + store, ok := s.store.(accessEvidenceStore) + if !ok { + return nil, AccessThresholdConfig{}, fmt.Errorf("store does not provide access evidence") + } + config, err := s.AccessThresholds(ctx) + if err != nil { + return nil, AccessThresholdConfig{}, err + } + evidence, err := store.AccessEvidence(ctx) + if err != nil { + return nil, AccessThresholdConfig{}, err + } + now := time.Now() + rows := make([]AccessVehicleRow, 0, len(evidence)) + for _, item := range evidence { + row := buildAccessVehicleRow(item, config, now) + if keepAccessRow(row, query) { + rows = append(rows, row) + } + } + sort.SliceStable(rows, func(i, j int) bool { + left, right := accessStateRank(rows[i].OnlineState), accessStateRank(rows[j].OnlineState) + if left != right { + return left < right + } + if rows[i].LatestReceivedAt != rows[j].LatestReceivedAt { + return rows[i].LatestReceivedAt > rows[j].LatestReceivedAt + } + return rows[i].VIN < rows[j].VIN + }) + return rows, config, nil +} + +func buildAccessVehicleRow(item AccessEvidenceRow, config AccessThresholdConfig, now time.Time) AccessVehicleRow { + threshold := config.DefaultThresholdSec + for _, override := range config.Protocols { + if strings.EqualFold(strings.TrimSpace(override.Protocol), strings.TrimSpace(item.Protocol)) { + threshold = override.ThresholdSec + break + } + } + row := AccessVehicleRow{ + VIN: strings.TrimSpace(item.VIN), + Plate: strings.TrimSpace(item.Plate), + OEM: strings.TrimSpace(item.OEM), + Model: strings.TrimSpace(item.Model), + Company: strings.TrimSpace(item.Company), + Protocol: strings.TrimSpace(item.Protocol), + Provider: strings.TrimSpace(item.Provider), + Source: strings.TrimSpace(item.Source), + FirstSeenAt: normalizeAccessTime(item.FirstSeenAt), + LatestEventAt: normalizeAccessTime(item.LatestEventAt), + LatestReceivedAt: normalizeAccessTime(item.LatestReceivedAt), + ReportIntervalSec: item.ReportIntervalSec, + ThresholdSec: threshold, + LatestMessageType: firstNonEmpty(strings.TrimSpace(item.LatestMessageType), accessMessageType(item.Protocol)), + LatestEventID: strings.TrimSpace(item.LatestEventID), + LatestError: strings.TrimSpace(item.LatestError), + FirstSeenEvidence: strings.TrimSpace(item.FirstSeenEvidence), + FirstSeenSource: strings.TrimSpace(item.FirstSeenSource), + ReportIntervalProof: strings.TrimSpace(item.ReportIntervalProof), + ReportSampleCount: item.ReportSampleCount, + } + if row.FirstSeenEvidence == "" { + row.FirstSeenEvidence = "现有实时快照不保存首次接入时间" + } + if row.ReportIntervalProof == "" { + row.ReportIntervalProof = "需要连续上报样本后才能计算" + } + eventAt, eventOK := parseAccessTime(item.LatestEventAt) + receivedAt, receivedOK := parseAccessTime(item.LatestReceivedAt) + if eventOK && receivedOK { + delay := int(receivedAt.Sub(eventAt).Seconds()) + row.DataDelaySec = &delay + row.DelayAbnormal = delay < 0 || delay > config.DelayThresholdSec + if delay < 0 && row.LatestError == "" { + row.LatestError = "接收时间早于事件时间" + } + } + latest, latestOK := receivedAt, receivedOK + if !latestOK { + latest, latestOK = parseAccessTime(item.LatestUpdatedAt) + } + switch { + case strings.TrimSpace(item.Protocol) == "" && !latestOK && !eventOK: + row.OnlineState = "never_reported" + case !latestOK: + row.OnlineState = "unknown" + if row.LatestError == "" { + row.LatestError = "缺少可解析的接收时间" + } + default: + freshness := int(now.Sub(latest).Seconds()) + if freshness < 0 { + freshness = 0 + } + row.FreshnessSec = &freshness + if freshness <= threshold { + row.OnlineState = "online" + } else { + row.OnlineState = "offline" + } + } + return row +} + +func keepAccessRow(row AccessVehicleRow, query AccessQuery) bool { + keyword := strings.ToLower(strings.TrimSpace(query.Keyword)) + if keyword != "" && !strings.Contains(strings.ToLower(row.VIN), keyword) && !strings.Contains(strings.ToLower(row.Plate), keyword) { + return false + } + if value := strings.TrimSpace(query.Protocol); value != "" && !strings.EqualFold(value, row.Protocol) { + return false + } + if value := strings.TrimSpace(query.OEM); value != "" && !strings.EqualFold(value, row.OEM) { + return false + } + if value := strings.ToLower(strings.TrimSpace(query.Model)); value != "" && !strings.Contains(strings.ToLower(row.Model), value) { + return false + } + if value := strings.ToLower(strings.TrimSpace(query.Provider)); value != "" && !strings.Contains(strings.ToLower(row.Provider), value) { + return false + } + if !accessTimeMatches(row.FirstSeenAt, query.FirstSeenFrom, query.FirstSeenTo) || !accessTimeMatches(row.LatestReceivedAt, query.LatestSeenFrom, query.LatestSeenTo) { + return false + } + if value := strings.TrimSpace(query.OnlineState); value != "" && value != "all" && value != row.OnlineState { + return false + } + switch strings.TrimSpace(query.DelayState) { + case "abnormal": + return row.DelayAbnormal + case "normal": + return row.DataDelaySec != nil && !row.DelayAbnormal + } + return true +} + +func validateAccessQuery(query AccessQuery) error { + for _, item := range []struct{ name, from, to string }{{"首次接入", query.FirstSeenFrom, query.FirstSeenTo}, {"最新上报", query.LatestSeenFrom, query.LatestSeenTo}} { + start, startOK := parseAccessFilterTime(item.from) + end, endOK := parseAccessFilterTime(item.to) + if strings.TrimSpace(item.from) != "" && !startOK || strings.TrimSpace(item.to) != "" && !endOK { + return clientError{Code: "ACCESS_TIME_INVALID", Message: item.name + "时间格式无效"} + } + if startOK && endOK && start.After(end) { + return clientError{Code: "ACCESS_TIME_RANGE_INVALID", Message: item.name + "开始时间不能晚于结束时间"} + } + } + return nil +} + +func parseAccessFilterTime(value string) (time.Time, bool) { + if strings.TrimSpace(value) == "" { + return time.Time{}, false + } + if parsed, ok := parseTrackRequestTime(value); ok { + return parsed, true + } + return parseAccessTime(value) +} + +func accessTimeMatches(value, from, to string) bool { + if strings.TrimSpace(from) == "" && strings.TrimSpace(to) == "" { + return true + } + actual, ok := parseAccessTime(value) + if !ok { + return false + } + if start, ok := parseAccessFilterTime(from); ok && actual.Before(start) { + return false + } + if end, ok := parseAccessFilterTime(to); ok && actual.After(end) { + return false + } + return true +} + +func validateAccessThresholdUpdate(update AccessThresholdUpdate) error { + if update.Version <= 0 { + return clientError{Code: "ACCESS_THRESHOLD_VERSION_REQUIRED", Message: "阈值版本不能为空"} + } + if update.DefaultThresholdSec < 30 || update.DefaultThresholdSec > 86400 { + return clientError{Code: "ACCESS_THRESHOLD_INVALID", Message: "全局在线阈值必须在 30 秒到 24 小时之间"} + } + if update.DelayThresholdSec < 1 || update.DelayThresholdSec > 3600 { + return clientError{Code: "ACCESS_DELAY_THRESHOLD_INVALID", Message: "延迟阈值必须在 1 秒到 1 小时之间"} + } + if update.LongOfflineSec < update.DefaultThresholdSec || update.LongOfflineSec > 604800 { + return clientError{Code: "ACCESS_LONG_OFFLINE_INVALID", Message: "长离线阈值必须不小于在线阈值且不超过 7 天"} + } + seen := map[string]struct{}{} + for _, item := range update.Protocols { + protocol := strings.ToUpper(strings.TrimSpace(item.Protocol)) + if protocol == "" || item.ThresholdSec < 30 || item.ThresholdSec > 86400 { + return clientError{Code: "ACCESS_PROTOCOL_THRESHOLD_INVALID", Message: "协议阈值必须包含协议名且位于 30 秒到 24 小时之间"} + } + if _, exists := seen[protocol]; exists { + return clientError{Code: "ACCESS_PROTOCOL_THRESHOLD_DUPLICATED", Message: "协议阈值不能重复"} + } + seen[protocol] = struct{}{} + } + return nil +} + +func normalizeAccessThresholdConfig(config AccessThresholdConfig, now time.Time) AccessThresholdConfig { + defaults := defaultAccessThresholds(now) + if config.Version <= 0 { + return defaults + } + if config.DefaultThresholdSec <= 0 { + config.DefaultThresholdSec = defaults.DefaultThresholdSec + } + if config.DelayThresholdSec <= 0 { + config.DelayThresholdSec = defaults.DelayThresholdSec + } + if config.LongOfflineSec <= 0 { + config.LongOfflineSec = defaults.LongOfflineSec + } + if config.Protocols == nil { + config.Protocols = []AccessProtocolThreshold{} + } + if config.Audit == nil { + config.Audit = []AccessThresholdAudit{} + } + return config +} + +func parseAccessTime(value string) (time.Time, bool) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed, true + } + } + for _, layout := range []string{"2006-01-02 15:04:05.000", "2006-01-02 15:04:05"} { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return parsed, true + } + } + return time.Time{}, false +} + +func normalizeAccessTime(value string) string { + parsed, ok := parseAccessTime(value) + if !ok { + return "" + } + return parsed.Format(time.RFC3339) +} + +func accessMessageType(protocol string) string { + switch strings.ToUpper(strings.TrimSpace(protocol)) { + case "GB32960": + return "实时信息上报" + case "JT808": + return "位置信息汇报" + case "YUTONG_MQTT": + return "实时遥测" + default: + return "" + } +} + +func accessStateRank(state string) int { + switch state { + case "online": + return 0 + case "offline": + return 1 + case "never_reported": + return 2 + default: + return 3 + } +} + +func reportedOnDay(value string, day time.Time) bool { + parsed, ok := parseAccessTime(value) + if !ok { + return false + } + year, month, date := parsed.In(day.Location()).Date() + wantYear, wantMonth, wantDate := day.Date() + return year == wantYear && month == wantMonth && date == wantDate +} + +func addAccessDistribution(items map[string]*AccessDistribution, name string, online bool) { + item := items[name] + if item == nil { + item = &AccessDistribution{Name: name} + items[name] = item + } + item.Total++ + if online { + item.Online++ + } +} + +func sortedAccessDistributions(items map[string]*AccessDistribution) []AccessDistribution { + result := make([]AccessDistribution, 0, len(items)) + for _, item := range items { + value := *item + if value.Total > 0 { + value.OnlineRate = float64(value.Online) / float64(value.Total) * 100 + } + result = append(result, value) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Total != result[j].Total { + return result[i].Total > result[j].Total + } + return result[i].Name < result[j].Name + }) + return result +} diff --git a/vehicle-data-platform/apps/api/internal/platform/access_store.go b/vehicle-data-platform/apps/api/internal/platform/access_store.go new file mode 100644 index 00000000..a9bd7a23 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/access_store.go @@ -0,0 +1,255 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +const accessEvidenceLimit = 50000 + +func buildAccessUnresolvedIdentityWhere(query AccessUnresolvedIdentityQuery) (string, []any) { + where := []string{ + `COALESCE(NULLIF(TRIM(b.vin),''),'')=''`, + `(r.vin IS NULL OR TRIM(r.vin)='' OR LOWER(TRIM(r.vin))='unknown')`, + } + args := []any{} + if keyword := strings.TrimSpace(query.Keyword); keyword != "" { + like := "%" + keyword + "%" + where = append(where, `(r.phone LIKE ? OR r.plate LIKE ? OR r.manufacturer LIKE ? OR r.source_endpoint LIKE ?)`) + args = append(args, like, like, like, like) + } + return strings.Join(where, " AND "), args +} + +func (s *ProductionStore) AccessUnresolvedIdentities(ctx context.Context, query AccessUnresolvedIdentityQuery) (Page[AccessUnresolvedIdentity], error) { + where, args := buildAccessUnresolvedIdentityWhere(query) + from := ` FROM jt808_registration r LEFT JOIN vehicle_identity_binding b ON b.phone=r.phone WHERE ` + where + var total int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*)`+from, args...).Scan(&total); err != nil { + return Page[AccessUnresolvedIdentity]{}, err + } + listArgs := append(append([]any(nil), args...), query.Limit, query.Offset) + rows, err := s.db.QueryContext(ctx, `SELECT +SHA2(CONCAT('JT808:',r.phone),256), +'JT808', +CASE WHEN CHAR_LENGTH(r.phone)>=7 THEN CONCAT(LEFT(r.phone,3),'****',RIGHT(r.phone,4)) ELSE '***' END, +COALESCE(r.plate,''),COALESCE(r.manufacturer,''),COALESCE(r.source_endpoint,''), +COALESCE(DATE_FORMAT(r.first_registered_at,'%Y-%m-%d %H:%i:%s'),''), +COALESCE(DATE_FORMAT(r.latest_registered_at,'%Y-%m-%d %H:%i:%s'),''), +COALESCE(DATE_FORMAT(r.latest_authenticated_at,'%Y-%m-%d %H:%i:%s'),''), +COALESCE(DATE_FORMAT(r.latest_seen_at,'%Y-%m-%d %H:%i:%s'),''), +GREATEST(0,TIMESTAMPDIFF(SECOND,r.latest_seen_at,NOW())), +'missing_vin_jt808', +'核对终端手机号、车牌和厂家后维护 phone→VIN 权威绑定；禁止猜测 VIN' +`+from+` ORDER BY r.latest_seen_at DESC,r.phone ASC LIMIT ? OFFSET ?`, listArgs...) + if err != nil { + return Page[AccessUnresolvedIdentity]{}, err + } + defer rows.Close() + items := make([]AccessUnresolvedIdentity, 0, query.Limit) + for rows.Next() { + var item AccessUnresolvedIdentity + if err := rows.Scan(&item.ID, &item.Protocol, &item.IdentifierMasked, &item.Plate, &item.Manufacturer, &item.SourceEndpoint, &item.FirstRegisteredAt, &item.LatestRegisteredAt, &item.LatestAuthenticatedAt, &item.LatestSeenAt, &item.FreshnessSec, &item.IssueCode, &item.RecommendedAction); err != nil { + return Page[AccessUnresolvedIdentity]{}, err + } + items = append(items, item) + } + return Page[AccessUnresolvedIdentity]{Items: items, Total: total, Limit: query.Limit, Offset: query.Offset}, rows.Err() +} + +func (s *ProductionStore) AccessEvidence(ctx context.Context) ([]AccessEvidenceRow, error) { + rows, err := s.db.QueryContext(ctx, `SELECT +v.vin, +COALESCE(NULLIF(s.plate, ''), NULLIF(b.plate, ''), '') AS plate, +COALESCE(NULLIF(b.oem, ''), '') AS oem, +COALESCE(p.model_name, '') AS model_name, +COALESCE(p.company_name, '') AS company_name, +COALESCE(s.protocol, '') AS protocol, +COALESCE(s.platform_name, '') AS provider, +COALESCE(DATE_FORMAT(s.access_first_seen_at, '%Y-%m-%d %H:%i:%s.%f'), '') AS first_seen_at, +COALESCE(s.access_first_seen_source, '') AS first_seen_source, +COALESCE(DATE_FORMAT(s.event_time, '%Y-%m-%d %H:%i:%s.%f'), '') AS event_time, +COALESCE(DATE_FORMAT(s.access_latest_received_at, '%Y-%m-%d %H:%i:%s.%f'), '') AS received_at, +COALESCE(DATE_FORMAT(s.access_latest_received_at, '%Y-%m-%d %H:%i:%s.%f'), '') AS updated_at, +COALESCE(s.access_report_interval_ms, -1) AS report_interval_ms, +COALESCE(s.access_sample_count, 0) AS report_sample_count, +COALESCE(s.event_id, '') AS event_id +FROM ( + SELECT vin FROM vehicle_identity_binding WHERE vin IS NOT NULL AND vin <> '' + UNION + SELECT vin FROM vehicle_realtime_snapshot WHERE vin IS NOT NULL AND vin <> '' +) v +LEFT JOIN vehicle_identity_binding b ON b.vin = v.vin +LEFT JOIN vehicle_profile p ON p.vin = v.vin +LEFT JOIN vehicle_realtime_snapshot s ON s.vin = v.vin + AND NOT EXISTS ( + SELECT 1 FROM vehicle_realtime_snapshot newer + WHERE newer.vin = s.vin + AND (newer.access_latest_received_at > s.access_latest_received_at OR (newer.access_latest_received_at = s.access_latest_received_at AND newer.protocol < s.protocol)) + ) +ORDER BY s.access_latest_received_at DESC, v.vin ASC +LIMIT ?`, accessEvidenceLimit+1) + if err != nil { + return nil, err + } + defer rows.Close() + items := make([]AccessEvidenceRow, 0, 1024) + for rows.Next() { + var row AccessEvidenceRow + var reportIntervalMS int64 + if err := rows.Scan(&row.VIN, &row.Plate, &row.OEM, &row.Model, &row.Company, &row.Protocol, &row.Provider, &row.FirstSeenAt, &row.FirstSeenSource, &row.LatestEventAt, &row.LatestReceivedAt, &row.LatestUpdatedAt, &reportIntervalMS, &row.ReportSampleCount, &row.LatestEventID); err != nil { + return nil, err + } + if reportIntervalMS >= 0 { + seconds := int((reportIntervalMS + 500) / 1000) + row.ReportIntervalSec = &seconds + } + if row.Protocol == "" { + row.Source = "vehicle_identity_binding" + row.LatestError = "车辆已建档但从未形成实时快照" + } else { + row.Source = "vehicle_realtime_snapshot" + switch row.FirstSeenSource { + case "live_writer": + row.FirstSeenEvidence = "网关实时写入首次观测" + case "snapshot_backfill": + row.FirstSeenEvidence = "上线基线：由实时快照回填（非历史首次接入）" + } + if row.ReportSampleCount >= 2 && row.ReportIntervalSec != nil { + row.ReportIntervalProof = fmt.Sprintf("网关连续接收时间差（持久样本 %d 条）", row.ReportSampleCount) + } else if row.ReportSampleCount == 1 { + row.ReportIntervalProof = "仅有 1 个持久接收样本，等待下一次上报" + } + } + items = append(items, row) + if len(items) > accessEvidenceLimit { + return nil, fmt.Errorf("access evidence exceeds safety limit %d", accessEvidenceLimit) + } + } + return items, rows.Err() +} + +func (s *ProductionStore) ensureAccessSchema(ctx context.Context) error { + s.accessSchemaOnce.Do(func() { + statements := []string{ + `CREATE TABLE IF NOT EXISTS vehicle_access_threshold_config ( + id TINYINT NOT NULL PRIMARY KEY, + version INT NOT NULL, + default_threshold_sec INT NOT NULL, + delay_threshold_sec INT NOT NULL, + long_offline_sec INT NOT NULL, + protocol_overrides_json LONGTEXT NOT NULL, + updated_by VARCHAR(128) NOT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +)`, + `CREATE TABLE IF NOT EXISTS vehicle_access_threshold_audit ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + version INT NOT NULL, + actor VARCHAR(128) NOT NULL, + summary VARCHAR(255) NOT NULL, + config_json LONGTEXT NOT NULL, + changed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY idx_access_threshold_audit_version (version), + KEY idx_access_threshold_audit_changed (changed_at) +)`, + } + for _, statement := range statements { + if _, err := s.db.ExecContext(ctx, statement); err != nil { + s.accessSchemaErr = err + return + } + } + defaults := defaultAccessThresholds(time.Now()) + protocols, _ := json.Marshal(defaults.Protocols) + _, s.accessSchemaErr = s.db.ExecContext(ctx, `INSERT IGNORE INTO vehicle_access_threshold_config +(id, version, default_threshold_sec, delay_threshold_sec, long_offline_sec, protocol_overrides_json, updated_by) +VALUES (1, ?, ?, ?, ?, ?, ?)`, defaults.Version, defaults.DefaultThresholdSec, defaults.DelayThresholdSec, defaults.LongOfflineSec, string(protocols), defaults.UpdatedBy) + }) + return s.accessSchemaErr +} + +func (s *ProductionStore) AccessThresholds(ctx context.Context) (AccessThresholdConfig, error) { + if err := s.ensureAccessSchema(ctx); err != nil { + return AccessThresholdConfig{}, err + } + var config AccessThresholdConfig + var protocolsJSON string + err := s.db.QueryRowContext(ctx, `SELECT version, default_threshold_sec, delay_threshold_sec, long_offline_sec, +protocol_overrides_json, updated_by, DATE_FORMAT(updated_at, '%Y-%m-%d %H:%i:%s') +FROM vehicle_access_threshold_config WHERE id = 1`).Scan( + &config.Version, &config.DefaultThresholdSec, &config.DelayThresholdSec, &config.LongOfflineSec, + &protocolsJSON, &config.UpdatedBy, &config.UpdatedAt, + ) + if err != nil { + return AccessThresholdConfig{}, err + } + if err := json.Unmarshal([]byte(protocolsJSON), &config.Protocols); err != nil { + return AccessThresholdConfig{}, fmt.Errorf("decode access protocol thresholds: %w", err) + } + config.UpdatedAt = normalizeAccessTime(config.UpdatedAt) + auditRows, err := s.db.QueryContext(ctx, `SELECT version, actor, DATE_FORMAT(changed_at, '%Y-%m-%d %H:%i:%s'), summary +FROM vehicle_access_threshold_audit ORDER BY changed_at DESC, id DESC LIMIT 10`) + if err != nil { + return AccessThresholdConfig{}, err + } + defer auditRows.Close() + for auditRows.Next() { + var item AccessThresholdAudit + if err := auditRows.Scan(&item.Version, &item.Actor, &item.ChangedAt, &item.Summary); err != nil { + return AccessThresholdConfig{}, err + } + item.ChangedAt = normalizeAccessTime(item.ChangedAt) + config.Audit = append(config.Audit, item) + } + return config, auditRows.Err() +} + +func (s *ProductionStore) SaveAccessThresholds(ctx context.Context, update AccessThresholdUpdate) (AccessThresholdConfig, error) { + if err := s.ensureAccessSchema(ctx); err != nil { + return AccessThresholdConfig{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return AccessThresholdConfig{}, err + } + defer tx.Rollback() + var currentVersion int + if err := tx.QueryRowContext(ctx, `SELECT version FROM vehicle_access_threshold_config WHERE id = 1 FOR UPDATE`).Scan(¤tVersion); err != nil { + return AccessThresholdConfig{}, err + } + if currentVersion != update.Version { + return AccessThresholdConfig{}, clientError{Code: "ACCESS_THRESHOLD_VERSION_CONFLICT", Message: "阈值配置已被其他用户更新，请刷新后重试"} + } + protocols, err := json.Marshal(update.Protocols) + if err != nil { + return AccessThresholdConfig{}, err + } + nextVersion := currentVersion + 1 + result, err := tx.ExecContext(ctx, `UPDATE vehicle_access_threshold_config SET +version = ?, default_threshold_sec = ?, delay_threshold_sec = ?, long_offline_sec = ?, +protocol_overrides_json = ?, updated_by = ?, updated_at = CURRENT_TIMESTAMP +WHERE id = 1 AND version = ?`, nextVersion, update.DefaultThresholdSec, update.DelayThresholdSec, update.LongOfflineSec, string(protocols), update.Actor, currentVersion) + if err != nil { + return AccessThresholdConfig{}, err + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + return AccessThresholdConfig{}, clientError{Code: "ACCESS_THRESHOLD_VERSION_CONFLICT", Message: "阈值配置更新冲突，请刷新后重试"} + } + snapshot, _ := json.Marshal(map[string]any{ + "version": nextVersion, "defaultThresholdSec": update.DefaultThresholdSec, + "delayThresholdSec": update.DelayThresholdSec, "longOfflineSec": update.LongOfflineSec, + "protocols": update.Protocols, + }) + if _, err := tx.ExecContext(ctx, `INSERT INTO vehicle_access_threshold_audit +(version, actor, summary, config_json) VALUES (?, ?, ?, ?)`, nextVersion, update.Actor, "更新在线、延迟和长离线阈值", string(snapshot)); err != nil { + return AccessThresholdConfig{}, err + } + if err := tx.Commit(); err != nil { + return AccessThresholdConfig{}, err + } + return s.AccessThresholds(ctx) +} diff --git a/vehicle-data-platform/apps/api/internal/platform/access_test.go b/vehicle-data-platform/apps/api/internal/platform/access_test.go new file mode 100644 index 00000000..2e3abcc4 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/access_test.go @@ -0,0 +1,164 @@ +package platform + +import ( + "context" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestAccessSummaryUsesDynamicFreshnessAndDelay(t *testing.T) { + service := NewService(NewMockStore()) + summary, err := service.AccessSummary(context.Background(), AccessQuery{}) + if err != nil { + t.Fatalf("AccessSummary returned error: %v", err) + } + if summary.TotalVehicles != 6 || summary.OnlineVehicles != 2 || summary.OfflineVehicles != 2 { + t.Fatalf("unexpected access states: %+v", summary) + } + if summary.NeverReported != 1 || summary.UnknownVehicles != 1 || summary.LongOfflineVehicles != 2 { + t.Fatalf("summary must distinguish never/offline/unknown: %+v", summary) + } + if summary.DelayAbnormal != 1 || summary.ThresholdVersion != 1 { + t.Fatalf("summary must expose dynamic delay and threshold version: %+v", summary) + } +} + +func TestAccessVehiclesFiltersAndKeepsEvidenceGapsExplicit(t *testing.T) { + service := NewService(NewMockStore()) + page, err := service.AccessVehicles(context.Background(), AccessQuery{OnlineState: "never_reported", Limit: 10}) + if err != nil { + t.Fatalf("AccessVehicles returned error: %v", err) + } + if page.Total != 1 || len(page.Items) != 1 || page.Items[0].OnlineState != "never_reported" { + t.Fatalf("unexpected never-reported page: %+v", page) + } + if page.Items[0].FirstSeenAt != "" || page.Items[0].FirstSeenEvidence == "" || page.Items[0].ReportIntervalProof == "" { + t.Fatalf("missing source evidence must stay explicit: %+v", page.Items[0]) + } + + delayed, err := service.AccessVehicles(context.Background(), AccessQuery{DelayState: "abnormal", Limit: 10}) + if err != nil || delayed.Total != 1 || delayed.Items[0].DataDelaySec == nil || *delayed.Items[0].DataDelaySec != 45 { + t.Fatalf("delay filter should use event/receive difference: page=%+v err=%v", delayed, err) + } +} + +func TestAccessVehiclesSupportsModelProviderAndTimeFilters(t *testing.T) { + service := NewService(NewMockStore()) + page, err := service.AccessVehicles(t.Context(), AccessQuery{Model: "氢燃料", Provider: "车厂", LatestSeenFrom: time.Now().Add(-time.Minute).Format("2006-01-02T15:04"), Limit: 10}) + if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].VIN != "LNXNEGRR7SR318212" { + t.Fatalf("advanced access filters failed: page=%+v err=%v", page, err) + } + _, err = service.AccessVehicles(t.Context(), AccessQuery{FirstSeenFrom: "not-a-time"}) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "ACCESS_TIME_INVALID" { + t.Fatalf("invalid access time must be rejected, got %v", err) + } + _, err = service.AccessSummary(t.Context(), AccessQuery{LatestSeenFrom: "2026-07-14T09:00", LatestSeenTo: "2026-07-14T08:00"}) + clientErr, ok = asClientError(err) + if !ok || clientErr.Code != "ACCESS_TIME_RANGE_INVALID" { + t.Fatalf("reversed access time range must be rejected, got %v", err) + } +} + +func TestAccessThresholdUpdateUsesOptimisticVersion(t *testing.T) { + store := NewMockStore() + service := NewService(store) + updated, err := service.UpdateAccessThresholds(context.Background(), AccessThresholdUpdate{ + Version: 1, DefaultThresholdSec: 600, DelayThresholdSec: 60, LongOfflineSec: 3600, + Protocols: []AccessProtocolThreshold{{Protocol: "JT808", ThresholdSec: 300}}, Actor: "tester", + }) + if err != nil { + t.Fatalf("UpdateAccessThresholds returned error: %v", err) + } + if updated.Version != 2 || updated.UpdatedBy != "tester" || len(updated.Audit) != 1 { + t.Fatalf("threshold update should version and audit: %+v", updated) + } + _, err = service.UpdateAccessThresholds(context.Background(), AccessThresholdUpdate{ + Version: 1, DefaultThresholdSec: 600, DelayThresholdSec: 60, LongOfflineSec: 3600, + }) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "ACCESS_THRESHOLD_VERSION_CONFLICT" { + t.Fatalf("stale update should conflict, got %v", err) + } +} + +func TestAccessVehicleCarriesDurableProjectionEvidenceAndMasterData(t *testing.T) { + service := NewService(NewMockStore()) + page, err := service.AccessVehicles(context.Background(), AccessQuery{Keyword: "LB9A32A24R0LS1426", Limit: 10}) + if err != nil || len(page.Items) != 1 { + t.Fatalf("access vehicle query failed: page=%+v err=%v", page, err) + } + row := page.Items[0] + if row.Company != "岭牛示范车队" || row.FirstSeenSource != "live_writer" || row.ReportSampleCount != 120 || row.ReportIntervalSec == nil { + t.Fatalf("durable access evidence/master data missing: %+v", row) + } +} + +func TestProductionAccessEvidenceReadsDurableWriterProjection(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + mock.ExpectQuery("(?s)SELECT.*access_first_seen_at.*access_first_seen_source.*access_report_interval_ms.*vehicle_profile.*access_latest_received_at"). + WithArgs(accessEvidenceLimit + 1). + WillReturnRows(sqlmock.NewRows([]string{ + "vin", "plate", "oem", "model_name", "company_name", "protocol", "provider", "first_seen_at", "first_seen_source", + "event_time", "received_at", "updated_at", "report_interval_ms", "report_sample_count", "event_id", + }).AddRow("VIN001", "粤A1", "示范厂家", "纯电客车", "示范公交", "GB32960", "车厂平台", "2026-07-14 05:00:00.000", "snapshot_backfill", "2026-07-14 05:01:00.000", "2026-07-14 05:01:30.500", "2026-07-14 05:01:30.500", int64(30500), int64(3), "evt-3")) + rows, err := store.AccessEvidence(context.Background()) + if err != nil || len(rows) != 1 { + t.Fatalf("AccessEvidence rows=%+v err=%v", rows, err) + } + row := rows[0] + if row.ReportIntervalSec == nil || *row.ReportIntervalSec != 31 || row.ReportSampleCount != 3 || row.Model != "纯电客车" || row.Company != "示范公交" { + t.Fatalf("unexpected durable projection row: %+v", row) + } + if row.FirstSeenEvidence != "上线基线：由实时快照回填（非历史首次接入）" || row.ReportIntervalProof == "" { + t.Fatalf("projection evidence boundary missing: %+v", row) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestAccessUnresolvedIdentitiesRemainMaskedAndProtocolScoped(t *testing.T) { + service := NewService(NewMockStore()) + page, err := service.AccessUnresolvedIdentities(t.Context(), AccessUnresolvedIdentityQuery{Protocol: "jt808", Limit: 20}) + if err != nil || page.Total != 1 || len(page.Items) != 1 { + t.Fatalf("unresolved identity queue failed: page=%+v err=%v", page, err) + } + if page.Items[0].IdentifierMasked != "138****0001" || page.Items[0].IssueCode != "missing_vin_jt808" || page.Items[0].RecommendedAction == "" { + t.Fatalf("unresolved evidence is not actionable and masked: %+v", page.Items[0]) + } + empty, err := service.AccessUnresolvedIdentities(t.Context(), AccessUnresolvedIdentityQuery{Protocol: "GB32960"}) + if err != nil || empty.Total != 0 || len(empty.Items) != 0 { + t.Fatalf("non-JT808 unresolved query must be empty: page=%+v err=%v", empty, err) + } +} + +func TestProductionUnresolvedIdentityQueueExcludesBoundPhonesAndNeverReturnsRawIdentifier(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + mock.ExpectQuery(`(?s)SELECT COUNT\(\*\).*jt808_registration.*COALESCE.*b\.vin`).WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`(?s)SELECT.*SHA2\(CONCAT\('JT808:',r\.phone\),256\).*CHAR_LENGTH\(r\.phone\).*missing_vin_jt808.*ORDER BY r\.latest_seen_at`).WithArgs(20, 0).WillReturnRows(sqlmock.NewRows([]string{ + "id", "protocol", "identifier_masked", "plate", "manufacturer", "source_endpoint", "first_registered_at", "latest_registered_at", "latest_authenticated_at", "latest_seen_at", "freshness_sec", "issue_code", "recommended_action", + }).AddRow("hash-id", "JT808", "410****3543", "", "示范终端", "gateway-a", "", "", "", "2026-07-14 07:52:09", 20, "missing_vin_jt808", "核对后绑定")) + page, err := store.AccessUnresolvedIdentities(t.Context(), AccessUnresolvedIdentityQuery{Limit: 20}) + if err != nil || page.Total != 1 || len(page.Items) != 1 { + t.Fatalf("production unresolved query failed: page=%+v err=%v", page, err) + } + if page.Items[0].ID != "hash-id" || page.Items[0].IdentifierMasked != "410****3543" || page.Items[0].FreshnessSec != 20 { + t.Fatalf("production unresolved evidence lost: %+v", page.Items[0]) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert.go b/vehicle-data-platform/apps/api/internal/platform/alert.go new file mode 100644 index 00000000..29fb6f85 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert.go @@ -0,0 +1,314 @@ +package platform + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "strings" +) + +type alertStore interface { + AlertSummary(context.Context, AlertQuery) (AlertSummary, error) + AlertEvents(context.Context, AlertQuery) (Page[AlertEvent], error) + AlertEvent(context.Context, string) (AlertEvent, error) + AlertRules(context.Context) ([]AlertRule, error) + SaveAlertRule(context.Context, AlertRuleInput) (AlertRule, error) + SetAlertRuleEnabled(context.Context, string, AlertRuleEnabledUpdate) (AlertRule, error) + ActOnAlert(context.Context, string, AlertActionRequest) (AlertEvent, error) + AlertNotifications(context.Context, AlertNotificationQuery) (Page[AlertNotification], error) + MarkAlertNotificationsRead(context.Context, AlertNotificationReadRequest) (int, error) +} + +type alertEvaluatorStore interface { + EvaluateAlerts(context.Context) (AlertEvaluationResult, error) +} + +func (s *Service) alertStore() (alertStore, error) { + store, ok := s.store.(alertStore) + if !ok { + return nil, fmt.Errorf("store does not provide durable alert center") + } + return store, nil +} + +func (s *Service) AlertSummary(ctx context.Context, query AlertQuery) (AlertSummary, error) { + store, err := s.alertStore() + if err != nil { + return AlertSummary{}, err + } + return store.AlertSummary(ctx, normalizeAlertQuery(query)) +} + +func (s *Service) AlertEvents(ctx context.Context, query AlertQuery) (Page[AlertEvent], error) { + store, err := s.alertStore() + if err != nil { + return Page[AlertEvent]{}, err + } + return store.AlertEvents(ctx, normalizeAlertQuery(query)) +} + +func (s *Service) AlertEvent(ctx context.Context, id string) (AlertEvent, error) { + if strings.TrimSpace(id) == "" { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_ID_REQUIRED", Message: "告警事件 ID 不能为空"} + } + store, err := s.alertStore() + if err != nil { + return AlertEvent{}, err + } + return store.AlertEvent(ctx, strings.TrimSpace(id)) +} + +func (s *Service) AlertRules(ctx context.Context) ([]AlertRule, error) { + store, err := s.alertStore() + if err != nil { + return nil, err + } + return store.AlertRules(ctx) +} + +func (s *Service) SaveAlertRule(ctx context.Context, input AlertRuleInput) (AlertRule, error) { + definitions, err := s.metricDefinitions(ctx) + if err != nil { + return AlertRule{}, err + } + if err := validateAlertRule(input, definitions...); err != nil { + return AlertRule{}, err + } + input.ID = strings.TrimSpace(input.ID) + if input.ID == "" { + id, err := newAlertID("rule") + if err != nil { + return AlertRule{}, err + } + input.ID = id + } + input.Actor = firstNonEmpty(strings.TrimSpace(input.Actor), "platform-admin") + input.ScopeProtocols = cleanUniqueStrings(input.ScopeProtocols) + input.ScopeVINs = cleanUniqueStrings(input.ScopeVINs) + input.ScopeOEMs = cleanUniqueStrings(input.ScopeOEMs) + input.ScopeModels = cleanUniqueStrings(input.ScopeModels) + input.ScopeCompanies = cleanUniqueStrings(input.ScopeCompanies) + if err := validateAlertRuleScopes(input); err != nil { + return AlertRule{}, err + } + input.NotificationChannels = normalizeAlertChannels(input.NotificationChannels) + if strings.EqualFold(input.Operator, "changed") { + input.DurationSec = 0 + } + if strings.EqualFold(input.ValueType, "boolean") && input.BooleanThreshold != nil { + if *input.BooleanThreshold { + input.Threshold = 1 + } else { + input.Threshold = 0 + } + } + store, err := s.alertStore() + if err != nil { + return AlertRule{}, err + } + return store.SaveAlertRule(ctx, input) +} + +func validateAlertRuleScopes(input AlertRuleInput) error { + for _, scope := range []struct { + name string + values []string + }{ + {"协议", input.ScopeProtocols}, {"VIN", input.ScopeVINs}, {"厂家", input.ScopeOEMs}, + {"车型", input.ScopeModels}, {"企业", input.ScopeCompanies}, + } { + if len(scope.values) > 500 { + return clientError{Code: "ALERT_RULE_SCOPE_TOO_LARGE", Message: scope.name + "范围最多允许 500 项"} + } + for _, value := range scope.values { + if len([]rune(value)) > 128 { + return clientError{Code: "ALERT_RULE_SCOPE_VALUE_INVALID", Message: scope.name + "范围值不能超过 128 个字符"} + } + } + } + return nil +} + +func (s *Service) SetAlertRuleEnabled(ctx context.Context, id string, update AlertRuleEnabledUpdate) (AlertRule, error) { + if strings.TrimSpace(id) == "" || update.Version <= 0 { + return AlertRule{}, clientError{Code: "ALERT_RULE_VERSION_REQUIRED", Message: "规则 ID 和版本不能为空"} + } + update.Actor = firstNonEmpty(strings.TrimSpace(update.Actor), "platform-admin") + store, err := s.alertStore() + if err != nil { + return AlertRule{}, err + } + return store.SetAlertRuleEnabled(ctx, strings.TrimSpace(id), update) +} + +func (s *Service) ActOnAlert(ctx context.Context, id string, request AlertActionRequest) (AlertEvent, error) { + if strings.TrimSpace(id) == "" || request.Version <= 0 { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_VERSION_REQUIRED", Message: "事件 ID 和版本不能为空"} + } + request.Action = strings.ToLower(strings.TrimSpace(request.Action)) + if request.Action != "acknowledge" && request.Action != "close" && request.Action != "ignore" { + return AlertEvent{}, clientError{Code: "ALERT_ACTION_INVALID", Message: "仅支持确认、关闭或忽略告警"} + } + if len([]rune(request.Note)) > 200 { + return AlertEvent{}, clientError{Code: "ALERT_NOTE_TOO_LONG", Message: "处置备注不能超过 200 字"} + } + request.Actor = firstNonEmpty(strings.TrimSpace(request.Actor), "platform-admin") + store, err := s.alertStore() + if err != nil { + return AlertEvent{}, err + } + return store.ActOnAlert(ctx, strings.TrimSpace(id), request) +} + +func (s *Service) AlertNotifications(ctx context.Context, query AlertNotificationQuery) (Page[AlertNotification], error) { + if query.Limit <= 0 { + query.Limit = 20 + } + if query.Limit > 100 { + query.Limit = 100 + } + if query.Offset < 0 { + query.Offset = 0 + } + store, err := s.alertStore() + if err != nil { + return Page[AlertNotification]{}, err + } + return store.AlertNotifications(ctx, query) +} + +func (s *Service) MarkAlertNotificationsRead(ctx context.Context, request AlertNotificationReadRequest) (int, error) { + if len(request.IDs) == 0 || len(request.IDs) > 100 { + return 0, clientError{Code: "ALERT_NOTIFICATION_IDS_INVALID", Message: "请选择 1 到 100 条通知"} + } + request.Actor = firstNonEmpty(strings.TrimSpace(request.Actor), "platform-admin") + store, err := s.alertStore() + if err != nil { + return 0, err + } + return store.MarkAlertNotificationsRead(ctx, request) +} + +func (s *Service) EvaluateAlerts(ctx context.Context) (AlertEvaluationResult, error) { + store, ok := s.store.(alertEvaluatorStore) + if !ok { + return AlertEvaluationResult{}, fmt.Errorf("store does not provide alert evaluation") + } + return store.EvaluateAlerts(ctx) +} + +func normalizeAlertQuery(query AlertQuery) AlertQuery { + query.Keyword = strings.TrimSpace(query.Keyword) + query.Severity = strings.ToLower(strings.TrimSpace(query.Severity)) + query.Status = strings.ToLower(strings.TrimSpace(query.Status)) + query.RuleID = strings.TrimSpace(query.RuleID) + query.Protocol = strings.TrimSpace(query.Protocol) + if query.Limit <= 0 { + query.Limit = 20 + } + if query.Limit > 200 { + query.Limit = 200 + } + if query.Offset < 0 { + query.Offset = 0 + } + return query +} + +func validateAlertRule(input AlertRuleInput, catalog ...MetricDefinition) error { + if strings.TrimSpace(input.Name) == "" || len([]rune(input.Name)) > 80 { + return clientError{Code: "ALERT_RULE_NAME_INVALID", Message: "规则名称不能为空且不能超过 80 字"} + } + severity := strings.ToLower(strings.TrimSpace(input.Severity)) + if severity != "critical" && severity != "major" && severity != "minor" { + return clientError{Code: "ALERT_RULE_SEVERITY_INVALID", Message: "告警级别仅支持 critical、major、minor"} + } + valueType := strings.ToLower(strings.TrimSpace(input.ValueType)) + if valueType != "numeric" && valueType != "boolean" { + return clientError{Code: "ALERT_RULE_VALUE_TYPE_INVALID", Message: "规则值类型仅支持 numeric 或 boolean"} + } + if strings.TrimSpace(input.Metric) == "" { + return clientError{Code: "ALERT_RULE_METRIC_REQUIRED", Message: "规则指标不能为空"} + } + var metric *MetricDefinition + definitions := catalog + if len(definitions) == 0 { + definitions = metricDefinitions() + } + for index := range definitions { + candidate := definitions[index] + if candidate.Key == strings.TrimSpace(input.Metric) { + metric = &candidate + break + } + } + if metric == nil || !metric.Alertable { + return clientError{Code: "ALERT_RULE_METRIC_INVALID", Message: "规则指标不在可告警指标目录中"} + } + if _, supported := alertMetricValue(input.Metric, alertEvaluationEvidence{}); !supported { + return clientError{Code: "ALERT_RULE_METRIC_UNSUPPORTED", Message: "规则指标尚未接入告警评估器"} + } + if metric.ValueType != valueType { + return clientError{Code: "ALERT_RULE_METRIC_TYPE_MISMATCH", Message: "规则值类型与指标目录不一致"} + } + operator := strings.ToLower(strings.TrimSpace(input.Operator)) + allowed := map[string]bool{"gt": true, "gte": true, "lt": true, "lte": true, "eq": true, "neq": true, "between": true, "outside": true, "changed": true} + if !allowed[operator] { + return clientError{Code: "ALERT_RULE_OPERATOR_INVALID", Message: "规则比较符无效"} + } + if (operator == "between" || operator == "outside") && (valueType != "numeric" || input.ThresholdHigh <= input.Threshold) { + return clientError{Code: "ALERT_RULE_RANGE_INVALID", Message: "区间规则必须是数值型且上限大于下限"} + } + if operator == "changed" && valueType != "boolean" { + return clientError{Code: "ALERT_RULE_CHANGE_INVALID", Message: "状态变化规则必须使用布尔值类型"} + } + if valueType == "boolean" && operator != "changed" && input.BooleanThreshold == nil { + return clientError{Code: "ALERT_RULE_BOOLEAN_REQUIRED", Message: "布尔规则必须配置目标值"} + } + if input.DurationSec < 0 || input.DurationSec > 86400 || input.RepeatIntervalSec < 0 || input.RepeatIntervalSec > 604800 { + return clientError{Code: "ALERT_RULE_INTERVAL_INVALID", Message: "持续时间或重复间隔超出允许范围"} + } + if input.Version < 0 { + return clientError{Code: "ALERT_RULE_VERSION_INVALID", Message: "规则版本无效"} + } + return nil +} + +func normalizeAlertChannels(values []string) []string { + allowed := map[string]bool{"in_app": true, "sms": true, "email": true, "wecom": true} + out := make([]string, 0, len(values)+1) + seen := map[string]bool{} + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if allowed[value] && !seen[value] { + seen[value] = true + out = append(out, value) + } + } + if !seen["in_app"] { + out = append([]string{"in_app"}, out...) + } + return out +} + +func cleanUniqueStrings(values []string) []string { + out := make([]string, 0, len(values)) + seen := map[string]bool{} + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" && !seen[value] { + seen[value] = true + out = append(out, value) + } + } + return out +} + +func newAlertID(prefix string) (string, error) { + buf := make([]byte, 10) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return prefix + "-" + hex.EncodeToString(buf), nil +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_evaluator.go b/vehicle-data-platform/apps/api/internal/platform/alert_evaluator.go new file mode 100644 index 00000000..ffdc5e54 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_evaluator.go @@ -0,0 +1,553 @@ +package platform + +import ( + "context" + "database/sql" + "fmt" + "math" + "strings" + "time" +) + +const alertEvaluationVehicleLimit = 50000 +const alertCandidateBatchSize = 500 +const alertCandidateContinuityWindow = time.Minute +const alertDynamicEvidenceMaxAge = 5 * time.Minute + +type alertEvaluationEvidence struct { + VIN, Plate, Protocol, OEM, Model, Company, SourceEventID, EventAt, ReceivedAt, Location string + SpeedKmh, SOCPercent, Longitude, Latitude float64 + AlarmFlag int64 + FreshnessSec, DataDelaySec int +} + +type alertCandidateState struct { + FirstMatchedAt, LastMatchedAt time.Time + SourceEventID string +} +type alertActiveEvent struct{ ID, Status string } +type alertCandidateUpsert struct { + RuleID, VIN, Protocol, SourceEventID string + FirstMatchedAt, LastMatchedAt time.Time + LatestValue float64 +} +type alertRuleState struct { + LastValue float64 + LastObservedAt time.Time +} +type alertRuleStateUpsert struct { + RuleID, VIN, Protocol string + LastValue float64 + ObservedAt time.Time +} + +type alertObservationDecision uint8 + +const ( + alertObservationAdvanced alertObservationDecision = iota + alertObservationDuplicate + alertObservationLate +) + +const alertEventInsertSQL = `INSERT INTO vehicle_alert_event(id,fingerprint,rule_id,rule_name,rule_version,severity,status,vin,plate,protocol,metric,operator,trigger_value,threshold_value,threshold_high,unit,duration_sec,location_text,longitude,latitude,source_event_id,event_at,received_at,triggered_at) VALUES(?,?,?,?,?,?,'unprocessed',?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP(3))` + +func buildAlertEventInsert(id, fingerprint string, rule AlertRule, item alertEvaluationEvidence, value float64) (string, []any) { + return alertEventInsertSQL, []any{ + id, fingerprint, rule.ID, rule.Name, rule.Version, rule.Severity, + item.VIN, item.Plate, item.Protocol, rule.Metric, rule.Operator, value, + rule.Threshold, rule.ThresholdHigh, alertMetricUnit(rule.Metric), rule.DurationSec, + item.Location, item.Longitude, item.Latitude, item.SourceEventID, + nullableAlertTime(item.EventAt), nullableAlertTime(item.ReceivedAt), + } +} + +func (s *ProductionStore) EvaluateAlerts(ctx context.Context) (AlertEvaluationResult, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertEvaluationResult{}, err + } + rules, err := s.AlertRules(ctx) + if err != nil { + return AlertEvaluationResult{}, err + } + rules = snapshotAlertRules(rules, s.alertStreamMode) + if len(rules) == 0 { + return AlertEvaluationResult{AsOf: time.Now().Format(time.RFC3339)}, nil + } + evidence, err := s.alertEvaluationEvidence(ctx) + if err != nil { + return AlertEvaluationResult{}, err + } + result := AlertEvaluationResult{VehiclesScanned: len(evidence), AsOf: time.Now().Format(time.RFC3339)} + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return result, err + } + defer tx.Rollback() + candidates, err := loadAlertCandidates(ctx, tx) + if err != nil { + return result, err + } + activeEvents, err := loadActiveAlertEvents(ctx, tx) + if err != nil { + return result, err + } + ruleStates, err := loadAlertRuleStates(ctx, tx) + if err != nil { + return result, err + } + lastTriggered, err := loadAlertLastTriggered(ctx, tx) + if err != nil { + return result, err + } + now := time.Now() + for _, rule := range rules { + if !rule.Enabled { + continue + } + enabled, lockErr := lockAlertRuleForEvaluation(ctx, tx, rule.ID) + if lockErr != nil { + return result, lockErr + } + if !enabled { + continue + } + result.RulesEvaluated++ + candidateUpserts := make([]alertCandidateUpsert, 0, len(evidence)) + stateUpserts := make([]alertRuleStateUpsert, 0, len(evidence)) + for _, item := range evidence { + if !alertRuleInScope(rule, item) { + continue + } + value, supported := alertMetricValue(rule.Metric, item) + if !supported { + continue + } + fingerprint := rule.ID + "|" + item.VIN + "|" + item.Protocol + if rule.Metric != "freshness_sec" && time.Duration(item.FreshnessSec)*time.Second > alertDynamicEvidenceMaxAge { + result.StaleEvidenceSkipped++ + if _, exists := candidates[fingerprint]; exists { + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_candidate WHERE rule_id=? AND vin=? AND protocol=?`, rule.ID, item.VIN, item.Protocol); err != nil { + return result, err + } + delete(candidates, fingerprint) + } + continue + } + observedAt, clockDriven, observationOK := alertObservationTime(rule.Metric, item, now) + if !observationOK { + result.LateObservations++ + continue + } + matched := false + if strings.EqualFold(rule.Operator, "changed") { + normalized := 0.0 + if value != 0 { + normalized = 1 + } + previous, exists := ruleStates[fingerprint] + if exists && !observedAt.After(previous.LastObservedAt) { + result.LateObservations++ + continue + } + matched = exists && previous.LastValue != normalized + stateUpserts = append(stateUpserts, alertRuleStateUpsert{RuleID: rule.ID, VIN: item.VIN, Protocol: item.Protocol, LastValue: normalized, ObservedAt: observedAt}) + ruleStates[fingerprint] = alertRuleState{LastValue: normalized, LastObservedAt: observedAt} + } else { + matched = compareAlertRuleValue(value, rule) + } + if matched { + if len(activeEvents[fingerprint]) > 0 { + continue + } + candidate, exists := candidates[fingerprint] + candidate, decision := advanceAlertCandidate(candidate, exists, observedAt, item.SourceEventID, clockDriven) + switch decision { + case alertObservationDuplicate: + result.DuplicateObservations++ + continue + case alertObservationLate: + result.LateObservations++ + continue + } + candidates[fingerprint] = candidate + result.CandidatesAdvanced++ + duration := int(candidate.LastMatchedAt.Sub(candidate.FirstMatchedAt).Seconds()) + if duration < rule.DurationSec { + candidateUpserts = append(candidateUpserts, alertCandidateUpsert{RuleID: rule.ID, VIN: item.VIN, Protocol: item.Protocol, SourceEventID: candidate.SourceEventID, FirstMatchedAt: candidate.FirstMatchedAt, LastMatchedAt: candidate.LastMatchedAt, LatestValue: value}) + continue + } + if !alertRepeatAllowed(now, lastTriggered[fingerprint], rule.RepeatIntervalSec) { + candidateUpserts = append(candidateUpserts, alertCandidateUpsert{RuleID: rule.ID, VIN: item.VIN, Protocol: item.Protocol, SourceEventID: candidate.SourceEventID, FirstMatchedAt: candidate.FirstMatchedAt, LastMatchedAt: candidate.LastMatchedAt, LatestValue: value}) + continue + } + id, e := newAlertID("alert") + if e != nil { + return result, e + } + unit := alertMetricUnit(rule.Metric) + insertSQL, insertArgs := buildAlertEventInsert(id, fingerprint, rule, item, value) + _, err = tx.ExecContext(ctx, insertSQL, insertArgs...) + if err != nil { + return result, err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'trigger','','unprocessed','alert-evaluator','规则命中并达到持续时间')`, id); err != nil { + return result, err + } + for _, channel := range rule.NotificationChannels { + delivery := "reserved" + if channel == "in_app" { + delivery = "created" + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_notification(event_id,title,content,severity,channel,delivery_status) VALUES(?,?,?,?,?,?)`, id, rule.Name, fmt.Sprintf("%s / %s 触发%s：%.2f %s", item.Plate, item.VIN, rule.Name, value, unit), rule.Severity, channel, delivery); err != nil { + return result, err + } + } + activeEvents[fingerprint] = []alertActiveEvent{{ID: id, Status: "unprocessed"}} + lastTriggered[fingerprint] = now + if exists { + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_candidate WHERE rule_id=? AND vin=? AND protocol=?`, rule.ID, item.VIN, item.Protocol); err != nil { + return result, err + } + delete(candidates, fingerprint) + } + result.Opened++ + } else { + if _, exists := candidates[fingerprint]; exists { + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_candidate WHERE rule_id=? AND vin=? AND protocol=?`, rule.ID, item.VIN, item.Protocol); err != nil { + return result, err + } + delete(candidates, fingerprint) + } + recovered := true + if strings.TrimSpace(rule.RecoveryOperator) != "" { + recovered = compareAlertValue(value, rule.RecoveryOperator, rule.RecoveryThreshold) + } + if !recovered { + continue + } + for _, event := range activeEvents[fingerprint] { + if _, err = tx.ExecContext(ctx, `UPDATE vehicle_alert_event SET status='recovered',recovered_at=CURRENT_TIMESTAMP(3),version=version+1 WHERE id=? AND status=?`, event.ID, event.Status); err != nil { + return result, err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'recover',?,'recovered','alert-evaluator','恢复条件满足')`, event.ID, event.Status); err != nil { + return result, err + } + result.Recovered++ + } + delete(activeEvents, fingerprint) + } + } + for start := 0; start < len(candidateUpserts); start += alertCandidateBatchSize { + end := min(start+alertCandidateBatchSize, len(candidateUpserts)) + query, args := buildAlertCandidateUpsert(candidateUpserts[start:end]) + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return result, err + } + } + for start := 0; start < len(stateUpserts); start += alertCandidateBatchSize { + end := min(start+alertCandidateBatchSize, len(stateUpserts)) + query, args := buildAlertRuleStateUpsert(stateUpserts[start:end]) + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return result, err + } + } + } + if err = tx.Commit(); err != nil { + return result, err + } + return result, nil +} + +func snapshotAlertRules(rules []AlertRule, streamMode string) []AlertRule { + if !strings.EqualFold(strings.TrimSpace(streamMode), "active") { + return rules + } + filtered := make([]AlertRule, 0, len(rules)) + for _, rule := range rules { + if strings.EqualFold(strings.TrimSpace(rule.Metric), "freshness_sec") { + filtered = append(filtered, rule) + } + } + return filtered +} + +func lockAlertRuleForEvaluation(ctx context.Context, tx *sql.Tx, ruleID string) (bool, error) { + var enabled bool + err := tx.QueryRowContext(ctx, `SELECT enabled FROM vehicle_alert_rule WHERE id=? FOR UPDATE`, ruleID).Scan(&enabled) + if err == sql.ErrNoRows { + return false, nil + } + return enabled, err +} + +func loadAlertCandidates(ctx context.Context, tx *sql.Tx) (map[string]alertCandidateState, error) { + rows, err := tx.QueryContext(ctx, `SELECT c.rule_id,c.vin,c.protocol,c.first_matched_at,c.last_matched_at,c.source_event_id FROM vehicle_alert_candidate c JOIN vehicle_alert_rule r ON r.id=c.rule_id AND r.enabled=1`) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]alertCandidateState{} + for rows.Next() { + var ruleID, vin, protocol string + var state alertCandidateState + if err := rows.Scan(&ruleID, &vin, &protocol, &state.FirstMatchedAt, &state.LastMatchedAt, &state.SourceEventID); err != nil { + return nil, err + } + items[ruleID+"|"+vin+"|"+protocol] = state + } + return items, rows.Err() +} + +func buildAlertCandidateUpsert(items []alertCandidateUpsert) (string, []any) { + values := make([]string, 0, len(items)) + args := make([]any, 0, len(items)*7) + for _, item := range items { + values = append(values, "(?,?,?,?,?,?,?)") + args = append(args, item.RuleID, item.VIN, item.Protocol, item.FirstMatchedAt, item.LastMatchedAt, item.LatestValue, item.SourceEventID) + } + return `INSERT INTO vehicle_alert_candidate(rule_id,vin,protocol,first_matched_at,last_matched_at,latest_value,source_event_id) VALUES ` + strings.Join(values, ",") + ` ON DUPLICATE KEY UPDATE first_matched_at=VALUES(first_matched_at),last_matched_at=VALUES(last_matched_at),latest_value=VALUES(latest_value),source_event_id=VALUES(source_event_id)`, args +} + +func loadAlertRuleStates(ctx context.Context, tx *sql.Tx) (map[string]alertRuleState, error) { + rows, err := tx.QueryContext(ctx, `SELECT s.rule_id,s.vin,s.protocol,s.observed_value,s.last_observed_at FROM vehicle_alert_rule_state s JOIN vehicle_alert_rule r ON r.id=s.rule_id AND r.enabled=1`) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]alertRuleState{} + for rows.Next() { + var ruleID, vin, protocol string + var state alertRuleState + if err := rows.Scan(&ruleID, &vin, &protocol, &state.LastValue, &state.LastObservedAt); err != nil { + return nil, err + } + items[ruleID+"|"+vin+"|"+protocol] = state + } + return items, rows.Err() +} + +func loadAlertLastTriggered(ctx context.Context, tx *sql.Tx) (map[string]time.Time, error) { + rows, err := tx.QueryContext(ctx, `SELECT e.fingerprint,MAX(e.triggered_at) FROM vehicle_alert_event e JOIN vehicle_alert_rule r ON r.id=e.rule_id AND r.enabled=1 GROUP BY e.fingerprint`) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]time.Time{} + for rows.Next() { + var fingerprint string + var triggeredAt time.Time + if err := rows.Scan(&fingerprint, &triggeredAt); err != nil { + return nil, err + } + items[fingerprint] = triggeredAt + } + return items, rows.Err() +} + +func alertRepeatAllowed(now, last time.Time, intervalSec int) bool { + return intervalSec <= 0 || last.IsZero() || now.Sub(last) >= time.Duration(intervalSec)*time.Second +} + +func alertObservationTime(metric string, item alertEvaluationEvidence, now time.Time) (time.Time, bool, bool) { + if strings.EqualFold(strings.TrimSpace(metric), "freshness_sec") { + return now, true, true + } + for _, value := range []string{item.EventAt, item.ReceivedAt} { + if parsed, ok := parseAlertEvidenceTime(value); ok { + return parsed, false, true + } + } + return time.Time{}, false, false +} + +func parseAlertEvidenceTime(value string) (time.Time, bool) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, false + } + for _, layout := range []string{ + time.RFC3339Nano, + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05", + } { + parsed, err := time.ParseInLocation(layout, value, time.Local) + if err == nil { + return parsed, true + } + } + return time.Time{}, false +} + +// advanceAlertCandidate makes event-time, not evaluator wall time, the proof of +// duration. Re-reading the same snapshot cannot advance a telemetry candidate. +// freshness_sec is deliberately clock-driven because staleness itself changes +// while the latest source event remains the same. +func advanceAlertCandidate(current alertCandidateState, exists bool, observedAt time.Time, sourceEventID string, clockDriven bool) (alertCandidateState, alertObservationDecision) { + sourceEventID = strings.TrimSpace(sourceEventID) + if !exists { + return alertCandidateState{FirstMatchedAt: observedAt, LastMatchedAt: observedAt, SourceEventID: sourceEventID}, alertObservationAdvanced + } + if !clockDriven && sourceEventID != "" && sourceEventID == current.SourceEventID { + return current, alertObservationDuplicate + } + if observedAt.Before(current.LastMatchedAt) { + return current, alertObservationLate + } + if observedAt.Equal(current.LastMatchedAt) { + if clockDriven || sourceEventID == "" || sourceEventID == current.SourceEventID { + return current, alertObservationDuplicate + } + // A distinct event can share millisecond precision. Remember its ID so + // subsequent evaluator passes are idempotent, without inventing duration. + current.SourceEventID = sourceEventID + return current, alertObservationAdvanced + } + if observedAt.Sub(current.LastMatchedAt) > alertCandidateContinuityWindow { + return alertCandidateState{FirstMatchedAt: observedAt, LastMatchedAt: observedAt, SourceEventID: sourceEventID}, alertObservationAdvanced + } + current.LastMatchedAt = observedAt + current.SourceEventID = sourceEventID + return current, alertObservationAdvanced +} + +func buildAlertRuleStateUpsert(items []alertRuleStateUpsert) (string, []any) { + values := make([]string, 0, len(items)) + args := make([]any, 0, len(items)*5) + for _, item := range items { + values = append(values, "(?,?,?,?,?)") + args = append(args, item.RuleID, item.VIN, item.Protocol, item.LastValue, item.ObservedAt) + } + return `INSERT INTO vehicle_alert_rule_state(rule_id,vin,protocol,observed_value,last_observed_at) VALUES ` + strings.Join(values, ",") + ` ON DUPLICATE KEY UPDATE observed_value=VALUES(observed_value),last_observed_at=VALUES(last_observed_at)`, args +} + +func loadActiveAlertEvents(ctx context.Context, tx *sql.Tx) (map[string][]alertActiveEvent, error) { + rows, err := tx.QueryContext(ctx, `SELECT fingerprint,id,status FROM vehicle_alert_event WHERE status IN ('unprocessed','processing') FOR UPDATE`) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string][]alertActiveEvent{} + for rows.Next() { + var fingerprint string + var event alertActiveEvent + if err := rows.Scan(&fingerprint, &event.ID, &event.Status); err != nil { + return nil, err + } + items[fingerprint] = append(items[fingerprint], event) + } + return items, rows.Err() +} + +func (s *ProductionStore) alertEvaluationEvidence(ctx context.Context) ([]alertEvaluationEvidence, error) { + rows, err := s.db.QueryContext(ctx, `SELECT l.vin,COALESCE(NULLIF(l.plate,''),NULLIF(b.plate,''),''),l.protocol,COALESCE(b.oem,''),COALESCE(p.model_name,''),COALESCE(p.company_name,''),COALESCE(l.speed_kmh,0),COALESCE(l.soc_percent,0),COALESCE(l.alarm_flag,0),COALESCE(l.longitude,0),COALESCE(l.latitude,0),COALESCE(l.event_id,''),COALESCE(DATE_FORMAT(l.event_time,'%Y-%m-%d %H:%i:%s.%f'),''),COALESCE(DATE_FORMAT(l.received_at,'%Y-%m-%d %H:%i:%s.%f'),''),GREATEST(0,TIMESTAMPDIFF(SECOND,l.updated_at,NOW())),TIMESTAMPDIFF(SECOND,l.event_time,l.received_at),COALESCE(CONCAT(l.longitude,',',l.latitude),'') FROM vehicle_realtime_location l LEFT JOIN vehicle_identity_binding b ON b.vin=l.vin LEFT JOIN vehicle_profile p ON p.vin=l.vin WHERE l.vin<>'' ORDER BY l.updated_at DESC LIMIT ?`, alertEvaluationVehicleLimit+1) + if err != nil { + return nil, err + } + defer rows.Close() + items := make([]alertEvaluationEvidence, 0, 1024) + for rows.Next() { + var item alertEvaluationEvidence + var delay sql.NullInt64 + if err = rows.Scan(&item.VIN, &item.Plate, &item.Protocol, &item.OEM, &item.Model, &item.Company, &item.SpeedKmh, &item.SOCPercent, &item.AlarmFlag, &item.Longitude, &item.Latitude, &item.SourceEventID, &item.EventAt, &item.ReceivedAt, &item.FreshnessSec, &delay, &item.Location); err != nil { + return nil, err + } + if delay.Valid { + item.DataDelaySec = int(delay.Int64) + } + items = append(items, item) + if len(items) > alertEvaluationVehicleLimit { + return nil, fmt.Errorf("alert evaluation evidence exceeds safety limit %d", alertEvaluationVehicleLimit) + } + } + return items, rows.Err() +} + +func alertRuleInScope(rule AlertRule, item alertEvaluationEvidence) bool { + if len(rule.ScopeProtocols) > 0 && !containsFold(rule.ScopeProtocols, item.Protocol) { + return false + } + if len(rule.ScopeVINs) > 0 && !containsFold(rule.ScopeVINs, item.VIN) { + return false + } + if len(rule.ScopeOEMs) > 0 && !containsFold(rule.ScopeOEMs, item.OEM) { + return false + } + if len(rule.ScopeModels) > 0 && !containsFold(rule.ScopeModels, item.Model) { + return false + } + if len(rule.ScopeCompanies) > 0 && !containsFold(rule.ScopeCompanies, item.Company) { + return false + } + return true +} +func containsFold(values []string, target string) bool { + for _, value := range values { + if strings.EqualFold(value, target) { + return true + } + } + return false +} +func alertMetricValue(metric string, item alertEvaluationEvidence) (float64, bool) { + switch strings.ToLower(strings.TrimSpace(metric)) { + case "speed_kmh": + return item.SpeedKmh, true + case "soc_percent": + return item.SOCPercent, true + case "alarm_active": + if item.AlarmFlag != 0 { + return 1, true + } + return 0, true + case "freshness_sec": + return float64(item.FreshnessSec), true + case "data_delay_sec": + return float64(item.DataDelaySec), true + } + return 0, false +} +func compareAlertValue(value float64, operator string, threshold float64) bool { + switch strings.ToLower(operator) { + case "gt": + return value > threshold + case "gte": + return value >= threshold + case "lt": + return value < threshold + case "lte": + return value <= threshold + case "eq": + return math.Abs(value-threshold) < 1e-9 + case "neq": + return math.Abs(value-threshold) >= 1e-9 + } + return false +} + +func compareAlertRuleValue(value float64, rule AlertRule) bool { + switch strings.ToLower(rule.Operator) { + case "between": + return value >= rule.Threshold && value <= rule.ThresholdHigh + case "outside": + return value < rule.Threshold || value > rule.ThresholdHigh + default: + return compareAlertValue(value, rule.Operator, rule.Threshold) + } +} +func alertMetricUnit(metric string) string { + switch metric { + case "speed_kmh": + return "km/h" + case "soc_percent": + return "%" + case "freshness_sec", "data_delay_sec": + return "秒" + } + return "" +} +func nullableAlertTime(value string) any { + if strings.TrimSpace(value) == "" { + return nil + } + return value +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_mock.go b/vehicle-data-platform/apps/api/internal/platform/alert_mock.go new file mode 100644 index 00000000..9e2c07f9 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_mock.go @@ -0,0 +1,306 @@ +package platform + +import ( + "context" + "sort" + "strings" + "time" +) + +func (m *MockStore) seedAlertCenter() { + now := time.Now() + boolTrue := true + m.alertRules = []AlertRule{ + {ID: "rule-speeding", Name: "持续超速告警", Description: "速度持续高于阈值", Severity: "critical", ValueType: "numeric", Metric: "speed_kmh", Operator: "gt", Threshold: 80, DurationSec: 60, RecoveryOperator: "lte", RecoveryThreshold: 75, RepeatIntervalSec: 600, ScopeProtocols: []string{"JT808", "GB32960"}, ScopeVINs: []string{}, NotificationChannels: []string{"in_app"}, Enabled: true, Version: 2, CreatedBy: "system", UpdatedBy: "platform-admin", CreatedAt: now.AddDate(0, -2, 0).Format(time.RFC3339), UpdatedAt: now.Add(-48 * time.Hour).Format(time.RFC3339)}, + {ID: "rule-offline", Name: "离线超时", Description: "车辆超过阈值未上报", Severity: "major", ValueType: "numeric", Metric: "freshness_sec", Operator: "gt", Threshold: 3600, DurationSec: 0, RecoveryOperator: "lte", RecoveryThreshold: 300, RepeatIntervalSec: 3600, ScopeProtocols: []string{"JT808", "GB32960", "YUTONG_MQTT"}, ScopeVINs: []string{}, NotificationChannels: []string{"in_app"}, Enabled: true, Version: 1, CreatedBy: "system", UpdatedBy: "system", CreatedAt: now.AddDate(0, -3, 0).Format(time.RFC3339), UpdatedAt: now.AddDate(0, -3, 0).Format(time.RFC3339)}, + {ID: "rule-alarm", Name: "协议告警位", Description: "原始协议告警位非零", Severity: "major", ValueType: "boolean", Metric: "alarm_active", Operator: "eq", BooleanThreshold: &boolTrue, DurationSec: 0, RecoveryOperator: "eq", RecoveryThreshold: 0, RepeatIntervalSec: 300, ScopeProtocols: []string{"JT808"}, ScopeVINs: []string{}, NotificationChannels: []string{"in_app"}, Enabled: false, Version: 1, CreatedBy: "platform-admin", UpdatedBy: "platform-admin", CreatedAt: now.Add(-72 * time.Hour).Format(time.RFC3339), UpdatedAt: now.Add(-72 * time.Hour).Format(time.RFC3339)}, + } + locations := []string{"广东省深圳市南山区科技南路", "广东省广州市白云区机场高速", "广东省东莞市南城街道", "广东省佛山市顺德区伦教街道", "上海市临港新片区", "四川省成都市高新区"} + statuses := []string{"unprocessed", "unprocessed", "processing", "recovered", "closed", "ignored"} + severities := []string{"critical", "critical", "major", "major", "minor", "minor"} + plates := []string{"粤AG18312", "粤B7C526", "粤C9D872", "粤E6P987", "豫A88888", "川AHTWO1"} + vins := []string{"LB9A32A24R0LS1426", "LFP23A98V2P012345", "LS5A3A5E8N0123456", "LJ12BA3R1N0456789", "LMRKH9AC2R1004087", "LNXNEGRR7SR318212"} + for i := range statuses { + triggered := now.Add(-time.Duration(8+i*11) * time.Minute) + event := AlertEvent{ID: "alert-demo-" + string(rune('1'+i)), RuleID: "rule-speeding", RuleName: "持续超速告警", RuleVersion: 2, Severity: severities[i], Status: statuses[i], VIN: vins[i], Plate: plates[i], Protocol: []string{"JT808", "JT808", "JT808", "GB32960", "YUTONG_MQTT", "GB32960"}[i], Metric: "speed_kmh", Operator: "gt", TriggerValue: 96 - float64(i*3), Threshold: 80, Unit: "km/h", DurationSec: 60, Location: locations[i], SourceEventID: "source-event-00" + string(rune('1'+i)), EventAt: triggered.Add(-time.Second).Format(time.RFC3339), ReceivedAt: triggered.Format(time.RFC3339), TriggeredAt: triggered.Format(time.RFC3339), Version: 1} + if statuses[i] == "processing" { + event.Handler = "张三" + } + if statuses[i] == "recovered" || statuses[i] == "closed" || statuses[i] == "ignored" { + event.RecoveredAt = triggered.Add(5 * time.Minute).Format(time.RFC3339) + } + event.Actions = []AlertAction{{ID: int64(i + 1), Action: "trigger", ToStatus: "unprocessed", Actor: "alert-evaluator", Note: "规则命中并达到持续时间", CreatedAt: triggered.Format(time.RFC3339)}} + if statuses[i] != "unprocessed" { + event.Actions = append(event.Actions, AlertAction{ID: int64(20 + i), Action: statuses[i], FromStatus: "unprocessed", ToStatus: statuses[i], Actor: firstNonEmpty(event.Handler, "platform-admin"), CreatedAt: triggered.Add(time.Minute).Format(time.RFC3339)}) + } + m.alertEvents = append(m.alertEvents, event) + } + m.nextAlertActionID = 100 + for i := 0; i < 9; i++ { + m.alertNotifications = append(m.alertNotifications, AlertNotification{ID: int64(i + 1), EventID: m.alertEvents[i%len(m.alertEvents)].ID, Title: m.alertEvents[i%len(m.alertEvents)].RuleName, Content: plates[i%len(plates)] + " 触发告警，请及时处理", Severity: severities[i%len(severities)], Channel: "in_app", Read: i >= 7, CreatedAt: now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339)}) + } + m.nextNotificationID = 10 +} + +func (m *MockStore) AlertSummary(_ context.Context, query AlertQuery) (AlertSummary, error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + result := AlertSummary{AsOf: time.Now().Format(time.RFC3339)} + for _, event := range m.alertEvents { + if !keepMockAlertEvent(event, query) { + continue + } + switch event.Status { + case "unprocessed": + result.Unprocessed++ + result.Active++ + case "processing": + result.Processing++ + result.Active++ + case "recovered": + result.Recovered++ + case "closed": + result.Closed++ + case "ignored": + result.Ignored++ + } + } + for _, item := range m.alertNotifications { + if !item.Read { + result.UnreadNotifications++ + } + } + return result, nil +} + +func (m *MockStore) ActiveAlertVINs(_ context.Context, protocol string) ([]string, error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + seen := map[string]struct{}{} + vins := make([]string, 0) + for _, event := range m.alertEvents { + if event.Status != "unprocessed" && event.Status != "processing" { + continue + } + if protocol != "" && event.Protocol != protocol { + continue + } + if _, exists := seen[event.VIN]; exists { + continue + } + seen[event.VIN] = struct{}{} + vins = append(vins, event.VIN) + } + return vins, nil +} + +func (m *MockStore) AlertEvents(_ context.Context, query AlertQuery) (Page[AlertEvent], error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + items := make([]AlertEvent, 0, len(m.alertEvents)) + for _, event := range m.alertEvents { + if keepMockAlertEvent(event, query) { + event.Actions = nil + items = append(items, event) + } + } + sort.SliceStable(items, func(i, j int) bool { return items[i].TriggeredAt > items[j].TriggeredAt }) + total := len(items) + start := query.Offset + if start > total { + start = total + } + end := start + query.Limit + if end > total { + end = total + } + return Page[AlertEvent]{Items: append([]AlertEvent(nil), items[start:end]...), Total: total, Limit: query.Limit, Offset: query.Offset}, nil +} + +func keepMockAlertEvent(event AlertEvent, query AlertQuery) bool { + keyword := strings.ToLower(query.Keyword) + if keyword != "" && !strings.Contains(strings.ToLower(event.VIN), keyword) && !strings.Contains(strings.ToLower(event.Plate), keyword) && !strings.Contains(strings.ToLower(event.RuleName), keyword) { + return false + } + if query.Severity != "" && query.Severity != "all" && event.Severity != query.Severity { + return false + } + if query.Status == "active" && event.Status != "unprocessed" && event.Status != "processing" { + return false + } + if query.Status != "" && query.Status != "all" && query.Status != "active" && event.Status != query.Status { + return false + } + if query.RuleID != "" && event.RuleID != query.RuleID { + return false + } + if query.Protocol != "" && !strings.EqualFold(event.Protocol, query.Protocol) { + return false + } + if query.DateFrom != "" && event.TriggeredAt < query.DateFrom { + return false + } + if query.DateTo != "" && event.TriggeredAt > query.DateTo { + return false + } + return true +} + +func (m *MockStore) AlertEvent(_ context.Context, id string) (AlertEvent, error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + for _, event := range m.alertEvents { + if event.ID == id { + event.Actions = append([]AlertAction(nil), event.Actions...) + return event, nil + } + } + return AlertEvent{}, clientError{Code: "ALERT_EVENT_NOT_FOUND", Message: "告警事件不存在"} +} + +func (m *MockStore) AlertRules(context.Context) ([]AlertRule, error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + return append([]AlertRule(nil), m.alertRules...), nil +} + +func (m *MockStore) SaveAlertRule(_ context.Context, input AlertRuleInput) (AlertRule, error) { + m.alertMu.Lock() + defer m.alertMu.Unlock() + now := time.Now().Format(time.RFC3339) + for i, current := range m.alertRules { + if current.ID != input.ID { + continue + } + if input.Version != current.Version { + return AlertRule{}, clientError{Code: "ALERT_RULE_VERSION_CONFLICT", Message: "规则已被其他用户更新，请刷新后重试"} + } + next := ruleFromInput(input) + next.Version = current.Version + 1 + next.CreatedAt = current.CreatedAt + next.CreatedBy = current.CreatedBy + next.UpdatedAt = now + next.UpdatedBy = input.Actor + m.alertRules[i] = next + return next, nil + } + if input.Version != 0 { + return AlertRule{}, clientError{Code: "ALERT_RULE_NOT_FOUND", Message: "待更新规则不存在"} + } + next := ruleFromInput(input) + next.Version = 1 + next.CreatedAt = now + next.UpdatedAt = now + next.CreatedBy = input.Actor + next.UpdatedBy = input.Actor + m.alertRules = append(m.alertRules, next) + return next, nil +} + +func ruleFromInput(input AlertRuleInput) AlertRule { + return AlertRule{ID: input.ID, Name: input.Name, Description: input.Description, Severity: strings.ToLower(input.Severity), ValueType: strings.ToLower(input.ValueType), Metric: input.Metric, Operator: strings.ToLower(input.Operator), Threshold: input.Threshold, ThresholdHigh: input.ThresholdHigh, BooleanThreshold: input.BooleanThreshold, DurationSec: input.DurationSec, RecoveryOperator: strings.ToLower(input.RecoveryOperator), RecoveryThreshold: input.RecoveryThreshold, RepeatIntervalSec: input.RepeatIntervalSec, ScopeProtocols: append([]string(nil), input.ScopeProtocols...), ScopeVINs: append([]string(nil), input.ScopeVINs...), ScopeOEMs: append([]string(nil), input.ScopeOEMs...), ScopeModels: append([]string(nil), input.ScopeModels...), ScopeCompanies: append([]string(nil), input.ScopeCompanies...), NotificationChannels: append([]string(nil), input.NotificationChannels...), Enabled: input.Enabled} +} + +func (m *MockStore) SetAlertRuleEnabled(_ context.Context, id string, update AlertRuleEnabledUpdate) (AlertRule, error) { + m.alertMu.Lock() + defer m.alertMu.Unlock() + for i := range m.alertRules { + if m.alertRules[i].ID == id { + if m.alertRules[i].Version != update.Version { + return AlertRule{}, clientError{Code: "ALERT_RULE_VERSION_CONFLICT", Message: "规则已被其他用户更新，请刷新后重试"} + } + m.alertRules[i].Enabled = update.Enabled + m.alertRules[i].Version++ + m.alertRules[i].UpdatedBy = update.Actor + m.alertRules[i].UpdatedAt = time.Now().Format(time.RFC3339) + return m.alertRules[i], nil + } + } + return AlertRule{}, clientError{Code: "ALERT_RULE_NOT_FOUND", Message: "规则不存在"} +} + +func (m *MockStore) ActOnAlert(_ context.Context, id string, request AlertActionRequest) (AlertEvent, error) { + m.alertMu.Lock() + defer m.alertMu.Unlock() + for i := range m.alertEvents { + event := &m.alertEvents[i] + if event.ID != id { + continue + } + if event.Version != request.Version { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_VERSION_CONFLICT", Message: "事件已被其他用户更新，请刷新后重试"} + } + from := event.Status + to := from + switch request.Action { + case "acknowledge": + if from != "unprocessed" { + return AlertEvent{}, clientError{Code: "ALERT_ACTION_NOT_ALLOWED", Message: "只有未处理告警可以确认"} + } + to = "processing" + event.Handler = request.Actor + case "close": + if from != "unprocessed" && from != "processing" && from != "recovered" { + return AlertEvent{}, clientError{Code: "ALERT_ACTION_NOT_ALLOWED", Message: "当前状态不能关闭"} + } + to = "closed" + event.Handler = request.Actor + case "ignore": + if from != "unprocessed" && from != "processing" { + return AlertEvent{}, clientError{Code: "ALERT_ACTION_NOT_ALLOWED", Message: "当前状态不能忽略"} + } + to = "ignored" + event.Handler = request.Actor + } + m.nextAlertActionID++ + event.Actions = append(event.Actions, AlertAction{ID: m.nextAlertActionID, Action: request.Action, FromStatus: from, ToStatus: to, Actor: request.Actor, Note: request.Note, CreatedAt: time.Now().Format(time.RFC3339)}) + event.Status = to + event.Version++ + return *event, nil + } + return AlertEvent{}, clientError{Code: "ALERT_EVENT_NOT_FOUND", Message: "告警事件不存在"} +} + +func (m *MockStore) AlertNotifications(_ context.Context, query AlertNotificationQuery) (Page[AlertNotification], error) { + m.alertMu.RLock() + defer m.alertMu.RUnlock() + items := make([]AlertNotification, 0, len(m.alertNotifications)) + for _, item := range m.alertNotifications { + if !query.UnreadOnly || !item.Read { + items = append(items, item) + } + } + total := len(items) + start := query.Offset + if start > total { + start = total + } + end := start + query.Limit + if end > total { + end = total + } + return Page[AlertNotification]{Items: append([]AlertNotification(nil), items[start:end]...), Total: total, Limit: query.Limit, Offset: query.Offset}, nil +} + +func (m *MockStore) MarkAlertNotificationsRead(_ context.Context, request AlertNotificationReadRequest) (int, error) { + m.alertMu.Lock() + defer m.alertMu.Unlock() + ids := map[int64]bool{} + for _, id := range request.IDs { + ids[id] = true + } + count := 0 + now := time.Now().Format(time.RFC3339) + for i := range m.alertNotifications { + if ids[m.alertNotifications[i].ID] && !m.alertNotifications[i].Read { + m.alertNotifications[i].Read = true + m.alertNotifications[i].ReadAt = now + count++ + } + } + return count, nil +} + +func (m *MockStore) EvaluateAlerts(context.Context) (AlertEvaluationResult, error) { + return AlertEvaluationResult{RulesEvaluated: len(m.alertRules), VehiclesScanned: len(m.vehicles), AsOf: time.Now().Format(time.RFC3339)}, nil +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_store.go b/vehicle-data-platform/apps/api/internal/platform/alert_store.go new file mode 100644 index 00000000..7fca1e69 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_store.go @@ -0,0 +1,457 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" +) + +func (s *ProductionStore) ensureAlertSchema(ctx context.Context) error { + s.alertSchemaOnce.Do(func() { + var count int + s.alertSchemaErr = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_alert_rule`).Scan(&count) + if s.alertSchemaErr != nil { + s.alertSchemaErr = fmt.Errorf("alert center schema unavailable; apply deploy/migrations/002_alert_center.sql: %w", s.alertSchemaErr) + } + }) + return s.alertSchemaErr +} + +func (s *ProductionStore) AlertSummary(ctx context.Context, query AlertQuery) (AlertSummary, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertSummary{}, err + } + where, args := buildAlertWhere(query) + var result AlertSummary + err := s.db.QueryRowContext(ctx, `SELECT + COALESCE(SUM(CASE WHEN status IN ('unprocessed','processing') THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN status='unprocessed' THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN status='processing' THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN status='recovered' THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN status='closed' THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN status='ignored' THEN 1 ELSE 0 END),0) +FROM vehicle_alert_event e WHERE `+where, args...).Scan(&result.Active, &result.Unprocessed, &result.Processing, &result.Recovered, &result.Closed, &result.Ignored) + if err != nil { + return AlertSummary{}, err + } + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_alert_notification WHERE channel='in_app' AND is_read=0`).Scan(&result.UnreadNotifications); err != nil { + return AlertSummary{}, err + } + result.AsOf = time.Now().Format(time.RFC3339) + return result, nil +} + +func (s *ProductionStore) ActiveAlertVINs(ctx context.Context, protocol string) ([]string, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return nil, err + } + query := `SELECT DISTINCT vin FROM vehicle_alert_event WHERE status IN ('unprocessed','processing') AND vin<>''` + args := []any{} + if protocol = strings.TrimSpace(protocol); protocol != "" { + query += ` AND protocol=?` + args = append(args, protocol) + } + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + vins := make([]string, 0) + for rows.Next() { + var vin string + if err := rows.Scan(&vin); err != nil { + return nil, err + } + vins = append(vins, vin) + } + return vins, rows.Err() +} + +func buildAlertWhere(query AlertQuery) (string, []any) { + where := []string{"1=1"} + args := []any{} + if query.Keyword != "" { + like := "%" + query.Keyword + "%" + where = append(where, "(e.vin LIKE ? OR e.plate LIKE ? OR e.rule_name LIKE ?)") + args = append(args, like, like, like) + } + if query.Severity != "" && query.Severity != "all" { + where = append(where, "e.severity=?") + args = append(args, query.Severity) + } + if query.Status == "active" { + where = append(where, "e.status IN ('unprocessed','processing')") + } else if query.Status != "" && query.Status != "all" { + where = append(where, "e.status=?") + args = append(args, query.Status) + } + if query.RuleID != "" { + where = append(where, "e.rule_id=?") + args = append(args, query.RuleID) + } + if query.Protocol != "" { + where = append(where, "e.protocol=?") + args = append(args, query.Protocol) + } + if query.DateFrom != "" { + where = append(where, "e.triggered_at>=?") + args = append(args, query.DateFrom) + } + if query.DateTo != "" { + where = append(where, "e.triggered_at<=?") + args = append(args, query.DateTo) + } + return strings.Join(where, " AND "), args +} + +const alertEventSelect = `SELECT e.id,e.rule_id,e.rule_name,e.rule_version,e.severity,e.status,e.vin,e.plate,e.protocol, +e.metric,e.operator,e.trigger_value,e.threshold_value,e.threshold_high,e.unit,e.duration_sec,e.location_text,e.longitude,e.latitude, +e.source_event_id,COALESCE(DATE_FORMAT(e.event_at,'%Y-%m-%dT%H:%i:%s.%fZ'),''),COALESCE(DATE_FORMAT(e.received_at,'%Y-%m-%dT%H:%i:%s.%fZ'),''), +DATE_FORMAT(e.triggered_at,'%Y-%m-%dT%H:%i:%s.%fZ'),COALESCE(DATE_FORMAT(e.recovered_at,'%Y-%m-%dT%H:%i:%s.%fZ'),''),e.handler,e.version +FROM vehicle_alert_event e ` + +func scanAlertEvent(scanner interface{ Scan(...any) error }) (AlertEvent, error) { + var event AlertEvent + var longitude, latitude sql.NullFloat64 + err := scanner.Scan(&event.ID, &event.RuleID, &event.RuleName, &event.RuleVersion, &event.Severity, &event.Status, &event.VIN, &event.Plate, &event.Protocol, + &event.Metric, &event.Operator, &event.TriggerValue, &event.Threshold, &event.ThresholdHigh, &event.Unit, &event.DurationSec, &event.Location, &longitude, &latitude, + &event.SourceEventID, &event.EventAt, &event.ReceivedAt, &event.TriggeredAt, &event.RecoveredAt, &event.Handler, &event.Version) + if longitude.Valid { + event.Longitude = &longitude.Float64 + } + if latitude.Valid { + event.Latitude = &latitude.Float64 + } + return event, err +} + +func (s *ProductionStore) AlertEvents(ctx context.Context, query AlertQuery) (Page[AlertEvent], error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return Page[AlertEvent]{}, err + } + where, args := buildAlertWhere(query) + var total int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_alert_event e WHERE `+where, args...).Scan(&total); err != nil { + return Page[AlertEvent]{}, err + } + listArgs := append(append([]any(nil), args...), query.Limit, query.Offset) + rows, err := s.db.QueryContext(ctx, alertEventSelect+`WHERE `+where+` ORDER BY e.triggered_at DESC,e.id DESC LIMIT ? OFFSET ?`, listArgs...) + if err != nil { + return Page[AlertEvent]{}, err + } + defer rows.Close() + items := make([]AlertEvent, 0, query.Limit) + for rows.Next() { + event, err := scanAlertEvent(rows) + if err != nil { + return Page[AlertEvent]{}, err + } + items = append(items, event) + } + return Page[AlertEvent]{Items: items, Total: total, Limit: query.Limit, Offset: query.Offset}, rows.Err() +} + +func (s *ProductionStore) AlertEvent(ctx context.Context, id string) (AlertEvent, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertEvent{}, err + } + event, err := scanAlertEvent(s.db.QueryRowContext(ctx, alertEventSelect+`WHERE e.id=?`, id)) + if err == sql.ErrNoRows { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_NOT_FOUND", Message: "告警事件不存在"} + } + if err != nil { + return AlertEvent{}, err + } + rows, err := s.db.QueryContext(ctx, `SELECT id,action,from_status,to_status,actor,note,DATE_FORMAT(created_at,'%Y-%m-%dT%H:%i:%s.%fZ') FROM vehicle_alert_event_action WHERE event_id=? ORDER BY created_at,id`, id) + if err != nil { + return AlertEvent{}, err + } + defer rows.Close() + for rows.Next() { + var item AlertAction + if err := rows.Scan(&item.ID, &item.Action, &item.FromStatus, &item.ToStatus, &item.Actor, &item.Note, &item.CreatedAt); err != nil { + return AlertEvent{}, err + } + event.Actions = append(event.Actions, item) + } + return event, rows.Err() +} + +const alertRuleSelect = `SELECT id,name,description,severity,value_type,metric,operator,threshold_value,threshold_high,boolean_threshold,duration_sec, +recovery_operator,recovery_threshold,repeat_interval_sec,scope_protocols_json,scope_vins_json,COALESCE(scope_oems_json,'[]'),COALESCE(scope_models_json,'[]'),COALESCE(scope_companies_json,'[]'),notification_channels_json,enabled,version, +created_by,updated_by,DATE_FORMAT(created_at,'%Y-%m-%dT%H:%i:%s.%fZ'),DATE_FORMAT(updated_at,'%Y-%m-%dT%H:%i:%s.%fZ') FROM vehicle_alert_rule ` + +func scanAlertRule(scanner interface{ Scan(...any) error }) (AlertRule, error) { + var rule AlertRule + var boolean sql.NullBool + var protocols, vins, oems, models, companies, channels string + err := scanner.Scan(&rule.ID, &rule.Name, &rule.Description, &rule.Severity, &rule.ValueType, &rule.Metric, &rule.Operator, &rule.Threshold, &rule.ThresholdHigh, &boolean, &rule.DurationSec, &rule.RecoveryOperator, &rule.RecoveryThreshold, &rule.RepeatIntervalSec, &protocols, &vins, &oems, &models, &companies, &channels, &rule.Enabled, &rule.Version, &rule.CreatedBy, &rule.UpdatedBy, &rule.CreatedAt, &rule.UpdatedAt) + if boolean.Valid { + rule.BooleanThreshold = &boolean.Bool + } + if err == nil { + if e := json.Unmarshal([]byte(protocols), &rule.ScopeProtocols); e != nil { + return AlertRule{}, e + } + if e := json.Unmarshal([]byte(vins), &rule.ScopeVINs); e != nil { + return AlertRule{}, e + } + if e := json.Unmarshal([]byte(oems), &rule.ScopeOEMs); e != nil { + return AlertRule{}, e + } + if e := json.Unmarshal([]byte(models), &rule.ScopeModels); e != nil { + return AlertRule{}, e + } + if e := json.Unmarshal([]byte(companies), &rule.ScopeCompanies); e != nil { + return AlertRule{}, e + } + if e := json.Unmarshal([]byte(channels), &rule.NotificationChannels); e != nil { + return AlertRule{}, e + } + } + return rule, err +} + +func (s *ProductionStore) AlertRules(ctx context.Context) ([]AlertRule, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return nil, err + } + rows, err := s.db.QueryContext(ctx, alertRuleSelect+`ORDER BY enabled DESC,severity,name`) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AlertRule{} + for rows.Next() { + item, err := scanAlertRule(rows) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, rows.Err() +} + +func marshalAlertRuleLists(input AlertRuleInput) (string, string, string, string, string, string, error) { + p, e := json.Marshal(input.ScopeProtocols) + if e != nil { + return "", "", "", "", "", "", e + } + v, e := json.Marshal(input.ScopeVINs) + if e != nil { + return "", "", "", "", "", "", e + } + o, e := json.Marshal(input.ScopeOEMs) + if e != nil { + return "", "", "", "", "", "", e + } + m, e := json.Marshal(input.ScopeModels) + if e != nil { + return "", "", "", "", "", "", e + } + co, e := json.Marshal(input.ScopeCompanies) + if e != nil { + return "", "", "", "", "", "", e + } + c, e := json.Marshal(input.NotificationChannels) + return string(p), string(v), string(o), string(m), string(co), string(c), e +} + +const alertRuleInsertSQL = `INSERT INTO vehicle_alert_rule(id,name,description,severity,value_type,metric,operator,threshold_value,threshold_high,boolean_threshold,duration_sec,recovery_operator,recovery_threshold,repeat_interval_sec,scope_protocols_json,scope_vins_json,scope_oems_json,scope_models_json,scope_companies_json,notification_channels_json,enabled,version,created_by,updated_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?)` +const alertRuleUpdateSQL = `UPDATE vehicle_alert_rule SET name=?,description=?,severity=?,value_type=?,metric=?,operator=?,threshold_value=?,threshold_high=?,boolean_threshold=?,duration_sec=?,recovery_operator=?,recovery_threshold=?,repeat_interval_sec=?,scope_protocols_json=?,scope_vins_json=?,scope_oems_json=?,scope_models_json=?,scope_companies_json=?,notification_channels_json=?,enabled=?,version=version+1,updated_by=? WHERE id=? AND version=?` + +func (s *ProductionStore) SaveAlertRule(ctx context.Context, input AlertRuleInput) (AlertRule, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertRule{}, err + } + p, v, o, m, co, c, err := marshalAlertRuleLists(input) + if err != nil { + return AlertRule{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return AlertRule{}, err + } + defer tx.Rollback() + var current int + err = tx.QueryRowContext(ctx, `SELECT version FROM vehicle_alert_rule WHERE id=? FOR UPDATE`, input.ID).Scan(¤t) + if err == sql.ErrNoRows { + if input.Version != 0 { + return AlertRule{}, clientError{Code: "ALERT_RULE_NOT_FOUND", Message: "待更新规则不存在"} + } + _, err = tx.ExecContext(ctx, alertRuleInsertSQL, input.ID, input.Name, input.Description, strings.ToLower(input.Severity), strings.ToLower(input.ValueType), input.Metric, strings.ToLower(input.Operator), input.Threshold, input.ThresholdHigh, input.BooleanThreshold, input.DurationSec, strings.ToLower(input.RecoveryOperator), input.RecoveryThreshold, input.RepeatIntervalSec, p, v, o, m, co, c, input.Enabled, input.Actor, input.Actor) + current = 0 + } else if err != nil { + return AlertRule{}, err + } else { + if current != input.Version { + return AlertRule{}, clientError{Code: "ALERT_RULE_VERSION_CONFLICT", Message: "规则已被其他用户更新，请刷新后重试"} + } + res, e := tx.ExecContext(ctx, alertRuleUpdateSQL, input.Name, input.Description, strings.ToLower(input.Severity), strings.ToLower(input.ValueType), input.Metric, strings.ToLower(input.Operator), input.Threshold, input.ThresholdHigh, input.BooleanThreshold, input.DurationSec, strings.ToLower(input.RecoveryOperator), input.RecoveryThreshold, input.RepeatIntervalSec, p, v, o, m, co, c, input.Enabled, input.Actor, input.ID, current) + err = e + if err == nil { + n, _ := res.RowsAffected() + if n != 1 { + err = clientError{Code: "ALERT_RULE_VERSION_CONFLICT", Message: "规则更新冲突，请刷新后重试"} + } + } + } + if err != nil { + return AlertRule{}, err + } + snapshot, _ := json.Marshal(input) + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_rule_audit(rule_id,rule_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`, input.ID, current+1, input.Actor, firstNonEmpty(map[bool]string{true: "create", false: "update"}[current == 0], "update"), string(snapshot)); err != nil { + return AlertRule{}, err + } + if err = tx.Commit(); err != nil { + return AlertRule{}, err + } + return scanAlertRule(s.db.QueryRowContext(ctx, alertRuleSelect+`WHERE id=?`, input.ID)) +} + +func (s *ProductionStore) SetAlertRuleEnabled(ctx context.Context, id string, update AlertRuleEnabledUpdate) (AlertRule, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertRule{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return AlertRule{}, err + } + defer tx.Rollback() + res, err := tx.ExecContext(ctx, `UPDATE vehicle_alert_rule SET enabled=?,version=version+1,updated_by=? WHERE id=? AND version=?`, update.Enabled, update.Actor, id, update.Version) + if err != nil { + return AlertRule{}, err + } + n, _ := res.RowsAffected() + if n != 1 { + return AlertRule{}, clientError{Code: "ALERT_RULE_VERSION_CONFLICT", Message: "规则不存在或已被其他用户更新"} + } + rule, err := scanAlertRule(tx.QueryRowContext(ctx, alertRuleSelect+`WHERE id=?`, id)) + if err != nil { + return AlertRule{}, err + } + if !update.Enabled { + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_candidate WHERE rule_id=?`, id); err != nil { + return AlertRule{}, err + } + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_rule_state WHERE rule_id=?`, id); err != nil { + return AlertRule{}, err + } + } + snapshot, _ := json.Marshal(rule) + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_rule_audit(rule_id,rule_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`, id, rule.Version, update.Actor, map[bool]string{true: "enable", false: "disable"}[update.Enabled], string(snapshot)); err != nil { + return AlertRule{}, err + } + if err = tx.Commit(); err != nil { + return AlertRule{}, err + } + return rule, nil +} + +func alertActionTarget(status, action string) (string, error) { + switch action { + case "acknowledge": + if status == "unprocessed" { + return "processing", nil + } + case "close": + if status == "unprocessed" || status == "processing" || status == "recovered" { + return "closed", nil + } + case "ignore": + if status == "unprocessed" || status == "processing" { + return "ignored", nil + } + } + return "", clientError{Code: "ALERT_ACTION_NOT_ALLOWED", Message: "当前状态不允许该处置动作"} +} + +func (s *ProductionStore) ActOnAlert(ctx context.Context, id string, request AlertActionRequest) (AlertEvent, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return AlertEvent{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return AlertEvent{}, err + } + defer tx.Rollback() + var status string + var version int + if err = tx.QueryRowContext(ctx, `SELECT status,version FROM vehicle_alert_event WHERE id=? FOR UPDATE`, id).Scan(&status, &version); err == sql.ErrNoRows { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_NOT_FOUND", Message: "告警事件不存在"} + } else if err != nil { + return AlertEvent{}, err + } + if version != request.Version { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_VERSION_CONFLICT", Message: "事件已被其他用户更新，请刷新后重试"} + } + target, err := alertActionTarget(status, request.Action) + if err != nil { + return AlertEvent{}, err + } + res, err := tx.ExecContext(ctx, `UPDATE vehicle_alert_event SET status=?,handler=?,version=version+1 WHERE id=? AND version=?`, target, request.Actor, id, version) + if err != nil { + return AlertEvent{}, err + } + n, _ := res.RowsAffected() + if n != 1 { + return AlertEvent{}, clientError{Code: "ALERT_EVENT_VERSION_CONFLICT", Message: "事件更新冲突，请刷新后重试"} + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,?,?,?,?,?)`, id, request.Action, status, target, request.Actor, request.Note); err != nil { + return AlertEvent{}, err + } + if err = tx.Commit(); err != nil { + return AlertEvent{}, err + } + return s.AlertEvent(ctx, id) +} + +func (s *ProductionStore) AlertNotifications(ctx context.Context, query AlertNotificationQuery) (Page[AlertNotification], error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return Page[AlertNotification]{}, err + } + where := "channel='in_app'" + if query.UnreadOnly { + where += " AND is_read=0" + } + var total int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_alert_notification WHERE `+where).Scan(&total); err != nil { + return Page[AlertNotification]{}, err + } + rows, err := s.db.QueryContext(ctx, `SELECT id,event_id,title,content,severity,channel,is_read,DATE_FORMAT(created_at,'%Y-%m-%dT%H:%i:%s.%fZ'),COALESCE(DATE_FORMAT(read_at,'%Y-%m-%dT%H:%i:%s.%fZ'),'') FROM vehicle_alert_notification WHERE `+where+` ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?`, query.Limit, query.Offset) + if err != nil { + return Page[AlertNotification]{}, err + } + defer rows.Close() + items := []AlertNotification{} + for rows.Next() { + var item AlertNotification + if err := rows.Scan(&item.ID, &item.EventID, &item.Title, &item.Content, &item.Severity, &item.Channel, &item.Read, &item.CreatedAt, &item.ReadAt); err != nil { + return Page[AlertNotification]{}, err + } + items = append(items, item) + } + return Page[AlertNotification]{Items: items, Total: total, Limit: query.Limit, Offset: query.Offset}, rows.Err() +} + +func (s *ProductionStore) MarkAlertNotificationsRead(ctx context.Context, request AlertNotificationReadRequest) (int, error) { + if err := s.ensureAlertSchema(ctx); err != nil { + return 0, err + } + placeholders := make([]string, len(request.IDs)) + args := make([]any, 0, len(request.IDs)+1) + args = append(args, request.Actor) + for i, id := range request.IDs { + placeholders[i] = "?" + args = append(args, id) + } + result, err := s.db.ExecContext(ctx, `UPDATE vehicle_alert_notification SET is_read=1,read_by=?,read_at=CURRENT_TIMESTAMP(3) WHERE channel='in_app' AND is_read=0 AND id IN (`+strings.Join(placeholders, ",")+`)`, args...) + if err != nil { + return 0, err + } + n, err := result.RowsAffected() + return int(n), err +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_stream.go b/vehicle-data-platform/apps/api/internal/platform/alert_stream.go new file mode 100644 index 00000000..0f05c61f --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_stream.go @@ -0,0 +1,271 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +const alertStreamMaxFutureSkew = 10 * time.Minute + +type AlertStreamRecord struct { + Topic string + Partition int + Offset int64 + HighWatermark int64 + Protocol string + VIN string + Plate string + SourceEventID string + EventAt time.Time + ReceivedAt time.Time + Fields map[string]json.RawMessage + Valid bool + Late bool + ErrorCode string +} + +type AlertStreamBatchResult struct { + Fetched int + Processed int + Valid int + Invalid int + Late int + ReplaySkipped int + Partitions int + RulesEvaluated int + CandidatesAdvanced int + DuplicateObservations int + LateObservations int + Opened int + Recovered int +} + +type alertStreamEnvelope struct { + EventID string `json:"event_id"` + EventKind string `json:"event_kind"` + SourceEventID string `json:"source_event_id"` + FieldMapping string `json:"field_mapping"` + Protocol string `json:"protocol"` + VIN string `json:"vin"` + Plate string `json:"plate"` + EventTimeMS int64 `json:"event_time_ms"` + ReceivedAtMS int64 `json:"received_at_ms"` + Fields map[string]json.RawMessage `json:"fields"` +} + +func DecodeAlertStreamRecord(topic string, partition int, offset, highWatermark int64, payload []byte, lateness time.Duration) AlertStreamRecord { + record := AlertStreamRecord{Topic: strings.TrimSpace(topic), Partition: partition, Offset: offset, HighWatermark: highWatermark} + var envelope alertStreamEnvelope + if err := json.Unmarshal(payload, &envelope); err != nil { + record.ErrorCode = "invalid_json" + return record + } + record.Protocol = strings.TrimSpace(envelope.Protocol) + record.VIN = strings.TrimSpace(envelope.VIN) + record.Plate = strings.TrimSpace(envelope.Plate) + record.SourceEventID = firstNonEmpty(strings.TrimSpace(envelope.SourceEventID), strings.TrimSpace(envelope.EventID)) + record.Fields = envelope.Fields + if expected, known := alertStreamTopicProtocol(record.Topic); known && !strings.EqualFold(expected, record.Protocol) { + record.ErrorCode = "protocol_topic_mismatch" + return record + } + if !strings.EqualFold(strings.TrimSpace(envelope.EventKind), "FIELDS") { + record.ErrorCode = "event_kind_mismatch" + return record + } + if strings.TrimSpace(envelope.FieldMapping) == "" { + record.ErrorCode = "missing_field_mapping" + return record + } + if record.VIN == "" { + record.ErrorCode = "missing_vin_" + strings.ToLower(record.Protocol) + return record + } + if record.SourceEventID == "" { + record.ErrorCode = "missing_source_event_id" + return record + } + if len(envelope.Fields) == 0 { + record.ErrorCode = "missing_fields" + return record + } + prefix := map[string]string{"GB32960": "gb32960.", "JT808": "jt808.", "YUTONG_MQTT": "yutong_mqtt."}[strings.ToUpper(record.Protocol)] + for field := range envelope.Fields { + if prefix != "" && (field != strings.TrimSpace(field) || !strings.HasPrefix(field, prefix) || len(field) <= len(prefix)) { + record.ErrorCode = "invalid_field_name" + return record + } + } + if envelope.ReceivedAtMS <= 0 { + record.ErrorCode = "missing_received_time" + return record + } + receivedAt := time.UnixMilli(envelope.ReceivedAtMS) + eventMS := envelope.EventTimeMS + if eventMS <= 0 || eventMS > envelope.ReceivedAtMS+alertStreamMaxFutureSkew.Milliseconds() { + eventMS = envelope.ReceivedAtMS + } + record.EventAt = time.UnixMilli(eventMS) + record.ReceivedAt = receivedAt + if lateness > 0 && receivedAt.Sub(record.EventAt) > lateness { + record.Late = true + } + record.Valid = true + return record +} + +func alertStreamTopicProtocol(topic string) (string, bool) { + switch strings.TrimSpace(topic) { + case "vehicle.fields.go.gb32960.v1": + return "GB32960", true + case "vehicle.fields.go.jt808.v1": + return "JT808", true + case "vehicle.fields.go.yutong-mqtt.v1": + return "YUTONG_MQTT", true + default: + return "", false + } +} + +type alertStreamPartition struct { + topic string + partition int +} + +type alertStreamCheckpointUpdate struct { + key alertStreamPartition + nextOffset, highWatermark int64 + processed, valid, invalid, late, replay int64 + lastEventAt, lastReceivedAt any + lastSourceEventID string + lastInvalidCode string +} + +func (s *ProductionStore) RecordAlertStreamBatch(ctx context.Context, consumerGroup string, records []AlertStreamRecord) (AlertStreamBatchResult, error) { + result := AlertStreamBatchResult{Fetched: len(records)} + consumerGroup = strings.TrimSpace(consumerGroup) + if consumerGroup == "" || len(consumerGroup) > 128 { + return result, fmt.Errorf("alert stream consumer group is empty or too long") + } + if len(records) == 0 { + return result, nil + } + grouped := map[alertStreamPartition][]AlertStreamRecord{} + for _, record := range records { + if record.Topic == "" || len(record.Topic) > 255 || record.Partition < 0 || record.Offset < 0 { + return result, fmt.Errorf("invalid alert stream message position topic=%q partition=%d offset=%d", record.Topic, record.Partition, record.Offset) + } + key := alertStreamPartition{topic: record.Topic, partition: record.Partition} + grouped[key] = append(grouped[key], record) + } + keys := make([]alertStreamPartition, 0, len(grouped)) + for key := range grouped { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].topic == keys[j].topic { + return keys[i].partition < keys[j].partition + } + return keys[i].topic < keys[j].topic + }) + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return result, err + } + defer tx.Rollback() + updates := make([]alertStreamCheckpointUpdate, 0, len(keys)) + accepted := make([]AlertStreamRecord, 0, len(records)) + for _, key := range keys { + partitionRecords := grouped[key] + sort.Slice(partitionRecords, func(i, j int) bool { return partitionRecords[i].Offset < partitionRecords[j].Offset }) + var nextOffset int64 + hasCheckpoint := true + if err = tx.QueryRowContext(ctx, `SELECT next_offset FROM vehicle_alert_stream_checkpoint WHERE consumer_group=? AND topic=? AND partition_id=? FOR UPDATE`, consumerGroup, key.topic, key.partition).Scan(&nextOffset); err == sql.ErrNoRows { + hasCheckpoint = false + nextOffset = -1 + } else if err != nil { + return result, err + } + update := alertStreamCheckpointUpdate{key: key} + maxOffset := nextOffset - 1 + var lastEventTime time.Time + var lastPosition int64 + hasPosition := false + for _, record := range partitionRecords { + if record.HighWatermark > update.highWatermark { + update.highWatermark = record.HighWatermark + } + if hasCheckpoint && record.Offset < nextOffset { + update.replay++ + continue + } + if hasPosition && record.Offset == lastPosition { + update.replay++ + continue + } + lastPosition = record.Offset + hasPosition = true + update.processed++ + if record.Valid { + update.valid++ + if record.Late { + update.late++ + } + if lastEventTime.IsZero() || record.EventAt.After(lastEventTime) { + lastEventTime = record.EventAt + update.lastEventAt = record.EventAt + update.lastReceivedAt = record.ReceivedAt + update.lastSourceEventID = record.SourceEventID + } + accepted = append(accepted, record) + } else { + update.invalid++ + update.lastInvalidCode = record.ErrorCode + } + if record.Offset > maxOffset { + maxOffset = record.Offset + } + } + update.nextOffset = nextOffset + if maxOffset >= 0 && maxOffset+1 > update.nextOffset { + update.nextOffset = maxOffset + 1 + } + if update.nextOffset < 0 { + update.nextOffset = 0 + } + updates = append(updates, update) + result.Processed += int(update.processed) + result.Valid += int(update.valid) + result.Invalid += int(update.invalid) + result.Late += int(update.late) + result.ReplaySkipped += int(update.replay) + } + if strings.EqualFold(s.alertStreamMode, "active") && len(accepted) > 0 { + evaluation, evaluationErr := evaluateAlertStreamRecordsTx(ctx, tx, accepted, time.Now()) + if evaluationErr != nil { + return result, evaluationErr + } + result.RulesEvaluated = evaluation.RulesEvaluated + result.CandidatesAdvanced = evaluation.CandidatesAdvanced + result.DuplicateObservations = evaluation.DuplicateObservations + result.LateObservations = evaluation.LateObservations + result.Opened = evaluation.Opened + result.Recovered = evaluation.Recovered + } + for _, update := range updates { + _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_stream_checkpoint(consumer_group,topic,partition_id,next_offset,high_watermark,processed_count,valid_count,invalid_count,late_count,replay_skipped_count,last_event_at,last_received_at,last_source_event_id,last_invalid_code,last_invalid_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,IF(?='',NULL,CURRENT_TIMESTAMP(3))) ON DUPLICATE KEY UPDATE next_offset=GREATEST(next_offset,VALUES(next_offset)),high_watermark=GREATEST(high_watermark,VALUES(high_watermark)),processed_count=processed_count+VALUES(processed_count),valid_count=valid_count+VALUES(valid_count),invalid_count=invalid_count+VALUES(invalid_count),late_count=late_count+VALUES(late_count),replay_skipped_count=replay_skipped_count+VALUES(replay_skipped_count),last_event_at=COALESCE(VALUES(last_event_at),last_event_at),last_received_at=COALESCE(VALUES(last_received_at),last_received_at),last_source_event_id=IF(VALUES(last_source_event_id)='',last_source_event_id,VALUES(last_source_event_id)),last_invalid_code=IF(VALUES(last_invalid_code)='',last_invalid_code,VALUES(last_invalid_code)),last_invalid_at=COALESCE(VALUES(last_invalid_at),last_invalid_at)`, consumerGroup, update.key.topic, update.key.partition, update.nextOffset, update.highWatermark, update.processed, update.valid, update.invalid, update.late, update.replay, update.lastEventAt, update.lastReceivedAt, update.lastSourceEventID, update.lastInvalidCode, update.lastInvalidCode) + if err != nil { + return result, err + } + } + result.Partitions = len(keys) + if err = tx.Commit(); err != nil { + return result, err + } + return result, nil +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go b/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go new file mode 100644 index 00000000..8d3ee430 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go @@ -0,0 +1,471 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +type alertStreamVehicleMetadata struct { + Plate, OEM, Model, Company string +} + +type alertStreamMetricMapping map[string]map[string]string + +func evaluateAlertStreamRecordsTx(ctx context.Context, tx *sql.Tx, records []AlertStreamRecord, now time.Time) (AlertEvaluationResult, error) { + result := AlertEvaluationResult{VehiclesScanned: len(records), AsOf: now.Format(time.RFC3339)} + rules, err := loadActiveAlertStreamRules(ctx, tx) + if err != nil { + return result, err + } + if len(rules) == 0 { + return result, nil + } + mappings, err := loadAlertStreamMetricMappings(ctx, tx) + if err != nil { + return result, err + } + metadata, err := loadAlertStreamVehicleMetadata(ctx, tx, records) + if err != nil { + return result, err + } + candidates, err := loadAlertStreamCandidates(ctx, tx, rules, records) + if err != nil { + return result, err + } + activeEvents, err := loadActiveAlertStreamEvents(ctx, tx, rules, records) + if err != nil { + return result, err + } + ruleStates, err := loadAlertStreamRuleStates(ctx, tx, rules, records) + if err != nil { + return result, err + } + lastTriggered, err := loadAlertStreamLastTriggered(ctx, tx, rules, records) + if err != nil { + return result, err + } + candidateUpserts := map[string]alertCandidateUpsert{} + candidateDeletes := map[string]alertCandidateUpsert{} + stateUpserts := map[string]alertRuleStateUpsert{} + for _, rule := range rules { + result.RulesEvaluated++ + for _, record := range records { + item := alertStreamEvidence(record, metadata[record.VIN], mappings) + if !alertRuleInScope(rule, item) { + continue + } + value, supported := alertStreamMetricValue(rule.Metric, record, mappings) + if !supported { + continue + } + // Delayed device observations remain useful evidence for data-delay + // rules, but must not open or recover telemetry-value alerts using + // evidence outside the configured ingestion lateness window. + if record.Late && !strings.EqualFold(rule.Metric, "data_delay_sec") { + result.LateObservations++ + continue + } + observedAt := record.EventAt + if strings.EqualFold(rule.Metric, "data_delay_sec") { + observedAt = record.ReceivedAt + } + fingerprint := rule.ID + "|" + record.VIN + "|" + record.Protocol + matched := false + if strings.EqualFold(rule.Operator, "changed") { + normalized := 0.0 + if value != 0 { + normalized = 1 + } + previous, exists := ruleStates[fingerprint] + if exists && !observedAt.After(previous.LastObservedAt) { + result.LateObservations++ + continue + } + matched = exists && previous.LastValue != normalized + ruleStates[fingerprint] = alertRuleState{LastValue: normalized, LastObservedAt: observedAt} + stateUpserts[fingerprint] = alertRuleStateUpsert{RuleID: rule.ID, VIN: record.VIN, Protocol: record.Protocol, LastValue: normalized, ObservedAt: observedAt} + } else { + matched = compareAlertRuleValue(value, rule) + } + if matched { + if len(activeEvents[fingerprint]) > 0 { + continue + } + candidate, exists := candidates[fingerprint] + candidate, decision := advanceAlertCandidate(candidate, exists, observedAt, record.SourceEventID, false) + switch decision { + case alertObservationDuplicate: + result.DuplicateObservations++ + continue + case alertObservationLate: + result.LateObservations++ + continue + } + candidates[fingerprint] = candidate + result.CandidatesAdvanced++ + candidateUpserts[fingerprint] = alertCandidateUpsert{RuleID: rule.ID, VIN: record.VIN, Protocol: record.Protocol, SourceEventID: candidate.SourceEventID, FirstMatchedAt: candidate.FirstMatchedAt, LastMatchedAt: candidate.LastMatchedAt, LatestValue: value} + delete(candidateDeletes, fingerprint) + if int(candidate.LastMatchedAt.Sub(candidate.FirstMatchedAt).Seconds()) < rule.DurationSec || !alertRepeatAllowed(now, lastTriggered[fingerprint], rule.RepeatIntervalSec) { + continue + } + id, idErr := newAlertID("alert") + if idErr != nil { + return result, idErr + } + insertSQL, insertArgs := buildAlertEventInsert(id, fingerprint, rule, item, value) + if _, err = tx.ExecContext(ctx, insertSQL, insertArgs...); err != nil { + return result, err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'trigger','','unprocessed','alert-stream-evaluator','Kafka 事件时间规则命中并达到持续时间')`, id); err != nil { + return result, err + } + unit := alertMetricUnit(rule.Metric) + for _, channel := range rule.NotificationChannels { + delivery := "reserved" + if channel == "in_app" { + delivery = "created" + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_notification(event_id,title,content,severity,channel,delivery_status) VALUES(?,?,?,?,?,?)`, id, rule.Name, fmt.Sprintf("%s / %s 触发%s：%.2f %s", item.Plate, item.VIN, rule.Name, value, unit), rule.Severity, channel, delivery); err != nil { + return result, err + } + } + activeEvents[fingerprint] = []alertActiveEvent{{ID: id, Status: "unprocessed"}} + lastTriggered[fingerprint] = now + delete(candidates, fingerprint) + delete(candidateUpserts, fingerprint) + candidateDeletes[fingerprint] = alertCandidateUpsert{RuleID: rule.ID, VIN: record.VIN, Protocol: record.Protocol} + result.Opened++ + continue + } + if _, exists := candidates[fingerprint]; exists { + delete(candidates, fingerprint) + delete(candidateUpserts, fingerprint) + candidateDeletes[fingerprint] = alertCandidateUpsert{RuleID: rule.ID, VIN: record.VIN, Protocol: record.Protocol} + } + recovered := true + if strings.TrimSpace(rule.RecoveryOperator) != "" { + recovered = compareAlertValue(value, rule.RecoveryOperator, rule.RecoveryThreshold) + } + if !recovered { + continue + } + for _, event := range activeEvents[fingerprint] { + if _, err = tx.ExecContext(ctx, `UPDATE vehicle_alert_event SET status='recovered',recovered_at=CURRENT_TIMESTAMP(3),version=version+1 WHERE id=? AND status=?`, event.ID, event.Status); err != nil { + return result, err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'recover',?,'recovered','alert-stream-evaluator','Kafka 事件时间恢复条件满足')`, event.ID, event.Status); err != nil { + return result, err + } + result.Recovered++ + } + delete(activeEvents, fingerprint) + } + } + for _, item := range candidateDeletes { + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_candidate WHERE rule_id=? AND vin=? AND protocol=?`, item.RuleID, item.VIN, item.Protocol); err != nil { + return result, err + } + } + candidateItems := make([]alertCandidateUpsert, 0, len(candidateUpserts)) + for _, item := range candidateUpserts { + candidateItems = append(candidateItems, item) + } + for start := 0; start < len(candidateItems); start += alertCandidateBatchSize { + end := min(start+alertCandidateBatchSize, len(candidateItems)) + query, args := buildAlertCandidateUpsert(candidateItems[start:end]) + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return result, err + } + } + stateItems := make([]alertRuleStateUpsert, 0, len(stateUpserts)) + for _, item := range stateUpserts { + stateItems = append(stateItems, item) + } + for start := 0; start < len(stateItems); start += alertCandidateBatchSize { + end := min(start+alertCandidateBatchSize, len(stateItems)) + query, args := buildAlertRuleStateUpsert(stateItems[start:end]) + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return result, err + } + } + return result, nil +} + +func alertStreamScope(rules []AlertRule, records []AlertStreamRecord) ([]string, []string, []string) { + ruleIDs := make([]string, 0, len(rules)) + vins := make([]string, 0, len(records)) + protocols := make([]string, 0, 3) + seenRules, seenVINs, seenProtocols := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, rule := range rules { + if !seenRules[rule.ID] { + seenRules[rule.ID] = true + ruleIDs = append(ruleIDs, rule.ID) + } + } + for _, record := range records { + if !seenVINs[record.VIN] { + seenVINs[record.VIN] = true + vins = append(vins, record.VIN) + } + if !seenProtocols[record.Protocol] { + seenProtocols[record.Protocol] = true + protocols = append(protocols, record.Protocol) + } + } + return ruleIDs, vins, protocols +} + +func alertStreamScopeWhere(rules []AlertRule, records []AlertStreamRecord) (string, []any) { + ruleIDs, vins, protocols := alertStreamScope(rules, records) + placeholders := func(count int) string { return strings.TrimSuffix(strings.Repeat("?,", count), ",") } + args := make([]any, 0, len(ruleIDs)+len(vins)+len(protocols)) + for _, value := range ruleIDs { + args = append(args, value) + } + for _, value := range vins { + args = append(args, value) + } + for _, value := range protocols { + args = append(args, value) + } + return `rule_id IN (` + placeholders(len(ruleIDs)) + `) AND vin IN (` + placeholders(len(vins)) + `) AND protocol IN (` + placeholders(len(protocols)) + `)`, args +} + +func loadAlertStreamCandidates(ctx context.Context, tx *sql.Tx, rules []AlertRule, records []AlertStreamRecord) (map[string]alertCandidateState, error) { + where, args := alertStreamScopeWhere(rules, records) + rows, err := tx.QueryContext(ctx, `SELECT rule_id,vin,protocol,first_matched_at,last_matched_at,source_event_id FROM vehicle_alert_candidate WHERE `+where, args...) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]alertCandidateState{} + for rows.Next() { + var ruleID, vin, protocol string + var state alertCandidateState + if err := rows.Scan(&ruleID, &vin, &protocol, &state.FirstMatchedAt, &state.LastMatchedAt, &state.SourceEventID); err != nil { + return nil, err + } + items[ruleID+"|"+vin+"|"+protocol] = state + } + return items, rows.Err() +} + +func loadActiveAlertStreamEvents(ctx context.Context, tx *sql.Tx, rules []AlertRule, records []AlertStreamRecord) (map[string][]alertActiveEvent, error) { + where, args := alertStreamScopeWhere(rules, records) + rows, err := tx.QueryContext(ctx, `SELECT fingerprint,id,status FROM vehicle_alert_event WHERE status IN ('unprocessed','processing') AND `+where+` FOR UPDATE`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string][]alertActiveEvent{} + for rows.Next() { + var fingerprint string + var event alertActiveEvent + if err := rows.Scan(&fingerprint, &event.ID, &event.Status); err != nil { + return nil, err + } + items[fingerprint] = append(items[fingerprint], event) + } + return items, rows.Err() +} + +func loadAlertStreamRuleStates(ctx context.Context, tx *sql.Tx, rules []AlertRule, records []AlertStreamRecord) (map[string]alertRuleState, error) { + where, args := alertStreamScopeWhere(rules, records) + rows, err := tx.QueryContext(ctx, `SELECT rule_id,vin,protocol,observed_value,last_observed_at FROM vehicle_alert_rule_state WHERE `+where, args...) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]alertRuleState{} + for rows.Next() { + var ruleID, vin, protocol string + var state alertRuleState + if err := rows.Scan(&ruleID, &vin, &protocol, &state.LastValue, &state.LastObservedAt); err != nil { + return nil, err + } + items[ruleID+"|"+vin+"|"+protocol] = state + } + return items, rows.Err() +} + +func loadAlertStreamLastTriggered(ctx context.Context, tx *sql.Tx, rules []AlertRule, records []AlertStreamRecord) (map[string]time.Time, error) { + where, args := alertStreamScopeWhere(rules, records) + rows, err := tx.QueryContext(ctx, `SELECT fingerprint,MAX(triggered_at) FROM vehicle_alert_event WHERE `+where+` GROUP BY fingerprint`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + items := map[string]time.Time{} + for rows.Next() { + var fingerprint string + var triggeredAt time.Time + if err := rows.Scan(&fingerprint, &triggeredAt); err != nil { + return nil, err + } + items[fingerprint] = triggeredAt + } + return items, rows.Err() +} + +func loadActiveAlertStreamRules(ctx context.Context, tx *sql.Tx) ([]AlertRule, error) { + rows, err := tx.QueryContext(ctx, alertRuleSelect+`WHERE enabled=1 AND metric<>'freshness_sec' ORDER BY id FOR UPDATE`) + if err != nil { + return nil, err + } + defer rows.Close() + rules := []AlertRule{} + for rows.Next() { + rule, scanErr := scanAlertRule(rows) + if scanErr != nil { + return nil, scanErr + } + rules = append(rules, rule) + } + return rules, rows.Err() +} + +func loadAlertStreamMetricMappings(ctx context.Context, tx *sql.Tx) (alertStreamMetricMapping, error) { + rows, err := tx.QueryContext(ctx, `SELECT m.metric_key,m.protocol,m.source_field FROM vehicle_metric_protocol_mapping m JOIN vehicle_metric_definition d ON d.metric_key=m.metric_key WHERE d.enabled=1 AND (d.alertable=1 OR d.metric_key IN ('longitude','latitude'))`) + if err != nil { + return nil, err + } + defer rows.Close() + result := alertStreamMetricMapping{} + for rows.Next() { + var metric, protocol, source string + if err := rows.Scan(&metric, &protocol, &source); err != nil { + return nil, err + } + if result[metric] == nil { + result[metric] = map[string]string{} + } + result[metric][strings.ToUpper(protocol)] = source + } + return result, rows.Err() +} + +func loadAlertStreamVehicleMetadata(ctx context.Context, tx *sql.Tx, records []AlertStreamRecord) (map[string]alertStreamVehicleMetadata, error) { + vins := make([]string, 0, len(records)) + seen := map[string]bool{} + for _, record := range records { + if !seen[record.VIN] { + seen[record.VIN] = true + vins = append(vins, record.VIN) + } + } + if len(vins) == 0 { + return map[string]alertStreamVehicleMetadata{}, nil + } + selects := make([]string, len(vins)) + args := make([]any, len(vins)) + for i, vin := range vins { + selects[i] = "SELECT ? AS vin" + args[i] = vin + } + query := `SELECT v.vin,COALESCE(MAX(NULLIF(b.plate,'')),''),COALESCE(MAX(b.oem),''),COALESCE(MAX(p.model_name),''),COALESCE(MAX(p.company_name),'') FROM (` + strings.Join(selects, " UNION ALL ") + `) v LEFT JOIN vehicle_identity_binding b ON b.vin=v.vin LEFT JOIN vehicle_profile p ON p.vin=v.vin GROUP BY v.vin` + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + result := map[string]alertStreamVehicleMetadata{} + for rows.Next() { + var vin string + var item alertStreamVehicleMetadata + if err := rows.Scan(&vin, &item.Plate, &item.OEM, &item.Model, &item.Company); err != nil { + return nil, err + } + result[vin] = item + } + return result, rows.Err() +} + +func alertStreamEvidence(record AlertStreamRecord, metadata alertStreamVehicleMetadata, mappings alertStreamMetricMapping) alertEvaluationEvidence { + longitude, _ := alertStreamLocationValue("longitude", record, mappings) + latitude, _ := alertStreamLocationValue("latitude", record, mappings) + plate := firstNonEmpty(record.Plate, metadata.Plate) + return alertEvaluationEvidence{ + VIN: record.VIN, Plate: plate, Protocol: record.Protocol, OEM: metadata.OEM, Model: metadata.Model, Company: metadata.Company, + SourceEventID: record.SourceEventID, + EventAt: record.EventAt.Format("2006-01-02 15:04:05.999"), + ReceivedAt: record.ReceivedAt.Format("2006-01-02 15:04:05.999"), + Longitude: longitude, Latitude: latitude, + Location: fmt.Sprintf("%.6f,%.6f", longitude, latitude), + } +} + +func alertStreamLocationValue(metric string, record AlertStreamRecord, mappings alertStreamMetricMapping) (float64, bool) { + if source := mappings[metric][strings.ToUpper(record.Protocol)]; source != "" && !strings.EqualFold(source, "vehicle_realtime_location."+metric) { + return alertStreamRawNumber(record.Fields[source]) + } + aliases := map[string]map[string][]string{ + "longitude": { + "GB32960": {"gb32960.position.longitude"}, "JT808": {"jt808.location.longitude"}, "YUTONG_MQTT": {"yutong_mqtt.data.longitude", "yutong_mqtt.root.data.longitude"}, + }, + "latitude": { + "GB32960": {"gb32960.position.latitude"}, "JT808": {"jt808.location.latitude"}, "YUTONG_MQTT": {"yutong_mqtt.data.latitude", "yutong_mqtt.root.data.latitude"}, + }, + } + for _, source := range aliases[metric][strings.ToUpper(record.Protocol)] { + if value, ok := alertStreamRawNumber(record.Fields[source]); ok { + return value, true + } + } + return 0, false +} + +func alertStreamMetricValue(metric string, record AlertStreamRecord, mappings alertStreamMetricMapping) (float64, bool) { + metric = strings.ToLower(strings.TrimSpace(metric)) + if metric == "data_delay_sec" { + return record.ReceivedAt.Sub(record.EventAt).Seconds(), true + } + source := mappings[metric][strings.ToUpper(record.Protocol)] + if source == "" || strings.EqualFold(source, "received_at-event_time") || strings.HasPrefix(source, "vehicle_realtime_location.") { + return 0, false + } + value, ok := alertStreamRawNumber(record.Fields[source]) + if ok && metric == "alarm_active" { + if value != 0 { + return 1, true + } + return 0, true + } + return value, ok +} + +func alertStreamRawNumber(raw json.RawMessage) (float64, bool) { + if len(raw) == 0 || string(raw) == "null" { + return 0, false + } + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return 0, false + } + switch typed := value.(type) { + case json.Number: + parsed, err := typed.Float64() + return parsed, err == nil + case string: + text := strings.TrimSpace(typed) + if strings.HasPrefix(strings.ToLower(text), "0x") { + parsed, err := strconv.ParseUint(text[2:], 16, 64) + return float64(parsed), err == nil + } + parsed, err := strconv.ParseFloat(text, 64) + return parsed, err == nil + case bool: + if typed { + return 1, true + } + return 0, true + default: + return 0, false + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_stream_test.go b/vehicle-data-platform/apps/api/internal/platform/alert_stream_test.go new file mode 100644 index 00000000..d50aa985 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_stream_test.go @@ -0,0 +1,210 @@ +package platform + +import ( + "encoding/json" + "errors" + "reflect" + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestDecodeAlertStreamRecordValidatesContractAndLateness(t *testing.T) { + payload := []byte(`{"event_id":"evt:fields","event_kind":"FIELDS","source_event_id":"evt","field_mapping":"2026-07-03.v1","protocol":"JT808","vin":"VIN1","plate":"川A10001","event_time_ms":1783980000000,"received_at_ms":1783980181000,"fields":{"jt808.location.speed_kmh":88}}`) + record := DecodeAlertStreamRecord("vehicle.fields.go.jt808.v1", 2, 10, 20, payload, 2*time.Minute) + if !record.Valid || !record.Late || record.ErrorCode != "" || record.SourceEventID != "evt" || record.Protocol != "JT808" { + t.Fatalf("unexpected valid record: %+v", record) + } + if delay := record.ReceivedAt.Sub(record.EventAt); delay != 181*time.Second { + t.Fatalf("event/receipt delay=%s", delay) + } + if value, ok := alertStreamRawNumber(record.Fields["jt808.location.speed_kmh"]); !ok || value != 88 { + t.Fatalf("decoded canonical fields were not retained: value=%v ok=%v", value, ok) + } + + badTopic := DecodeAlertStreamRecord("vehicle.fields.go.gb32960.v1", 0, 1, 2, payload, time.Minute) + if badTopic.Valid || badTopic.ErrorCode != "protocol_topic_mismatch" { + t.Fatalf("topic/protocol mismatch accepted: %+v", badTopic) + } + badField := []byte(`{"event_id":"evt:fields","event_kind":"FIELDS","source_event_id":"evt","field_mapping":"v1","protocol":"JT808","vin":"VIN1","event_time_ms":1783980000000,"received_at_ms":1783980001000,"fields":{"speed_kmh":88}}`) + record = DecodeAlertStreamRecord("vehicle.fields.go.jt808.v1", 0, 1, 2, badField, time.Minute) + if record.Valid || record.ErrorCode != "invalid_field_name" { + t.Fatalf("unscoped field accepted: %+v", record) + } +} + +func TestAlertStreamMetricValueUsesCatalogMappingAndStrictMissingSemantics(t *testing.T) { + record := AlertStreamRecord{ + Protocol: "GB32960", EventAt: time.Unix(100, 0), ReceivedAt: time.Unix(130, 0), + Fields: map[string]json.RawMessage{ + "gb32960.vehicle.speed_kmh": json.RawMessage(`"82.5"`), + "gb32960.alarm.general_alarm_flag": json.RawMessage(`"0x00000004"`), + }, + } + mappings := alertStreamMetricMapping{ + "speed_kmh": {"GB32960": "gb32960.vehicle.speed_kmh"}, + "soc_percent": {"GB32960": "gb32960.vehicle.soc_percent"}, + "alarm_active": {"GB32960": "gb32960.alarm.general_alarm_flag"}, + } + if value, ok := alertStreamMetricValue("speed_kmh", record, mappings); !ok || value != 82.5 { + t.Fatalf("mapped numeric field = %v,%v", value, ok) + } + if value, ok := alertStreamMetricValue("alarm_active", record, mappings); !ok || value != 1 { + t.Fatalf("hex alarm field = %v,%v", value, ok) + } + if _, ok := alertStreamMetricValue("soc_percent", record, mappings); ok { + t.Fatal("missing SOC must not be coerced to zero") + } + if value, ok := alertStreamMetricValue("data_delay_sec", record, mappings); !ok || value != 30 { + t.Fatalf("derived delay = %v,%v", value, ok) + } +} + +func TestAlertStreamStateQueriesAreScopedToBatchRulesVehiclesAndProtocols(t *testing.T) { + rules := []AlertRule{{ID: "r2"}, {ID: "r1"}, {ID: "r2"}} + records := []AlertStreamRecord{{VIN: "VIN1", Protocol: "JT808"}, {VIN: "VIN2", Protocol: "GB32960"}, {VIN: "VIN1", Protocol: "JT808"}} + where, args := alertStreamScopeWhere(rules, records) + if where != "rule_id IN (?,?) AND vin IN (?,?) AND protocol IN (?,?)" { + t.Fatalf("unexpected scoped predicate: %s", where) + } + want := []any{"r2", "r1", "VIN1", "VIN2", "JT808", "GB32960"} + if !reflect.DeepEqual(args, want) { + t.Fatalf("scope args=%#v want=%#v", args, want) + } +} + +func TestActiveAlertStreamCommitsRuleEffectsBeforeCheckpointInSameTransaction(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "").WithAlertStreamConfig("active", "alert-active") + record := AlertStreamRecord{Topic: "vehicle.fields.go.jt808.v1", Partition: 0, Offset: 7, HighWatermark: 8, Protocol: "JT808", VIN: "VIN1", SourceEventID: "evt-7", EventAt: time.Now(), ReceivedAt: time.Now(), Fields: map[string]json.RawMessage{"jt808.location.speed_kmh": json.RawMessage(`10`)}, Valid: true} + checkpointQuery := `SELECT next_offset FROM vehicle_alert_stream_checkpoint WHERE consumer_group=? AND topic=? AND partition_id=? FOR UPDATE` + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(checkpointQuery)).WithArgs("alert-active", record.Topic, 0).WillReturnRows(sqlmock.NewRows([]string{"next_offset"})) + mock.ExpectQuery(regexp.QuoteMeta(alertRuleSelect + `WHERE enabled=1 AND metric<>'freshness_sec' ORDER BY id FOR UPDATE`)).WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`INSERT INTO vehicle_alert_stream_checkpoint`).WithArgs("alert-active", record.Topic, 0, int64(8), int64(8), int64(1), int64(1), int64(0), int64(0), int64(0), record.EventAt, record.ReceivedAt, "evt-7", "", "").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + result, err := store.RecordAlertStreamBatch(t.Context(), "alert-active", []AlertStreamRecord{record}) + if err != nil || result.Processed != 1 { + t.Fatalf("active batch failed: result=%+v err=%v", result, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestActiveAlertStreamRollsBackCheckpointWhenRuleEvaluationFails(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "").WithAlertStreamConfig("active", "alert-active") + record := AlertStreamRecord{Topic: "vehicle.fields.go.jt808.v1", Partition: 0, Offset: 9, HighWatermark: 10, Protocol: "JT808", VIN: "VIN1", SourceEventID: "evt-9", EventAt: time.Now(), ReceivedAt: time.Now(), Fields: map[string]json.RawMessage{"jt808.location.speed_kmh": json.RawMessage(`10`)}, Valid: true} + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT next_offset FROM vehicle_alert_stream_checkpoint WHERE consumer_group=? AND topic=? AND partition_id=? FOR UPDATE`)).WithArgs("alert-active", record.Topic, 0).WillReturnRows(sqlmock.NewRows([]string{"next_offset"})) + mock.ExpectQuery(regexp.QuoteMeta(alertRuleSelect + `WHERE enabled=1 AND metric<>'freshness_sec' ORDER BY id FOR UPDATE`)).WillReturnError(errors.New("rule read failed")) + mock.ExpectRollback() + if _, err := store.RecordAlertStreamBatch(t.Context(), "alert-active", []AlertStreamRecord{record}); err == nil { + t.Fatal("rule evaluation failure must abort checkpoint transaction") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestDecodeAlertStreamRecordNormalizesMissingAndFutureEventTime(t *testing.T) { + for _, eventMS := range []int64{0, 1783981000001} { + payload := []byte(`{"event_id":"evt:fields","event_kind":"FIELDS","source_event_id":"evt","field_mapping":"v1","protocol":"JT808","vin":"VIN1","event_time_ms":` + itoa64(eventMS) + `,"received_at_ms":1783980000000,"fields":{"jt808.location.speed_kmh":1}}`) + record := DecodeAlertStreamRecord("vehicle.fields.go.jt808.v1", 0, 1, 2, payload, time.Minute) + if !record.Valid || !record.EventAt.Equal(record.ReceivedAt) || record.Late { + t.Fatalf("event time was not normalized: %+v", record) + } + } +} + +func TestAlertStreamCheckpointMakesDatabaseOffsetAuthoritative(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + eventAt := time.Date(2026, 7, 14, 7, 0, 0, 123000000, time.Local) + receivedAt := eventAt.Add(3 * time.Minute) + records := []AlertStreamRecord{ + {Topic: "vehicle.fields.go.jt808.v1", Partition: 0, Offset: 10, HighWatermark: 20, Valid: true, Late: true, EventAt: eventAt, ReceivedAt: receivedAt, SourceEventID: "evt-10"}, + {Topic: "vehicle.fields.go.jt808.v1", Partition: 0, Offset: 11, HighWatermark: 20, Valid: false, ErrorCode: "invalid_json"}, + } + checkpointQuery := `SELECT next_offset FROM vehicle_alert_stream_checkpoint WHERE consumer_group=? AND topic=? AND partition_id=? FOR UPDATE` + checkpointInsert := `INSERT INTO vehicle_alert_stream_checkpoint` + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(checkpointQuery)).WithArgs("alert-shadow", "vehicle.fields.go.jt808.v1", 0).WillReturnRows(sqlmock.NewRows([]string{"next_offset"})) + mock.ExpectExec(checkpointInsert).WithArgs("alert-shadow", "vehicle.fields.go.jt808.v1", 0, int64(12), int64(20), int64(2), int64(1), int64(1), int64(1), int64(0), eventAt, receivedAt, "evt-10", "invalid_json", "invalid_json").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + result, err := store.RecordAlertStreamBatch(t.Context(), "alert-shadow", records) + if err != nil { + t.Fatal(err) + } + if result.Processed != 2 || result.Valid != 1 || result.Invalid != 1 || result.Late != 1 || result.ReplaySkipped != 0 || result.Partitions != 1 { + t.Fatalf("unexpected first checkpoint result: %+v", result) + } + + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(checkpointQuery)).WithArgs("alert-shadow", "vehicle.fields.go.jt808.v1", 0).WillReturnRows(sqlmock.NewRows([]string{"next_offset"}).AddRow(12)) + mock.ExpectExec(checkpointInsert).WithArgs("alert-shadow", "vehicle.fields.go.jt808.v1", 0, int64(12), int64(20), int64(0), int64(0), int64(0), int64(0), int64(2), nil, nil, "", "", "").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + replayed, err := store.RecordAlertStreamBatch(t.Context(), "alert-shadow", records) + if err != nil { + t.Fatal(err) + } + if replayed.Processed != 0 || replayed.ReplaySkipped != 2 { + t.Fatalf("database checkpoint did not suppress replay: %+v", replayed) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestAlertStreamCheckpointHealthExposesLagAndQuality(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "").WithAlertStreamConfig("shadow", "alert-shadow") + query := `SELECT COUNT(*),COALESCE(SUM(GREATEST(high_watermark-next_offset,0)),0),COALESCE(SUM(processed_count),0),COALESCE(SUM(valid_count),0),COALESCE(SUM(invalid_count),0),COALESCE(SUM(late_count),0),COALESCE(SUM(replay_skipped_count),0),MAX(updated_at),COALESCE((SELECT c2.last_invalid_code FROM vehicle_alert_stream_checkpoint c2 WHERE c2.consumer_group=? AND c2.last_invalid_at IS NOT NULL ORDER BY c2.last_invalid_at DESC LIMIT 1),''),MAX(last_invalid_at) FROM vehicle_alert_stream_checkpoint WHERE consumer_group=?` + now := time.Now() + mock.ExpectQuery(regexp.QuoteMeta(query)).WithArgs("alert-shadow", "alert-shadow").WillReturnRows(sqlmock.NewRows([]string{"partitions", "lag", "processed", "valid", "invalid", "late", "replay", "updated_at", "last_invalid_code", "last_invalid_at"}).AddRow(9, 3, 1000, 999, 1, 4, 2, now, "missing_vin_jt808", now)) + health, status := store.alertStreamCheckpointHealth(t.Context()) + if health.Status != "warning" || status.Mode != "shadow" || status.Partitions != 9 || status.Lag != 3 || status.Processed != 1000 || status.Invalid != 1 || status.Late != 4 || status.ReplaySkipped != 2 || status.UpdatedAt == "" || status.LastInvalidCode != "missing_vin_jt808" || status.LastInvalidAt == "" { + t.Fatalf("checkpoint health lost evidence: health=%+v status=%+v", health, status) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func itoa64(value int64) string { + if value == 0 { + return "0" + } + negative := value < 0 + if negative { + value = -value + } + digits := make([]byte, 0, 20) + for value > 0 { + digits = append([]byte{byte('0' + value%10)}, digits...) + value /= 10 + } + if negative { + digits = append([]byte{'-'}, digits...) + } + return string(digits) +} diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_test.go b/vehicle-data-platform/apps/api/internal/platform/alert_test.go new file mode 100644 index 00000000..c878ee2b --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/alert_test.go @@ -0,0 +1,499 @@ +package platform + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +type configuredMetricStore struct { + *MockStore + definitions []MetricDefinition +} + +func (s *configuredMetricStore) MetricDefinitions(context.Context) ([]MetricDefinition, error) { + return append([]MetricDefinition(nil), s.definitions...), nil +} + +func BenchmarkAlertEvaluationPlan10000x20(b *testing.B) { + evidence := make([]alertEvaluationEvidence, 10_000) + for index := range evidence { + evidence[index] = alertEvaluationEvidence{VIN: "VIN" + strconv.Itoa(index), Protocol: "JT808", SpeedKmh: float64(index % 120), FreshnessSec: 5} + } + rules := make([]AlertRule, 20) + for index := range rules { + rules[index] = AlertRule{ID: "rule" + strconv.Itoa(index), Enabled: true, Metric: "speed_kmh", Operator: "gt", Threshold: float64(60 + index)} + } + now := time.Now() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + upserts := make([]alertCandidateUpsert, 0, len(evidence)) + for _, rule := range rules { + for _, item := range evidence { + value, ok := alertMetricValue(rule.Metric, item) + if ok && alertRuleInScope(rule, item) && compareAlertValue(value, rule.Operator, rule.Threshold) { + upserts = append(upserts, alertCandidateUpsert{RuleID: rule.ID, VIN: item.VIN, Protocol: item.Protocol, FirstMatchedAt: now, LastMatchedAt: now, LatestValue: value}) + } + } + } + for start := 0; start < len(upserts); start += alertCandidateBatchSize { + end := min(start+alertCandidateBatchSize, len(upserts)) + _, args := buildAlertCandidateUpsert(upserts[start:end]) + if len(args) == 0 { + b.Fatal("expected batched candidates") + } + } + } +} + +func TestAlertEventWorkflowIsVersionedAndAudited(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + list := httptest.NewRecorder() + handler.ServeHTTP(list, httptest.NewRequest(http.MethodPost, "/api/v2/alerts/events", bytes.NewBufferString(`{"status":"unprocessed","limit":20}`))) + if list.Code != http.StatusOK { + t.Fatalf("list status=%d body=%s", list.Code, list.Body.String()) + } + var body struct { + Data Page[AlertEvent] `json:"data"` + } + if err := json.Unmarshal(list.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Data.Total < 2 { + t.Fatalf("expected seeded unprocessed events, got %+v", body.Data) + } + event := body.Data.Items[0] + action := func(version int) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v2/alerts/events/"+event.ID+"/actions", bytes.NewBufferString(`{"version":`+itoa(version)+`,"action":"acknowledge","note":"已联系司机","actor":"spoofed-body-user"}`)) + req.Header.Set("X-User-Name", "spoofed-header-user") + req = req.WithContext(WithPrincipal(req.Context(), Principal{Name: "operator-a", Role: "operator"})) + handler.ServeHTTP(rec, req) + return rec + } + rec := action(event.Version) + if rec.Code != http.StatusOK { + t.Fatalf("action status=%d body=%s", rec.Code, rec.Body.String()) + } + var updated struct { + Data AlertEvent `json:"data"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &updated) + if updated.Data.Status != "processing" || updated.Data.Handler != "operator-a" || len(updated.Data.Actions) < 2 { + t.Fatalf("workflow not persisted: %+v", updated.Data) + } + stale := action(event.Version) + if stale.Code != http.StatusConflict { + t.Fatalf("stale update should conflict, status=%d body=%s", stale.Code, stale.Body.String()) + } +} + +func TestAlertEventsActiveStatusIncludesOnlyOpenWorkflowStates(t *testing.T) { + service := NewService(NewMockStore()) + page, err := service.AlertEvents(t.Context(), AlertQuery{Status: "active", Limit: 200}) + if err != nil { + t.Fatal(err) + } + if page.Total == 0 { + t.Fatal("expected seeded active alerts") + } + for _, event := range page.Items { + if event.Status != "unprocessed" && event.Status != "processing" { + t.Fatalf("active query returned terminal state: %+v", event) + } + } + if where, args := buildAlertWhere(AlertQuery{Status: "active"}); !strings.Contains(where, "status IN ('unprocessed','processing')") || len(args) != 0 { + t.Fatalf("active SQL filter is not authoritative: where=%s args=%v", where, args) + } +} + +func TestAlertRuleCreationNormalizesBooleanAndChannels(t *testing.T) { + service := NewService(NewMockStore()) + truth := true + rule, err := service.SaveAlertRule(t.Context(), AlertRuleInput{Name: "主电源异常", Severity: "major", ValueType: "boolean", Metric: "alarm_active", Operator: "eq", BooleanThreshold: &truth, ScopeModels: []string{"纯电客车", "纯电客车"}, ScopeCompanies: []string{"示范公交"}, NotificationChannels: []string{"sms", "sms"}, Enabled: true}) + if err != nil { + t.Fatal(err) + } + if rule.Threshold != 1 { + t.Fatalf("boolean threshold should normalize to numeric evaluator value: %+v", rule) + } + if len(rule.NotificationChannels) != 2 || rule.NotificationChannels[0] != "in_app" { + t.Fatalf("channels should include unique in-app truth: %+v", rule.NotificationChannels) + } + if len(rule.ScopeModels) != 1 || rule.ScopeModels[0] != "纯电客车" || len(rule.ScopeCompanies) != 1 { + t.Fatalf("master-data scopes should normalize and round-trip: %+v", rule) + } + _, err = service.SaveAlertRule(t.Context(), AlertRuleInput{ID: rule.ID, Version: 99, Name: rule.Name, Severity: rule.Severity, ValueType: rule.ValueType, Metric: rule.Metric, Operator: rule.Operator, BooleanThreshold: &truth}) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "ALERT_RULE_VERSION_CONFLICT" { + t.Fatalf("expected optimistic conflict, got %v", err) + } +} + +func TestAlertNotificationsReadUpdatesUnreadSummary(t *testing.T) { + service := NewService(NewMockStore()) + before, err := service.AlertSummary(t.Context(), AlertQuery{}) + if err != nil { + t.Fatal(err) + } + page, err := service.AlertNotifications(t.Context(), AlertNotificationQuery{UnreadOnly: true, Limit: 20}) + if err != nil { + t.Fatal(err) + } + if page.Total != before.UnreadNotifications || page.Total == 0 { + t.Fatalf("unread truth differs: summary=%+v page=%+v", before, page) + } + count, err := service.MarkAlertNotificationsRead(t.Context(), AlertNotificationReadRequest{IDs: []int64{page.Items[0].ID}, Actor: "operator-a"}) + if err != nil || count != 1 { + t.Fatalf("mark read count=%d err=%v", count, err) + } + after, _ := service.AlertSummary(t.Context(), AlertQuery{}) + if after.UnreadNotifications != before.UnreadNotifications-1 { + t.Fatalf("unread should recalculate: before=%d after=%d", before.UnreadNotifications, after.UnreadNotifications) + } +} + +func TestAlertEvaluatorComparisonsAndScope(t *testing.T) { + item := alertEvaluationEvidence{VIN: "VIN1", Protocol: "JT808", OEM: "宇通", Model: "纯电客车", Company: "示范公交", SpeedKmh: 96, AlarmFlag: 1} + rule := AlertRule{Metric: "speed_kmh", Operator: "gt", Threshold: 80, ScopeProtocols: []string{"JT808"}, ScopeVINs: []string{"VIN1"}} + value, ok := alertMetricValue(rule.Metric, item) + if !ok || !compareAlertValue(value, rule.Operator, rule.Threshold) || !alertRuleInScope(rule, item) { + t.Fatalf("numeric rule should match current evidence") + } + if compareAlertValue(75, "gt", 80) || !compareAlertValue(75, "lte", 75) { + t.Fatal("comparison boundary incorrect") + } + rule.ScopeVINs = []string{"OTHER"} + if alertRuleInScope(rule, item) { + t.Fatal("VIN scope must prevent cross-vehicle evaluation") + } + rule.ScopeVINs = nil + rule.ScopeOEMs = []string{"其他厂家"} + if alertRuleInScope(rule, item) { + t.Fatal("OEM scope must prevent cross-manufacturer evaluation") + } + rule.ScopeOEMs = nil + rule.ScopeModels = []string{"氢燃料重卡"} + if alertRuleInScope(rule, item) { + t.Fatal("model scope must use authoritative vehicle profile") + } + rule.ScopeModels = []string{"纯电客车"} + rule.ScopeCompanies = []string{"其他企业"} + if alertRuleInScope(rule, item) { + t.Fatal("company scope must use authoritative vehicle profile") + } + if !compareAlertRuleValue(80, AlertRule{Operator: "between", Threshold: 70, ThresholdHigh: 90}) || compareAlertRuleValue(95, AlertRule{Operator: "between", Threshold: 70, ThresholdHigh: 90}) { + t.Fatal("between comparison boundary incorrect") + } + if !compareAlertRuleValue(95, AlertRule{Operator: "outside", Threshold: 70, ThresholdHigh: 90}) || compareAlertRuleValue(80, AlertRule{Operator: "outside", Threshold: 70, ThresholdHigh: 90}) { + t.Fatal("outside comparison boundary incorrect") + } +} + +func TestAlertRuleSQLMatchesMasterDataScopeArguments(t *testing.T) { + if placeholders := strings.Count(alertRuleInsertSQL, "?"); placeholders != 23 { + t.Fatalf("alert insert placeholders=%d query=%s", placeholders, alertRuleInsertSQL) + } + if placeholders := strings.Count(alertRuleUpdateSQL, "?"); placeholders != 23 { + t.Fatalf("alert update placeholders=%d query=%s", placeholders, alertRuleUpdateSQL) + } +} + +func TestAlertRuleScopeBoundsProtectEvaluator(t *testing.T) { + values := make([]string, 501) + for index := range values { + values[index] = "company-" + strconv.Itoa(index) + } + err := validateAlertRuleScopes(AlertRuleInput{ScopeCompanies: values}) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "ALERT_RULE_SCOPE_TOO_LARGE" { + t.Fatalf("large scope should be rejected, got %v", err) + } +} + +func TestAlertCandidateUpsertBatchesRows(t *testing.T) { + now := time.Now() + query, args := buildAlertCandidateUpsert([]alertCandidateUpsert{ + {RuleID: "r1", VIN: "v1", Protocol: "JT808", FirstMatchedAt: now, LastMatchedAt: now, LatestValue: 91, SourceEventID: "e1"}, + {RuleID: "r1", VIN: "v2", Protocol: "GB32960", FirstMatchedAt: now, LastMatchedAt: now, LatestValue: 92, SourceEventID: "e2"}, + }) + if strings.Count(query, "(?,?,?,?,?,?,?)") != 2 || len(args) != 14 || !strings.Contains(query, "ON DUPLICATE KEY UPDATE") { + t.Fatalf("unexpected batch query=%s args=%#v", query, args) + } +} + +func TestAlertStateLoadersPreserveMySQLDatetimeMilliseconds(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatal(err) + } + first := time.Date(2026, 7, 14, 10, 0, 0, 123000000, time.Local) + last := first.Add(20*time.Second + 333*time.Millisecond) + candidateSQL := `SELECT c.rule_id,c.vin,c.protocol,c.first_matched_at,c.last_matched_at,c.source_event_id FROM vehicle_alert_candidate c JOIN vehicle_alert_rule r ON r.id=c.rule_id AND r.enabled=1` + mock.ExpectQuery(regexp.QuoteMeta(candidateSQL)).WillReturnRows(sqlmock.NewRows([]string{"rule_id", "vin", "protocol", "first_matched_at", "last_matched_at", "source_event_id"}).AddRow("r1", "v1", "JT808", first, last, "event-2")) + candidates, err := loadAlertCandidates(t.Context(), tx) + if err != nil { + t.Fatal(err) + } + candidate := candidates["r1|v1|JT808"] + if !candidate.FirstMatchedAt.Equal(first) || !candidate.LastMatchedAt.Equal(last) || candidate.SourceEventID != "event-2" { + t.Fatalf("candidate datetime precision lost: %+v", candidate) + } + + stateSQL := `SELECT s.rule_id,s.vin,s.protocol,s.observed_value,s.last_observed_at FROM vehicle_alert_rule_state s JOIN vehicle_alert_rule r ON r.id=s.rule_id AND r.enabled=1` + mock.ExpectQuery(regexp.QuoteMeta(stateSQL)).WillReturnRows(sqlmock.NewRows([]string{"rule_id", "vin", "protocol", "observed_value", "last_observed_at"}).AddRow("r2", "v2", "GB32960", 1.0, last)) + states, err := loadAlertRuleStates(t.Context(), tx) + if err != nil { + t.Fatal(err) + } + state := states["r2|v2|GB32960"] + if state.LastValue != 1 || !state.LastObservedAt.Equal(last) { + t.Fatalf("rule state datetime precision lost: %+v", state) + } + + lastTriggeredSQL := `SELECT e.fingerprint,MAX(e.triggered_at) FROM vehicle_alert_event e JOIN vehicle_alert_rule r ON r.id=e.rule_id AND r.enabled=1 GROUP BY e.fingerprint` + mock.ExpectQuery(regexp.QuoteMeta(lastTriggeredSQL)).WillReturnRows(sqlmock.NewRows([]string{"fingerprint", "triggered_at"}).AddRow("r1|v1|JT808", last)) + lastTriggered, err := loadAlertLastTriggered(t.Context(), tx) + if err != nil { + t.Fatal(err) + } + if !lastTriggered["r1|v1|JT808"].Equal(last) { + t.Fatalf("repeat timestamp precision lost: %s", lastTriggered["r1|v1|JT808"]) + } + mock.ExpectRollback() + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestAlertRuleDisableIsAtomicAndClearsEvaluatorState(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + mock.ExpectQuery(regexp.QuoteMeta(`SELECT COUNT(*) FROM vehicle_alert_rule`)).WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta(`UPDATE vehicle_alert_rule SET enabled=?,version=version+1,updated_by=? WHERE id=? AND version=?`)).WithArgs(false, "admin-a", "rule-1", 1).WillReturnResult(sqlmock.NewResult(0, 1)) + columns := []string{"id", "name", "description", "severity", "value_type", "metric", "operator", "threshold_value", "threshold_high", "boolean_threshold", "duration_sec", "recovery_operator", "recovery_threshold", "repeat_interval_sec", "scope_protocols_json", "scope_vins_json", "scope_oems_json", "scope_models_json", "scope_companies_json", "notification_channels_json", "enabled", "version", "created_by", "updated_by", "created_at", "updated_at"} + mock.ExpectQuery(regexp.QuoteMeta(alertRuleSelect + `WHERE id=?`)).WithArgs("rule-1").WillReturnRows(sqlmock.NewRows(columns).AddRow("rule-1", "超速", "", "minor", "numeric", "speed_kmh", "gte", 0.0, 0.0, nil, 20, "", 0.0, 3600, `["JT808"]`, `["VIN1"]`, `[]`, `[]`, `[]`, `["in_app"]`, false, 2, "admin-a", "admin-a", "2026-07-14T07:00:00.000000Z", "2026-07-14T07:01:00.000000Z")) + mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM vehicle_alert_candidate WHERE rule_id=?`)).WithArgs("rule-1").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM vehicle_alert_rule_state WHERE rule_id=?`)).WithArgs("rule-1").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO vehicle_alert_rule_audit(rule_id,rule_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`)).WithArgs("rule-1", 2, "admin-a", "disable", sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + rule, err := store.SetAlertRuleEnabled(t.Context(), "rule-1", AlertRuleEnabledUpdate{Version: 1, Enabled: false, Actor: "admin-a"}) + if err != nil { + t.Fatal(err) + } + if rule.Enabled || rule.Version != 2 { + t.Fatalf("unexpected disabled rule: %+v", rule) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestAlertEvaluationLocksRuleAgainstConcurrentDisable(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(`SELECT enabled FROM vehicle_alert_rule WHERE id=? FOR UPDATE`)).WithArgs("rule-1").WillReturnRows(sqlmock.NewRows([]string{"enabled"}).AddRow(true)) + enabled, err := lockAlertRuleForEvaluation(t.Context(), tx, "rule-1") + if err != nil || !enabled { + t.Fatalf("rule lock enabled=%t err=%v", enabled, err) + } + mock.ExpectRollback() + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestAlertRuleStateUpsertBatchesChangedValues(t *testing.T) { + now := time.Now() + query, args := buildAlertRuleStateUpsert([]alertRuleStateUpsert{{RuleID: "r1", VIN: "v1", Protocol: "JT808", LastValue: 1, ObservedAt: now}, {RuleID: "r1", VIN: "v2", Protocol: "GB32960", LastValue: 0, ObservedAt: now}}) + if strings.Count(query, "(?,?,?,?,?)") != 2 || len(args) != 10 || !strings.Contains(query, "ON DUPLICATE KEY UPDATE") { + t.Fatalf("unexpected state batch query=%s args=%#v", query, args) + } +} + +func TestAlertEventInsertContractMatchesArguments(t *testing.T) { + rule := AlertRule{ID: "rule-1", Name: "超速", Version: 3, Severity: "major", Metric: "speed_kmh", Operator: "between", Threshold: 60, ThresholdHigh: 90, DurationSec: 30} + item := alertEvaluationEvidence{VIN: "VIN1", Plate: "川A10001", Protocol: "JT808", SourceEventID: "event-1", EventAt: "2026-07-14 10:00:00.000", ReceivedAt: "2026-07-14 10:00:01.000", Location: "104.1,30.6", Longitude: 104.1, Latitude: 30.6} + query, args := buildAlertEventInsert("alert-1", "rule-1|VIN1|JT808", rule, item, 75) + if placeholders := strings.Count(query, "?"); placeholders != len(args) { + t.Fatalf("event insert placeholders=%d args=%d query=%s", placeholders, len(args), query) + } + if len(args) != 22 || !strings.Contains(query, "'unprocessed'") || !strings.Contains(query, "CURRENT_TIMESTAMP(3)") { + t.Fatalf("unexpected event insert contract args=%#v query=%s", args, query) + } +} + +func TestAlertRuleValidationCoversRangesAndStateChange(t *testing.T) { + base := AlertRuleInput{Name: "区间", Severity: "major", ValueType: "numeric", Metric: "soc_percent", Operator: "between", Threshold: 20, ThresholdHigh: 80} + if err := validateAlertRule(base); err != nil { + t.Fatalf("valid range rejected: %v", err) + } + base.ThresholdHigh = 10 + if err := validateAlertRule(base); err == nil { + t.Fatal("inverted range should be rejected") + } + changed := AlertRuleInput{Name: "状态变化", Severity: "major", ValueType: "boolean", Metric: "alarm_active", Operator: "changed"} + if err := validateAlertRule(changed); err != nil { + t.Fatalf("boolean state change rejected: %v", err) + } + unknown := AlertRuleInput{Name: "未知指标", Severity: "major", ValueType: "numeric", Metric: "client_supplied_sql", Operator: "gt"} + if err := validateAlertRule(unknown); err == nil { + t.Fatal("metric outside the server catalog should be rejected") + } + mismatched := AlertRuleInput{Name: "类型错误", Severity: "major", ValueType: "boolean", Metric: "speed_kmh", Operator: "eq", BooleanThreshold: new(bool)} + if err := validateAlertRule(mismatched); err == nil { + t.Fatal("metric catalog type mismatch should be rejected") + } +} + +func TestMetricCatalogAndAlertValidationShareStoreConfiguration(t *testing.T) { + definitions := metricDefinitions() + for index := range definitions { + if definitions[index].Key == "speed_kmh" { + definitions[index].Label = "数据库配置速度" + definitions[index].Alertable = false + } + } + store := &configuredMetricStore{MockStore: NewMockStore(), definitions: definitions} + service := NewService(store) + catalog, err := service.MetricCatalog(t.Context()) + if err != nil { + t.Fatal(err) + } + if catalog.Metrics[0].Label != "数据库配置速度" { + t.Fatalf("catalog did not use store definitions: %+v", catalog.Metrics[0]) + } + _, err = service.SaveAlertRule(t.Context(), AlertRuleInput{Name: "应被禁用", Severity: "major", ValueType: "numeric", Metric: "speed_kmh", Operator: "gt"}) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "ALERT_RULE_METRIC_INVALID" { + t.Fatalf("rule validation must share configured catalog, got %v", err) + } +} + +func TestAlertRepeatIntervalBoundary(t *testing.T) { + now := time.Now() + if alertRepeatAllowed(now, now.Add(-59*time.Second), 60) { + t.Fatal("repeat should remain suppressed inside interval") + } + if !alertRepeatAllowed(now, now.Add(-60*time.Second), 60) || !alertRepeatAllowed(now, time.Time{}, 60) { + t.Fatal("repeat should open at boundary or without history") + } +} + +func TestAlertCandidateDurationOnlyAdvancesWithNewEventTime(t *testing.T) { + first := time.Date(2026, 7, 14, 10, 0, 0, 0, time.Local) + candidate, decision := advanceAlertCandidate(alertCandidateState{}, false, first, "event-1", false) + if decision != alertObservationAdvanced || !candidate.FirstMatchedAt.Equal(first) || !candidate.LastMatchedAt.Equal(first) { + t.Fatalf("first observation did not create candidate: decision=%v candidate=%+v", decision, candidate) + } + + unchanged, decision := advanceAlertCandidate(candidate, true, first.Add(30*time.Second), "event-1", false) + if decision != alertObservationDuplicate || !unchanged.LastMatchedAt.Equal(first) { + t.Fatalf("same source event must not invent duration: decision=%v candidate=%+v", decision, unchanged) + } + + advanced, decision := advanceAlertCandidate(candidate, true, first.Add(30*time.Second), "event-2", false) + if decision != alertObservationAdvanced || advanced.LastMatchedAt.Sub(advanced.FirstMatchedAt) != 30*time.Second { + t.Fatalf("newer event should advance event-time duration: decision=%v candidate=%+v", decision, advanced) + } + + late, decision := advanceAlertCandidate(advanced, true, first.Add(20*time.Second), "event-late", false) + if decision != alertObservationLate || late.SourceEventID != "event-2" { + t.Fatalf("late event should not regress candidate: decision=%v candidate=%+v", decision, late) + } +} + +func TestAlertCandidateContinuityGapResetsEventTimeWindow(t *testing.T) { + first := time.Date(2026, 7, 14, 10, 0, 0, 0, time.Local) + current := alertCandidateState{FirstMatchedAt: first, LastMatchedAt: first.Add(30 * time.Second), SourceEventID: "event-2"} + nextAt := current.LastMatchedAt.Add(alertCandidateContinuityWindow + time.Second) + next, decision := advanceAlertCandidate(current, true, nextAt, "event-3", false) + if decision != alertObservationAdvanced || !next.FirstMatchedAt.Equal(nextAt) || !next.LastMatchedAt.Equal(nextAt) { + t.Fatalf("continuity gap should start a new window: decision=%v candidate=%+v", decision, next) + } +} + +func TestFreshnessCandidateIsExplicitlyClockDriven(t *testing.T) { + first := time.Date(2026, 7, 14, 10, 0, 0, 0, time.Local) + item := alertEvaluationEvidence{SourceEventID: "same-event", EventAt: "2026-07-14 09:55:00.000000"} + observedAt, clockDriven, ok := alertObservationTime("freshness_sec", item, first) + if !ok || !clockDriven || !observedAt.Equal(first) { + t.Fatalf("freshness observation must use evaluator clock: at=%s clock=%t ok=%t", observedAt, clockDriven, ok) + } + candidate := alertCandidateState{FirstMatchedAt: first, LastMatchedAt: first, SourceEventID: item.SourceEventID} + next, decision := advanceAlertCandidate(candidate, true, first.Add(10*time.Second), item.SourceEventID, true) + if decision != alertObservationAdvanced || next.LastMatchedAt.Sub(next.FirstMatchedAt) != 10*time.Second { + t.Fatalf("freshness should advance while source event is unchanged: decision=%v candidate=%+v", decision, next) + } +} + +func TestAlertObservationTimePrefersEventTimeAndFallsBackToReceipt(t *testing.T) { + now := time.Date(2026, 7, 14, 12, 0, 0, 0, time.Local) + eventItem := alertEvaluationEvidence{EventAt: "2026-07-14 10:00:30.123000", ReceivedAt: "2026-07-14 10:00:31.000000"} + observedAt, clockDriven, ok := alertObservationTime("speed_kmh", eventItem, now) + if !ok || clockDriven || observedAt.Second() != 30 || observedAt.Nanosecond() != 123000000 { + t.Fatalf("event time not selected: at=%s clock=%t ok=%t", observedAt, clockDriven, ok) + } + receiptItem := alertEvaluationEvidence{ReceivedAt: "2026-07-14 10:00:31.000000"} + observedAt, _, ok = alertObservationTime("speed_kmh", receiptItem, now) + if !ok || observedAt.Second() != 31 { + t.Fatalf("receipt fallback not selected: at=%s ok=%t", observedAt, ok) + } +} + +func TestSnapshotEvaluatorKeepsOnlyFreshnessRulesWhenStreamIsActive(t *testing.T) { + rules := []AlertRule{{ID: "speed", Metric: "speed_kmh"}, {ID: "fresh", Metric: "freshness_sec"}, {ID: "delay", Metric: "data_delay_sec"}} + if got := snapshotAlertRules(rules, "shadow"); len(got) != 3 { + t.Fatalf("shadow mode filtered rules: %+v", got) + } + got := snapshotAlertRules(rules, "active") + if len(got) != 1 || got[0].ID != "fresh" { + t.Fatalf("active snapshot ownership is not freshness-only: %+v", got) + } +} + +func itoa(value int) string { + if value == 0 { + return "0" + } + digits := []byte{} + for value > 0 { + digits = append([]byte{byte('0' + value%10)}, digits...) + value /= 10 + } + return string(digits) +} diff --git a/vehicle-data-platform/apps/api/internal/platform/handler.go b/vehicle-data-platform/apps/api/internal/platform/handler.go index 93acfc59..51100eb9 100644 --- a/vehicle-data-platform/apps/api/internal/platform/handler.go +++ b/vehicle-data-platform/apps/api/internal/platform/handler.go @@ -52,6 +52,258 @@ func (h *Handler) routes() { h.mux.HandleFunc("GET /api/alert-events/notification-plan", h.handleQualityNotificationPlan) h.mux.HandleFunc("GET /api/ops/health", h.handleOpsHealth) h.mux.HandleFunc("GET /api/ops/source-readiness", h.handleSourceReadiness) + h.mux.HandleFunc("GET /api/v2/monitor/summary", h.handleMonitorSummary) + h.mux.HandleFunc("GET /api/v2/monitor/map", h.handleMonitorMap) + h.mux.HandleFunc("GET /api/v2/vehicles/{vin}/profile", h.handleVehicleProfile) + h.mux.HandleFunc("PUT /api/v2/vehicles/{vin}/profile", h.handleSaveVehicleProfile) + h.mux.HandleFunc("GET /api/v2/vehicles/{vin}/telemetry/latest", h.handleLatestTelemetry) + h.mux.HandleFunc("POST /api/v2/vehicle-profiles/sync", h.handleSyncVehicleProfiles) + h.mux.HandleFunc("GET /api/v2/tracks", h.handleTrackPlayback) + h.mux.HandleFunc("GET /api/v2/metrics", h.handleMetricCatalog) + h.mux.HandleFunc("GET /api/v2/history/metrics", h.handleHistoryMetricCatalog) + h.mux.HandleFunc("GET /api/v2/history/query", h.handleHistoryData) + h.mux.HandleFunc("GET /api/v2/history/series", h.handleHistorySeries) + h.mux.HandleFunc("POST /api/v2/exports", h.handleCreateHistoryExport) + h.mux.HandleFunc("GET /api/v2/exports", h.handleListHistoryExports) + h.mux.HandleFunc("GET /api/v2/exports/{id}/download", h.handleDownloadHistoryExport) + h.mux.HandleFunc("POST /api/v2/access/summary", h.handleAccessSummary) + h.mux.HandleFunc("POST /api/v2/access/vehicles", h.handleAccessVehicles) + h.mux.HandleFunc("POST /api/v2/access/unresolved-identities", h.handleAccessUnresolvedIdentities) + h.mux.HandleFunc("GET /api/v2/access/thresholds", h.handleAccessThresholds) + h.mux.HandleFunc("PUT /api/v2/access/thresholds", h.handleUpdateAccessThresholds) + h.mux.HandleFunc("POST /api/v2/alerts/summary", h.handleAlertSummary) + h.mux.HandleFunc("POST /api/v2/alerts/events", h.handleAlertEvents) + h.mux.HandleFunc("GET /api/v2/alerts/events/{id}", h.handleAlertEvent) + h.mux.HandleFunc("POST /api/v2/alerts/events/{id}/actions", h.handleAlertAction) + h.mux.HandleFunc("GET /api/v2/alerts/rules", h.handleAlertRules) + h.mux.HandleFunc("POST /api/v2/alerts/rules", h.handleSaveAlertRule) + h.mux.HandleFunc("PUT /api/v2/alerts/rules/{id}", h.handleSaveAlertRule) + h.mux.HandleFunc("PUT /api/v2/alerts/rules/{id}/enabled", h.handleAlertRuleEnabled) + h.mux.HandleFunc("GET /api/v2/alerts/notifications", h.handleAlertNotifications) + h.mux.HandleFunc("POST /api/v2/alerts/notifications/read", h.handleAlertNotificationsRead) +} + +func (h *Handler) handleAccessUnresolvedIdentities(w http.ResponseWriter, r *http.Request) { + var query AccessUnresolvedIdentityQuery + if !decodeJSONBody(w, r, &query) { + return + } + data, err := h.service.AccessUnresolvedIdentities(r.Context(), query) + h.write(w, r, data, err) +} + +func (h *Handler) handleVehicleProfile(w http.ResponseWriter, r *http.Request) { + data, err := h.service.VehicleProfile(r.Context(), r.PathValue("vin")) + h.write(w, r, data, err) +} + +func (h *Handler) handleLatestTelemetry(w http.ResponseWriter, r *http.Request) { + data, err := h.service.LatestTelemetry(r.Context(), r.PathValue("vin")) + h.write(w, r, data, err) +} + +func (h *Handler) handleSaveVehicleProfile(w http.ResponseWriter, r *http.Request) { + var input VehicleProfileInput + if !decodeJSONBody(w, r, &input) { + return + } + input.Actor = ActorFromContext(r.Context()) + data, err := h.service.SaveVehicleProfile(r.Context(), r.PathValue("vin"), input) + h.write(w, r, data, err) +} + +func (h *Handler) handleSyncVehicleProfiles(w http.ResponseWriter, r *http.Request) { + var request VehicleProfileSyncRequest + if !decodeJSONBody(w, r, &request) { + return + } + request.Actor = ActorFromContext(r.Context()) + data, err := h.service.SyncVehicleProfiles(r.Context(), request) + h.write(w, r, data, err) +} + +func (h *Handler) handleMetricCatalog(w http.ResponseWriter, r *http.Request) { + data, err := h.service.MetricCatalog(r.Context()) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertSummary(w http.ResponseWriter, r *http.Request) { + var query AlertQuery + if !decodeJSONBody(w, r, &query) { + return + } + data, err := h.service.AlertSummary(r.Context(), query) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertEvents(w http.ResponseWriter, r *http.Request) { + var query AlertQuery + if !decodeJSONBody(w, r, &query) { + return + } + data, err := h.service.AlertEvents(r.Context(), query) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertEvent(w http.ResponseWriter, r *http.Request) { + data, err := h.service.AlertEvent(r.Context(), r.PathValue("id")) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertAction(w http.ResponseWriter, r *http.Request) { + var request AlertActionRequest + if !decodeJSONBody(w, r, &request) { + return + } + request.Actor = ActorFromContext(r.Context()) + data, err := h.service.ActOnAlert(r.Context(), r.PathValue("id"), request) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertRules(w http.ResponseWriter, r *http.Request) { + data, err := h.service.AlertRules(r.Context()) + h.write(w, r, data, err) +} + +func (h *Handler) handleSaveAlertRule(w http.ResponseWriter, r *http.Request) { + var input AlertRuleInput + if !decodeJSONBody(w, r, &input) { + return + } + if pathID := r.PathValue("id"); pathID != "" { + input.ID = pathID + } + input.Actor = ActorFromContext(r.Context()) + data, err := h.service.SaveAlertRule(r.Context(), input) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertRuleEnabled(w http.ResponseWriter, r *http.Request) { + var update AlertRuleEnabledUpdate + if !decodeJSONBody(w, r, &update) { + return + } + update.Actor = ActorFromContext(r.Context()) + data, err := h.service.SetAlertRuleEnabled(r.Context(), r.PathValue("id"), update) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertNotifications(w http.ResponseWriter, r *http.Request) { + limit := parsePositive(r.URL.Query().Get("limit"), 20) + offset := parsePositive(r.URL.Query().Get("offset"), 0) + unread := strings.EqualFold(r.URL.Query().Get("unreadOnly"), "true") || r.URL.Query().Get("unreadOnly") == "1" + data, err := h.service.AlertNotifications(r.Context(), AlertNotificationQuery{UnreadOnly: unread, Limit: limit, Offset: offset}) + h.write(w, r, data, err) +} + +func (h *Handler) handleAlertNotificationsRead(w http.ResponseWriter, r *http.Request) { + var request AlertNotificationReadRequest + if !decodeJSONBody(w, r, &request) { + return + } + request.Actor = ActorFromContext(r.Context()) + count, err := h.service.MarkAlertNotificationsRead(r.Context(), request) + h.write(w, r, map[string]int{"updated": count}, err) +} + +func (h *Handler) handleAccessSummary(w http.ResponseWriter, r *http.Request) { + var query AccessQuery + if !decodeJSONBody(w, r, &query) { + return + } + data, err := h.service.AccessSummary(r.Context(), query) + h.write(w, r, data, err) +} + +func (h *Handler) handleAccessVehicles(w http.ResponseWriter, r *http.Request) { + var query AccessQuery + if !decodeJSONBody(w, r, &query) { + return + } + data, err := h.service.AccessVehicles(r.Context(), query) + h.write(w, r, data, err) +} + +func (h *Handler) handleAccessThresholds(w http.ResponseWriter, r *http.Request) { + data, err := h.service.AccessThresholds(r.Context()) + h.write(w, r, data, err) +} + +func (h *Handler) handleUpdateAccessThresholds(w http.ResponseWriter, r *http.Request) { + var update AccessThresholdUpdate + if !decodeJSONBody(w, r, &update) { + return + } + update.Actor = ActorFromContext(r.Context()) + data, err := h.service.UpdateAccessThresholds(r.Context(), update) + h.write(w, r, data, err) +} + +func decodeJSONBody(w http.ResponseWriter, r *http.Request, target any) bool { + defer r.Body.Close() + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + httpx.WriteError(w, http.StatusBadRequest, "BAD_JSON", "请求 JSON 解析失败", err.Error(), traceID(r)) + return false + } + return true +} + +func (h *Handler) handleCreateHistoryExport(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var request HistoryExportRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + httpx.WriteError(w, http.StatusBadRequest, "BAD_JSON", "请求 JSON 解析失败", err.Error(), traceID(r)) + return + } + data, err := h.service.CreateHistoryExport(request) + h.write(w, r, data, err) +} + +func (h *Handler) handleListHistoryExports(w http.ResponseWriter, r *http.Request) { + h.write(w, r, h.service.ListHistoryExports(), nil) +} + +func (h *Handler) handleDownloadHistoryExport(w http.ResponseWriter, r *http.Request) { + path, name, err := h.service.HistoryExportFile(r.PathValue("id")) + if err != nil { + h.write(w, r, nil, err) + return + } + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", `attachment; filename="history-export.csv"`) + w.Header().Set("X-Export-Name", name) + http.ServeFile(w, r, path) +} + +func (h *Handler) handleHistoryMetricCatalog(w http.ResponseWriter, r *http.Request) { + h.write(w, r, h.service.HistoryMetricCatalog(), nil) +} + +func (h *Handler) handleHistoryData(w http.ResponseWriter, r *http.Request) { + data, err := h.service.HistoryData(r.Context(), r.URL.Query()) + h.write(w, r, data, err) +} + +func (h *Handler) handleHistorySeries(w http.ResponseWriter, r *http.Request) { + data, err := h.service.HistorySeries(r.Context(), r.URL.Query()) + h.write(w, r, data, err) +} + +func (h *Handler) handleTrackPlayback(w http.ResponseWriter, r *http.Request) { + data, err := h.service.TrackPlayback(r.Context(), r.URL.Query()) + h.write(w, r, data, err) +} + +func (h *Handler) handleMonitorSummary(w http.ResponseWriter, r *http.Request) { + data, err := h.service.MonitorSummary(r.Context(), r.URL.Query()) + h.write(w, r, data, err) +} + +func (h *Handler) handleMonitorMap(w http.ResponseWriter, r *http.Request) { + data, err := h.service.MonitorMap(r.Context(), r.URL.Query()) + h.write(w, r, data, err) } func (h *Handler) handleDashboardSummary(w http.ResponseWriter, r *http.Request) { @@ -214,7 +466,14 @@ func (h *Handler) handleSourceReadiness(w http.ResponseWriter, r *http.Request) func (h *Handler) write(w http.ResponseWriter, r *http.Request, data any, err error) { if err != nil { if clientErr, ok := asClientError(err); ok { - httpx.WriteError(w, http.StatusBadRequest, clientErr.Code, clientErr.Message, "", traceID(r)) + status := http.StatusBadRequest + switch { + case strings.HasSuffix(clientErr.Code, "_NOT_FOUND"): + status = http.StatusNotFound + case strings.HasSuffix(clientErr.Code, "_CONFLICT"), clientErr.Code == "ALERT_ACTION_NOT_ALLOWED": + status = http.StatusConflict + } + httpx.WriteError(w, status, clientErr.Code, clientErr.Message, "", traceID(r)) return } httpx.WriteError(w, http.StatusInternalServerError, "INTERNAL", "服务处理失败", err.Error(), traceID(r)) diff --git a/vehicle-data-platform/apps/api/internal/platform/handler_test.go b/vehicle-data-platform/apps/api/internal/platform/handler_test.go index b948ae4c..44a9e4f9 100644 --- a/vehicle-data-platform/apps/api/internal/platform/handler_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/handler_test.go @@ -8,6 +8,7 @@ import ( "net/url" "strings" "testing" + "time" ) func TestHandlerDashboardSummary(t *testing.T) { @@ -26,6 +27,140 @@ func TestHandlerDashboardSummary(t *testing.T) { } } +func TestHandlerV2MonitorSummaryAndMap(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + for _, test := range []struct { + path string + want string + }{ + {path: "/api/v2/monitor/summary", want: "drivingVehicles"}, + {path: "/api/v2/monitor/map?zoom=5", want: "clusters"}, + {path: "/api/v2/monitor/map?zoom=11", want: "points"}, + } { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, test.path, nil) + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s status = %d body=%s", test.path, rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), test.want) { + t.Fatalf("%s missing %q: %s", test.path, test.want, rec.Body.String()) + } + if test.path == "/api/v2/monitor/summary" && (!strings.Contains(rec.Body.String(), `"alertDataAvailable":true`) || !strings.Contains(rec.Body.String(), `"alertVehicles":1`)) { + t.Fatalf("monitor summary must expose filtered active alert vehicles: %s", rec.Body.String()) + } + } +} + +func TestHandlerRejectsInvalidMonitorBounds(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + request := httptest.NewRequest(http.MethodGet, "/api/v2/monitor/map?zoom=12&bounds=105,31,103,29", nil) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusBadRequest || !strings.Contains(response.Body.String(), "MONITOR_BOUNDS_INVALID") { + t.Fatalf("unexpected invalid bounds response: status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestHandlerV2AccessManagement(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + for _, test := range []struct { + method string + path string + body string + want string + }{ + {method: http.MethodPost, path: "/api/v2/access/summary", body: `{}`, want: `"neverReported":1`}, + {method: http.MethodPost, path: "/api/v2/access/vehicles", body: `{"onlineState":"offline","limit":10}`, want: `"onlineState":"offline"`}, + {method: http.MethodGet, path: "/api/v2/access/thresholds", want: `"defaultThresholdSec":300`}, + {method: http.MethodPut, path: "/api/v2/access/thresholds", body: `{"version":1,"defaultThresholdSec":600,"delayThresholdSec":60,"longOfflineSec":3600,"protocols":[{"protocol":"JT808","thresholdSec":300}],"actor":"handler-test"}`, want: `"version":2`}, + } { + rec := httptest.NewRecorder() + req := httptest.NewRequest(test.method, test.path, strings.NewReader(test.body)) + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s %s status=%d body=%s", test.method, test.path, rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), test.want) { + t.Fatalf("%s %s missing %s body=%s", test.method, test.path, test.want, rec.Body.String()) + } + } +} + +func TestHandlerV2TrackPlayback(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v2/tracks?keyword=%E5%B7%9DAHTWO1&maxPoints=8", nil) + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + var body struct { + Data TrackPlaybackResponse `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("response JSON should decode: %v body=%s", err, rec.Body.String()) + } + if body.Data.VIN != "LNXNEGRR7SR318212" || body.Data.Summary.PointCount < 10 { + t.Fatalf("track should resolve vehicle and summarize full point set: %+v", body.Data) + } + if len(body.Data.Points) != 8 || !body.Data.Sampled { + t.Fatalf("track should server-sample map payload to requested bound: %+v", body.Data) + } + if len(body.Data.Events) < 2 || len(body.Data.Sources) != 1 { + t.Fatalf("track should expose events and source evidence: %+v", body.Data) + } +} + +func TestHandlerV2HistoryCatalogAndMultiVehicleQuery(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + for _, test := range []struct{ path, want string }{ + {path: "/api/v2/metrics", want: "sourceFields"}, + {path: "/api/v2/vehicles/%E5%B7%9DAHTWO1/telemetry/latest", want: "sourceField"}, + {path: "/api/v2/history/metrics", want: "dailyMileageKm"}, + {path: "/api/v2/history/query?keywords=%E5%B7%9DAHTWO1,%E7%B2%A4AG18312&category=location&limit=10", want: "queryDurationMs"}, + {path: "/api/v2/history/query?keyword=%E5%B7%9DAHTWO1&category=raw&limit=10", want: "gb32960.vehicle.speed_kmh"}, + {path: "/api/v2/history/series?keyword=%E5%B7%9DAHTWO1&dateFrom=2026-07-03T18%3A00&dateTo=2026-07-03T21%3A00&targetPoints=120", want: "grainSeconds"}, + } { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, test.path, nil) + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s status=%d body=%s", test.path, rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), test.want) { + t.Fatalf("%s missing %q body=%s", test.path, test.want, rec.Body.String()) + } + } +} + +func TestHistorySeriesGrainAndBounds(t *testing.T) { + if got := historySeriesGrain(24*time.Hour, 240); got != 900 { + t.Fatalf("24h / 240 should select the next safe nice bucket, got %d", got) + } + handler := NewHandler(NewService(NewMockStore())) + for _, test := range []struct{ path, code string }{ + {path: "/api/v2/history/series?category=raw&keyword=%E5%B7%9DAHTWO1", code: "HISTORY_SERIES_CATEGORY_UNSUPPORTED"}, + {path: "/api/v2/history/series?keyword=%E5%B7%9DAHTWO1&dateFrom=2026-01-01T00%3A00&dateTo=2026-03-01T00%3A00", code: "HISTORY_TIME_RANGE_TOO_LARGE"}, + } { + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, test.path, nil)) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), test.code) { + t.Fatalf("%s expected %s, status=%d body=%s", test.path, test.code, rec.Code, rec.Body.String()) + } + } +} + +func TestHandlerV2HistoryRequiresBoundedVehicleScope(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v2/history/query?category=location", nil) + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "VEHICLE_KEY_REQUIRED") { + t.Fatalf("unbounded history query should be rejected: status=%d body=%s", rec.Code, rec.Body.String()) + } +} + func TestHandlerVehicles(t *testing.T) { handler := NewHandler(NewService(NewMockStore())) rec := httptest.NewRecorder() diff --git a/vehicle-data-platform/apps/api/internal/platform/metric_store.go b/vehicle-data-platform/apps/api/internal/platform/metric_store.go new file mode 100644 index 00000000..28be36f7 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/metric_store.go @@ -0,0 +1,54 @@ +package platform + +import ( + "context" + "encoding/json" + "fmt" +) + +func (s *ProductionStore) MetricDefinitions(ctx context.Context) ([]MetricDefinition, error) { + rows, err := s.db.QueryContext(ctx, `SELECT metric_key,label,description,unit,category,value_type,protocols_json,searchable,chartable,alertable +FROM vehicle_metric_definition WHERE enabled=1 ORDER BY sort_order,metric_key`) + if err != nil { + return nil, fmt.Errorf("metric catalog unavailable; apply deploy/migrations/005_metric_catalog.sql: %w", err) + } + defer rows.Close() + definitions := make([]MetricDefinition, 0, 16) + byKey := map[string]int{} + for rows.Next() { + var definition MetricDefinition + var protocols string + if err := rows.Scan(&definition.Key, &definition.Label, &definition.Description, &definition.Unit, &definition.Category, &definition.ValueType, &protocols, &definition.Searchable, &definition.Chartable, &definition.Alertable); err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(protocols), &definition.Protocols); err != nil { + return nil, fmt.Errorf("decode metric %s protocols: %w", definition.Key, err) + } + definition.SourceFields = map[string]string{} + definitions = append(definitions, definition) + byKey[definition.Key] = len(definitions) - 1 + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(definitions) == 0 { + return nil, fmt.Errorf("metric catalog has no enabled definitions") + } + mappings, err := s.db.QueryContext(ctx, `SELECT m.metric_key,m.protocol,m.source_field +FROM vehicle_metric_protocol_mapping m JOIN vehicle_metric_definition d ON d.metric_key=m.metric_key AND d.enabled=1 +ORDER BY m.metric_key,m.protocol`) + if err != nil { + return nil, err + } + defer mappings.Close() + for mappings.Next() { + var key, protocol, sourceField string + if err := mappings.Scan(&key, &protocol, &sourceField); err != nil { + return nil, err + } + if index, ok := byKey[key]; ok { + definitions[index].SourceFields[protocol] = sourceField + } + } + return definitions, mappings.Err() +} diff --git a/vehicle-data-platform/apps/api/internal/platform/mock_store.go b/vehicle-data-platform/apps/api/internal/platform/mock_store.go index 384d4d88..32e5734b 100644 --- a/vehicle-data-platform/apps/api/internal/platform/mock_store.go +++ b/vehicle-data-platform/apps/api/internal/platform/mock_store.go @@ -6,11 +6,23 @@ import ( "sort" "strconv" "strings" + "sync" + "time" ) type MockStore struct { - vehicles []VehicleRow - locations []RealtimeLocationRow + vehicles []VehicleRow + locations []RealtimeLocationRow + accessMu sync.RWMutex + accessThresholds AccessThresholdConfig + profileMu sync.RWMutex + profiles map[string]VehicleProfile + alertMu sync.RWMutex + alertRules []AlertRule + alertEvents []AlertEvent + alertNotifications []AlertNotification + nextAlertActionID int64 + nextNotificationID int64 } func NewMockStore() *MockStore { @@ -21,14 +33,157 @@ func NewMockStore() *MockStore { {VIN: "LMRKH9AC2R1004087", Plate: "豫A88888", Phone: "", OEM: "宇通", Protocol: "YUTONG_MQTT", Online: true, LastSeen: "2026-07-03 20:11:59", LocationText: "上海市临港", BindingScore: 92}, {VIN: "LB9A32A24P0LS1230", Plate: "粤AFF7936", Phone: "13307795426", OEM: "广安车联", Protocol: "JT808", Online: false, LastSeen: "2026-07-03 19:58:00", LocationText: "广东省佛山市", BindingScore: 88}, } - return &MockStore{ - vehicles: vehicles, + store := &MockStore{ + vehicles: vehicles, + accessThresholds: defaultAccessThresholds(time.Now()), + profiles: map[string]VehicleProfile{ + "LB9A32A24R0LS1426": {VIN: "LB9A32A24R0LS1426", ModelName: "新能源运营车", VehicleType: "乘用车", CompanyName: "岭牛示范车队", OperationStatus: "active", AccessProvider: "G7", FirstAccessAt: "2026-03-01T08:00:00+08:00", RuntimeSeconds: int64Pointer(1263600), SourceSystem: "manual", Version: 1, UpdatedBy: "demo-admin", UpdatedAt: "2026-07-03T20:12:10+08:00"}, + }, locations: []RealtimeLocationRow{ {VIN: vehicles[0].VIN, Plate: vehicles[0].Plate, Protocol: vehicles[0].Protocol, Longitude: 113.2644, Latitude: 23.1291, SpeedKmh: 42.5, SOCPercent: 76.2, TotalMileageKm: 48798.9, LastSeen: vehicles[0].LastSeen}, {VIN: vehicles[1].VIN, Plate: vehicles[1].Plate, Protocol: vehicles[1].Protocol, Longitude: 104.0668, Latitude: 30.5728, SpeedKmh: 18.3, SOCPercent: 64.8, TotalMileageKm: 119925, LastSeen: vehicles[1].LastSeen}, {VIN: vehicles[2].VIN, Plate: vehicles[2].Plate, Protocol: vehicles[2].Protocol, Longitude: 121.075044, Latitude: 30.590921, SpeedKmh: 27, SOCPercent: 78.4, TotalMileageKm: 119925, LastSeen: vehicles[2].LastSeen}, }, } + store.seedAlertCenter() + return store +} + +func int64Pointer(value int64) *int64 { return &value } + +func (m *MockStore) VehicleProfile(_ context.Context, vin string) (VehicleProfile, bool, error) { + m.profileMu.RLock() + defer m.profileMu.RUnlock() + profile, ok := m.profiles[vin] + return profile, ok, nil +} + +func (m *MockStore) SaveVehicleProfile(_ context.Context, vin string, input VehicleProfileInput) (VehicleProfile, error) { + m.profileMu.Lock() + defer m.profileMu.Unlock() + current, exists := m.profiles[vin] + if !exists && input.Version != 0 { + return VehicleProfile{}, clientError{Code: "VEHICLE_PROFILE_NOT_FOUND", Message: "待更新车辆档案不存在"} + } + if exists && input.Version != current.Version { + return VehicleProfile{}, clientError{Code: "VEHICLE_PROFILE_VERSION_CONFLICT", Message: "车辆档案已被其他用户更新，请刷新后重试"} + } + version := 1 + if exists { + version = current.Version + 1 + } + profile := VehicleProfile{VIN: vin, ModelName: input.ModelName, VehicleType: input.VehicleType, CompanyName: input.CompanyName, OperationStatus: input.OperationStatus, AccessProvider: input.AccessProvider, FirstAccessAt: input.FirstAccessAt, RuntimeSeconds: input.RuntimeSeconds, SourceSystem: "manual", Version: version, UpdatedBy: input.Actor, UpdatedAt: time.Now().Format(time.RFC3339)} + m.profiles[vin] = profile + return profile, nil +} + +func (m *MockStore) SyncVehicleProfiles(_ context.Context, request VehicleProfileSyncRequest) (VehicleProfileSyncResult, error) { + m.profileMu.Lock() + defer m.profileMu.Unlock() + result := VehicleProfileSyncResult{ + SourceSystem: request.SourceSystem, SourceVersion: request.SourceVersion, + DryRun: request.DryRun, Received: len(request.Items), + Items: make([]VehicleProfileSyncItemResult, 0, len(request.Items)), + } + known := make(map[string]bool, len(m.vehicles)) + for _, vehicle := range m.vehicles { + known[vehicle.VIN] = true + } + now := time.Now().Format(time.RFC3339) + for _, item := range request.Items { + itemResult := VehicleProfileSyncItemResult{VIN: item.VIN} + if !known[item.VIN] { + itemResult.Status = profileSyncMissingVehicle + countVehicleProfileSyncResult(&result, itemResult.Status) + result.Items = append(result.Items, itemResult) + continue + } + current, exists := m.profiles[item.VIN] + if exists { + itemResult.PreviousSource = current.SourceSystem + itemResult.PreviousVersion = current.SourceVersion + } + itemResult.Status = vehicleProfileSyncDecision(current, exists, request, item) + itemResult.ProfileVersion = current.Version + if itemResult.Status == profileSyncCreated { + itemResult.ProfileVersion = 1 + } else if itemResult.Status == profileSyncUpdated { + itemResult.ProfileVersion = current.Version + 1 + } + countVehicleProfileSyncResult(&result, itemResult.Status) + result.Items = append(result.Items, itemResult) + if request.DryRun || (itemResult.Status != profileSyncCreated && itemResult.Status != profileSyncUpdated) { + continue + } + m.profiles[item.VIN] = VehicleProfile{ + VIN: item.VIN, ModelName: item.ModelName, VehicleType: item.VehicleType, + CompanyName: item.CompanyName, OperationStatus: item.OperationStatus, + AccessProvider: item.AccessProvider, FirstAccessAt: item.FirstAccessAt, + RuntimeSeconds: item.RuntimeSeconds, SourceSystem: request.SourceSystem, + SourceVersion: request.SourceVersion, SyncedAt: now, + Version: itemResult.ProfileVersion, UpdatedBy: request.Actor, UpdatedAt: now, + } + } + return result, nil +} + +func (m *MockStore) AccessEvidence(context.Context) ([]AccessEvidenceRow, error) { + now := time.Now() + seconds := func(value int) string { return now.Add(-time.Duration(value) * time.Second).Format(time.RFC3339) } + interval10, interval30, interval60 := 10, 30, 60 + return []AccessEvidenceRow{ + {VIN: "LB9A32A24R0LS1426", Plate: "粤AG18312", OEM: "比亚迪", Model: "G7s", Company: "岭牛示范车队", Protocol: "JT808", Provider: "G7", Source: "vehicle_realtime_snapshot", FirstSeenAt: now.AddDate(0, -4, 0).Format(time.RFC3339), LatestEventAt: seconds(13), LatestReceivedAt: seconds(12), LatestUpdatedAt: seconds(12), ReportIntervalSec: &interval10, ReportSampleCount: 120, LatestMessageType: "位置信息汇报", LatestEventID: "evt-jt808-001", FirstSeenEvidence: "网关实时写入首次观测", FirstSeenSource: "live_writer", ReportIntervalProof: "网关连续接收时间差（持久样本 120 条）"}, + {VIN: "LNXNEGRR7SR318212", Plate: "川AHTWO1", OEM: "现代", Model: "氢燃料车型", Company: "示范物流", Protocol: "GB32960", Provider: "车厂平台", Source: "vehicle_realtime_snapshot", FirstSeenAt: now.AddDate(0, -2, 0).Format(time.RFC3339), LatestEventAt: seconds(57), LatestReceivedAt: seconds(12), LatestUpdatedAt: seconds(12), ReportIntervalSec: &interval30, ReportSampleCount: 48, LatestMessageType: "实时信息上报", LatestEventID: "evt-gb32960-002", LatestError: "上游事件时间延迟", FirstSeenEvidence: "网关实时写入首次观测", FirstSeenSource: "live_writer", ReportIntervalProof: "网关连续接收时间差（持久样本 48 条）"}, + {VIN: "LMRKH9AC2R1004087", Plate: "豫A88888", OEM: "宇通", Model: "纯电客车", Protocol: "YUTONG_MQTT", Provider: "宇通云", Source: "vehicle_realtime_snapshot", FirstSeenAt: now.AddDate(0, -8, 0).Format(time.RFC3339), LatestEventAt: seconds(2400), LatestReceivedAt: seconds(2398), LatestUpdatedAt: seconds(2398), ReportIntervalSec: &interval60, LatestMessageType: "实时遥测", LatestEventID: "evt-mqtt-003", FirstSeenEvidence: "测试接入台账", ReportIntervalProof: "最近两次接收时间差"}, + {VIN: "LB9A32A24P0LS1230", Plate: "粤AFF7936", OEM: "广安车联", Model: "运营车辆", Protocol: "JT808", Provider: "广安车联", Source: "vehicle_realtime_snapshot", FirstSeenAt: now.AddDate(-1, 0, 0).Format(time.RFC3339), LatestEventAt: seconds(9100), LatestReceivedAt: seconds(9098), LatestUpdatedAt: seconds(9098), ReportIntervalSec: &interval30, LatestMessageType: "位置信息汇报", LatestEventID: "evt-jt808-004", LatestError: "来源链路长时间无更新", FirstSeenEvidence: "测试接入台账", ReportIntervalProof: "最近两次接收时间差"}, + {VIN: "LUNKNOWNACCESS01", Plate: "粤A待核1", OEM: "待维护", Protocol: "UNKNOWN", Provider: "未知平台", Source: "vehicle_realtime_snapshot", LatestEventAt: "invalid-time", LatestMessageType: "协议未识别", LatestError: "协议或时间字段无法识别"}, + {VIN: "LNEVERREPORT0001", Plate: "粤A未报1", OEM: "档案车辆", Model: "待接入", Source: "vehicle_identity_binding", LatestError: "车辆已建档但从未形成实时快照"}, + }, nil +} + +func (m *MockStore) AccessUnresolvedIdentities(_ context.Context, query AccessUnresolvedIdentityQuery) (Page[AccessUnresolvedIdentity], error) { + items := []AccessUnresolvedIdentity{{ + ID: "mock-unresolved-jt808", Protocol: "JT808", IdentifierMasked: "138****0001", Plate: "待维护", + Manufacturer: "示范终端", SourceEndpoint: "gateway-a", LatestSeenAt: time.Now().Add(-30 * time.Second).Format(time.RFC3339), + FreshnessSec: 30, IssueCode: "missing_vin_jt808", RecommendedAction: "核对终端手机号、车牌和厂家后维护 phone→VIN 权威绑定；禁止猜测 VIN", + }} + if query.Offset >= len(items) { + return Page[AccessUnresolvedIdentity]{Items: []AccessUnresolvedIdentity{}, Total: len(items), Limit: query.Limit, Offset: query.Offset}, nil + } + return Page[AccessUnresolvedIdentity]{Items: items, Total: len(items), Limit: query.Limit, Offset: query.Offset}, nil +} + +func (m *MockStore) AccessThresholds(context.Context) (AccessThresholdConfig, error) { + m.accessMu.RLock() + defer m.accessMu.RUnlock() + config := m.accessThresholds + config.Protocols = append([]AccessProtocolThreshold(nil), config.Protocols...) + config.Audit = append([]AccessThresholdAudit(nil), config.Audit...) + return config, nil +} + +func (m *MockStore) SaveAccessThresholds(_ context.Context, update AccessThresholdUpdate) (AccessThresholdConfig, error) { + m.accessMu.Lock() + defer m.accessMu.Unlock() + if update.Version != m.accessThresholds.Version { + return AccessThresholdConfig{}, clientError{Code: "ACCESS_THRESHOLD_VERSION_CONFLICT", Message: "阈值配置已被其他用户更新，请刷新后重试"} + } + now := time.Now().Format(time.RFC3339) + next := AccessThresholdConfig{ + Version: update.Version + 1, + DefaultThresholdSec: update.DefaultThresholdSec, + DelayThresholdSec: update.DelayThresholdSec, + LongOfflineSec: update.LongOfflineSec, + Protocols: append([]AccessProtocolThreshold(nil), update.Protocols...), + UpdatedBy: update.Actor, + UpdatedAt: now, + Audit: append([]AccessThresholdAudit{{Version: update.Version + 1, Actor: update.Actor, ChangedAt: now, Summary: "更新在线、延迟和长离线阈值"}}, m.accessThresholds.Audit...), + } + if len(next.Audit) > 10 { + next.Audit = next.Audit[:10] + } + m.accessThresholds = next + return next, nil } func (m *MockStore) DashboardSummary(context.Context) (DashboardSummary, error) { @@ -396,6 +551,9 @@ func (m *MockStore) VehicleRealtime(_ context.Context, query url.Values) (Page[V continue } } + if status := strings.TrimSpace(query.Get("status")); status != "" && !matchesMonitorStatus(*row, status) { + continue + } if !keepServiceStatus(row.ServiceStatus, query.Get("serviceStatus")) { continue } @@ -521,14 +679,105 @@ func (m *MockStore) HistoryLocations(ctx context.Context, query url.Values) (Pag func (m *MockStore) HistoryLocationsFromTDengine(ctx context.Context, query url.Values) (Page[HistoryLocationRow], error) { realtime, _ := m.RealtimeLocations(ctx, query) - rows := make([]HistoryLocationRow, 0, len(realtime.Items)) + rows := make([]HistoryLocationRow, 0, len(realtime.Items)*18) + speeds := []float64{0, 8, 18, 32, 61, 44, 12, 0, 16, 43, 72, 38, 26, 54, 80, 41, 14, 0} for _, row := range realtime.Items { - rows = append(rows, HistoryLocationRow{ - VIN: row.VIN, Plate: row.Plate, Protocol: row.Protocol, Longitude: row.Longitude, Latitude: row.Latitude, - SpeedKmh: row.SpeedKmh, TotalMileageKm: row.TotalMileageKm, DeviceTime: row.LastSeen, ServerTime: row.LastSeen, - }) + endTime, ok := parseVehicleServiceTime(row.LastSeen) + if !ok { + endTime = time.Date(2026, 7, 3, 20, 12, 0, 0, time.UTC) + } + for index, speed := range speeds { + remaining := len(speeds) - 1 - index + observedAt := endTime.Add(-time.Duration(remaining*5) * time.Minute).Format("2006-01-02 15:04:05") + rows = append(rows, HistoryLocationRow{ + VIN: row.VIN, Plate: row.Plate, Protocol: row.Protocol, + Longitude: row.Longitude - float64(remaining)*0.0062, + Latitude: row.Latitude - float64(remaining)*0.0021 + float64(index%3)*0.0008, + SpeedKmh: speed, TotalMileageKm: row.TotalMileageKm - float64(remaining)*1.35, + DeviceTime: observedAt, ServerTime: observedAt, + }) + } + } + return page(rows, query), nil +} + +func (m *MockStore) HistoryLocationSeries(ctx context.Context, query HistoryLocationSeriesQuery) ([]HistoryLocationSeriesBucket, error) { + page, err := m.HistoryLocationsFromTDengine(ctx, url.Values{"vin": {query.VIN}, "protocol": {query.Protocol}, "limit": {"1000"}}) + if err != nil { + return nil, err + } + buckets := make([]HistoryLocationSeriesBucket, 0, len(page.Items)) + for _, row := range page.Items { + speed, mileage := row.SpeedKmh, row.TotalMileageKm + buckets = append(buckets, HistoryLocationSeriesBucket{Time: row.DeviceTime, Protocol: row.Protocol, Count: 1, SpeedAverage: &speed, SpeedMinimum: &speed, SpeedMaximum: &speed, SpeedLast: &speed, MileageAverage: &mileage, MileageMinimum: &mileage, MileageMaximum: &mileage, MileageLast: &mileage}) + } + sort.SliceStable(buckets, func(i, j int) bool { return buckets[i].Time < buckets[j].Time }) + return buckets, nil +} + +func (m *MockStore) HistoryExportCount(ctx context.Context, query HistoryExportStoreQuery) (int64, error) { + rows, err := m.mockHistoryExportRows(ctx, query) + return int64(len(rows)), err +} + +func (m *MockStore) HistoryExportBatch(ctx context.Context, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) ([]HistoryDataRow, HistoryExportCursor, error) { + rows, err := m.mockHistoryExportRows(ctx, query) + if err != nil { + return nil, cursor, err + } + if limit <= 0 { + limit = 5000 + } + start := cursor.Offset + if start > len(rows) { + start = len(rows) + } + end := start + limit + if end > len(rows) { + end = len(rows) + } + result := append([]HistoryDataRow(nil), rows[start:end]...) + cursor.Offset = end + return result, cursor, nil +} + +func (m *MockStore) mockHistoryExportRows(ctx context.Context, query HistoryExportStoreQuery) ([]HistoryDataRow, error) { + switch query.Category { + case "raw": + page, err := m.RawFrames(ctx, RawFrameQuery{VIN: query.VIN, Protocol: query.Protocol, IncludeFields: true, Fields: query.Metrics, Limit: 1000}) + if err != nil { + return nil, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for _, item := range page.Items { + values := map[string]any{"frameType": item.FrameType, "rawSizeBytes": item.RawSizeBytes} + for key, value := range item.ParsedFields { + values[key] = value + } + rows = append(rows, HistoryDataRow{ID: item.ID, VIN: item.VIN, Plate: item.Plate, Protocol: item.Protocol, DeviceTime: item.DeviceTime, ServerTime: item.ServerTime, Quality: "normal", EvidenceID: item.ID, Values: values}) + } + return rows, nil + case "mileage": + page, err := m.DailyMileage(ctx, url.Values{"vin": {query.VIN}, "protocol": {query.Protocol}, "limit": {"1000"}}) + if err != nil { + return nil, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for index, item := range page.Items { + rows = append(rows, HistoryDataRow{ID: "mileage-" + item.VIN + "-" + strconv.Itoa(index), VIN: item.VIN, Plate: item.Plate, Protocol: item.Source, DeviceTime: item.Date, ServerTime: item.Date, Quality: "normal", Values: map[string]any{"startMileageKm": item.StartMileageKm, "endMileageKm": item.EndMileageKm, "dailyMileageKm": item.DailyMileageKm}}) + } + return rows, nil + default: + page, err := m.HistoryLocationsFromTDengine(ctx, url.Values{"vin": {query.VIN}, "protocol": {query.Protocol}, "limit": {"1000"}}) + if err != nil { + return nil, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for index, item := range page.Items { + rows = append(rows, HistoryDataRow{ID: "location-" + item.VIN + "-" + strconv.Itoa(index), VIN: item.VIN, Plate: item.Plate, Protocol: item.Protocol, DeviceTime: item.DeviceTime, ServerTime: item.ServerTime, Quality: "normal", Values: map[string]any{"speedKmh": item.SpeedKmh, "totalMileageKm": item.TotalMileageKm, "longitude": item.Longitude, "latitude": item.Latitude}}) + } + return rows, nil } - return Page[HistoryLocationRow]{Items: rows, Total: len(rows), Limit: realtime.Limit, Offset: realtime.Offset}, nil } func (m *MockStore) RawFrames(_ context.Context, query RawFrameQuery) (Page[RawFrameRow], error) { @@ -551,8 +800,8 @@ func (m *MockStore) RawFrames(_ context.Context, query RawFrameQuery) (Page[RawF fields = nil } rows := []RawFrameRow{ - {ID: "raw-20260703-001", VIN: "LB9A32A24R0LS1426", Plate: "粤AG18312", Protocol: "GB32960", FrameType: "realtime", DeviceTime: "2026-07-03 20:12:06", ServerTime: "2026-07-03 20:12:06", RawSizeBytes: 430, ParsedFields: fields}, - {ID: "raw-20260703-002", VIN: "LNXNEGRR7SR318212", Plate: "川AHTWO1", Protocol: "GB32960", FrameType: "realtime", DeviceTime: "2026-07-03 20:12:06", ServerTime: "2026-07-03 20:12:06", RawSizeBytes: 430, ParsedFields: fields}, + {ID: "raw-20260703-001", VIN: "LB9A32A24R0LS1426", Plate: "粤AG18312", Protocol: "GB32960", FrameType: "realtime", DeviceTime: "2026-07-03 20:12:06", ServerTime: "2026-07-03 20:12:06", RawSizeBytes: 430, ParseStatus: "ok", SourceEndpoint: "gb32960-gateway", ParsedFields: fields}, + {ID: "raw-20260703-002", VIN: "LNXNEGRR7SR318212", Plate: "川AHTWO1", Protocol: "GB32960", FrameType: "realtime", DeviceTime: "2026-07-03 20:12:06", ServerTime: "2026-07-03 20:12:06", RawSizeBytes: 430, ParseStatus: "ok", SourceEndpoint: "gb32960-gateway", ParsedFields: fields}, } if vin := strings.TrimSpace(query.VIN); vin != "" { rows = keep(rows, func(row RawFrameRow) bool { return row.VIN == vin }) diff --git a/vehicle-data-platform/apps/api/internal/platform/model.go b/vehicle-data-platform/apps/api/internal/platform/model.go index c7885ca6..1df5b440 100644 --- a/vehicle-data-platform/apps/api/internal/platform/model.go +++ b/vehicle-data-platform/apps/api/internal/platform/model.go @@ -7,6 +7,661 @@ type Page[T any] struct { Offset int `json:"offset"` } +type MonitorSummary struct { + TotalVehicles int `json:"totalVehicles"` + OnlineVehicles int `json:"onlineVehicles"` + OfflineVehicles int `json:"offlineVehicles"` + DrivingVehicles int `json:"drivingVehicles"` + IdleVehicles int `json:"idleVehicles"` + AlertVehicles int `json:"alertVehicles"` + UnknownVehicles int `json:"unknownVehicles"` + ActiveToday int `json:"activeToday"` + FrameToday int `json:"frameToday"` + AlertDataAvailable bool `json:"alertDataAvailable"` + Truncated bool `json:"truncated"` + AsOf string `json:"asOf"` +} + +type MonitorMapResponse struct { + Mode string `json:"mode"` + Zoom int `json:"zoom"` + Total int `json:"total"` + Truncated bool `json:"truncated"` + Points []MonitorMapPoint `json:"points"` + Clusters []MonitorMapCluster `json:"clusters"` + AsOf string `json:"asOf"` +} + +type MonitorMapPoint struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + Protocols []string `json:"protocols"` + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` + SpeedKmh float64 `json:"speedKmh"` + SOCPercent float64 `json:"socPercent"` + TotalMileageKm float64 `json:"totalMileageKm"` + LastSeen string `json:"lastSeen"` + Status string `json:"status"` +} + +type MonitorMapCluster struct { + ID string `json:"id"` + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` + Count int `json:"count"` + Online int `json:"online"` + Offline int `json:"offline"` + Driving int `json:"driving"` + Idle int `json:"idle"` + Unknown int `json:"unknown"` +} + +type TrackPlaybackResponse struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + Points []HistoryLocationRow `json:"points"` + Events []TrackPlaybackEvent `json:"events"` + Sources []TrackPlaybackSource `json:"sources"` + Segments []TrackSegment `json:"segments"` + Stops []TrackStop `json:"stops"` + Summary TrackPlaybackSummary `json:"summary"` + Coverage TrackCoverage `json:"coverage"` + Quality TrackQuality `json:"quality"` + Total int `json:"total"` + Truncated bool `json:"truncated"` + Sampled bool `json:"sampled"` + AsOf string `json:"asOf"` +} + +type TrackPlaybackSummary struct { + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + DistanceKm float64 `json:"distanceKm"` + DurationSeconds int64 `json:"durationSeconds"` + AverageSpeedKmh float64 `json:"averageSpeedKmh"` + MaximumSpeedKmh float64 `json:"maximumSpeedKmh"` + PointCount int `json:"pointCount"` + MovingSeconds int64 `json:"movingSeconds"` + StoppedSeconds int64 `json:"stoppedSeconds"` + StopCount int `json:"stopCount"` + SegmentCount int `json:"segmentCount"` +} + +type TrackPlaybackEvent struct { + Index int `json:"index"` + SampledIndex int `json:"sampledIndex"` + Type string `json:"type"` + Title string `json:"title"` + Time string `json:"time"` + SpeedKmh float64 `json:"speedKmh"` + SOCPercent float64 `json:"socPercent"` + SOCAvailable bool `json:"socAvailable"` + DirectionDeg *int64 `json:"directionDeg,omitempty"` + AlarmFlag *int64 `json:"alarmFlag,omitempty"` + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` +} + +type TrackPlaybackSource struct { + Protocol string `json:"protocol"` + PointCount int `json:"pointCount"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` +} + +// TrackCoverage makes a bounded replay query explicit. Complete=false means +// summaries describe only the fetched slice, never the whole requested range. +type TrackCoverage struct { + RequestedStart string `json:"requestedStart"` + RequestedEnd string `json:"requestedEnd"` + ActualStart string `json:"actualStart"` + ActualEnd string `json:"actualEnd"` + TotalPoints int `json:"totalPoints"` + FetchedPoints int `json:"fetchedPoints"` + ProcessedPoints int `json:"processedPoints"` + ReturnedPoints int `json:"returnedPoints"` + Complete bool `json:"complete"` + LimitReasons []string `json:"limitReasons"` + Evidence string `json:"evidence"` +} + +// TrackSegment is inferred from GPS movement and gaps. It deliberately does +// not claim ignition state because vehicle_locations does not contain it. +type TrackSegment struct { + Index int `json:"index"` + Type string `json:"type"` + Title string `json:"title"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + DurationSeconds int64 `json:"durationSeconds"` + DistanceKm float64 `json:"distanceKm"` + PointCount int `json:"pointCount"` + StartIndex int `json:"startIndex"` + EndIndex int `json:"endIndex"` + SampledStartIndex int `json:"sampledStartIndex"` + SampledEndIndex int `json:"sampledEndIndex"` +} + +type TrackStop struct { + Index int `json:"index"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + DurationSeconds int64 `json:"durationSeconds"` + PointCount int `json:"pointCount"` + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` + SampledIndex int `json:"sampledIndex"` + Evidence string `json:"evidence"` + startIndex int + endIndex int +} + +type TrackQuality struct { + Status string `json:"status"` + SelectedProtocol string `json:"selectedProtocol"` + RawPoints int `json:"rawPoints"` + ValidPoints int `json:"validPoints"` + AlternateSourcePoints int `json:"alternateSourcePoints"` + InvalidCoordinatePoints int `json:"invalidCoordinatePoints"` + DuplicatePoints int `json:"duplicatePoints"` + DriftPoints int `json:"driftPoints"` + SourceSwitches int `json:"sourceSwitches"` + LargeGapCount int `json:"largeGapCount"` + MaximumGapSeconds int64 `json:"maximumGapSeconds"` + Evidence string `json:"evidence"` +} + +type HistoryMetricCatalog struct { + Categories []HistoryDataCategory `json:"categories"` + Metrics []HistoryMetricDefinition `json:"metrics"` +} + +type MetricCatalog struct { + Metrics []MetricDefinition `json:"metrics"` + AsOf string `json:"asOf"` +} + +type MetricDefinition struct { + Key string `json:"key"` + Label string `json:"label"` + Description string `json:"description"` + Unit string `json:"unit"` + Category string `json:"category"` + ValueType string `json:"valueType"` + Protocols []string `json:"protocols"` + SourceFields map[string]string `json:"sourceFields"` + Searchable bool `json:"searchable"` + Chartable bool `json:"chartable"` + Alertable bool `json:"alertable"` +} + +type HistoryDataCategory struct { + Key string `json:"key"` + Label string `json:"label"` +} + +type HistoryMetricDefinition struct { + Key string `json:"key"` + Label string `json:"label"` + Unit string `json:"unit"` + Category string `json:"category"` + ValueType string `json:"valueType"` + DefaultVisible bool `json:"defaultVisible"` +} + +type HistoryDataResponse struct { + Category string `json:"category"` + Columns []HistoryMetricDefinition `json:"columns"` + Rows []HistoryDataRow `json:"rows"` + Summary HistoryDataSummary `json:"summary"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + AsOf string `json:"asOf"` +} + +type HistoryDataSummary struct { + ResultRows int `json:"resultRows"` + VehicleCount int `json:"vehicleCount"` + Sources []string `json:"sources"` + QueryDuration int64 `json:"queryDurationMs"` +} + +type HistoryDataRow struct { + ID string `json:"id"` + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + DeviceTime string `json:"deviceTime"` + ServerTime string `json:"serverTime"` + Quality string `json:"quality"` + EvidenceID string `json:"evidenceId,omitempty"` + Values map[string]any `json:"values"` +} + +type HistorySeriesResponse struct { + Metrics []HistoryMetricDefinition `json:"metrics"` + Series []HistorySeries `json:"series"` + Summary HistorySeriesSummary `json:"summary"` + DateFrom string `json:"dateFrom"` + DateTo string `json:"dateTo"` + AsOf string `json:"asOf"` +} + +type HistorySeries struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + Metric string `json:"metric"` + Label string `json:"label"` + Unit string `json:"unit"` + Aggregation string `json:"aggregation"` + Points []HistorySeriesPoint `json:"points"` +} + +type HistorySeriesPoint struct { + Time string `json:"time"` + Value *float64 `json:"value"` + Min *float64 `json:"min"` + Max *float64 `json:"max"` + Count int64 `json:"count"` +} + +type HistorySeriesSummary struct { + RawPointCount int64 `json:"rawPointCount"` + BucketCount int `json:"bucketCount"` + ReturnedPointCount int `json:"returnedPointCount"` + SeriesCount int `json:"seriesCount"` + GrainSeconds int `json:"grainSeconds"` + TargetPoints int `json:"targetPoints"` + ExpectedBucketCount int `json:"expectedBucketCount"` + MissingBucketCount int `json:"missingBucketCount"` + QueryDuration int64 `json:"queryDurationMs"` + Complete bool `json:"complete"` + Evidence string `json:"evidence"` +} + +type HistoryLocationSeriesQuery struct { + VIN string + Protocol string + DateFrom string + DateTo string + GrainSeconds int +} + +type HistoryLocationSeriesBucket struct { + Time string + Protocol string + Count int64 + SpeedAverage *float64 + SpeedMinimum *float64 + SpeedMaximum *float64 + SpeedLast *float64 + MileageAverage *float64 + MileageMinimum *float64 + MileageMaximum *float64 + MileageLast *float64 +} + +type HistoryExportRequest struct { + Keywords []string `json:"keywords"` + Category string `json:"category"` + Protocol string `json:"protocol"` + DateFrom string `json:"dateFrom"` + DateTo string `json:"dateTo"` + Metrics []string `json:"metrics"` + Format string `json:"format"` +} + +type HistoryExportJob struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Progress int `json:"progress"` + Format string `json:"format"` + Category string `json:"category"` + Keywords []string `json:"keywords"` + RowCount int `json:"rowCount"` + TotalRows int64 `json:"totalRows"` + ProcessedRows int64 `json:"processedRows"` + FileSizeBytes int64 `json:"fileSizeBytes"` + Error string `json:"error,omitempty"` + DownloadURL string `json:"downloadUrl,omitempty"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + CompletedAt string `json:"completedAt,omitempty"` + Evidence string `json:"evidence"` + filePath string +} + +type HistoryExportStoreQuery struct { + Category string + VIN string + Protocol string + DateFrom string + DateTo string + Metrics []string +} + +type HistoryExportCursor struct { + Time string + Protocol string + ID string + Offset int +} + +type AccessQuery struct { + Keyword string `json:"keyword"` + Protocol string `json:"protocol"` + OEM string `json:"oem"` + Model string `json:"model"` + Provider string `json:"provider"` + FirstSeenFrom string `json:"firstSeenFrom"` + FirstSeenTo string `json:"firstSeenTo"` + LatestSeenFrom string `json:"latestSeenFrom"` + LatestSeenTo string `json:"latestSeenTo"` + OnlineState string `json:"onlineState"` + DelayState string `json:"delayState"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type AccessEvidenceRow struct { + VIN string + Plate string + OEM string + Model string + Company string + Protocol string + Provider string + Source string + FirstSeenAt string + LatestEventAt string + LatestReceivedAt string + LatestUpdatedAt string + ReportIntervalSec *int + LatestMessageType string + LatestEventID string + LatestError string + FirstSeenEvidence string + FirstSeenSource string + ReportIntervalProof string + ReportSampleCount int64 +} + +type AccessVehicleRow struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + OEM string `json:"oem"` + Model string `json:"model"` + Company string `json:"company"` + Protocol string `json:"protocol"` + Provider string `json:"provider"` + Source string `json:"source"` + FirstSeenAt string `json:"firstSeenAt"` + LatestEventAt string `json:"latestEventAt"` + LatestReceivedAt string `json:"latestReceivedAt"` + ReportIntervalSec *int `json:"reportIntervalSec"` + DataDelaySec *int `json:"dataDelaySec"` + FreshnessSec *int `json:"freshnessSec"` + OnlineState string `json:"onlineState"` + ThresholdSec int `json:"thresholdSec"` + LatestMessageType string `json:"latestMessageType"` + LatestEventID string `json:"latestEventId"` + LatestError string `json:"latestError"` + DelayAbnormal bool `json:"delayAbnormal"` + FirstSeenEvidence string `json:"firstSeenEvidence"` + FirstSeenSource string `json:"firstSeenSource"` + ReportIntervalProof string `json:"reportIntervalEvidence"` + ReportSampleCount int64 `json:"reportSampleCount"` +} + +type AccessUnresolvedIdentityQuery struct { + Keyword string `json:"keyword"` + Protocol string `json:"protocol"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type AccessUnresolvedIdentity struct { + ID string `json:"id"` + Protocol string `json:"protocol"` + IdentifierMasked string `json:"identifierMasked"` + Plate string `json:"plate"` + Manufacturer string `json:"manufacturer"` + SourceEndpoint string `json:"sourceEndpoint"` + FirstRegisteredAt string `json:"firstRegisteredAt"` + LatestRegisteredAt string `json:"latestRegisteredAt"` + LatestAuthenticatedAt string `json:"latestAuthenticatedAt"` + LatestSeenAt string `json:"latestSeenAt"` + FreshnessSec int `json:"freshnessSec"` + IssueCode string `json:"issueCode"` + RecommendedAction string `json:"recommendedAction"` +} + +type AccessDistribution struct { + Name string `json:"name"` + Total int `json:"total"` + Online int `json:"online"` + OnlineRate float64 `json:"onlineRate"` +} + +type AccessSummary struct { + TotalVehicles int `json:"totalVehicles"` + OnlineVehicles int `json:"onlineVehicles"` + OfflineVehicles int `json:"offlineVehicles"` + LongOfflineVehicles int `json:"longOfflineVehicles"` + NeverReported int `json:"neverReported"` + UnknownVehicles int `json:"unknownVehicles"` + DelayAbnormal int `json:"delayAbnormal"` + ReportedToday int `json:"reportedToday"` + OnlineRate float64 `json:"onlineRate"` + Protocols []AccessDistribution `json:"protocols"` + OEMs []AccessDistribution `json:"oems"` + AsOf string `json:"asOf"` + ThresholdVersion int `json:"thresholdVersion"` +} + +type AccessProtocolThreshold struct { + Protocol string `json:"protocol"` + ThresholdSec int `json:"thresholdSec"` +} + +type AccessThresholdAudit struct { + Version int `json:"version"` + Actor string `json:"actor"` + ChangedAt string `json:"changedAt"` + Summary string `json:"summary"` +} + +type AccessThresholdConfig struct { + Version int `json:"version"` + DefaultThresholdSec int `json:"defaultThresholdSec"` + DelayThresholdSec int `json:"delayThresholdSec"` + LongOfflineSec int `json:"longOfflineSec"` + Protocols []AccessProtocolThreshold `json:"protocols"` + UpdatedBy string `json:"updatedBy"` + UpdatedAt string `json:"updatedAt"` + Audit []AccessThresholdAudit `json:"audit"` +} + +type AccessThresholdUpdate struct { + Version int `json:"version"` + DefaultThresholdSec int `json:"defaultThresholdSec"` + DelayThresholdSec int `json:"delayThresholdSec"` + LongOfflineSec int `json:"longOfflineSec"` + Protocols []AccessProtocolThreshold `json:"protocols"` + Actor string `json:"actor"` +} + +type AlertQuery struct { + Keyword string `json:"keyword"` + Severity string `json:"severity"` + Status string `json:"status"` + RuleID string `json:"ruleId"` + Protocol string `json:"protocol"` + DateFrom string `json:"dateFrom"` + DateTo string `json:"dateTo"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type AlertSummary struct { + Active int `json:"active"` + Unprocessed int `json:"unprocessed"` + Processing int `json:"processing"` + Recovered int `json:"recovered"` + Closed int `json:"closed"` + Ignored int `json:"ignored"` + UnreadNotifications int `json:"unreadNotifications"` + AsOf string `json:"asOf"` +} + +type AlertRule struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Severity string `json:"severity"` + ValueType string `json:"valueType"` + Metric string `json:"metric"` + Operator string `json:"operator"` + Threshold float64 `json:"threshold"` + ThresholdHigh float64 `json:"thresholdHigh"` + BooleanThreshold *bool `json:"booleanThreshold,omitempty"` + DurationSec int `json:"durationSec"` + RecoveryOperator string `json:"recoveryOperator"` + RecoveryThreshold float64 `json:"recoveryThreshold"` + RepeatIntervalSec int `json:"repeatIntervalSec"` + ScopeProtocols []string `json:"scopeProtocols"` + ScopeVINs []string `json:"scopeVins"` + ScopeOEMs []string `json:"scopeOems"` + ScopeModels []string `json:"scopeModels"` + ScopeCompanies []string `json:"scopeCompanies"` + NotificationChannels []string `json:"notificationChannels"` + Enabled bool `json:"enabled"` + Version int `json:"version"` + CreatedBy string `json:"createdBy"` + UpdatedBy string `json:"updatedBy"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +type AlertRuleInput struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Severity string `json:"severity"` + ValueType string `json:"valueType"` + Metric string `json:"metric"` + Operator string `json:"operator"` + Threshold float64 `json:"threshold"` + ThresholdHigh float64 `json:"thresholdHigh"` + BooleanThreshold *bool `json:"booleanThreshold,omitempty"` + DurationSec int `json:"durationSec"` + RecoveryOperator string `json:"recoveryOperator"` + RecoveryThreshold float64 `json:"recoveryThreshold"` + RepeatIntervalSec int `json:"repeatIntervalSec"` + ScopeProtocols []string `json:"scopeProtocols"` + ScopeVINs []string `json:"scopeVins"` + ScopeOEMs []string `json:"scopeOems"` + ScopeModels []string `json:"scopeModels"` + ScopeCompanies []string `json:"scopeCompanies"` + NotificationChannels []string `json:"notificationChannels"` + Enabled bool `json:"enabled"` + Version int `json:"version"` + Actor string `json:"actor"` +} + +type AlertRuleEnabledUpdate struct { + Version int `json:"version"` + Enabled bool `json:"enabled"` + Actor string `json:"actor"` +} + +type AlertEvent struct { + ID string `json:"id"` + RuleID string `json:"ruleId"` + RuleName string `json:"ruleName"` + RuleVersion int `json:"ruleVersion"` + Severity string `json:"severity"` + Status string `json:"status"` + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + Metric string `json:"metric"` + Operator string `json:"operator"` + TriggerValue float64 `json:"triggerValue"` + Threshold float64 `json:"threshold"` + ThresholdHigh float64 `json:"thresholdHigh"` + Unit string `json:"unit"` + DurationSec int `json:"durationSec"` + Location string `json:"location"` + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + SourceEventID string `json:"sourceEventId"` + EventAt string `json:"eventAt"` + ReceivedAt string `json:"receivedAt"` + TriggeredAt string `json:"triggeredAt"` + RecoveredAt string `json:"recoveredAt"` + Handler string `json:"handler"` + Version int `json:"version"` + Actions []AlertAction `json:"actions,omitempty"` +} + +type AlertAction struct { + ID int64 `json:"id"` + Action string `json:"action"` + FromStatus string `json:"fromStatus"` + ToStatus string `json:"toStatus"` + Actor string `json:"actor"` + Note string `json:"note"` + CreatedAt string `json:"createdAt"` +} + +type AlertActionRequest struct { + Version int `json:"version"` + Action string `json:"action"` + Actor string `json:"actor"` + Note string `json:"note"` +} + +type AlertNotification struct { + ID int64 `json:"id"` + EventID string `json:"eventId"` + Title string `json:"title"` + Content string `json:"content"` + Severity string `json:"severity"` + Channel string `json:"channel"` + Read bool `json:"read"` + CreatedAt string `json:"createdAt"` + ReadAt string `json:"readAt"` +} + +type AlertNotificationQuery struct { + UnreadOnly bool `json:"unreadOnly"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +type AlertNotificationReadRequest struct { + IDs []int64 `json:"ids"` + Actor string `json:"actor"` +} + +type AlertEvaluationResult struct { + RulesEvaluated int `json:"rulesEvaluated"` + VehiclesScanned int `json:"vehiclesScanned"` + CandidatesAdvanced int `json:"candidatesAdvanced"` + DuplicateObservations int `json:"duplicateObservations"` + LateObservations int `json:"lateObservations"` + StaleEvidenceSkipped int `json:"staleEvidenceSkipped"` + Opened int `json:"opened"` + Recovered int `json:"recovered"` + AsOf string `json:"asOf"` +} + type DashboardSummary struct { OnlineVehicles int `json:"onlineVehicles"` ActiveToday int `json:"activeToday"` @@ -97,6 +752,7 @@ type VehicleDetail struct { LookupResolved bool `json:"lookupResolved"` Resolution *VehicleIdentityResolution `json:"resolution,omitempty"` Identity *VehicleRow `json:"identity,omitempty"` + Profile *VehicleProfile `json:"profile,omitempty"` RealtimeSummary *VehicleRealtimeRow `json:"realtimeSummary,omitempty"` ServiceStatus *VehicleServiceStatus `json:"serviceStatus,omitempty"` ServiceOverview *VehicleServiceOverview `json:"serviceOverview,omitempty"` @@ -110,6 +766,78 @@ type VehicleDetail struct { Quality Page[QualityIssueRow] `json:"quality"` } +type VehicleProfile struct { + VIN string `json:"vin"` + ModelName string `json:"modelName"` + VehicleType string `json:"vehicleType"` + CompanyName string `json:"companyName"` + OperationStatus string `json:"operationStatus"` + AccessProvider string `json:"accessProvider"` + FirstAccessAt string `json:"firstAccessAt"` + RuntimeSeconds *int64 `json:"runtimeSeconds"` + SourceSystem string `json:"sourceSystem"` + SourceVersion string `json:"sourceVersion"` + SyncedAt string `json:"syncedAt"` + Version int `json:"version"` + UpdatedBy string `json:"updatedBy"` + UpdatedAt string `json:"updatedAt"` + Completeness int `json:"completeness"` + MissingFields []string `json:"missingFields"` +} + +type VehicleProfileInput struct { + ModelName string `json:"modelName"` + VehicleType string `json:"vehicleType"` + CompanyName string `json:"companyName"` + OperationStatus string `json:"operationStatus"` + AccessProvider string `json:"accessProvider"` + FirstAccessAt string `json:"firstAccessAt"` + RuntimeSeconds *int64 `json:"runtimeSeconds"` + Version int `json:"version"` + Actor string `json:"actor"` +} + +type VehicleProfileSyncItem struct { + VIN string `json:"vin"` + ModelName string `json:"modelName"` + VehicleType string `json:"vehicleType"` + CompanyName string `json:"companyName"` + OperationStatus string `json:"operationStatus"` + AccessProvider string `json:"accessProvider"` + FirstAccessAt string `json:"firstAccessAt"` + RuntimeSeconds *int64 `json:"runtimeSeconds"` +} + +type VehicleProfileSyncRequest struct { + SourceSystem string `json:"sourceSystem"` + SourceVersion string `json:"sourceVersion"` + ConflictPolicy string `json:"conflictPolicy"` + DryRun bool `json:"dryRun"` + Items []VehicleProfileSyncItem `json:"items"` + Actor string `json:"actor"` +} + +type VehicleProfileSyncItemResult struct { + VIN string `json:"vin"` + Status string `json:"status"` + PreviousSource string `json:"previousSource,omitempty"` + PreviousVersion string `json:"previousVersion,omitempty"` + ProfileVersion int `json:"profileVersion,omitempty"` +} + +type VehicleProfileSyncResult struct { + SourceSystem string `json:"sourceSystem"` + SourceVersion string `json:"sourceVersion"` + DryRun bool `json:"dryRun"` + Received int `json:"received"` + Created int `json:"created"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Conflicted int `json:"conflicted"` + Missing int `json:"missing"` + Items []VehicleProfileSyncItemResult `json:"items"` +} + type VehicleSourceConsistency struct { SourceCount int `json:"sourceCount"` OnlineSourceCount int `json:"onlineSourceCount"` @@ -257,21 +985,64 @@ type HistoryLocationRow struct { Longitude float64 `json:"longitude"` Latitude float64 `json:"latitude"` SpeedKmh float64 `json:"speedKmh"` + SOCPercent float64 `json:"socPercent"` + SOCAvailable bool `json:"socAvailable"` + DirectionDeg *int64 `json:"directionDeg,omitempty"` + AlarmFlag *int64 `json:"alarmFlag,omitempty"` TotalMileageKm float64 `json:"totalMileageKm"` DeviceTime string `json:"deviceTime"` ServerTime string `json:"serverTime"` } type RawFrameRow struct { - ID string `json:"id"` - VIN string `json:"vin"` - Plate string `json:"plate"` - Protocol string `json:"protocol"` - FrameType string `json:"frameType"` - DeviceTime string `json:"deviceTime"` - ServerTime string `json:"serverTime"` - RawSizeBytes int `json:"rawSizeBytes"` - ParsedFields map[string]any `json:"parsedFields"` + ID string `json:"id"` + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + FrameType string `json:"frameType"` + DeviceTime string `json:"deviceTime"` + ServerTime string `json:"serverTime"` + RawSizeBytes int `json:"rawSizeBytes"` + ParseStatus string `json:"parseStatus,omitempty"` + ParseError string `json:"parseError,omitempty"` + SourceEndpoint string `json:"sourceEndpoint,omitempty"` + ParsedFields map[string]any `json:"parsedFields"` +} + +type LatestTelemetryCategory struct { + Key string `json:"key"` + Label string `json:"label"` + Count int `json:"count"` +} + +type LatestTelemetryValue struct { + Key string `json:"key"` + SourceField string `json:"sourceField"` + Label string `json:"label"` + Description string `json:"description,omitempty"` + Unit string `json:"unit"` + Category string `json:"category"` + ValueType string `json:"valueType"` + Value any `json:"value"` + Protocol string `json:"protocol"` + SourceEndpoint string `json:"sourceEndpoint,omitempty"` + FrameID string `json:"frameId"` + DeviceTime string `json:"deviceTime"` + ServerTime string `json:"serverTime"` + Quality string `json:"quality"` + QualityReason string `json:"qualityReason"` + FreshnessSeconds int64 `json:"freshnessSeconds"` + DataDelaySeconds *int64 `json:"dataDelaySeconds,omitempty"` +} + +type LatestTelemetryResponse struct { + VIN string `json:"vin"` + Categories []LatestTelemetryCategory `json:"categories"` + Values []LatestTelemetryValue `json:"values"` + AsOf string `json:"asOf"` + StaleAfterSeconds int64 `json:"staleAfterSeconds"` + ScannedFrames int `json:"scannedFrames"` + Evidence string `json:"evidence"` } type DailyMileageRow struct { @@ -387,15 +1158,31 @@ type QualityPriorityIssue struct { } type OpsHealth struct { - LinkHealth []LinkHealth `json:"linkHealth"` - KafkaLag *int `json:"kafkaLag"` - ActiveConnections *int `json:"activeConnections"` - CapacityMetrics CapacityMetrics `json:"capacityMetrics"` - CapacityFindings []string `json:"capacityFindings"` - RedisOnlineKeys *int `json:"redisOnlineKeys"` - TDengineWritable bool `json:"tdengineWritable"` - MySQLWritable bool `json:"mysqlWritable"` - Runtime RuntimeInfo `json:"runtime"` + LinkHealth []LinkHealth `json:"linkHealth"` + KafkaLag *int `json:"kafkaLag"` + ActiveConnections *int `json:"activeConnections"` + CapacityMetrics CapacityMetrics `json:"capacityMetrics"` + CapacityFindings []string `json:"capacityFindings"` + RedisOnlineKeys *int `json:"redisOnlineKeys"` + TDengineWritable bool `json:"tdengineWritable"` + MySQLWritable bool `json:"mysqlWritable"` + AlertStream AlertStreamHealth `json:"alertStream"` + Runtime RuntimeInfo `json:"runtime"` +} + +type AlertStreamHealth struct { + Mode string `json:"mode"` + ConsumerGroup string `json:"consumerGroup"` + Partitions int `json:"partitions"` + Lag int64 `json:"lag"` + Processed int64 `json:"processed"` + Valid int64 `json:"valid"` + Invalid int64 `json:"invalid"` + Late int64 `json:"late"` + ReplaySkipped int64 `json:"replaySkipped"` + UpdatedAt string `json:"updatedAt"` + LastInvalidCode string `json:"lastInvalidCode"` + LastInvalidAt string `json:"lastInvalidAt"` } type CapacityMetrics struct { @@ -412,6 +1199,8 @@ type CapacityMetrics struct { } type RuntimeInfo struct { + DataMode string `json:"dataMode"` + ExportDir string `json:"-"` RequestTimeoutMs int `json:"requestTimeoutMs"` AMapWebJSConfigured bool `json:"amapWebJsConfigured"` AMapAPIConfigured bool `json:"amapApiConfigured"` @@ -419,4 +1208,6 @@ type RuntimeInfo struct { AMapSecurityCodeExposed bool `json:"amapSecurityCodeExposed"` AMapSecurityServiceHost string `json:"amapSecurityServiceHost"` PlatformRelease string `json:"platformRelease"` + AlertStreamMode string `json:"alertStreamMode"` + AlertStreamConsumerGroup string `json:"alertStreamConsumerGroup"` } diff --git a/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go b/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go index caa0a78e..fe3481b3 100644 --- a/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go +++ b/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go @@ -337,6 +337,19 @@ func buildVehicleRealtimeSQL(query url.Values) SQLQuery { case "offline": having = append(having, "COUNT(DISTINCT CASE WHEN l.updated_at >= DATE_SUB(NOW(), INTERVAL 1 MINUTE) THEN l.protocol END) = 0") } + primarySpeed := "CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(l.speed_kmh AS CHAR) ORDER BY l.updated_at DESC, l.protocol ASC), ',', 1) AS DECIMAL(18,6))" + switch strings.TrimSpace(query.Get("status")) { + case "driving": + having = append(having, + "COUNT(DISTINCT CASE WHEN l.updated_at >= DATE_SUB(NOW(), INTERVAL 1 MINUTE) THEN l.protocol END) > 0", + primarySpeed+" > 3", + ) + case "idle": + having = append(having, + "COUNT(DISTINCT CASE WHEN l.updated_at >= DATE_SUB(NOW(), INTERVAL 1 MINUTE) THEN l.protocol END) > 0", + primarySpeed+" <= 3", + ) + } switch strings.TrimSpace(query.Get("serviceStatus")) { case "healthy": having = append(having, @@ -424,7 +437,8 @@ func buildDailyMileageSQL(query url.Values) SQLQuery { fromSQL := `FROM vehicle_daily_mileage m LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin WHERE ` + strings.Join(where, " AND ") return SQLQuery{ Text: `SELECT m.vin, COALESCE(b.plate, '') AS plate, DATE_FORMAT(m.stat_date, '%Y-%m-%d') AS stat_date, ` + - `m.first_total_mileage_km, m.latest_total_mileage_km, m.daily_mileage_km, m.protocol ` + + `COALESCE(m.latest_total_mileage_km - m.daily_mileage_km, 0) AS start_mileage_km, ` + + `COALESCE(m.latest_total_mileage_km, 0) AS end_mileage_km, m.daily_mileage_km, m.protocol ` + fromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?`, Args: args, CountText: `SELECT COUNT(*) ` + fromSQL, diff --git a/vehicle-data-platform/apps/api/internal/platform/principal.go b/vehicle-data-platform/apps/api/internal/platform/principal.go new file mode 100644 index 00000000..cc532462 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/principal.go @@ -0,0 +1,26 @@ +package platform + +import "context" + +type Principal struct { + Name string `json:"name"` + Role string `json:"role"` +} + +type principalContextKey struct{} + +func WithPrincipal(ctx context.Context, principal Principal) context.Context { + return context.WithValue(ctx, principalContextKey{}, principal) +} + +func PrincipalFromContext(ctx context.Context) (Principal, bool) { + principal, ok := ctx.Value(principalContextKey{}).(Principal) + return principal, ok +} + +func ActorFromContext(ctx context.Context) string { + if principal, ok := PrincipalFromContext(ctx); ok && principal.Name != "" { + return principal.Name + } + return "system" +} diff --git a/vehicle-data-platform/apps/api/internal/platform/production_store.go b/vehicle-data-platform/apps/api/internal/platform/production_store.go index 6600e706..e0fba8cb 100644 --- a/vehicle-data-platform/apps/api/internal/platform/production_store.go +++ b/vehicle-data-platform/apps/api/internal/platform/production_store.go @@ -7,15 +7,24 @@ import ( "net/url" "strconv" "strings" + "sync" "time" ) type ProductionStore struct { - db *sql.DB - tdengine *sql.DB - tdDatabase string - redisOnline redisOnlineKeyCounter - capacityCheck capacityChecker + db *sql.DB + tdengine *sql.DB + tdDatabase string + redisOnline redisOnlineKeyCounter + capacityCheck capacityChecker + alertStreamGroup string + alertStreamMode string + accessSchemaOnce sync.Once + accessSchemaErr error + alertSchemaOnce sync.Once + alertSchemaErr error + profileSchemaOnce sync.Once + profileSchemaErr error } type redisOnlineKeyCounter interface { @@ -43,6 +52,12 @@ func (s *ProductionStore) WithCapacityChecker(checker capacityChecker) *Producti return s } +func (s *ProductionStore) WithAlertStreamConfig(mode, consumerGroup string) *ProductionStore { + s.alertStreamMode = strings.TrimSpace(mode) + s.alertStreamGroup = strings.TrimSpace(consumerGroup) + return s +} + func OpenSQL(ctx context.Context, driver, dsn string) (*sql.DB, error) { db, err := sql.Open(driver, dsn) if err != nil { @@ -575,7 +590,7 @@ func (s *ProductionStore) HistoryLocations(ctx context.Context, query url.Values for _, row := range realtime.Items { items = append(items, HistoryLocationRow{ VIN: row.VIN, Plate: row.Plate, Protocol: row.Protocol, Longitude: row.Longitude, Latitude: row.Latitude, - SpeedKmh: row.SpeedKmh, TotalMileageKm: row.TotalMileageKm, DeviceTime: row.LastSeen, ServerTime: row.LastSeen, + SpeedKmh: row.SpeedKmh, SOCPercent: row.SOCPercent, SOCAvailable: row.SOCPercent > 0, TotalMileageKm: row.TotalMileageKm, DeviceTime: row.LastSeen, ServerTime: row.LastSeen, }) } return Page[HistoryLocationRow]{Items: items, Total: realtime.Total, Limit: realtime.Limit, Offset: realtime.Offset}, nil @@ -619,17 +634,26 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer items := make([]HistoryLocationRow, 0) for rows.Next() { var row HistoryLocationRow - var ts, receivedAt string - var longitude, latitude, speed, mileage sql.NullFloat64 - if err := rows.Scan(&ts, &row.VIN, &row.Protocol, &longitude, &latitude, &speed, &mileage, &receivedAt); err != nil { + var ts int64 + var receivedAt sql.NullInt64 + var longitude, latitude, speed, soc, mileage sql.NullFloat64 + var direction, alarm sql.NullInt64 + if err := rows.Scan(&ts, &row.VIN, &row.Protocol, &longitude, &latitude, &speed, &soc, &direction, &alarm, &mileage, &receivedAt); err != nil { return Page[HistoryLocationRow]{}, err } row.Longitude = nullFloat64(longitude) row.Latitude = nullFloat64(latitude) row.SpeedKmh = nullFloat64(speed) + row.SOCPercent = nullFloat64(soc) + row.SOCAvailable = soc.Valid + row.DirectionDeg = nullInt64Pointer(direction) + row.AlarmFlag = nullInt64Pointer(alarm) row.TotalMileageKm = nullFloat64(mileage) - row.DeviceTime = ts - row.ServerTime = firstNonEmpty(receivedAt, ts) + row.DeviceTime = time.UnixMilli(ts).UTC().Format(time.RFC3339Nano) + row.ServerTime = row.DeviceTime + if receivedAt.Valid { + row.ServerTime = time.UnixMilli(receivedAt.Int64).UTC().Format(time.RFC3339Nano) + } items = append(items, row) } if err := rows.Err(); err != nil { @@ -644,6 +668,172 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer return Page[HistoryLocationRow]{Items: items, Total: total, Limit: limit, Offset: offset}, nil } +func (s *ProductionStore) HistoryLocationSeries(ctx context.Context, query HistoryLocationSeriesQuery) ([]HistoryLocationSeriesBucket, error) { + if s.tdengine == nil { + return []HistoryLocationSeriesBucket{}, nil + } + built := buildHistoryLocationSeriesSQL(s.tdDatabase, query) + rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...) + if err != nil { + if isTDengineTableNotExist(err) { + return []HistoryLocationSeriesBucket{}, nil + } + return nil, err + } + defer rows.Close() + buckets := make([]HistoryLocationSeriesBucket, 0) + for rows.Next() { + var bucket HistoryLocationSeriesBucket + var speedAverage, speedMinimum, speedMaximum, speedLast sql.NullFloat64 + var mileageAverage, mileageMinimum, mileageMaximum, mileageLast sql.NullFloat64 + if err := rows.Scan(&bucket.Time, &bucket.Protocol, &bucket.Count, &speedAverage, &speedMinimum, &speedMaximum, &speedLast, &mileageAverage, &mileageMinimum, &mileageMaximum, &mileageLast); err != nil { + return nil, err + } + if parsed, ok := parseVehicleServiceTime(bucket.Time); ok { + bucket.Time = parsed.UTC().Format(time.RFC3339) + } + bucket.SpeedAverage = nullFloat64Pointer(speedAverage) + bucket.SpeedMinimum = nullFloat64Pointer(speedMinimum) + bucket.SpeedMaximum = nullFloat64Pointer(speedMaximum) + bucket.SpeedLast = nullFloat64Pointer(speedLast) + bucket.MileageAverage = nullFloat64Pointer(mileageAverage) + bucket.MileageMinimum = nullFloat64Pointer(mileageMinimum) + bucket.MileageMaximum = nullFloat64Pointer(mileageMaximum) + bucket.MileageLast = nullFloat64Pointer(mileageLast) + buckets = append(buckets, bucket) + } + return buckets, rows.Err() +} + +func (s *ProductionStore) HistoryExportCount(ctx context.Context, query HistoryExportStoreQuery) (int64, error) { + if query.Category == "mileage" { + page, err := s.DailyMileage(ctx, historyExportMileageQuery(query, 1, 0)) + return int64(page.Total), err + } + if s.tdengine == nil { + return 0, nil + } + built := buildHistoryExportBatchSQL(s.tdDatabase, query, HistoryExportCursor{}, 1) + var total int64 + if err := s.tdengine.QueryRowContext(ctx, built.CountText, built.CountArgs...).Scan(&total); err != nil { + if isTDengineTableNotExist(err) { + return 0, nil + } + return 0, err + } + return total, nil +} + +func (s *ProductionStore) HistoryExportBatch(ctx context.Context, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) ([]HistoryDataRow, HistoryExportCursor, error) { + if query.Category == "mileage" { + page, err := s.DailyMileage(ctx, historyExportMileageQuery(query, limit, cursor.Offset)) + if err != nil { + return nil, cursor, err + } + result := make([]HistoryDataRow, 0, len(page.Items)) + for index, item := range page.Items { + result = append(result, HistoryDataRow{ID: "mileage-" + item.VIN + "-" + strconv.Itoa(cursor.Offset+index), VIN: item.VIN, Plate: item.Plate, Protocol: item.Source, DeviceTime: item.Date, ServerTime: item.Date, Quality: "normal", Values: map[string]any{"startMileageKm": item.StartMileageKm, "endMileageKm": item.EndMileageKm, "dailyMileageKm": item.DailyMileageKm}}) + } + cursor.Offset += len(result) + return result, cursor, nil + } + if s.tdengine == nil { + return []HistoryDataRow{}, cursor, nil + } + built := buildHistoryExportBatchSQL(s.tdDatabase, query, cursor, limit) + rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...) + if err != nil { + if isTDengineTableNotExist(err) { + return []HistoryDataRow{}, cursor, nil + } + return nil, cursor, err + } + defer rows.Close() + plates, err := s.platesByVIN(ctx, []string{query.VIN}) + if err != nil { + return nil, cursor, err + } + if query.Category == "raw" { + return scanRawHistoryExportRows(rows, query, cursor, plates[query.VIN]) + } + return scanLocationHistoryExportRows(rows, cursor, plates[query.VIN]) +} + +func scanLocationHistoryExportRows(rows *sql.Rows, cursor HistoryExportCursor, plate string) ([]HistoryDataRow, HistoryExportCursor, error) { + items := make([]HistoryDataRow, 0) + for rows.Next() { + var vin, protocol, ts, receivedAt string + var longitude, latitude, speed, mileage sql.NullFloat64 + if err := rows.Scan(&ts, &vin, &protocol, &longitude, &latitude, &speed, &mileage, &receivedAt); err != nil { + return nil, cursor, err + } + quality := "normal" + location := HistoryLocationRow{Longitude: nullFloat64(longitude), Latitude: nullFloat64(latitude)} + if !validTrackCoordinate(location) { + quality = "warning" + } + items = append(items, HistoryDataRow{ID: "location-" + vin + "-" + protocol + "-" + ts, VIN: vin, Plate: plate, Protocol: protocol, DeviceTime: ts, ServerTime: firstNonEmpty(receivedAt, ts), Quality: quality, Values: map[string]any{"speedKmh": nullFloat64(speed), "totalMileageKm": nullFloat64(mileage), "longitude": nullFloat64(longitude), "latitude": nullFloat64(latitude)}}) + cursor = HistoryExportCursor{Time: ts, Protocol: protocol} + } + return items, cursor, rows.Err() +} + +func scanRawHistoryExportRows(rows *sql.Rows, query HistoryExportStoreQuery, cursor HistoryExportCursor, plate string) ([]HistoryDataRow, HistoryExportCursor, error) { + items := make([]HistoryDataRow, 0) + for rows.Next() { + var ts, frameID, eventTime, receivedAt, parsedFields, parseStatus, parseError, sourceEndpoint, protocol, vehicleKey, vin, phone string + var rawSizeBytes int + if err := rows.Scan(&ts, &frameID, &eventTime, &receivedAt, &rawSizeBytes, &parsedFields, &parseStatus, &parseError, &sourceEndpoint, &protocol, &vehicleKey, &vin, &phone); err != nil { + return nil, cursor, err + } + values := map[string]any{"frameType": vehicleKey, "rawSizeBytes": rawSizeBytes} + fields := parsedFieldsFromString(parsedFields) + if len(query.Metrics) > 0 { + fields = filterParsedFieldsMap(fields, query.Metrics) + } + for key, value := range fields { + values[key] = value + } + quality := "normal" + if parseStatus != "" && !strings.EqualFold(parseStatus, "ok") { + quality = "warning" + } + items = append(items, HistoryDataRow{ID: frameID, VIN: vin, Plate: plate, Protocol: protocol, DeviceTime: firstNonEmpty(eventTime, ts), ServerTime: firstNonEmpty(receivedAt, ts), Quality: quality, EvidenceID: frameID, Values: values}) + cursor = HistoryExportCursor{Time: ts, Protocol: protocol, ID: frameID} + } + return items, cursor, rows.Err() +} + +func historyExportMileageQuery(query HistoryExportStoreQuery, limit, offset int) url.Values { + values := url.Values{"vin": {query.VIN}, "limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(offset)}} + if query.Protocol != "" { + values.Set("protocol", query.Protocol) + } + if query.DateFrom != "" { + values.Set("dateFrom", strings.Split(query.DateFrom, "T")[0]) + } + if query.DateTo != "" { + values.Set("dateTo", strings.Split(query.DateTo, "T")[0]) + } + return values +} + +func nullFloat64Pointer(value sql.NullFloat64) *float64 { + if !value.Valid { + return nil + } + result := value.Float64 + return &result +} + +func nullInt64Pointer(value sql.NullInt64) *int64 { + if !value.Valid { + return nil + } + result := value.Int64 + return &result +} + func (s *ProductionStore) enrichHistoryLocationPlates(ctx context.Context, items []HistoryLocationRow) error { vins := make([]string, 0) seen := map[string]struct{}{} @@ -758,6 +948,9 @@ func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (P row.DeviceTime = firstNonEmpty(eventTime, ts) row.ServerTime = firstNonEmpty(receivedAt, ts) row.RawSizeBytes = rawSizeBytes + row.ParseStatus = parseStatus + row.ParseError = parseError + row.SourceEndpoint = sourceEndpoint row.ParsedFields = parsedFieldsFromString(parsedFields) if len(query.Fields) > 0 { row.ParsedFields = filterParsedFieldsMap(row.ParsedFields, query.Fields) @@ -930,6 +1123,7 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) { tdengineHealth := s.tdengineRawFrameHealth(ctx) redisHealth, redisOnlineKeys := s.redisOnlineKeyHealth(ctx) capacityHealth, kafkaLag, activeConnections, capacityMetrics, capacityFindings := s.capacityCheckHealth(ctx) + alertStreamHealth, alertStream := s.alertStreamCheckpointHealth(ctx) mysqlWritable := mysqlStatus == "ok" && snapshotHealth.Status == "ok" && locationHealth.Status == "ok" return OpsHealth{ LinkHealth: []LinkHealth{ @@ -939,6 +1133,7 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) { tdengineHealth, capacityHealth, redisHealth, + alertStreamHealth, }, KafkaLag: kafkaLag, ActiveConnections: activeConnections, @@ -947,9 +1142,49 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) { RedisOnlineKeys: redisOnlineKeys, TDengineWritable: tdengineHealth.Status == "ok", MySQLWritable: mysqlWritable, + AlertStream: alertStream, }, nil } +func (s *ProductionStore) alertStreamCheckpointHealth(ctx context.Context) (LinkHealth, AlertStreamHealth) { + result := AlertStreamHealth{Mode: firstNonEmpty(s.alertStreamMode, "disabled"), ConsumerGroup: s.alertStreamGroup} + health := LinkHealth{Name: "Alert Kafka stream", Status: "warning", Detail: "Kafka event-time consumer 未配置"} + if s.alertStreamGroup == "" { + return health, result + } + var updatedAt, lastInvalidAt sql.NullTime + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(GREATEST(high_watermark-next_offset,0)),0),COALESCE(SUM(processed_count),0),COALESCE(SUM(valid_count),0),COALESCE(SUM(invalid_count),0),COALESCE(SUM(late_count),0),COALESCE(SUM(replay_skipped_count),0),MAX(updated_at),COALESCE((SELECT c2.last_invalid_code FROM vehicle_alert_stream_checkpoint c2 WHERE c2.consumer_group=? AND c2.last_invalid_at IS NOT NULL ORDER BY c2.last_invalid_at DESC LIMIT 1),''),MAX(last_invalid_at) FROM vehicle_alert_stream_checkpoint WHERE consumer_group=?`, s.alertStreamGroup, s.alertStreamGroup).Scan(&result.Partitions, &result.Lag, &result.Processed, &result.Valid, &result.Invalid, &result.Late, &result.ReplaySkipped, &updatedAt, &result.LastInvalidCode, &lastInvalidAt) + if err != nil { + health.Status = "error" + health.Detail = "读取告警流 checkpoint 失败：" + err.Error() + return health, result + } + if updatedAt.Valid { + result.UpdatedAt = updatedAt.Time.Format(time.RFC3339) + } + if lastInvalidAt.Valid { + result.LastInvalidAt = lastInvalidAt.Time.Format(time.RFC3339) + } + if result.Partitions == 0 || !updatedAt.Valid { + health.Detail = "告警流尚未建立 Kafka 分区 checkpoint" + return health, result + } + age := time.Since(updatedAt.Time) + health.Status = "ok" + health.Detail = result.Mode + " checkpoint 正常" + if age > 2*time.Minute { + health.Status = "error" + health.Detail = "告警流 checkpoint 超过 2 分钟未推进" + } else if result.Lag > 1000 || (lastInvalidAt.Valid && time.Since(lastInvalidAt.Time) < 5*time.Minute) { + health.Status = "warning" + health.Detail = "告警流存在积压或最近非法消息" + if result.LastInvalidCode != "" { + health.Detail += "：" + result.LastInvalidCode + } + } + return health, result +} + func (s *ProductionStore) capacityCheckHealth(ctx context.Context) (LinkHealth, *int, *int, CapacityMetrics, []string) { health := LinkHealth{Name: "Capacity check", Status: "warning", Detail: "平台暂未接入 capacity-check"} if s.capacityCheck == nil { diff --git a/vehicle-data-platform/apps/api/internal/platform/profile.go b/vehicle-data-platform/apps/api/internal/platform/profile.go new file mode 100644 index 00000000..a3b52f39 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/profile.go @@ -0,0 +1,309 @@ +package platform + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" + "time" +) + +var vehicleOperationStatuses = map[string]bool{ + "unknown": true, "active": true, "inactive": true, "maintenance": true, "retired": true, +} + +const ( + vehicleProfileSyncLimit = 500 + profileSyncCreated = "created" + profileSyncUpdated = "updated" + profileSyncUnchanged = "unchanged" + profileSyncConflictSource = "conflict_source" + profileSyncConflictSourceVersion = "conflict_source_version" + profileSyncMissingVehicle = "missing_vehicle" + profileSyncConflictPolicyPreserve = "preserve" + profileSyncConflictPolicyOverwrite = "overwrite" +) + +var vehicleProfileSourceSystemPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9._:-]*$`) + +func (s *Service) VehicleProfile(ctx context.Context, vin string) (VehicleProfile, error) { + vin = strings.TrimSpace(vin) + if err := validateProfileVIN(vin); err != nil { + return VehicleProfile{}, err + } + store, ok := s.store.(VehicleProfileStore) + if !ok { + return decorateVehicleProfile(VehicleProfile{VIN: vin, OperationStatus: "unknown", SourceSystem: "unconfigured", MissingFields: []string{}}), nil + } + profile, exists, err := store.VehicleProfile(ctx, vin) + if err != nil { + return VehicleProfile{}, err + } + if !exists { + profile = VehicleProfile{VIN: vin, OperationStatus: "unknown", SourceSystem: "unconfigured"} + } + return decorateVehicleProfile(profile), nil +} + +func (s *Service) SaveVehicleProfile(ctx context.Context, vin string, input VehicleProfileInput) (VehicleProfile, error) { + vin = strings.TrimSpace(vin) + if err := validateProfileVIN(vin); err != nil { + return VehicleProfile{}, err + } + store, ok := s.store.(VehicleProfileStore) + if !ok { + return VehicleProfile{}, fmt.Errorf("store does not provide durable vehicle profiles") + } + exists, err := s.vehicleExists(ctx, vin) + if err != nil { + return VehicleProfile{}, err + } + if !exists { + return VehicleProfile{}, clientError{Code: "VEHICLE_NOT_FOUND", Message: "车辆身份不存在，不能创建主档"} + } + input = normalizeVehicleProfileInput(input) + if err := validateVehicleProfileInput(input); err != nil { + return VehicleProfile{}, err + } + profile, err := store.SaveVehicleProfile(ctx, vin, input) + if err != nil { + return VehicleProfile{}, err + } + return decorateVehicleProfile(profile), nil +} + +func (s *Service) SyncVehicleProfiles(ctx context.Context, request VehicleProfileSyncRequest) (VehicleProfileSyncResult, error) { + store, ok := s.store.(VehicleProfileStore) + if !ok { + return VehicleProfileSyncResult{}, fmt.Errorf("store does not provide durable vehicle profiles") + } + request = normalizeVehicleProfileSyncRequest(request) + if err := validateVehicleProfileSyncRequest(request); err != nil { + return VehicleProfileSyncResult{}, err + } + return store.SyncVehicleProfiles(ctx, request) +} + +func normalizeVehicleProfileSyncRequest(request VehicleProfileSyncRequest) VehicleProfileSyncRequest { + request.SourceSystem = strings.ToLower(strings.TrimSpace(request.SourceSystem)) + request.SourceVersion = strings.TrimSpace(request.SourceVersion) + request.ConflictPolicy = strings.ToLower(strings.TrimSpace(request.ConflictPolicy)) + request.Actor = strings.TrimSpace(request.Actor) + if request.ConflictPolicy == "" { + request.ConflictPolicy = profileSyncConflictPolicyPreserve + } + for index := range request.Items { + item := &request.Items[index] + item.VIN = strings.ToUpper(strings.TrimSpace(item.VIN)) + normalized := normalizeVehicleProfileInput(vehicleProfileSyncInput(*item, request.Actor)) + item.ModelName = normalized.ModelName + item.VehicleType = normalized.VehicleType + item.CompanyName = normalized.CompanyName + item.OperationStatus = normalized.OperationStatus + item.AccessProvider = normalized.AccessProvider + item.FirstAccessAt = normalized.FirstAccessAt + if item.FirstAccessAt != "" { + if parsed, err := parseVehicleProfileTime(item.FirstAccessAt); err == nil { + item.FirstAccessAt = parsed.Format(time.RFC3339) + } + } + } + return request +} + +func validateVehicleProfileSyncRequest(request VehicleProfileSyncRequest) error { + if request.SourceSystem == "" || len(request.SourceSystem) > 64 || !vehicleProfileSourceSystemPattern.MatchString(request.SourceSystem) || request.SourceSystem == "manual" || request.SourceSystem == "unconfigured" { + return clientError{Code: "VEHICLE_PROFILE_SYNC_SOURCE_INVALID", Message: "sourceSystem 必须是 64 字符内的外部系统标识"} + } + if request.SourceVersion == "" || len([]rune(request.SourceVersion)) > 128 { + return clientError{Code: "VEHICLE_PROFILE_SYNC_VERSION_INVALID", Message: "sourceVersion 不能为空且不能超过 128 个字符"} + } + if request.ConflictPolicy != profileSyncConflictPolicyPreserve && request.ConflictPolicy != profileSyncConflictPolicyOverwrite { + return clientError{Code: "VEHICLE_PROFILE_SYNC_POLICY_INVALID", Message: "conflictPolicy 仅支持 preserve 或 overwrite"} + } + if len(request.Items) == 0 || len(request.Items) > vehicleProfileSyncLimit { + return clientError{Code: "VEHICLE_PROFILE_SYNC_SIZE_INVALID", Message: "单批车辆主档必须为 1 至 500 条"} + } + if request.Actor == "" { + return clientError{Code: "VEHICLE_PROFILE_ACTOR_REQUIRED", Message: "缺少档案同步操作人"} + } + seen := make(map[string]struct{}, len(request.Items)) + for _, item := range request.Items { + if err := validateProfileVIN(item.VIN); err != nil { + return err + } + if _, exists := seen[item.VIN]; exists { + return clientError{Code: "VEHICLE_PROFILE_SYNC_DUPLICATE_VIN", Message: "同一批次不能包含重复 VIN"} + } + seen[item.VIN] = struct{}{} + if err := validateVehicleProfileInput(vehicleProfileSyncInput(item, request.Actor)); err != nil { + return err + } + } + return nil +} + +func vehicleProfileSyncInput(item VehicleProfileSyncItem, actor string) VehicleProfileInput { + return VehicleProfileInput{ + ModelName: item.ModelName, VehicleType: item.VehicleType, CompanyName: item.CompanyName, + OperationStatus: item.OperationStatus, AccessProvider: item.AccessProvider, + FirstAccessAt: item.FirstAccessAt, RuntimeSeconds: item.RuntimeSeconds, Actor: actor, + } +} + +func vehicleProfileSyncDecision(current VehicleProfile, exists bool, request VehicleProfileSyncRequest, item VehicleProfileSyncItem) string { + if !exists { + return profileSyncCreated + } + if current.SourceSystem == request.SourceSystem && current.SourceVersion == request.SourceVersion { + if vehicleProfileSyncFieldsEqual(current, item) { + return profileSyncUnchanged + } + return profileSyncConflictSourceVersion + } + if request.ConflictPolicy == profileSyncConflictPolicyPreserve && current.SourceSystem != request.SourceSystem { + return profileSyncConflictSource + } + return profileSyncUpdated +} + +func vehicleProfileSyncFieldsEqual(current VehicleProfile, item VehicleProfileSyncItem) bool { + return current.ModelName == item.ModelName && current.VehicleType == item.VehicleType && + current.CompanyName == item.CompanyName && current.OperationStatus == item.OperationStatus && + current.AccessProvider == item.AccessProvider && current.FirstAccessAt == item.FirstAccessAt && + equalOptionalInt64(current.RuntimeSeconds, item.RuntimeSeconds) +} + +func equalOptionalInt64(left, right *int64) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} + +func countVehicleProfileSyncResult(result *VehicleProfileSyncResult, status string) { + switch status { + case profileSyncCreated: + result.Created++ + case profileSyncUpdated: + result.Updated++ + case profileSyncUnchanged: + result.Unchanged++ + case profileSyncMissingVehicle: + result.Missing++ + default: + result.Conflicted++ + } +} + +func (s *Service) vehicleExists(ctx context.Context, vin string) (bool, error) { + page, err := s.store.Vehicles(ctx, url.Values{"keyword": {vin}, "limit": {"20"}}) + if err != nil { + return false, err + } + for _, item := range page.Items { + if strings.EqualFold(strings.TrimSpace(item.VIN), vin) { + return true, nil + } + } + return false, nil +} + +func validateProfileVIN(vin string) error { + if vin == "" || len(vin) > 32 { + return clientError{Code: "VEHICLE_PROFILE_VIN_INVALID", Message: "VIN 不能为空且不能超过 32 个字符"} + } + return nil +} + +func normalizeVehicleProfileInput(input VehicleProfileInput) VehicleProfileInput { + input.ModelName = strings.TrimSpace(input.ModelName) + input.VehicleType = strings.TrimSpace(input.VehicleType) + input.CompanyName = strings.TrimSpace(input.CompanyName) + input.OperationStatus = strings.ToLower(strings.TrimSpace(input.OperationStatus)) + input.AccessProvider = strings.TrimSpace(input.AccessProvider) + input.FirstAccessAt = strings.TrimSpace(input.FirstAccessAt) + input.Actor = strings.TrimSpace(input.Actor) + if input.OperationStatus == "" { + input.OperationStatus = "unknown" + } + return input +} + +func validateVehicleProfileInput(input VehicleProfileInput) error { + lengths := []struct { + value string + max int + name string + }{ + {input.ModelName, 128, "车型"}, {input.VehicleType, 64, "车辆类型"}, + {input.CompanyName, 128, "所属企业"}, {input.AccessProvider, 128, "接入服务商"}, + } + for _, field := range lengths { + if len([]rune(field.value)) > field.max { + return clientError{Code: "VEHICLE_PROFILE_FIELD_INVALID", Message: field.name + "长度超出限制"} + } + } + if !vehicleOperationStatuses[input.OperationStatus] { + return clientError{Code: "VEHICLE_PROFILE_STATUS_INVALID", Message: "运营状态不在允许范围内"} + } + if input.RuntimeSeconds != nil && *input.RuntimeSeconds < 0 { + return clientError{Code: "VEHICLE_PROFILE_RUNTIME_INVALID", Message: "累计运行时长不能为负数"} + } + if input.FirstAccessAt != "" { + if _, err := parseVehicleProfileTime(input.FirstAccessAt); err != nil { + return clientError{Code: "VEHICLE_PROFILE_TIME_INVALID", Message: "首次接入时间格式无效"} + } + } + if input.Version < 0 { + return clientError{Code: "VEHICLE_PROFILE_VERSION_INVALID", Message: "档案版本无效"} + } + if input.Actor == "" { + return clientError{Code: "VEHICLE_PROFILE_ACTOR_REQUIRED", Message: "缺少档案维护人"} + } + return nil +} + +func parseVehicleProfileTime(value string) (time.Time, error) { + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04", "2006-01-02 15:04:05"} { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return parsed, nil + } + } + return time.Time{}, fmt.Errorf("unsupported vehicle profile time") +} + +func decorateVehicleProfile(profile VehicleProfile) VehicleProfile { + missing := make([]string, 0, 7) + if profile.ModelName == "" { + missing = append(missing, "modelName") + } + if profile.VehicleType == "" { + missing = append(missing, "vehicleType") + } + if profile.CompanyName == "" { + missing = append(missing, "companyName") + } + if profile.OperationStatus == "" || profile.OperationStatus == "unknown" { + missing = append(missing, "operationStatus") + } + if profile.AccessProvider == "" { + missing = append(missing, "accessProvider") + } + if profile.FirstAccessAt == "" { + missing = append(missing, "firstAccessAt") + } + if profile.RuntimeSeconds == nil { + missing = append(missing, "runtimeSeconds") + } + profile.MissingFields = missing + profile.Completeness = (7 - len(missing)) * 100 / 7 + if profile.OperationStatus == "" { + profile.OperationStatus = "unknown" + } + if profile.SourceSystem == "" { + profile.SourceSystem = "unconfigured" + } + return profile +} diff --git a/vehicle-data-platform/apps/api/internal/platform/profile_store.go b/vehicle-data-platform/apps/api/internal/platform/profile_store.go new file mode 100644 index 00000000..4b7d9e24 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/profile_store.go @@ -0,0 +1,226 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "strings" + "time" +) + +const vehicleProfileSelect = `SELECT vin,model_name,vehicle_type,company_name,operation_status,access_provider,first_access_at,runtime_seconds,source_system,source_version,synced_at,version,updated_by,updated_at FROM vehicle_profile ` + +func (s *ProductionStore) ensureProfileSchema(ctx context.Context) error { + s.profileSchemaOnce.Do(func() { + var present int + s.profileSchemaErr = s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('vehicle_profile','vehicle_profile_audit')`).Scan(&present) + if s.profileSchemaErr == nil && present != 2 { + s.profileSchemaErr = sql.ErrNoRows + } + }) + return s.profileSchemaErr +} + +func (s *ProductionStore) VehicleProfile(ctx context.Context, vin string) (VehicleProfile, bool, error) { + if err := s.ensureProfileSchema(ctx); err != nil { + return VehicleProfile{}, false, err + } + profile, err := scanVehicleProfile(s.db.QueryRowContext(ctx, vehicleProfileSelect+`WHERE vin=?`, vin)) + if err == sql.ErrNoRows { + return VehicleProfile{}, false, nil + } + return profile, err == nil, err +} + +func (s *ProductionStore) SaveVehicleProfile(ctx context.Context, vin string, input VehicleProfileInput) (VehicleProfile, error) { + if err := s.ensureProfileSchema(ctx); err != nil { + return VehicleProfile{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return VehicleProfile{}, err + } + defer tx.Rollback() + var current int + err = tx.QueryRowContext(ctx, `SELECT version FROM vehicle_profile WHERE vin=? FOR UPDATE`, vin).Scan(¤t) + firstAccess := nullableVehicleProfileTime(input.FirstAccessAt) + if err == sql.ErrNoRows { + if input.Version != 0 { + return VehicleProfile{}, clientError{Code: "VEHICLE_PROFILE_NOT_FOUND", Message: "待更新车辆档案不存在"} + } + _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_profile(vin,model_name,vehicle_type,company_name,operation_status,access_provider,first_access_at,runtime_seconds,source_system,source_version,synced_at,version,updated_by) VALUES(?,?,?,?,?,?,?,?, 'manual','',NULL,1,?)`, vin, input.ModelName, input.VehicleType, input.CompanyName, input.OperationStatus, input.AccessProvider, firstAccess, input.RuntimeSeconds, input.Actor) + current = 0 + } else if err != nil { + return VehicleProfile{}, err + } else { + if input.Version != current { + return VehicleProfile{}, clientError{Code: "VEHICLE_PROFILE_VERSION_CONFLICT", Message: "车辆档案已被其他用户更新，请刷新后重试"} + } + result, updateErr := tx.ExecContext(ctx, `UPDATE vehicle_profile SET model_name=?,vehicle_type=?,company_name=?,operation_status=?,access_provider=?,first_access_at=?,runtime_seconds=?,source_system='manual',source_version='',synced_at=NULL,version=version+1,updated_by=? WHERE vin=? AND version=?`, input.ModelName, input.VehicleType, input.CompanyName, input.OperationStatus, input.AccessProvider, firstAccess, input.RuntimeSeconds, input.Actor, vin, current) + err = updateErr + if err == nil { + rows, _ := result.RowsAffected() + if rows != 1 { + err = clientError{Code: "VEHICLE_PROFILE_VERSION_CONFLICT", Message: "车辆档案更新冲突，请刷新后重试"} + } + } + } + if err != nil { + return VehicleProfile{}, err + } + snapshot, _ := json.Marshal(input) + action := "update" + if current == 0 { + action = "create" + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_profile_audit(vin,profile_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`, vin, current+1, input.Actor, action, string(snapshot)); err != nil { + return VehicleProfile{}, err + } + if err = tx.Commit(); err != nil { + return VehicleProfile{}, err + } + return scanVehicleProfile(s.db.QueryRowContext(ctx, vehicleProfileSelect+`WHERE vin=?`, vin)) +} + +func (s *ProductionStore) SyncVehicleProfiles(ctx context.Context, request VehicleProfileSyncRequest) (VehicleProfileSyncResult, error) { + result := VehicleProfileSyncResult{ + SourceSystem: request.SourceSystem, SourceVersion: request.SourceVersion, + DryRun: request.DryRun, Received: len(request.Items), + Items: make([]VehicleProfileSyncItemResult, 0, len(request.Items)), + } + if err := s.ensureProfileSchema(ctx); err != nil { + return VehicleProfileSyncResult{}, err + } + tx, err := s.db.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return VehicleProfileSyncResult{}, err + } + defer tx.Rollback() + + identityArgs := make([]any, 0, len(request.Items)) + for _, item := range request.Items { + identityArgs = append(identityArgs, item.VIN) + } + rows, err := tx.QueryContext(ctx, `SELECT DISTINCT vin FROM vehicle_identity_binding WHERE vin IN (`+repeatPlaceholders(len(identityArgs))+`)`, identityArgs...) + if err != nil { + return VehicleProfileSyncResult{}, err + } + knownVINs := make(map[string]bool, len(request.Items)) + for rows.Next() { + var vin string + if err := rows.Scan(&vin); err != nil { + rows.Close() + return VehicleProfileSyncResult{}, err + } + knownVINs[vin] = true + } + if err := rows.Close(); err != nil { + return VehicleProfileSyncResult{}, err + } + if err := rows.Err(); err != nil { + return VehicleProfileSyncResult{}, err + } + + syncedAt := time.Now() + for _, item := range request.Items { + itemResult := VehicleProfileSyncItemResult{VIN: item.VIN} + if !knownVINs[item.VIN] { + itemResult.Status = profileSyncMissingVehicle + countVehicleProfileSyncResult(&result, itemResult.Status) + result.Items = append(result.Items, itemResult) + continue + } + current, scanErr := scanVehicleProfile(tx.QueryRowContext(ctx, vehicleProfileSelect+`WHERE vin=? FOR UPDATE`, item.VIN)) + exists := scanErr == nil + if scanErr != nil && scanErr != sql.ErrNoRows { + return VehicleProfileSyncResult{}, scanErr + } + if exists { + itemResult.PreviousSource = current.SourceSystem + itemResult.PreviousVersion = current.SourceVersion + } + itemResult.Status = vehicleProfileSyncDecision(current, exists, request, item) + itemResult.ProfileVersion = current.Version + if itemResult.Status == profileSyncCreated { + itemResult.ProfileVersion = 1 + } else if itemResult.Status == profileSyncUpdated { + itemResult.ProfileVersion = current.Version + 1 + } + countVehicleProfileSyncResult(&result, itemResult.Status) + result.Items = append(result.Items, itemResult) + if request.DryRun || (itemResult.Status != profileSyncCreated && itemResult.Status != profileSyncUpdated) { + continue + } + + firstAccess := nullableVehicleProfileTime(item.FirstAccessAt) + if itemResult.Status == profileSyncCreated { + _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_profile(vin,model_name,vehicle_type,company_name,operation_status,access_provider,first_access_at,runtime_seconds,source_system,source_version,synced_at,version,updated_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,1,?)`, item.VIN, item.ModelName, item.VehicleType, item.CompanyName, item.OperationStatus, item.AccessProvider, firstAccess, item.RuntimeSeconds, request.SourceSystem, request.SourceVersion, syncedAt, request.Actor) + } else { + var updateResult sql.Result + updateResult, err = tx.ExecContext(ctx, `UPDATE vehicle_profile SET model_name=?,vehicle_type=?,company_name=?,operation_status=?,access_provider=?,first_access_at=?,runtime_seconds=?,source_system=?,source_version=?,synced_at=?,version=version+1,updated_by=? WHERE vin=? AND version=?`, item.ModelName, item.VehicleType, item.CompanyName, item.OperationStatus, item.AccessProvider, firstAccess, item.RuntimeSeconds, request.SourceSystem, request.SourceVersion, syncedAt, request.Actor, item.VIN, current.Version) + if err == nil { + rowsAffected, rowsErr := updateResult.RowsAffected() + if rowsErr != nil { + err = rowsErr + } else if rowsAffected != 1 { + err = clientError{Code: "VEHICLE_PROFILE_VERSION_CONFLICT", Message: "车辆档案同步冲突，请重新预演"} + } + } + } + if err != nil { + return VehicleProfileSyncResult{}, err + } + snapshot, _ := json.Marshal(struct { + SourceSystem string `json:"sourceSystem"` + SourceVersion string `json:"sourceVersion"` + Profile VehicleProfileSyncItem `json:"profile"` + }{request.SourceSystem, request.SourceVersion, item}) + action := "sync_" + itemResult.Status + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_profile_audit(vin,profile_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`, item.VIN, itemResult.ProfileVersion, request.Actor, action, string(snapshot)); err != nil { + return VehicleProfileSyncResult{}, err + } + } + if request.DryRun { + return result, nil + } + if err := tx.Commit(); err != nil { + return VehicleProfileSyncResult{}, err + } + return result, nil +} + +func nullableVehicleProfileTime(value string) any { + if strings.TrimSpace(value) == "" { + return nil + } + parsed, err := parseVehicleProfileTime(value) + if err != nil { + return nil + } + return parsed +} + +type vehicleProfileScanner interface{ Scan(...any) error } + +func scanVehicleProfile(scanner vehicleProfileScanner) (VehicleProfile, error) { + var profile VehicleProfile + var firstAccess, syncedAt sql.NullTime + var runtime sql.NullInt64 + var updatedAt time.Time + err := scanner.Scan(&profile.VIN, &profile.ModelName, &profile.VehicleType, &profile.CompanyName, &profile.OperationStatus, &profile.AccessProvider, &firstAccess, &runtime, &profile.SourceSystem, &profile.SourceVersion, &syncedAt, &profile.Version, &profile.UpdatedBy, &updatedAt) + if err != nil { + return VehicleProfile{}, err + } + if firstAccess.Valid { + profile.FirstAccessAt = firstAccess.Time.Format(time.RFC3339) + } + if runtime.Valid { + value := runtime.Int64 + profile.RuntimeSeconds = &value + } + if syncedAt.Valid { + profile.SyncedAt = syncedAt.Time.Format(time.RFC3339) + } + profile.UpdatedAt = updatedAt.Format(time.RFC3339) + return profile, nil +} diff --git a/vehicle-data-platform/apps/api/internal/platform/profile_test.go b/vehicle-data-platform/apps/api/internal/platform/profile_test.go new file mode 100644 index 00000000..0ddec18b --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/profile_test.go @@ -0,0 +1,265 @@ +package platform + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestVehicleProfileReturnsCompletenessAndMissingFields(t *testing.T) { + service := NewService(NewMockStore()) + profile, err := service.VehicleProfile(context.Background(), "LB9A32A24R0LS1426") + if err != nil { + t.Fatal(err) + } + if profile.Completeness != 100 || len(profile.MissingFields) != 0 || profile.RuntimeSeconds == nil { + t.Fatalf("expected complete seeded profile, got %+v", profile) + } + empty, err := service.VehicleProfile(context.Background(), "LNXNEGRR7SR318212") + if err != nil { + t.Fatal(err) + } + if empty.Version != 0 || empty.Completeness != 0 || len(empty.MissingFields) != 7 { + t.Fatalf("expected explicit empty profile, got %+v", empty) + } +} + +func TestSaveVehicleProfileCreatesAndUsesOptimisticVersion(t *testing.T) { + service := NewService(NewMockStore()) + runtime := int64(3600) + input := VehicleProfileInput{ModelName: "氢燃料重卡", VehicleType: "重卡", CompanyName: "示范物流", OperationStatus: "active", AccessProvider: "车厂平台", FirstAccessAt: "2026-07-01T08:30:00+08:00", RuntimeSeconds: &runtime, Actor: "admin-a"} + profile, err := service.SaveVehicleProfile(context.Background(), "LNXNEGRR7SR318212", input) + if err != nil { + t.Fatal(err) + } + if profile.Version != 1 || profile.Completeness != 100 || profile.UpdatedBy != "admin-a" { + t.Fatalf("unexpected created profile: %+v", profile) + } + input.Version = 0 + _, err = service.SaveVehicleProfile(context.Background(), "LNXNEGRR7SR318212", input) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "VEHICLE_PROFILE_VERSION_CONFLICT" { + t.Fatalf("stale write should conflict, got %v", err) + } +} + +func TestVehicleProfileValidationRejectsInventedVINAndInvalidValues(t *testing.T) { + service := NewService(NewMockStore()) + negative := int64(-1) + _, err := service.SaveVehicleProfile(context.Background(), "LNOTPRESENT000000", VehicleProfileInput{OperationStatus: "active", Actor: "admin"}) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "VEHICLE_NOT_FOUND" { + t.Fatalf("invented VIN should be rejected, got %v", err) + } + _, err = service.SaveVehicleProfile(context.Background(), "LNXNEGRR7SR318212", VehicleProfileInput{OperationStatus: "flying", RuntimeSeconds: &negative, Actor: "admin"}) + clientErr, ok = asClientError(err) + if !ok || clientErr.Code != "VEHICLE_PROFILE_STATUS_INVALID" { + t.Fatalf("invalid status should be rejected first, got %v", err) + } +} + +func TestVehicleProfileSyncIsDryRunnableIdempotentAndVersionSafe(t *testing.T) { + service := NewService(NewMockStore()) + runtime := int64(7200) + request := VehicleProfileSyncRequest{ + SourceSystem: " OEM-TSP ", SourceVersion: "snapshot-20260714-01", DryRun: true, Actor: "sync-admin", + Items: []VehicleProfileSyncItem{{VIN: " lnxnegrr7sr318212 ", ModelName: "氢燃料重卡", VehicleType: "重卡", CompanyName: "示范物流", OperationStatus: "ACTIVE", AccessProvider: "车厂平台", FirstAccessAt: "2026-07-01T08:30:00+08:00", RuntimeSeconds: &runtime}}, + } + dryRun, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if !dryRun.DryRun || dryRun.Created != 1 || dryRun.Items[0].VIN != "LNXNEGRR7SR318212" { + t.Fatalf("unexpected dry-run result: %+v", dryRun) + } + profile, err := service.VehicleProfile(context.Background(), "LNXNEGRR7SR318212") + if err != nil || profile.Version != 0 { + t.Fatalf("dry-run must not persist profile: profile=%+v err=%v", profile, err) + } + + request.DryRun = false + created, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil || created.Created != 1 { + t.Fatalf("expected created sync: result=%+v err=%v", created, err) + } + profile, err = service.VehicleProfile(context.Background(), "LNXNEGRR7SR318212") + if err != nil || profile.SourceSystem != "oem-tsp" || profile.SourceVersion != request.SourceVersion || profile.Version != 1 { + t.Fatalf("unexpected synced profile: %+v err=%v", profile, err) + } + + repeated, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil || repeated.Unchanged != 1 || repeated.Items[0].Status != profileSyncUnchanged { + t.Fatalf("same source version should be idempotent: result=%+v err=%v", repeated, err) + } + request.Items[0].CompanyName = "同版本篡改" + conflict, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil || conflict.Conflicted != 1 || conflict.Items[0].Status != profileSyncConflictSourceVersion { + t.Fatalf("changed payload under same source version should conflict: result=%+v err=%v", conflict, err) + } + request.SourceVersion = "snapshot-20260714-02" + updated, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil || updated.Updated != 1 || updated.Items[0].ProfileVersion != 2 { + t.Fatalf("new source version should update: result=%+v err=%v", updated, err) + } +} + +func TestVehicleProfileSyncProtectsManualOwnershipAndReportsMissingVIN(t *testing.T) { + service := NewService(NewMockStore()) + request := VehicleProfileSyncRequest{ + SourceSystem: "fleet-gps", SourceVersion: "42", Actor: "sync-admin", + Items: []VehicleProfileSyncItem{ + {VIN: "LB9A32A24R0LS1426", ModelName: "外部车型", OperationStatus: "active"}, + {VIN: "LNOTPRESENT000000", ModelName: "未知车辆", OperationStatus: "active"}, + }, + } + result, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if result.Conflicted != 1 || result.Missing != 1 || result.Items[0].Status != profileSyncConflictSource || result.Items[1].Status != profileSyncMissingVehicle { + t.Fatalf("default policy must preserve manual profile and report missing VIN: %+v", result) + } + request.ConflictPolicy = profileSyncConflictPolicyOverwrite + request.Items = request.Items[:1] + overwritten, err := service.SyncVehicleProfiles(context.Background(), request) + if err != nil || overwritten.Updated != 1 { + t.Fatalf("explicit overwrite should take ownership: result=%+v err=%v", overwritten, err) + } + profile, _ := service.VehicleProfile(context.Background(), "LB9A32A24R0LS1426") + if profile.SourceSystem != "fleet-gps" || profile.ModelName != "外部车型" { + t.Fatalf("overwrite did not update source ownership: %+v", profile) + } +} + +func TestVehicleProfileSyncValidatesBatchBoundsAndDuplicateVIN(t *testing.T) { + service := NewService(NewMockStore()) + base := VehicleProfileSyncRequest{SourceSystem: "oem-tsp", SourceVersion: "1", Actor: "admin"} + base.Items = []VehicleProfileSyncItem{{VIN: "LNXNEGRR7SR318212"}, {VIN: "lnxnegrr7sr318212"}} + _, err := service.SyncVehicleProfiles(context.Background(), base) + clientErr, ok := asClientError(err) + if !ok || clientErr.Code != "VEHICLE_PROFILE_SYNC_DUPLICATE_VIN" { + t.Fatalf("duplicate VIN should fail validation, got %v", err) + } + base.Items = make([]VehicleProfileSyncItem, vehicleProfileSyncLimit+1) + _, err = service.SyncVehicleProfiles(context.Background(), base) + clientErr, ok = asClientError(err) + if !ok || clientErr.Code != "VEHICLE_PROFILE_SYNC_SIZE_INVALID" { + t.Fatalf("oversized batch should fail validation, got %v", err) + } +} + +func TestVehicleDetailIncludesProfile(t *testing.T) { + service := NewService(NewMockStore()) + detail, err := service.VehicleDetail(context.Background(), "LB9A32A24R0LS1426", "") + if err != nil { + t.Fatal(err) + } + if detail.Profile == nil || detail.Profile.CompanyName != "岭牛示范车队" || detail.Profile.Completeness != 100 { + t.Fatalf("vehicle detail missing master profile: %+v", detail.Profile) + } +} + +func TestVehicleProfileHandlersReadAndUpdate(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + read := httptest.NewRecorder() + handler.ServeHTTP(read, httptest.NewRequest(http.MethodGet, "/api/v2/vehicles/LNXNEGRR7SR318212/profile", nil)) + if read.Code != http.StatusOK || !bytes.Contains(read.Body.Bytes(), []byte(`"completeness":0`)) { + t.Fatalf("empty profile read status=%d body=%s", read.Code, read.Body.String()) + } + update := httptest.NewRecorder() + body := bytes.NewBufferString(`{"modelName":"氢燃料重卡","vehicleType":"重卡","companyName":"示范物流","operationStatus":"active","accessProvider":"车厂平台","firstAccessAt":"2026-07-01T08:30","runtimeSeconds":3600,"version":0}`) + handler.ServeHTTP(update, httptest.NewRequest(http.MethodPut, "/api/v2/vehicles/LNXNEGRR7SR318212/profile", body)) + if update.Code != http.StatusOK { + t.Fatalf("profile update status=%d body=%s", update.Code, update.Body.String()) + } + var response struct { + Data VehicleProfile `json:"data"` + } + if err := json.Unmarshal(update.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Data.Version != 1 || response.Data.Completeness != 100 || response.Data.UpdatedBy != "system" { + t.Fatalf("unexpected handler profile: %+v", response.Data) + } +} + +func TestVehicleProfileSyncHandlerUsesAuthenticatedActor(t *testing.T) { + handler := NewHandler(NewService(NewMockStore())) + recorder := httptest.NewRecorder() + body := bytes.NewBufferString(`{"sourceSystem":"oem-tsp","sourceVersion":"v1","items":[{"vin":"LNXNEGRR7SR318212","modelName":"氢燃料重卡","operationStatus":"active"}]}`) + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/v2/vehicle-profiles/sync", body)) + if recorder.Code != http.StatusOK || !bytes.Contains(recorder.Body.Bytes(), []byte(`"created":1`)) { + t.Fatalf("profile sync status=%d body=%s", recorder.Code, recorder.Body.String()) + } + profile, _ := handler.service.VehicleProfile(context.Background(), "LNXNEGRR7SR318212") + if profile.UpdatedBy != "system" { + t.Fatalf("handler must override body actor with authenticated actor, got %+v", profile) + } +} + +func TestProductionVehicleProfileCreateIsTransactionalAndAudited(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + now := time.Now() + runtime := int64(7200) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('vehicle_profile','vehicle_profile_audit')`)).WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2)) + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT version FROM vehicle_profile WHERE vin=? FOR UPDATE`)).WithArgs("VIN001").WillReturnRows(sqlmock.NewRows([]string{"version"})) + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO vehicle_profile(vin,model_name,vehicle_type,company_name,operation_status,access_provider,first_access_at,runtime_seconds,source_system,source_version,synced_at,version,updated_by) VALUES(?,?,?,?,?,?,?,?, 'manual','',NULL,1,?)`)).WithArgs("VIN001", "示范车型", "重卡", "示范物流", "active", "车厂平台", sqlmock.AnyArg(), runtime, "admin-a").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO vehicle_profile_audit(vin,profile_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`)).WithArgs("VIN001", 1, "admin-a", "create", sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + columns := []string{"vin", "model_name", "vehicle_type", "company_name", "operation_status", "access_provider", "first_access_at", "runtime_seconds", "source_system", "source_version", "synced_at", "version", "updated_by", "updated_at"} + mock.ExpectQuery(regexp.QuoteMeta(vehicleProfileSelect + `WHERE vin=?`)).WithArgs("VIN001").WillReturnRows(sqlmock.NewRows(columns).AddRow("VIN001", "示范车型", "重卡", "示范物流", "active", "车厂平台", now, runtime, "manual", "", nil, 1, "admin-a", now)) + profile, err := store.SaveVehicleProfile(context.Background(), "VIN001", VehicleProfileInput{ModelName: "示范车型", VehicleType: "重卡", CompanyName: "示范物流", OperationStatus: "active", AccessProvider: "车厂平台", FirstAccessAt: now.Format(time.RFC3339), RuntimeSeconds: &runtime, Actor: "admin-a"}) + if err != nil { + t.Fatal(err) + } + if profile.Version != 1 || profile.RuntimeSeconds == nil || *profile.RuntimeSeconds != runtime { + t.Fatalf("unexpected stored profile: %+v", profile) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestProductionVehicleProfileSyncCreatesAndAuditsInOneTransaction(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + store := NewProductionStore(db, nil, "") + runtime := int64(3600) + request := VehicleProfileSyncRequest{ + SourceSystem: "oem-tsp", SourceVersion: "snapshot-1", Actor: "sync-admin", + Items: []VehicleProfileSyncItem{{VIN: "VIN001", ModelName: "示范车型", VehicleType: "重卡", CompanyName: "示范物流", OperationStatus: "active", AccessProvider: "车厂平台", FirstAccessAt: "2026-07-01T08:30:00+08:00", RuntimeSeconds: &runtime}}, + } + mock.ExpectQuery(regexp.QuoteMeta(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ('vehicle_profile','vehicle_profile_audit')`)).WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(2)) + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(`SELECT DISTINCT vin FROM vehicle_identity_binding WHERE vin IN (?)`)).WithArgs("VIN001").WillReturnRows(sqlmock.NewRows([]string{"vin"}).AddRow("VIN001")) + mock.ExpectQuery(regexp.QuoteMeta(vehicleProfileSelect + `WHERE vin=? FOR UPDATE`)).WithArgs("VIN001").WillReturnRows(sqlmock.NewRows([]string{"vin", "model_name", "vehicle_type", "company_name", "operation_status", "access_provider", "first_access_at", "runtime_seconds", "source_system", "source_version", "synced_at", "version", "updated_by", "updated_at"})) + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO vehicle_profile(vin,model_name,vehicle_type,company_name,operation_status,access_provider,first_access_at,runtime_seconds,source_system,source_version,synced_at,version,updated_by) VALUES(?,?,?,?,?,?,?,?,?,?,?,1,?)`)).WithArgs("VIN001", "示范车型", "重卡", "示范物流", "active", "车厂平台", sqlmock.AnyArg(), runtime, "oem-tsp", "snapshot-1", sqlmock.AnyArg(), "sync-admin").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO vehicle_profile_audit(vin,profile_version,actor,action,snapshot_json) VALUES(?,?,?,?,?)`)).WithArgs("VIN001", 1, "sync-admin", "sync_created", sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + result, err := store.SyncVehicleProfiles(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if result.Created != 1 || result.Items[0].ProfileVersion != 1 { + t.Fatalf("unexpected sync result: %+v", result) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go b/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go index 316d3361..52262112 100644 --- a/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go @@ -232,6 +232,15 @@ func TestBuildVehicleRealtimeSQLFiltersServiceStatus(t *testing.T) { } } +func TestBuildVehicleRealtimeSQLFiltersMotionStatusBeforePagination(t *testing.T) { + built := buildVehicleRealtimeSQL(url.Values{"status": {"driving"}, "limit": {"200"}}) + for _, want := range []string{"HAVING", "INTERVAL 1 MINUTE", "GROUP_CONCAT(CAST(l.speed_kmh AS CHAR)", "> 3"} { + if !strings.Contains(built.Text, want) || !strings.Contains(built.CountText, want) { + t.Fatalf("motion filter must constrain rows and count before LIMIT, missing %q: %s / %s", want, built.Text, built.CountText) + } + } +} + func TestBuildDailyMileageSQL(t *testing.T) { query := url.Values{"vin": {"VIN001"}, "protocol": {"JT808"}, "dateFrom": {"2026-07-01"}, "dateTo": {"2026-07-03"}} built := buildDailyMileageSQL(query) @@ -241,6 +250,9 @@ func TestBuildDailyMileageSQL(t *testing.T) { if !strings.Contains(built.Text, "ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC") { t.Fatalf("SQL should keep stable pagination order: %s", built.Text) } + if strings.Contains(built.Text, "m.first_total_mileage_km") || !strings.Contains(built.Text, "m.latest_total_mileage_km - m.daily_mileage_km") { + t.Fatalf("daily mileage must follow the current projection schema and derive its start value: %s", built.Text) + } if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") { t.Fatalf("count SQL = %s", built.CountText) } @@ -285,6 +297,13 @@ func TestBuildRawFrameSQL(t *testing.T) { } } +func TestBuildRawFrameSQLCanSkipUnneededCount(t *testing.T) { + built := buildRawFrameSQL("lingniu_vehicle_ts", RawFrameQuery{VIN: "VIN001", IncludeFields: true, Limit: 100, SkipCount: true}) + if built.CountText != "" || !strings.Contains(built.Text, "LIMIT 100") { + t.Fatalf("bounded latest query should skip count: %+v", built) + } +} + func TestBuildHistoryLocationSQL(t *testing.T) { built := buildHistoryLocationSQL("lingniu_vehicle_ts", map[string]string{ "protocol": "JT808", @@ -295,6 +314,14 @@ func TestBuildHistoryLocationSQL(t *testing.T) { if !strings.Contains(built.Text, "lingniu_vehicle_ts.loc_jt808_") || !strings.Contains(built.Text, "vin = 'VIN001'") { t.Fatalf("SQL = %s", built.Text) } + for _, field := range []string{"soc_percent", "direction_deg", "alarm_flag"} { + if !strings.Contains(built.Text, field) { + t.Fatalf("track evidence field %s missing from SQL: %s", field, built.Text) + } + } + if !strings.Contains(built.Text, "CAST(ts AS BIGINT)") || !strings.Contains(built.Text, "CAST(received_at AS BIGINT)") { + t.Fatalf("history location time must use epoch scans: %s", built.Text) + } if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") { t.Fatalf("count SQL = %s", built.CountText) } @@ -303,10 +330,61 @@ func TestBuildHistoryLocationSQL(t *testing.T) { } } +func TestBuildHistoryLocationSQLNormalizesDatetimeLocalMinutePrecision(t *testing.T) { + built := buildHistoryLocationSQL("lingniu_vehicle_ts", map[string]string{ + "vin": "VIN001", "dateFrom": "2026-07-14T00:00", "dateTo": "2026-07-14T05:56", "limit": "10", + }) + for _, want := range []string{"ts >= '2026-07-14T00:00:00+08:00'", "ts <= '2026-07-14T05:56:00+08:00'"} { + if !strings.Contains(built.Text, want) { + t.Fatalf("datetime-local minute precision must be normalized for TDengine, missing %q in %s", want, built.Text) + } + } +} + +func TestTDengineTimeLiteralsAlwaysCarryAnExplicitOffset(t *testing.T) { + for _, value := range []string{"2026-07-14T08:24", "2026-07-14 08:24:00", "2026-07-14T00:24:00Z"} { + normalized := normalizeTDengineTime(value) + if !strings.HasSuffix(normalized, "+08:00") && !strings.HasSuffix(normalized, "Z") { + t.Fatalf("TDengine time literal lost timezone: input=%s normalized=%s", value, normalized) + } + } +} + +func TestBuildHistoryLocationSeriesSQLUsesWhitelistedAggregationWindow(t *testing.T) { + built := buildHistoryLocationSeriesSQL("lingniu_vehicle_ts", HistoryLocationSeriesQuery{VIN: "VIN'001", Protocol: "jt808", DateFrom: "2026-07-14T00:00", DateTo: "2026-07-14T05:56", GrainSeconds: 60}) + for _, want := range []string{"SELECT _wstart, protocol, COUNT(*)", "AVG(speed_kmh)", "LAST(total_mileage_km)", "vin = 'VIN''001'", "protocol = 'JT808'", "ts >= '2026-07-14T00:00:00+08:00'", "PARTITION BY protocol INTERVAL(60s)", "ORDER BY _wstart ASC, protocol ASC"} { + if !strings.Contains(built.Text, want) { + t.Fatalf("series SQL missing %q: %s", want, built.Text) + } + } + if strings.Contains(built.Text, "FILL") || strings.Contains(built.Text, "GROUP BY") { + t.Fatalf("series SQL must preserve missing windows and use TDengine window syntax: %s", built.Text) + } +} + +func TestBuildHistoryExportBatchSQLUsesStableForwardCursor(t *testing.T) { + built := buildHistoryExportBatchSQL("lingniu_vehicle_ts", HistoryExportStoreQuery{Category: "location", VIN: "VIN001", DateFrom: "2026-07-14T00:00", DateTo: "2026-07-14T06:00"}, HistoryExportCursor{Time: "2026-07-14T01:00:00+08:00", Protocol: "GB32960"}, 5000) + for _, want := range []string{"ts >= '2026-07-14T00:00:00+08:00'", "ts <= '2026-07-14T06:00:00+08:00'", "ts > '2026-07-14T01:00:00+08:00'", "protocol > 'GB32960'", "ORDER BY ts ASC, protocol ASC LIMIT 5000"} { + if !strings.Contains(built.Text, want) { + t.Fatalf("location export SQL missing %q: %s", want, built.Text) + } + } + if strings.Contains(built.Text, "OFFSET") { + t.Fatalf("million-row export must not use OFFSET: %s", built.Text) + } + + raw := buildHistoryExportBatchSQL("lingniu_vehicle_ts", HistoryExportStoreQuery{Category: "raw", VIN: "VIN001"}, HistoryExportCursor{Time: "2026-07-14T01:00:00+08:00", Protocol: "JT808", ID: "frame-9"}, 5000) + for _, want := range []string{"frame_id > 'frame-9'", "ORDER BY ts ASC, protocol ASC, frame_id ASC", "parsed_json"} { + if !strings.Contains(raw.Text, want) { + t.Fatalf("raw export SQL missing %q: %s", want, raw.Text) + } + } +} + func TestBuildTodayRawFrameCountSQL(t *testing.T) { now := time.Date(2026, 7, 3, 22, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)) built := buildTodayRawFrameCountSQL("lingniu_vehicle_ts", now) - for _, want := range []string{"SELECT COUNT(*)", "lingniu_vehicle_ts.raw_frames", "ts >= '2026-07-02 16:00:00'"} { + for _, want := range []string{"SELECT COUNT(*)", "lingniu_vehicle_ts.raw_frames", "ts >= '2026-07-03T00:00:00+08:00'"} { if !strings.Contains(built.Text, want) { t.Fatalf("SQL missing %q: %s", want, built.Text) } diff --git a/vehicle-data-platform/apps/api/internal/platform/service.go b/vehicle-data-platform/apps/api/internal/platform/service.go index 9e00e4e1..395f75f8 100644 --- a/vehicle-data-platform/apps/api/internal/platform/service.go +++ b/vehicle-data-platform/apps/api/internal/platform/service.go @@ -1,13 +1,22 @@ package platform import ( + "bufio" "context" + "crypto/rand" + "encoding/csv" + "encoding/hex" + "encoding/json" "errors" + "fmt" "math" "net/url" + "os" + "path/filepath" "sort" "strconv" "strings" + "sync" "time" ) @@ -33,6 +42,12 @@ type VehicleOverviewBatchStore interface { VehicleServiceOverviews(context.Context, VehicleOverviewBatchQuery) (Page[VehicleServiceOverview], error) } +type VehicleProfileStore interface { + VehicleProfile(context.Context, string) (VehicleProfile, bool, error) + SaveVehicleProfile(context.Context, string, VehicleProfileInput) (VehicleProfile, error) + SyncVehicleProfiles(context.Context, VehicleProfileSyncRequest) (VehicleProfileSyncResult, error) +} + type RawFrameQuery struct { Protocol string `json:"protocol"` VIN string `json:"vin"` @@ -43,6 +58,7 @@ type RawFrameQuery struct { IncludeFields bool `json:"includeFields"` Limit int `json:"limit"` Offset int `json:"offset"` + SkipCount bool `json:"-"` } type VehicleOverviewBatchQuery struct { @@ -53,8 +69,12 @@ type VehicleOverviewBatchQuery struct { } type Service struct { - store Store - runtime RuntimeInfo + store Store + runtime RuntimeInfo + exportsMu sync.RWMutex + exports map[string]*HistoryExportJob + exportDir string + exportSlots chan struct{} } type clientError struct { @@ -321,17 +341,274 @@ func buildVehicleCoverageSourceStatus(protocols []string, onlineProtocols []stri } func NewService(store Store) *Service { - return &Service{store: store} + return newService(store, RuntimeInfo{}) } func NewServiceWithRuntime(store Store, runtime RuntimeInfo) *Service { - return &Service{store: store, runtime: runtime} + return newService(store, runtime) +} + +func newService(store Store, runtime RuntimeInfo) *Service { + exportDir := strings.TrimSpace(runtime.ExportDir) + if exportDir == "" { + exportDir = filepath.Join(os.TempDir(), "lingniu-vehicle-platform-exports") + } + service := &Service{store: store, runtime: runtime, exports: map[string]*HistoryExportJob{}, exportDir: exportDir, exportSlots: make(chan struct{}, 1)} + service.loadHistoryExports() + return service } func (s *Service) DashboardSummary(ctx context.Context) (DashboardSummary, error) { return s.store.DashboardSummary(ctx) } +const ( + monitorVehicleLimit = 10000 + monitorPointLimit = 2000 + monitorPointZoom = 11 + monitorClusterGridPixels = 64 +) + +type monitorBounds struct { + minLongitude float64 + minLatitude float64 + maxLongitude float64 + maxLatitude float64 +} + +type monitorAlertStore interface { + ActiveAlertVINs(context.Context, string) ([]string, error) +} + +func parseMonitorBounds(value string) (monitorBounds, bool, error) { + value = strings.TrimSpace(value) + if value == "" { + return monitorBounds{}, false, nil + } + parts := strings.Split(value, ",") + if len(parts) != 4 { + return monitorBounds{}, false, clientError{Code: "MONITOR_BOUNDS_INVALID", Message: "地图视口必须为最小经度、最小纬度、最大经度、最大纬度"} + } + values := [4]float64{} + for index, part := range parts { + parsed, err := strconv.ParseFloat(strings.TrimSpace(part), 64) + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) { + return monitorBounds{}, false, clientError{Code: "MONITOR_BOUNDS_INVALID", Message: "地图视口坐标无效"} + } + values[index] = parsed + } + if values[0] < -180 || values[2] > 180 || values[1] < -90 || values[3] > 90 || values[0] >= values[2] || values[1] >= values[3] { + return monitorBounds{}, false, clientError{Code: "MONITOR_BOUNDS_INVALID", Message: "地图视口范围无效"} + } + return monitorBounds{minLongitude: values[0], minLatitude: values[1], maxLongitude: values[2], maxLatitude: values[3]}, true, nil +} + +func (bounds monitorBounds) contains(longitude, latitude float64) bool { + return longitude >= bounds.minLongitude && longitude <= bounds.maxLongitude && latitude >= bounds.minLatitude && latitude <= bounds.maxLatitude +} + +func monitorVehicleStatus(row VehicleRealtimeRow) string { + if strings.TrimSpace(row.LastSeen) == "" { + return "unknown" + } + if !row.Online { + return "offline" + } + if row.SpeedKmh > 3 { + return "driving" + } + return "idle" +} + +func normalizeMonitorQuery(query url.Values) url.Values { + next := make(url.Values, len(query)+2) + for key, values := range query { + next[key] = append([]string(nil), values...) + } + next.Set("limit", strconv.Itoa(monitorVehicleLimit)) + next.Set("offset", "0") + if next.Get("vin") == "" && strings.TrimSpace(next.Get("keyword")) != "" { + next.Set("vin", strings.TrimSpace(next.Get("keyword"))) + } + switch strings.TrimSpace(next.Get("status")) { + case "online": + next.Set("online", "online") + case "offline": + next.Set("online", "offline") + } + return next +} + +func matchesMonitorStatus(row VehicleRealtimeRow, requested string) bool { + requested = strings.TrimSpace(requested) + return requested == "" || monitorVehicleStatus(row) == requested || requested == "online" && row.Online +} + +func (s *Service) MonitorSummary(ctx context.Context, query url.Values) (MonitorSummary, error) { + vehicles, err := s.store.VehicleRealtime(ctx, normalizeMonitorQuery(query)) + if err != nil { + return MonitorSummary{}, err + } + dashboard, err := s.store.DashboardSummary(ctx) + if err != nil { + return MonitorSummary{}, err + } + result := MonitorSummary{ + ActiveToday: dashboard.ActiveToday, + FrameToday: dashboard.FrameToday, + Truncated: vehicles.Total > len(vehicles.Items), + AsOf: time.Now().UTC().Format(time.RFC3339), + } + activeAlertVINs := map[string]struct{}{} + if store, ok := s.store.(monitorAlertStore); ok { + vins, alertErr := store.ActiveAlertVINs(ctx, strings.TrimSpace(query.Get("protocol"))) + if alertErr != nil { + return MonitorSummary{}, alertErr + } + for _, vin := range vins { + activeAlertVINs[strings.TrimSpace(vin)] = struct{}{} + } + result.AlertDataAvailable = true + } + for _, vehicle := range vehicles.Items { + if !matchesMonitorStatus(vehicle, query.Get("status")) { + continue + } + result.TotalVehicles++ + if _, active := activeAlertVINs[vehicle.VIN]; active { + result.AlertVehicles++ + } + switch monitorVehicleStatus(vehicle) { + case "driving": + result.DrivingVehicles++ + result.OnlineVehicles++ + case "idle": + result.IdleVehicles++ + result.OnlineVehicles++ + case "offline": + result.OfflineVehicles++ + default: + result.UnknownVehicles++ + } + } + return result, nil +} + +func (s *Service) MonitorMap(ctx context.Context, query url.Values) (MonitorMapResponse, error) { + vehicles, err := s.store.VehicleRealtime(ctx, normalizeMonitorQuery(query)) + if err != nil { + return MonitorMapResponse{}, err + } + return buildMonitorMapResponse(vehicles, query) +} + +func buildMonitorMapResponse(vehicles Page[VehicleRealtimeRow], query url.Values) (MonitorMapResponse, error) { + zoom := parsePositive(query.Get("zoom"), 5) + if zoom > 20 { + zoom = 20 + } + bounds, hasBounds, err := parseMonitorBounds(query.Get("bounds")) + if err != nil { + return MonitorMapResponse{}, err + } + result := MonitorMapResponse{ + Mode: "clusters", + Zoom: zoom, + Truncated: vehicles.Total > len(vehicles.Items), + Points: []MonitorMapPoint{}, + Clusters: []MonitorMapCluster{}, + AsOf: time.Now().UTC().Format(time.RFC3339), + } + points := make([]MonitorMapPoint, 0, len(vehicles.Items)) + for _, vehicle := range vehicles.Items { + if !matchesMonitorStatus(vehicle, query.Get("status")) { + continue + } + result.Total++ + if vehicle.Longitude < 73 || vehicle.Longitude > 135 || vehicle.Latitude < 18 || vehicle.Latitude > 54 { + continue + } + if hasBounds && !bounds.contains(vehicle.Longitude, vehicle.Latitude) { + continue + } + points = append(points, MonitorMapPoint{ + VIN: vehicle.VIN, Plate: vehicle.Plate, Protocol: vehicle.PrimaryProtocol, Protocols: vehicle.Protocols, + Longitude: vehicle.Longitude, Latitude: vehicle.Latitude, SpeedKmh: vehicle.SpeedKmh, + SOCPercent: vehicle.SOCPercent, TotalMileageKm: vehicle.TotalMileageKm, LastSeen: vehicle.LastSeen, + Status: monitorVehicleStatus(vehicle), + }) + } + if zoom >= monitorPointZoom && len(points) <= monitorPointLimit { + result.Mode = "points" + result.Points = points + return result, nil + } + type clusterBucket struct { + cluster MonitorMapCluster + points []MonitorMapPoint + } + cellSize := 360 * monitorClusterGridPixels / (256 * math.Pow(2, float64(zoom))) + if cellSize < 0.01 { + cellSize = 0.01 + } + clusters := map[string]*clusterBucket{} + for { + clusters = make(map[string]*clusterBucket) + for _, point := range points { + x := int(math.Floor(point.Longitude / cellSize)) + y := int(math.Floor(point.Latitude / cellSize)) + key := strconv.Itoa(x) + ":" + strconv.Itoa(y) + bucket := clusters[key] + if bucket == nil { + bucket = &clusterBucket{cluster: MonitorMapCluster{ID: key}} + clusters[key] = bucket + } + cluster := &bucket.cluster + bucket.points = append(bucket.points, point) + cluster.Count++ + cluster.Longitude += (point.Longitude - cluster.Longitude) / float64(cluster.Count) + cluster.Latitude += (point.Latitude - cluster.Latitude) / float64(cluster.Count) + switch point.Status { + case "driving": + cluster.Driving++ + cluster.Online++ + case "idle": + cluster.Idle++ + cluster.Online++ + case "offline": + cluster.Offline++ + default: + cluster.Unknown++ + } + } + if len(clusters) <= monitorPointLimit || cellSize >= 180 { + break + } + cellSize *= 2 + } + keys := make([]string, 0, len(clusters)) + for key := range clusters { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + bucket := clusters[key] + if bucket.cluster.Count == 1 { + result.Points = append(result.Points, bucket.points[0]) + continue + } + result.Clusters = append(result.Clusters, bucket.cluster) + } + if len(result.Points) > 0 { + if len(result.Clusters) > 0 { + result.Mode = "mixed" + } else { + result.Mode = "points" + } + } + return result, nil +} + func (s *Service) Vehicles(ctx context.Context, query url.Values) (Page[VehicleRow], error) { return s.store.Vehicles(ctx, query) } @@ -570,12 +847,17 @@ func (s *Service) VehicleDetail(ctx context.Context, vin string, protocol string } serviceOverview := buildVehicleServiceOverview(resolvedVIN, keyword, &resolution, identity, summary, sourceStatus, history, raw, mileage, quality) serviceOverview.ServiceStatus = serviceStatus + profile, err := s.VehicleProfile(ctx, resolvedVIN) + if err != nil { + return VehicleDetail{}, err + } return VehicleDetail{ VIN: resolvedVIN, LookupKey: keyword, LookupResolved: resolvedVIN != "", Resolution: &resolution, Identity: identity, + Profile: &profile, RealtimeSummary: summary, ServiceStatus: serviceStatus, ServiceOverview: serviceOverview, @@ -954,6 +1236,1828 @@ func (s *Service) HistoryLocations(ctx context.Context, query url.Values) (Page[ return s.store.HistoryLocationsFromTDengine(ctx, resolvedQuery) } +func (s *Service) TrackPlayback(ctx context.Context, query url.Values) (TrackPlaybackResponse, error) { + keyword := strings.TrimSpace(firstNonEmpty(query.Get("keyword"), query.Get("vin"))) + if keyword == "" { + return TrackPlaybackResponse{}, clientError{Code: "VEHICLE_KEY_REQUIRED", Message: "轨迹查询需要车牌、VIN 或终端标识"} + } + if err := validateTrackWindow(query.Get("dateFrom"), query.Get("dateTo")); err != nil { + return TrackPlaybackResponse{}, err + } + maxPoints := parsePositive(query.Get("maxPoints"), 1200) + if maxPoints < 2 { + maxPoints = 2 + } + if maxPoints > 2000 { + maxPoints = 2000 + } + resolvedQuery, err := s.resolveVehicleQuery(ctx, query) + if err != nil { + return TrackPlaybackResponse{}, err + } + resolvedQuery.Set("limit", "5000") + resolvedQuery.Set("offset", "0") + page, err := s.store.HistoryLocationsFromTDengine(ctx, resolvedQuery) + if err != nil { + return TrackPlaybackResponse{}, err + } + rawPoints := append([]HistoryLocationRow(nil), page.Items...) + sort.SliceStable(rawPoints, func(i, j int) bool { + left, leftOK := parseVehicleServiceTime(rawPoints[i].DeviceTime) + right, rightOK := parseVehicleServiceTime(rawPoints[j].DeviceTime) + if leftOK && rightOK && !left.Equal(right) { + return left.Before(right) + } + return rawPoints[i].DeviceTime < rawPoints[j].DeviceTime + }) + cleanPoints, quality := analyzeTrackPoints(rawPoints) + points, selectedProtocol, alternateSourcePoints := selectTrackPrimarySource(cleanPoints, query.Get("protocol")) + quality.SelectedProtocol = selectedProtocol + quality.AlternateSourcePoints = alternateSourcePoints + quality.ValidPoints = len(points) + applyTrackSequenceQuality(points, &quality) + result := TrackPlaybackResponse{ + Points: []HistoryLocationRow{}, + Events: []TrackPlaybackEvent{}, + Sources: summarizeTrackSources(cleanPoints), + Segments: []TrackSegment{}, + Stops: []TrackStop{}, + Total: page.Total, + Truncated: page.Total > len(rawPoints), + Quality: quality, + AsOf: time.Now().UTC().Format(time.RFC3339), + } + result.Coverage = trackCoverage(query, page.Total, len(rawPoints), len(points), 0, result.Truncated, false) + applyTrackPrimarySourceCoverage(&result.Coverage, selectedProtocol, alternateSourcePoints) + if len(points) == 0 { + return result, nil + } + result.VIN = points[0].VIN + result.Plate = points[0].Plate + segments := buildTrackSegments(points) + stops := buildTrackStops(points, segments) + result.Summary = summarizeTrack(points) + result.Coverage.ActualStart = result.Summary.StartTime + result.Coverage.ActualEnd = result.Summary.EndTime + result.Summary.SegmentCount = len(segments) + result.Summary.StopCount = len(stops) + for _, segment := range segments { + switch segment.Type { + case "moving": + result.Summary.MovingSeconds += segment.DurationSeconds + case "stopped": + result.Summary.StoppedSeconds += segment.DurationSeconds + } + } + result.Segments = segments + result.Stops = stops + result.Events = buildTrackEvents(points, segments, stops) + keyIndexes := trackKeyIndexes(result.Events, segments, stops) + var sampledIndexes []int + result.Points, sampledIndexes = sampleTrackPointsWithKeys(points, maxPoints, keyIndexes) + result.Sampled = len(result.Points) < len(points) + for index := range result.Events { + result.Events[index].SampledIndex = nearestSampledIndex(result.Events[index].Index, sampledIndexes) + } + for index := range result.Segments { + result.Segments[index].SampledStartIndex = nearestSampledIndex(result.Segments[index].StartIndex, sampledIndexes) + result.Segments[index].SampledEndIndex = nearestSampledIndex(result.Segments[index].EndIndex, sampledIndexes) + } + for index := range result.Stops { + result.Stops[index].SampledIndex = nearestSampledIndex((result.Stops[index].startIndex+result.Stops[index].endIndex)/2, sampledIndexes) + } + result.Coverage = trackCoverage(query, page.Total, len(rawPoints), len(points), len(result.Points), result.Truncated, result.Sampled) + applyTrackPrimarySourceCoverage(&result.Coverage, selectedProtocol, alternateSourcePoints) + result.Coverage.ActualStart = result.Summary.StartTime + result.Coverage.ActualEnd = result.Summary.EndTime + return result, nil +} + +func validateTrackWindow(dateFrom, dateTo string) error { + dateFrom = strings.TrimSpace(dateFrom) + dateTo = strings.TrimSpace(dateTo) + if dateFrom == "" && dateTo == "" { + return nil + } + if dateFrom == "" || dateTo == "" { + return clientError{Code: "TRACK_TIME_RANGE_REQUIRED", Message: "轨迹时间范围需要同时提供开始和结束时间"} + } + start, startOK := parseTrackRequestTime(dateFrom) + end, endOK := parseTrackRequestTime(dateTo) + if !startOK || !endOK { + return clientError{Code: "TRACK_TIME_INVALID", Message: "轨迹时间格式无效"} + } + if !end.After(start) { + return clientError{Code: "TRACK_TIME_ORDER_INVALID", Message: "轨迹结束时间必须晚于开始时间"} + } + if end.Sub(start) > 7*24*time.Hour { + return clientError{Code: "TRACK_TIME_RANGE_TOO_LARGE", Message: "单次轨迹回放最长支持 7 天，请缩小时间范围"} + } + return nil +} + +func parseTrackRequestTime(value string) (time.Time, bool) { + shanghai := time.FixedZone("Asia/Shanghai", 8*60*60) + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04", "2006-01-02T15:04:05", "2006-01-02 15:04:05"} { + if layout == time.RFC3339 { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed, true + } + continue + } + if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil { + return parsed, true + } + } + return time.Time{}, false +} + +func (s *Service) HistoryMetricCatalog() HistoryMetricCatalog { + return HistoryMetricCatalog{ + Categories: []HistoryDataCategory{{Key: "location", Label: "位置数据"}, {Key: "raw", Label: "原始报文"}, {Key: "mileage", Label: "日里程"}}, + Metrics: append(append(append([]HistoryMetricDefinition{}, historyLocationMetrics()...), historyRawMetrics()...), historyMileageMetrics()...), + } +} + +type metricCatalogStore interface { + MetricDefinitions(context.Context) ([]MetricDefinition, error) +} + +func (s *Service) metricDefinitions(ctx context.Context) ([]MetricDefinition, error) { + if store, ok := s.store.(metricCatalogStore); ok { + return store.MetricDefinitions(ctx) + } + return metricDefinitions(), nil +} + +func (s *Service) MetricCatalog(ctx context.Context) (MetricCatalog, error) { + definitions, err := s.metricDefinitions(ctx) + if err != nil { + return MetricCatalog{}, err + } + return MetricCatalog{Metrics: definitions, AsOf: time.Now().UTC().Format(time.RFC3339)}, nil +} + +const latestTelemetryStaleAfter = 5 * time.Minute + +type latestTelemetryRawResult struct { + page Page[RawFrameRow] + err error +} + +type latestTelemetryCatalogResult struct { + definitions []MetricDefinition + err error +} + +// LatestTelemetry keeps the latest scalar value for every unified metric or +// source field. RAW and catalog reads are independent, so they run in parallel. +func (s *Service) LatestTelemetry(ctx context.Context, vehicleKey string) (LatestTelemetryResponse, error) { + vehicleKey = strings.TrimSpace(vehicleKey) + if vehicleKey == "" { + return LatestTelemetryResponse{}, clientError{Code: "VEHICLE_KEY_REQUIRED", Message: "最新遥测查询需要车牌、VIN 或终端标识"} + } + resolvedVIN, err := s.resolveVehicleVIN(ctx, vehicleKey, "") + if err != nil { + return LatestTelemetryResponse{}, err + } + rawResult := make(chan latestTelemetryRawResult, len(canonicalVehicleProtocols)) + catalogResult := make(chan latestTelemetryCatalogResult, 1) + for _, protocol := range canonicalVehicleProtocols { + go func(protocol string) { + page, queryErr := s.store.RawFrames(ctx, RawFrameQuery{VIN: resolvedVIN, Protocol: protocol, IncludeFields: true, Limit: 5, SkipCount: true}) + rawResult <- latestTelemetryRawResult{page: page, err: queryErr} + }(protocol) + } + go func() { + definitions, err := s.metricDefinitions(ctx) + catalogResult <- latestTelemetryCatalogResult{definitions: definitions, err: err} + }() + frames := make([]RawFrameRow, 0, len(canonicalVehicleProtocols)*5) + for range canonicalVehicleProtocols { + raw := <-rawResult + if raw.err != nil { + return LatestTelemetryResponse{}, raw.err + } + frames = append(frames, raw.page.Items...) + } + catalog := <-catalogResult + if catalog.err != nil { + return LatestTelemetryResponse{}, catalog.err + } + return buildLatestTelemetryResponse(resolvedVIN, frames, catalog.definitions, time.Now()), nil +} + +type latestTelemetryCandidate struct { + value LatestTelemetryValue + sortTime time.Time + timeValid bool +} + +func buildLatestTelemetryResponse(vehicleKey string, frames []RawFrameRow, definitions []MetricDefinition, now time.Time) LatestTelemetryResponse { + asOf := now.UTC().Format(time.RFC3339) + response := LatestTelemetryResponse{ + VIN: strings.TrimSpace(vehicleKey), + Categories: []LatestTelemetryCategory{}, + Values: []LatestTelemetryValue{}, + AsOf: asOf, + StaleAfterSeconds: int64(latestTelemetryStaleAfter / time.Second), + ScannedFrames: len(frames), + } + definitionBySource := make(map[string]MetricDefinition, len(definitions)*2) + for _, definition := range definitions { + for protocol, sourceField := range definition.SourceFields { + definitionBySource[telemetrySourceLookupKey(protocol, sourceField)] = definition + } + } + candidates := make(map[string]latestTelemetryCandidate, 64) + for _, frame := range frames { + if response.VIN == vehicleKey && strings.TrimSpace(frame.VIN) != "" { + response.VIN = strings.TrimSpace(frame.VIN) + } + observed, observedOK := latestTelemetryObservedTime(frame) + quality, qualityReason, freshness, delay := latestTelemetryQuality(frame, now) + normalizedProtocol := strings.ToUpper(strings.TrimSpace(frame.Protocol)) + sourceLookupPrefix := normalizedProtocol + "\x00" + for sourceField, rawValue := range frame.ParsedFields { + if !isTelemetryScalar(rawValue) { + continue + } + definition, unified := definitionBySource[sourceLookupPrefix+strings.TrimSpace(sourceField)] + key := sourceField + if unified { + key = definition.Key + } + if previous, exists := candidates[key]; exists && (!observedOK || (previous.timeValid && !observed.After(previous.sortTime))) { + continue + } + label, unit := latestTelemetryRawMetadata(sourceField) + category := telemetrySourceCategory(sourceField) + valueType := "dynamic" + description := "" + value := rawValue + if unified { + label = definition.Label + unit = definition.Unit + category = normalizeTelemetryCategory(definition.Category) + valueType = definition.ValueType + description = definition.Description + value = normalizeTelemetryValue(rawValue, definition.ValueType) + } + candidates[key] = latestTelemetryCandidate{ + value: LatestTelemetryValue{ + Key: key, SourceField: sourceField, Label: label, Description: description, Unit: unit, + Category: category, ValueType: valueType, Value: value, Protocol: normalizedProtocol, + SourceEndpoint: frame.SourceEndpoint, FrameID: frame.ID, DeviceTime: frame.DeviceTime, ServerTime: frame.ServerTime, + Quality: quality, QualityReason: qualityReason, FreshnessSeconds: freshness, DataDelaySeconds: delay, + }, + sortTime: observed, timeValid: observedOK, + } + } + } + for _, candidate := range candidates { + response.Values = append(response.Values, candidate.value) + } + sort.Slice(response.Values, func(i, j int) bool { + left, right := telemetryCategoryRank(response.Values[i].Category), telemetryCategoryRank(response.Values[j].Category) + if left != right { + return left < right + } + if response.Values[i].Label != response.Values[j].Label { + return response.Values[i].Label < response.Values[j].Label + } + return response.Values[i].Key < response.Values[j].Key + }) + categoryIndexes := map[string]int{} + for _, value := range response.Values { + index, exists := categoryIndexes[value.Category] + if !exists { + categoryIndexes[value.Category] = len(response.Categories) + response.Categories = append(response.Categories, LatestTelemetryCategory{Key: value.Category, Label: telemetryCategoryLabel(value.Category)}) + index = len(response.Categories) - 1 + } + response.Categories[index].Count++ + } + response.Evidence = fmt.Sprintf("scanned %d newest RAW frames; selected %d newest scalar values by unified metric/source field", len(frames), len(response.Values)) + return response +} + +func telemetrySourceLookupKey(protocol, sourceField string) string { + return strings.ToUpper(strings.TrimSpace(protocol)) + "\x00" + strings.TrimSpace(sourceField) +} + +func latestTelemetryObservedTime(frame RawFrameRow) (time.Time, bool) { + if parsed, ok := parseLatestTelemetryTime(frame.ServerTime); ok { + return parsed, true + } + return parseLatestTelemetryTime(frame.DeviceTime) +} + +func parseLatestTelemetryTime(value string) (time.Time, bool) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed, true + } + } + shanghai := time.FixedZone("Asia/Shanghai", 8*60*60) + for _, layout := range []string{"2006-01-02 15:04:05.999999999", "2006-01-02 15:04:05", "2006-01-02T15:04:05.999999999", "2006-01-02T15:04:05"} { + if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil { + return parsed, true + } + } + return time.Time{}, false +} + +func latestTelemetryQuality(frame RawFrameRow, now time.Time) (string, string, int64, *int64) { + serverTime, serverOK := parseLatestTelemetryTime(frame.ServerTime) + deviceTime, deviceOK := parseLatestTelemetryTime(frame.DeviceTime) + var delay *int64 + if serverOK && deviceOK { + seconds := int64(math.Abs(serverTime.Sub(deviceTime).Seconds())) + delay = &seconds + } + if status := strings.TrimSpace(frame.ParseStatus); status != "" && !strings.EqualFold(status, "ok") { + reason := "原始帧解析状态为 " + status + if detail := strings.TrimSpace(frame.ParseError); detail != "" { + reason += "：" + detail + } + return "warning", reason, telemetryFreshnessSeconds(serverTime, serverOK, now), delay + } + if !serverOK { + return "warning", "缺少可解析的服务接收时间", 0, delay + } + if serverTime.After(now.Add(latestTelemetryStaleAfter)) { + return "warning", "服务接收时间晚于平台当前时间超过 5 分钟", 0, delay + } + freshness := telemetryFreshnessSeconds(serverTime, true, now) + if delay != nil && *delay > int64(latestTelemetryStaleAfter/time.Second) { + return "warning", "设备时间与服务接收时间相差超过 5 分钟", freshness, delay + } + if freshness > int64(latestTelemetryStaleAfter/time.Second) { + return "stale", "最新值距平台当前时间超过 5 分钟", freshness, delay + } + return "good", "解析成功且接收时间在 5 分钟内", freshness, delay +} + +func telemetryFreshnessSeconds(serverTime time.Time, valid bool, now time.Time) int64 { + if !valid || serverTime.After(now) { + return 0 + } + return int64(now.Sub(serverTime).Seconds()) +} + +func isTelemetryScalar(value any) bool { + switch value.(type) { + case nil, map[string]any, []any: + return false + case string, bool, float64, float32, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, json.Number: + return true + default: + return false + } +} + +func normalizeTelemetryValue(value any, valueType string) any { + if valueType != "boolean" { + return value + } + switch typed := value.(type) { + case bool: + return typed + case float64: + return typed != 0 + case json.Number: + number, err := typed.Float64() + return err == nil && number != 0 + case string: + return strings.EqualFold(typed, "true") || typed == "1" + default: + return value + } +} + +func latestTelemetryRawMetadata(sourceField string) (string, string) { + label, unit := rawMetricMetadata(sourceField) + protocol := strings.ToUpper(strings.Split(sourceField, ".")[0]) + label = strings.TrimSuffix(label, " · "+protocol) + return label, unit +} + +func normalizeTelemetryCategory(category string) string { + switch strings.ToLower(strings.TrimSpace(category)) { + case "driving", "vehicle": + return "vehicle" + case "energy", "battery": + return "battery" + case "safety", "alarm": + return "alarm" + case "location": + return "location" + case "quality": + return "quality" + default: + return "extension" + } +} + +func telemetrySourceCategory(sourceField string) string { + normalized := strings.ToLower(sourceField) + switch { + case strings.Contains(normalized, "alarm") || strings.Contains(normalized, "fault") || strings.Contains(normalized, "warning"): + return "alarm" + case strings.Contains(normalized, "fuel_cell") || strings.Contains(normalized, "fuelcell") || strings.Contains(normalized, "fc_stack"): + return "fuel-cell" + case strings.Contains(normalized, "motor") || strings.Contains(normalized, "engine") || strings.Contains(normalized, "rpm"): + return "motor" + case strings.Contains(normalized, "battery") || strings.Contains(normalized, "soc") || strings.Contains(normalized, "cell_") || strings.Contains(normalized, "voltage") || strings.Contains(normalized, "current"): + return "battery" + case strings.Contains(normalized, "location") || strings.Contains(normalized, "longitude") || strings.Contains(normalized, "latitude") || strings.Contains(normalized, "direction") || strings.Contains(normalized, "altitude"): + return "location" + case strings.Contains(normalized, ".vehicle.") || strings.HasPrefix(normalized, "vehicle.") || strings.Contains(normalized, "speed") || strings.Contains(normalized, "mileage") || strings.Contains(normalized, "gear") || strings.Contains(normalized, "brake") || strings.Contains(normalized, "accelerator"): + return "vehicle" + default: + return "extension" + } +} + +func telemetryCategoryRank(category string) int { + for index, key := range []string{"vehicle", "motor", "fuel-cell", "battery", "location", "alarm", "quality", "extension"} { + if category == key { + return index + } + } + return 99 +} + +func telemetryCategoryLabel(category string) string { + labels := map[string]string{"vehicle": "整车数据", "motor": "驱动电机", "fuel-cell": "燃料电池", "battery": "电池", "location": "定位", "alarm": "报警", "quality": "数据质量", "extension": "扩展字段"} + if label := labels[category]; label != "" { + return label + } + return category +} + +func metricDefinitions() []MetricDefinition { + allProtocols := append([]string(nil), canonicalVehicleProtocols...) + return []MetricDefinition{ + {Key: "speed_kmh", Label: "速度", Description: "车辆最新行驶速度", Unit: "km/h", Category: "driving", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"GB32960": "gb32960.vehicle.speed_kmh", "JT808": "jt808.location.speed_kmh", "YUTONG_MQTT": "yutong_mqtt.data.meter_speed"}, Searchable: true, Chartable: true, Alertable: true}, + {Key: "soc_percent", Label: "SOC", Description: "动力电池剩余电量百分比", Unit: "%", Category: "energy", ValueType: "numeric", Protocols: []string{"GB32960", "YUTONG_MQTT"}, SourceFields: map[string]string{"GB32960": "gb32960.vehicle.soc_percent", "YUTONG_MQTT": "yutong_mqtt.data.battery_capacity_soc"}, Searchable: true, Chartable: true, Alertable: true}, + {Key: "alarm_active", Label: "协议告警位", Description: "协议报文是否携带活动告警", Unit: "", Category: "safety", ValueType: "boolean", Protocols: []string{"GB32960", "JT808"}, SourceFields: map[string]string{"GB32960": "gb32960.alarm.general_alarm_flag", "JT808": "jt808.location.alarm_flag"}, Searchable: true, Chartable: false, Alertable: true}, + {Key: "freshness_sec", Label: "离线时长", Description: "最新数据距当前时间的秒数", Unit: "s", Category: "quality", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"PLATFORM": "vehicle_realtime_location.updated_at"}, Searchable: true, Chartable: true, Alertable: true}, + {Key: "data_delay_sec", Label: "数据延迟", Description: "服务接收时间与设备事件时间的差值", Unit: "s", Category: "quality", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"PLATFORM": "received_at-event_time"}, Searchable: true, Chartable: true, Alertable: true}, + {Key: "total_mileage_km", Label: "总里程", Description: "车辆累计行驶里程", Unit: "km", Category: "driving", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"GB32960": "gb32960.vehicle.total_mileage_km", "JT808": "jt808.location.total_mileage_km", "YUTONG_MQTT": "yutong.vehicle.total_mileage_km"}, Searchable: true, Chartable: true, Alertable: false}, + {Key: "longitude", Label: "经度", Description: "WGS84/协议归一化经度", Unit: "°", Category: "location", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"PLATFORM": "vehicle_realtime_location.longitude"}, Searchable: true, Chartable: false, Alertable: false}, + {Key: "latitude", Label: "纬度", Description: "WGS84/协议归一化纬度", Unit: "°", Category: "location", ValueType: "numeric", Protocols: allProtocols, SourceFields: map[string]string{"PLATFORM": "vehicle_realtime_location.latitude"}, Searchable: true, Chartable: false, Alertable: false}, + } +} + +func (s *Service) HistoryData(ctx context.Context, query url.Values) (HistoryDataResponse, error) { + started := time.Now() + category := strings.ToLower(strings.TrimSpace(query.Get("category"))) + if category == "" { + category = "location" + } + if category != "location" && category != "raw" && category != "mileage" { + return HistoryDataResponse{}, clientError{Code: "HISTORY_CATEGORY_INVALID", Message: "历史数据类型仅支持 location、raw 或 mileage"} + } + keywords := historyQueryKeywords(query) + if len(keywords) == 0 { + return HistoryDataResponse{}, clientError{Code: "VEHICLE_KEY_REQUIRED", Message: "历史查询至少需要一个车牌、VIN 或终端标识"} + } + if len(keywords) > 5 { + return HistoryDataResponse{}, clientError{Code: "VEHICLE_LIMIT_EXCEEDED", Message: "单次历史查询最多支持 5 台车辆"} + } + limit := parsePositive(query.Get("limit"), 50) + if limit > 200 { + limit = 200 + } + offset := parsePositive(query.Get("offset"), 0) + fetchLimit := limit + offset + if fetchLimit > 1000 { + fetchLimit = 1000 + } + rows := make([]HistoryDataRow, 0, fetchLimit*len(keywords)) + total := 0 + for _, keyword := range keywords { + vehicleRows, vehicleTotal, err := s.historyDataForVehicle(ctx, category, keyword, query, fetchLimit) + if err != nil { + return HistoryDataResponse{}, err + } + rows = append(rows, vehicleRows...) + total += vehicleTotal + } + sort.SliceStable(rows, func(i, j int) bool { + if rows[i].DeviceTime == rows[j].DeviceTime { + return rows[i].VIN < rows[j].VIN + } + return rows[i].DeviceTime > rows[j].DeviceTime + }) + if offset > len(rows) { + offset = len(rows) + } + end := offset + limit + if end > len(rows) { + end = len(rows) + } + pageRows := append([]HistoryDataRow(nil), rows[offset:end]...) + columns := historyMetricsForCategory(category) + if category == "raw" { + columns = mergeDiscoveredRawMetrics(columns, pageRows) + } + vehicles := map[string]struct{}{} + sources := map[string]struct{}{} + for _, row := range rows { + vehicles[row.VIN] = struct{}{} + sources[row.Protocol] = struct{}{} + } + sourceNames := make([]string, 0, len(sources)) + for source := range sources { + if source != "" { + sourceNames = append(sourceNames, source) + } + } + sort.Strings(sourceNames) + return HistoryDataResponse{ + Category: category, Columns: columns, Rows: pageRows, Total: total, Limit: limit, Offset: offset, + Summary: HistoryDataSummary{ResultRows: total, VehicleCount: len(vehicles), Sources: sourceNames, QueryDuration: time.Since(started).Milliseconds()}, + AsOf: time.Now().UTC().Format(time.RFC3339), + }, nil +} + +type historyLocationSeriesStore interface { + HistoryLocationSeries(context.Context, HistoryLocationSeriesQuery) ([]HistoryLocationSeriesBucket, error) +} + +type historySeriesMetricSpec struct { + definition HistoryMetricDefinition + aggregation string + value func(HistoryLocationSeriesBucket) *float64 + minimum func(HistoryLocationSeriesBucket) *float64 + maximum func(HistoryLocationSeriesBucket) *float64 +} + +func historySeriesMetricSpecs() map[string]historySeriesMetricSpec { + return map[string]historySeriesMetricSpec{ + "speedKmh": { + definition: HistoryMetricDefinition{Key: "speedKmh", Label: "速度", Unit: "km/h", Category: "location", ValueType: "number", DefaultVisible: true}, + aggregation: "avg", value: func(row HistoryLocationSeriesBucket) *float64 { return row.SpeedAverage }, + minimum: func(row HistoryLocationSeriesBucket) *float64 { return row.SpeedMinimum }, maximum: func(row HistoryLocationSeriesBucket) *float64 { return row.SpeedMaximum }, + }, + "totalMileageKm": { + definition: HistoryMetricDefinition{Key: "totalMileageKm", Label: "总里程", Unit: "km", Category: "location", ValueType: "number", DefaultVisible: true}, + aggregation: "last", value: func(row HistoryLocationSeriesBucket) *float64 { return row.MileageLast }, + minimum: func(row HistoryLocationSeriesBucket) *float64 { return row.MileageMinimum }, maximum: func(row HistoryLocationSeriesBucket) *float64 { return row.MileageMaximum }, + }, + } +} + +func (s *Service) HistorySeries(ctx context.Context, query url.Values) (HistorySeriesResponse, error) { + started := time.Now() + keywords := historyQueryKeywords(query) + if len(keywords) == 0 { + return HistorySeriesResponse{}, clientError{Code: "VEHICLE_KEY_REQUIRED", Message: "趋势查询至少需要一个车牌、VIN 或终端标识"} + } + if len(keywords) > 5 { + return HistorySeriesResponse{}, clientError{Code: "VEHICLE_LIMIT_EXCEEDED", Message: "单次趋势查询最多支持 5 台车辆"} + } + category := strings.ToLower(strings.TrimSpace(query.Get("category"))) + if category == "" { + category = "location" + } + if category != "location" { + return HistorySeriesResponse{}, clientError{Code: "HISTORY_SERIES_CATEGORY_UNSUPPORTED", Message: "当前仅位置数据支持聚合趋势，原始报文和日里程请查看明细"} + } + dateFrom, dateTo, start, end, err := historySeriesWindow(query.Get("dateFrom"), query.Get("dateTo"), time.Now()) + if err != nil { + return HistorySeriesResponse{}, err + } + targetPoints := parsePositive(query.Get("targetPoints"), 240) + if targetPoints < 60 { + targetPoints = 60 + } + if targetPoints > 600 { + targetPoints = 600 + } + grainSeconds := historySeriesGrain(end.Sub(start), targetPoints) + specs := historySeriesMetricSpecs() + metricKeys := historySeriesMetrics(query.Get("metrics"), specs) + definitions := make([]HistoryMetricDefinition, 0, len(metricKeys)) + for _, key := range metricKeys { + definitions = append(definitions, specs[key].definition) + } + store, ok := s.store.(historyLocationSeriesStore) + if !ok { + return HistorySeriesResponse{}, fmt.Errorf("history series aggregation is unavailable") + } + protocol := strings.TrimSpace(query.Get("protocol")) + type vehicleBuckets struct { + vin, plate string + buckets []HistoryLocationSeriesBucket + } + vehicles := make([]vehicleBuckets, 0, len(keywords)) + for _, keyword := range keywords { + vin, resolveErr := s.resolveVehicleVIN(ctx, keyword, protocol) + if resolveErr != nil { + return HistorySeriesResponse{}, resolveErr + } + buckets, aggregateErr := store.HistoryLocationSeries(ctx, HistoryLocationSeriesQuery{VIN: vin, Protocol: protocol, DateFrom: dateFrom, DateTo: dateTo, GrainSeconds: grainSeconds}) + if aggregateErr != nil { + return HistorySeriesResponse{}, aggregateErr + } + vehicles = append(vehicles, vehicleBuckets{vin: vin, plate: s.historySeriesPlate(ctx, vin), buckets: buckets}) + } + seriesByKey := map[string]*HistorySeries{} + seriesOrder := make([]string, 0) + groupBuckets := map[string]int{} + var rawPointCount int64 + for _, vehicle := range vehicles { + for _, bucket := range vehicle.buckets { + groupKey := vehicle.vin + "\x00" + bucket.Protocol + groupBuckets[groupKey]++ + rawPointCount += bucket.Count + for _, metricKey := range metricKeys { + spec := specs[metricKey] + key := groupKey + "\x00" + metricKey + series := seriesByKey[key] + if series == nil { + series = &HistorySeries{VIN: vehicle.vin, Plate: vehicle.plate, Protocol: bucket.Protocol, Metric: metricKey, Label: spec.definition.Label, Unit: spec.definition.Unit, Aggregation: spec.aggregation, Points: []HistorySeriesPoint{}} + seriesByKey[key] = series + seriesOrder = append(seriesOrder, key) + } + series.Points = append(series.Points, HistorySeriesPoint{Time: historySeriesPointTime(bucket.Time), Value: spec.value(bucket), Min: spec.minimum(bucket), Max: spec.maximum(bucket), Count: bucket.Count}) + } + } + } + series := make([]HistorySeries, 0, len(seriesOrder)) + returnedPoints := 0 + for _, key := range seriesOrder { + current := *seriesByKey[key] + returnedPoints += len(current.Points) + series = append(series, current) + } + windowBuckets := int(math.Ceil(end.Sub(start).Seconds() / float64(grainSeconds))) + groupCount := len(groupBuckets) + if groupCount == 0 { + groupCount = len(vehicles) + } + expectedBuckets := windowBuckets * groupCount + actualBuckets := 0 + for _, count := range groupBuckets { + actualBuckets += count + } + missingBuckets := expectedBuckets - actualBuckets + if missingBuckets < 0 { + missingBuckets = 0 + } + evidence := fmt.Sprintf("TDengine INTERVAL(%ds)，按车辆与协议分区；空窗不补值", grainSeconds) + return HistorySeriesResponse{Metrics: definitions, Series: series, DateFrom: start.UTC().Format(time.RFC3339), DateTo: end.UTC().Format(time.RFC3339), AsOf: time.Now().UTC().Format(time.RFC3339), Summary: HistorySeriesSummary{RawPointCount: rawPointCount, BucketCount: actualBuckets, ReturnedPointCount: returnedPoints, SeriesCount: len(series), GrainSeconds: grainSeconds, TargetPoints: targetPoints, ExpectedBucketCount: expectedBuckets, MissingBucketCount: missingBuckets, QueryDuration: time.Since(started).Milliseconds(), Complete: missingBuckets == 0, Evidence: evidence}}, nil +} + +func historySeriesMetrics(raw string, specs map[string]historySeriesMetricSpec) []string { + requested := strings.FieldsFunc(raw, func(char rune) bool { return char == ',' || char == ';' }) + if len(requested) == 0 { + requested = []string{"speedKmh", "totalMileageKm"} + } + result := make([]string, 0, len(requested)) + seen := map[string]bool{} + for _, key := range requested { + key = strings.TrimSpace(key) + if _, ok := specs[key]; ok && !seen[key] { + result = append(result, key) + seen[key] = true + } + } + if len(result) == 0 { + return []string{"speedKmh", "totalMileageKm"} + } + return result +} + +func historySeriesWindow(rawFrom, rawTo string, now time.Time) (string, string, time.Time, time.Time, error) { + shanghai := time.FixedZone("Asia/Shanghai", 8*60*60) + localNow := now.In(shanghai) + if strings.TrimSpace(rawFrom) == "" { + rawFrom = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, shanghai).Format("2006-01-02T15:04") + } + if strings.TrimSpace(rawTo) == "" { + rawTo = localNow.Format("2006-01-02T15:04") + } + start, startOK := parseTrackRequestTime(rawFrom) + end, endOK := parseTrackRequestTime(rawTo) + if !startOK || !endOK { + return "", "", time.Time{}, time.Time{}, clientError{Code: "HISTORY_TIME_INVALID", Message: "趋势时间格式无效"} + } + if !end.After(start) { + return "", "", time.Time{}, time.Time{}, clientError{Code: "HISTORY_TIME_ORDER_INVALID", Message: "趋势结束时间必须晚于开始时间"} + } + if end.Sub(start) > 31*24*time.Hour { + return "", "", time.Time{}, time.Time{}, clientError{Code: "HISTORY_TIME_RANGE_TOO_LARGE", Message: "单次趋势分析最长支持 31 天"} + } + return rawFrom, rawTo, start, end, nil +} + +func historySeriesGrain(duration time.Duration, targetPoints int) int { + required := int(math.Ceil(duration.Seconds() / float64(targetPoints))) + for _, grain := range []int{1, 5, 10, 30, 60, 300, 900, 1800, 3600, 21600, 86400} { + if required <= grain { + return grain + } + } + return 86400 +} + +func historySeriesPointTime(value string) string { + if parsed, ok := parseVehicleServiceTime(value); ok { + return parsed.UTC().Format(time.RFC3339) + } + return value +} + +func (s *Service) historySeriesPlate(ctx context.Context, vin string) string { + page, err := s.store.Vehicles(ctx, url.Values{"keyword": {vin}, "limit": {"1"}}) + if err == nil && len(page.Items) > 0 { + return page.Items[0].Plate + } + return "" +} + +func historyQueryKeywords(query url.Values) []string { + raw := firstNonEmpty(query.Get("keywords"), query.Get("keyword"), query.Get("vin")) + parts := strings.FieldsFunc(raw, func(char rune) bool { return char == ',' || char == ';' || char == '\n' }) + result := make([]string, 0, len(parts)) + seen := map[string]struct{}{} + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + key := strings.ToLower(part) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, part) + } + return result +} + +func (s *Service) historyDataForVehicle(ctx context.Context, category, keyword string, query url.Values, fetchLimit int) ([]HistoryDataRow, int, error) { + protocol := strings.TrimSpace(query.Get("protocol")) + switch category { + case "location": + resolved := url.Values{"keyword": {keyword}, "limit": {strconv.Itoa(fetchLimit)}, "offset": {"0"}} + copyHistoryFilters(resolved, query, protocol) + resolvedQuery, err := s.resolveVehicleQuery(ctx, resolved) + if err != nil { + return nil, 0, err + } + page, err := s.store.HistoryLocationsFromTDengine(ctx, resolvedQuery) + if err != nil { + return nil, 0, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for index, item := range page.Items { + quality := "normal" + if !validTrackCoordinate(item) { + quality = "warning" + } + rows = append(rows, HistoryDataRow{ID: "location-" + item.VIN + "-" + strconv.Itoa(index) + "-" + item.DeviceTime, VIN: item.VIN, Plate: item.Plate, Protocol: item.Protocol, DeviceTime: item.DeviceTime, ServerTime: item.ServerTime, Quality: quality, Values: map[string]any{"speedKmh": item.SpeedKmh, "totalMileageKm": item.TotalMileageKm, "longitude": item.Longitude, "latitude": item.Latitude}}) + } + return rows, page.Total, nil + case "raw": + resolvedVIN, err := s.resolveVehicleVIN(ctx, keyword, protocol) + if err != nil { + return nil, 0, err + } + page, err := s.store.RawFrames(ctx, RawFrameQuery{VIN: resolvedVIN, Protocol: protocol, DateFrom: query.Get("dateFrom"), DateTo: query.Get("dateTo"), IncludeFields: true, Limit: fetchLimit}) + if err != nil { + return nil, 0, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for _, item := range page.Items { + values := map[string]any{"frameType": item.FrameType, "rawSizeBytes": item.RawSizeBytes} + for key, value := range item.ParsedFields { + values[key] = value + } + rows = append(rows, HistoryDataRow{ID: item.ID, VIN: item.VIN, Plate: item.Plate, Protocol: item.Protocol, DeviceTime: item.DeviceTime, ServerTime: item.ServerTime, Quality: "normal", EvidenceID: item.ID, Values: values}) + } + return rows, page.Total, nil + case "mileage": + resolved := url.Values{"keyword": {keyword}, "limit": {strconv.Itoa(fetchLimit)}, "offset": {"0"}} + copyHistoryFilters(resolved, query, protocol) + resolvedQuery, err := s.resolveVehicleQuery(ctx, resolved) + if err != nil { + return nil, 0, err + } + page, err := s.store.DailyMileage(ctx, resolvedQuery) + if err != nil { + return nil, 0, err + } + rows := make([]HistoryDataRow, 0, len(page.Items)) + for index, item := range page.Items { + quality := "normal" + if item.AnomalySeverity != "" { + quality = item.AnomalySeverity + } + rows = append(rows, HistoryDataRow{ID: "mileage-" + item.VIN + "-" + strconv.Itoa(index) + "-" + item.Date, VIN: item.VIN, Plate: item.Plate, Protocol: item.Source, DeviceTime: item.Date, ServerTime: item.Date, Quality: quality, Values: map[string]any{"startMileageKm": item.StartMileageKm, "endMileageKm": item.EndMileageKm, "dailyMileageKm": item.DailyMileageKm}}) + } + return rows, page.Total, nil + } + return nil, 0, nil +} + +func copyHistoryFilters(target, source url.Values, protocol string) { + if protocol != "" { + target.Set("protocol", protocol) + } + if value := strings.TrimSpace(source.Get("dateFrom")); value != "" { + target.Set("dateFrom", value) + } + if value := strings.TrimSpace(source.Get("dateTo")); value != "" { + target.Set("dateTo", value) + } +} + +func historyLocationMetrics() []HistoryMetricDefinition { + return []HistoryMetricDefinition{ + {Key: "speedKmh", Label: "速度", Unit: "km/h", Category: "location", ValueType: "number", DefaultVisible: true}, + {Key: "totalMileageKm", Label: "总里程", Unit: "km", Category: "location", ValueType: "number", DefaultVisible: true}, + {Key: "longitude", Label: "经度", Unit: "°", Category: "location", ValueType: "number", DefaultVisible: true}, + {Key: "latitude", Label: "纬度", Unit: "°", Category: "location", ValueType: "number", DefaultVisible: true}, + } +} + +func historyRawMetrics() []HistoryMetricDefinition { + return []HistoryMetricDefinition{ + {Key: "frameType", Label: "报文类型", Category: "raw", ValueType: "string", DefaultVisible: true}, + {Key: "rawSizeBytes", Label: "报文大小", Unit: "B", Category: "raw", ValueType: "number", DefaultVisible: true}, + } +} + +func historyMileageMetrics() []HistoryMetricDefinition { + return []HistoryMetricDefinition{ + {Key: "startMileageKm", Label: "起始里程", Unit: "km", Category: "mileage", ValueType: "number", DefaultVisible: true}, + {Key: "endMileageKm", Label: "结束里程", Unit: "km", Category: "mileage", ValueType: "number", DefaultVisible: true}, + {Key: "dailyMileageKm", Label: "日里程", Unit: "km", Category: "mileage", ValueType: "number", DefaultVisible: true}, + } +} + +func historyMetricsForCategory(category string) []HistoryMetricDefinition { + switch category { + case "raw": + return historyRawMetrics() + case "mileage": + return historyMileageMetrics() + default: + return historyLocationMetrics() + } +} + +func mergeDiscoveredRawMetrics(base []HistoryMetricDefinition, rows []HistoryDataRow) []HistoryMetricDefinition { + result := append([]HistoryMetricDefinition(nil), base...) + seen := map[string]struct{}{} + for _, metric := range result { + seen[metric.Key] = struct{}{} + } + keys := make([]string, 0) + for _, row := range rows { + for key := range row.Values { + if _, ok := seen[key]; !ok { + seen[key] = struct{}{} + keys = append(keys, key) + } + } + } + sort.Strings(keys) + for _, key := range keys { + label, unit := rawMetricMetadata(key) + result = append(result, HistoryMetricDefinition{Key: key, Label: label, Unit: unit, Category: "raw", ValueType: "dynamic", DefaultVisible: len(result) < 7}) + } + return result +} + +func rawMetricMetadata(key string) (string, string) { + parts := strings.Split(key, ".") + tail := parts[len(parts)-1] + protocol := strings.ToUpper(parts[0]) + labels := map[string]string{"speed_kmh": "速度", "total_mileage_km": "总里程", "avg_voltage_v": "平均电压", "soc_percent": "SOC", "longitude": "经度", "latitude": "纬度", "direction": "方向", "current_a": "电流", "temperature_c": "温度"} + label := labels[tail] + if label == "" { + label = strings.ReplaceAll(tail, "_", " ") + } + if protocol != "" { + label += " · " + protocol + } + unit := "" + switch { + case strings.HasSuffix(tail, "_kmh"): + unit = "km/h" + case strings.HasSuffix(tail, "_km"): + unit = "km" + case strings.HasSuffix(tail, "_percent"): + unit = "%" + case strings.HasSuffix(tail, "_voltage_v") || strings.HasSuffix(tail, "_v"): + unit = "V" + case strings.HasSuffix(tail, "_current_a") || strings.HasSuffix(tail, "_a"): + unit = "A" + case strings.HasSuffix(tail, "_temperature_c") || strings.HasSuffix(tail, "_c"): + unit = "℃" + } + return label, unit +} + +func (s *Service) CreateHistoryExport(request HistoryExportRequest) (HistoryExportJob, error) { + request.Keywords = normalizedKeywords(request.Keywords) + if len(request.Keywords) == 0 || len(request.Keywords) > 5 { + return HistoryExportJob{}, clientError{Code: "EXPORT_SCOPE_INVALID", Message: "导出任务需要 1 至 5 台车辆"} + } + request.Format = strings.ToLower(strings.TrimSpace(request.Format)) + if request.Format == "" { + request.Format = "csv" + } + if request.Format != "csv" { + return HistoryExportJob{}, clientError{Code: "EXPORT_FORMAT_INVALID", Message: "当前版本的受控导出仅支持 CSV"} + } + if request.Category == "" { + request.Category = "location" + } + request.Category = strings.ToLower(strings.TrimSpace(request.Category)) + if request.Category != "location" && request.Category != "raw" && request.Category != "mileage" { + return HistoryExportJob{}, clientError{Code: "EXPORT_CATEGORY_INVALID", Message: "导出类型仅支持 location、raw 或 mileage"} + } + dateFrom, dateTo, _, _, windowErr := historySeriesWindow(request.DateFrom, request.DateTo, time.Now()) + if windowErr != nil { + return HistoryExportJob{}, windowErr + } + request.DateFrom, request.DateTo = dateFrom, dateTo + request.Metrics = normalizedKeywords(request.Metrics) + if len(request.Metrics) > 32 { + return HistoryExportJob{}, clientError{Code: "EXPORT_METRIC_LIMIT_EXCEEDED", Message: "单次导出最多支持 32 个指标"} + } + identifier, err := randomExportID() + if err != nil { + return HistoryExportJob{}, err + } + now := time.Now().UTC().Format(time.RFC3339) + job := &HistoryExportJob{ID: identifier, Name: "历史数据_" + time.Now().Format("20060102_150405"), Status: "queued", Progress: 0, Format: request.Format, Category: request.Category, Keywords: append([]string(nil), request.Keywords...), CreatedAt: now, UpdatedAt: now, Evidence: "单并发流式任务；最多 1,000,000 行；完成文件原子发布"} + s.exportsMu.Lock() + s.exports[job.ID] = job + if err := s.persistHistoryExportsLocked(); err != nil { + delete(s.exports, job.ID) + s.exportsMu.Unlock() + return HistoryExportJob{}, fmt.Errorf("persist export job: %w", err) + } + s.exportsMu.Unlock() + go s.runHistoryExport(job.ID, request) + return copyHistoryExportJob(job), nil +} + +func (s *Service) ListHistoryExports() []HistoryExportJob { + s.exportsMu.RLock() + result := make([]HistoryExportJob, 0, len(s.exports)) + for _, job := range s.exports { + result = append(result, copyHistoryExportJob(job)) + } + s.exportsMu.RUnlock() + sort.Slice(result, func(i, j int) bool { return result[i].CreatedAt > result[j].CreatedAt }) + if len(result) > 20 { + result = result[:20] + } + return result +} + +func (s *Service) HistoryExportFile(id string) (string, string, error) { + s.exportsMu.RLock() + job := s.exports[id] + if job == nil { + s.exportsMu.RUnlock() + return "", "", clientError{Code: "EXPORT_NOT_FOUND", Message: "导出任务不存在"} + } + copy := copyHistoryExportJob(job) + path := job.filePath + s.exportsMu.RUnlock() + if copy.Status != "completed" || path == "" { + return "", "", clientError{Code: "EXPORT_NOT_READY", Message: "导出文件尚未生成"} + } + return path, copy.Name + ".csv", nil +} + +func (s *Service) runHistoryExport(id string, request HistoryExportRequest) { + s.exportSlots <- struct{}{} + defer func() { <-s.exportSlots }() + s.updateHistoryExport(id, func(job *HistoryExportJob) { job.Status = "running"; job.Progress = 1 }) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + store, ok := s.store.(interface { + HistoryExportCount(context.Context, HistoryExportStoreQuery) (int64, error) + HistoryExportBatch(context.Context, HistoryExportStoreQuery, HistoryExportCursor, int) ([]HistoryDataRow, HistoryExportCursor, error) + }) + if !ok { + s.failHistoryExport(id, errors.New("当前数据存储不支持流式历史导出")) + return + } + resolvedVINs := make([]string, 0, len(request.Keywords)) + seenVINs := map[string]bool{} + for _, keyword := range request.Keywords { + vin, err := s.resolveVehicleVIN(ctx, keyword, request.Protocol) + if err != nil { + s.failHistoryExport(id, err) + return + } + if vin = strings.TrimSpace(vin); vin != "" && !seenVINs[vin] { + seenVINs[vin] = true + resolvedVINs = append(resolvedVINs, vin) + } + } + queries := make([]HistoryExportStoreQuery, 0, len(resolvedVINs)) + var totalRows int64 + for _, vin := range resolvedVINs { + query := HistoryExportStoreQuery{Category: request.Category, VIN: vin, Protocol: request.Protocol, DateFrom: request.DateFrom, DateTo: request.DateTo, Metrics: append([]string(nil), request.Metrics...)} + count, err := store.HistoryExportCount(ctx, query) + if err != nil { + s.failHistoryExport(id, err) + return + } + totalRows += count + if totalRows > 1_000_000 { + s.failHistoryExport(id, fmt.Errorf("结果 %d 行，超过单任务 1,000,000 行上限，请缩小车辆或时间范围", totalRows)) + return + } + queries = append(queries, query) + } + s.updateHistoryExport(id, func(job *HistoryExportJob) { job.TotalRows = totalRows; job.Progress = 3 }) + columns := historyExportColumns(request.Category, request.Metrics) + if err := os.MkdirAll(s.exportDir, 0o750); err != nil { + s.failHistoryExport(id, err) + return + } + path := filepath.Join(s.exportDir, id+".csv") + partPath := path + ".part" + _ = os.Remove(partPath) + file, err := os.OpenFile(partPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o640) + if err != nil { + s.failHistoryExport(id, err) + return + } + completed := false + defer func() { + if !completed { + _ = file.Close() + _ = os.Remove(partPath) + } + }() + if _, err := file.WriteString("\uFEFF"); err != nil { + s.failHistoryExport(id, err) + return + } + buffered := bufio.NewWriterSize(file, 1<<20) + writer := csv.NewWriter(buffered) + if err := writeHistoryExportMetadata(writer, request, columns); err != nil { + s.failHistoryExport(id, err) + return + } + var processed int64 + for _, query := range queries { + cursor := HistoryExportCursor{} + for { + rows, nextCursor, err := store.HistoryExportBatch(ctx, query, cursor, 5000) + if err != nil { + s.failHistoryExport(id, err) + return + } + if len(rows) == 0 { + break + } + if processed+int64(len(rows)) > 1_000_000 { + s.failHistoryExport(id, errors.New("导出期间数据增长后超过 1,000,000 行上限，任务已安全停止")) + return + } + for _, row := range rows { + if err := writer.Write(historyExportRecord(row, columns)); err != nil { + s.failHistoryExport(id, err) + return + } + } + writer.Flush() + if err := writer.Error(); err != nil { + s.failHistoryExport(id, err) + return + } + if err := buffered.Flush(); err != nil { + s.failHistoryExport(id, err) + return + } + processed += int64(len(rows)) + stat, _ := file.Stat() + progress := 95 + if totalRows > 0 { + progress = 3 + int(float64(processed)/float64(totalRows)*92) + if progress > 95 { + progress = 95 + } + } + s.updateHistoryExport(id, func(job *HistoryExportJob) { + job.ProcessedRows = processed + job.Progress = progress + if stat != nil { + job.FileSizeBytes = stat.Size() + } + }) + if nextCursor == cursor { + s.failHistoryExport(id, errors.New("导出游标未推进，任务已安全停止")) + return + } + cursor = nextCursor + } + } + writer.Flush() + writeErr := writer.Error() + bufferErr := buffered.Flush() + syncErr := file.Sync() + closeErr := file.Close() + if writeErr != nil { + s.failHistoryExport(id, writeErr) + return + } + if bufferErr != nil { + s.failHistoryExport(id, bufferErr) + return + } + if syncErr != nil { + s.failHistoryExport(id, syncErr) + return + } + if closeErr != nil { + s.failHistoryExport(id, closeErr) + return + } + if err := os.Rename(partPath, path); err != nil { + s.failHistoryExport(id, err) + return + } + completed = true + stat, _ := os.Stat(path) + completedAt := time.Now().UTC().Format(time.RFC3339) + s.updateHistoryExport(id, func(job *HistoryExportJob) { + job.Status = "completed" + job.Progress = 100 + job.RowCount = int(processed) + job.ProcessedRows = processed + job.TotalRows = totalRows + job.CompletedAt = completedAt + if stat != nil { + job.FileSizeBytes = stat.Size() + } + job.filePath = path + job.DownloadURL = "/api/v2/exports/" + job.ID + "/download" + }) +} + +func (s *Service) updateHistoryExport(id string, update func(*HistoryExportJob)) { + s.exportsMu.Lock() + defer s.exportsMu.Unlock() + if job := s.exports[id]; job != nil { + update(job) + job.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + _ = s.persistHistoryExportsLocked() + } +} + +func (s *Service) failHistoryExport(id string, err error) { + s.updateHistoryExport(id, func(job *HistoryExportJob) { job.Status = "failed"; job.Error = err.Error(); job.Progress = 0 }) +} + +func (s *Service) exportIndexPath() string { + return filepath.Join(s.exportDir, "jobs.json") +} + +func (s *Service) persistHistoryExportsLocked() error { + if err := os.MkdirAll(s.exportDir, 0o750); err != nil { + return err + } + jobs := make([]HistoryExportJob, 0, len(s.exports)) + for _, job := range s.exports { + jobs = append(jobs, copyHistoryExportJob(job)) + } + sort.Slice(jobs, func(i, j int) bool { return jobs[i].CreatedAt > jobs[j].CreatedAt }) + if len(jobs) > 100 { + jobs = jobs[:100] + } + contents, err := json.MarshalIndent(jobs, "", " ") + if err != nil { + return err + } + temporary := s.exportIndexPath() + ".tmp" + if err := os.WriteFile(temporary, contents, 0o640); err != nil { + return err + } + return os.Rename(temporary, s.exportIndexPath()) +} + +func (s *Service) loadHistoryExports() { + contents, err := os.ReadFile(s.exportIndexPath()) + if err != nil { + return + } + var jobs []HistoryExportJob + if json.Unmarshal(contents, &jobs) != nil { + return + } + now := time.Now().UTC().Format(time.RFC3339) + for index := range jobs { + job := jobs[index] + if job.Status == "queued" || job.Status == "running" { + job.Status = "failed" + job.Progress = 0 + job.Error = "服务重启，任务未完成，请重新创建导出" + job.UpdatedAt = now + } + if job.Status == "completed" { + job.filePath = filepath.Join(s.exportDir, job.ID+".csv") + if _, err := os.Stat(job.filePath); err != nil { + job.Status = "failed" + job.Progress = 0 + job.Error = "导出文件已不存在，请重新创建导出" + job.DownloadURL = "" + job.filePath = "" + job.UpdatedAt = now + } + } + s.exports[job.ID] = &job + } + s.exportsMu.Lock() + _ = s.persistHistoryExportsLocked() + s.exportsMu.Unlock() +} + +func randomExportID() (string, error) { + bytes := make([]byte, 8) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return "exp_" + hex.EncodeToString(bytes), nil +} + +func copyHistoryExportJob(job *HistoryExportJob) HistoryExportJob { + if job == nil { + return HistoryExportJob{} + } + copy := *job + copy.Keywords = append([]string(nil), job.Keywords...) + return copy +} + +func selectedExportMetrics(columns []HistoryMetricDefinition, requested []string) []HistoryMetricDefinition { + if len(requested) == 0 { + return append([]HistoryMetricDefinition(nil), columns...) + } + wanted := map[string]struct{}{} + for _, key := range requested { + wanted[key] = struct{}{} + } + result := make([]HistoryMetricDefinition, 0, len(requested)) + for _, column := range columns { + if _, ok := wanted[column.Key]; ok { + result = append(result, column) + } + } + return result +} + +func historyExportColumns(category string, requested []string) []HistoryMetricDefinition { + base := historyMetricsForCategory(category) + selected := selectedExportMetrics(base, requested) + if category != "raw" || len(requested) == 0 { + return selected + } + known := map[string]bool{} + for _, column := range selected { + known[column.Key] = true + } + for _, key := range requested { + if known[key] { + continue + } + label, unit := rawMetricMetadata(key) + selected = append(selected, HistoryMetricDefinition{Key: key, Label: label, Unit: unit, Category: "raw", ValueType: "dynamic", DefaultVisible: true}) + known[key] = true + } + return selected +} + +func writeHistoryExportMetadata(writer *csv.Writer, request HistoryExportRequest, columns []HistoryMetricDefinition) error { + rows := [][]string{ + {"导出元数据", "查询开始", request.DateFrom, "查询结束", request.DateTo, "车辆", strings.Join(request.Keywords, "、"), "数据类型", request.Category, "协议", firstNonEmpty(request.Protocol, "全部")}, + {"指标与单位", strings.Join(exportMetricLabels(columns), "；")}, + append([]string{"设备时间", "服务时间", "车牌", "VIN", "协议", "数据质量", "证据ID"}, exportMetricLabels(columns)...), + } + for _, row := range rows { + if err := writer.Write(row); err != nil { + return err + } + } + return nil +} + +func historyExportRecord(row HistoryDataRow, columns []HistoryMetricDefinition) []string { + record := []string{row.DeviceTime, row.ServerTime, row.Plate, row.VIN, row.Protocol, row.Quality, row.EvidenceID} + for _, column := range columns { + value, ok := row.Values[column.Key] + if !ok || value == nil { + record = append(record, "") + continue + } + record = append(record, fmt.Sprint(value)) + } + return record +} + +func exportMetricLabels(columns []HistoryMetricDefinition) []string { + labels := make([]string, 0, len(columns)) + for _, column := range columns { + label := column.Label + if column.Unit != "" { + label += "(" + column.Unit + ")" + } + labels = append(labels, label) + } + return labels +} + +func summarizeTrack(points []HistoryLocationRow) TrackPlaybackSummary { + summary := TrackPlaybackSummary{PointCount: len(points)} + if len(points) == 0 { + return summary + } + summary.StartTime = points[0].DeviceTime + summary.EndTime = points[len(points)-1].DeviceTime + var speedTotal float64 + var speedCount int + var coordinateDistance float64 + for index, point := range points { + if point.SpeedKmh >= 0 { + speedTotal += point.SpeedKmh + speedCount++ + } + if point.SpeedKmh > summary.MaximumSpeedKmh { + summary.MaximumSpeedKmh = point.SpeedKmh + } + if index > 0 && validTrackCoordinate(points[index-1]) && validTrackCoordinate(point) { + previousTime, previousOK := parseVehicleServiceTime(points[index-1].DeviceTime) + currentTime, currentOK := parseVehicleServiceTime(point.DeviceTime) + if previousOK && currentOK && currentTime.After(previousTime) && currentTime.Sub(previousTime) <= time.Duration(trackGapThresholdSeconds)*time.Second { + coordinateDistance += haversineKm(points[index-1].Latitude, points[index-1].Longitude, point.Latitude, point.Longitude) + } + } + } + if speedCount > 0 { + summary.AverageSpeedKmh = speedTotal / float64(speedCount) + } + mileageDistance := points[len(points)-1].TotalMileageKm - points[0].TotalMileageKm + if points[0].Protocol == points[len(points)-1].Protocol && mileageDistance >= 0 && mileageDistance < 10000 { + summary.DistanceKm = mileageDistance + } else { + summary.DistanceKm = coordinateDistance + } + start, startOK := parseVehicleServiceTime(summary.StartTime) + end, endOK := parseVehicleServiceTime(summary.EndTime) + if startOK && endOK && end.After(start) { + summary.DurationSeconds = int64(end.Sub(start).Seconds()) + } + return summary +} + +func summarizeTrackSources(points []HistoryLocationRow) []TrackPlaybackSource { + byProtocol := map[string]*TrackPlaybackSource{} + for _, point := range points { + protocol := strings.TrimSpace(point.Protocol) + if protocol == "" { + protocol = "UNKNOWN" + } + source := byProtocol[protocol] + if source == nil { + source = &TrackPlaybackSource{Protocol: protocol, StartTime: point.DeviceTime} + byProtocol[protocol] = source + } + source.PointCount++ + source.EndTime = point.DeviceTime + } + keys := make([]string, 0, len(byProtocol)) + for key := range byProtocol { + keys = append(keys, key) + } + sort.Strings(keys) + result := make([]TrackPlaybackSource, 0, len(keys)) + for _, key := range keys { + result = append(result, *byProtocol[key]) + } + return result +} + +const ( + trackGapThresholdSeconds = int64(10 * 60) + trackStopMinimumSeconds = int64(3 * 60) + trackStopSpeedKmh = 3.0 + trackDriftSpeedKmh = 220.0 + trackEventLimit = 64 +) + +func trackCoverage(query url.Values, total, fetched, processed, returned int, truncated, sampled bool) TrackCoverage { + coverage := TrackCoverage{ + RequestedStart: strings.TrimSpace(query.Get("dateFrom")), + RequestedEnd: strings.TrimSpace(query.Get("dateTo")), + TotalPoints: total, + FetchedPoints: fetched, + ProcessedPoints: processed, + ReturnedPoints: returned, + Complete: !truncated, + LimitReasons: []string{}, + } + if truncated { + coverage.LimitReasons = append(coverage.LimitReasons, "latest_source_slice") + } + if processed < fetched { + coverage.LimitReasons = append(coverage.LimitReasons, "quality_filter") + } + if sampled { + coverage.LimitReasons = append(coverage.LimitReasons, "map_sampling") + } + if truncated { + coverage.Evidence = fmt.Sprintf("查询共 %d 个源点，仅加工最新 %d 个；摘要不代表完整时间窗", total, fetched) + } else { + coverage.Evidence = fmt.Sprintf("查询窗 %d 个源点已完整读取", total) + } + return coverage +} + +func analyzeTrackPoints(raw []HistoryLocationRow) ([]HistoryLocationRow, TrackQuality) { + quality := TrackQuality{RawPoints: len(raw)} + points := make([]HistoryLocationRow, 0, len(raw)) + lastByProtocol := map[string]HistoryLocationRow{} + for _, point := range raw { + if !validTrackCoordinate(point) { + quality.InvalidCoordinatePoints++ + continue + } + protocol := firstNonEmpty(strings.TrimSpace(point.Protocol), "UNKNOWN") + if previous, ok := lastByProtocol[protocol]; ok { + if point.DeviceTime == previous.DeviceTime && point.Protocol == previous.Protocol && + math.Abs(point.Longitude-previous.Longitude) < 0.000001 && math.Abs(point.Latitude-previous.Latitude) < 0.000001 { + quality.DuplicatePoints++ + continue + } + previousTime, previousOK := parseVehicleServiceTime(previous.DeviceTime) + currentTime, currentOK := parseVehicleServiceTime(point.DeviceTime) + if previousOK && currentOK && currentTime.After(previousTime) { + seconds := currentTime.Sub(previousTime).Seconds() + distance := haversineKm(previous.Latitude, previous.Longitude, point.Latitude, point.Longitude) + impliedSpeed := distance / seconds * 3600 + if seconds <= 300 && distance >= 0.5 && impliedSpeed > trackDriftSpeedKmh { + quality.DriftPoints++ + continue + } + } + } + points = append(points, point) + lastByProtocol[protocol] = point + } + quality.ValidPoints = len(points) + issues := quality.InvalidCoordinatePoints + quality.DuplicatePoints + quality.DriftPoints + if issues == 0 { + quality.Status = "good" + quality.Evidence = "坐标、重复点、漂移和时间间隔检查未发现异常" + } else { + quality.Status = "warning" + quality.Evidence = fmt.Sprintf("过滤无效坐标 %d、重复 %d、疑似漂移 %d；检测到大间隔 %d", quality.InvalidCoordinatePoints, quality.DuplicatePoints, quality.DriftPoints, quality.LargeGapCount) + } + return points, quality +} + +func selectTrackPrimarySource(points []HistoryLocationRow, requestedProtocol string) ([]HistoryLocationRow, string, int) { + requestedProtocol = strings.ToUpper(strings.TrimSpace(requestedProtocol)) + counts := map[string]int{} + latest := map[string]time.Time{} + for _, point := range points { + protocol := strings.ToUpper(firstNonEmpty(strings.TrimSpace(point.Protocol), "UNKNOWN")) + counts[protocol]++ + if observedAt, ok := parseVehicleServiceTime(point.DeviceTime); ok && observedAt.After(latest[protocol]) { + latest[protocol] = observedAt + } + } + selected := requestedProtocol + if selected == "" { + protocols := make([]string, 0, len(counts)) + for protocol := range counts { + protocols = append(protocols, protocol) + } + sort.Slice(protocols, func(i, j int) bool { + if counts[protocols[i]] != counts[protocols[j]] { + return counts[protocols[i]] > counts[protocols[j]] + } + if !latest[protocols[i]].Equal(latest[protocols[j]]) { + return latest[protocols[i]].After(latest[protocols[j]]) + } + return protocols[i] < protocols[j] + }) + if len(protocols) > 0 { + selected = protocols[0] + } + } + result := make([]HistoryLocationRow, 0, counts[selected]) + for _, point := range points { + if strings.EqualFold(firstNonEmpty(strings.TrimSpace(point.Protocol), "UNKNOWN"), selected) { + result = append(result, point) + } + } + return result, selected, len(points) - len(result) +} + +func applyTrackSequenceQuality(points []HistoryLocationRow, quality *TrackQuality) { + quality.SourceSwitches = 0 + quality.LargeGapCount = 0 + quality.MaximumGapSeconds = 0 + for index := 1; index < len(points); index++ { + if points[index].Protocol != points[index-1].Protocol { + quality.SourceSwitches++ + } + previousTime, previousOK := parseVehicleServiceTime(points[index-1].DeviceTime) + currentTime, currentOK := parseVehicleServiceTime(points[index].DeviceTime) + if !previousOK || !currentOK || !currentTime.After(previousTime) { + continue + } + gap := int64(currentTime.Sub(previousTime).Seconds()) + if gap > quality.MaximumGapSeconds { + quality.MaximumGapSeconds = gap + } + if gap > trackGapThresholdSeconds { + quality.LargeGapCount++ + } + } + issues := quality.InvalidCoordinatePoints + quality.DuplicatePoints + quality.DriftPoints + quality.LargeGapCount + if issues == 0 { + quality.Status = "good" + quality.Evidence = "主来源坐标、重复点、漂移和时间间隔检查未发现异常" + } else { + quality.Status = "warning" + quality.Evidence = fmt.Sprintf("主来源过滤无效坐标 %d、重复 %d、疑似漂移 %d；检测到大间隔 %d", quality.InvalidCoordinatePoints, quality.DuplicatePoints, quality.DriftPoints, quality.LargeGapCount) + } +} + +func applyTrackPrimarySourceCoverage(coverage *TrackCoverage, protocol string, alternatePoints int) { + if alternatePoints <= 0 { + return + } + coverage.LimitReasons = append(coverage.LimitReasons, "primary_source_projection") + coverage.Evidence += fmt.Sprintf("；地图选用 %s 主来源，另 %d 个有效点保留在来源统计", firstNonEmpty(protocol, "UNKNOWN"), alternatePoints) +} + +func buildTrackSegments(points []HistoryLocationRow) []TrackSegment { + if len(points) == 0 { + return []TrackSegment{} + } + if len(points) == 1 { + return []TrackSegment{{Index: 0, Type: "point", Title: "单点", StartTime: points[0].DeviceTime, EndTime: points[0].DeviceTime, PointCount: 1}} + } + segments := make([]TrackSegment, 0) + for index := 1; index < len(points); index++ { + previous, current := points[index-1], points[index] + previousTime, previousOK := parseVehicleServiceTime(previous.DeviceTime) + currentTime, currentOK := parseVehicleServiceTime(current.DeviceTime) + duration := int64(0) + validInterval := previousOK && currentOK && currentTime.After(previousTime) + if validInterval { + duration = int64(currentTime.Sub(previousTime).Seconds()) + } + distance := haversineKm(previous.Latitude, previous.Longitude, current.Latitude, current.Longitude) + typeName, title := "moving", "行驶" + switch { + case !validInterval || currentTime.Sub(previousTime) > time.Duration(trackGapThresholdSeconds)*time.Second: + typeName, title = "gap", "数据间隔" + case previous.SpeedKmh <= trackStopSpeedKmh && current.SpeedKmh <= trackStopSpeedKmh && distance <= 0.15: + typeName, title = "stopped", "停车（GPS 推断）" + } + if typeName == "gap" { + distance = 0 + } + if len(segments) > 0 && segments[len(segments)-1].Type == typeName && segments[len(segments)-1].EndIndex == index-1 { + segment := &segments[len(segments)-1] + segment.EndIndex = index + segment.EndTime = current.DeviceTime + segment.DurationSeconds += duration + segment.DistanceKm += distance + segment.PointCount++ + continue + } + segments = append(segments, TrackSegment{ + Index: len(segments), Type: typeName, Title: title, + StartTime: previous.DeviceTime, EndTime: current.DeviceTime, + DurationSeconds: duration, DistanceKm: distance, PointCount: 2, + StartIndex: index - 1, EndIndex: index, + }) + } + return segments +} + +func buildTrackStops(points []HistoryLocationRow, segments []TrackSegment) []TrackStop { + stops := make([]TrackStop, 0) + for _, segment := range segments { + if segment.Type != "stopped" || segment.DurationSeconds < trackStopMinimumSeconds { + continue + } + longitude, latitude := 0.0, 0.0 + count := 0 + for index := segment.StartIndex; index <= segment.EndIndex && index < len(points); index++ { + longitude += points[index].Longitude + latitude += points[index].Latitude + count++ + } + if count > 0 { + longitude /= float64(count) + latitude /= float64(count) + } + stops = append(stops, TrackStop{ + Index: len(stops), StartTime: segment.StartTime, EndTime: segment.EndTime, + DurationSeconds: segment.DurationSeconds, PointCount: segment.PointCount, + Longitude: longitude, Latitude: latitude, + Evidence: "由连续低速且小位移位置点推断，不代表点火状态", + startIndex: segment.StartIndex, endIndex: segment.EndIndex, + }) + } + return stops +} + +func buildTrackEvents(points []HistoryLocationRow, segments []TrackSegment, stops []TrackStop) []TrackPlaybackEvent { + if len(points) == 0 { + return []TrackPlaybackEvent{} + } + events := []TrackPlaybackEvent{trackEvent(0, "start", "开始行驶", points[0])} + for _, stop := range stops { + events = append(events, trackEvent(stop.startIndex, "stop", "停车（GPS 推断）", points[stop.startIndex])) + } + for _, segment := range segments { + if segment.Type == "gap" && segment.EndIndex < len(points) { + events = append(events, trackEvent(segment.EndIndex, "gap", "数据间隔", points[segment.EndIndex])) + } + } + for index := 1; index < len(points)-1; index++ { + delta := points[index].SpeedKmh - points[index-1].SpeedKmh + typeName, title := "", "" + switch { + case points[index].Protocol != points[index-1].Protocol: + typeName, title = "source_switch", "数据源切换" + case delta >= 25: + typeName, title = "acceleration", "急加速" + case delta <= -25: + typeName, title = "braking", "急减速" + } + if typeName != "" { + events = append(events, trackEvent(index, typeName, title, points[index])) + } + } + events = append(events, trackEvent(len(points)-1, "end", "结束行驶", points[len(points)-1])) + sort.SliceStable(events, func(i, j int) bool { return events[i].Index < events[j].Index }) + if len(events) > trackEventLimit { + limited := append([]TrackPlaybackEvent(nil), events[:trackEventLimit-1]...) + limited = append(limited, events[len(events)-1]) + events = limited + } + return events +} + +func trackEvent(index int, eventType, title string, point HistoryLocationRow) TrackPlaybackEvent { + return TrackPlaybackEvent{Index: index, Type: eventType, Title: title, Time: point.DeviceTime, SpeedKmh: point.SpeedKmh, SOCPercent: point.SOCPercent, SOCAvailable: point.SOCAvailable, DirectionDeg: point.DirectionDeg, AlarmFlag: point.AlarmFlag, Longitude: point.Longitude, Latitude: point.Latitude} +} + +func trackKeyIndexes(events []TrackPlaybackEvent, segments []TrackSegment, stops []TrackStop) []int { + indexes := make([]int, 0, len(events)+len(segments)*2+len(stops)) + for _, event := range events { + indexes = append(indexes, event.Index) + } + for _, segment := range segments { + indexes = append(indexes, segment.StartIndex, segment.EndIndex) + } + for _, stop := range stops { + indexes = append(indexes, (stop.startIndex+stop.endIndex)/2) + } + return indexes +} + +func sampleTrackPoints(points []HistoryLocationRow, maxPoints int) []HistoryLocationRow { + result, _ := sampleTrackPointsWithKeys(points, maxPoints, nil) + return result +} + +func sampleTrackPointsWithKeys(points []HistoryLocationRow, maxPoints int, keyIndexes []int) ([]HistoryLocationRow, []int) { + if len(points) <= maxPoints { + indexes := make([]int, len(points)) + for index := range points { + indexes[index] = index + } + return append([]HistoryLocationRow(nil), points...), indexes + } + if maxPoints < 2 { + maxPoints = 2 + } + selected := map[int]struct{}{0: {}, len(points) - 1: {}} + for _, index := range keyIndexes { + if index >= 0 && index < len(points) { + selected[index] = struct{}{} + } + } + indexes := sortedTrackIndexes(selected) + if len(indexes) > maxPoints { + indexes = uniformlySelectTrackIndexes(indexes, maxPoints) + selected = make(map[int]struct{}, len(indexes)) + for _, index := range indexes { + selected[index] = struct{}{} + } + } + remaining := maxPoints - len(selected) + for slot := 1; slot <= remaining; slot++ { + index := int(math.Round(float64(slot) * float64(len(points)-1) / float64(remaining+1))) + selected[index] = struct{}{} + } + for len(selected) < maxPoints { + indexes = sortedTrackIndexes(selected) + largestGap, midpoint := 0, -1 + for index := 1; index < len(indexes); index++ { + gap := indexes[index] - indexes[index-1] + if gap > largestGap { + largestGap = gap + midpoint = indexes[index-1] + gap/2 + } + } + if midpoint < 0 || largestGap <= 1 { + break + } + selected[midpoint] = struct{}{} + } + indexes = sortedTrackIndexes(selected) + if len(indexes) > maxPoints { + indexes = uniformlySelectTrackIndexes(indexes, maxPoints) + } + result := make([]HistoryLocationRow, 0, len(indexes)) + for _, index := range indexes { + result = append(result, points[index]) + } + return result, indexes +} + +func sortedTrackIndexes(selected map[int]struct{}) []int { + indexes := make([]int, 0, len(selected)) + for index := range selected { + indexes = append(indexes, index) + } + sort.Ints(indexes) + return indexes +} + +func uniformlySelectTrackIndexes(indexes []int, limit int) []int { + if len(indexes) <= limit { + return append([]int(nil), indexes...) + } + result := make([]int, 0, limit) + for slot := 0; slot < limit; slot++ { + index := int(math.Round(float64(slot) * float64(len(indexes)-1) / float64(limit-1))) + result = append(result, indexes[index]) + } + return result +} + +func nearestSampledIndex(original int, sampled []int) int { + if len(sampled) == 0 { + return 0 + } + position := sort.SearchInts(sampled, original) + if position == 0 { + return 0 + } + if position == len(sampled) { + return len(sampled) - 1 + } + if original-sampled[position-1] <= sampled[position]-original { + return position - 1 + } + return position +} + +func validTrackCoordinate(point HistoryLocationRow) bool { + return point.Longitude >= 72 && point.Longitude <= 138 && point.Latitude >= 0.8 && point.Latitude <= 56 +} + +func haversineKm(lat1, lon1, lat2, lon2 float64) float64 { + const earthRadiusKm = 6371.0088 + toRadians := math.Pi / 180 + dLat := (lat2 - lat1) * toRadians + dLon := (lon2 - lon1) * toRadians + a := math.Sin(dLat/2)*math.Sin(dLat/2) + math.Cos(lat1*toRadians)*math.Cos(lat2*toRadians)*math.Sin(dLon/2)*math.Sin(dLon/2) + return earthRadiusKm * 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a)) +} + func (s *Service) RawFrames(ctx context.Context, query RawFrameQuery) (Page[RawFrameRow], error) { if query.Limit <= 0 || query.Limit > 500 { query.Limit = 100 diff --git a/vehicle-data-platform/apps/api/internal/platform/service_test.go b/vehicle-data-platform/apps/api/internal/platform/service_test.go index ac7bcf88..f7b0b395 100644 --- a/vehicle-data-platform/apps/api/internal/platform/service_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/service_test.go @@ -2,10 +2,50 @@ package platform import ( "context" + "fmt" "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" "testing" + "time" ) +func TestHistoryExportIndexSurvivesServiceRestart(t *testing.T) { + dir := t.TempDir() + first := NewServiceWithRuntime(NewMockStore(), RuntimeInfo{ExportDir: dir}) + completedAt := time.Now().UTC().Format(time.RFC3339) + filePath := filepath.Join(dir, "exp_completed.csv") + if err := os.WriteFile(filePath, []byte("vin\nVIN001\n"), 0o640); err != nil { + t.Fatal(err) + } + first.exportsMu.Lock() + first.exports["exp_completed"] = &HistoryExportJob{ID: "exp_completed", Name: "完成任务", Status: "completed", Progress: 100, Format: "csv", Category: "location", CreatedAt: completedAt, UpdatedAt: completedAt, DownloadURL: "/api/v2/exports/exp_completed/download", filePath: filePath} + first.exports["exp_running"] = &HistoryExportJob{ID: "exp_running", Name: "中断任务", Status: "running", Progress: 50, Format: "csv", Category: "location", CreatedAt: completedAt, UpdatedAt: completedAt} + if err := first.persistHistoryExportsLocked(); err != nil { + first.exportsMu.Unlock() + t.Fatal(err) + } + first.exportsMu.Unlock() + + restarted := NewServiceWithRuntime(NewMockStore(), RuntimeInfo{ExportDir: dir}) + jobs := restarted.ListHistoryExports() + if len(jobs) != 2 { + t.Fatalf("jobs=%+v", jobs) + } + path, _, err := restarted.HistoryExportFile("exp_completed") + if err != nil || path != filePath { + t.Fatalf("completed export not restored: path=%q err=%v", path, err) + } + for _, job := range jobs { + if job.ID == "exp_running" && (job.Status != "failed" || !strings.Contains(job.Error, "服务重启")) { + t.Fatalf("interrupted export should be failed: %+v", job) + } + } +} + type countingStore struct { *MockStore vehiclesCalls int @@ -102,3 +142,467 @@ func TestCompleteProtocolStatsIncludesCanonicalSlots(t *testing.T) { t.Fatalf("missing canonical protocol should be exposed as zero slot, got %+v", byProtocol["GB32960"]) } } + +func TestSummarizeTrackUsesMileageAndChronology(t *testing.T) { + points := []HistoryLocationRow{ + {DeviceTime: "2026-07-03 10:00:00", TotalMileageKm: 100, SpeedKmh: 0, Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03 10:30:00", TotalMileageKm: 112.5, SpeedKmh: 50, Longitude: 113.2, Latitude: 23.2}, + } + summary := summarizeTrack(points) + if summary.DistanceKm != 12.5 || summary.DurationSeconds != 1800 || summary.MaximumSpeedKmh != 50 { + t.Fatalf("unexpected track summary: %+v", summary) + } +} + +func TestTrackEventKeepsSynchronizedTelemetryEvidence(t *testing.T) { + direction := int64(88) + alarm := int64(2) + event := trackEvent(7, "alarm", "报警点", HistoryLocationRow{DeviceTime: "2026-07-03 10:00:00", SpeedKmh: 36, SOCPercent: 78.5, SOCAvailable: true, DirectionDeg: &direction, AlarmFlag: &alarm, Longitude: 113.2, Latitude: 23.1}) + if event.Index != 7 || !event.SOCAvailable || event.SOCPercent != 78.5 || event.DirectionDeg == nil || *event.DirectionDeg != 88 || event.AlarmFlag == nil || *event.AlarmFlag != 2 { + t.Fatalf("track event lost synchronized telemetry evidence: %+v", event) + } +} + +func TestSampleTrackPointsPreservesEndpoints(t *testing.T) { + points := make([]HistoryLocationRow, 10) + for index := range points { + points[index].DeviceTime = strconv.Itoa(index) + } + sampled := sampleTrackPoints(points, 4) + if len(sampled) != 4 || sampled[0].DeviceTime != "0" || sampled[len(sampled)-1].DeviceTime != "9" { + t.Fatalf("sampling should preserve endpoints: %+v", sampled) + } +} + +func TestTrackAnalysisFiltersInvalidDuplicateAndDriftPoints(t *testing.T) { + points := []HistoryLocationRow{ + {DeviceTime: "2026-07-03 10:00:00", Protocol: "JT808", Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03 10:00:00", Protocol: "JT808", Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03 10:01:00", Protocol: "JT808", Longitude: 114.1, Latitude: 24.1}, + {DeviceTime: "2026-07-03 10:02:00", Protocol: "JT808", Longitude: 0, Latitude: 0}, + {DeviceTime: "2026-07-03 10:03:00", Protocol: "JT808", Longitude: 113.101, Latitude: 23.101}, + } + clean, quality := analyzeTrackPoints(points) + if len(clean) != 2 || quality.ValidPoints != 2 || quality.DuplicatePoints != 1 || quality.DriftPoints != 1 || quality.InvalidCoordinatePoints != 1 { + t.Fatalf("unexpected track quality projection: clean=%+v quality=%+v", clean, quality) + } + if quality.Status != "warning" { + t.Fatalf("filtered evidence must surface as warning: %+v", quality) + } +} + +func TestTrackPrimarySourcePreventsParallelProtocolsFromBecomingOneRoute(t *testing.T) { + points := []HistoryLocationRow{ + {DeviceTime: "2026-07-03 10:00:00", Protocol: "GB32960", Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03 10:00:00", Protocol: "JT808", Longitude: 103.1, Latitude: 30.1}, + {DeviceTime: "2026-07-03 10:01:00", Protocol: "GB32960", Longitude: 113.101, Latitude: 23.101}, + {DeviceTime: "2026-07-03 10:01:00", Protocol: "JT808", Longitude: 103.101, Latitude: 30.101}, + {DeviceTime: "2026-07-03 10:02:00", Protocol: "JT808", Longitude: 103.102, Latitude: 30.102}, + } + clean, quality := analyzeTrackPoints(points) + if len(clean) != 5 || quality.DriftPoints != 0 { + t.Fatalf("parallel protocols must be checked within source: clean=%+v quality=%+v", clean, quality) + } + selected, protocol, alternate := selectTrackPrimarySource(clean, "") + if protocol != "JT808" || len(selected) != 3 || alternate != 2 { + t.Fatalf("unexpected primary source projection: protocol=%s selected=%+v alternate=%d", protocol, selected, alternate) + } + explicit, protocol, alternate := selectTrackPrimarySource(clean, "GB32960") + if protocol != "GB32960" || len(explicit) != 2 || alternate != 3 { + t.Fatalf("explicit source must win: protocol=%s selected=%+v alternate=%d", protocol, explicit, alternate) + } +} + +func TestTrackSegmentsExposeStopsAndDataGapsWithoutClaimingIgnition(t *testing.T) { + points := []HistoryLocationRow{ + {DeviceTime: "2026-07-03 10:00:00", SpeedKmh: 0, Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03 10:02:00", SpeedKmh: 0, Longitude: 113.10001, Latitude: 23.10001}, + {DeviceTime: "2026-07-03 10:04:00", SpeedKmh: 0, Longitude: 113.10002, Latitude: 23.10002}, + {DeviceTime: "2026-07-03 10:05:00", SpeedKmh: 30, Longitude: 113.11, Latitude: 23.11}, + {DeviceTime: "2026-07-03 10:20:00", SpeedKmh: 30, Longitude: 113.12, Latitude: 23.12}, + } + segments := buildTrackSegments(points) + if len(segments) != 3 || segments[0].Type != "stopped" || segments[1].Type != "moving" || segments[2].Type != "gap" { + t.Fatalf("unexpected inferred segments: %+v", segments) + } + stops := buildTrackStops(points, segments) + if len(stops) != 1 || stops[0].DurationSeconds != 240 || !strings.Contains(stops[0].Evidence, "不代表点火状态") { + t.Fatalf("unexpected inferred stops: %+v", stops) + } +} + +func TestTrackSegmentsDoNotTreatSubsecondSamplesAsDataGaps(t *testing.T) { + points := []HistoryLocationRow{ + {DeviceTime: "2026-07-03T10:00:00.100+08:00", SpeedKmh: 30, Longitude: 113.1, Latitude: 23.1}, + {DeviceTime: "2026-07-03T10:00:00.900+08:00", SpeedKmh: 31, Longitude: 113.1001, Latitude: 23.1001}, + {DeviceTime: "2026-07-03T10:00:01.700+08:00", SpeedKmh: 32, Longitude: 113.1002, Latitude: 23.1002}, + } + segments := buildTrackSegments(points) + if len(segments) != 1 || segments[0].Type != "moving" || segments[0].PointCount != 3 { + t.Fatalf("subsecond telemetry must remain a continuous moving segment: %+v", segments) + } +} + +func TestMonitorBoundsAreValidatedAndApplied(t *testing.T) { + bounds, ok, err := parseMonitorBounds("103,29,105,31") + if err != nil || !ok || !bounds.contains(104, 30) || bounds.contains(106, 30) { + t.Fatalf("unexpected monitor bounds: bounds=%+v ok=%t err=%v", bounds, ok, err) + } + for _, invalid := range []string{"103,29,105", "east,29,105,31", "105,31,103,29", "-181,0,1,1"} { + if _, _, invalidErr := parseMonitorBounds(invalid); invalidErr == nil { + t.Fatalf("invalid monitor bounds must be rejected: %q", invalid) + } + } +} + +func TestMonitorQueryKeepsServerOwnedKeywordAndMotionStatus(t *testing.T) { + query := normalizeMonitorQuery(url.Values{"keyword": {"沪A"}, "status": {"driving"}}) + if query.Get("vin") != "沪A" || query.Get("limit") != "10000" { + t.Fatalf("monitor query did not normalize keyword and bound: %v", query) + } + if !matchesMonitorStatus(VehicleRealtimeRow{Online: true, SpeedKmh: 20, LastSeen: "2026-07-14T08:00:00+08:00"}, "driving") { + t.Fatal("driving row must match driving filter") + } + if matchesMonitorStatus(VehicleRealtimeRow{Online: true, SpeedKmh: 0, LastSeen: "2026-07-14T08:00:00+08:00"}, "driving") { + t.Fatal("idle row must not match driving filter") + } +} + +func TestMonitorMapTenThousandVehiclesNeverReturnsTenThousandPoints(t *testing.T) { + vehicles := syntheticMonitorVehicles(10_000) + result, err := buildMonitorMapResponse(vehicles, url.Values{"zoom": {"13"}}) + if err != nil { + t.Fatal(err) + } + if (result.Mode != "clusters" && result.Mode != "mixed") || len(result.Clusters) == 0 || len(result.Points)+len(result.Clusters) > monitorPointLimit { + t.Fatalf("10k unbounded map must stay aggregated: mode=%s points=%d clusters=%d", result.Mode, len(result.Points), len(result.Clusters)) + } + for _, cluster := range result.Clusters { + if cluster.Count < 2 { + t.Fatalf("singleton must be released as a point instead of a fake cluster: %+v", cluster) + } + } + viewport, err := buildMonitorMapResponse(vehicles, url.Values{"zoom": {"11"}, "bounds": {"113,22,114,24"}}) + if err != nil { + t.Fatal(err) + } + if viewport.Mode != "points" || len(viewport.Points) >= monitorPointLimit { + t.Fatalf("bounded high zoom should return a controlled point set: mode=%s points=%d", viewport.Mode, len(viewport.Points)) + } +} + +func TestMonitorMapUsesMeaningfulClustersAndReleasesPointsAtDetailZoom(t *testing.T) { + vehicles := Page[VehicleRealtimeRow]{Items: []VehicleRealtimeRow{ + {VIN: "NEAR-1", Plate: "粤A00001", Online: true, Longitude: 113.260, Latitude: 23.130}, + {VIN: "NEAR-2", Plate: "粤A00002", Online: true, Longitude: 113.265, Latitude: 23.135}, + {VIN: "SINGLE", Plate: "粤A00003", Online: true, Longitude: 113.600, Latitude: 23.500}, + }, Total: 3, Limit: 3} + + mixed, err := buildMonitorMapResponse(vehicles, url.Values{"zoom": {"10"}}) + if err != nil { + t.Fatal(err) + } + if mixed.Mode != "mixed" || len(mixed.Clusters) != 1 || mixed.Clusters[0].Count != 2 || len(mixed.Points) != 1 || mixed.Points[0].VIN != "SINGLE" { + t.Fatalf("zoom below the detail threshold must keep only a meaningful cluster and release its singleton: %+v", mixed) + } + + detail, err := buildMonitorMapResponse(vehicles, url.Values{"zoom": {"11"}}) + if err != nil { + t.Fatal(err) + } + if detail.Mode != "points" || len(detail.Points) != 3 || len(detail.Clusters) != 0 { + t.Fatalf("zoom 11 must release controlled data as exact points: %+v", detail) + } +} + +func BenchmarkMonitorMapTenThousandVehicles(b *testing.B) { + vehicles := syntheticMonitorVehicles(10_000) + query := url.Values{"zoom": {"5"}} + b.ReportAllocs() + b.ResetTimer() + for index := 0; index < b.N; index++ { + if _, err := buildMonitorMapResponse(vehicles, query); err != nil { + b.Fatal(err) + } + } +} + +func syntheticMonitorVehicles(count int) Page[VehicleRealtimeRow] { + items := make([]VehicleRealtimeRow, count) + for index := range items { + items[index] = VehicleRealtimeRow{ + VIN: "SYNTH" + strconv.Itoa(index), Online: true, SpeedKmh: float64(index % 90), + Longitude: 73 + float64((index/100)%200)*0.3, Latitude: 18 + float64(index%100)*0.3, + LastSeen: "2026-07-14T08:00:00+08:00", + } + } + return Page[VehicleRealtimeRow]{Items: items, Total: count, Limit: count} +} + +func TestKeyPointSamplingPreservesEventsAndMapsToReturnedIndexes(t *testing.T) { + points := make([]HistoryLocationRow, 20) + for index := range points { + points[index].DeviceTime = strconv.Itoa(index) + } + sampled, indexes := sampleTrackPointsWithKeys(points, 6, []int{5, 11}) + if len(sampled) != 6 || !containsInt(indexes, 0) || !containsInt(indexes, 5) || !containsInt(indexes, 11) || !containsInt(indexes, 19) { + t.Fatalf("key-point sampling lost evidence: indexes=%+v", indexes) + } + mapped := nearestSampledIndex(11, indexes) + if mapped < 0 || mapped >= len(indexes) || indexes[mapped] != 11 { + t.Fatalf("event should map to its preserved returned point: mapped=%d indexes=%+v", mapped, indexes) + } +} + +func TestTrackCoverageDoesNotPresentLatestSliceAsCompleteWindow(t *testing.T) { + coverage := trackCoverage(url.Values{"dateFrom": {"2026-07-01T00:00:00"}}, 71000, 5000, 4990, 1200, true, true) + if coverage.Complete || coverage.TotalPoints != 71000 || coverage.FetchedPoints != 5000 || len(coverage.LimitReasons) != 3 { + t.Fatalf("unexpected bounded coverage: %+v", coverage) + } + if !strings.Contains(coverage.Evidence, "不代表完整时间窗") { + t.Fatalf("coverage boundary must be explicit: %+v", coverage) + } +} + +func TestTrackWindowRequiresBoundedOrderedPair(t *testing.T) { + if err := validateTrackWindow("2026-07-01T00:00", ""); err == nil { + t.Fatal("one-sided track window must be rejected") + } + if err := validateTrackWindow("2026-07-08T00:00", "2026-07-01T00:00"); err == nil { + t.Fatal("reversed track window must be rejected") + } + if err := validateTrackWindow("2026-07-01T00:00", "2026-07-09T00:00"); err == nil { + t.Fatal("track window over seven days must be rejected") + } + if err := validateTrackWindow("2026-07-01T00:00", "2026-07-08T00:00"); err != nil { + t.Fatalf("seven-day track window should be accepted: %v", err) + } + if err := validateTrackWindow("", ""); err != nil { + t.Fatalf("unbounded latest-slice compatibility mode should remain available: %v", err) + } +} + +func containsInt(values []int, wanted int) bool { + for _, value := range values { + if value == wanted { + return true + } + } + return false +} + +func TestHistoryExportRunsAsControlledAsyncJob(t *testing.T) { + service := NewService(NewMockStore()) + job, err := service.CreateHistoryExport(HistoryExportRequest{Keywords: []string{"川AHTWO1"}, Category: "location", Metrics: []string{"speedKmh"}, Format: "csv"}) + if err != nil { + t.Fatalf("create export: %v", err) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + jobs := service.ListHistoryExports() + if len(jobs) > 0 && jobs[0].ID == job.ID && jobs[0].Status == "completed" { + path, _, fileErr := service.HistoryExportFile(job.ID) + if fileErr != nil { + t.Fatalf("export file: %v", fileErr) + } + body, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatalf("read export: %v", readErr) + } + if len(body) < 3 || string(body[:3]) != "\xEF\xBB\xBF" { + t.Fatalf("CSV should include UTF-8 BOM") + } + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("export job did not complete: %+v", service.ListHistoryExports()) +} + +type largeHistoryExportStore struct { + *MockStore + total int64 +} + +func (s *largeHistoryExportStore) HistoryExportCount(context.Context, HistoryExportStoreQuery) (int64, error) { + return s.total, nil +} + +func (s *largeHistoryExportStore) HistoryExportBatch(_ context.Context, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) ([]HistoryDataRow, HistoryExportCursor, error) { + remaining := int(s.total) - cursor.Offset + if remaining <= 0 { + return []HistoryDataRow{}, cursor, nil + } + if remaining < limit { + limit = remaining + } + rows := make([]HistoryDataRow, limit) + for index := range rows { + number := cursor.Offset + index + rows[index] = HistoryDataRow{ID: "row-" + strconv.Itoa(number), VIN: query.VIN, Plate: "粤A测试", Protocol: "JT808", DeviceTime: "2026-07-14T00:00:00+08:00", ServerTime: "2026-07-14T00:00:01+08:00", Quality: "normal", Values: map[string]any{"speedKmh": float64(number % 120)}} + } + cursor.Offset += len(rows) + return rows, cursor, nil +} + +func TestHistoryExportStreamsBeyondLegacyLimit(t *testing.T) { + assertLargeHistoryExport(t, 12_345, 10*time.Second) +} + +func TestHistoryExportMillionRows(t *testing.T) { + if os.Getenv("EXPORT_MILLION_TEST") != "1" { + t.Skip("set EXPORT_MILLION_TEST=1 for the release capacity gate") + } + assertLargeHistoryExport(t, 1_000_000, 2*time.Minute) +} + +func assertLargeHistoryExport(t *testing.T, total int64, timeout time.Duration) { + t.Helper() + dir := t.TempDir() + store := &largeHistoryExportStore{MockStore: NewMockStore(), total: total} + service := NewServiceWithRuntime(store, RuntimeInfo{ExportDir: dir}) + job, err := service.CreateHistoryExport(HistoryExportRequest{Keywords: []string{"LNXNEGRR7SR318212"}, Category: "location", DateFrom: "2026-07-14T00:00", DateTo: "2026-07-14T06:00", Metrics: []string{"speedKmh"}, Format: "csv"}) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + current := service.ListHistoryExports()[0] + if current.Status == "failed" { + t.Fatalf("export failed: %+v", current) + } + if current.Status == "completed" { + if current.RowCount != int(total) || current.ProcessedRows != total || current.TotalRows != total || current.FileSizeBytes == 0 || current.CompletedAt == "" { + t.Fatalf("incomplete evidence: %+v", current) + } + path, _, fileErr := service.HistoryExportFile(job.ID) + if fileErr != nil { + t.Fatal(fileErr) + } + if _, statErr := os.Stat(path + ".part"); !os.IsNotExist(statErr) { + t.Fatalf("part file should be atomically removed: %v", statErr) + } + body, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(body[:min(len(body), 512)]), "查询开始") || !strings.Contains(string(body[:min(len(body), 512)]), "速度(km/h)") { + t.Fatalf("CSV metadata missing: %q", body[:min(len(body), 512)]) + } + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("export timed out: %+v", service.ListHistoryExports()) +} + +func TestRawMetricMetadataKeepsProtocolAndUnit(t *testing.T) { + label, unit := rawMetricMetadata("jt808.location.additional.total_mileage_km") + if label != "总里程 · JT808" || unit != "km" { + t.Fatalf("unexpected RAW metric metadata: %q %q", label, unit) + } +} + +func TestBuildLatestTelemetryResponseUsesCatalogAndNewestSourceEvidence(t *testing.T) { + now := time.Date(2026, 7, 14, 9, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) + definitions := []MetricDefinition{ + {Key: "speed_kmh", Label: "速度", Description: "车辆最新行驶速度", Unit: "km/h", Category: "driving", ValueType: "numeric", SourceFields: map[string]string{"GB32960": "gb32960.vehicle.speed_kmh"}}, + {Key: "alarm_active", Label: "协议告警位", Unit: "", Category: "safety", ValueType: "boolean", SourceFields: map[string]string{"GB32960": "gb32960.alarm.general_alarm_flag"}}, + } + frames := []RawFrameRow{ + {ID: "new", VIN: "VIN001", Protocol: "GB32960", DeviceTime: "2026-07-14 09:29:58", ServerTime: "2026-07-14 09:29:59", ParseStatus: "ok", SourceEndpoint: "gateway-a", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 42.5, "gb32960.alarm.general_alarm_flag": float64(1), "vendor.custom_temperature_c": 31.2}}, + {ID: "old", VIN: "VIN001", Protocol: "GB32960", DeviceTime: "2026-07-14 09:20:00", ServerTime: "2026-07-14 09:20:01", ParseStatus: "ok", ParsedFields: map[string]any{"gb32960.vehicle.speed_kmh": 10.0}}, + } + response := buildLatestTelemetryResponse("VIN001", frames, definitions, now) + if response.VIN != "VIN001" || response.ScannedFrames != 2 || len(response.Values) != 3 || len(response.Categories) != 3 { + t.Fatalf("unexpected response: %+v", response) + } + byKey := map[string]LatestTelemetryValue{} + for _, value := range response.Values { + byKey[value.Key] = value + } + if speed := byKey["speed_kmh"]; speed.Value != 42.5 || speed.SourceField != "gb32960.vehicle.speed_kmh" || speed.FrameID != "new" || speed.Quality != "good" || speed.FreshnessSeconds != 1 { + t.Fatalf("unified speed should use newest frame and evidence: %+v", speed) + } + if alarm := byKey["alarm_active"]; alarm.Value != true || alarm.Category != "alarm" { + t.Fatalf("catalog boolean should be normalized: %+v", alarm) + } + if extension := byKey["vendor.custom_temperature_c"]; extension.Category != "extension" || extension.Unit != "℃" || extension.Protocol != "GB32960" { + t.Fatalf("manufacturer extension should retain source semantics: %+v", extension) + } +} + +func TestLatestTelemetryQualityDistinguishesStaleAndWarnings(t *testing.T) { + now := time.Date(2026, 7, 14, 9, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) + stale, reason, freshness, delay := latestTelemetryQuality(RawFrameRow{DeviceTime: "2026-07-14 09:19:59", ServerTime: "2026-07-14 09:20:00", ParseStatus: "ok"}, now) + if stale != "stale" || freshness != 600 || delay == nil || *delay != 1 || !strings.Contains(reason, "超过 5 分钟") { + t.Fatalf("unexpected stale evidence: %s %s %d %v", stale, reason, freshness, delay) + } + warning, reason, _, _ := latestTelemetryQuality(RawFrameRow{ServerTime: "2026-07-14 09:29:59", ParseStatus: "failed", ParseError: "checksum"}, now) + if warning != "warning" || !strings.Contains(reason, "checksum") { + t.Fatalf("parse warning should retain reason: %s %s", warning, reason) + } +} + +type latestTelemetryQueryStore struct { + *MockStore + mu sync.Mutex + queries []RawFrameQuery +} + +func (s *latestTelemetryQueryStore) RawFrames(ctx context.Context, query RawFrameQuery) (Page[RawFrameRow], error) { + s.mu.Lock() + s.queries = append(s.queries, query) + s.mu.Unlock() + return s.MockStore.RawFrames(ctx, query) +} + +func TestLatestTelemetryBoundsProtocolReads(t *testing.T) { + store := &latestTelemetryQueryStore{MockStore: NewMockStore()} + response, err := NewService(store).LatestTelemetry(t.Context(), "LNXNEGRR7SR318212") + if err != nil { + t.Fatal(err) + } + if len(response.Values) == 0 || response.ScannedFrames != 1 { + t.Fatalf("expected one mock GB frame: %+v", response) + } + store.mu.Lock() + defer store.mu.Unlock() + if len(store.queries) != len(canonicalVehicleProtocols) { + t.Fatalf("queries = %+v", store.queries) + } + protocols := map[string]bool{} + for _, query := range store.queries { + protocols[query.Protocol] = true + if query.VIN != "LNXNEGRR7SR318212" || query.Limit != 5 || !query.IncludeFields || !query.SkipCount { + t.Fatalf("unbounded latest query: %+v", query) + } + } + for _, protocol := range canonicalVehicleProtocols { + if !protocols[protocol] { + t.Fatalf("missing protocol %s: %+v", protocol, store.queries) + } + } +} + +func BenchmarkLatestTelemetryHundredFrames(b *testing.B) { + now := time.Date(2026, 7, 14, 9, 30, 0, 0, time.FixedZone("Asia/Shanghai", 8*60*60)) + definitions := metricDefinitions() + frames := make([]RawFrameRow, 100) + for frameIndex := range frames { + fields := make(map[string]any, 50) + for fieldIndex := 0; fieldIndex < 50; fieldIndex++ { + fields[fmt.Sprintf("vendor.section_%02d.metric_%02d_voltage_v", fieldIndex/10, fieldIndex)] = float64(frameIndex + fieldIndex) + } + fields["gb32960.vehicle.speed_kmh"] = float64(frameIndex) + frames[frameIndex] = RawFrameRow{ID: fmt.Sprintf("frame-%03d", frameIndex), VIN: "VIN001", Protocol: "GB32960", DeviceTime: "2026-07-14 09:29:58", ServerTime: "2026-07-14 09:29:59", ParseStatus: "ok", ParsedFields: fields} + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + response := buildLatestTelemetryResponse("VIN001", frames, definitions, now) + if len(response.Values) != 51 { + b.Fatalf("values = %d", len(response.Values)) + } + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go b/vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go index 0eec668f..aba78e4e 100644 --- a/vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go +++ b/vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go @@ -33,6 +33,9 @@ func buildRawFrameSQL(database string, query RawFrameQuery) SQLQuery { if len(where) > 0 { countText += ` WHERE ` + strings.Join(where, " AND ") } + if query.SkipCount { + countText = "" + } text += ` ORDER BY ts DESC, frame_id ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset) return SQLQuery{Text: text, Args: args, CountText: countText} } @@ -71,7 +74,7 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery where = append(where, "ts <= '"+quoteTDengine(normalizeTDengineTime(dateTo))+"'") } limit, offset := parseLimitOffset(query["limit"], query["offset"]) - text := `SELECT ts, vin, protocol, longitude, latitude, speed_kmh, total_mileage_km, received_at FROM ` + table + text := `SELECT CAST(ts AS BIGINT), vin, protocol, longitude, latitude, speed_kmh, soc_percent, direction_deg, alarm_flag, total_mileage_km, CAST(received_at AS BIGINT) FROM ` + table if len(where) > 0 { text += ` WHERE ` + strings.Join(where, " AND ") } @@ -83,6 +86,90 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery return SQLQuery{Text: text, Args: args, CountText: countText} } +func buildHistoryLocationSeriesSQL(database string, query HistoryLocationSeriesQuery) SQLQuery { + tableQuery := map[string]string{"vin": query.VIN, "protocol": query.Protocol} + where := []string{"vin = '" + quoteTDengine(strings.TrimSpace(query.VIN)) + "'"} + if protocol := strings.TrimSpace(query.Protocol); protocol != "" { + where = append(where, "protocol = '"+quoteTDengine(strings.ToUpper(protocol))+"'") + } + where = append(where, + "ts >= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateFrom))+"'", + "ts <= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateTo))+"'", + ) + grain := query.GrainSeconds + if grain <= 0 { + grain = 60 + } + text := `SELECT _wstart, protocol, COUNT(*), ` + + `AVG(speed_kmh), MIN(speed_kmh), MAX(speed_kmh), LAST(speed_kmh), ` + + `AVG(total_mileage_km), MIN(total_mileage_km), MAX(total_mileage_km), LAST(total_mileage_km) ` + + `FROM ` + locationTable(database, tableQuery) + ` WHERE ` + strings.Join(where, " AND ") + + ` PARTITION BY protocol INTERVAL(` + strconv.Itoa(grain) + `s) ORDER BY _wstart ASC, protocol ASC` + return SQLQuery{Text: text} +} + +func buildHistoryExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery { + if limit <= 0 || limit > 5000 { + limit = 5000 + } + switch query.Category { + case "raw": + return buildRawExportBatchSQL(database, query, cursor, limit) + default: + return buildLocationExportBatchSQL(database, query, cursor, limit) + } +} + +func buildLocationExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery { + tableQuery := map[string]string{"vin": query.VIN, "protocol": query.Protocol} + where := historyExportTDengineWhere(query) + if cursor.Time != "" { + cursorTime := quoteTDengine(cursor.Time) + cursorProtocol := quoteTDengine(cursor.Protocol) + where = append(where, "(ts > '"+cursorTime+"' OR (ts = '"+cursorTime+"' AND protocol > '"+cursorProtocol+"'))") + } + table := locationTable(database, tableQuery) + return SQLQuery{ + Text: `SELECT ts, vin, protocol, longitude, latitude, speed_kmh, total_mileage_km, received_at FROM ` + table + ` WHERE ` + strings.Join(where, " AND ") + ` ORDER BY ts ASC, protocol ASC LIMIT ` + strconv.Itoa(limit), + CountText: `SELECT COUNT(*) FROM ` + table + ` WHERE ` + strings.Join(historyExportTDengineWhere(query), " AND "), + } +} + +func buildRawExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery { + rawQuery := RawFrameQuery{VIN: query.VIN, Protocol: query.Protocol} + where := historyExportTDengineWhere(query) + if cursor.Time != "" { + timeValue := quoteTDengine(cursor.Time) + protocol := quoteTDengine(cursor.Protocol) + identifier := quoteTDengine(cursor.ID) + where = append(where, "(ts > '"+timeValue+"' OR (ts = '"+timeValue+"' AND (protocol > '"+protocol+"' OR (protocol = '"+protocol+"' AND frame_id > '"+identifier+"'))))") + } + table := rawFrameTable(database, rawQuery) + selectText := `SELECT ts, frame_id, event_time, received_at, raw_size_bytes, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone FROM ` + table + ` WHERE ` + strings.Join(where, " AND ") + return SQLQuery{Text: selectText + ` ORDER BY ts ASC, protocol ASC, frame_id ASC LIMIT ` + strconv.Itoa(limit), CountText: `SELECT COUNT(*) FROM ` + table + ` WHERE ` + strings.Join(historyExportTDengineWhere(query), " AND ")} +} + +func historyExportTDengineWhere(query HistoryExportStoreQuery) []string { + where := []string{"vin = '" + quoteTDengine(strings.TrimSpace(query.VIN)) + "'"} + if protocol := strings.TrimSpace(query.Protocol); protocol != "" { + where = append(where, "protocol = '"+quoteTDengine(strings.ToUpper(protocol))+"'") + } + if query.DateFrom != "" { + where = append(where, "ts >= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateFrom))+"'") + } + if query.DateTo != "" { + where = append(where, "ts <= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateTo))+"'") + } + return where +} + +func normalizeTDengineSeriesTime(value string) string { + if parsed, ok := parseTrackRequestTime(value); ok { + return parsed.Format(time.RFC3339) + } + return strings.TrimSpace(value) +} + func buildTodayRawFrameCountSQL(database string, now time.Time) SQLQuery { start := shanghaiDayStartUTC(now) return SQLQuery{ @@ -133,13 +220,13 @@ func normalizeTDengineTime(value string) string { return "" } shanghai := time.FixedZone("Asia/Shanghai", 8*3600) - for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} { + for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02T15:04", "2006-01-02 15:04:05", "2006-01-02 15:04", "2006-01-02"} { if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil { - return parsed.UTC().Format("2006-01-02 15:04:05") + return parsed.Format(time.RFC3339) } } if parsed, err := time.Parse(time.RFC3339, value); err == nil { - return parsed.UTC().Format("2006-01-02 15:04:05") + return parsed.Format(time.RFC3339) } return value } @@ -148,5 +235,5 @@ func shanghaiDayStartUTC(now time.Time) string { shanghai := time.FixedZone("Asia/Shanghai", 8*3600) local := now.In(shanghai) start := time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, shanghai) - return start.UTC().Format("2006-01-02 15:04:05") + return start.Format(time.RFC3339) } diff --git a/vehicle-data-platform/apps/web/package.json b/vehicle-data-platform/apps/web/package.json index 727ae5e2..a16bb016 100644 --- a/vehicle-data-platform/apps/web/package.json +++ b/vehicle-data-platform/apps/web/package.json @@ -10,10 +10,12 @@ "@douyinfe/semi-icons": "^2.71.0", "@douyinfe/semi-theme-default": "^2.71.0", "@douyinfe/semi-ui": "^2.71.0", + "@tanstack/react-query": "5.101.2", "@vitejs/plugin-react": "^4.3.4", "echarts": "^5.6.0", "react": "^18.3.1", "react-dom": "^18.3.1", + "react-router-dom": "6.30.1", "vite": "^6.0.0" }, "devDependencies": { diff --git a/vehicle-data-platform/apps/web/pnpm-lock.yaml b/vehicle-data-platform/apps/web/pnpm-lock.yaml index a5d5c3e9..a88527b9 100644 --- a/vehicle-data-platform/apps/web/pnpm-lock.yaml +++ b/vehicle-data-platform/apps/web/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@douyinfe/semi-ui': specifier: ^2.71.0 version: 2.101.0(@floating-ui/dom@1.7.6)(@tiptap/suggestion@3.27.1(@floating-ui/dom@1.7.6)(@tiptap/core@3.27.1(@tiptap/pm@3.27.1))(@tiptap/pm@3.27.1))(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@tanstack/react-query': + specifier: 5.101.2 + version: 5.101.2(react@18.3.1) '@vitejs/plugin-react': specifier: ^4.3.4 version: 4.7.0(vite@6.4.3(sass@1.101.0)) @@ -29,6 +32,9 @@ importers: react-dom: specifier: ^18.3.1 version: 18.3.1(react@18.3.1) + react-router-dom: + specifier: 6.30.1 + version: 6.30.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1) vite: specifier: ^6.0.0 version: 6.4.3(sass@1.101.0) @@ -647,6 +653,10 @@ packages: resolution: {integrity: sha512-tmmZ3lQxAe/k/+rNnXQRawJ4NjxO2hqiOLTHvWchtGZULp4RyFeh6aU4XdOYBFe2KE1oShQTv4AblOs2iOrNnQ==} engines: {node: '>= 10.0.0'} + '@remix-run/router@1.23.0': + resolution: {integrity: sha512-O3rHJzAQKamUz1fvE0Qaw0xSFqsA/yafi2iqeE0pvdFtCO1viYx8QL6f3Ln/aCCTLxs68SLf0KPM9eSeM8yBnA==} + engines: {node: '>=14.0.0'} + '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} @@ -788,6 +798,14 @@ packages: cpu: [x64] os: [win32] + '@tanstack/query-core@5.101.2': + resolution: {integrity: sha512-hH5MLoJhF7KaIGd7q3xTXGXvslI+GYlM1Z/35aSHHWaCJWB7XvTSHYuV3eM7tw+aE0mT/xMro4M4Q9rCGHT0lw==} + + '@tanstack/react-query@5.101.2': + resolution: {integrity: sha512-seDkr6kzGzX1okaaTtZPtgA688CDPlXUz1C6xSg0ESqn04Vuc8tlrYms1s3de+znBqhPVxFRfpAfUf+6XvfPWg==} + peerDependencies: + react: ^18 || ^19 + '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -1842,6 +1860,19 @@ packages: react: '>= 16.3' react-dom: '>= 16.3' + react-router-dom@6.30.1: + resolution: {integrity: sha512-llKsgOkZdbPU1Eg3zK8lCn+sjD9wMRZZPuzmdWWX5SUs8OFkN5HnFVC0u5KMeMaC9aoancFI/KoLuKPqN+hxHw==} + engines: {node: '>=14.0.0'} + peerDependencies: + react: '>=16.8' + react-dom: '>=16.8' + + react-router@6.30.1: + resolution: {integrity: sha512-X1m21aEmxGXqENEPG3T6u0Th7g0aS4ZmoNynhbs+Cn+q+QGTLt+d5IQ2bHAXKzKcxGJjxACpVbnYQSCRcfxHlQ==} + engines: {node: '>=14.0.0'} + peerDependencies: + react: '>=16.8' + react-window@1.8.11: resolution: {integrity: sha512-+SRbUVT2scadgFSWx+R1P754xHPEqvcfSfVX10QYg6POOz+WNgkN48pS+BtZNIMGiL1HYrSEiCkwsMS15QogEQ==} engines: {node: '>8.0.0'} @@ -2751,6 +2782,8 @@ snapshots: '@parcel/watcher-win32-x64': 2.5.6 optional: true + '@remix-run/router@1.23.0': {} + '@rolldown/pluginutils@1.0.0-beta.27': {} '@rollup/rollup-android-arm-eabi@4.62.2': @@ -2828,6 +2861,13 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true + '@tanstack/query-core@5.101.2': {} + + '@tanstack/react-query@5.101.2(react@18.3.1)': + dependencies: + '@tanstack/query-core': 5.101.2 + react: 18.3.1 + '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -4267,6 +4307,18 @@ snapshots: react-dom: 18.3.1(react@18.3.1) react-draggable: 4.7.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react-router-dom@6.30.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + '@remix-run/router': 1.23.0 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-router: 6.30.1(react@18.3.1) + + react-router@6.30.1(react@18.3.1): + dependencies: + '@remix-run/router': 1.23.0 + react: 18.3.1 + react-window@1.8.11(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@babel/runtime': 7.29.7 diff --git a/vehicle-data-platform/apps/web/public/app-config.example.js b/vehicle-data-platform/apps/web/public/app-config.example.js new file mode 100644 index 00000000..878c2453 --- /dev/null +++ b/vehicle-data-platform/apps/web/public/app-config.example.js @@ -0,0 +1,8 @@ +window.__LINGNIU_APP_CONFIG__ = { + amapWebJsKey: '', + // Production should prefer amapSecurityServiceHost and keep amapSecurityJsCode on the API service. + amapSecurityJsCode: '', + amapSecurityServiceHost: '', + apiBaseUrl: '', + prototypeUseRealData: true +}; diff --git a/vehicle-data-platform/apps/web/src/PrototypeApp.tsx b/vehicle-data-platform/apps/web/src/PrototypeApp.tsx new file mode 100644 index 00000000..8012f8bf --- /dev/null +++ b/vehicle-data-platform/apps/web/src/PrototypeApp.tsx @@ -0,0 +1,7819 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { Dispatch, ReactNode, SetStateAction } from 'react'; +import { Button, Input, Tag } from '@douyinfe/semi-ui'; +import { + IconBarChartHStroked, + IconBell, + IconCopy, + IconHistogram, + IconHome, + IconPulse, + IconSearch, + IconServer, + IconSetting +} from '@douyinfe/semi-icons'; +import { + alertEvents, + dataSources, + fieldOptions, + getVehiclePrimarySource, + historyRecords, + protocolMeta, + sourceHealthLabel, + statusLabel, + trackPoints, + vehicles +} from './prototype/mockData'; +import { + buildDataCapabilityRadar, + buildHistoryQueryPreview, + buildReplayRawLookupWindow, + capabilityBoundaries, + filterVehicles, + formatNumber, + getAlertEvidenceItems, + getAlertRows, + getCommandPriorityItems, + getKpis, + getMileageSourceDecisions, + getRealtimeEvidence, + getRealtimeSummary, + getReplayEvidence, + getReplaySegments, + getReplaySummary, + getSampleBoundaryCopy, + getSourceConsistency, + getSourceName, + getStatsInterpretationItems, + getVehicleDiagnosticSteps, + getVehicleDetailDigest, + getVehicleSourceResponsibilityRows, + historyPageSizeOptions, + includesQuery, + protocolFilterOptions, + sectionDataBoundaries, + sectionTitle, + statusFilterOptions +} from './prototype/viewModel'; +import type { HistoryQueryPreview, ProtocolFilter, SectionKey, StatusFilter } from './prototype/viewModel'; +import { + loadPrototypeAlertSignals, + loadPrototypeMapAddress, + loadPrototypeRawFrames, + loadPrototypeRealtimeVehicles, + loadPrototypeSourceMonitor, + loadPrototypeTrackPoints +} from './prototype/realData'; +import { getAMapConfig, getApiBaseUrl, isAMapConfigured } from './config/appConfig'; +import { buildAMapMarkerURL, isValidAMapCoordinate, loadAMap } from './integrations/amap'; +import type { + PrototypeAlertSignal, + PrototypeAlertSignalState, + PrototypeMapAddressState, + PrototypeRawFrameState, + PrototypeRealDataState, + PrototypeSourceMonitorState, + PrototypeTrackState +} from './prototype/realData'; +import { rawCoordinateAddressEvidence, rawCoordinateEvidence, type RawCoordinateEvidence } from './prototype/rawEvidence'; +import type { + AlertEvent, + DataSourceRecord, + Protocol, + SourceHealth, + TrackPoint, + VehicleRecord, + VehicleSourceSnapshot +} from './prototype/types'; +import { VehicleMap } from './components/VehicleMap'; +import type { VehicleMapPoint } from './components/VehicleMap'; +import './styles/prototype.css'; + +const navItems: Array<{ key: SectionKey; label: string; icon: ReactNode; metric: string }> = [ + { key: 'command', label: '实时指挥台', icon: , metric: '地图看车' }, + { key: 'replay', label: '轨迹回放', icon: , metric: '证据链' }, + { key: 'history', label: '历史查询', icon: , metric: 'Raw/Fields' }, + { key: 'alerts', label: '告警中心', icon: , metric: '断链/异常' }, + { key: 'stats', label: '统计分析', icon: , metric: '里程/在线' }, + { key: 'sources', label: '数据源监控', icon: , metric: '平台质量' } +]; + +const protocolOrder: Protocol[] = ['GB32960', 'JT808', 'YUTONG_MQTT']; +const defaultHistoryFields = ['speed_kph', 'total_mileage_km', 'lat_lon']; + +const sectionKeys = new Set(navItems.map((item) => item.key)); + +type FreshnessFilter = 'ALL' | '10S' | '60S' | 'STALE'; +type DetailTab = '概况' | '实时字段' | '来源' | '告警'; +type StatsApiProbeStatus = 'checking' | 'ready' | 'missing' | 'error'; +type RealtimeQualityTone = 'green' | 'blue' | 'orange' | 'red'; +type DiagnosticTone = 'ok' | 'warn' | 'error' | 'idle'; +type AMapRuntimeStatus = 'not-configured' | 'loading' | 'ready' | 'error'; + +type AMapRuntimeState = { + status: AMapRuntimeStatus; + label: string; + detail: string; + diagnosticValue: string; + tagLabel: string; + tone: DiagnosticTone; + error?: string; +}; + +type RealtimeQualityBucket = { + key: string; + title: string; + count: number; + detail: string; + tone: RealtimeQualityTone; + vehicles: VehicleRecord[]; +}; + +type StatsApiProbeDefinition = { + key: string; + label: string; + capability: string; + endpoint: string; + required: boolean; +}; + +type StatsApiProbeResult = StatsApiProbeDefinition & { + status: StatsApiProbeStatus; + httpStatus?: number; + detail: string; + checkedAt?: string; +}; + +const compactViewportQuery = '(max-width: 720px)'; +const statsDesktopRealtimeLimit = 20; +const statsCompactRealtimeLimit = 8; +const statsDesktopConsistencyLimit = 10; +const statsCompactConsistencyLimit = 6; +const sourceDesktopVisibleLimit = 12; +const sourceCompactVisibleLimit = 8; +const realtimeRefreshIntervalMs = 15_000; +const realtimeQueryLimit = 300; +const realtimeLocationQuerySource = `/api/realtime/locations?limit=${realtimeQueryLimit}&offset=0`; +const realtimeSnapshotQuerySource = `/api/realtime/snapshots?limit=${realtimeQueryLimit}&offset=0`; +const realtimeQuerySource = `${realtimeLocationQuerySource} + ${realtimeSnapshotQuerySource}`; + +function isCompactViewportNow() { + return typeof window !== 'undefined' && window.matchMedia(compactViewportQuery).matches; +} + +function defaultStatsRealtimeLimit() { + return isCompactViewportNow() ? statsCompactRealtimeLimit : statsDesktopRealtimeLimit; +} + +function defaultStatsConsistencyLimit() { + return isCompactViewportNow() ? statsCompactConsistencyLimit : statsDesktopConsistencyLimit; +} + +function defaultSourceVisibleLimit() { + return isCompactViewportNow() ? sourceCompactVisibleLimit : sourceDesktopVisibleLimit; +} + +function useCompactViewport() { + const [matches, setMatches] = useState(isCompactViewportNow); + + useEffect(() => { + const media = window.matchMedia(compactViewportQuery); + const sync = () => setMatches(media.matches); + sync(); + media.addEventListener?.('change', sync); + return () => { + media.removeEventListener?.('change', sync); + }; + }, []); + + return matches; +} + +function amapRuntimeState(status: AMapRuntimeStatus, error?: string): AMapRuntimeState { + if (status === 'not-configured') { + return { + status, + label: '未配置', + detail: '降级为坐标预览', + diagnosticValue: '坐标预览', + tagLabel: '坐标预览', + tone: 'warn' + }; + } + if (status === 'loading') { + return { + status, + label: '加载中', + detail: '正在加载高德 Web JS SDK', + diagnosticValue: 'SDK 加载中', + tagLabel: 'AMap 加载中', + tone: 'warn' + }; + } + if (status === 'ready') { + return { + status, + label: '已加载', + detail: '高德 Web JS SDK 可用', + diagnosticValue: 'SDK 已加载', + tagLabel: 'AMap 已加载', + tone: 'ok' + }; + } + return { + status, + label: '加载失败', + detail: error ? `高德 Web JS SDK 加载失败：${error}` : '高德 Web JS SDK 加载失败', + diagnosticValue: 'SDK 失败', + tagLabel: 'AMap 失败', + tone: 'error', + error + }; +} + +function useAMapRuntimeProbe() { + const config = getAMapConfig(); + const [state, setState] = useState(() => + isAMapConfigured(config) ? amapRuntimeState('loading') : amapRuntimeState('not-configured') + ); + + useEffect(() => { + let mounted = true; + if (!isAMapConfigured(config)) { + setState(amapRuntimeState('not-configured')); + return () => { + mounted = false; + }; + } + setState(amapRuntimeState('loading')); + loadAMap(['AMap.Scale']) + .then(() => { + if (mounted) { + setState(amapRuntimeState('ready')); + } + }) + .catch((error: unknown) => { + if (mounted) { + setState(amapRuntimeState('error', error instanceof Error ? error.message : String(error))); + } + }); + return () => { + mounted = false; + }; + }, [config.securityJsCode, config.securityServiceHost, config.webJsKey]); + + return state; +} + +const freshnessFilterOptions: Array<{ value: FreshnessFilter; label: string }> = [ + { value: 'ALL', label: '全部新鲜度' }, + { value: '10S', label: '10秒内' }, + { value: '60S', label: '1分钟内' }, + { value: 'STALE', label: '超过1分钟' } +]; + +const detailTabs: DetailTab[] = ['概况', '实时字段', '来源', '告警']; +const detailTabURLValues: Record = { + 概况: 'overview', + 实时字段: 'fields', + 来源: 'sources', + 告警: 'alerts' +}; +const detailTabByURLValue = new Map( + Object.entries(detailTabURLValues).map(([label, value]) => [value, label as DetailTab]) +); + +const statsApiProbeDefinitions: StatsApiProbeDefinition[] = [ + { + key: 'realtime-locations', + label: '实时位置接口', + capability: '在线、位置、速度、最新总里程', + endpoint: '/api/realtime/locations?limit=1&offset=0', + required: true + }, + { + key: 'realtime-snapshots', + label: '实时快照接口', + capability: '全量扁平 parsed_json 字段', + endpoint: '/api/realtime/snapshots?limit=1&offset=0', + required: true + }, + { + key: 'daily-mileage', + label: '日里程聚合接口', + capability: '日里程、趋势、排名', + endpoint: '/api/mileage/daily?limit=1&offset=0', + required: false + } +]; + +type ChinaTimeRangePreset = { + from: string; + to: string; + label: string; +}; + +type HistoryHintOrigin = 'realtime-field' | 'track-node'; + +type HistoryFieldHint = { + protocol: Protocol; + fieldKey: string; + query: string; + selectedField: string; + origin?: HistoryHintOrigin; +}; + +type HistoryOpenContext = { + dateFrom?: string; + dateTo?: string; + fields?: string[]; + hint?: HistoryFieldHint; +}; + +type AppURLState = { + protocol?: ProtocolFilter; + status?: StatusFilter; + source?: string; + freshness?: FreshnessFilter; + detailTab?: DetailTab; + query?: string; + rawField?: string; + rawQuery?: string; + fieldPreset?: string; + rawSource?: HistoryHintOrigin; + dateFrom?: string; + dateTo?: string; +}; + +function chinaDateParts(date = new Date()) { + const formatter = new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit' + }); + const parts = formatter.formatToParts(date); + const value = (type: Intl.DateTimeFormatPartTypes) => parts.find((part) => part.type === type)?.value ?? ''; + return `${value('year')}-${value('month')}-${value('day')}`; +} + +function chinaDateTime(date = new Date()) { + const formatter = new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, + hourCycle: 'h23' + }); + const parts = formatter.formatToParts(date); + const value = (type: Intl.DateTimeFormatPartTypes) => parts.find((part) => part.type === type)?.value ?? '00'; + return `${value('year')}-${value('month')}-${value('day')}T${value('hour')}:${value('minute')}:${value('second')}+08:00`; +} + +function chinaShortDateTime(value?: string) { + const timestamp = value ? Date.parse(value) : NaN; + if (!Number.isFinite(timestamp)) { + return '暂无'; + } + return new Intl.DateTimeFormat('zh-CN', { + timeZone: 'Asia/Shanghai', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false + }).format(new Date(timestamp)); +} + +function appApiURL(path: string) { + const base = getApiBaseUrl(); + return base ? `${base}${path}` : path; +} + +function payloadItemsCount(payload: unknown) { + if (!payload || typeof payload !== 'object') { + return undefined; + } + const root = payload as { data?: unknown; items?: unknown }; + if (Array.isArray(root.items)) { + return root.items.length; + } + if (Array.isArray(root.data)) { + return root.data.length; + } + if (root.data && typeof root.data === 'object') { + const data = root.data as { items?: unknown }; + if (Array.isArray(data.items)) { + return data.items.length; + } + } + return undefined; +} + +function payloadTotalCount(payload: unknown) { + if (!payload || typeof payload !== 'object') { + return undefined; + } + const root = payload as { data?: unknown; total?: unknown }; + const directTotal = Number(root.total); + if (Number.isFinite(directTotal)) { + return directTotal; + } + if (root.data && typeof root.data === 'object') { + const dataTotal = Number((root.data as { total?: unknown }).total); + if (Number.isFinite(dataTotal)) { + return dataTotal; + } + } + return undefined; +} + +function initialStatsApiProbes(): StatsApiProbeResult[] { + return statsApiProbeDefinitions.map((definition) => ({ + ...definition, + status: 'checking', + detail: '等待浏览器探测真实接口' + })); +} + +async function probeStatsApi(definition: StatsApiProbeDefinition): Promise { + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), 4500); + const checkedAt = chinaShortDateTime(new Date().toISOString()); + try { + const response = await fetch(appApiURL(definition.endpoint), { + method: 'GET', + signal: controller.signal + }); + if (!response.ok) { + return { + ...definition, + status: response.status === 404 ? 'missing' : 'error', + httpStatus: response.status, + checkedAt, + detail: `HTTP ${response.status}，${response.status === 404 ? '后端暂未提供该能力' : response.statusText || '接口异常'}` + }; + } + const text = await response.text(); + let payload: unknown; + try { + payload = text ? JSON.parse(text) : undefined; + } catch { + payload = undefined; + } + const sampleCount = payloadItemsCount(payload); + const totalCount = payloadTotalCount(payload); + const sampleText = sampleCount == null ? '已返回响应' : `样本 ${sampleCount} 条`; + const totalText = totalCount != null && sampleCount != null && totalCount > sampleCount + ? ` / 总计 ${formatNumber(totalCount, 0)} 条` + : ''; + return { + ...definition, + status: 'ready', + httpStatus: response.status, + checkedAt, + detail: `HTTP ${response.status}，${sampleText}${totalText}` + }; + } catch (error) { + const isAbort = error instanceof DOMException && error.name === 'AbortError'; + return { + ...definition, + status: 'error', + checkedAt, + detail: isAbort ? '探测超时，暂不能作为页面能力依据' : `探测失败：${error instanceof Error ? error.message : String(error)}` + }; + } finally { + window.clearTimeout(timeout); + } +} + +function statsApiProbeTag(probe: StatsApiProbeResult) { + if (probe.status === 'ready') { + return { color: 'green', label: '可用' } as const; + } + if (probe.status === 'missing') { + return { color: 'orange', label: '未接入' } as const; + } + if (probe.status === 'error') { + return { color: 'red', label: '异常' } as const; + } + return { color: 'blue', label: '探测中' } as const; +} + +function StatsApiProbePanel({ probes }: { probes: StatsApiProbeResult[] }) { + const readyCount = probes.filter((probe) => probe.status === 'ready').length; + const checkingCount = probes.filter((probe) => probe.status === 'checking').length; + const requiredBlocked = probes.some((probe) => probe.required && probe.status !== 'ready'); + const summaryColor = checkingCount > 0 + ? 'blue' + : readyCount === probes.length + ? 'green' + : requiredBlocked + ? 'red' + : 'orange'; + return ( +
+
+
+ 真实统计接口探测 + 浏览器直接请求后端，只展示源头已支撑的统计能力 +
+ + {readyCount}/{probes.length} 可用 + +
+
+ {probes.map((probe) => { + const tag = statsApiProbeTag(probe); + return ( +
+
+ {probe.label} + {tag.label} +
+ {probe.endpoint} + {probe.capability} +

{probe.detail}

+ {probe.required ? '页面核心依赖' : '接入后再开放展示'}{probe.checkedAt ? ` / ${probe.checkedAt}` : ''} +
+ ); + })} +
+
+ ); +} + +function getStatsProbe(probes: StatsApiProbeResult[], key: string) { + return probes.find((probe) => probe.key === key); +} + +function statsInterpretationTag(tone: ReturnType[number]['tone']) { + if (tone === 'ready') return { color: 'green', label: '可用' } as const; + if (tone === 'error') return { color: 'red', label: '阻塞' } as const; + if (tone === 'waiting') return { color: 'blue', label: '等待' } as const; + return { color: 'orange', label: '关注' } as const; +} + +function StatsInterpretationPanel({ + items, + dailyMileageReady +}: { + items: ReturnType; + dailyMileageReady: boolean; +}) { + return ( +
+
+
+ 统计判读 + 先判断真实数据是否足够可信，再开放日里程、趋势和排名；不把多协议里程简单相加。 +
+ + {dailyMileageReady ? '日里程可用' : '日里程待接入'} + +
+
+ {items.map((item) => { + const tag = statsInterpretationTag(item.tone); + return ( +
+
+ {item.title} + {tag.label} +
+ {item.metric} + {item.detail} + {item.evidence} +
+ ); + })} +
+
+ ); +} + +function StatsProductionReadiness({ + probes, + realtimeRows, + usingRealData +}: { + probes: StatsApiProbeResult[]; + realtimeRows: number; + usingRealData: boolean; +}) { + const realtimeProbe = getStatsProbe(probes, 'realtime-locations'); + const snapshotProbe = getStatsProbe(probes, 'realtime-snapshots'); + const dailyMileageProbe = getStatsProbe(probes, 'daily-mileage'); + const realtimeReady = realtimeProbe?.status === 'ready' && snapshotProbe?.status === 'ready' && usingRealData && realtimeRows > 0; + const snapshotReady = snapshotProbe?.status === 'ready'; + const dailyMileageReady = dailyMileageProbe?.status === 'ready'; + const tone = realtimeReady ? dailyMileageReady ? 'ready' : 'partial' : 'blocked'; + const title = realtimeReady + ? dailyMileageReady + ? '统计生产就绪' + : '当前态统计可用，日里程待接入' + : '统计真值暂不可用'; + const detail = realtimeReady + ? dailyMileageReady + ? '实时位置、实时快照和日里程聚合接口均可用，可展示当前态、日里程和趋势入口。' + : '实时位置和快照接口已有真实来源行，只展示在线、速度、最新总里程和字段覆盖；日里程、趋势、排名继续收起。' + : '实时位置或快照接口没有可用真实来源行，不展示日里程、趋势、排名或多源统计真值。'; + + return ( +
+
+ 生产就绪度 + {title} + {detail} +
+ + + + +
+ ); +} + +function lastSeenSecondsFromText(value: string) { + const matched = value.match(/(\d+(?:\.\d+)?)\s*(秒|分钟|小时)/); + if (!matched) { + return Number.POSITIVE_INFINITY; + } + const amount = Number(matched[1]); + if (!Number.isFinite(amount)) { + return Number.POSITIVE_INFINITY; + } + if (matched[2] === '小时') { + return amount * 3600; + } + if (matched[2] === '分钟') { + return amount * 60; + } + return amount; +} + +function sourceLastSeenSeconds(source: VehicleSourceSnapshot) { + const absoluteTimestamp = source.lastSeenAt ? Date.parse(source.lastSeenAt) : NaN; + if (Number.isFinite(absoluteTimestamp)) { + return Math.max(0, Math.round((Date.now() - absoluteTimestamp) / 1000)); + } + return lastSeenSecondsFromText(source.lastSeen); +} + +function vehicleMatchesFreshness(vehicle: VehicleRecord, freshnessFilter: FreshnessFilter) { + if (freshnessFilter === 'ALL') { + return true; + } + const seconds = sourceLastSeenSeconds(getVehiclePrimarySource(vehicle)); + if (freshnessFilter === '10S') { + return seconds <= 10; + } + if (freshnessFilter === '60S') { + return seconds <= 60; + } + return seconds > 60; +} + +function vehicleHasRealtimeCoordinate(vehicle: VehicleRecord) { + return isValidAMapCoordinate(vehicle.longitude, vehicle.latitude); +} + +function firstVehicleWithUsableCoordinate(records: VehicleRecord[]) { + return records.find(vehicleHasRealtimeCoordinate) ?? records[0]; +} + +function vehicleRealtimeFieldCount(vehicle: VehicleRecord) { + return Object.values(vehicle.flatFields).reduce((sum, fields) => sum + Object.keys(fields).length, 0); +} + +function buildRealtimeQualityBuckets(records: VehicleRecord[]): RealtimeQualityBucket[] { + const fresh = records.filter((vehicle) => sourceLastSeenSeconds(getVehiclePrimarySource(vehicle)) <= 60); + const stale = records.filter((vehicle) => sourceLastSeenSeconds(getVehiclePrimarySource(vehicle)) > 60); + const missingCoordinate = records.filter((vehicle) => !vehicleHasRealtimeCoordinate(vehicle)); + const missingMileage = records.filter((vehicle) => + vehicle.sources.every((source) => source.totalMileageKm == null || source.totalMileageKm <= 0) + ); + const thinFields = records.filter((vehicle) => vehicleRealtimeFieldCount(vehicle) < 6); + + return [ + { + key: 'fresh', + title: '1分钟内活跃', + count: fresh.length, + detail: '可直接进入地图和实时字段核验', + tone: 'green', + vehicles: fresh + }, + { + key: 'stale', + title: '超过1分钟', + count: stale.length, + detail: '优先排查链路、平台转发或车辆断联', + tone: stale.length > 0 ? 'orange' : 'blue', + vehicles: stale + }, + { + key: 'coordinate', + title: '坐标不可用', + count: missingCoordinate.length, + detail: '地图无法落点，需要回查 RAW 坐标字段', + tone: missingCoordinate.length > 0 ? 'red' : 'green', + vehicles: missingCoordinate + }, + { + key: 'mileage', + title: '总里程缺失', + count: missingMileage.length, + detail: '影响里程统计，需要确认协议字段映射', + tone: missingMileage.length > 0 ? 'orange' : 'green', + vehicles: missingMileage + }, + { + key: 'fields', + title: '字段偏少', + count: thinFields.length, + detail: '实时解析字段少于 6 个，适合查 RAW 对照', + tone: thinFields.length > 0 ? 'orange' : 'green', + vehicles: thinFields + } + ]; +} + +function chinaDayRange(offsetDays = 0): ChinaTimeRangePreset { + const day = chinaDateParts(new Date(Date.now() + offsetDays * 24 * 60 * 60 * 1000)); + return { + from: `${day}T00:00:00+08:00`, + to: `${day}T23:59:59+08:00`, + label: offsetDays === 0 ? '今日' : offsetDays === -1 ? '昨日' : day + }; +} + +function chinaLastHourRange(): ChinaTimeRangePreset { + const now = new Date(); + return { + from: chinaDateTime(new Date(now.getTime() - 60 * 60 * 1000)), + to: chinaDateTime(now), + label: '近1小时' + }; +} + +function initialChinaDayRange() { + return chinaDayRange(0); +} + +const timeRangePresets: Array<{ key: 'today' | 'yesterday' | 'lastHour'; label: string; range: () => ChinaTimeRangePreset }> = [ + { key: 'today', label: '今日', range: () => chinaDayRange(0) }, + { key: 'yesterday', label: '昨日', range: () => chinaDayRange(-1) }, + { key: 'lastHour', label: '近1小时', range: chinaLastHourRange } +]; + +function isSectionKey(value: string | null): value is SectionKey { + return Boolean(value && sectionKeys.has(value as SectionKey)); +} + +function initialSectionFromURL(): SectionKey { + const params = new URLSearchParams(window.location.search); + const querySection = params.get('section'); + if (isSectionKey(querySection)) { + return querySection; + } + const hashSection = window.location.hash.replace(/^#\/?/, ''); + if (isSectionKey(hashSection)) { + return hashSection; + } + return 'command'; +} + +function initialVehicleKeyFromURL() { + const params = new URLSearchParams(window.location.search); + return (params.get('vin') || params.get('vehicle') || '').trim(); +} + +function initialQueryFromURL() { + const params = new URLSearchParams(window.location.search); + return (params.get('q') || '').trim(); +} + +function isProtocol(value: string | null): value is Protocol { + return Boolean(value && protocolOrder.includes(value as Protocol)); +} + +function isStatusFilter(value: string | null): value is StatusFilter { + return value === 'ALL' || value === 'online' || value === 'warning' || value === 'offline'; +} + +function initialStatusFromURL(): StatusFilter { + const params = new URLSearchParams(window.location.search); + const status = params.get('status'); + return isStatusFilter(status) ? status : 'ALL'; +} + +function initialSourceFromURL() { + const params = new URLSearchParams(window.location.search); + return (params.get('source') || 'ALL').trim() || 'ALL'; +} + +function isFreshnessFilter(value: string | null): value is FreshnessFilter { + return value === 'ALL' || value === '10S' || value === '60S' || value === 'STALE'; +} + +function initialFreshnessFromURL(): FreshnessFilter { + const params = new URLSearchParams(window.location.search); + const freshness = params.get('freshness'); + return isFreshnessFilter(freshness) ? freshness : 'ALL'; +} + +function isDetailTab(value: string | null): value is DetailTab { + return Boolean(value && detailTabs.includes(value as DetailTab)); +} + +function initialDetailTabFromURL(): DetailTab { + const params = new URLSearchParams(window.location.search); + const tab = params.get('tab'); + if (isDetailTab(tab)) { + return tab; + } + return detailTabByURLValue.get(tab ?? '') ?? '概况'; +} + +function initialProtocolFromURL(): ProtocolFilter { + const params = new URLSearchParams(window.location.search); + const protocol = params.get('protocol'); + if (protocol === 'ALL') { + return 'ALL'; + } + const rawField = params.get('rawField'); + return isProtocol(protocol) ? protocol : rawField ? inferProtocolFromRealtimeField(rawField) ?? 'ALL' : 'ALL'; +} + +function inferProtocolFromRealtimeField(fieldKey: string): Protocol | undefined { + const normalized = fieldKey.trim().toLowerCase(); + if (normalized.startsWith('gb32960.') || normalized.startsWith('gb.')) { + return 'GB32960'; + } + if (normalized.startsWith('jt808.') || normalized.startsWith('808.')) { + return 'JT808'; + } + if (normalized.startsWith('yutong_mqtt.') || normalized.startsWith('mqtt.')) { + return 'YUTONG_MQTT'; + } + return undefined; +} + +function updateAppURL(section: SectionKey, vehicleKey?: string, state: AppURLState = {}) { + const nextURL = new URL(window.location.href); + nextURL.searchParams.set('section', section); + if (vehicleKey) { + nextURL.searchParams.set('vin', vehicleKey); + } else { + nextURL.searchParams.delete('vin'); + } + if (state.protocol && state.protocol !== 'ALL') { + nextURL.searchParams.set('protocol', state.protocol); + } else { + nextURL.searchParams.delete('protocol'); + } + if (section === 'command') { + if (state.status && state.status !== 'ALL') { + nextURL.searchParams.set('status', state.status); + } else { + nextURL.searchParams.delete('status'); + } + if (state.source && state.source !== 'ALL') { + nextURL.searchParams.set('source', state.source); + } else { + nextURL.searchParams.delete('source'); + } + if (state.freshness && state.freshness !== 'ALL') { + nextURL.searchParams.set('freshness', state.freshness); + } else { + nextURL.searchParams.delete('freshness'); + } + if (state.detailTab && state.detailTab !== '概况') { + nextURL.searchParams.set('tab', detailTabURLValues[state.detailTab]); + } else { + nextURL.searchParams.delete('tab'); + } + if (state.query) { + nextURL.searchParams.set('q', state.query); + } else { + nextURL.searchParams.delete('q'); + } + } else { + nextURL.searchParams.delete('status'); + nextURL.searchParams.delete('source'); + nextURL.searchParams.delete('freshness'); + nextURL.searchParams.delete('tab'); + nextURL.searchParams.delete('q'); + } + if (section === 'history' && state.rawField) { + nextURL.searchParams.set('rawField', state.rawField); + if (state.rawSource) { + nextURL.searchParams.set('rawSource', state.rawSource); + } else { + nextURL.searchParams.delete('rawSource'); + } + if (state.rawQuery) { + nextURL.searchParams.set('rawQuery', state.rawQuery); + } else { + nextURL.searchParams.delete('rawQuery'); + } + if (state.fieldPreset) { + nextURL.searchParams.set('fieldPreset', state.fieldPreset); + } else { + nextURL.searchParams.delete('fieldPreset'); + } + } else { + nextURL.searchParams.delete('rawField'); + nextURL.searchParams.delete('rawQuery'); + nextURL.searchParams.delete('fieldPreset'); + nextURL.searchParams.delete('rawSource'); + } + if (section === 'history') { + if (state.dateFrom) { + nextURL.searchParams.set('dateFrom', state.dateFrom); + } else { + nextURL.searchParams.delete('dateFrom'); + } + if (state.dateTo) { + nextURL.searchParams.set('dateTo', state.dateTo); + } else { + nextURL.searchParams.delete('dateTo'); + } + } else { + nextURL.searchParams.delete('dateFrom'); + nextURL.searchParams.delete('dateTo'); + } + nextURL.hash = ''; + window.history.replaceState(null, '', nextURL); +} + +function findVehicleByKey(records: VehicleRecord[], key: string) { + const normalized = key.trim().toLowerCase(); + if (!normalized) { + return undefined; + } + return records.find((vehicle) => + [vehicle.vin, vehicle.plate, vehicle.phone].some((value) => value?.toLowerCase() === normalized) + ); +} + +function rawFieldQueryFromRealtimeKey(fieldKey: string) { + const normalized = fieldKey.toLowerCase(); + if (normalized.includes('speed')) return 'speed'; + if (normalized.includes('total_mileage') || normalized.includes('mileage')) return 'total_mileage'; + if (normalized.includes('soc') || normalized.includes('battery_capacity')) return 'soc'; + if (normalized.includes('hydrogen') || normalized.includes('left_hydrogen')) return 'hydrogen'; + if (normalized.includes('latitude') || normalized.endsWith('.lat')) return 'latitude'; + if (normalized.includes('longitude') || normalized.endsWith('.lng')) return 'longitude'; + if (normalized.includes('direction')) return 'direction'; + if (normalized.includes('fc_stack') || normalized.includes('fuel_cell')) return 'fc_stack'; + if (normalized.includes('event_time') || normalized.includes('device_time') || normalized.includes('received_at')) return 'time'; + const parts = fieldKey.split('.').filter(Boolean); + return parts.slice(-2).join('.') || fieldKey; +} + +function selectedHistoryFieldFromRealtimeKey(fieldKey: string) { + const normalized = fieldKey.toLowerCase(); + if (normalized.includes('speed')) return 'speed_kph'; + if (normalized.includes('total_mileage') || normalized.includes('mileage')) return 'total_mileage_km'; + if (normalized.includes('soc') || normalized.includes('battery_capacity')) return 'soc_pct'; + if (normalized.includes('hydrogen') || normalized.includes('left_hydrogen')) return 'left_hydrogen_kg'; + if (normalized.includes('latitude') || normalized.includes('longitude') || normalized.endsWith('.lat') || normalized.endsWith('.lng')) return 'lat_lon'; + if (normalized.includes('direction')) return 'direction_deg'; + if (normalized.includes('fc_stack') || normalized.includes('fuel_cell')) return 'fc_stack'; + if (normalized.includes('event_time') || normalized.includes('device_time') || normalized.includes('received_at')) return 'quality_delay'; + return fieldKey; +} + +function historyFieldHintFromURL(): HistoryFieldHint | undefined { + const params = new URLSearchParams(window.location.search); + const fieldKey = (params.get('rawField') || '').trim(); + if (!fieldKey) { + return undefined; + } + const protocolParam = params.get('protocol'); + const protocol = isProtocol(protocolParam) + ? protocolParam + : inferProtocolFromRealtimeField(fieldKey) ?? 'GB32960'; + const query = (params.get('rawQuery') || '').trim() || rawFieldQueryFromRealtimeKey(fieldKey); + const selectedField = (params.get('fieldPreset') || '').trim() || selectedHistoryFieldFromRealtimeKey(fieldKey); + return { + protocol, + fieldKey, + query, + selectedField, + origin: params.get('rawSource') === 'track-node' ? 'track-node' : 'realtime-field' + }; +} + +function initialHistoryDateFrom() { + const params = new URLSearchParams(window.location.search); + const range = initialChinaDayRange(); + return initialSectionFromURL() === 'history' ? params.get('dateFrom') || range.from : range.from; +} + +function initialHistoryDateTo() { + const params = new URLSearchParams(window.location.search); + const range = initialChinaDayRange(); + return initialSectionFromURL() === 'history' ? params.get('dateTo') || range.to : range.to; +} + +function historyVehicleFromKey(key: string, base: VehicleRecord): VehicleRecord { + const normalized = key.trim(); + const sourceId = `history-source-${normalized}`; + return { + ...base, + id: `history-vin-${normalized}`, + plate: normalized, + vin: normalized, + phone: undefined, + oem: '历史查询', + status: 'offline', + city: '--', + address: '按 VIN 查询历史轨迹', + primarySourceId: sourceId, + sources: [{ + id: sourceId, + protocol: 'GB32960', + sourceName: 'TDengine 历史轨迹', + endpoint: '/api/history/locations', + platformAccount: 'vehicle_locations', + status: 'offline', + latencyMs: 0, + lastSeen: '按时间窗查询', + fieldCount: 0 + }], + flatFields: { + GB32960: {}, + JT808: {}, + YUTONG_MQTT: {} + } + }; +} + +function AppSelect({ + value, + options, + onChange, + ariaLabel +}: { + value: T; + options: Array<{ value: T; label: string }>; + onChange: (value: T) => void; + ariaLabel: string; +}) { + return ( + + ); +} + +function TimeRangePresetButtons({ + onApply +}: { + onApply: (range: ChinaTimeRangePreset) => void; +}) { + return ( +
+ {timeRangePresets.map((preset) => ( + + ))} +
+ ); +} + +export default function PrototypeApp() { + const [activeSection, setActiveSection] = useState(() => initialSectionFromURL()); + const [query, setQuery] = useState(() => initialQueryFromURL()); + const [protocolFilter, setProtocolFilter] = useState(() => initialProtocolFromURL()); + const [statusFilter, setStatusFilter] = useState(() => initialStatusFromURL()); + const [sourceFilter, setSourceFilter] = useState(() => initialSourceFromURL()); + const [freshnessFilter, setFreshnessFilter] = useState(() => initialFreshnessFromURL()); + const [activeVehicleId, setActiveVehicleId] = useState(vehicles[0].id); + const [detailTab, setDetailTab] = useState(() => initialDetailTabFromURL()); + const [playbackIndex, setPlaybackIndex] = useState(0); + const [isPlaying, setIsPlaying] = useState(false); + const [historyQueried, setHistoryQueried] = useState(true); + const [selectedFields, setSelectedFields] = useState(() => { + const hint = initialSectionFromURL() === 'history' ? historyFieldHintFromURL() : undefined; + return hint ? [hint.selectedField] : defaultHistoryFields; + }); + const [historyPage, setHistoryPage] = useState(1); + const [historyPageSize, setHistoryPageSize] = useState(3); + const [historyDateFrom, setHistoryDateFrom] = useState(initialHistoryDateFrom); + const [historyDateTo, setHistoryDateTo] = useState(initialHistoryDateTo); + const [historyFieldHint, setHistoryFieldHint] = useState(() => + initialSectionFromURL() === 'history' ? historyFieldHintFromURL() : undefined + ); + const [realData, setRealData] = useState({ + vehicles: [], + status: 'loading', + source: realtimeQuerySource + }); + const [autoRealtimeRefresh, setAutoRealtimeRefresh] = useState(true); + const realtimeRequestId = useRef(0); + const amapRuntime = useAMapRuntimeProbe(); + const refreshRealtimeData = useCallback((reason: 'initial' | 'manual' | 'auto' = 'manual') => { + const requestId = realtimeRequestId.current + 1; + realtimeRequestId.current = requestId; + setRealData((current) => ({ + ...current, + status: 'loading', + source: realtimeQuerySource, + error: reason === 'auto' ? current.error : undefined + })); + loadPrototypeRealtimeVehicles(realtimeQueryLimit).then((next) => { + if (realtimeRequestId.current !== requestId) { + return; + } + setRealData((current) => { + if (next.status === 'error' && current.vehicles.length > 0) { + return { + ...next, + status: 'stale', + vehicles: current.vehicles, + fetchedAt: current.fetchedAt, + total: current.total ?? current.vehicles.length, + source: `${next.source} / in-memory cache`, + error: `live request failed: ${next.error ?? 'unknown error'}` + }; + } + return next; + }); + }); + }, []); + + const realVehicles = realData.vehicles; + const allowMockDataset = realData.status === 'disabled'; + const canQueryRealRaw = realData.status !== 'disabled'; + const canQueryRealTrack = realData.status !== 'disabled'; + const canQueryRealSources = realData.status !== 'disabled'; + const canQueryRealAlerts = realData.status !== 'disabled'; + const vehicleDataset = realVehicles.length > 0 ? realVehicles : allowMockDataset ? vehicles : []; + const usingRealData = realVehicles.length > 0; + const realtimeMode = realtimeModeSummary(realData, usingRealData); + const sourceOptions = useMemo(() => { + if (!allowMockDataset && realVehicles.length === 0) { + return []; + } + const options = new Map(); + if (allowMockDataset) { + dataSources.forEach((source) => options.set(source.id, source.name)); + } + vehicleDataset.forEach((vehicle) => { + vehicle.sources.forEach((source) => options.set(source.id, source.sourceName)); + }); + return Array.from(options.entries()).map(([value, label]) => ({ value, label })); + }, [allowMockDataset, realVehicles.length, vehicleDataset]); + const baseFilteredVehicles = useMemo( + () => filterVehicles(query, protocolFilter, statusFilter, sourceFilter, vehicleDataset), + [protocolFilter, query, sourceFilter, statusFilter, vehicleDataset] + ); + const filteredVehicles = useMemo( + () => activeSection === 'command' + ? baseFilteredVehicles.filter((vehicle) => vehicleMatchesFreshness(vehicle, freshnessFilter)) + : baseFilteredVehicles, + [activeSection, baseFilteredVehicles, freshnessFilter] + ); + const statsVehicles = useMemo( + () => (query.trim() ? vehicleDataset.filter((vehicle) => includesQuery(vehicle, query)) : vehicleDataset), + [query, vehicleDataset] + ); + const routeVehicleKey = initialVehicleKeyFromURL(); + const routeVehicle = useMemo( + () => findVehicleByKey(vehicleDataset, routeVehicleKey), + [routeVehicleKey, vehicleDataset] + ); + + const activeVehicle = useMemo(() => { + if (activeSection === 'replay' && routeVehicleKey) { + return routeVehicle ?? historyVehicleFromKey(routeVehicleKey, vehicleDataset[0] ?? vehicles[0]); + } + const current = filteredVehicles.find((vehicle) => vehicle.id === activeVehicleId); + return current && vehicleHasRealtimeCoordinate(current) + ? current + : firstVehicleWithUsableCoordinate(filteredVehicles) ?? firstVehicleWithUsableCoordinate(vehicleDataset) ?? vehicles[0]; + }, [activeSection, activeVehicleId, filteredVehicles, routeVehicle, routeVehicleKey, vehicleDataset]); + + const activeSource = getVehiclePrimarySource(activeVehicle); + const commandURLState = (overrides: AppURLState = {}): AppURLState => ({ + protocol: protocolFilter, + status: statusFilter, + source: sourceFilter, + freshness: freshnessFilter, + detailTab, + query, + ...overrides + }); + const syncCommandURL = (vehicleKey = activeVehicle.vin, overrides: AppURLState = {}) => { + if (activeSection === 'command') { + updateAppURL('command', vehicleKey, commandURLState(overrides)); + } + }; + const setCommandQuery = (value: string) => { + setQuery(value); + syncCommandURL(activeVehicle.vin, { query: value }); + }; + const setCommandProtocolFilter = (value: ProtocolFilter) => { + setProtocolFilter(value); + syncCommandURL(activeVehicle.vin, { protocol: value }); + }; + const setCommandStatusFilter = (value: StatusFilter) => { + setStatusFilter(value); + syncCommandURL(activeVehicle.vin, { status: value }); + }; + const setCommandSourceFilter = (value: string) => { + setSourceFilter(value); + syncCommandURL(activeVehicle.vin, { source: value }); + }; + const setCommandFreshnessFilter = (value: FreshnessFilter) => { + setFreshnessFilter(value); + syncCommandURL(activeVehicle.vin, { freshness: value }); + }; + const setCommandDetailTab = (tab: DetailTab) => { + setDetailTab(tab); + syncCommandURL(activeVehicle.vin, { detailTab: tab }); + }; + const selectSection = (section: SectionKey) => { + setActiveSection(section); + updateAppURL(section, activeVehicle.vin, section === 'command' ? commandURLState() : {}); + }; + const selectVehicle = (vehicleId: string) => { + setActiveVehicleId(vehicleId); + const vehicle = vehicleDataset.find((item) => item.id === vehicleId); + updateAppURL(activeSection, vehicle?.vin, activeSection === 'command' ? commandURLState() : {}); + }; + const openVehicleReplayForVin = (vin = activeVehicle.vin) => { + const matchedVehicle = vehicleDataset.find((vehicle) => vehicle.vin === vin); + if (matchedVehicle) { + setActiveVehicleId(matchedVehicle.id); + } + setPlaybackIndex(0); + setIsPlaying(false); + setActiveSection('replay'); + updateAppURL('replay', matchedVehicle?.vin ?? vin); + }; + const openVehicleCommandForVin = (vin = activeVehicle.vin) => { + const matchedVehicle = vehicleDataset.find((vehicle) => vehicle.vin === vin); + if (matchedVehicle) { + setActiveVehicleId(matchedVehicle.id); + setQuery(matchedVehicle.vin); + } + setProtocolFilter('ALL'); + setStatusFilter('ALL'); + setSourceFilter('ALL'); + setFreshnessFilter('ALL'); + setDetailTab('概况'); + setActiveSection('command'); + updateAppURL('command', matchedVehicle?.vin ?? vin); + }; + const openVehicleHistoryForVin = (vin = activeVehicle.vin, protocol?: Protocol, context?: HistoryOpenContext) => { + const matchedVehicle = vehicleDataset.find((vehicle) => vehicle.vin === vin); + if (matchedVehicle) { + setActiveVehicleId(matchedVehicle.id); + } + setHistoryFieldHint(context?.hint); + if (context?.fields?.length) { + setSelectedFields(context.fields); + } + if (context?.dateFrom) { + setHistoryDateFrom(context.dateFrom); + } + if (context?.dateTo) { + setHistoryDateTo(context.dateTo); + } + setQuery(matchedVehicle?.vin ?? vin); + setProtocolFilter(protocol ?? 'ALL'); + setStatusFilter('ALL'); + setSourceFilter('ALL'); + setHistoryPage(1); + setHistoryQueried(true); + setActiveSection('history'); + updateAppURL('history', matchedVehicle?.vin ?? vin, { + protocol: protocol ?? 'ALL', + dateFrom: context?.dateFrom, + dateTo: context?.dateTo, + rawField: context?.hint?.fieldKey, + rawQuery: context?.hint?.query, + fieldPreset: context?.hint?.selectedField, + rawSource: context?.hint?.origin + }); + }; + const openVehicleHistoryForField = (vin: string, protocol: Protocol, fieldKey: string) => { + const matchedVehicle = vehicleDataset.find((vehicle) => vehicle.vin === vin); + const selectedField = selectedHistoryFieldFromRealtimeKey(fieldKey); + const nextHint = { + protocol, + fieldKey, + query: rawFieldQueryFromRealtimeKey(fieldKey), + selectedField, + origin: 'realtime-field' as const + }; + if (matchedVehicle) { + setActiveVehicleId(matchedVehicle.id); + } + setHistoryFieldHint(nextHint); + setSelectedFields([selectedField]); + setQuery(matchedVehicle?.vin ?? vin); + setProtocolFilter(protocol); + setStatusFilter('ALL'); + setSourceFilter('ALL'); + setHistoryPage(1); + setHistoryQueried(true); + setActiveSection('history'); + updateAppURL('history', matchedVehicle?.vin ?? vin, { + protocol, + rawField: nextHint.fieldKey, + rawQuery: nextHint.query, + fieldPreset: nextHint.selectedField, + rawSource: nextHint.origin + }); + }; + const openProtocolHistory = (protocol: Protocol) => { + setHistoryFieldHint(undefined); + setQuery(''); + setProtocolFilter(protocol); + setStatusFilter('ALL'); + setSourceFilter('ALL'); + setHistoryPage(1); + setHistoryQueried(true); + setActiveSection('history'); + updateAppURL('history', undefined, { protocol }); + }; + const kpis = usingRealData + ? getRealtimeKpisForVehicles(filteredVehicles, protocolFilter) + : getKpis(filteredVehicles, protocolFilter); + const activeTrack = trackPoints.filter((point) => point.vehicleId === activeVehicle.id); + const activeAlerts = alertEvents.filter((alert) => alert.vehicleId === activeVehicle.id); + const isEvidenceSection = activeSection === 'history' || activeSection === 'replay' || activeSection === 'alerts'; + const isCommandSection = activeSection === 'command'; + const isStatsSection = activeSection === 'stats'; + const isSourceSection = activeSection === 'sources'; + const showOperationalSummary = !isCommandSection && !isEvidenceSection && !isStatsSection && !isSourceSection; + const visibleHistory = historyRecords.filter((record) => { + const vehicle = vehicleDataset.find((item) => item.id === record.vehicleId); + const protocolMatched = protocolFilter === 'ALL' || record.protocol === protocolFilter; + const queryMatched = vehicle ? includesQuery(vehicle, query) : true; + return protocolMatched && queryMatched; + }); + const historyContextVehicle = query.trim() ? findVehicleByKey(vehicleDataset, query) : undefined; + const historyQueryPreview = useMemo( + () => + buildHistoryQueryPreview({ + keyword: query, + protocolFilter, + selectedFields, + dateFrom: historyDateFrom, + dateTo: historyDateTo, + limit: historyPageSize, + offset: (historyPage - 1) * historyPageSize + }), + [historyDateFrom, historyDateTo, historyPage, historyPageSize, protocolFilter, query, selectedFields] + ); + + useEffect(() => { + refreshRealtimeData('initial'); + }, [refreshRealtimeData]); + + useEffect(() => { + if (!autoRealtimeRefresh) { + return undefined; + } + const timer = window.setInterval(() => { + refreshRealtimeData('auto'); + }, realtimeRefreshIntervalMs); + return () => { + window.clearInterval(timer); + }; + }, [autoRealtimeRefresh, refreshRealtimeData]); + + useEffect(() => { + if (sourceFilter === 'ALL') { + return; + } + if (!sourceOptions.some((option) => option.value === sourceFilter)) { + setSourceFilter('ALL'); + } + }, [sourceFilter, sourceOptions]); + + useEffect(() => { + if (filteredVehicles.length > 0 && !filteredVehicles.some((vehicle) => vehicle.id === activeVehicleId)) { + setActiveVehicleId(firstVehicleWithUsableCoordinate(filteredVehicles)?.id ?? filteredVehicles[0].id); + } + }, [activeVehicleId, filteredVehicles]); + + useEffect(() => { + const vehicleKey = initialVehicleKeyFromURL().toLowerCase(); + if (!vehicleKey) { + return; + } + const matchedVehicle = findVehicleByKey(vehicleDataset, vehicleKey); + if (matchedVehicle && matchedVehicle.id !== activeVehicleId) { + setActiveVehicleId(matchedVehicle.id); + } + }, [activeVehicleId, vehicleDataset]); + + useEffect(() => { + const vehicleKey = initialVehicleKeyFromURL(); + if (activeSection !== 'history' || !vehicleKey || query === vehicleKey) { + return; + } + setQuery(vehicleKey); + setHistoryPage(1); + setHistoryQueried(true); + }, [activeSection, query]); + + useEffect(() => { + setPlaybackIndex(0); + }, [activeVehicle.id]); + + useEffect(() => { + window.scrollTo({ top: 0, left: 0 }); + }, [activeSection]); + + useEffect(() => { + const totalPages = Math.max(1, Math.ceil(visibleHistory.length / historyPageSize)); + if (historyPage > totalPages) { + setHistoryPage(totalPages); + } + }, [historyPage, historyPageSize, visibleHistory.length]); + + return ( +
+ + +
+
+
+

{sectionTitle[activeSection].title}

+

{sectionTitle[activeSection].subtitle}

+
+
+ } placeholder="搜索车牌 / VIN / 手机号 / 城市" value={query} onChange={setCommandQuery} /> +
+
+ + + + + {!isEvidenceSection && !isSourceSection && ( + refreshRealtimeData('manual')} + realData={realData} + usingRealData={usingRealData} + /> + )} + + {showOperationalSummary && ( + + )} + + {showOperationalSummary && ( +
+ {kpis.map((kpi) => ( +
+ {kpi.label} + {kpi.value} + {kpi.sub} +
+ ))} +
+ )} + + {activeSection === 'command' && ( + openVehicleHistoryForVin(activeVehicle.vin, protocol)} + onOpenHistoryField={openVehicleHistoryForField} + onOpenReplay={() => openVehicleReplayForVin()} + onSelectVehicle={selectVehicle} + realData={realData} + setDetailTab={setCommandDetailTab} + /> + )} + + {activeSection === 'replay' && ( + + )} + + {activeSection === 'history' && ( + { + setHistoryFieldHint(undefined); + setSelectedFields(defaultHistoryFields); + setHistoryPage(1); + updateAppURL('history', query || activeVehicle.vin, { protocol: protocolFilter }); + }} + onOpenCommand={openVehicleCommandForVin} + onOpenReplay={openVehicleReplayForVin} + protocolFilter={protocolFilter} + query={query} + queryPreview={historyQueryPreview} + records={visibleHistory} + selectedFields={selectedFields} + setHistoryDateFrom={setHistoryDateFrom} + setHistoryDateTo={setHistoryDateTo} + setHistoryPage={setHistoryPage} + setHistoryPageSize={setHistoryPageSize} + setHistoryQueried={setHistoryQueried} + setSelectedFields={setSelectedFields} + /> + )} + + {activeSection === 'alerts' && ( + + )} + + {activeSection === 'stats' && ( + + )} + + {activeSection === 'sources' && ( + + )} +
+
+ ); +} + +type RealtimeModeSummary = { + boundary: string; + sideDetail: string; + sideLabel: string; + tone: 'ready' | 'loading' | 'stale' | 'error' | 'fallback' | 'disabled'; +}; + +function realtimeModeSummary(realData: PrototypeRealDataState, usingRealData: boolean): RealtimeModeSummary { + if (realData.status === 'ready' && usingRealData) { + return { + boundary: `真实实时 / ${realData.vehicles.length}/${realData.total ?? realData.vehicles.length} 条位置`, + sideDetail: 'ECS 实时投影', + sideLabel: '真实实时', + tone: 'ready' + }; + } + if (realData.status === 'loading' && usingRealData) { + return { + boundary: '真实刷新中 / 保持上一批点位', + sideDetail: '刷新中', + sideLabel: '真实刷新中', + tone: 'loading' + }; + } + if (realData.status === 'stale' && usingRealData) { + return { + boundary: '真实缓存 / 实时 API 当前失败', + sideDetail: '最近一次真实快照', + sideLabel: '真实缓存', + tone: 'stale' + }; + } + if (realData.status === 'loading') { + if (realData.error && realData.vehicles.length === 0) { + return { + boundary: '重试中 / 上次 realtime API 失败', + sideDetail: '无生产车辆', + sideLabel: '重试中', + tone: 'loading' + }; + } + return { + boundary: '读取中 / 等待 realtime API', + sideDetail: '等待接口', + sideLabel: '读取中', + tone: 'loading' + }; + } + if (realData.status === 'error') { + return { + boundary: 'API 失败 / 不展示样例当前态', + sideDetail: '无生产车辆', + sideLabel: 'API 失败', + tone: 'error' + }; + } + if (realData.status === 'fallback') { + return { + boundary: '真实为空 / 不展示样例当前态', + sideDetail: '无真实车辆', + sideLabel: '真实为空', + tone: 'fallback' + }; + } + return { + boundary: '真实数据未启用 / 当前为样例模式', + sideDetail: '原型模式', + sideLabel: '样例模式', + tone: 'disabled' + }; +} + +function DataBoundary({ + activeSection, + canQueryRealAlerts, + canQueryRealSources, + realtimeMode, + usingRealData +}: { + activeSection: SectionKey; + canQueryRealAlerts: boolean; + canQueryRealSources: boolean; + realtimeMode: RealtimeModeSummary; + usingRealData: boolean; +}) { + const currentBoundary = (() => { + if (realtimeMode.tone === 'disabled') { + return realtimeMode.boundary; + } + if (activeSection === 'history') { + return '历史 RAW 独立查询 / 实时当前态不参与'; + } + if (activeSection === 'replay') { + return '历史轨迹独立查询 / 实时当前态不参与'; + } + if (activeSection === 'stats') { + return usingRealData + ? '真实实时来源统计 / 统计 API 独立探测' + : '统计 API 独立探测 / 不展示样例真值'; + } + if (activeSection === 'alerts' && canQueryRealAlerts) { + return '告警观测独立探测 / durable alert API 未接入'; + } + if (activeSection === 'sources' && canQueryRealSources) { + return '来源证据独立探测 / realtime 与 RAW 可部分成功'; + } + return realtimeMode.boundary; + })(); + const sourceBoundaries = activeSection === 'stats' + ? usingRealData + ? [ + { label: 'Realtime API', value: 'locations + snapshots 当前可用' }, + { label: '未接入', value: '/api/mileage/daily、/api/statistics 当前 404' } + ] + : [ + { label: 'Realtime API', value: 'locations + snapshots 等待返回真实来源行' }, + { label: '统计 API', value: '日里程、趋势、排名未接入前不展示真值' } + ] + : activeSection === 'alerts' + ? canQueryRealAlerts + ? [ + { label: '观测信号', value: '/api/realtime/locations 与 /api/history/locations 独立探测' }, + { label: '持久告警', value: 'durable alert 事件、通知、派单接口未接入' } + ] + : sectionDataBoundaries[activeSection] + : activeSection === 'sources' + ? canQueryRealSources + ? [ + { label: '来源证据', value: '/api/realtime/locations 与 raw_frames/query 独立探测' }, + { label: '平台台账', value: 'vehicle_data_source / ops API 未接入前只展示端点证据' } + ] + : [ + { label: '来源证据', value: '等待 realtime locations 与 raw_frames 返回真实样本' }, + { label: '平台台账', value: 'vehicle_data_source / ops API 未接入前不展示真值' } + ] + : sectionDataBoundaries[activeSection]; + const boundaries = [ + { label: '当前口径', value: currentBoundary }, + ...sourceBoundaries + ]; + return ( +
+ 当前模块数据来源 +
+ {boundaries.map((item) => ( + + {item.label} + {' '} + {item.value} + + ))} +
+
+ ); +} + +function OperationFlow({ + filteredVehicles, + activeVehicle, + activeSource, + usingRealData +}: { + filteredVehicles: VehicleRecord[]; + activeVehicle: VehicleRecord; + activeSource: ReturnType; + usingRealData: boolean; +}) { + const sourceCount = activeVehicle.sources.length; + return ( +
+
+ 01 + 接收实时帧 + {filteredVehicles.length} 台筛选车辆进入地图与列表 +
+
+ 02 + 解析扁平字段 + {activeSource.sourceName} 当前 {activeSource.fieldCount} 个字段 +
+
+ 03 + 合并实时投影 + {activeVehicle.plate} 关联 {sourceCount} 路来源 +
+
+ 04 + 证据与边界 + {usingRealData ? '告警/质量 API 未接入，当前只输出观测信号' : `${alertEvents.length} 条本地样例事件，仅验证交互`} +
+
+ ); +} + +function getRealtimeKpisForVehicles(filteredVehicles: VehicleRecord[], protocolFilter: ProtocolFilter = 'ALL') { + const sources = filteredVehicles.flatMap((vehicle) => + vehicle.sources.filter((source) => protocolFilter === 'ALL' || source.protocol === protocolFilter) + ); + const online = filteredVehicles.filter((vehicle) => vehicle.status === 'online').length; + const movingSources = sources.filter((source) => (source.speedKph ?? 0) > 0).length; + const totalMileage = sources.reduce((sum, source) => sum + (source.totalMileageKm ?? 0), 0); + const fieldCount = sources.reduce((sum, source) => sum + source.fieldCount, 0); + const avgLatency = Math.round(sources.reduce((sum, source) => sum + source.latencyMs, 0) / Math.max(sources.length, 1)); + return [ + { label: '筛选车辆', value: formatNumber(filteredVehicles.length), sub: `在线 ${online} / 来源行 ${sources.length}`, tone: 'blue' }, + { label: '实时行驶', value: String(movingSources), sub: '按来源车速 > 0 判断', tone: 'green' }, + { label: '最新总里程', value: `${formatNumber(totalMileage, 1)} km`, sub: '按当前来源行合计', tone: 'orange' }, + { label: '字段/延迟', value: `${formatNumber(fieldCount)}`, sub: `字段数 / 平均 ${avgLatency} ms`, tone: 'cyan' } + ] as const; +} + +function GlobalFilters({ + activeSection, + protocolFilter, + setProtocolFilter, + freshnessFilter, + setFreshnessFilter, + statusFilter, + setStatusFilter, + sourceFilter, + setSourceFilter, + sourceOptions +}: { + activeSection: SectionKey; + protocolFilter: ProtocolFilter; + setProtocolFilter: (value: ProtocolFilter) => void; + freshnessFilter: FreshnessFilter; + setFreshnessFilter: (value: FreshnessFilter) => void; + statusFilter: StatusFilter; + setStatusFilter: (value: StatusFilter) => void; + sourceFilter: string; + setSourceFilter: (value: string) => void; + sourceOptions: Array<{ value: string; label: string }>; +}) { + const isHistory = activeSection === 'history'; + const isReplay = activeSection === 'replay'; + const isStats = activeSection === 'stats'; + const isAlerts = activeSection === 'alerts'; + const isSources = activeSection === 'sources'; + const isEvidenceQuery = isHistory || isReplay; + const onlyUsesProtocol = isEvidenceQuery || isStats || isAlerts || isSources; + const sourceSelectOptions = [ + { value: 'ALL', label: sourceOptions.length === 0 ? '真实来源待读取' : '全部来源' }, + ...sourceOptions + ]; + const sourceSelectValue = sourceSelectOptions.some((option) => option.value === sourceFilter) ? sourceFilter : 'ALL'; + return ( +
+ + {!onlyUsesProtocol && } + {!onlyUsesProtocol && ( + + )} + {!onlyUsesProtocol && } +
+ {isHistory ? ( + <> + RAW 按 VIN / 协议 / 字段查询 + 状态和来源不参与历史 RAW 请求 + + ) : isReplay ? ( + <> + 轨迹按 VIN / 时间窗查询 + 状态和来源不参与轨迹回放请求 + + ) : isStats ? ( + <> + 统计按搜索车辆池 / 协议计算 + 状态和来源筛选不参与统计快照 + + ) : isAlerts ? ( + <> + 告警按观测信号 / 协议查询 + 状态和来源筛选不参与告警观测 + + ) : isSources ? ( + <> + 来源按搜索词 / 协议筛选 + 状态和来源筛选不参与端点证据 + + ) : ( + <> + 实时帧进入展示，支持最近上报时间筛选 + {sourceOptions.length === 0 ? '真实来源未返回，来源筛选等待 realtime 数据' : '登录/鉴权只进入接入审计'} + + )} +
+
+ ); +} + +function realtimeErrorDetail(error?: string) { + const message = String(error ?? '').trim(); + if (!message) { + return '等待接口响应'; + } + const normalized = message.toLowerCase(); + if ( + normalized.includes('request timeout') || + normalized.includes('abort') || + normalized.includes('signal is aborted') + ) { + const timeoutMatch = message.match(/after\s+(\d+)ms/i); + const timeoutText = timeoutMatch ? `${Math.round(Number(timeoutMatch[1]) / 1000)} 秒` : '超时时间内'; + return `实时接口超时，后端未在 ${timeoutText}内返回`; + } + if (normalized.includes('http 500')) { + return normalized.includes('empty response body') + ? '实时接口返回 500 且响应体为空，需要检查 BFF 日志或数据库依赖' + : '实时接口返回 500，需要检查 BFF 或数据库依赖'; + } + if (normalized.includes('http 404')) { + return '实时接口未部署或路径不匹配'; + } + if (normalized.includes('failed to fetch') || normalized.includes('network')) { + return '实时接口网络不可达或连接被关闭'; + } + if (normalized.includes('empty')) { + return '实时接口暂未返回车辆数据'; + } + return `实时接口异常：${message}`; +} + +function realtimeHTTPStatus(error?: string) { + const match = String(error ?? '').match(/HTTP\s+(\d{3})/i); + return match ? match[1] : undefined; +} + +function realtimeHTTPBodyState(error?: string) { + const message = String(error ?? '').trim(); + if (!message || !/HTTP\s+\d{3}/i.test(message)) { + return undefined; + } + return message.toLowerCase().includes('empty response body') ? '空 body' : '有错误明细'; +} + +function firstRequestSource(source: string) { + return String(source).split(' / ')[0].trim(); +} + +function absoluteRuntimeURL(source: string) { + const requestSource = firstRequestSource(source); + if (!requestSource) { + return ''; + } + if (/^https?:\/\//i.test(requestSource)) { + return requestSource; + } + if (typeof window === 'undefined') { + return requestSource; + } + const path = requestSource.startsWith('/') ? requestSource : `/${requestSource}`; + return `${window.location.origin}${path}`; +} + +function shellQuote(value: string) { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +function curlForRuntimeSource(source: string) { + const url = absoluteRuntimeURL(source); + return url ? `curl -i --max-time 12 ${shellQuote(url)}` : 'n/a'; +} + +function amapConfigDiagnostic() { + const config = getAMapConfig(); + if (!config.webJsKey) { + return 'web-js-key:missing, security:not-configured'; + } + if (config.securityServiceHost) { + return `web-js-key:configured, security-proxy:${config.securityServiceHost}`; + } + if (config.securityJsCode) { + return 'web-js-key:configured, security:browser-js-code'; + } + return 'web-js-key:configured, security:not-configured'; +} + +function realtimeDiagnosticText( + realData: PrototypeRealDataState, + usingRealData: boolean, + amapRuntime: AMapRuntimeState +) { + const httpStatus = realtimeHTTPStatus(realData.error); + const httpBodyState = realtimeHTTPBodyState(realData.error); + const diagnosticItems = realtimeDiagnosticItems(realData, usingRealData, amapRuntime); + const realtimeURL = absoluteRuntimeURL(realData.source); + return [ + '[车辆数据中台 realtime 诊断]', + `time=${new Date().toLocaleString('zh-CN', { hour12: false })}`, + `source=${realData.source}`, + `request=${realtimeURL || firstRequestSource(realData.source) || 'n/a'}`, + `curl=${curlForRuntimeSource(realData.source)}`, + `status=${realData.status}`, + `vehicles=${realData.vehicles.length}`, + `total=${realData.total ?? realData.vehicles.length}`, + `usingRealData=${usingRealData ? 'yes' : 'no'}`, + `httpStatus=${httpStatus ?? 'n/a'}`, + `responseBody=${httpBodyState ?? 'n/a'}`, + `error=${realData.error ?? 'n/a'}`, + `message=${realtimeErrorDetail(realData.error)}`, + `amap=${amapRuntime.label} / ${amapRuntime.detail}`, + `amapConfig=${amapConfigDiagnostic()}`, + `diagnostics=${diagnosticItems.map((item) => `${item.label}:${item.value}`).join(', ')}` + ].join('\n'); +} + +async function copyTextWithFallback(value: string) { + try { + await navigator.clipboard.writeText(value); + return; + } catch { + // Some embedded/local browser contexts deny Clipboard API writes even after a click. + } + const textarea = document.createElement('textarea'); + textarea.value = value; + textarea.setAttribute('readonly', 'true'); + textarea.style.position = 'fixed'; + textarea.style.left = '-9999px'; + textarea.style.top = '0'; + document.body.appendChild(textarea); + textarea.select(); + const copied = document.execCommand('copy'); + document.body.removeChild(textarea); + if (!copied) { + throw new Error('copy command failed'); + } +} + +function RealDataStatus({ + amapRuntime, + realData, + usingRealData, + autoRefresh, + onAutoRefreshChange, + onRefresh +}: { + amapRuntime: AMapRuntimeState; + realData: PrototypeRealDataState; + usingRealData: boolean; + autoRefresh: boolean; + onAutoRefreshChange: Dispatch>; + onRefresh: () => void; +}) { + const [copyState, setCopyState] = useState<'idle' | 'copied' | 'manual'>('idle'); + const diagnosticTextareaRef = useRef(null); + const retryingWithoutRealtime = realData.status === 'loading' && Boolean(realData.error) && realData.vehicles.length === 0; + const statusText: Record = { + disabled: '真实数据未启用', + loading: '读取真实数据中', + ready: '真实数据已接入', + stale: '真实缓存数据', + fallback: '真实数据为空', + error: '真实数据读取失败' + }; + const statusTitle = retryingWithoutRealtime ? '真实接口重试中' : statusText[realData.status]; + const statusClass = realData.status; + const statusDetail = realData.status === 'stale' + ? `${realData.fetchedAt ? `缓存 ${realData.fetchedAt}` : '使用最近一次真实缓存'}${realData.error ? ` / ${realtimeErrorDetail(realData.error)}` : ''}` + : realData.fetchedAt + ? `刷新 ${realData.fetchedAt}` + : realtimeErrorDetail(realData.error); + const dataModeText = usingRealData + ? `${realData.vehicles.length} 台车辆 / ${realData.total ?? realData.vehicles.length} 条实时记录` + : realData.status === 'disabled' + ? '当前展示本地样例数据' + : '未展示生产车辆，等待真实接口'; + const diagnosticItems = realtimeDiagnosticItems(realData, usingRealData, amapRuntime); + const diagnosticText = realtimeDiagnosticText(realData, usingRealData, amapRuntime); + const selectDiagnosticText = useCallback(() => { + const textarea = diagnosticTextareaRef.current; + if (!textarea) { + return; + } + textarea.focus(); + textarea.select(); + textarea.setSelectionRange(0, textarea.value.length); + }, []); + + useEffect(() => { + if (copyState !== 'manual') { + return; + } + const timer = window.setTimeout(selectDiagnosticText, 0); + return () => { + window.clearTimeout(timer); + }; + }, [copyState, diagnosticText, selectDiagnosticText]); + + const copyDiagnostics = async () => { + if (copyState === 'manual') { + selectDiagnosticText(); + return; + } + try { + await copyTextWithFallback(diagnosticText); + setCopyState('copied'); + window.setTimeout(() => setCopyState('idle'), 1600); + } catch { + setCopyState('manual'); + } + }; + return ( +
+
+ {statusTitle} + {dataModeText} +
+ {statusDetail} + {realData.source} +
+ {diagnosticItems.map((item) => ( + + {item.label} + {item.value} + + ))} +
+
+ + + +
+ {copyState === 'manual' && ( +
+
+ 剪贴板受限，已选中诊断文本 + +
+