From 37844e44251414ca6e0dab22cb1417073a615099 Mon Sep 17 00:00:00 2001 From: lingniu Date: Wed, 1 Jul 2026 16:56:46 +0800 Subject: [PATCH] chore: remove local service runners --- deploy/local/launchctl/README.md | 119 ----------- .../com.lingniu.gb32960.plist.template | 61 ------ .../com.lingniu.jt808.plist.template | 57 ----- ...m.lingniu.vehicle-analytics.plist.template | 55 ----- ...com.lingniu.vehicle-history.plist.template | 75 ------- .../com.lingniu.yutong-mqtt.plist.template | 87 -------- deploy/local/launchctl/render.py | 95 --------- .../gb32960-service-split-runbook.md | 152 +++++--------- .../vehicle-ingest-tdengine-verification.md | 120 +++-------- .../historyapp/MavenModuleProfileTest.java | 3 +- .../PortainerComposeResourceLimitsTest.java | 194 +----------------- 11 files changed, 83 insertions(+), 935 deletions(-) delete mode 100644 deploy/local/launchctl/README.md delete mode 100644 deploy/local/launchctl/com.lingniu.gb32960.plist.template delete mode 100644 deploy/local/launchctl/com.lingniu.jt808.plist.template delete mode 100644 deploy/local/launchctl/com.lingniu.vehicle-analytics.plist.template delete mode 100644 deploy/local/launchctl/com.lingniu.vehicle-history.plist.template delete mode 100644 deploy/local/launchctl/com.lingniu.yutong-mqtt.plist.template delete mode 100644 deploy/local/launchctl/render.py diff --git a/deploy/local/launchctl/README.md b/deploy/local/launchctl/README.md deleted file mode 100644 index 0e9ca21e..00000000 --- a/deploy/local/launchctl/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# macOS launchctl 本机部署模板 - -本目录用于把本机生产验证服务的启动方式仓库化。模板面向当前生产验证链路： - -- `gb32960-ingest-app`: TCP `32960`，HTTP `20100` -- `jt808-ingest-app`: TCP `808`，HTTP `20400` -- `yutong-mqtt-app`: HTTP `20500` -- `vehicle-history-app`: HTTP `20200` -- `vehicle-analytics-app`: HTTP `20300` - -`vehicle-history-app` 的热历史查询只依赖 Kafka 与 TDengine：RAW 帧的 `parsedJson`、`metadataJson`、`rawUri` 已进入 TDengine `raw_frames`。GB32960/JT808 接入服务可以继续保留 `SINK_ARCHIVE_PATH` 作为原始 `.bin` 冷备，history 本机模板不再要求共享 archive root。 - -`vehicle-history-app` 的生产高吞吐模板是 TDengine-only： - -- 不暴露 DuckDB 或文件型明细库启动变量，避免本地文件索引进入实时消费热路径。 -- 只写 `raw_frames` 和协议位置表，避免逐字段写放大。 -- 通用 RAW 查询直接返回 TDengine `raw_frames` 中的 `parsedJson`/`parsedFields`；字段趋势宽表由独立字段解析服务消费 Kafka RAW/事件后维护。 - -## 生成 plist - -在仓库根目录执行： - -```bash -export PROJECT_ROOT="$(pwd)" -export JAVA_BIN="$(/usr/libexec/java_home 2>/dev/null)/bin/java" -export INGEST_ARCHIVE_PATH="$PROJECT_ROOT/data/archive-live" -export TDENGINE_JDBC_URL='jdbc:TAOS-WS://:6041/vehicle_ts' -export TDENGINE_USERNAME='root' -export TDENGINE_PASSWORD='' -export VEHICLE_IDENTITY_MYSQL_JDBC_URL='jdbc:mysql://:3306/vehicle_ingest' -export VEHICLE_IDENTITY_MYSQL_USERNAME='' -export VEHICLE_IDENTITY_MYSQL_PASSWORD='' -export VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL='60s' -export MYSQL_JDBC_URL="$VEHICLE_IDENTITY_MYSQL_JDBC_URL" -export MYSQL_USERNAME="$VEHICLE_IDENTITY_MYSQL_USERNAME" -export MYSQL_PASSWORD="$VEHICLE_IDENTITY_MYSQL_PASSWORD" -export KAFKA_TOPIC_HISTORY_DLQ='vehicle.dlq.history.v1' -export YUTONG_MQTT_ENABLED='false' -export YUTONG_MQTT_URI='' -export YUTONG_MQTT_TOPIC='#' -export YUTONG_MQTT_USERNAME='' -export YUTONG_MQTT_PASSWORD='' -export YUTONG_MQTT_KEEP_ALIVE_SECONDS='20' -export YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS='10' -export YUTONG_MQTT_TLS_CA_PEM='' -export YUTONG_MQTT_TLS_CLIENT_PEM='' -export YUTONG_MQTT_TLS_CLIENT_KEY='' -export YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED='true' - -mkdir -p "$HOME/Library/LaunchAgents" "$PROJECT_ROOT/data" -mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-yutong-mqtt-live /tmp/lingniu-vehicle-history-live /tmp/lingniu-vehicle-analytics-live - -python3 deploy/local/launchctl/render.py -``` - -渲染脚本固定生成 `gb32960 jt808 yutong-mqtt vehicle-history vehicle-analytics` 五个 plist，并会对密码、JDBC URL、MQTT topic、keepAlive、连接超时和 TLS PEM 参数中的 XML 特殊字符做转义，避免 `#`、`&` 等字符破坏 plist。 - -如果生产配置由 Nacos 下发，可以把模板里的 `NACOS_CONFIG_ENABLED` 改成 `true`，并补充 `NACOS_SERVER_ADDR`、`NACOS_NAMESPACE`、`NACOS_GROUP`、`NACOS_USERNAME`、`NACOS_PASSWORD`。端口、Kafka topic、接入服务冷备 archive 路径、TDengine 连接建议仍保留为启动环境变量，便于 Portainer、launchctl 和临时压测保持一致。 - -history 消费者会按协议和 raw/event 自动拆分 consumer group。设置 `KAFKA_GROUP_HISTORY=vehicle-history-live` 后，实际 group 为： - -- `vehicle-history-live-gb32960-event` -- `vehicle-history-live-gb32960-raw` -- `vehicle-history-live-jt808-event` -- `vehicle-history-live-jt808-raw` -- `vehicle-history-live-yutong-mqtt-event` -- `vehicle-history-live-yutong-mqtt-raw` - -本地模板与 ECS 生产口径一致，身份绑定只使用 MySQL。启动前必须提供 `VEHICLE_IDENTITY_MYSQL_*` 连接参数；服务会自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表：前者只保存 `plate`、`vin`，后者保存注册字段；服务按刷新周期用注册表里的车牌关联 VIN，后续 raw/event 会带真实 VIN。 - -## 启停 - -```bash -launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist" -launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist" -launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.yutong-mqtt.plist" -launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist" -launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-analytics.plist" - -launchctl list | rg 'com.lingniu.(gb32960|jt808|yutong-mqtt|vehicle-history|vehicle-analytics)' -``` - -替换 jar 前先停止服务，避免 KeepAlive 在 jar 拷贝中途拉起半包： - -```bash -launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.gb32960.plist" || true -launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.jt808.plist" || true -launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.yutong-mqtt.plist" || true -launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-history.plist" || true -launchctl bootout "gui/$(id -u)" "$HOME/Library/LaunchAgents/com.lingniu.vehicle-analytics.plist" || true -``` - -## 验证地址 - -```bash -curl -sS http://127.0.0.1:20100/actuator/health -curl -sS http://127.0.0.1:20100/actuator/health/liveness -curl -sS http://127.0.0.1:20100/actuator/health/readiness -curl -sS http://127.0.0.1:20400/actuator/health -curl -sS http://127.0.0.1:20400/actuator/health/liveness -curl -sS http://127.0.0.1:20400/actuator/health/readiness -curl -sS http://127.0.0.1:20500/actuator/health -curl -sS http://127.0.0.1:20500/actuator/health/liveness -curl -sS http://127.0.0.1:20500/actuator/health/readiness -curl -sS http://127.0.0.1:20200/actuator/health -curl -sS http://127.0.0.1:20200/actuator/health/liveness -curl -sS http://127.0.0.1:20200/actuator/health/readiness -curl -sS http://127.0.0.1:20300/actuator/health -curl -sS http://127.0.0.1:20300/actuator/health/liveness -curl -sS http://127.0.0.1:20300/actuator/health/readiness -``` - -API / Swagger： - -- GB32960 ingest health: `http://127.0.0.1:20100/actuator/health` -- JT808 ingest health: `http://127.0.0.1:20400/actuator/health` -- Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health` -- History query: `http://127.0.0.1:20200/swagger-ui/index.html` -- Analytics metrics: `http://127.0.0.1:20300/swagger-ui/index.html` diff --git a/deploy/local/launchctl/com.lingniu.gb32960.plist.template b/deploy/local/launchctl/com.lingniu.gb32960.plist.template deleted file mode 100644 index 62e78b79..00000000 --- a/deploy/local/launchctl/com.lingniu.gb32960.plist.template +++ /dev/null @@ -1,61 +0,0 @@ - - - - - Label - com.lingniu.gb32960 - WorkingDirectory - __PROJECT_ROOT__ - ProgramArguments - - __JAVA_BIN__ - -jar - __PROJECT_ROOT__/modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar - - EnvironmentVariables - - HTTP_PORT - 20100 - GB32960_PORT - 32960 - GB32960_AUTH_ENABLED - false - GB32960_PLATFORM_IP_HYUNDAI - 115.29.187.205 - KAFKA_BROKERS - 114.55.58.251:9092 - KAFKA_NODE_ID - gb32960-live-32960 - KAFKA_TOPIC_GB32960_EVENT - vehicle.event.gb32960.v1 - KAFKA_TOPIC_GB32960_RAW - vehicle.raw.gb32960.v1 - KAFKA_TOPIC_GB32960_DLQ - vehicle.dlq.gb32960.v1 - SINK_ARCHIVE_PATH - __SHARED_ARCHIVE_PATH__ - VEHICLE_IDENTITY_MYSQL_TABLE - vehicle_identity_binding - VEHICLE_IDENTITY_MYSQL_JDBC_URL - __VEHICLE_IDENTITY_MYSQL_JDBC_URL__ - VEHICLE_IDENTITY_MYSQL_USERNAME - __VEHICLE_IDENTITY_MYSQL_USERNAME__ - VEHICLE_IDENTITY_MYSQL_PASSWORD - __VEHICLE_IDENTITY_MYSQL_PASSWORD__ - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL - __VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__ - NACOS_CONFIG_ENABLED - false - MANAGEMENT_HEALTH_REDIS_ENABLED - false - - StandardOutPath - /tmp/lingniu-gb32960-live/launchctl-gb32960.log - StandardErrorPath - /tmp/lingniu-gb32960-live/launchctl-gb32960.err.log - RunAtLoad - - KeepAlive - - - diff --git a/deploy/local/launchctl/com.lingniu.jt808.plist.template b/deploy/local/launchctl/com.lingniu.jt808.plist.template deleted file mode 100644 index f47fe057..00000000 --- a/deploy/local/launchctl/com.lingniu.jt808.plist.template +++ /dev/null @@ -1,57 +0,0 @@ - - - - - Label - com.lingniu.jt808 - WorkingDirectory - __PROJECT_ROOT__ - ProgramArguments - - __JAVA_BIN__ - -jar - __PROJECT_ROOT__/modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar - - EnvironmentVariables - - HTTP_PORT - 20400 - JT808_PORT - 808 - KAFKA_BROKERS - 114.55.58.251:9092 - KAFKA_NODE_ID - jt808-live-808 - KAFKA_TOPIC_JT808_EVENT - vehicle.event.jt808.v1 - KAFKA_TOPIC_JT808_RAW - vehicle.raw.jt808.v1 - KAFKA_TOPIC_JT808_DLQ - vehicle.dlq.jt808.v1 - SINK_ARCHIVE_PATH - __SHARED_ARCHIVE_PATH__ - VEHICLE_IDENTITY_MYSQL_TABLE - vehicle_identity_binding - VEHICLE_IDENTITY_MYSQL_JDBC_URL - __VEHICLE_IDENTITY_MYSQL_JDBC_URL__ - VEHICLE_IDENTITY_MYSQL_USERNAME - __VEHICLE_IDENTITY_MYSQL_USERNAME__ - VEHICLE_IDENTITY_MYSQL_PASSWORD - __VEHICLE_IDENTITY_MYSQL_PASSWORD__ - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL - __VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__ - NACOS_CONFIG_ENABLED - false - MANAGEMENT_HEALTH_REDIS_ENABLED - false - - StandardOutPath - /tmp/lingniu-jt808-live/launchctl-jt808.log - StandardErrorPath - /tmp/lingniu-jt808-live/launchctl-jt808.err.log - RunAtLoad - - KeepAlive - - - diff --git a/deploy/local/launchctl/com.lingniu.vehicle-analytics.plist.template b/deploy/local/launchctl/com.lingniu.vehicle-analytics.plist.template deleted file mode 100644 index 2d29d6be..00000000 --- a/deploy/local/launchctl/com.lingniu.vehicle-analytics.plist.template +++ /dev/null @@ -1,55 +0,0 @@ - - - - - Label - com.lingniu.vehicle-analytics - WorkingDirectory - __PROJECT_ROOT__ - ProgramArguments - - __JAVA_BIN__ - -jar - __PROJECT_ROOT__/modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar - - EnvironmentVariables - - HTTP_PORT - 20300 - KAFKA_CONSUMER_ENABLED - true - KAFKA_CONSUMER_AUTO_OFFSET_RESET - latest - KAFKA_CONSUMER_CLIENT_ID_PREFIX - vehicle-analytics-live - KAFKA_BROKERS - 114.55.58.251:9092 - KAFKA_GROUP_STAT - vehicle-stat-live - VEHICLE_STAT_ENABLED - true - VEHICLE_STAT_ZONE_ID - Asia/Shanghai - VEHICLE_STAT_JT808_MILEAGE_ENABLED - true - MYSQL_JDBC_URL - __MYSQL_JDBC_URL__ - MYSQL_USERNAME - __MYSQL_USERNAME__ - MYSQL_PASSWORD - __MYSQL_PASSWORD__ - NACOS_CONFIG_ENABLED - false - MANAGEMENT_HEALTH_REDIS_ENABLED - false - - StandardOutPath - /tmp/lingniu-vehicle-analytics-live/vehicle-analytics.log - StandardErrorPath - /tmp/lingniu-vehicle-analytics-live/vehicle-analytics.err.log - RunAtLoad - - KeepAlive - - - diff --git a/deploy/local/launchctl/com.lingniu.vehicle-history.plist.template b/deploy/local/launchctl/com.lingniu.vehicle-history.plist.template deleted file mode 100644 index d3b82d2f..00000000 --- a/deploy/local/launchctl/com.lingniu.vehicle-history.plist.template +++ /dev/null @@ -1,75 +0,0 @@ - - - - - Label - com.lingniu.vehicle-history - WorkingDirectory - __PROJECT_ROOT__ - ProgramArguments - - __JAVA_BIN__ - -jar - __PROJECT_ROOT__/modules/apps/vehicle-history-app/target/vehicle-history-app.jar - - EnvironmentVariables - - HTTP_PORT - 20200 - KAFKA_CONSUMER_ENABLED - true - KAFKA_CONSUMER_AUTO_OFFSET_RESET - latest - KAFKA_CONSUMER_CLIENT_ID_PREFIX - vehicle-history-live - KAFKA_GROUP_HISTORY - vehicle-history-live - KAFKA_CONSUMER_CONCURRENCY - 3 - KAFKA_BROKERS - 114.55.58.251:9092 - KAFKA_TOPIC_GB32960_EVENT - vehicle.event.gb32960.v1 - KAFKA_TOPIC_GB32960_RAW - vehicle.raw.gb32960.v1 - KAFKA_TOPIC_JT808_EVENT - vehicle.event.jt808.v1 - KAFKA_TOPIC_JT808_RAW - vehicle.raw.jt808.v1 - KAFKA_TOPIC_YUTONG_MQTT_EVENT - vehicle.event.mqtt-yutong.v1 - KAFKA_TOPIC_YUTONG_MQTT_RAW - vehicle.raw.mqtt-yutong.v1 - KAFKA_TOPIC_HISTORY_DLQ - __KAFKA_TOPIC_HISTORY_DLQ__ - TDENGINE_HISTORY_ENABLED - true - TDENGINE_HISTORY_DATABASE - vehicle_ts - TDENGINE_JDBC_URL - __TDENGINE_JDBC_URL__ - TDENGINE_DRIVER_CLASS_NAME - com.taosdata.jdbc.ws.WebSocketDriver - TDENGINE_USERNAME - __TDENGINE_USERNAME__ - TDENGINE_PASSWORD - __TDENGINE_PASSWORD__ - TDENGINE_MIN_IDLE - 0 - TDENGINE_MAX_POOL_SIZE - 16 - NACOS_CONFIG_ENABLED - false - MANAGEMENT_HEALTH_REDIS_ENABLED - false - - StandardOutPath - /tmp/lingniu-vehicle-history-live/vehicle-history.log - StandardErrorPath - /tmp/lingniu-vehicle-history-live/vehicle-history.err.log - RunAtLoad - - KeepAlive - - - diff --git a/deploy/local/launchctl/com.lingniu.yutong-mqtt.plist.template b/deploy/local/launchctl/com.lingniu.yutong-mqtt.plist.template deleted file mode 100644 index f5031663..00000000 --- a/deploy/local/launchctl/com.lingniu.yutong-mqtt.plist.template +++ /dev/null @@ -1,87 +0,0 @@ - - - - - Label - com.lingniu.yutong-mqtt - WorkingDirectory - __PROJECT_ROOT__ - ProgramArguments - - __JAVA_BIN__ - -jar - __PROJECT_ROOT__/modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar - - EnvironmentVariables - - HTTP_PORT - 20500 - YUTONG_MQTT_ENABLED - __YUTONG_MQTT_ENABLED__ - YUTONG_MQTT_AUTO_STARTUP - true - YUTONG_MQTT_ENDPOINT_NAME - yutong - YUTONG_MQTT_URI - __YUTONG_MQTT_URI__ - YUTONG_MQTT_TOPIC - __YUTONG_MQTT_TOPIC__ - YUTONG_MQTT_QOS - 1 - YUTONG_MQTT_CLIENT_ID - lingniu-yutong-mqtt-live - YUTONG_MQTT_USERNAME - __YUTONG_MQTT_USERNAME__ - YUTONG_MQTT_PASSWORD - __YUTONG_MQTT_PASSWORD__ - YUTONG_MQTT_CLEAN_SESSION - false - YUTONG_MQTT_KEEP_ALIVE_SECONDS - __YUTONG_MQTT_KEEP_ALIVE_SECONDS__ - YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS - __YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS__ - YUTONG_MQTT_TLS_CA_PEM - __YUTONG_MQTT_TLS_CA_PEM__ - YUTONG_MQTT_TLS_CLIENT_PEM - __YUTONG_MQTT_TLS_CLIENT_PEM__ - YUTONG_MQTT_TLS_CLIENT_KEY - __YUTONG_MQTT_TLS_CLIENT_KEY__ - YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED - __YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED__ - KAFKA_BROKERS - 114.55.58.251:9092 - KAFKA_NODE_ID - yutong-mqtt-live - KAFKA_TOPIC_YUTONG_MQTT_EVENT - vehicle.event.mqtt-yutong.v1 - KAFKA_TOPIC_YUTONG_MQTT_RAW - vehicle.raw.mqtt-yutong.v1 - KAFKA_TOPIC_YUTONG_MQTT_DLQ - vehicle.dlq.mqtt-yutong.v1 - SINK_ARCHIVE_PATH - __SHARED_ARCHIVE_PATH__ - VEHICLE_IDENTITY_MYSQL_TABLE - vehicle_identity_binding - VEHICLE_IDENTITY_MYSQL_JDBC_URL - __VEHICLE_IDENTITY_MYSQL_JDBC_URL__ - VEHICLE_IDENTITY_MYSQL_USERNAME - __VEHICLE_IDENTITY_MYSQL_USERNAME__ - VEHICLE_IDENTITY_MYSQL_PASSWORD - __VEHICLE_IDENTITY_MYSQL_PASSWORD__ - VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL - __VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__ - NACOS_CONFIG_ENABLED - false - MANAGEMENT_HEALTH_REDIS_ENABLED - false - - StandardOutPath - /tmp/lingniu-yutong-mqtt-live/launchctl-yutong-mqtt.log - StandardErrorPath - /tmp/lingniu-yutong-mqtt-live/launchctl-yutong-mqtt.err.log - RunAtLoad - - KeepAlive - - - diff --git a/deploy/local/launchctl/render.py b/deploy/local/launchctl/render.py deleted file mode 100644 index 1f77f3a8..00000000 --- a/deploy/local/launchctl/render.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -import html -import os -import re -from pathlib import Path - - -SERVICES = ( - "gb32960", - "jt808", - "yutong-mqtt", - "vehicle-history", - "vehicle-analytics", -) - -LOG_DIRS = ( - "lingniu-gb32960-live", - "lingniu-jt808-live", - "lingniu-yutong-mqtt-live", - "lingniu-vehicle-history-live", - "lingniu-vehicle-analytics-live", -) - - -def env(name: str, default: str = "") -> str: - return os.environ.get(name, default) - - -def xml_value(value: str) -> str: - return html.escape(value, quote=False) - - -def replacements() -> dict[str, str]: - identity_jdbc = env("VEHICLE_IDENTITY_MYSQL_JDBC_URL", "jdbc:mysql://127.0.0.1:3306/vehicle_ingest") - identity_username = env("VEHICLE_IDENTITY_MYSQL_USERNAME", "") - identity_password = env("VEHICLE_IDENTITY_MYSQL_PASSWORD", "") - return { - "__PROJECT_ROOT__": env("PROJECT_ROOT", str(Path.cwd())), - "__JAVA_BIN__": env("JAVA_BIN", "/usr/bin/java"), - "__SHARED_ARCHIVE_PATH__": env("INGEST_ARCHIVE_PATH", str(Path.cwd() / "data/archive-live")), - "__TDENGINE_JDBC_URL__": env("TDENGINE_JDBC_URL", "jdbc:TAOS-WS://127.0.0.1:6041/vehicle_ts"), - "__TDENGINE_USERNAME__": env("TDENGINE_USERNAME", "root"), - "__TDENGINE_PASSWORD__": env("TDENGINE_PASSWORD", ""), - "__VEHICLE_IDENTITY_MYSQL_JDBC_URL__": identity_jdbc, - "__VEHICLE_IDENTITY_MYSQL_USERNAME__": identity_username, - "__VEHICLE_IDENTITY_MYSQL_PASSWORD__": identity_password, - "__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__": env("VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL", "60s"), - "__MYSQL_JDBC_URL__": env("MYSQL_JDBC_URL", identity_jdbc), - "__MYSQL_USERNAME__": env("MYSQL_USERNAME", identity_username), - "__MYSQL_PASSWORD__": env("MYSQL_PASSWORD", identity_password), - "__KAFKA_TOPIC_HISTORY_DLQ__": env("KAFKA_TOPIC_HISTORY_DLQ", "vehicle.dlq.history.v1"), - "__YUTONG_MQTT_ENABLED__": env("YUTONG_MQTT_ENABLED", "false"), - "__YUTONG_MQTT_URI__": env("YUTONG_MQTT_URI", ""), - "__YUTONG_MQTT_TOPIC__": env("YUTONG_MQTT_TOPIC", "#"), - "__YUTONG_MQTT_USERNAME__": env("YUTONG_MQTT_USERNAME", ""), - "__YUTONG_MQTT_PASSWORD__": env("YUTONG_MQTT_PASSWORD", ""), - "__YUTONG_MQTT_KEEP_ALIVE_SECONDS__": env("YUTONG_MQTT_KEEP_ALIVE_SECONDS", "20"), - "__YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS__": env("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", "10"), - "__YUTONG_MQTT_TLS_CA_PEM__": env("YUTONG_MQTT_TLS_CA_PEM", ""), - "__YUTONG_MQTT_TLS_CLIENT_PEM__": env("YUTONG_MQTT_TLS_CLIENT_PEM", ""), - "__YUTONG_MQTT_TLS_CLIENT_KEY__": env("YUTONG_MQTT_TLS_CLIENT_KEY", ""), - "__YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED__": env( - "YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", "true"), - } - - -def render_template(template: str, values: dict[str, str]) -> str: - rendered = template - for placeholder, value in values.items(): - rendered = rendered.replace(placeholder, xml_value(value)) - unresolved = sorted(set(re.findall(r"__[A-Z0-9_]+__", rendered))) - if unresolved: - raise RuntimeError("unresolved launchctl placeholder(s): " + ", ".join(unresolved)) - return rendered - - -def main() -> None: - root = Path(env("PROJECT_ROOT", str(Path.cwd()))).resolve() - template_dir = Path(__file__).resolve().parent - launch_agents = Path.home() / "Library/LaunchAgents" - launch_agents.mkdir(parents=True, exist_ok=True) - (root / "data").mkdir(parents=True, exist_ok=True) - for log_dir in LOG_DIRS: - (Path("/tmp") / log_dir).mkdir(parents=True, exist_ok=True) - - values = replacements() - for service in SERVICES: - template_path = template_dir / f"com.lingniu.{service}.plist.template" - output_path = launch_agents / f"com.lingniu.{service}.plist" - output_path.write_text(render_template(template_path.read_text(), values), encoding="utf-8") - print(output_path) - - -if __name__ == "__main__": - main() diff --git a/docs/operations/gb32960-service-split-runbook.md b/docs/operations/gb32960-service-split-runbook.md index 1c16384d..53efacdd 100644 --- a/docs/operations/gb32960-service-split-runbook.md +++ b/docs/operations/gb32960-service-split-runbook.md @@ -4,7 +4,7 @@ This runbook covers the default production runtimes: GB32960 ingest, JT808 inges ## Service Roles -| Service | Module | Role | Local ports | +| Service | Module | Role | Production ports | | --- | --- | --- | --- | | GB32960 ingest | `:gb32960-ingest-app` | Accept GB32960 TCP connections, decode/authenticate frames, produce raw and normalized records to Kafka, and send protocol ACKs after required Kafka production succeeds. | TCP `32960`, HTTP `20100` | | JT808 ingest | `:jt808-ingest-app` | Accept JT/T 808 TCP connections on the production receive port, parse raw frames, archive frame bytes, and produce raw/event envelopes to Kafka. | TCP `808`, HTTP `20400` | @@ -27,7 +27,7 @@ This runbook covers the default production runtimes: GB32960 ingest, JT808 inges | `vehicle.dlq.mqtt-yutong.v1` | Kafka producer/consumer error paths | Operators/replay tooling | Dead-letter records for failed Yutong MQTT production or consumer processing. | | `vehicle.dlq.history.v1` | History app consumer error paths | Operators/replay tooling | Dead-letter records for failed history storage processing. | -Default local consumer groups: +Default consumer groups: - History: `vehicle-history` - Analytics statistics: `vehicle-stat` @@ -40,13 +40,13 @@ History and analytics failures do not block GB32960 ACKs. They are downstream Ka Authorization failures are rejected before the Kafka durability boundary and can be ACKed/rejected immediately according to protocol handling. -## Local Prerequisites +## Build Prerequisites - Java 25 and Maven available on PATH. -- Local Kafka reachable at `127.0.0.1:9092`, plus Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer`. +- Kafka CLI tools such as `kafka-topics` and optionally `kafka-console-consumer` available on the ECS/operator host. - Session state requires Redis. GB32960/JT808 ingest services keep live Netty channels in-process, but session indexes and TTL metadata use Redis only. -- No repository-local Kafka bootstrap script or compose file was found during Task 8 verification on 2026-06-23. +- Redis is configured through Portainer environment variables such as `REDIS_HOST` and `REDIS_DATABASE=50`; there is no memory session-store fallback. If your shell needs an explicit JDK: @@ -73,99 +73,50 @@ Expected result: `BUILD SUCCESS` and these runnable jars: ## Create Topics -Run these only after Kafka is reachable at `127.0.0.1:9092`: +Run these only on the ECS/operator host after Kafka is reachable. The default bootstrap address is the ECS private Kafka address used by the production stack: ```bash -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1 -kafka-topics --bootstrap-server 127.0.0.1:9092 --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1 +KAFKA_BOOTSTRAP="${KAFKA_BOOTSTRAP:-172.17.111.56:9092}" +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.gb32960.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.gb32960.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.gb32960.v1 --partitions 3 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.jt808.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.jt808.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.jt808.v1 --partitions 3 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.raw.mqtt-yutong.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.event.mqtt-yutong.v1 --partitions 12 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.mqtt-yutong.v1 --partitions 3 --replication-factor 1 +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --create --if-not-exists --topic vehicle.dlq.history.v1 --partitions 3 --replication-factor 1 ``` Optional sanity check: ```bash -kafka-topics --bootstrap-server 127.0.0.1:9092 --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1' +kafka-topics --bootstrap-server "$KAFKA_BOOTSTRAP" --list | grep -E 'vehicle\.(raw|event|dlq)\.(gb32960|jt808|mqtt-yutong|history)\.v1' ``` -## Start Split Services Locally +## Production Runtime -Run each command in a separate terminal from the repository root. +Production services run on ECS through Portainer/Docker. Do not start GB32960, JT808, Yutong MQTT, history, or analytics services on local developer machines; local machines are for code build, unit tests, and source inspection only. -GB32960 ingest: +Use `deploy/portainer/docker-compose.yml` as the production runtime manifest. The default stack is: -```bash -KAFKA_BROKERS=127.0.0.1:9092 \ -REDIS_HOST=127.0.0.1 \ -REDIS_PORT=6379 \ -REDIS_DATABASE=50 \ -GB32960_PORT=32960 \ -HTTP_PORT=20100 \ -java --sun-misc-unsafe-memory-access=allow \ - -jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar -``` +- `gb32960-ingest-app` +- `jt808-ingest-app` +- `yutong-mqtt-app` +- `vehicle-history-app` +- `vehicle-analytics-app` -JT808 ingest: +`vehicle-history-app` intentionally has no raw archive volume. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount. -```bash -KAFKA_BROKERS=127.0.0.1:9092 \ -REDIS_HOST=127.0.0.1 \ -REDIS_PORT=6379 \ -REDIS_DATABASE=50 \ -JT808_PORT=808 \ -HTTP_PORT=20400 \ -java --sun-misc-unsafe-memory-access=allow \ - -jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar -``` - -Yutong MQTT ingest: - -```bash -KAFKA_BROKERS=127.0.0.1:9092 \ -YUTONG_MQTT_ENABLED=true \ -YUTONG_MQTT_URI='' \ -YUTONG_MQTT_USERNAME='' \ -YUTONG_MQTT_PASSWORD='' \ -HTTP_PORT=20500 \ -java --sun-misc-unsafe-memory-access=allow \ - -jar modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar -``` - -Vehicle history: - -```bash -KAFKA_BROKERS=127.0.0.1:9092 \ -HTTP_PORT=20200 \ -TDENGINE_HISTORY_ENABLED=true \ -TDENGINE_HISTORY_DATABASE=vehicle_ts \ -TDENGINE_JDBC_URL='jdbc:TAOS-WS://:6041/vehicle_ts' \ -TDENGINE_USERNAME=root \ -TDENGINE_PASSWORD='' \ -java --sun-misc-unsafe-memory-access=allow \ - -jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar -``` - -`SINK_ARCHIVE_PATH` is intentionally omitted from the history JVM. The history hot path is Kafka to TDengine: raw envelopes are stored in TDengine `raw_frames` with `parsedJson`, `metadataJson`, and `rawUri`. Ingest services may still write original `.bin` files as cold backup, but history API queries must not depend on a shared local archive mount. - -Vehicle analytics: - -```bash -KAFKA_BROKERS=127.0.0.1:9092 \ -HTTP_PORT=20300 \ -java --sun-misc-unsafe-memory-access=allow \ - -jar modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar -``` +Keep `TDENGINE_HISTORY_ENABLED=true` in the `vehicle-history-app` environment so history reads and writes use TDengine in production. `vehicle-analytics-app` is intentionally a metric runtime only. Latest-state APIs belong to `vehicle-state-service`, which stays outside the default reactor and should be built with `-Poptional-latest-state` and deployed as a separate consumer if that surface is needed. ## Health Verification +Run these from the ECS host or inside the matching container: + ```bash curl -sS http://127.0.0.1:20100/actuator/health curl -sS http://127.0.0.1:20100/actuator/health/liveness @@ -210,10 +161,10 @@ Expected: the ACK file is non-empty and the decoded bytes begin with the GB32960 Verify only what was actually run in your environment: ```bash -kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000 -kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000 -kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000 -kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000 +kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000 +kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.gb32960.v1 --from-beginning --max-messages 1 --timeout-ms 10000 +kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.raw.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000 +kafka-console-consumer --bootstrap-server "$KAFKA_BOOTSTRAP" --topic vehicle.event.mqtt-yutong.v1 --from-beginning --max-messages 1 --timeout-ms 10000 ``` Use TDengine CLI or a JDBC client to verify history writes: @@ -251,17 +202,15 @@ Do not expect `vehicle-history-app` to create raw `.bin` archive files from Kafk Do not mark any of these as verified unless the matching command was run and the output was inspected. -## Local launchctl Deployment +## Runtime Location Policy -Use the templates under `deploy/local/launchctl/` for the local TCP-focused production-verification machine. They currently cover GB32960, JT808, and history; run Yutong MQTT separately with the command above or deploy it through Portainer. +Production services run on ECS through Portainer/Docker. -- GB32960 ingest: `com.lingniu.gb32960`, TCP `32960`, HTTP `20100` -- JT808 ingest: `com.lingniu.jt808`, TCP `808`, HTTP `20400` -- History: `com.lingniu.vehicle-history`, HTTP `20200` +- Do not create local service runners for GB32960, JT808, Yutong MQTT, history, or analytics. +- Do not run the service jars directly on developer machines for production verification. +- Keep local work limited to source edits, builds, and automated tests. -The launchctl history template does not inject `SINK_ARCHIVE_PATH`. Use TDengine `raw_frames` as the authoritative hot-query store; keep any ingest-side raw archive path as an optional cold backup for operator inspection. - -For Portainer deployments, mirror this same shape: no history archive volume, history consumes raw/event Kafka topics and writes TDengine history rows. +For Portainer deployments, keep history without an archive volume: history consumes raw/event Kafka topics and writes TDengine history rows. ## Rollback Guidance @@ -273,7 +222,7 @@ If the split deployment is unhealthy: 4. If Kafka itself is unavailable, stop `gb32960-ingest-app` or route GB32960 traffic to a previously verified release that preserves the Kafka durability boundary. 5. After rollback, compare Kafka consumer group lag and DLQ contents before resuming the split services. -Rollback commands depend on the deployment environment. For local testing, stop the active Java processes and restart the previous verified app jars or images. +Rollback commands depend on the deployment environment. For the default production stack, roll back the Portainer stack image version and let Docker restart the affected containers. ## Task 8 Verification Notes @@ -285,19 +234,19 @@ Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`: - `kafka-topics`, `kafka-topics.sh`, `docker`, and `docker-compose` were not found on PATH. - Kafka topic creation, service startup, health checks, Kafka record checks, archive checks, TDengine checks, stat output checks, and ACK observation were not run because the local Kafka prerequisite was absent. -## Latest Local Verification +## Latest Build Verification Observed on 2026-06-23 in worktree `.worktrees/gb32960-service-split`: - Targeted module tests: `mvn -pl :sink-kafka,:ingest-core,:protocol-gb32960,:event-history-service,:vehicle-state-service,:vehicle-stat-service test` ended with `BUILD SUCCESS`; Maven reported 55 protocol/sink/core tests, 30 event-history tests, 14 vehicle-state tests, and 19 vehicle-stat tests with 0 failures/errors/skips. - Split app tests should cover the current active apps listed in the Service Roles section. - Package verification should use the active-app package command from the Build section. -- Split app startup: not run in this verification pass because local Kafka was still absent. -- Kafka raw records: not verified; no local Kafka broker or Kafka CLI tools were available. -- Kafka event records: not verified; no local Kafka broker or Kafka CLI tools were available. +- Split app startup: not run in this verification pass because an ECS/Portainer runtime was outside that build-only check. +- Kafka raw records: not verified in that build-only check. +- Kafka event records: not verified in that build-only check. - TDengine history rows: not verified; downstream services were not started without Kafka. - Analytics output: not verified; downstream services were not started without Kafka. -- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above in a Kafka-backed local or staging environment. +- ACK behavior: not verified against a live broker in this pass. Unit tests cover the GB32960 ACK boundary and ordering, but an operator should still run the ACK capture command above on ECS or staging. ## Remote Kafka Live Verification @@ -313,15 +262,12 @@ Observed on 2026-06-23 with Kafka `114.55.58.251:9092`: This verifies the live path `GB32960 TCP 32960 -> gb32960-ingest-app -> Kafka raw/event topics` against the remote Kafka broker. Downstream `vehicle-history-app` and `vehicle-analytics-app` were not started in this pass. -## 2026-06-29 Local Production-Path Verification +## 2026-06-29 Superseded Local Verification Record -Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`: +Observed on 2026-06-29 in worktree `.worktrees/vehicle-ingest-redesign-phase1`. This record is retained only as historical evidence of parser/storage behavior; do not repeat it as a deployment procedure because production services now run on ECS through Portainer/Docker. -- `jt808-ingest-app` was running through launchctl on TCP `808` and HTTP `20400`; `curl -sS http://127.0.0.1:20400/actuator/health` returned top-level status `UP`. -- `gb32960-ingest-app` was running through launchctl on TCP `32960` and HTTP `20100`; `curl -sS http://127.0.0.1:20100/actuator/health` returned top-level status `UP`. -- `vehicle-history-app` was running through launchctl on HTTP `20200`; `curl -sS http://127.0.0.1:20200/actuator/health` returned top-level status `UP`. - External JT808 traffic on TCP `808` was parsed and written through Kafka into TDengine. Direct TDengine checks showed more than `6000` JT808 `raw_frames` and more than `1900` JT808 `vehicle_locations`; current production history does not maintain a `telemetry_fields` table. - A GB32960 synthetic realtime frame sent to TCP `32960` received a binary ACK beginning with `2323`. The same VIN `LTEST202606290001` was queryable from TDengine-backed history APIs with speed, mileage, longitude, and latitude fields. - History raw and event Kafka consumption used separate bindings so raw-frame writes remained current while location/field event ingestion continued. -This verifies the local path `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`. +The verified behavior was `GB32960/JT808 TCP -> protocol app -> Kafka raw/event topics -> vehicle-history-app -> TDengine raw_frames/location tables -> Swagger/API query`; the deployment method from that historical run is superseded. diff --git a/docs/operations/vehicle-ingest-tdengine-verification.md b/docs/operations/vehicle-ingest-tdengine-verification.md index f5bd1efe..7106aae5 100644 --- a/docs/operations/vehicle-ingest-tdengine-verification.md +++ b/docs/operations/vehicle-ingest-tdengine-verification.md @@ -44,125 +44,51 @@ mvn -pl :gb32960-ingest-app,:jt808-ingest-app,:yutong-mqtt-app,:vehicle-history- - `modules/apps/vehicle-history-app/target/vehicle-history-app.jar` - `modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar` -## 启动 JT808 接入服务，监听 808 端口 +## ECS 服务状态验证 + +生产服务只在 ECS 上运行，通过 Portainer/Docker 管理。不要在本机用 `java -jar`、plist 或其它守护进程方式启动 GB32960、JT808、Yutong MQTT、history、analytics。 ```bash -HTTP_PORT=20400 \ -JT808_PORT=808 \ -KAFKA_CONSUMER_ENABLED=false \ -java -jar modules/apps/jt808-ingest-app/target/jt808-ingest-app.jar -``` +docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' \ + | egrep 'gb32960|jt808|yutong|vehicle-history|vehicle-analytics|NAMES' -验证： +ss -lntp | egrep ':(808|32960|20100|20200|20300|20400|20500)\b' -```bash curl -sS http://127.0.0.1:20400/actuator/health curl -sS http://127.0.0.1:20400/actuator/health/liveness curl -sS http://127.0.0.1:20400/actuator/health/readiness -lsof -nP -iTCP:808 -sTCP:LISTEN -``` - -预期：健康检查为 `UP`，并且 Java 进程监听 TCP `808`。 - -## 启动 GB32960 接入服务 - -```bash -HTTP_PORT=20100 \ -GB32960_PORT=32960 \ -KAFKA_CONSUMER_ENABLED=false \ -java -jar modules/apps/gb32960-ingest-app/target/gb32960-ingest-app.jar -``` - -验证： - -```bash curl -sS http://127.0.0.1:20100/actuator/health curl -sS http://127.0.0.1:20100/actuator/health/liveness curl -sS http://127.0.0.1:20100/actuator/health/readiness -lsof -nP -iTCP:32960 -sTCP:LISTEN -``` - -## 启动 Yutong MQTT 接入服务 - -```bash -HTTP_PORT=20500 \ -YUTONG_MQTT_ENABLED=true \ -YUTONG_MQTT_AUTO_STARTUP=true \ -KAFKA_CONSUMER_ENABLED=false \ -java -jar modules/apps/yutong-mqtt-app/target/yutong-mqtt-app.jar -``` - -验证： - -```bash curl -sS http://127.0.0.1:20500/actuator/health curl -sS http://127.0.0.1:20500/actuator/health/liveness curl -sS http://127.0.0.1:20500/actuator/health/readiness -``` - -预期：健康检查为 `UP`，并且 Kafka 中 `vehicle.raw.mqtt-yutong.v1`、`vehicle.event.mqtt-yutong.v1` 能随真实 MQTT 消息增长。 - -## 启动 TDengine 版历史服务 - -```bash -HTTP_PORT=20200 \ -KAFKA_CONSUMER_ENABLED=true \ -java -jar modules/apps/vehicle-history-app/target/vehicle-history-app.jar -``` - -验证： - -```bash curl -sS http://127.0.0.1:20200/actuator/health curl -sS http://127.0.0.1:20200/actuator/health/liveness curl -sS http://127.0.0.1:20200/actuator/health/readiness curl -sS http://127.0.0.1:20200/v3/api-docs \ | grep -E '/api/event-history/locations|/api/event-history/raw-frames' -``` - -预期：健康检查为 `UP`，OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`，也不持续写入逐字段宽表。 - -## 启动 JT808 指标统计服务 - -```bash -HTTP_PORT=20300 \ -KAFKA_CONSUMER_ENABLED=true \ -VEHICLE_STAT_ENABLED=true \ -VEHICLE_STAT_JT808_MILEAGE_ENABLED=true \ -java -jar modules/apps/vehicle-analytics-app/target/vehicle-analytics-app.jar -``` - -验证： - -```bash curl -sS http://127.0.0.1:20300/actuator/health curl -sS http://127.0.0.1:20300/actuator/health/liveness curl -sS http://127.0.0.1:20300/actuator/health/readiness ``` -预期：健康检查为 `UP`；JT808 位置帧中有 GPS 总里程时，服务按差值法写入 MySQL `vehicle_stat_metric` 的 `daily_mileage_km` 指标。 +预期：五个容器均为运行状态；健康检查为 `UP`；JT808/GB32960 TCP 端口在 ECS 上监听；OpenAPI 中包含通用位置分页查询和 RAW 帧查询接口。history-app 不暴露 `/api/event-history/telemetry/fields`，也不持续写入逐字段宽表。JT808 位置帧中有 GPS 总里程时，`vehicle-analytics-app` 按差值法写入 MySQL `vehicle_stat_metric` 的 `daily_mileage_km` 指标。 -## 本机 launchctl 部署 +## 运行位置约束 -当前本机生产验证建议使用仓库模板生成 plist： - -```bash -deploy/local/launchctl/ -``` - -关键约束： +生产身份绑定固定使用 MySQL，不再提供 file/memory/sqlite 运行时切换入口。生产服务固定运行在 ECS/Portainer，不提供本机守护进程模板。 - `jt808-ingest-app` 监听 TCP `808`，HTTP `20400`。 - `gb32960-ingest-app` 监听 TCP `32960`，HTTP `20100`。 +- `yutong-mqtt-app` 监听 HTTP `20500`。 - `vehicle-history-app` 监听 HTTP `20200`。 -- Yutong MQTT 与 `vehicle-analytics-app` 可以按上面的 jar 启动命令手工验收；Portainer/ECS 生产部署已包含这两个服务。 -- history 热查询只依赖 Kafka 和 TDengine `raw_frames`；本地模板不会给 history 注入 `SINK_ARCHIVE_PATH`。 -- GB32960/JT808 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始 `.bin` 冷备，但这不是 history API 的实时查询前置条件。 -- 替换 jar 前先 `launchctl bootout`，再复制 jar 和 `launchctl bootstrap`，避免 KeepAlive 在 jar 拷贝中途重启。 +- `vehicle-analytics-app` 监听 HTTP `20300`。 +- history 热查询只依赖 Kafka 和 TDengine `raw_frames`；history 不需要共享 archive volume。 +- GB32960/JT808/Yutong MQTT 接入服务可以继续使用 `SINK_ARCHIVE_PATH` 保存原始冷备，但这不是 history API 的实时查询前置条件。 JT808 注册身份绑定： -- 生产身份绑定固定使用 MySQL，不再提供 file/memory/sqlite 运行时切换入口。 - 生产需要把 0x0100 注册信息维护到 MySQL 时，只需要配置 `VEHICLE_IDENTITY_MYSQL_*` 连接参数。 - 需要提供 `VEHICLE_IDENTITY_MYSQL_JDBC_URL`、`VEHICLE_IDENTITY_MYSQL_USERNAME`、`VEHICLE_IDENTITY_MYSQL_PASSWORD`。 - 服务启动时自动创建 `vehicle_identity_binding` 和 `vehicle_identity_binding_registration` 两张表。`vehicle_identity_binding` 只维护 `plate`、`vin` 两列；`vehicle_identity_binding_registration` 按 `protocol + phone` 保存 JT808 0x0100 注册字段。 @@ -220,7 +146,7 @@ API / Swagger： ## JT808 实时转发验收 -如果外部平台已经把 JT808 报文转发到本机 TCP `808`，至少观察 60 秒日志和 Kafka 数据。 +如果外部平台已经把 JT808 报文转发到 ECS TCP `808`，至少观察 60 秒日志和 Kafka 数据。 必须拿到以下证据： @@ -272,13 +198,15 @@ SELECT COUNT(*) FROM jt808_locations; - TDengine 未启用时，TDengine 专属接口不应该出现在 OpenAPI 中。 - 路径不存在时，API 层应该返回 HTTP `404`，而不是泛化的存储失败。 -## 2026-06-29 本机实时链路验证结果 +## 2026-06-29 Superseded Local Live Verification Record -运行服务： +该记录仅保留为解析和存储行为的历史证据；生产运行位置已经收敛到 ECS/Portainer。 -- `com.lingniu.jt808.808`: `jt808-ingest-app`，TCP `808`，HTTP `20400` -- `com.lingniu.gb32960.32960`: `gb32960-ingest-app`，TCP `32960`，HTTP `20100` -- `com.lingniu.vehicle-history.jt808`: `vehicle-history-app`，HTTP `20200` +当时参与验证的服务： + +- `jt808-ingest-app`：TCP `808`，HTTP `20400` +- `gb32960-ingest-app`：TCP `32960`，HTTP `20100` +- `vehicle-history-app`：HTTP `20200` 当次健康检查均返回 `{"status":"UP"}`。 @@ -300,7 +228,7 @@ python3 tools/vehicle_ingest_live_verify.py \ JT808 真实转发链路已验证： -- 本机 TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。 +- ECS TCP `808` 有外部平台 `222.66.200.68` 持续发送 JT808 报文。 - `vehicle.raw.jt808.v1` 和 `vehicle.event.jt808.v1` 被 history 消费。 - `raw_frames` 中 `protocol='JT808'` 的真实行数已超过 `36000`。 - 外部平台真实 0x0100 注册帧已超过 `347` 条，真实 0x0200 位置帧已超过 `12400` 条。 @@ -309,7 +237,7 @@ JT808 真实转发链路已验证： - 分页 API 已用终端号 `13079963301` 验证第一页和 nextCursor 第二页，样例位置包含 `longitude=118.913846`、`latitude=31.927309`、`statusFlag=3` 和 `archive://2026/06/29/JT808/...` 原始帧引用。 - 注册帧样例：终端号 `13079963320`、deviceId `9963320`、deviceType `SEG-9888G`、plate `沪A61559F`、maker `70112`、province `31`、city `113`、plateColor `2`。生产 VIN 反写依赖 MySQL identity store，后续 raw/event 的 VIN 解析也从 MySQL 绑定读取。 -## JT808 本机 smoke / 轻压测工具 +## JT808 ECS smoke / 轻压测工具 仓库提供 `tools/jt808_e2e_smoke.py`，用于重复验证 TCP `808` 到 TDengine history 的闭环： @@ -416,7 +344,7 @@ curl -sS 'http://127.0.0.1:20200/api/event-history/gb32960/snapshots/fields?vin= ## 当前注意事项 -- 当前本机使用 TDengine WebSocket JDBC，例如 `jdbc:TAOS-WS://:6041/vehicle_ts`。 +- 当前生产链路使用 TDengine WebSocket JDBC，例如 `jdbc:TAOS-WS://:6041/vehicle_ts`。 - `KAFKA_CONSUMER_AUTO_OFFSET_RESET=latest` 适合生产接入新流量；如果要回放历史 Kafka 数据，需要切换 consumer group 或重置 offset。 - `vehicle-history-app` raw 和 event 使用不同 consumer binding；raw consumer 必须开启，否则 `raw_frames` 不会持续增长。 - JT808 设备没有 VIN 映射时，`vehicle_key` 使用 `jt808:`；启用 MySQL identity store 后，registration 表中已反写 VIN 的 phone/deviceId/plate 会用于后续 raw/event 的 VIN 解析。 diff --git a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/MavenModuleProfileTest.java b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/MavenModuleProfileTest.java index 58652ac8..b3f4f2e6 100644 --- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/MavenModuleProfileTest.java +++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/MavenModuleProfileTest.java @@ -402,7 +402,7 @@ class MavenModuleProfileTest { assertThat(runbook) .contains("Session state requires Redis") - .contains("REDIS_HOST=127.0.0.1") + .contains("REDIS_HOST") .contains("REDIS_DATABASE=50") .doesNotContain("SESSION_STORE=memory") .doesNotContain("without Redis"); @@ -447,7 +447,6 @@ class MavenModuleProfileTest { repositoryRoot().resolve("README.md"), repositoryRoot().resolve("pom.xml"), repositoryRoot().resolve("deploy/portainer/docker-compose.yml"), - repositoryRoot().resolve("deploy/local/launchctl/README.md"), repositoryRoot().resolve("docs/operations/gb32960-service-split-runbook.md"), repositoryRoot().resolve("docs/operations/vehicle-ingest-tdengine-verification.md"), repositoryRoot().resolve("docs/operations/jt808-daily-mileage-runbook.md"), diff --git a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java index 14e3f53e..f937f979 100644 --- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java +++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java @@ -5,7 +5,6 @@ import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.core.io.FileSystemResource; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -123,141 +122,17 @@ class PortainerComposeResourceLimitsTest { } @Test - void localLaunchctlReadmeChecksLivenessAndReadinessForManagedApps() throws IOException { - String readme = Files.readString(repositoryRoot() - .resolve("deploy/local/launchctl/README.md")); - - for (String httpPort : List.of("20100", "20400", "20500", "20200", "20300")) { - assertThat(readme) - .contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/liveness") - .contains("curl -sS http://127.0.0.1:" + httpPort + "/actuator/health/readiness"); - } - } - - @Test - void localLaunchctlReadmeDocumentsYutongMqttProductionTuning() throws IOException { - String readme = Files.readString(repositoryRoot() - .resolve("deploy/local/launchctl/README.md")); - - assertThat(readme) - .contains("export YUTONG_MQTT_KEEP_ALIVE_SECONDS='20'") - .contains("export YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS='10'") - .contains("export YUTONG_MQTT_TLS_CA_PEM=''") - .contains("export YUTONG_MQTT_TLS_CLIENT_PEM=''") - .contains("export YUTONG_MQTT_TLS_CLIENT_KEY=''") - .contains("export YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED='true'") - .contains("keepAlive、连接超时和 TLS PEM 参数"); - } - - @Test - void localLaunchctlTemplatesExposeHistoryConsumerDeadLetterTopic() throws IOException { - Path launchctl = repositoryRoot().resolve("deploy/local/launchctl"); - String readme = Files.readString(launchctl.resolve("README.md")); - String renderer = Files.readString(launchctl.resolve("render.py")); - String historyTemplate = Files.readString(launchctl.resolve("com.lingniu.vehicle-history.plist.template")); - - assertThat(readme) - .contains("export KAFKA_TOPIC_HISTORY_DLQ='vehicle.dlq.history.v1'"); - assertThat(renderer) - .contains("\"__KAFKA_TOPIC_HISTORY_DLQ__\": env(\"KAFKA_TOPIC_HISTORY_DLQ\", \"vehicle.dlq.history.v1\")"); - assertThat(historyTemplate) - .contains("KAFKA_TOPIC_HISTORY_DLQ\n __KAFKA_TOPIC_HISTORY_DLQ__"); - } - - @Test - void localLaunchctlTemplatesCoverActiveProductionApps() throws IOException { - Path launchctl = repositoryRoot().resolve("deploy/local/launchctl"); - String readme = Files.readString(launchctl.resolve("README.md")); - - assertThat(readme) - .contains("python3 deploy/local/launchctl/render.py") - .contains("gb32960 jt808 yutong-mqtt vehicle-history vehicle-analytics") - .contains("mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-yutong-mqtt-live /tmp/lingniu-vehicle-history-live /tmp/lingniu-vehicle-analytics-live") - .contains("launchctl bootstrap \"gui/$(id -u)\" \"$HOME/Library/LaunchAgents/com.lingniu.yutong-mqtt.plist\"") - .contains("launchctl bootstrap \"gui/$(id -u)\" \"$HOME/Library/LaunchAgents/com.lingniu.vehicle-analytics.plist\"") - .contains("launchctl bootout \"gui/$(id -u)\" \"$HOME/Library/LaunchAgents/com.lingniu.yutong-mqtt.plist\" || true") - .contains("launchctl bootout \"gui/$(id -u)\" \"$HOME/Library/LaunchAgents/com.lingniu.vehicle-analytics.plist\" || true") - .contains("launchctl list | rg 'com.lingniu.(gb32960|jt808|yutong-mqtt|vehicle-history|vehicle-analytics)'") - .contains("Yutong MQTT ingest health: `http://127.0.0.1:20500/actuator/health`") - .contains("Analytics metrics: `http://127.0.0.1:20300/swagger-ui/index.html`") - .doesNotContain("Yutong MQTT ingest: `http://127.0.0.1:20500/swagger-ui/index.html`") - .doesNotContain("sed \\") - .doesNotContain("三个服务"); - - assertLaunchctlTemplate(launchctl, "gb32960", "gb32960-ingest-app", "20100"); - assertLaunchctlTemplate(launchctl, "jt808", "jt808-ingest-app", "20400"); - assertLaunchctlTemplate(launchctl, "yutong-mqtt", "yutong-mqtt-app", "20500"); - assertLaunchctlTemplate(launchctl, "vehicle-history", "vehicle-history-app", "20200"); - assertLaunchctlTemplate(launchctl, "vehicle-analytics", "vehicle-analytics-app", "20300"); - } - - @Test - void localLaunchctlRendererPreservesSpecialCharactersSafely() throws Exception { + void defaultProductionSurfaceDoesNotShipLocalLaunchctlRunners() throws IOException { Path root = repositoryRoot(); - Path launchctl = root.resolve("deploy/local/launchctl"); - Path home = Files.createTempDirectory("lingniu-launchctl-home"); + String runbook = Files.readString(root.resolve("docs/operations/gb32960-service-split-runbook.md")); - ProcessBuilder processBuilder = new ProcessBuilder( - "python3", - launchctl.resolve("render.py").toString()); - processBuilder.directory(root.toFile()); - processBuilder.environment().put("HOME", home.toString()); - processBuilder.environment().put("PROJECT_ROOT", "/tmp/lingniu project & live"); - processBuilder.environment().put("JAVA_BIN", "/usr/bin/java"); - processBuilder.environment().put("INGEST_ARCHIVE_PATH", "/tmp/archive&live__v1"); - processBuilder.environment().put("TDENGINE_JDBC_URL", "jdbc:TAOS-WS://host:6041/vehicle_ts"); - processBuilder.environment().put("TDENGINE_USERNAME", "root"); - processBuilder.environment().put("TDENGINE_PASSWORD", "td&engine#pwd"); - processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_JDBC_URL", - "jdbc:mysql://host:3306/vehicle_ingest?useSSL=false&serverTimezone=Asia/Shanghai"); - processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_USERNAME", "lingniu_vehicle"); - processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_PASSWORD", "id&pwd#1"); - processBuilder.environment().put("MYSQL_JDBC_URL", - "jdbc:mysql://host:3306/vehicle_stat?useSSL=false&serverTimezone=Asia/Shanghai"); - processBuilder.environment().put("MYSQL_USERNAME", "stat_user"); - processBuilder.environment().put("MYSQL_PASSWORD", "stat&pwd#2"); - processBuilder.environment().put("YUTONG_MQTT_ENABLED", "true"); - processBuilder.environment().put("YUTONG_MQTT_URI", "ssl://mqtt.example.com:8883"); - processBuilder.environment().put("YUTONG_MQTT_TOPIC", "#"); - processBuilder.environment().put("YUTONG_MQTT_USERNAME", "mqtt_user"); - processBuilder.environment().put("YUTONG_MQTT_PASSWORD", "mqtt&pwd#3"); - processBuilder.environment().put("YUTONG_MQTT_KEEP_ALIVE_SECONDS", "45"); - processBuilder.environment().put("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS", "12"); - processBuilder.environment().put("YUTONG_MQTT_TLS_CA_PEM", "ca&pem"); - processBuilder.environment().put("YUTONG_MQTT_TLS_CLIENT_PEM", "client"); - processBuilder.environment().put("YUTONG_MQTT_TLS_CLIENT_KEY", "key&value"); - processBuilder.environment().put("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED", "false"); - - Process process = processBuilder.start(); - String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8) - + new String(process.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); - - assertThat(process.waitFor()).as(output).isZero(); - - Path launchAgents = home.resolve("Library/LaunchAgents"); - String mqtt = Files.readString(launchAgents.resolve("com.lingniu.yutong-mqtt.plist")); - String analytics = Files.readString(launchAgents.resolve("com.lingniu.vehicle-analytics.plist")); - - assertThat(mqtt) - .contains("#") - .contains("mqtt&pwd#3") - .contains("YUTONG_MQTT_KEEP_ALIVE_SECONDS\n 45") - .contains("YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS\n 12") - .contains("YUTONG_MQTT_TLS_CA_PEM\n ca&pem") - .contains("YUTONG_MQTT_TLS_CLIENT_PEM\n client<pem>") - .contains("YUTONG_MQTT_TLS_CLIENT_KEY\n key&value") - .contains("YUTONG_MQTT_TLS_HOSTNAME_VERIFICATION_ENABLED\n false") - .contains("/tmp/archive&live__v1") - .doesNotContain("__YUTONG_MQTT_TOPIC__") - .doesNotContain("__YUTONG_MQTT_PASSWORD__") - .doesNotContain("__YUTONG_MQTT_TLS"); - assertThat(analytics) - .contains("stat&pwd#2") - .doesNotContain("__MYSQL_PASSWORD__"); - assertThat(Files.list(launchAgents) - .filter(path -> path.getFileName().toString().endsWith(".plist")) - .count()) - .isEqualTo(5); + assertThat(root.resolve("deploy/local/launchctl")).doesNotExist(); + assertThat(runbook) + .contains("Production services run on ECS through Portainer/Docker.") + .doesNotContain("Local launchctl Deployment") + .doesNotContain("deploy/local/launchctl") + .doesNotContain("launchctl bootstrap") + .doesNotContain("launchctl bootout"); } @Test @@ -440,39 +315,6 @@ class PortainerComposeResourceLimitsTest { assertThat(compose).contains("\n yutong-mqtt-data:\n"); } - @Test - void localLaunchctlHistoryRuntimeDoesNotRequireSharedArchiveRoot() throws IOException { - Path launchctl = repositoryRoot().resolve("deploy/local/launchctl"); - String historyTemplate = Files.readString(launchctl.resolve("com.lingniu.vehicle-history.plist.template")); - String readme = Files.readString(launchctl.resolve("README.md")); - - assertThat(historyTemplate) - .doesNotContain("SINK_ARCHIVE_PATH") - .doesNotContain("__SHARED_ARCHIVE_PATH__"); - assertThat(readme) - .doesNotContain("三个服务必须使用同一个 `SINK_ARCHIVE_PATH`") - .doesNotContain("history 服务按同一个 archive root 读取原始帧") - .doesNotContain("EventFileStore") - .doesNotContain("/api/event-history/records"); - } - - @Test - void localLaunchctlJt808UsesCurrentHttpPort() throws IOException { - Path launchctl = repositoryRoot().resolve("deploy/local/launchctl"); - String jt808Template = Files.readString(launchctl.resolve("com.lingniu.jt808.plist.template")); - String readme = Files.readString(launchctl.resolve("README.md")); - - assertThat(jt808Template) - .contains("HTTP_PORT\n 20400") - .doesNotContain("20482"); - assertThat(readme) - .contains("`jt808-ingest-app`: TCP `808`，HTTP `20400`") - .contains("curl -sS http://127.0.0.1:20400/actuator/health") - .contains("JT808 ingest health: `http://127.0.0.1:20400/actuator/health`") - .doesNotContain("JT808 ingest: `http://127.0.0.1:20400/swagger-ui/index.html`") - .doesNotContain("20482"); - } - @Test void operationRunbooksDescribeTdengineRawJsonHotPath() throws IOException { Path operations = repositoryRoot().resolve("docs/operations"); @@ -746,24 +588,6 @@ class PortainerComposeResourceLimitsTest { .doesNotContain(" start_period: 60s\n"); } - private static void assertLaunchctlTemplate(Path launchctl, - String serviceName, - String appName, - String httpPort) throws IOException { - Path template = launchctl.resolve("com.lingniu." + serviceName + ".plist.template"); - assertThat(template).exists(); - assertThat(Files.readString(template)) - .contains("com.lingniu." + serviceName + "") - .contains("__PROJECT_ROOT__/modules/apps/" + appName + "/target/" + appName + ".jar") - .contains("HTTP_PORT\n " + httpPort + "") - .contains("NACOS_CONFIG_ENABLED\n false") - .contains("MANAGEMENT_HEALTH_REDIS_ENABLED\n false") - .contains("/tmp/lingniu-" + serviceName + "-live/") - .doesNotContain("VEHICLE_IDENTITY_STORE") - .doesNotContain("KAFKA_ENABLED") - .doesNotContain("TDENGINE_INITIALIZE_SCHEMA"); - } - private static String serviceBlock(String compose, String serviceName) { String marker = "\n " + serviceName + ":\n"; int start = compose.indexOf(marker);