From 78c39036cd11b44b1f59543cc2ddf0cded013fb5 Mon Sep 17 00:00:00 2001 From: lingniu Date: Wed, 1 Jul 2026 13:03:17 +0800 Subject: [PATCH] deploy: render launchctl templates safely --- deploy/local/launchctl/README.md | 26 +----- deploy/local/launchctl/render.py | 87 +++++++++++++++++++ .../PortainerComposeResourceLimitsTest.java | 59 +++++++++++++ 3 files changed, 149 insertions(+), 23 deletions(-) create mode 100644 deploy/local/launchctl/render.py diff --git a/deploy/local/launchctl/README.md b/deploy/local/launchctl/README.md index 6b03d4bf..c5c30a94 100644 --- a/deploy/local/launchctl/README.md +++ b/deploy/local/launchctl/README.md @@ -44,31 +44,11 @@ export YUTONG_MQTT_PASSWORD='' mkdir -p "$HOME/Library/LaunchAgents" "$PROJECT_ROOT/data" mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-yutong-mqtt-live /tmp/lingniu-vehicle-history-live /tmp/lingniu-vehicle-analytics-live -for service in gb32960 jt808 yutong-mqtt vehicle-history vehicle-analytics; do - sed \ - -e "s#__PROJECT_ROOT__#$PROJECT_ROOT#g" \ - -e "s#__JAVA_BIN__#$JAVA_BIN#g" \ - -e "s#__SHARED_ARCHIVE_PATH__#$INGEST_ARCHIVE_PATH#g" \ - -e "s#__TDENGINE_JDBC_URL__#$TDENGINE_JDBC_URL#g" \ - -e "s#__TDENGINE_USERNAME__#$TDENGINE_USERNAME#g" \ - -e "s#__TDENGINE_PASSWORD__#$TDENGINE_PASSWORD#g" \ - -e "s#__VEHICLE_IDENTITY_MYSQL_JDBC_URL__#$VEHICLE_IDENTITY_MYSQL_JDBC_URL#g" \ - -e "s#__VEHICLE_IDENTITY_MYSQL_USERNAME__#$VEHICLE_IDENTITY_MYSQL_USERNAME#g" \ - -e "s#__VEHICLE_IDENTITY_MYSQL_PASSWORD__#$VEHICLE_IDENTITY_MYSQL_PASSWORD#g" \ - -e "s#__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__#$VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL#g" \ - -e "s#__MYSQL_JDBC_URL__#$MYSQL_JDBC_URL#g" \ - -e "s#__MYSQL_USERNAME__#$MYSQL_USERNAME#g" \ - -e "s#__MYSQL_PASSWORD__#$MYSQL_PASSWORD#g" \ - -e "s#__YUTONG_MQTT_ENABLED__#$YUTONG_MQTT_ENABLED#g" \ - -e "s#__YUTONG_MQTT_URI__#$YUTONG_MQTT_URI#g" \ - -e "s#__YUTONG_MQTT_TOPIC__#$YUTONG_MQTT_TOPIC#g" \ - -e "s#__YUTONG_MQTT_USERNAME__#$YUTONG_MQTT_USERNAME#g" \ - -e "s#__YUTONG_MQTT_PASSWORD__#$YUTONG_MQTT_PASSWORD#g" \ - "deploy/local/launchctl/com.lingniu.${service}.plist.template" \ - > "$HOME/Library/LaunchAgents/com.lingniu.${service}.plist" -done +python3 deploy/local/launchctl/render.py ``` +渲染脚本固定生成 `gb32960 jt808 yutong-mqtt vehicle-history vehicle-analytics` 五个 plist，并会对密码、JDBC URL、MQTT topic 中的 XML 特殊字符做转义，避免 `#`、`&` 等字符破坏 plist。 + 如果生产配置由 Nacos 下发，可以把模板里的 `NACOS_CONFIG_ENABLED` 改成 `true`，并补充 `NACOS_SERVER_ADDR`、`NACOS_NAMESPACE`、`NACOS_GROUP`、`NACOS_USERNAME`、`NACOS_PASSWORD`。端口、Kafka topic、接入服务冷备 archive 路径、TDengine 连接建议仍保留为启动环境变量，便于 Portainer、launchctl 和临时压测保持一致。 history 消费者会按协议和 raw/event 自动拆分 consumer group。设置 `KAFKA_GROUP_HISTORY=vehicle-history-live` 后，实际 group 为： diff --git a/deploy/local/launchctl/render.py b/deploy/local/launchctl/render.py new file mode 100644 index 00000000..3f804ac6 --- /dev/null +++ b/deploy/local/launchctl/render.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +import html +import os +import re +from pathlib import Path + + +SERVICES = ( + "gb32960", + "jt808", + "yutong-mqtt", + "vehicle-history", + "vehicle-analytics", +) + +LOG_DIRS = ( + "lingniu-gb32960-live", + "lingniu-jt808-live", + "lingniu-yutong-mqtt-live", + "lingniu-vehicle-history-live", + "lingniu-vehicle-analytics-live", +) + + +def env(name: str, default: str = "") -> str: + return os.environ.get(name, default) + + +def xml_value(value: str) -> str: + return html.escape(value, quote=False) + + +def replacements() -> dict[str, str]: + identity_jdbc = env("VEHICLE_IDENTITY_MYSQL_JDBC_URL", "jdbc:mysql://127.0.0.1:3306/vehicle_ingest") + identity_username = env("VEHICLE_IDENTITY_MYSQL_USERNAME", "") + identity_password = env("VEHICLE_IDENTITY_MYSQL_PASSWORD", "") + return { + "__PROJECT_ROOT__": env("PROJECT_ROOT", str(Path.cwd())), + "__JAVA_BIN__": env("JAVA_BIN", "/usr/bin/java"), + "__SHARED_ARCHIVE_PATH__": env("INGEST_ARCHIVE_PATH", str(Path.cwd() / "data/archive-live")), + "__TDENGINE_JDBC_URL__": env("TDENGINE_JDBC_URL", "jdbc:TAOS-WS://127.0.0.1:6041/vehicle_ts"), + "__TDENGINE_USERNAME__": env("TDENGINE_USERNAME", "root"), + "__TDENGINE_PASSWORD__": env("TDENGINE_PASSWORD", ""), + "__VEHICLE_IDENTITY_MYSQL_JDBC_URL__": identity_jdbc, + "__VEHICLE_IDENTITY_MYSQL_USERNAME__": identity_username, + "__VEHICLE_IDENTITY_MYSQL_PASSWORD__": identity_password, + "__VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL__": env("VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL", "60s"), + "__MYSQL_JDBC_URL__": env("MYSQL_JDBC_URL", identity_jdbc), + "__MYSQL_USERNAME__": env("MYSQL_USERNAME", identity_username), + "__MYSQL_PASSWORD__": env("MYSQL_PASSWORD", identity_password), + "__YUTONG_MQTT_ENABLED__": env("YUTONG_MQTT_ENABLED", "false"), + "__YUTONG_MQTT_URI__": env("YUTONG_MQTT_URI", ""), + "__YUTONG_MQTT_TOPIC__": env("YUTONG_MQTT_TOPIC", "#"), + "__YUTONG_MQTT_USERNAME__": env("YUTONG_MQTT_USERNAME", ""), + "__YUTONG_MQTT_PASSWORD__": env("YUTONG_MQTT_PASSWORD", ""), + } + + +def render_template(template: str, values: dict[str, str]) -> str: + rendered = template + for placeholder, value in values.items(): + rendered = rendered.replace(placeholder, xml_value(value)) + unresolved = sorted(set(re.findall(r"__[A-Z0-9_]+__", rendered))) + if unresolved: + raise RuntimeError("unresolved launchctl placeholder(s): " + ", ".join(unresolved)) + return rendered + + +def main() -> None: + root = Path(env("PROJECT_ROOT", str(Path.cwd()))).resolve() + template_dir = Path(__file__).resolve().parent + launch_agents = Path.home() / "Library/LaunchAgents" + launch_agents.mkdir(parents=True, exist_ok=True) + (root / "data").mkdir(parents=True, exist_ok=True) + for log_dir in LOG_DIRS: + (Path("/tmp") / log_dir).mkdir(parents=True, exist_ok=True) + + values = replacements() + for service in SERVICES: + template_path = template_dir / f"com.lingniu.{service}.plist.template" + output_path = launch_agents / f"com.lingniu.{service}.plist" + output_path.write_text(render_template(template_path.read_text(), values), encoding="utf-8") + print(output_path) + + +if __name__ == "__main__": + main() diff --git a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java index ffac3144..130f342a 100644 --- a/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java +++ b/modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/PortainerComposeResourceLimitsTest.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.core.io.FileSystemResource; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -138,6 +139,7 @@ class PortainerComposeResourceLimitsTest { String readme = Files.readString(launchctl.resolve("README.md")); assertThat(readme) + .contains("python3 deploy/local/launchctl/render.py") .contains("gb32960 jt808 yutong-mqtt vehicle-history vehicle-analytics") .contains("mkdir -p /tmp/lingniu-gb32960-live /tmp/lingniu-jt808-live /tmp/lingniu-yutong-mqtt-live /tmp/lingniu-vehicle-history-live /tmp/lingniu-vehicle-analytics-live") .contains("launchctl bootstrap \"gui/$(id -u)\" \"$HOME/Library/LaunchAgents/com.lingniu.yutong-mqtt.plist\"") @@ -147,6 +149,7 @@ class PortainerComposeResourceLimitsTest { .contains("launchctl list | rg 'com.lingniu.(gb32960|jt808|yutong-mqtt|vehicle-history|vehicle-analytics)'") .contains("Yutong MQTT ingest: `http://127.0.0.1:20500/swagger-ui/index.html`") .contains("Analytics metrics: `http://127.0.0.1:20300/swagger-ui/index.html`") + .doesNotContain("sed \\") .doesNotContain("三个服务"); assertLaunchctlTemplate(launchctl, "gb32960", "gb32960-ingest-app", "20100"); @@ -156,6 +159,62 @@ class PortainerComposeResourceLimitsTest { assertLaunchctlTemplate(launchctl, "vehicle-analytics", "vehicle-analytics-app", "20300"); } + @Test + void localLaunchctlRendererPreservesSpecialCharactersSafely() throws Exception { + Path root = repositoryRoot(); + Path launchctl = root.resolve("deploy/local/launchctl"); + Path home = Files.createTempDirectory("lingniu-launchctl-home"); + + ProcessBuilder processBuilder = new ProcessBuilder( + "python3", + launchctl.resolve("render.py").toString()); + processBuilder.directory(root.toFile()); + processBuilder.environment().put("HOME", home.toString()); + processBuilder.environment().put("PROJECT_ROOT", "/tmp/lingniu project & live"); + processBuilder.environment().put("JAVA_BIN", "/usr/bin/java"); + processBuilder.environment().put("INGEST_ARCHIVE_PATH", "/tmp/archive&live__v1"); + processBuilder.environment().put("TDENGINE_JDBC_URL", "jdbc:TAOS-WS://host:6041/vehicle_ts"); + processBuilder.environment().put("TDENGINE_USERNAME", "root"); + processBuilder.environment().put("TDENGINE_PASSWORD", "td&engine#pwd"); + processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_JDBC_URL", + "jdbc:mysql://host:3306/vehicle_ingest?useSSL=false&serverTimezone=Asia/Shanghai"); + processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_USERNAME", "lingniu_vehicle"); + processBuilder.environment().put("VEHICLE_IDENTITY_MYSQL_PASSWORD", "id&pwd#1"); + processBuilder.environment().put("MYSQL_JDBC_URL", + "jdbc:mysql://host:3306/vehicle_stat?useSSL=false&serverTimezone=Asia/Shanghai"); + processBuilder.environment().put("MYSQL_USERNAME", "stat_user"); + processBuilder.environment().put("MYSQL_PASSWORD", "stat&pwd#2"); + processBuilder.environment().put("YUTONG_MQTT_ENABLED", "true"); + processBuilder.environment().put("YUTONG_MQTT_URI", "ssl://mqtt.example.com:8883"); + processBuilder.environment().put("YUTONG_MQTT_TOPIC", "#"); + processBuilder.environment().put("YUTONG_MQTT_USERNAME", "mqtt_user"); + processBuilder.environment().put("YUTONG_MQTT_PASSWORD", "mqtt&pwd#3"); + + Process process = processBuilder.start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8) + + new String(process.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + + assertThat(process.waitFor()).as(output).isZero(); + + Path launchAgents = home.resolve("Library/LaunchAgents"); + String mqtt = Files.readString(launchAgents.resolve("com.lingniu.yutong-mqtt.plist")); + String analytics = Files.readString(launchAgents.resolve("com.lingniu.vehicle-analytics.plist")); + + assertThat(mqtt) + .contains("#") + .contains("mqtt&pwd#3") + .contains("/tmp/archive&live__v1") + .doesNotContain("__YUTONG_MQTT_TOPIC__") + .doesNotContain("__YUTONG_MQTT_PASSWORD__"); + assertThat(analytics) + .contains("stat&pwd#2") + .doesNotContain("__MYSQL_PASSWORD__"); + assertThat(Files.list(launchAgents) + .filter(path -> path.getFileName().toString().endsWith(".plist")) + .count()) + .isEqualTo(5); + } + @Test void tdengineVerificationRunbookChecksLivenessAndReadinessForManagedApps() throws IOException { String runbook = Files.readString(repositoryRoot()