diff --git a/docs/operations/current-ecs-deployment.md b/docs/operations/current-ecs-deployment.md index 711c89ea..cd6b8bf2 100644 --- a/docs/operations/current-ecs-deployment.md +++ b/docs/operations/current-ecs-deployment.md @@ -173,13 +173,13 @@ python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8 ## 部署命令 -推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,并在发布后运行聚合生产验收: +推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收: ```bash python3 tools/go_native_deploy.py --date 2026-07-02 ``` -脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。 +脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。 脚本执行的发布动作等价于: diff --git a/tools/go_native_deploy.py b/tools/go_native_deploy.py index 6c6c1ed0..8d7e5358 100644 --- a/tools/go_native_deploy.py +++ b/tools/go_native_deploy.py @@ -11,6 +11,7 @@ import subprocess import sys import tarfile import tempfile +import time from dataclasses import dataclass @@ -41,6 +42,12 @@ class BuildCommand: cwd: str +@dataclass(frozen=True) +class SpoolStatus: + files: int + recent: int + + def git_short_sha(cwd: str) -> str: return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip() @@ -137,10 +144,55 @@ def deploy(args: argparse.Namespace) -> None: target, remote_switch_command(release, args.release_root, remote_archive), ]) + wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval) if not args.skip_acceptance: run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo)) +def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None: + deadline = time.monotonic() + timeout + last = SpoolStatus(files=-1, recent=-1) + while time.monotonic() <= deadline: + output = run_capture([ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + target, + spool_status_command(release_root), + ]) + last = parse_spool_status(output) + print(f"spool files={last.files} recent={last.recent}", flush=True) + if last.files == 0: + return + time.sleep(max(1.0, poll_interval)) + raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}") + + +def spool_status_command(release_root: str) -> str: + spool_dir = release_root.rstrip("/") + "/spool/gateway" + return ( + "spool=" + shell_quote(spool_dir) + "; " + "files=$(find \"$spool\" -type f 2>/dev/null | wc -l); " + "recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); " + "printf 'files=%s recent=%s\\n' \"$files\" \"$recent\"" + ) + + +def parse_spool_status(output: str) -> SpoolStatus: + values: dict[str, int] = {} + for part in output.split(): + if "=" not in part: + continue + key, value = part.split("=", 1) + try: + values[key] = int(value) + except ValueError: + continue + return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1)) + + def ssh_target(user: str, host: str) -> str: return user + "@" + host if user else host @@ -175,6 +227,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--release", default="") parser.add_argument("--date", default="") parser.add_argument("--timeout", type=float, default=20.0) + parser.add_argument("--spool-drain-timeout", type=float, default=900.0) + parser.add_argument("--spool-poll-interval", type=float, default=10.0) parser.add_argument("--skip-acceptance", action="store_true") return parser.parse_args(argv) @@ -189,6 +243,9 @@ def main(argv: list[str]) -> int: except (OSError, shutil.Error) as exc: print("deploy failed: " + str(exc), file=sys.stderr) return 1 + except RuntimeError as exc: + print("deploy failed: " + str(exc), file=sys.stderr) + return 1 return 0 diff --git a/tools/test_go_native_deploy.py b/tools/test_go_native_deploy.py index 9d7d25fb..24110410 100644 --- a/tools/test_go_native_deploy.py +++ b/tools/test_go_native_deploy.py @@ -45,6 +45,19 @@ class GoNativeDeployTest(unittest.TestCase): self.assertIn("--date", command) self.assertIn("2026-07-02", command) + def test_parse_spool_status_reads_file_and_recent_counts(self): + status = deploy.parse_spool_status("files=42 recent=3") + + self.assertEqual(status.files, 42) + self.assertEqual(status.recent, 3) + + def test_spool_status_command_targets_gateway_spool_directory(self): + command = deploy.spool_status_command("/opt/lingniu-go-native") + + self.assertIn("/opt/lingniu-go-native/spool/gateway", command) + self.assertIn("files=", command) + self.assertIn("recent=", command) + if __name__ == "__main__": unittest.main()