chore: wait for gateway spool before acceptance
This commit is contained in:
@@ -173,13 +173,13 @@ python3 tools/go_native_prod_smoke.py --date 2026-07-02 --timeout 8
|
|||||||
|
|
||||||
## 部署命令
|
## 部署命令
|
||||||
|
|
||||||
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,并在发布后运行聚合生产验收:
|
推荐使用仓库脚本发布完整 Go 原生 release。脚本会在本机/CI 构建 Linux amd64 四个二进制,打包上传到应用 ECS,切换 `/opt/lingniu-go-native/current`,逐个重启四个 systemd 服务,等待 gateway Kafka spool 清空,并在发布后运行聚合生产验收:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 tools/go_native_deploy.py --date 2026-07-02
|
python3 tools/go_native_deploy.py --date 2026-07-02
|
||||||
```
|
```
|
||||||
|
|
||||||
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。
|
脚本不会保存 SSH 或数据库密钥;认证仍使用当前操作环境可用的 SSH 凭据。需要跳过发布后验收时可显式追加 `--skip-acceptance`,但生产发布默认应保留验收。发布后 spool 默认最多等待 `900` 秒,可用 `--spool-drain-timeout` 和 `--spool-poll-interval` 调整。
|
||||||
|
|
||||||
脚本执行的发布动作等价于:
|
脚本执行的发布动作等价于:
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import tarfile
|
import tarfile
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
@@ -41,6 +42,12 @@ class BuildCommand:
|
|||||||
cwd: str
|
cwd: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SpoolStatus:
|
||||||
|
files: int
|
||||||
|
recent: int
|
||||||
|
|
||||||
|
|
||||||
def git_short_sha(cwd: str) -> str:
|
def git_short_sha(cwd: str) -> str:
|
||||||
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
|
return run_capture(["git", "rev-parse", "--short=7", "HEAD"], cwd=cwd).strip()
|
||||||
|
|
||||||
@@ -137,10 +144,55 @@ def deploy(args: argparse.Namespace) -> None:
|
|||||||
target,
|
target,
|
||||||
remote_switch_command(release, args.release_root, remote_archive),
|
remote_switch_command(release, args.release_root, remote_archive),
|
||||||
])
|
])
|
||||||
|
wait_for_spool_drain(target, args.release_root, args.spool_drain_timeout, args.spool_poll_interval)
|
||||||
if not args.skip_acceptance:
|
if not args.skip_acceptance:
|
||||||
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
|
run(acceptance_command(args.app_host, args.kafka_host, args.date, args.timeout), cwd=str(repo))
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_spool_drain(target: str, release_root: str, timeout: float, poll_interval: float) -> None:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
last = SpoolStatus(files=-1, recent=-1)
|
||||||
|
while time.monotonic() <= deadline:
|
||||||
|
output = run_capture([
|
||||||
|
"ssh",
|
||||||
|
"-o",
|
||||||
|
"StrictHostKeyChecking=no",
|
||||||
|
"-o",
|
||||||
|
"UserKnownHostsFile=/dev/null",
|
||||||
|
target,
|
||||||
|
spool_status_command(release_root),
|
||||||
|
])
|
||||||
|
last = parse_spool_status(output)
|
||||||
|
print(f"spool files={last.files} recent={last.recent}", flush=True)
|
||||||
|
if last.files == 0:
|
||||||
|
return
|
||||||
|
time.sleep(max(1.0, poll_interval))
|
||||||
|
raise RuntimeError(f"gateway spool did not drain: files={last.files} recent={last.recent}")
|
||||||
|
|
||||||
|
|
||||||
|
def spool_status_command(release_root: str) -> str:
|
||||||
|
spool_dir = release_root.rstrip("/") + "/spool/gateway"
|
||||||
|
return (
|
||||||
|
"spool=" + shell_quote(spool_dir) + "; "
|
||||||
|
"files=$(find \"$spool\" -type f 2>/dev/null | wc -l); "
|
||||||
|
"recent=$(find \"$spool\" -type f -mmin -5 2>/dev/null | wc -l); "
|
||||||
|
"printf 'files=%s recent=%s\\n' \"$files\" \"$recent\""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_spool_status(output: str) -> SpoolStatus:
|
||||||
|
values: dict[str, int] = {}
|
||||||
|
for part in output.split():
|
||||||
|
if "=" not in part:
|
||||||
|
continue
|
||||||
|
key, value = part.split("=", 1)
|
||||||
|
try:
|
||||||
|
values[key] = int(value)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return SpoolStatus(files=values.get("files", -1), recent=values.get("recent", -1))
|
||||||
|
|
||||||
|
|
||||||
def ssh_target(user: str, host: str) -> str:
|
def ssh_target(user: str, host: str) -> str:
|
||||||
return user + "@" + host if user else host
|
return user + "@" + host if user else host
|
||||||
|
|
||||||
@@ -175,6 +227,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
|
|||||||
parser.add_argument("--release", default="")
|
parser.add_argument("--release", default="")
|
||||||
parser.add_argument("--date", default="")
|
parser.add_argument("--date", default="")
|
||||||
parser.add_argument("--timeout", type=float, default=20.0)
|
parser.add_argument("--timeout", type=float, default=20.0)
|
||||||
|
parser.add_argument("--spool-drain-timeout", type=float, default=900.0)
|
||||||
|
parser.add_argument("--spool-poll-interval", type=float, default=10.0)
|
||||||
parser.add_argument("--skip-acceptance", action="store_true")
|
parser.add_argument("--skip-acceptance", action="store_true")
|
||||||
return parser.parse_args(argv)
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
@@ -189,6 +243,9 @@ def main(argv: list[str]) -> int:
|
|||||||
except (OSError, shutil.Error) as exc:
|
except (OSError, shutil.Error) as exc:
|
||||||
print("deploy failed: " + str(exc), file=sys.stderr)
|
print("deploy failed: " + str(exc), file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
except RuntimeError as exc:
|
||||||
|
print("deploy failed: " + str(exc), file=sys.stderr)
|
||||||
|
return 1
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -45,6 +45,19 @@ class GoNativeDeployTest(unittest.TestCase):
|
|||||||
self.assertIn("--date", command)
|
self.assertIn("--date", command)
|
||||||
self.assertIn("2026-07-02", command)
|
self.assertIn("2026-07-02", command)
|
||||||
|
|
||||||
|
def test_parse_spool_status_reads_file_and_recent_counts(self):
|
||||||
|
status = deploy.parse_spool_status("files=42 recent=3")
|
||||||
|
|
||||||
|
self.assertEqual(status.files, 42)
|
||||||
|
self.assertEqual(status.recent, 3)
|
||||||
|
|
||||||
|
def test_spool_status_command_targets_gateway_spool_directory(self):
|
||||||
|
command = deploy.spool_status_command("/opt/lingniu-go-native")
|
||||||
|
|
||||||
|
self.assertIn("/opt/lingniu-go-native/spool/gateway", command)
|
||||||
|
self.assertIn("files=", command)
|
||||||
|
self.assertIn("recent=", command)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user