test: verify gateway spool is drained

tools/go_systemd_prod_smoke.py

@@ -23,6 +23,7 @@ DEFAULT_PORTS = {
     32960: "gateway",
     20210: "realtime-api",
 }
+DEFAULT_SPOOL_DIR = "/opt/lingniu-go-native/spool/gateway"
 
 
 @dataclass(frozen=True)
@@ -83,6 +84,24 @@ def has_port(line: str, port: int) -> bool:
     return needle + " " in line or needle + "\t" in line
 
 
+def spool_check(output: str) -> Check:
+    values = parse_key_values(output)
+    files = int(values.get("files", "-1"))
+    recent = int(values.get("recent", "-1"))
+    status = "pass" if files == 0 and recent == 0 else "fail"
+    return Check("spool.gateway", status, output.strip() or "missing")
+
+
+def parse_key_values(output: str) -> dict[str, str]:
+    values: dict[str, str] = {}
+    for part in output.split():
+        if "=" not in part:
+            continue
+        key, value = part.split("=", 1)
+        values[key] = value
+    return values
+
+
 def ssh(host: str, user: str, command: str, timeout: float) -> str:
     target = user + "@" + host if user else host
     completed = subprocess.run(
@@ -105,22 +124,38 @@ def remote_command(services: list[str]) -> str:
         "systemctl is-enabled \"$svc\" 2>/dev/null || true; "
         "done; "
         "printf '\\n--PORTS--\\n'; "
-        "ss -lntp"
+        "ss -lntp; "
+        "printf '\\n--SPOOL--\\n'; "
+        "spool='" + DEFAULT_SPOOL_DIR + "'; "
+        "if test -d \"$spool\"; then "
+        "files=$(find \"$spool\" -type f | wc -l); "
+        "bytes=$(du -sb \"$spool\" 2>/dev/null | awk '{print $1}'); "
+        "recent=$(find \"$spool\" -type f -mmin -5 | wc -l); "
+        "printf 'files=%s bytes=%s recent=%s\\n' \"$files\" \"${bytes:-0}\" \"$recent\"; "
+        "else printf 'files=-1 bytes=0 recent=-1 missing=%s\\n' \"$spool\"; fi"
     )
 
 
-def split_remote_output(output: str) -> tuple[str, str]:
-    marker = "\n--PORTS--\n"
-    if marker not in output:
-        return output, ""
-    before, after = output.split(marker, 1)
-    return before, after
+def split_remote_output(output: str) -> tuple[str, str, str]:
+    ports_marker = "\n--PORTS--\n"
+    spool_marker = "\n--SPOOL--\n"
+    if ports_marker not in output:
+        return output, "", ""
+    service_output, rest = output.split(ports_marker, 1)
+    if spool_marker not in rest:
+        return service_output, rest, ""
+    port_output, spool_output = rest.split(spool_marker, 1)
+    return service_output, port_output, spool_output
 
 
 def run(args: argparse.Namespace) -> tuple[str, list[Check]]:
     output = ssh(args.host, args.user, remote_command(DEFAULT_SERVICES), args.timeout)
-    service_output, port_output = split_remote_output(output)
-    checks = service_checks(service_output, DEFAULT_SERVICES) + port_checks(port_output, DEFAULT_PORTS)
+    service_output, port_output, spool_output = split_remote_output(output)
+    checks = (
+        service_checks(service_output, DEFAULT_SERVICES)
+        + port_checks(port_output, DEFAULT_PORTS)
+        + [spool_check(spool_output)]
+    )
     status = "fail" if any(check.status == "fail" for check in checks) else "pass"
     return status, checks
 

tools/test_go_systemd_prod_smoke.py

@@ -66,6 +66,18 @@ LISTEN 0 4096 *:20210 *:* users:(("realtime-api",pid=11,fd=3))
         self.assertEqual(by_name["port.808"].status, "fail")
         self.assertIn("java", by_name["port.808"].message)
 
+    def test_spool_check_passes_when_no_pending_files(self):
+        check = smoke.spool_check("files=0 bytes=208896 recent=0")
+
+        self.assertEqual(check.status, "pass")
+        self.assertIn("files=0", check.message)
+
+    def test_spool_check_fails_when_pending_or_recent_files_exist(self):
+        for output in ["files=2 bytes=300 recent=0", "files=0 bytes=208896 recent=1"]:
+            check = smoke.spool_check(output)
+
+            self.assertEqual(check.status, "fail")
+
 
 if __name__ == "__main__":
     unittest.main()