fix: prevent tdengine raw frame overwrite

tools/jt808_e2e_smoke.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import argparse
 import datetime as dt
 import json
+import os
 import pathlib
 import socket
 import time
@@ -13,6 +14,11 @@ import urllib.parse
 import urllib.request
 from typing import Any
 
+try:
+    from tools import tdengine_smoke
+except ModuleNotFoundError:
+    import tdengine_smoke
+
 
 def bcd(value: str) -> bytes:
     digits = "".join(ch for ch in value if ch.isdigit())
@@ -247,6 +253,69 @@ def archive_path(root: pathlib.Path, uri: str) -> pathlib.Path:
     return root / uri[len(prefix):]
 
 
+def raw_frame_count_sql(phone: str, items: list[dict[str, Any]]) -> str:
+    sqls = raw_frame_count_sqls(phone, items)
+    if not sqls:
+        raise ValueError("no raw uri found in jt808 location items")
+    return sqls[0]
+
+
+def raw_frame_count_sqls(phone: str, items: list[dict[str, Any]]) -> list[str]:
+    seen: set[str] = set()
+    sqls: list[str] = []
+    for item in items:
+        raw_uri = item.get("rawUri") or ""
+        if not raw_uri or raw_uri in seen:
+            continue
+        seen.add(raw_uri)
+        sqls.append(tdengine_smoke.raw_frame_count_sql(
+            protocol="JT808",
+            vehicle_key=item.get("vehicleKey") or ("jt808:" + phone),
+            phone=phone,
+            raw_uri=raw_uri,
+        ))
+    return sqls
+
+
+def verify_tdengine_raw_frames(
+    rest_url: str,
+    username: str,
+    password: str,
+    sqls: list[str],
+    timeout_seconds: float,
+    request_timeout: float,
+) -> int:
+    if not sqls:
+        raise RuntimeError("no raw uri available for tdengine raw_frames verification")
+    verified = 0
+    for sql in sqls:
+        count = wait_for_tdengine_raw_frames(
+            rest_url, username, password, sql, 1, timeout_seconds, request_timeout)
+        if count < 1:
+            raise RuntimeError(f"tdengine raw_frames row not visible for query: {sql}")
+        verified += 1
+    return verified
+
+
+def wait_for_tdengine_raw_frames(
+    rest_url: str,
+    username: str,
+    password: str,
+    sql: str,
+    minimum: int,
+    timeout_seconds: float,
+    request_timeout: float,
+) -> int:
+    deadline = time.monotonic() + timeout_seconds
+    last_count = 0
+    while time.monotonic() < deadline:
+        last_count = tdengine_smoke.query_count(rest_url, username, password, sql, request_timeout)
+        if last_count >= minimum:
+            return last_count
+        time.sleep(1)
+    return last_count
+
+
 def wait_for_locations(
     history_base_url: str,
     phone: str,
@@ -285,6 +354,13 @@ def generated_phone(start_phone: str, offset: int) -> str:
     return str(int(start_phone) + offset)
 
 
+def expected_history_count(args: argparse.Namespace) -> int:
+    minimum = min(args.frames, args.expect_history_count)
+    if args.verify_pagination and args.frames > 1:
+        minimum = max(minimum, 2)
+    return minimum
+
+
 def run(args: argparse.Namespace) -> dict[str, Any]:
     event_time = parse_event_time(args.event_time)
     date_from = (event_time - dt.timedelta(minutes=5)).isoformat()
@@ -339,12 +415,12 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
         first_phone,
         date_from,
         date_to,
-        min(args.frames, args.expect_history_count),
+        expected_history_count(args),
         args.history_timeout,
         args.http_timeout,
     )
     items = page.get("items") or []
-    if len(items) < min(args.frames, args.expect_history_count):
+    if len(items) < expected_history_count(args):
         raise RuntimeError(f"history rows not visible for {first_phone}: got {len(items)}")
 
     archive_checked = 0
@@ -356,6 +432,17 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
                 raise RuntimeError(f"raw archive missing: {path}")
             archive_checked += 1
 
+    tdengine_raw_frames = None
+    if args.tdengine_rest_url:
+        tdengine_raw_frames = verify_tdengine_raw_frames(
+            args.tdengine_rest_url,
+            args.tdengine_username,
+            args.tdengine_password,
+            raw_frame_count_sqls(first_phone, items),
+            args.history_timeout,
+            args.http_timeout,
+        )
+
     pagination_ok = None
     if args.verify_pagination and args.frames > 1:
         pagination_ok = verify_pagination(args.history_base_url, first_phone, date_from, date_to, args.http_timeout)
@@ -372,6 +459,7 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
         "sendElapsedSeconds": round(elapsed, 3),
         "sendLocationsPerSecond": round(sent_locations / elapsed, 2),
         "historyRowsForFirstPhone": len(items),
+        "tdengineRawFrames": tdengine_raw_frames,
         "archiveChecked": archive_checked,
         "paginationOk": pagination_ok,
         "firstFrameId": items[0].get("frameId") if items else None,
@@ -396,6 +484,9 @@ def parser() -> argparse.ArgumentParser:
     p.add_argument("--verify-pagination", action="store_true")
     p.add_argument("--archive-root", default="")
     p.add_argument("--archive-check-limit", type=int, default=3)
+    p.add_argument("--tdengine-rest-url", default=os.environ.get("TDENGINE_REST_URL", ""))
+    p.add_argument("--tdengine-username", default=os.environ.get("TDENGINE_USERNAME", "root"))
+    p.add_argument("--tdengine-password", default=os.environ.get("TDENGINE_PASSWORD", "taosdata"))
     p.add_argument("--require-register-ack", action="store_true", default=True)
     p.add_argument("--tcp-timeout", type=float, default=2)
     p.add_argument("--http-timeout", type=float, default=5)