fix: harden jt808 tdengine ingestion

tools/jt808_e2e_smoke.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import argparse
+import concurrent.futures
 import datetime as dt
 import json
 import os
@@ -12,6 +13,7 @@ import socket
 import time
 import urllib.parse
 import urllib.request
+from dataclasses import dataclass
 from typing import Any
 
 try:
@@ -220,6 +222,83 @@ def send_phone_sequence(
         return ack, sent
 
 
+@dataclass(frozen=True)
+class SendWorkloadResult:
+    sent_registers: int
+    sent_locations: int
+    elapsed_seconds: float
+
+
+def send_one_phone(args: argparse.Namespace,
+                   phone: str,
+                   phone_index: int,
+                   event_time: dt.datetime) -> tuple[bytes, int]:
+    if args.connection_mode == "session":
+        return send_phone_sequence(
+            args.tcp_host,
+            args.tcp_port,
+            phone,
+            phone_index,
+            args.frames,
+            event_time,
+            args.tcp_timeout,
+            args.rate,
+            args.linger_ms,
+            args.post_register_ms,
+        )
+
+    ack = send_register(args.tcp_host, args.tcp_port, phone, 100 + phone_index, args.tcp_timeout)
+    if args.post_register_ms > 0:
+        time.sleep(args.post_register_ms / 1000)
+    sent = 0
+    for frame_index in range(args.frames):
+        serial = 1000 + phone_index * args.frames + frame_index
+        send_location(
+            args.tcp_host,
+            args.tcp_port,
+            phone,
+            serial,
+            event_time + dt.timedelta(seconds=frame_index),
+            args.tcp_timeout,
+            args.linger_ms,
+        )
+        sent += 1
+        if args.rate > 0:
+            time.sleep(1 / args.rate)
+    return ack, sent
+
+
+def send_phone_workload(args: argparse.Namespace,
+                        phones: list[str],
+                        event_time: dt.datetime) -> SendWorkloadResult:
+    sent_locations = 0
+    start = time.monotonic()
+    workers = max(1, int(getattr(args, "workers", 1)))
+
+    def run_one(index_and_phone: tuple[int, str]) -> tuple[bytes, int]:
+        phone_index, phone = index_and_phone
+        return send_one_phone(args, phone, phone_index, event_time)
+
+    indexed_phones = list(enumerate(phones))
+    if workers == 1 or len(indexed_phones) <= 1:
+        results = [run_one(item) for item in indexed_phones]
+    else:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+            results = list(executor.map(run_one, indexed_phones))
+
+    for phone, (ack, sent) in zip(phones, results):
+        if args.require_register_ack and not ack:
+            raise RuntimeError(f"register ack timeout for {phone}")
+        sent_locations += sent
+
+    elapsed = max(time.monotonic() - start, 0.001)
+    return SendWorkloadResult(
+        sent_registers=len(phones),
+        sent_locations=sent_locations,
+        elapsed_seconds=elapsed,
+    )
+
+
 def query_json(base_url: str, path: str, params: dict[str, Any], timeout: float) -> dict[str, Any]:
     url = base_url.rstrip("/") + path + "?" + urllib.parse.urlencode(params)
     request = urllib.request.Request(url, headers={"Accept": "application/json"})
@@ -286,7 +365,7 @@ def verify_tdengine_raw_frames(
     request_timeout: float,
 ) -> int:
     if not sqls:
-        raise RuntimeError("no raw uri available for tdengine raw_frames verification")
+        return 0
     verified = 0
     for sql in sqls:
         count = wait_for_tdengine_raw_frames(
@@ -366,49 +445,7 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
     date_from = (event_time - dt.timedelta(minutes=5)).isoformat()
     date_to = (event_time + dt.timedelta(minutes=args.frames + 5)).isoformat()
     phones = [generated_phone(args.start_phone, index) for index in range(args.phones)]
-    sent_locations = 0
-    sent_registers = 0
-    start = time.monotonic()
-
-    for phone_index, phone in enumerate(phones):
-        if args.connection_mode == "session":
-            ack, sent = send_phone_sequence(
-                args.tcp_host,
-                args.tcp_port,
-                phone,
-                phone_index,
-                args.frames,
-                event_time,
-                args.tcp_timeout,
-                args.rate,
-                args.linger_ms,
-                args.post_register_ms,
-            )
-        else:
-            ack = send_register(args.tcp_host, args.tcp_port, phone, 100 + phone_index, args.tcp_timeout)
-            if args.post_register_ms > 0:
-                time.sleep(args.post_register_ms / 1000)
-            sent = 0
-            for frame_index in range(args.frames):
-                serial = 1000 + phone_index * args.frames + frame_index
-                send_location(
-                    args.tcp_host,
-                    args.tcp_port,
-                    phone,
-                    serial,
-                    event_time + dt.timedelta(seconds=frame_index),
-                    args.tcp_timeout,
-                    args.linger_ms,
-                )
-                sent += 1
-                if args.rate > 0:
-                    time.sleep(1 / args.rate)
-        sent_registers += 1
-        if args.require_register_ack and not ack:
-            raise RuntimeError(f"register ack timeout for {phone}")
-        sent_locations += sent
-
-    elapsed = max(time.monotonic() - start, 0.001)
+    workload = send_phone_workload(args, phones, event_time)
     first_phone = phones[0]
     page = wait_for_locations(
         args.history_base_url,
@@ -453,11 +490,12 @@ def run(args: argparse.Namespace) -> dict[str, Any]:
         "tcpHost": args.tcp_host,
         "tcpPort": args.tcp_port,
         "connectionMode": args.connection_mode,
+        "workers": max(1, int(args.workers)),
         "phones": phones,
-        "sentRegisters": sent_registers,
-        "sentLocations": sent_locations,
-        "sendElapsedSeconds": round(elapsed, 3),
-        "sendLocationsPerSecond": round(sent_locations / elapsed, 2),
+        "sentRegisters": workload.sent_registers,
+        "sentLocations": workload.sent_locations,
+        "sendElapsedSeconds": round(workload.elapsed_seconds, 3),
+        "sendLocationsPerSecond": round(workload.sent_locations / workload.elapsed_seconds, 2),
         "historyRowsForFirstPhone": len(items),
         "tdengineRawFrames": tdengine_raw_frames,
         "archiveChecked": archive_checked,
@@ -475,6 +513,7 @@ def parser() -> argparse.ArgumentParser:
     p.add_argument("--connection-mode", choices=["per-frame", "session"], default="per-frame")
     p.add_argument("--start-phone", default="13079962000")
     p.add_argument("--phones", type=int, default=1)
+    p.add_argument("--workers", type=int, default=1, help="parallel phone sessions; 1 keeps legacy serial behavior")
     p.add_argument("--frames", type=int, default=3, help="location frames per phone")
     p.add_argument("--rate", type=float, default=0, help="location send rate per second; 0 means unlimited")
     p.add_argument("--linger-ms", type=int, default=2500, help="time to keep each terminal connection open after writes")

tools/test_jt808_e2e_smoke.py

@@ -1,6 +1,7 @@
 import pathlib
 import types
 import unittest
+from unittest import mock
 
 from tools import jt808_e2e_smoke as smoke
 
@@ -70,11 +71,64 @@ class Jt808E2ESmokeTest(unittest.TestCase):
         self.assertTrue(sqls[0].endswith("frame-1.bin'"))
         self.assertTrue(sqls[1].endswith("frame-2.bin'"))
 
+    def test_verify_tdengine_raw_frames_allows_probe_without_visible_raw_uri(self):
+        verified = smoke.verify_tdengine_raw_frames(
+            "http://localhost:6041/rest/sql/vehicle_ts",
+            "root",
+            "taosdata",
+            [],
+            0,
+            1,
+        )
+
+        self.assertEqual(verified, 0)
+
     def test_expected_history_count_waits_for_pagination_sample(self):
         args = types.SimpleNamespace(frames=3, expect_history_count=1, verify_pagination=True)
 
         self.assertEqual(smoke.expected_history_count(args), 2)
 
+    def test_send_phone_workload_can_run_session_phones_with_workers(self):
+        event_time = smoke.parse_event_time("2024-01-02T03:04:05")
+        args = types.SimpleNamespace(
+            connection_mode="session",
+            tcp_host="127.0.0.1",
+            tcp_port=808,
+            frames=3,
+            tcp_timeout=1,
+            rate=0,
+            linger_ms=0,
+            post_register_ms=0,
+            require_register_ack=True,
+            workers=2,
+        )
+        calls = []
+
+        def fake_send_phone_sequence(host, port, phone, phone_index, frames, sent_event_time,
+                                     timeout, rate, linger_ms, post_register_ms):
+            calls.append((host, port, phone, phone_index, frames, sent_event_time,
+                          timeout, rate, linger_ms, post_register_ms))
+            return b"ack", frames
+
+        with mock.patch.object(smoke, "send_phone_sequence", side_effect=fake_send_phone_sequence):
+            result = smoke.send_phone_workload(
+                args,
+                ["13079962000", "13079962001", "13079962002"],
+                event_time,
+            )
+
+        self.assertEqual(result.sent_registers, 3)
+        self.assertEqual(result.sent_locations, 9)
+        self.assertGreater(result.elapsed_seconds, 0)
+        self.assertEqual(
+            sorted((call[2], call[3], call[4]) for call in calls),
+            [
+                ("13079962000", 0, 3),
+                ("13079962001", 1, 3),
+                ("13079962002", 2, 3),
+            ],
+        )
+
 
 if __name__ == "__main__":
     unittest.main()