feat: productionize raw history ingestion

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java

@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
                 raw.getParseError(),
                 raw.getPeer(),
                 json(metadata),
+                parsedJson(envelope, metadata),
                 protocol(envelope),
                 vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
                 firstNonBlank(raw.getVin(), envelope.getVin()),
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
                 location.getStatusFlag(),
                 totalMileage(envelope),
                 rawUri,
-                json(envelope.getMetadataMap()),
                 protocol(envelope),
                 vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
                         envelope.getMetadataOrDefault("phone", "")),
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
         return Double.parseDouble(value);
     }
 
+    private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
+        if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
+            return envelope.getRawArchive().getParsedJson();
+        }
+        return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
+    }
+
     private static Double valueDouble(TelemetryField field) {
         String valueType = field.getValueType();
         if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java

@@ -6,11 +6,11 @@ import java.util.List;
 public final class TdengineHistoryQueries {
 
     private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
-            + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
+            + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
             + "protocol, vehicle_key, vin, phone";
     private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
             + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
-            + "metadata_json, protocol, vehicle_key, vin, phone";
+            + "protocol, vehicle_key, vin, phone";
     private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
             + "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
             + "metadata_json, protocol, vehicle_key, vin, phone";

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java

@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
                   parse_status NCHAR(16),
                   parse_error NCHAR(512),
                   peer NCHAR(128),
-                  metadata_json NCHAR(4096)
+                  metadata_json NCHAR(4096),
+                  parsed_json NCHAR(16374)
                 ) TAGS (
                   protocol NCHAR(16),
                   vehicle_key NCHAR(128),
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
                   alarm_flag BIGINT,
                   status_flag BIGINT,
                   total_mileage_km DOUBLE,
-                  raw_uri NCHAR(512),
-                  metadata_json NCHAR(4096)
+                  raw_uri NCHAR(512)
                 ) TAGS (
                   protocol NCHAR(16),
                   vehicle_key NCHAR(128),

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java

@@ -6,12 +6,12 @@ import java.util.Arrays;
 public final class TdengineHistoryStatements {
 
     private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
-            + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json";
-    private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+            + "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
+    private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
 
     private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
-            + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json";
-    private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+            + "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
+    private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
 
     private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
             + "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
                 values(
                         row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
                         row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
-                        row.parseError(), row.peer(), row.metadataJson()
+                        row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
                 )
         );
     }
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
                 values(
                         row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
                         row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
-                        row.totalMileageKm(), row.rawUri(), row.metadataJson()
+                        row.totalMileageKm(), row.rawUri()
                 )
         );
     }

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java

@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
                 rs.getString("parse_error"),
                 rs.getString("peer"),
                 rs.getString("metadata_json"),
+                rs.getString("parsed_json"),
                 rs.getString("protocol"),
                 rs.getString("vehicle_key"),
                 rs.getString("vin"),
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
                 rs.getLong("status_flag"),
                 nullableDouble(rs, "total_mileage_km"),
                 rs.getString("raw_uri"),
-                rs.getString("metadata_json"),
                 rs.getString("protocol"),
                 rs.getString("vehicle_key"),
                 rs.getString("vin"),

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java

@@ -16,7 +16,6 @@ public record TdengineLocationRow(
         long statusFlag,
         Double totalMileageKm,
         String rawUri,
-        String metadataJson,
         String protocol,
         String vehicleKey,
         String vin,

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java

@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
         String parseError,
         String peer,
         String metadataJson,
+        String parsedJson,
         String protocol,
         String vehicleKey,
         String vin,