shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java
View File

@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
raw.getParseError(),
raw.getPeer(),
json(metadata),
parsedJson(envelope, metadata),
protocol(envelope),
vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
firstNonBlank(raw.getVin(), envelope.getVin()),
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
location.getStatusFlag(),
totalMileage(envelope),
rawUri,
json(envelope.getMetadataMap()),
protocol(envelope),
vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
envelope.getMetadataOrDefault("phone", "")),
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
return Double.parseDouble(value);
}
private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
return envelope.getRawArchive().getParsedJson();
}
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
}
private static Double valueDouble(TelemetryField field) {
String valueType = field.getValueType();
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {

4
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java
View File

@@ -6,11 +6,11 @@ import java.util.List;
public final class TdengineHistoryQueries {
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
+ "protocol, vehicle_key, vin, phone";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";
+ "protocol, vehicle_key, vin, phone";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";

6
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java
View File

@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
parse_status NCHAR(16),
parse_error NCHAR(512),
peer NCHAR(128),
metadata_json NCHAR(4096)
metadata_json NCHAR(4096),
parsed_json NCHAR(16374)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE,
raw_uri NCHAR(512),
metadata_json NCHAR(4096)
raw_uri NCHAR(512)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),

12
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java
View File

@@ -6,12 +6,12 @@ import java.util.Arrays;
public final class TdengineHistoryStatements {
private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson()
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
)
);
}
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson()
row.totalMileageKm(), row.rawUri()
)
);
}

2
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java
View File

@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getString("parse_error"),
rs.getString("peer"),
rs.getString("metadata_json"),
rs.getString("parsed_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getLong("status_flag"),
nullableDouble(rs, "total_mileage_km"),
rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),

1
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java
View File

@@ -16,7 +16,6 @@ public record TdengineLocationRow(
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson,
String protocol,
String vehicleKey,
String vin,

1
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java
View File

@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
String parseError,
String peer,
String metadataJson,
String parsedJson,
String protocol,
String vehicleKey,
String vin,

10
modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java
View File

@@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest {
"",
"10.0.0.1:808",
"{\"auth\":\"passed\"}",
"{\"body\":{\"speedKmh\":42.0}}",
"JT808",
"jt808:g7gps",
"VIN123",
@@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest {
.contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')");
assertThat(batch.insertSql())
.startsWith("INSERT INTO raw_jt808_")
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
assertThat(batch.values())
.containsExactly(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson());
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson());
}
@Test
@@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:O'HARE",
"O'HARE",
@@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest {
.containsExactly(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson());
row.totalMileageKm(), row.rawUri());
}
@Test

2
modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java
View File

@@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:VIN123",
"VIN123",
@@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest {
parseError,
"10.0.0.1:808",
"{}",
"{\"frame\":\"" + frameId + "\"}",
"JT808",
"jt808:g7gps",
"VIN123",