shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove telemetry fields from tdengine history

Browse Source
This commit is contained in:
lingniu
2026-07-01 14:03:32 +08:00
parent 68f45ee79c
commit c3d67e4339
22 changed files with 24 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java
View File

@@ -4,13 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.sink.kafka.proto.LocationPayload;
import com.lingniu.ingest.sink.kafka.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.kafka.proto.TelemetryField;
import com.lingniu.ingest.sink.kafka.proto.VehicleEnvelope;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -82,62 +79,6 @@ public final class TdengineEnvelopeRows {
));
}
public static List<TdengineTelemetryFieldRow> telemetryFields(VehicleEnvelope envelope) {
if (envelope == null || !envelope.hasTelemetrySnapshot()) {
return List.of();
}
var snapshot = envelope.getTelemetrySnapshot();
if (snapshot.getFieldsCount() == 0) {
return List.of();
}
List<TdengineTelemetryFieldRow> rows = new ArrayList<>(snapshot.getFieldsCount());
String protocol = protocol(envelope);
String vin = envelope.getVin();
String phone = firstNonBlank(
envelope.getMetadataOrDefault("phone", ""),
envelope.hasRawFrameFact() ? envelope.getRawFrameFact().getPhone() : "");
String vehicleKey = vehicleKey(envelope,
firstNonBlank(
envelope.getMetadataOrDefault("vehicle_key", ""),
envelope.hasRawFrameFact() ? envelope.getRawFrameFact().getVehicleKey() : ""),
phone);
String frameId = firstNonBlank(
envelope.getMetadataOrDefault("frame_id", ""),
envelope.hasRawFrameFact() ? envelope.getRawFrameFact().getFrameId() : "",
envelope.getEventId());
String rawUri = firstNonBlank(
snapshot.getRawArchiveUri(),
envelope.hasRawArchive() ? envelope.getRawArchive().getUri() : "");
String metadataJson = json(envelope.getMetadataMap());
for (int i = 0; i < snapshot.getFieldsCount(); i++) {
TelemetryField field = snapshot.getFields(i);
if (field.getKey().isBlank()) {
continue;
}
rows.add(new TdengineTelemetryFieldRow(
instant(envelope.getEventTimeMs()),
envelope.getEventId() + "#" + i,
frameId,
instant(envelope.getIngestTimeMs()),
field.getKey(),
field.getValueType(),
field.getValue(),
valueDouble(field),
valueLong(field),
field.getUnit(),
field.getQuality(),
field.getSourcePath(),
rawUri,
metadataJson,
protocol,
vehicleKey,
vin,
phone
));
}
return List.copyOf(rows);
}
private static Instant instant(long epochMillis) {
return Instant.ofEpochMilli(epochMillis);
}
@@ -195,41 +136,6 @@ public final class TdengineEnvelopeRows {
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
}
private static Double valueDouble(TelemetryField field) {
String valueType = field.getValueType();
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {
return null;
}
return parseDouble(field.getValue());
}
private static Long valueLong(TelemetryField field) {
String valueType = field.getValueType();
if (!"LONG".equals(valueType) && !"INT".equals(valueType) && !"INTEGER".equals(valueType)) {
return null;
}
String value = field.getValue();
if (value == null || value.isBlank()) {
return null;
}
try {
return Long.parseLong(value);
} catch (NumberFormatException ignored) {
return null;
}
}
private static Double parseDouble(String value) {
if (value == null || value.isBlank()) {
return null;
}
try {
return Double.parseDouble(value);
} catch (NumberFormatException ignored) {
return null;
}
}
private static String json(Map<String, String> metadata) {
try {
return OBJECT_MAPPER.writeValueAsString(metadata == null ? Map.of() : metadata);

13
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java
View File

@@ -11,9 +11,6 @@ public final class TdengineHistoryQueries {
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "protocol, vehicle_key, vin, phone";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";
private final TdengineHistorySchema schema;
@@ -39,20 +36,10 @@ public final class TdengineHistoryQueries {
query.order(), query.limit(), query.cursor());
}
public TdengineQueryStatement telemetryFields(TdengineTelemetryFieldQuery query) {
String table = schema.telemetryFieldTable(query.protocol(), query.vehicleKey(), query.fieldKey());
return query(table, TELEMETRY_FIELD_COLUMNS, "fact_id", query.from(), query.to(),
query.order(), query.limit(), query.cursor());
}
public TdengineQueryStatement locationsByRawUri(String protocol, String rawUri, int limit) {
return byRawUri(schema.locationsStableTable(), LOCATION_COLUMNS, protocol, rawUri, "fact_id", limit);
}
public TdengineQueryStatement telemetryFieldsByRawUri(String protocol, String rawUri, int limit) {
return byRawUri(schema.telemetryFieldsStableTable(), TELEMETRY_FIELD_COLUMNS, protocol, rawUri, "fact_id", limit);
}
private static TdengineQueryStatement query(String table,
String columns,
String tieBreakerColumn,

7
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryReader.java
View File

@@ -9,15 +9,8 @@ public interface TdengineHistoryReader {
TdenginePage<TdengineLocationRow> queryLocations(TdengineLocationQuery query) throws IOException;
TdenginePage<TdengineTelemetryFieldRow> queryTelemetryFields(TdengineTelemetryFieldQuery query) throws IOException;
default List<TdengineLocationRow> queryLocationsByRawUri(String protocol, String rawUri, int limit)
throws IOException {
return List.of();
}
default List<TdengineTelemetryFieldRow> queryTelemetryFieldsByRawUri(String protocol, String rawUri, int limit)
throws IOException {
return List.of();
}
}

37
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java
View File

@@ -18,8 +18,7 @@ public final class TdengineHistorySchema {
"CREATE DATABASE IF NOT EXISTS " + database + " PRECISION 'ms'",
"USE " + database,
rawFramesStableSql(),
vehicleLocationsStableSql(),
telemetryFieldsStableSql()
vehicleLocationsStableSql()
);
}
@@ -35,20 +34,10 @@ public final class TdengineHistorySchema {
return "vehicle_locations";
}
public String telemetryFieldsStableTable() {
return "telemetry_fields";
}
public String locationTable(String protocol, String vehicleKey) {
return "loc_" + TdengineIdentifier.fragment(protocol) + "_" + TdengineIdentifier.hash16(vehicleKey);
}
public String telemetryFieldTable(String protocol, String vehicleKey, String fieldKey) {
return "tf_" + TdengineIdentifier.fragment(protocol)
+ "_" + TdengineIdentifier.hash16(vehicleKey)
+ "_" + TdengineIdentifier.hash16(fieldKey);
}
private static String rawFramesStableSql() {
return """
CREATE STABLE IF NOT EXISTS raw_frames (
@@ -98,28 +87,4 @@ public final class TdengineHistorySchema {
)""";
}
private static String telemetryFieldsStableSql() {
return """
CREATE STABLE IF NOT EXISTS telemetry_fields (
ts TIMESTAMP,
fact_id NCHAR(96),
frame_id NCHAR(64),
received_at TIMESTAMP,
value_type NCHAR(16),
value_text NCHAR(1024),
value_double DOUBLE,
value_long BIGINT,
unit NCHAR(32),
quality NCHAR(16),
source_path NCHAR(256),
raw_uri NCHAR(512),
metadata_json NCHAR(4096)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),
vin NCHAR(64),
phone NCHAR(32),
field_key NCHAR(256)
)""";
}
}

23
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java
View File

@@ -13,10 +13,6 @@ public final class TdengineHistoryStatements {
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
private static final String TELEMETRY_FIELD_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private final TdengineHistorySchema schema;
public TdengineHistoryStatements(TdengineHistorySchema schema) {
@@ -54,29 +50,10 @@ public final class TdengineHistoryStatements {
);
}
public TdengineBatchStatement telemetryField(TdengineTelemetryFieldRow row) {
String table = schema.telemetryFieldTable(row.protocol(), row.vehicleKey(), row.fieldKey());
return new TdengineBatchStatement(
"CREATE TABLE IF NOT EXISTS " + table + " USING telemetry_fields TAGS ("
+ tags(row.protocol(), row.vehicleKey(), row.vin(), row.phone(), row.fieldKey()) + ")",
"INSERT INTO " + table + " (" + TELEMETRY_FIELD_COLUMNS + ") VALUES ("
+ TELEMETRY_FIELD_PLACEHOLDERS + ")",
values(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.valueType(), row.valueText(),
row.valueDouble(), row.valueLong(), row.unit(), row.quality(), row.sourcePath(),
row.rawUri(), row.metadataJson()
)
);
}
private static String tags(String protocol, String vehicleKey, String vin, String phone) {
return quote(protocol) + ", " + quote(vehicleKey) + ", " + quote(vin) + ", " + quote(phone);
}
private static String tags(String protocol, String vehicleKey, String vin, String phone, String fieldKey) {
return tags(protocol, vehicleKey, vin, phone) + ", " + quote(fieldKey);
}
private static String quote(String value) {
return "'" + (value == null ? "" : value.replace("'", "''")) + "'";
}

6
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryWriter.java
View File

@@ -16,10 +16,4 @@ public interface TdengineHistoryWriter {
}
void appendLocations(List<TdengineLocationRow> rows) throws IOException;
default void appendTelemetryField(TdengineTelemetryFieldRow row) throws IOException {
appendTelemetryFields(List.of(row));
}
void appendTelemetryFields(List<TdengineTelemetryFieldRow> rows) throws IOException;
}

47
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java
View File

@@ -40,15 +40,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
return readPage(statement, pageSize, this::location, row -> new TdenginePageCursor(row.ts(), row.factId()));
}
@Override
public TdenginePage<TdengineTelemetryFieldRow> queryTelemetryFields(TdengineTelemetryFieldQuery query)
throws IOException {
int pageSize = Math.min(query.limit(), 1000);
TdengineQueryStatement statement = queries.telemetryFields(query.withLimit(pageSize + 1));
return readPage(statement, pageSize, this::telemetryField,
row -> new TdenginePageCursor(row.ts(), row.factId()));
}
@Override
public List<TdengineLocationRow> queryLocationsByRawUri(String protocol, String rawUri, int limit)
throws IOException {
@@ -59,16 +50,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
return readList(statement, this::location);
}
@Override
public List<TdengineTelemetryFieldRow> queryTelemetryFieldsByRawUri(String protocol, String rawUri, int limit)
throws IOException {
if (rawUri == null || rawUri.isBlank()) {
return List.of();
}
TdengineQueryStatement statement = queries.telemetryFieldsByRawUri(protocol, rawUri, limit);
return readList(statement, this::telemetryField);
}
private <T> TdenginePage<T> readPage(TdengineQueryStatement statement,
int pageSize,
SqlRowMapper<T> mapper,
@@ -187,29 +168,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
);
}
private TdengineTelemetryFieldRow telemetryField(ResultSet rs) throws SQLException {
return new TdengineTelemetryFieldRow(
instant(rs, "ts"),
rs.getString("fact_id"),
rs.getString("frame_id"),
instant(rs, "received_at"),
rs.getString("field_key"),
rs.getString("value_type"),
rs.getString("value_text"),
nullableDouble(rs, "value_double"),
nullableLong(rs, "value_long"),
rs.getString("unit"),
rs.getString("quality"),
rs.getString("source_path"),
rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
rs.getString("phone")
);
}
private static Instant instant(ResultSet rs, String column) throws SQLException {
Timestamp timestamp = rs.getTimestamp(column);
return timestamp == null ? null : timestamp.toInstant();
@@ -220,11 +178,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
return value instanceof Number number ? number.doubleValue() : null;
}
private static Long nullableLong(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
return value instanceof Number number ? number.longValue() : null;
}
@FunctionalInterface
private interface SqlRowMapper<T> {
T map(ResultSet rs) throws SQLException;

5
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriter.java
View File

@@ -59,11 +59,6 @@ public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
append(rows, statements::location);
}
@Override
public void appendTelemetryFields(List<TdengineTelemetryFieldRow> rows) throws IOException {
append(rows, statements::telemetryField);
}
private <T> void append(List<T> rows, Function<T, TdengineBatchStatement> mapper) throws IOException {
if (rows == null || rows.isEmpty()) {
return;

35
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineTelemetryFieldQuery.java
View File

@@ -1,35 +0,0 @@
package com.lingniu.ingest.tdenginehistory;
import java.time.Instant;
public record TdengineTelemetryFieldQuery(
String protocol,
String vehicleKey,
String fieldKey,
Instant from,
Instant to,
TdengineQueryOrder order,
int limit,
TdenginePageCursor cursor
) {
public TdengineTelemetryFieldQuery {
if (protocol == null || protocol.isBlank()) {
throw new IllegalArgumentException("protocol must not be blank");
}
if (vehicleKey == null || vehicleKey.isBlank()) {
throw new IllegalArgumentException("vehicleKey must not be blank");
}
if (fieldKey == null || fieldKey.isBlank()) {
throw new IllegalArgumentException("fieldKey must not be blank");
}
if (from == null || to == null || !from.isBefore(to)) {
throw new IllegalArgumentException("query time range must be valid");
}
order = order == null ? TdengineQueryOrder.DESC : order;
limit = Math.max(1, Math.min(limit, 1001));
}
TdengineTelemetryFieldQuery withLimit(int newLimit) {
return new TdengineTelemetryFieldQuery(protocol, vehicleKey, fieldKey, from, to, order, newLimit, cursor);
}
}

25
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineTelemetryFieldRow.java
View File

@@ -1,25 +0,0 @@
package com.lingniu.ingest.tdenginehistory;
import java.time.Instant;
public record TdengineTelemetryFieldRow(
Instant ts,
String factId,
String frameId,
Instant receivedAt,
String fieldKey,
String valueType,
String valueText,
Double valueDouble,
Long valueLong,
String unit,
String quality,
String sourcePath,
String rawUri,
String metadataJson,
String protocol,
String vehicleKey,
String vin,
String phone
) {
}