shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add tdengine history keyset queries

Browse Source
This commit is contained in:
lingniu
2026-06-29 13:38:39 +08:00
parent e708091f2d
commit 8466d622f6
13 changed files with 573 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java Normal file
View File

@@ -0,0 +1,65 @@
package com.lingniu.ingest.tdenginehistory;
import java.util.ArrayList;
import java.util.List;
public final class TdengineHistoryQueries {
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
+ "protocol, vehicle_key, vin, phone";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";
private final TdengineHistorySchema schema;
public TdengineHistoryQueries(TdengineHistorySchema schema) {
if (schema == null) {
throw new IllegalArgumentException("schema must not be null");
}
this.schema = schema;
}
public TdengineQueryStatement rawFrames(TdengineRawFrameQuery query) {
String table = schema.rawFrameTable(query.protocol(), query.vehicleKey());
return query(table, RAW_COLUMNS, "frame_id", query.from(), query.to(),
query.order(), query.limit(), query.cursor());
}
public TdengineQueryStatement locations(TdengineLocationQuery query) {
String table = schema.locationTable(query.protocol(), query.vehicleKey());
return query(table, LOCATION_COLUMNS, "fact_id", query.from(), query.to(),
query.order(), query.limit(), query.cursor());
}
private static TdengineQueryStatement query(String table,
String columns,
String tieBreakerColumn,
Object from,
Object to,
TdengineQueryOrder order,
int limit,
TdenginePageCursor cursor) {
List<Object> values = new ArrayList<>();
StringBuilder sql = new StringBuilder(256)
.append("SELECT ").append(columns)
.append(" FROM ").append(table)
.append(" WHERE ts >= ? AND ts < ?");
values.add(from);
values.add(to);
if (cursor != null) {
String op = order == TdengineQueryOrder.ASC ? ">" : "<";
sql.append(" AND (ts ").append(op).append(" ? OR (ts = ? AND ")
.append(tieBreakerColumn).append(' ').append(op).append(" ?))");
values.add(cursor.ts());
values.add(cursor.ts());
values.add(cursor.tieBreaker());
}
sql.append(" ORDER BY ts ").append(order.name())
.append(", ").append(tieBreakerColumn).append(' ').append(order.name())
.append(" LIMIT ?");
values.add(limit);
return new TdengineQueryStatement(sql.toString(), values);
}
}

10
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryReader.java Normal file
View File

@@ -0,0 +1,10 @@
package com.lingniu.ingest.tdenginehistory;
import java.io.IOException;
public interface TdengineHistoryReader {
TdenginePage<TdengineRawFrameRow> queryRawFrames(TdengineRawFrameQuery query) throws IOException;
TdenginePage<TdengineLocationRow> queryLocations(TdengineLocationQuery query) throws IOException;
}

138
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java Normal file
View File

@@ -0,0 +1,138 @@
package com.lingniu.ingest.tdenginehistory;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
private final DataSource dataSource;
private final TdengineHistoryQueries queries;
public TdengineJdbcHistoryReader(DataSource dataSource, TdengineHistorySchema schema) {
if (dataSource == null) {
throw new IllegalArgumentException("dataSource must not be null");
}
this.dataSource = dataSource;
this.queries = new TdengineHistoryQueries(schema);
}
@Override
public TdenginePage<TdengineRawFrameRow> queryRawFrames(TdengineRawFrameQuery query) throws IOException {
int pageSize = Math.min(query.limit(), 1000);
TdengineQueryStatement statement = queries.rawFrames(query.withLimit(pageSize + 1));
return readPage(statement, pageSize, this::rawFrame, row -> new TdenginePageCursor(row.ts(), row.frameId()));
}
@Override
public TdenginePage<TdengineLocationRow> queryLocations(TdengineLocationQuery query) throws IOException {
int pageSize = Math.min(query.limit(), 1000);
TdengineQueryStatement statement = queries.locations(query.withLimit(pageSize + 1));
return readPage(statement, pageSize, this::location, row -> new TdenginePageCursor(row.ts(), row.factId()));
}
private <T> TdenginePage<T> readPage(TdengineQueryStatement statement,
int pageSize,
SqlRowMapper<T> mapper,
Function<T, TdenginePageCursor> cursor) throws IOException {
List<T> rows = new ArrayList<>();
try (Connection connection = dataSource.getConnection();
PreparedStatement prepared = connection.prepareStatement(statement.sql())) {
bind(prepared, statement.values());
try (ResultSet resultSet = prepared.executeQuery()) {
while (resultSet.next()) {
rows.add(mapper.map(resultSet));
}
}
} catch (SQLException e) {
throw new IOException("tdengine history query failed", e);
}
boolean hasMore = rows.size() > pageSize;
List<T> items = hasMore ? List.copyOf(rows.subList(0, pageSize)) : List.copyOf(rows);
Optional<TdenginePageCursor> next = hasMore && !items.isEmpty()
? Optional.of(cursor.apply(items.getLast()))
: Optional.empty();
return new TdenginePage<>(items, next);
}
private static void bind(PreparedStatement prepared, List<Object> values) throws SQLException {
for (int i = 0; i < values.size(); i++) {
Object value = values.get(i);
int parameterIndex = i + 1;
if (value instanceof Instant instant) {
prepared.setTimestamp(parameterIndex, Timestamp.from(instant));
} else {
prepared.setObject(parameterIndex, value);
}
}
}
private TdengineRawFrameRow rawFrame(ResultSet rs) throws SQLException {
return new TdengineRawFrameRow(
instant(rs, "ts"),
rs.getString("frame_id"),
instant(rs, "received_at"),
rs.getInt("message_id"),
rs.getInt("sub_type"),
instant(rs, "event_time"),
rs.getString("raw_uri"),
rs.getString("checksum"),
rs.getLong("raw_size_bytes"),
rs.getString("parse_status"),
rs.getString("parse_error"),
rs.getString("peer"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
rs.getString("phone")
);
}
private TdengineLocationRow location(ResultSet rs) throws SQLException {
return new TdengineLocationRow(
instant(rs, "ts"),
rs.getString("fact_id"),
rs.getString("frame_id"),
instant(rs, "received_at"),
rs.getDouble("longitude"),
rs.getDouble("latitude"),
rs.getDouble("altitude_m"),
rs.getDouble("speed_kmh"),
rs.getDouble("direction_deg"),
rs.getLong("alarm_flag"),
rs.getLong("status_flag"),
nullableDouble(rs, "total_mileage_km"),
rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
rs.getString("phone")
);
}
private static Instant instant(ResultSet rs, String column) throws SQLException {
Timestamp timestamp = rs.getTimestamp(column);
return timestamp == null ? null : timestamp.toInstant();
}
private static Double nullableDouble(ResultSet rs, String column) throws SQLException {
Object value = rs.getObject(column);
return value instanceof Number number ? number.doubleValue() : null;
}
@FunctionalInterface
private interface SqlRowMapper<T> {
T map(ResultSet rs) throws SQLException;
}
}

31
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationQuery.java Normal file
View File

@@ -0,0 +1,31 @@
package com.lingniu.ingest.tdenginehistory;
import java.time.Instant;
public record TdengineLocationQuery(
String protocol,
String vehicleKey,
Instant from,
Instant to,
TdengineQueryOrder order,
int limit,
TdenginePageCursor cursor
) {
public TdengineLocationQuery {
if (protocol == null || protocol.isBlank()) {
throw new IllegalArgumentException("protocol must not be blank");
}
if (vehicleKey == null || vehicleKey.isBlank()) {
throw new IllegalArgumentException("vehicleKey must not be blank");
}
if (from == null || to == null || !from.isBefore(to)) {
throw new IllegalArgumentException("query time range must be valid");
}
order = order == null ? TdengineQueryOrder.DESC : order;
limit = Math.max(1, Math.min(limit, 1001));
}
TdengineLocationQuery withLimit(int newLimit) {
return new TdengineLocationQuery(protocol, vehicleKey, from, to, order, newLimit, cursor);
}
}

12
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdenginePage.java Normal file
View File

@@ -0,0 +1,12 @@
package com.lingniu.ingest.tdenginehistory;
import java.util.List;
import java.util.Optional;
public record TdenginePage<T>(List<T> items, Optional<TdenginePageCursor> nextCursor) {
public TdenginePage {
items = items == null ? List.of() : List.copyOf(items);
nextCursor = nextCursor == null ? Optional.empty() : nextCursor;
}
}

13
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdenginePageCursor.java Normal file
View File

@@ -0,0 +1,13 @@
package com.lingniu.ingest.tdenginehistory;
import java.time.Instant;
public record TdenginePageCursor(Instant ts, String tieBreaker) {
public TdenginePageCursor {
if (ts == null) {
throw new IllegalArgumentException("cursor ts must not be null");
}
tieBreaker = tieBreaker == null ? "" : tieBreaker;
}
}

6
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineQueryOrder.java Normal file
View File

@@ -0,0 +1,6 @@
package com.lingniu.ingest.tdenginehistory;
public enum TdengineQueryOrder {
ASC,
DESC
}

15
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineQueryStatement.java Normal file
View File

@@ -0,0 +1,15 @@
package com.lingniu.ingest.tdenginehistory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public record TdengineQueryStatement(String sql, List<Object> values) {
public TdengineQueryStatement {
if (sql == null || sql.isBlank()) {
throw new IllegalArgumentException("sql must not be blank");
}
values = values == null ? List.of() : Collections.unmodifiableList(new ArrayList<>(values));
}
}

31
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameQuery.java Normal file
View File

@@ -0,0 +1,31 @@
package com.lingniu.ingest.tdenginehistory;
import java.time.Instant;
public record TdengineRawFrameQuery(
String protocol,
String vehicleKey,
Instant from,
Instant to,
TdengineQueryOrder order,
int limit,
TdenginePageCursor cursor
) {
public TdengineRawFrameQuery {
if (protocol == null || protocol.isBlank()) {
throw new IllegalArgumentException("protocol must not be blank");
}
if (vehicleKey == null || vehicleKey.isBlank()) {
throw new IllegalArgumentException("vehicleKey must not be blank");
}
if (from == null || to == null || !from.isBefore(to)) {
throw new IllegalArgumentException("query time range must be valid");
}
order = order == null ? TdengineQueryOrder.DESC : order;
limit = Math.max(1, Math.min(limit, 1001));
}
TdengineRawFrameQuery withLimit(int newLimit) {
return new TdengineRawFrameQuery(protocol, vehicleKey, from, to, order, newLimit, cursor);
}
}

13
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/config/TdengineHistoryAutoConfiguration.java
View File

@@ -2,7 +2,9 @@ package com.lingniu.ingest.tdenginehistory.config;
import com.lingniu.ingest.tdenginehistory.TdengineHistorySchema;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryStatements;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;
import com.lingniu.ingest.tdenginehistory.TdengineJdbcHistoryReader;
import com.lingniu.ingest.tdenginehistory.TdengineJdbcHistoryWriter;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
@@ -47,4 +49,15 @@ public class TdengineHistoryAutoConfiguration {
public TdengineHistoryWriter tdengineHistoryWriter(DataSource dataSource, TdengineHistorySchema schema) {
return new TdengineJdbcHistoryWriter(dataSource, schema);
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(DataSource.class)
@ConditionalOnProperty(
prefix = "lingniu.ingest.tdengine-history",
name = "enabled",
havingValue = "true")
public TdengineHistoryReader tdengineHistoryReader(DataSource dataSource, TdengineHistorySchema schema) {
return new TdengineJdbcHistoryReader(dataSource, schema);
}
}