build: remove raw archive store prototype
This commit is contained in:
12
DECISIONS.md
12
DECISIONS.md
@@ -91,14 +91,14 @@
|
|||||||
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
|
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
|
||||||
- **Consequences**: 默认构建和部署边界继续收窄;最新状态能力保持源码可用,但其测试需要通过 `-Poptional-latest-state` 显式运行。
|
- **Consequences**: 默认构建和部署边界继续收窄;最新状态能力保持源码可用,但其测试需要通过 `-Poptional-latest-state` 显式运行。
|
||||||
|
|
||||||
## ADR-014 raw-archive-store 原型实现:Optional only
|
## ADR-014 raw-archive-store 原型已删除
|
||||||
- **Status**: Accepted
|
- **Status**: Accepted
|
||||||
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责,历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯;`raw-archive-store` 当前只是独立读写接口和本地实现原型,没有默认 app 或 service 依赖。
|
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责,历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯;`raw-archive-store` 只是未部署的独立读写原型,继续保留会造成 raw archive 路径歧义。
|
||||||
- **Decision**:
|
- **Decision**:
|
||||||
1. `raw-archive-store` 不进入默认 Maven reactor。
|
1. 删除 `raw-archive-store` 模块和对应 optional profile。
|
||||||
2. raw-archive-store 仅保留在 `optional-raw-archive-store` profile,需要验证本地读写原型时显式构建。
|
2. 默认生产链路只保留 `sink-archive`、Kafka raw topic、TDengine raw_frames 这条 raw 路径。
|
||||||
3. 默认生产链路继续以 `sink-archive`、Kafka raw topic、TDengine raw_frames 为准。
|
3. 如需新的 archive store 能力,先以生产链路需求重新设计,不恢复旧原型。
|
||||||
- **Consequences**: 默认构建少一个未部署模块;本地 raw archive 原型仍可通过 `-Poptional-raw-archive-store` 保留和验证。
|
- **Consequences**: 默认构建面和可选构建面继续收窄;raw bytes 写入职责集中在 `sink-archive`。
|
||||||
|
|
||||||
## ADR-015 文件型事件索引:Removed
|
## ADR-015 文件型事件索引:Removed
|
||||||
- **Status**: Accepted
|
- **Status**: Accepted
|
||||||
|
|||||||
@@ -187,8 +187,8 @@ flowchart LR
|
|||||||
`optional-attachments`.
|
`optional-attachments`.
|
||||||
- vehicle-state-service is absent from the default Maven reactor and available
|
- vehicle-state-service is absent from the default Maven reactor and available
|
||||||
through `optional-latest-state`.
|
through `optional-latest-state`.
|
||||||
- raw-archive-store is absent from the default Maven reactor and available
|
- raw-archive-store prototype has been removed; raw bytes are written through
|
||||||
through `optional-raw-archive-store`.
|
`sink-archive`.
|
||||||
- History APIs read from TDengine; file-based event indexes are not maintained.
|
- History APIs read from TDengine; file-based event indexes are not maintained.
|
||||||
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
|
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
|
||||||
- Location queries page over compact rows and reference raw records instead of
|
- Location queries page over compact rows and reference raw records instead of
|
||||||
|
|||||||
@@ -53,14 +53,17 @@ class MavenModuleProfileTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void rawArchiveStorePrototypeIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
|
void rawArchiveStorePrototypeIsRemovedFromBuildSurface() throws Exception {
|
||||||
Document pom = rootPom();
|
Document pom = rootPom();
|
||||||
|
|
||||||
assertThat(defaultModules(pom))
|
assertThat(defaultModules(pom))
|
||||||
.doesNotContain("modules/sinks/raw-archive-store");
|
.doesNotContain("modules/sinks/raw-archive-store");
|
||||||
|
|
||||||
assertThat(profileModules(pom, "optional-raw-archive-store"))
|
assertThat(profileModules(pom, "optional-raw-archive-store"))
|
||||||
.containsExactly("modules/sinks/raw-archive-store");
|
.as("raw bytes are written by sink-archive; raw-archive-store was an unused prototype")
|
||||||
|
.isEmpty();
|
||||||
|
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/raw-archive-store")))
|
||||||
|
.isFalse();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -153,7 +156,7 @@ class MavenModuleProfileTest {
|
|||||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-latest-state"))
|
assertThat(profileDependencyManagementArtifacts(pom, "optional-latest-state"))
|
||||||
.containsExactly("vehicle-state-service");
|
.containsExactly("vehicle-state-service");
|
||||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-raw-archive-store"))
|
assertThat(profileDependencyManagementArtifacts(pom, "optional-raw-archive-store"))
|
||||||
.containsExactly("raw-archive-store");
|
.isEmpty();
|
||||||
assertThat(profileDependencyManagementArtifacts(pom, "optional-event-file-store"))
|
assertThat(profileDependencyManagementArtifacts(pom, "optional-event-file-store"))
|
||||||
.isEmpty();
|
.isEmpty();
|
||||||
assertThat(profileDependencyManagementArtifacts(pom, "legacy-xinda"))
|
assertThat(profileDependencyManagementArtifacts(pom, "legacy-xinda"))
|
||||||
@@ -328,11 +331,12 @@ class MavenModuleProfileTest {
|
|||||||
.contains("Xinda Push 仅保留在 `legacy-xinda` profile")
|
.contains("Xinda Push 仅保留在 `legacy-xinda` profile")
|
||||||
.contains("JSATL12 仅保留在 `optional-attachments` profile")
|
.contains("JSATL12 仅保留在 `optional-attachments` profile")
|
||||||
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
|
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
|
||||||
.contains("raw-archive-store 仅保留在 `optional-raw-archive-store` profile")
|
.contains("raw-archive-store 原型已删除")
|
||||||
.contains("协议接入应用不直接写业务库")
|
.contains("协议接入应用不直接写业务库")
|
||||||
.contains("`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`")
|
.contains("`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`")
|
||||||
.doesNotContain("event-file-store")
|
.doesNotContain("event-file-store")
|
||||||
.doesNotContain("optional-event-file-store")
|
.doesNotContain("optional-event-file-store")
|
||||||
|
.doesNotContain("optional-raw-archive-store")
|
||||||
.doesNotContain("## ADR-004 信达 Push:保留模块但彻底重写\n- **Status**: Accepted")
|
.doesNotContain("## ADR-004 信达 Push:保留模块但彻底重写\n- **Status**: Accepted")
|
||||||
.doesNotContain("Spring Boot 3.4.x");
|
.doesNotContain("Spring Boot 3.4.x");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
<parent>
|
|
||||||
<groupId>com.lingniu.ingest</groupId>
|
|
||||||
<artifactId>lingniu-vehicle-ingest</artifactId>
|
|
||||||
<version>0.1.0-SNAPSHOT</version>
|
|
||||||
<relativePath>../../../pom.xml</relativePath>
|
|
||||||
</parent>
|
|
||||||
<artifactId>raw-archive-store</artifactId>
|
|
||||||
<name>raw-archive-store</name>
|
|
||||||
<description>Raw frame archive writer and reader contracts.</description>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.lingniu.ingest</groupId>
|
|
||||||
<artifactId>ingest-api</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.junit.jupiter</groupId>
|
|
||||||
<artifactId>junit-jupiter</artifactId>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.assertj</groupId>
|
|
||||||
<artifactId>assertj-core</artifactId>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
</project>
|
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.LinkOption;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.StandardOpenOption;
|
|
||||||
import java.security.MessageDigest;
|
|
||||||
import java.security.NoSuchAlgorithmException;
|
|
||||||
import java.time.ZoneId;
|
|
||||||
import java.util.HexFormat;
|
|
||||||
|
|
||||||
public final class LocalRawArchiveStore implements RawArchiveWriter, RawArchiveReader {
|
|
||||||
|
|
||||||
private static final ZoneId PARTITION_ZONE = ZoneId.of("Asia/Shanghai");
|
|
||||||
|
|
||||||
private final Path root;
|
|
||||||
private final String uriPrefix;
|
|
||||||
|
|
||||||
public LocalRawArchiveStore(Path root, String uriPrefix) {
|
|
||||||
if (root == null) {
|
|
||||||
throw new IllegalArgumentException("root must not be null");
|
|
||||||
}
|
|
||||||
this.root = root.toAbsolutePath().normalize();
|
|
||||||
String prefix = uriPrefix == null || uriPrefix.isBlank() ? "archive://raw" : uriPrefix.trim();
|
|
||||||
this.uriPrefix = prefix.endsWith("/") ? prefix.substring(0, prefix.length() - 1) : prefix;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException {
|
|
||||||
String key = key(request);
|
|
||||||
Path target = resolveKey(key);
|
|
||||||
Files.createDirectories(target.getParent());
|
|
||||||
rejectSymlinkPath(target);
|
|
||||||
byte[] bytes = request.rawBytes();
|
|
||||||
Files.write(target, bytes, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
|
|
||||||
return new RawArchiveReceipt(uriPrefix + "/" + key, checksum(bytes), bytes.length);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public byte[] read(String rawUri) throws IOException {
|
|
||||||
String key = keyFromUri(rawUri);
|
|
||||||
Path target = resolveKey(key);
|
|
||||||
rejectSymlinkPath(target);
|
|
||||||
return Files.readAllBytes(target);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String key(RawArchiveWriteRequest request) {
|
|
||||||
var date = request.receivedAt().atZone(PARTITION_ZONE).toLocalDate();
|
|
||||||
return "%04d/%02d/%02d/%s/%s/%s.bin".formatted(
|
|
||||||
date.getYear(),
|
|
||||||
date.getMonthValue(),
|
|
||||||
date.getDayOfMonth(),
|
|
||||||
request.protocol().name(),
|
|
||||||
safeSegment(request.vehicleKey()),
|
|
||||||
safeSegment(request.frameId()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String keyFromUri(String rawUri) {
|
|
||||||
if (rawUri == null || rawUri.isBlank()) {
|
|
||||||
throw new IllegalArgumentException("rawUri must not be blank");
|
|
||||||
}
|
|
||||||
String normalized = rawUri.trim();
|
|
||||||
String expected = uriPrefix + "/";
|
|
||||||
if (!normalized.startsWith(expected)) {
|
|
||||||
throw new IllegalArgumentException("rawUri does not use expected prefix: " + rawUri);
|
|
||||||
}
|
|
||||||
return normalized.substring(expected.length());
|
|
||||||
}
|
|
||||||
|
|
||||||
private Path resolveKey(String key) {
|
|
||||||
Path target = root.resolve(key).normalize();
|
|
||||||
if (!target.startsWith(root)) {
|
|
||||||
throw new IllegalArgumentException("raw archive key escapes root: " + key);
|
|
||||||
}
|
|
||||||
return target;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void rejectSymlinkPath(Path target) throws IOException {
|
|
||||||
Path current = root;
|
|
||||||
Path relative = root.relativize(target);
|
|
||||||
for (Path segment : relative) {
|
|
||||||
current = current.resolve(segment);
|
|
||||||
if (Files.isSymbolicLink(current)) {
|
|
||||||
throw new IOException("raw archive path traverses symlink: " + current);
|
|
||||||
}
|
|
||||||
if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String safeSegment(String value) {
|
|
||||||
String cleaned = value == null ? "" : value.trim();
|
|
||||||
if (cleaned.isBlank()) {
|
|
||||||
return "unknown";
|
|
||||||
}
|
|
||||||
return cleaned.replaceAll("[^A-Za-z0-9._-]", "_");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String checksum(byte[] bytes) {
|
|
||||||
try {
|
|
||||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
||||||
return "sha256:" + HexFormat.of().formatHex(digest.digest(bytes));
|
|
||||||
} catch (NoSuchAlgorithmException e) {
|
|
||||||
throw new IllegalStateException("SHA-256 is unavailable", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
public interface RawArchiveReader {
|
|
||||||
byte[] read(String rawUri) throws IOException;
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
public record RawArchiveReceipt(String rawUri, String checksum, long sizeBytes) {
|
|
||||||
public RawArchiveReceipt {
|
|
||||||
rawUri = required(rawUri, "rawUri");
|
|
||||||
checksum = required(checksum, "checksum");
|
|
||||||
if (sizeBytes < 0) {
|
|
||||||
throw new IllegalArgumentException("sizeBytes must not be negative");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String required(String value, String field) {
|
|
||||||
String cleaned = value == null ? "" : value.trim();
|
|
||||||
if (cleaned.isBlank()) {
|
|
||||||
throw new IllegalArgumentException(field + " must not be blank");
|
|
||||||
}
|
|
||||||
return cleaned;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
|
||||||
|
|
||||||
import java.time.Instant;
|
|
||||||
|
|
||||||
public record RawArchiveWriteRequest(
|
|
||||||
ProtocolId protocol,
|
|
||||||
String vehicleKey,
|
|
||||||
String frameId,
|
|
||||||
Instant receivedAt,
|
|
||||||
byte[] rawBytes
|
|
||||||
) {
|
|
||||||
public RawArchiveWriteRequest {
|
|
||||||
if (protocol == null) {
|
|
||||||
throw new IllegalArgumentException("protocol must not be null");
|
|
||||||
}
|
|
||||||
vehicleKey = required(vehicleKey, "vehicleKey");
|
|
||||||
frameId = required(frameId, "frameId");
|
|
||||||
if (receivedAt == null) {
|
|
||||||
throw new IllegalArgumentException("receivedAt must not be null");
|
|
||||||
}
|
|
||||||
if (rawBytes == null || rawBytes.length == 0) {
|
|
||||||
throw new IllegalArgumentException("rawBytes must not be empty");
|
|
||||||
}
|
|
||||||
rawBytes = rawBytes.clone();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public byte[] rawBytes() {
|
|
||||||
return rawBytes.clone();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String required(String value, String field) {
|
|
||||||
String cleaned = value == null ? "" : value.trim();
|
|
||||||
if (cleaned.isBlank()) {
|
|
||||||
throw new IllegalArgumentException(field + " must not be blank");
|
|
||||||
}
|
|
||||||
return cleaned;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
public interface RawArchiveWriter {
|
|
||||||
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
|
|
||||||
}
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
package com.lingniu.ingest.rawarchive;
|
|
||||||
|
|
||||||
import com.lingniu.ingest.api.ProtocolId;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
|
||||||
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.time.Instant;
|
|
||||||
import java.util.HexFormat;
|
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat;
|
|
||||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
|
||||||
|
|
||||||
class LocalRawArchiveStoreTest {
|
|
||||||
|
|
||||||
@TempDir
|
|
||||||
Path tempDir;
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void writesReadableRawObjectWithArchiveUriAndChecksum() throws Exception {
|
|
||||||
LocalRawArchiveStore store = new LocalRawArchiveStore(tempDir, "archive://raw");
|
|
||||||
byte[] bytes = HexFormat.of().parseHex("7e020000007e");
|
|
||||||
|
|
||||||
RawArchiveReceipt receipt = store.write(new RawArchiveWriteRequest(
|
|
||||||
ProtocolId.JT808,
|
|
||||||
"jt808:013912345678",
|
|
||||||
"frame-1",
|
|
||||||
Instant.parse("2026-06-29T00:00:00Z"),
|
|
||||||
bytes));
|
|
||||||
|
|
||||||
assertThat(receipt.rawUri()).isEqualTo("archive://raw/2026/06/29/JT808/jt808_013912345678/frame-1.bin");
|
|
||||||
assertThat(receipt.checksum()).startsWith("sha256:");
|
|
||||||
assertThat(receipt.sizeBytes()).isEqualTo(bytes.length);
|
|
||||||
assertThat(store.read(receipt.rawUri())).containsExactly(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void rejectsBlankFrameId() {
|
|
||||||
LocalRawArchiveStore store = new LocalRawArchiveStore(tempDir, "archive://raw");
|
|
||||||
|
|
||||||
assertThatThrownBy(() -> store.write(new RawArchiveWriteRequest(
|
|
||||||
ProtocolId.GB32960,
|
|
||||||
"VIN001",
|
|
||||||
"",
|
|
||||||
Instant.parse("2026-06-29T00:00:00Z"),
|
|
||||||
new byte[]{1})))
|
|
||||||
.isInstanceOf(IllegalArgumentException.class)
|
|
||||||
.hasMessageContaining("frameId");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
15
pom.xml
15
pom.xml
@@ -91,21 +91,6 @@
|
|||||||
</dependencies>
|
</dependencies>
|
||||||
</dependencyManagement>
|
</dependencyManagement>
|
||||||
</profile>
|
</profile>
|
||||||
<profile>
|
|
||||||
<id>optional-raw-archive-store</id>
|
|
||||||
<modules>
|
|
||||||
<module>modules/sinks/raw-archive-store</module>
|
|
||||||
</modules>
|
|
||||||
<dependencyManagement>
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.lingniu.ingest</groupId>
|
|
||||||
<artifactId>raw-archive-store</artifactId>
|
|
||||||
<version>${project.version}</version>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
|
||||||
</dependencyManagement>
|
|
||||||
</profile>
|
|
||||||
<profile>
|
<profile>
|
||||||
<id>legacy-xinda</id>
|
<id>legacy-xinda</id>
|
||||||
<modules>
|
<modules>
|
||||||
|
|||||||
Reference in New Issue
Block a user