shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build: remove raw archive store prototype

Browse Source
This commit is contained in:
lingniu
2026-07-01 13:33:24 +08:00
parent cea8ceeabe
commit 27f9cbf0d2
11 changed files with 16 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
DECISIONS.md
View File

@@ -91,14 +91,14 @@
3. 默认优化和验证优先覆盖活跃接入、TDengine 历史和 MySQL 指标链路。
- **Consequences**: 默认构建和部署边界继续收窄；最新状态能力保持源码可用，但其测试需要通过 `-Poptional-latest-state` 显式运行。
## ADR-014 raw-archive-store 原型实现：Optional only
## ADR-014 raw-archive-store 原型已删除
- **Status**: Accepted
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责，历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯；`raw-archive-store` 当前只是独立读写接口和本地实现原型，没有默认 app 或 service 依赖。
- **Context**: 默认生产 raw bytes 写入由 `sink-archive` 负责，历史查询通过 TDengine `raw_frames` 和 `archive://...` 引用追溯；`raw-archive-store` 只是未部署的独立读写原型，继续保留会造成 raw archive 路径歧义。
- **Decision**:
1. `raw-archive-store` 不进入默认 Maven reactor。
2. raw-archive-store 仅保留在 `optional-raw-archive-store` profile，需要验证本地读写原型时显式构建。
3. 默认生产链路继续以 `sink-archive`、Kafka raw topic、TDengine raw_frames 为准。
- **Consequences**: 默认构建少一个未部署模块；本地 raw archive 原型仍可通过 `-Poptional-raw-archive-store` 保留和验证。
1. 删除 `raw-archive-store` 模块和对应 optional profile。
2. 默认生产链路只保留 `sink-archive`、Kafka raw topic、TDengine raw_frames 这条 raw 路径。
3. 如需新的 archive store 能力，先以生产链路需求重新设计，不恢复旧原型。
- **Consequences**: 默认构建面和可选构建面继续收窄；raw bytes 写入职责集中在 `sink-archive`。
## ADR-015 文件型事件索引：Removed
- **Status**: Accepted

4
docs/target-architecture.md
View File

@@ -187,8 +187,8 @@ flowchart LR
`optional-attachments`.
- vehicle-state-service is absent from the default Maven reactor and available
through `optional-latest-state`.
- raw-archive-store is absent from the default Maven reactor and available
through `optional-raw-archive-store`.
- raw-archive-store prototype has been removed; raw bytes are written through
`sink-archive`.
- History APIs read from TDengine; file-based event indexes are not maintained.
- Raw frame queries can return complete parsed JSON through `payloadJson.parsed`.
- Location queries page over compact rows and reference raw records instead of

12
modules/apps/vehicle-history-app/src/test/java/com/lingniu/ingest/historyapp/MavenModuleProfileTest.java
View File

@@ -53,14 +53,17 @@ class MavenModuleProfileTest {
}
@Test
void rawArchiveStorePrototypeIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
void rawArchiveStorePrototypeIsRemovedFromBuildSurface() throws Exception {
Document pom = rootPom();
assertThat(defaultModules(pom))
.doesNotContain("modules/sinks/raw-archive-store");
assertThat(profileModules(pom, "optional-raw-archive-store"))
.containsExactly("modules/sinks/raw-archive-store");
.as("raw bytes are written by sink-archive; raw-archive-store was an unused prototype")
.isEmpty();
assertThat(Files.exists(repositoryRoot().resolve("modules/sinks/raw-archive-store")))
.isFalse();
}
@Test
@@ -153,7 +156,7 @@ class MavenModuleProfileTest {
assertThat(profileDependencyManagementArtifacts(pom, "optional-latest-state"))
.containsExactly("vehicle-state-service");
assertThat(profileDependencyManagementArtifacts(pom, "optional-raw-archive-store"))
.containsExactly("raw-archive-store");
.isEmpty();
assertThat(profileDependencyManagementArtifacts(pom, "optional-event-file-store"))
.isEmpty();
assertThat(profileDependencyManagementArtifacts(pom, "legacy-xinda"))
@@ -328,11 +331,12 @@ class MavenModuleProfileTest {
.contains("Xinda Push 仅保留在 `legacy-xinda` profile")
.contains("JSATL12 仅保留在 `optional-attachments` profile")
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
.contains("raw-archive-store 仅保留在 `optional-raw-archive-store` profile")
.contains("raw-archive-store 原型已删除")
.contains("协议接入应用不直接写业务库")
.contains("`vehicle-analytics-app` 写入 MySQL `vehicle_stat_metric`")
.doesNotContain("event-file-store")
.doesNotContain("optional-event-file-store")
.doesNotContain("optional-raw-archive-store")
.doesNotContain("## ADR-004 信达 Push：保留模块但彻底重写\n- **Status**: Accepted")
.doesNotContain("Spring Boot 3.4.x");
}

30
modules/sinks/raw-archive-store/pom.xml
View File

@@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lingniu.ingest</groupId>
<artifactId>lingniu-vehicle-ingest</artifactId>
<version>0.1.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>raw-archive-store</artifactId>
<name>raw-archive-store</name>
<description>Raw frame archive writer and reader contracts.</description>
<dependencies>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>ingest-api</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

110
modules/sinks/raw-archive-store/src/main/java/com/lingniu/ingest/rawarchive/LocalRawArchiveStore.java
View File

@@ -1,110 +0,0 @@
package com.lingniu.ingest.rawarchive;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.ZoneId;
import java.util.HexFormat;
public final class LocalRawArchiveStore implements RawArchiveWriter, RawArchiveReader {
private static final ZoneId PARTITION_ZONE = ZoneId.of("Asia/Shanghai");
private final Path root;
private final String uriPrefix;
public LocalRawArchiveStore(Path root, String uriPrefix) {
if (root == null) {
throw new IllegalArgumentException("root must not be null");
}
this.root = root.toAbsolutePath().normalize();
String prefix = uriPrefix == null || uriPrefix.isBlank() ? "archive://raw" : uriPrefix.trim();
this.uriPrefix = prefix.endsWith("/") ? prefix.substring(0, prefix.length() - 1) : prefix;
}
@Override
public RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException {
String key = key(request);
Path target = resolveKey(key);
Files.createDirectories(target.getParent());
rejectSymlinkPath(target);
byte[] bytes = request.rawBytes();
Files.write(target, bytes, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
return new RawArchiveReceipt(uriPrefix + "/" + key, checksum(bytes), bytes.length);
}
@Override
public byte[] read(String rawUri) throws IOException {
String key = keyFromUri(rawUri);
Path target = resolveKey(key);
rejectSymlinkPath(target);
return Files.readAllBytes(target);
}
private String key(RawArchiveWriteRequest request) {
var date = request.receivedAt().atZone(PARTITION_ZONE).toLocalDate();
return "%04d/%02d/%02d/%s/%s/%s.bin".formatted(
date.getYear(),
date.getMonthValue(),
date.getDayOfMonth(),
request.protocol().name(),
safeSegment(request.vehicleKey()),
safeSegment(request.frameId()));
}
private String keyFromUri(String rawUri) {
if (rawUri == null || rawUri.isBlank()) {
throw new IllegalArgumentException("rawUri must not be blank");
}
String normalized = rawUri.trim();
String expected = uriPrefix + "/";
if (!normalized.startsWith(expected)) {
throw new IllegalArgumentException("rawUri does not use expected prefix: " + rawUri);
}
return normalized.substring(expected.length());
}
private Path resolveKey(String key) {
Path target = root.resolve(key).normalize();
if (!target.startsWith(root)) {
throw new IllegalArgumentException("raw archive key escapes root: " + key);
}
return target;
}
private void rejectSymlinkPath(Path target) throws IOException {
Path current = root;
Path relative = root.relativize(target);
for (Path segment : relative) {
current = current.resolve(segment);
if (Files.isSymbolicLink(current)) {
throw new IOException("raw archive path traverses symlink: " + current);
}
if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
return;
}
}
}
private static String safeSegment(String value) {
String cleaned = value == null ? "" : value.trim();
if (cleaned.isBlank()) {
return "unknown";
}
return cleaned.replaceAll("[^A-Za-z0-9._-]", "_");
}
private static String checksum(byte[] bytes) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
return "sha256:" + HexFormat.of().formatHex(digest.digest(bytes));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is unavailable", e);
}
}
}

7
modules/sinks/raw-archive-store/src/main/java/com/lingniu/ingest/rawarchive/RawArchiveReader.java
View File

@@ -1,7 +0,0 @@
package com.lingniu.ingest.rawarchive;
import java.io.IOException;
public interface RawArchiveReader {
byte[] read(String rawUri) throws IOException;
}

19
modules/sinks/raw-archive-store/src/main/java/com/lingniu/ingest/rawarchive/RawArchiveReceipt.java
View File

@@ -1,19 +0,0 @@
package com.lingniu.ingest.rawarchive;
public record RawArchiveReceipt(String rawUri, String checksum, long sizeBytes) {
public RawArchiveReceipt {
rawUri = required(rawUri, "rawUri");
checksum = required(checksum, "checksum");
if (sizeBytes < 0) {
throw new IllegalArgumentException("sizeBytes must not be negative");
}
}
private static String required(String value, String field) {
String cleaned = value == null ? "" : value.trim();
if (cleaned.isBlank()) {
throw new IllegalArgumentException(field + " must not be blank");
}
return cleaned;
}
}

41
modules/sinks/raw-archive-store/src/main/java/com/lingniu/ingest/rawarchive/RawArchiveWriteRequest.java
View File

@@ -1,41 +0,0 @@
package com.lingniu.ingest.rawarchive;
import com.lingniu.ingest.api.ProtocolId;
import java.time.Instant;
public record RawArchiveWriteRequest(
ProtocolId protocol,
String vehicleKey,
String frameId,
Instant receivedAt,
byte[] rawBytes
) {
public RawArchiveWriteRequest {
if (protocol == null) {
throw new IllegalArgumentException("protocol must not be null");
}
vehicleKey = required(vehicleKey, "vehicleKey");
frameId = required(frameId, "frameId");
if (receivedAt == null) {
throw new IllegalArgumentException("receivedAt must not be null");
}
if (rawBytes == null || rawBytes.length == 0) {
throw new IllegalArgumentException("rawBytes must not be empty");
}
rawBytes = rawBytes.clone();
}
@Override
public byte[] rawBytes() {
return rawBytes.clone();
}
private static String required(String value, String field) {
String cleaned = value == null ? "" : value.trim();
if (cleaned.isBlank()) {
throw new IllegalArgumentException(field + " must not be blank");
}
return cleaned;
}
}

7
modules/sinks/raw-archive-store/src/main/java/com/lingniu/ingest/rawarchive/RawArchiveWriter.java
View File

@@ -1,7 +0,0 @@
package com.lingniu.ingest.rawarchive;
import java.io.IOException;
public interface RawArchiveWriter {
RawArchiveReceipt write(RawArchiveWriteRequest request) throws IOException;
}

50
modules/sinks/raw-archive-store/src/test/java/com/lingniu/ingest/rawarchive/LocalRawArchiveStoreTest.java
View File

@@ -1,50 +0,0 @@
package com.lingniu.ingest.rawarchive;
import com.lingniu.ingest.api.ProtocolId;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
import java.time.Instant;
import java.util.HexFormat;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class LocalRawArchiveStoreTest {
@TempDir
Path tempDir;
@Test
void writesReadableRawObjectWithArchiveUriAndChecksum() throws Exception {
LocalRawArchiveStore store = new LocalRawArchiveStore(tempDir, "archive://raw");
byte[] bytes = HexFormat.of().parseHex("7e020000007e");
RawArchiveReceipt receipt = store.write(new RawArchiveWriteRequest(
ProtocolId.JT808,
"jt808:013912345678",
"frame-1",
Instant.parse("2026-06-29T00:00:00Z"),
bytes));
assertThat(receipt.rawUri()).isEqualTo("archive://raw/2026/06/29/JT808/jt808_013912345678/frame-1.bin");
assertThat(receipt.checksum()).startsWith("sha256:");
assertThat(receipt.sizeBytes()).isEqualTo(bytes.length);
assertThat(store.read(receipt.rawUri())).containsExactly(bytes);
}
@Test
void rejectsBlankFrameId() {
LocalRawArchiveStore store = new LocalRawArchiveStore(tempDir, "archive://raw");
assertThatThrownBy(() -> store.write(new RawArchiveWriteRequest(
ProtocolId.GB32960,
"VIN001",
"",
Instant.parse("2026-06-29T00:00:00Z"),
new byte[]{1})))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("frameId");
}
}

15
pom.xml
View File

@@ -91,21 +91,6 @@
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>optional-raw-archive-store</id>
<modules>
<module>modules/sinks/raw-archive-store</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>raw-archive-store</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</profile>
<profile>
<id>legacy-xinda</id>
<modules>