feat: persist raw frames and publish raw facts

2026-06-29 13:10:14 +08:00
parent 9c2779f82b
commit feb9552234
14 changed files with 300 additions and 71 deletions
--- a/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/RawArchiveEventSink.java
+++ b/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/RawArchiveEventSink.java
@@ -4,15 +4,31 @@ import com.lingniu.ingest.api.event.RawArchiveKeys;
 import com.lingniu.ingest.api.event.VehicleEvent;
 import com.lingniu.ingest.api.sink.EventSink;

-import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HexFormat;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;

-public final class RawArchiveEventSink implements EventSink {
+public final class RawArchiveEventSink implements EventSink, AutoCloseable {

-    private final ArchiveStore store;
+    private final Path root;
+    private final ExecutorService executor;

-    public RawArchiveEventSink(ArchiveStore store) {
-        this.store = store;
+    public RawArchiveEventSink(Path root) {
+        if (root == null) {
+            throw new IllegalArgumentException("root must not be null");
+        }
+        this.root = root.toAbsolutePath().normalize();
+        this.executor = Executors.newVirtualThreadPerTaskExecutor();
    }

    @Override
@@ -20,25 +36,115 @@ public final class RawArchiveEventSink implements EventSink {
        return "raw-archive";
    }

+    @Override
+    public boolean accepts(VehicleEvent event) {
+        return event instanceof VehicleEvent.RawArchive;
+    }
+
    @Override
    public CompletableFuture<Void> publish(VehicleEvent event) {
        if (!(event instanceof VehicleEvent.RawArchive raw)) {
            return CompletableFuture.completedFuture(null);
        }
-        try {
-            byte[] bytes = raw.rawBytes();
-            if (bytes == null) {
-                throw new IllegalArgumentException("raw archive bytes must not be null");
-            }
-            store.put(RawArchiveKeys.key(raw), new ByteArrayInputStream(bytes), bytes.length);
-            return CompletableFuture.completedFuture(null);
-        } catch (Exception e) {
-            return CompletableFuture.failedFuture(e);
-        }
+        return CompletableFuture.runAsync(() -> write(raw), executor);
    }

    @Override
-    public boolean accepts(VehicleEvent event) {
-        return event instanceof VehicleEvent.RawArchive;
+    public void close() {
+        executor.shutdown();
+    }
+
+    public Path root() {
+        return root;
+    }
+
+    private void write(VehicleEvent.RawArchive raw) {
+        byte[] bytes = raw.rawBytes() == null ? new byte[0] : raw.rawBytes();
+        if (bytes.length == 0) {
+            return;
+        }
+        String key = RawArchiveKeys.key(raw);
+        Path target = resolveKey(key);
+        try {
+            createDirectoriesInsideRoot(target.getParent());
+            rejectSymlinkPath(target);
+            try {
+                Files.write(target, bytes, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+            } catch (FileAlreadyExistsException duplicate) {
+                byte[] existing = Files.readAllBytes(target);
+                if (!Arrays.equals(sha256(existing), sha256(bytes))) {
+                    throw new IOException("raw archive duplicate key has different content: " + key, duplicate);
+                }
+            }
+        } catch (IOException e) {
+            throw new RawArchiveWriteException("failed to write raw archive " + RawArchiveKeys.logicalUri(key), e);
+        }
+    }
+
+    private Path resolveKey(String key) {
+        Path target = root.resolve(key == null ? "" : key).normalize();
+        if (!target.startsWith(root)) {
+            throw new IllegalArgumentException("raw archive key escapes root: " + key);
+        }
+        return target;
+    }
+
+    private void createDirectoriesInsideRoot(Path directory) throws IOException {
+        rejectEscapedPath(directory);
+        if (Files.notExists(root, LinkOption.NOFOLLOW_LINKS)) {
+            Files.createDirectories(root);
+        }
+        Path current = root;
+        Path relative = root.relativize(directory);
+        for (Path segment : relative) {
+            current = current.resolve(segment);
+            if (Files.isSymbolicLink(current)) {
+                throw new IOException("raw archive path traverses symlink: " + current);
+            }
+            if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
+                Files.createDirectory(current);
+            } else if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) {
+                throw new IOException("raw archive path segment is not a directory: " + current);
+            }
+        }
+    }
+
+    private void rejectSymlinkPath(Path target) throws IOException {
+        rejectEscapedPath(target);
+        Path current = root;
+        Path relative = root.relativize(target);
+        for (Path segment : relative) {
+            current = current.resolve(segment);
+            if (Files.isSymbolicLink(current)) {
+                throw new IOException("raw archive path traverses symlink: " + current);
+            }
+            if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
+                return;
+            }
+        }
+    }
+
+    private void rejectEscapedPath(Path path) {
+        if (!path.normalize().startsWith(root)) {
+            throw new IllegalArgumentException("raw archive path escapes root: " + path);
+        }
+    }
+
+    private static byte[] sha256(byte[] bytes) {
+        try {
+            return MessageDigest.getInstance("SHA-256").digest(bytes);
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalStateException("SHA-256 is unavailable", e);
+        }
+    }
+
+    static String checksum(byte[] bytes) {
+        return "sha256:" + HexFormat.of().formatHex(sha256(bytes == null ? new byte[0] : bytes));
+    }
+
+    public static final class RawArchiveWriteException extends RuntimeException {
+        public RawArchiveWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
    }
 }
--- a/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveAutoConfiguration.java
+++ b/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveAutoConfiguration.java
@@ -5,31 +5,28 @@ import com.lingniu.ingest.sink.archive.LocalArchiveStore;
 import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;

+import java.nio.file.Path;
+
@AutoConfiguration
@EnableConfigurationProperties(SinkArchiveProperties.class)
-@ConditionalOnProperty(
-        prefix = "lingniu.ingest.sink.archive",
-        name = "enabled",
-        havingValue = "true",
-        matchIfMissing = true)
+@ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "enabled", havingValue = "true", matchIfMissing = true)
 public class SinkArchiveAutoConfiguration {

    @Bean
    @ConditionalOnMissingBean
    @ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "type", havingValue = "local", matchIfMissing = true)
    public ArchiveStore archiveStore(SinkArchiveProperties properties) {
-        return new LocalArchiveStore(properties.getPath());
+        return new LocalArchiveStore(Path.of(properties.getPath()));
    }

    @Bean
    @ConditionalOnMissingBean
-    @ConditionalOnBean(ArchiveStore.class)
-    public RawArchiveEventSink rawArchiveEventSink(ArchiveStore store) {
-        return new RawArchiveEventSink(store);
+    @ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "type", havingValue = "local", matchIfMissing = true)
+    public RawArchiveEventSink rawArchiveEventSink(SinkArchiveProperties properties) {
+        return new RawArchiveEventSink(Path.of(properties.getPath()));
    }
 }
--- a/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveProperties.java
+++ b/modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveProperties.java
@@ -7,7 +7,7 @@ public class SinkArchiveProperties {

    private boolean enabled = true;
    private String type = "local";
-    private String path = System.getProperty("java.io.tmpdir") + "/lingniu-archive";
+    private String path = "./archive/";

    public boolean isEnabled() {
        return enabled;
--- a/modules/sinks/sink-archive/src/test/java/com/lingniu/ingest/sink/archive/RawArchiveEventSinkTest.java
+++ b/modules/sinks/sink-archive/src/test/java/com/lingniu/ingest/sink/archive/RawArchiveEventSinkTest.java
@@ -1,63 +1,77 @@
 package com.lingniu.ingest.sink.archive;

 import com.lingniu.ingest.api.ProtocolId;
+import com.lingniu.ingest.api.event.RawArchiveKeys;
 import com.lingniu.ingest.api.event.VehicleEvent;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;

-import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Instant;
 import java.util.Map;
-import java.util.concurrent.CompletionException;

+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;

 class RawArchiveEventSinkTest {

+    @TempDir
+    Path tempDir;
+
    @Test
-    void failsFastWhenRawBytesAreNull() {
-        RawArchiveEventSink sink = new RawArchiveEventSink(new CapturingArchiveStore());
-        VehicleEvent.RawArchive raw = new VehicleEvent.RawArchive(
-                "event-1",
-                "VIN123",
-                ProtocolId.GB32960,
-                Instant.parse("2026-06-23T00:00:00Z"),
-                Instant.parse("2026-06-23T00:00:01Z"),
-                "trace-1",
-                Map.of(),
-                2,
-                1,
-                null);
+    void writesRawBytesUsingSharedArchiveKey() throws Exception {
+        RawArchiveEventSink sink = new RawArchiveEventSink(tempDir);
+        VehicleEvent.RawArchive raw = rawArchive(Map.of(), new byte[]{0x23, 0x23, 0x01});
+
+        sink.publish(raw).join();
+
+        String key = RawArchiveKeys.key(raw);
+        assertThat(Files.readAllBytes(tempDir.resolve(key))).containsExactly(0x23, 0x23, 0x01);
+        assertThat(RawArchiveKeys.logicalUri(key)).isEqualTo("archive://" + key);
+    }
+
+    @Test
+    void rejectsArchiveKeysThatEscapeRoot() {
+        RawArchiveEventSink sink = new RawArchiveEventSink(tempDir);
+        VehicleEvent.RawArchive raw = rawArchive(
+                Map.of(RawArchiveKeys.META_KEY, "../escape.bin"),
+                new byte[]{0x01});

        assertThatThrownBy(() -> sink.publish(raw).join())
-                .isInstanceOf(CompletionException.class)
-                .hasCauseInstanceOf(IllegalArgumentException.class);
+                .hasCauseInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("raw archive key escapes root");
    }

-    private static final class CapturingArchiveStore implements ArchiveStore {
+    @Test
+    void rejectsSymlinkTraversalBeforeCreatingDirectories() throws Exception {
+        Path root = tempDir.resolve("archive");
+        Path outside = tempDir.resolve("outside");
+        Files.createDirectories(root);
+        Files.createDirectories(outside);
+        Files.createSymbolicLink(root.resolve("link"), outside);

-        @Override
-        public String put(String key, InputStream data, long length) {
-            return "archive://" + key;
-        }
+        RawArchiveEventSink sink = new RawArchiveEventSink(root);
+        VehicleEvent.RawArchive raw = rawArchive(
+                Map.of(RawArchiveKeys.META_KEY, "link/nested/raw.bin"),
+                new byte[]{0x01});

-        @Override
-        public String append(String key, byte[] chunk) {
-            return "archive://" + key;
-        }
+        assertThatThrownBy(() -> sink.publish(raw).join())
+                .hasRootCauseMessage("raw archive path traverses symlink: " + root.resolve("link"));
+        assertThat(outside.resolve("nested")).doesNotExist();
+    }

-        @Override
-        public InputStream get(String key) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public boolean exists(String key) {
-            return false;
-        }
-
-        @Override
-        public long size(String key) {
-            return 0;
-        }
+    private static VehicleEvent.RawArchive rawArchive(Map<String, String> metadata, byte[] rawBytes) {
+        return new VehicleEvent.RawArchive(
+                "raw-1",
+                "VIN001",
+                ProtocolId.GB32960,
+                Instant.parse("2026-06-22T08:00:00Z"),
+                Instant.parse("2026-06-22T08:00:01Z"),
+                "trace-1",
+                metadata,
+                0x02,
+                0,
+                rawBytes);
    }
 }