feat: persist raw frames and publish raw facts

modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/RawArchiveEventSink.java

@@ -4,15 +4,31 @@ import com.lingniu.ingest.api.event.RawArchiveKeys;
 import com.lingniu.ingest.api.event.VehicleEvent;
 import com.lingniu.ingest.api.sink.EventSink;
 
-import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.HexFormat;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
-public final class RawArchiveEventSink implements EventSink {
+public final class RawArchiveEventSink implements EventSink, AutoCloseable {
 
-    private final ArchiveStore store;
+    private final Path root;
+    private final ExecutorService executor;
 
-    public RawArchiveEventSink(ArchiveStore store) {
-        this.store = store;
+    public RawArchiveEventSink(Path root) {
+        if (root == null) {
+            throw new IllegalArgumentException("root must not be null");
+        }
+        this.root = root.toAbsolutePath().normalize();
+        this.executor = Executors.newVirtualThreadPerTaskExecutor();
     }
 
     @Override
@@ -20,25 +36,115 @@ public final class RawArchiveEventSink implements EventSink {
         return "raw-archive";
     }
 
+    @Override
+    public boolean accepts(VehicleEvent event) {
+        return event instanceof VehicleEvent.RawArchive;
+    }
+
     @Override
     public CompletableFuture<Void> publish(VehicleEvent event) {
         if (!(event instanceof VehicleEvent.RawArchive raw)) {
             return CompletableFuture.completedFuture(null);
         }
-        try {
-            byte[] bytes = raw.rawBytes();
-            if (bytes == null) {
-                throw new IllegalArgumentException("raw archive bytes must not be null");
-            }
-            store.put(RawArchiveKeys.key(raw), new ByteArrayInputStream(bytes), bytes.length);
-            return CompletableFuture.completedFuture(null);
-        } catch (Exception e) {
-            return CompletableFuture.failedFuture(e);
-        }
+        return CompletableFuture.runAsync(() -> write(raw), executor);
     }
 
     @Override
-    public boolean accepts(VehicleEvent event) {
-        return event instanceof VehicleEvent.RawArchive;
+    public void close() {
+        executor.shutdown();
+    }
+
+    public Path root() {
+        return root;
+    }
+
+    private void write(VehicleEvent.RawArchive raw) {
+        byte[] bytes = raw.rawBytes() == null ? new byte[0] : raw.rawBytes();
+        if (bytes.length == 0) {
+            return;
+        }
+        String key = RawArchiveKeys.key(raw);
+        Path target = resolveKey(key);
+        try {
+            createDirectoriesInsideRoot(target.getParent());
+            rejectSymlinkPath(target);
+            try {
+                Files.write(target, bytes, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+            } catch (FileAlreadyExistsException duplicate) {
+                byte[] existing = Files.readAllBytes(target);
+                if (!Arrays.equals(sha256(existing), sha256(bytes))) {
+                    throw new IOException("raw archive duplicate key has different content: " + key, duplicate);
+                }
+            }
+        } catch (IOException e) {
+            throw new RawArchiveWriteException("failed to write raw archive " + RawArchiveKeys.logicalUri(key), e);
+        }
+    }
+
+    private Path resolveKey(String key) {
+        Path target = root.resolve(key == null ? "" : key).normalize();
+        if (!target.startsWith(root)) {
+            throw new IllegalArgumentException("raw archive key escapes root: " + key);
+        }
+        return target;
+    }
+
+    private void createDirectoriesInsideRoot(Path directory) throws IOException {
+        rejectEscapedPath(directory);
+        if (Files.notExists(root, LinkOption.NOFOLLOW_LINKS)) {
+            Files.createDirectories(root);
+        }
+        Path current = root;
+        Path relative = root.relativize(directory);
+        for (Path segment : relative) {
+            current = current.resolve(segment);
+            if (Files.isSymbolicLink(current)) {
+                throw new IOException("raw archive path traverses symlink: " + current);
+            }
+            if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
+                Files.createDirectory(current);
+            } else if (!Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) {
+                throw new IOException("raw archive path segment is not a directory: " + current);
+            }
+        }
+    }
+
+    private void rejectSymlinkPath(Path target) throws IOException {
+        rejectEscapedPath(target);
+        Path current = root;
+        Path relative = root.relativize(target);
+        for (Path segment : relative) {
+            current = current.resolve(segment);
+            if (Files.isSymbolicLink(current)) {
+                throw new IOException("raw archive path traverses symlink: " + current);
+            }
+            if (Files.notExists(current, LinkOption.NOFOLLOW_LINKS)) {
+                return;
+            }
+        }
+    }
+
+    private void rejectEscapedPath(Path path) {
+        if (!path.normalize().startsWith(root)) {
+            throw new IllegalArgumentException("raw archive path escapes root: " + path);
+        }
+    }
+
+    private static byte[] sha256(byte[] bytes) {
+        try {
+            return MessageDigest.getInstance("SHA-256").digest(bytes);
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalStateException("SHA-256 is unavailable", e);
+        }
+    }
+
+    static String checksum(byte[] bytes) {
+        return "sha256:" + HexFormat.of().formatHex(sha256(bytes == null ? new byte[0] : bytes));
+    }
+
+    public static final class RawArchiveWriteException extends RuntimeException {
+        public RawArchiveWriteException(String message, Throwable cause) {
+            super(message, cause);
+        }
     }
 }

modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveAutoConfiguration.java

@@ -5,31 +5,28 @@ import com.lingniu.ingest.sink.archive.LocalArchiveStore;
 import com.lingniu.ingest.sink.archive.RawArchiveEventSink;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 
+import java.nio.file.Path;
+
 @AutoConfiguration
 @EnableConfigurationProperties(SinkArchiveProperties.class)
-@ConditionalOnProperty(
-        prefix = "lingniu.ingest.sink.archive",
-        name = "enabled",
-        havingValue = "true",
-        matchIfMissing = true)
+@ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "enabled", havingValue = "true", matchIfMissing = true)
 public class SinkArchiveAutoConfiguration {
 
     @Bean
     @ConditionalOnMissingBean
     @ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "type", havingValue = "local", matchIfMissing = true)
     public ArchiveStore archiveStore(SinkArchiveProperties properties) {
-        return new LocalArchiveStore(properties.getPath());
+        return new LocalArchiveStore(Path.of(properties.getPath()));
     }
 
     @Bean
     @ConditionalOnMissingBean
-    @ConditionalOnBean(ArchiveStore.class)
-    public RawArchiveEventSink rawArchiveEventSink(ArchiveStore store) {
-        return new RawArchiveEventSink(store);
+    @ConditionalOnProperty(prefix = "lingniu.ingest.sink.archive", name = "type", havingValue = "local", matchIfMissing = true)
+    public RawArchiveEventSink rawArchiveEventSink(SinkArchiveProperties properties) {
+        return new RawArchiveEventSink(Path.of(properties.getPath()));
     }
 }

modules/sinks/sink-archive/src/main/java/com/lingniu/ingest/sink/archive/config/SinkArchiveProperties.java

@@ -7,7 +7,7 @@ public class SinkArchiveProperties {
 
     private boolean enabled = true;
     private String type = "local";
-    private String path = System.getProperty("java.io.tmpdir") + "/lingniu-archive";
+    private String path = "./archive/";
 
     public boolean isEnabled() {
         return enabled;