shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
deploy/portainer/docker-compose.yml
View File

@@ -4,7 +4,7 @@ version: "3.8"
# main-0.1.0-SNAPSHOT-22468e7
x-common-env: &common-env
TZ: Asia/Shanghai
JAVA_OPTS: ${JAVA_OPTS:--Xms128m -Xmx768m -Duser.timezone=Asia/Shanghai}
JAVA_OPTS: ${JAVA_OPTS:--XX:+UseZGC -XX:MaxRAMPercentage=75}
NACOS_CONFIG_ENABLED: ${NACOS_CONFIG_ENABLED:-true}
NACOS_SERVER_ADDR: ${NACOS_SERVER_ADDR:-127.0.0.1:8848}
NACOS_NAMESPACE: ${NACOS_NAMESPACE:-}
@@ -13,32 +13,33 @@ x-common-env: &common-env
NACOS_REFRESH_ENABLED: ${NACOS_REFRESH_ENABLED:-true}
NACOS_USERNAME: ${NACOS_USERNAME:-}
NACOS_PASSWORD: ${NACOS_PASSWORD:-}
KAFKA_BROKERS: ${KAFKA_BROKERS:-114.55.58.251:9092}
KAFKA_BROKERS: ${KAFKA_BROKERS:-172.17.111.56:9092}
KAFKA_ENABLED: ${KAFKA_ENABLED:-true}
KAFKA_CONSUMER_ENABLED: ${KAFKA_CONSUMER_ENABLED:-true}
KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS: ${KAFKA_CONSUMER_MAX_POLL_INTERVAL_MILLIS:-900000}
KAFKA_TOPIC_GB32960_EVENT: ${KAFKA_TOPIC_GB32960_EVENT:-vehicle.event.gb32960.v1}
KAFKA_TOPIC_GB32960_RAW: ${KAFKA_TOPIC_GB32960_RAW:-vehicle.raw.gb32960.v1}
KAFKA_TOPIC_GB32960_DLQ: ${KAFKA_TOPIC_GB32960_DLQ:-vehicle.dlq.gb32960.v1}
KAFKA_TOPIC_JT808_EVENT: ${KAFKA_TOPIC_JT808_EVENT:-vehicle.event.jt808.v1}
KAFKA_TOPIC_JT808_RAW: ${KAFKA_TOPIC_JT808_RAW:-vehicle.raw.jt808.v1}
KAFKA_TOPIC_JT808_DLQ: ${KAFKA_TOPIC_JT808_DLQ:-vehicle.dlq.jt808.v1}
KAFKA_TOPIC_YUTONG_MQTT_EVENT: ${KAFKA_TOPIC_YUTONG_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_RAW: ${KAFKA_TOPIC_YUTONG_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_YUTONG_MQTT_DLQ: ${KAFKA_TOPIC_YUTONG_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
KAFKA_TOPIC_XINDA_PUSH_EVENT: ${KAFKA_TOPIC_XINDA_PUSH_EVENT:-vehicle.event.xinda-push.v1}
KAFKA_TOPIC_XINDA_PUSH_RAW: ${KAFKA_TOPIC_XINDA_PUSH_RAW:-vehicle.raw.xinda-push.v1}
KAFKA_TOPIC_XINDA_PUSH_DLQ: ${KAFKA_TOPIC_XINDA_PUSH_DLQ:-vehicle.dlq.xinda-push.v1}
KAFKA_TOPIC_MQTT_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_MQTT_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_MQTT_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
KAFKA_TOPIC_XINDA_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_XINDA_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_XINDA_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
MYSQL_USERNAME: ${MYSQL_USERNAME:-}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
services:
gb32960-ingest-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/gb32960-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: gb32960-ingest-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_GB32960:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20100
GB32960_PORT: 32960
KAFKA_CONSUMER_ENABLED: "false"
@@ -48,13 +49,11 @@ services:
GB32960_PLATFORM_PWD_HYUNDAI: ${GB32960_PLATFORM_PWD_HYUNDAI:-}
GB32960_PLATFORM_IP_HYUNDAI: ${GB32960_PLATFORM_IP_HYUNDAI:-}
GB32960_TLS_ENABLED: ${GB32960_TLS_ENABLED:-false}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
ports:
- "${GB32960_HTTP_PORT:-20100}:20100"
- "${GB32960_TCP_PORT:-32960}:32960"
volumes:
- vehicle-history-archive:/archive
- gb32960-ingest-data:/data
networks:
- vehicle-ingest
@@ -63,109 +62,82 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/jt808-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: jt808-ingest-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_JT808:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20482
HTTP_PORT: 20400
JT808_PORT: 808
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_JT808:-jt808-ingest-portainer}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
ports:
- "${JT808_HTTP_PORT:-20482}:20482"
- "${JT808_HTTP_PORT:-20400}:20400"
- "${JT808_TCP_PORT:-808}:808"
volumes:
- vehicle-history-archive:/archive
- jt808-ingest-data:/data
networks:
- vehicle-ingest
yutong-mqtt-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/yutong-mqtt-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: yutong-mqtt-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_YUTONG_MQTT:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20500
MQTT_ENABLED: "true"
XINDA_PUSH_ENABLED: "false"
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_YUTONG_MQTT:-yutong-mqtt-portainer}
YUTONG_MQTT_ENABLED: ${YUTONG_MQTT_ENABLED:-false}
YUTONG_MQTT_ENDPOINT_NAME: ${YUTONG_MQTT_ENDPOINT_NAME:-yutong}
YUTONG_MQTT_URI: ${YUTONG_MQTT_URI:-}
YUTONG_MQTT_TOPIC: ${YUTONG_MQTT_TOPIC:-#}
YUTONG_MQTT_QOS: ${YUTONG_MQTT_QOS:-1}
YUTONG_MQTT_CLIENT_ID: ${YUTONG_MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
YUTONG_MQTT_USERNAME: ${YUTONG_MQTT_USERNAME:-}
YUTONG_MQTT_PASSWORD: ${YUTONG_MQTT_PASSWORD:-}
YUTONG_MQTT_CLEAN_SESSION: ${YUTONG_MQTT_CLEAN_SESSION:-false}
YUTONG_MQTT_KEEP_ALIVE_SECONDS: ${YUTONG_MQTT_KEEP_ALIVE_SECONDS:-20}
YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS: ${YUTONG_MQTT_CONNECTION_TIMEOUT_SECONDS:-10}
YUTONG_MQTT_TLS_CA_PEM: ${YUTONG_MQTT_TLS_CA_PEM:-}
YUTONG_MQTT_TLS_CLIENT_PEM: ${YUTONG_MQTT_TLS_CLIENT_PEM:-}
YUTONG_MQTT_TLS_CLIENT_KEY: ${YUTONG_MQTT_TLS_CLIENT_KEY:-}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
KAFKA_NODE_ID: ${KAFKA_NODE_ID_MQTT:-yutong-mqtt-portainer}
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_MQTT_EVENT:-vehicle.event.mqtt-yutong.v1}
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_MQTT_RAW:-vehicle.raw.mqtt-yutong.v1}
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_MQTT_DLQ:-vehicle.dlq.mqtt-yutong.v1}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
MQTT_ENDPOINT_NAME: ${MQTT_ENDPOINT_NAME:-yutong}
MQTT_URI: ${MQTT_URI:-tcp://127.0.0.1:1883}
MQTT_TOPIC: "${MQTT_TOPIC:-#}"
MQTT_QOS: ${MQTT_QOS:-1}
MQTT_CLIENT_ID: ${MQTT_CLIENT_ID:-lingniu-yutong-mqtt}
MQTT_USERNAME: ${MQTT_USERNAME:-}
MQTT_PASSWORD: ${MQTT_PASSWORD:-}
MQTT_CLEAN_SESSION: ${MQTT_CLEAN_SESSION:-true}
MQTT_PROFILE: ${MQTT_PROFILE:-yutong}
ports:
- "${YUTONG_MQTT_HTTP_PORT:-20500}:20500"
volumes:
- vehicle-history-archive:/archive
- yutong-mqtt-data:/data
networks:
- vehicle-ingest
xinda-push-app:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/xinda-push-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/partner-ingest-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: xinda-push-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_XINDA_PUSH:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20600
MQTT_ENABLED: "false"
XINDA_PUSH_ENABLED: "true"
KAFKA_CONSUMER_ENABLED: "false"
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA_PUSH:-xinda-push-portainer}
XINDA_PUSH_ENABLED: ${XINDA_PUSH_ENABLED:-false}
KAFKA_NODE_ID: ${KAFKA_NODE_ID_XINDA:-xinda-push-portainer}
KAFKA_TOPIC_PARTNER_EVENT: ${KAFKA_TOPIC_XINDA_EVENT:-vehicle.event.xinda.v1}
KAFKA_TOPIC_PARTNER_RAW: ${KAFKA_TOPIC_XINDA_RAW:-vehicle.raw.xinda.v1}
KAFKA_TOPIC_PARTNER_DLQ: ${KAFKA_TOPIC_XINDA_DLQ:-vehicle.dlq.xinda.v1}
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-mysql}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${MYSQL_PASSWORD:-}
XINDA_PUSH_HOST: ${XINDA_PUSH_HOST:-}
XINDA_PUSH_PORT: ${XINDA_PUSH_PORT:-10100}
XINDA_PUSH_USERNAME: ${XINDA_PUSH_USERNAME:-}
XINDA_PUSH_PASSWORD: ${XINDA_PUSH_PASSWORD:-}
XINDA_PUSH_CLIENT_DESC: ${XINDA_PUSH_CLIENT_DESC:-lingniu-ingest}
XINDA_PUSH_MSG_ID_LOCATION: ${XINDA_PUSH_MSG_ID_LOCATION:-0200}
XINDA_PUSH_MSG_ID_ALARM: ${XINDA_PUSH_MSG_ID_ALARM:-0300}
XINDA_PUSH_MSG_ID_PASSTHROUGH: ${XINDA_PUSH_MSG_ID_PASSTHROUGH:-0401}
XINDA_PUSH_RECONNECT_INTERVAL_SEC: ${XINDA_PUSH_RECONNECT_INTERVAL_SEC:-5}
SINK_ARCHIVE_PATH: /archive/
VEHICLE_IDENTITY_STORE: ${VEHICLE_IDENTITY_STORE:-file}
VEHICLE_IDENTITY_FILE: /data/vehicle-identity.jsonl
VEHICLE_IDENTITY_MYSQL_TABLE: ${VEHICLE_IDENTITY_MYSQL_TABLE:-vehicle_identity_binding}
VEHICLE_IDENTITY_MYSQL_JDBC_URL: ${VEHICLE_IDENTITY_MYSQL_JDBC_URL:-}
VEHICLE_IDENTITY_MYSQL_USERNAME: ${VEHICLE_IDENTITY_MYSQL_USERNAME:-}
VEHICLE_IDENTITY_MYSQL_PASSWORD: ${VEHICLE_IDENTITY_MYSQL_PASSWORD:-}
VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL: ${VEHICLE_IDENTITY_MYSQL_REFRESH_INTERVAL:-60s}
XINDA_PUSH_SUBSCRIBE_MSG_IDS: "${XINDA_PUSH_SUBSCRIBE_MSG_IDS:-0200,0300,0401}"
ports:
- "${XINDA_PUSH_HTTP_PORT:-20600}:20600"
volumes:
- vehicle-history-archive:/archive
- xinda-push-data:/data
networks:
- vehicle-ingest
@@ -173,8 +145,6 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-history-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-history-app
restart: unless-stopped
mem_limit: 2560m
memswap_limit: 2560m
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
@@ -182,29 +152,28 @@ services:
- xinda-push-app
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_HISTORY:--Xms512m -Xmx1536m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20200
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_HISTORY:-vehicle-history}
KAFKA_CONSUMER_MAX_POLL_RECORDS: ${KAFKA_CONSUMER_MAX_POLL_RECORDS_HISTORY:-2000}
KAFKA_GROUP_HISTORY: ${KAFKA_GROUP_HISTORY:-vehicle-history}
KAFKA_CONSUMER_CONCURRENCY: ${KAFKA_CONSUMER_CONCURRENCY:-3}
KAFKA_GROUP_HISTORY_GB32960: ${KAFKA_GROUP_HISTORY_GB32960:-vehicle-history}
KAFKA_GROUP_HISTORY_JT808: ${KAFKA_GROUP_HISTORY_JT808:-vehicle-history}
KAFKA_GROUP_HISTORY_MQTT_YUTONG: ${KAFKA_GROUP_HISTORY_MQTT_YUTONG:-vehicle-history}
KAFKA_GROUP_HISTORY_XINDA: ${KAFKA_GROUP_HISTORY_XINDA:-vehicle-history}
SINK_ARCHIVE_ENABLED: ${SINK_ARCHIVE_ENABLED:-true}
SINK_ARCHIVE_PATH: /archive/
# 高吞吐生产热路径默认只写 TDengine。需要 GB32960 snapshot/fields 旧查询时再显式开启。
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-false}
EVENT_FILE_STORE_ENABLED: ${EVENT_FILE_STORE_ENABLED:-true}
EVENT_FILE_STORE_PATH: /event-store/
EVENT_FILE_STORE_ZONE_ID: ${EVENT_FILE_STORE_ZONE_ID:-Asia/Shanghai}
TDENGINE_HISTORY_ENABLED: ${TDENGINE_HISTORY_ENABLED:-true}
TDENGINE_HISTORY_DATABASE: ${TDENGINE_HISTORY_DATABASE:-vehicle_ts}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://tdengine:6041/vehicle_ts}
TDENGINE_DRIVER_CLASS_NAME: ${TDENGINE_DRIVER_CLASS_NAME:-com.taosdata.jdbc.ws.WebSocketDriver}
EVENT_FILE_STORE_BATCH_SIZE: ${EVENT_FILE_STORE_BATCH_SIZE:-500}
EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS: ${EVENT_FILE_STORE_FLUSH_INTERVAL_MILLIS:-1000}
EVENT_FILE_STORE_WRITE_PARQUET_ENABLED: ${EVENT_FILE_STORE_WRITE_PARQUET_ENABLED:-false}
TDENGINE_ENABLED: ${TDENGINE_ENABLED:-true}
TDENGINE_JDBC_URL: ${TDENGINE_JDBC_URL:-jdbc:TAOS-WS://172.17.111.57:6041/vehicle_ts}
TDENGINE_DATABASE: ${TDENGINE_DATABASE:-vehicle_ts}
TDENGINE_JT808_LOCATION_STABLE: ${TDENGINE_JT808_LOCATION_STABLE:-jt808_locations}
TDENGINE_USERNAME: ${TDENGINE_USERNAME:-root}
TDENGINE_PASSWORD: ${TDENGINE_PASSWORD:-taosdata}
TDENGINE_MAX_POOL_SIZE: ${TDENGINE_MAX_POOL_SIZE:-32}
TDENGINE_MIN_IDLE: ${TDENGINE_MIN_IDLE:-0}
TDENGINE_CONNECTION_TIMEOUT_MILLIS: ${TDENGINE_CONNECTION_TIMEOUT_MILLIS:-5000}
TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS: ${TDENGINE_INITIALIZATION_FAIL_TIMEOUT_MILLIS:--1}
# 默认关闭逐字段宽表，避免 telemetry_fields 写放大；需要字段趋势查询时显式打开。
TDENGINE_TELEMETRY_FIELDS_ENABLED: ${TDENGINE_TELEMETRY_FIELDS_ENABLED:-false}
ports:
- "${VEHICLE_HISTORY_HTTP_PORT:-20200}:20200"
volumes:
@@ -217,16 +186,11 @@ services:
image: crpi-85r4m0ackrm3qpje.cn-shanghai.personal.cr.aliyuncs.com/oneos/vehicle-analytics-app:${LINGNIU_IMAGE_VERSION:?set LINGNIU_IMAGE_VERSION}
container_name: vehicle-analytics-app
restart: unless-stopped
mem_limit: 1536m
memswap_limit: 1536m
depends_on:
- gb32960-ingest-app
- jt808-ingest-app
- yutong-mqtt-app
- xinda-push-app
environment:
<<: *common-env
JAVA_OPTS: ${JAVA_OPTS_ANALYTICS:--Xms256m -Xmx768m -Duser.timezone=Asia/Shanghai}
HTTP_PORT: 20300
KAFKA_CONSUMER_CLIENT_ID_PREFIX: ${KAFKA_CONSUMER_CLIENT_ID_PREFIX_ANALYTICS:-vehicle-analytics}
KAFKA_GROUP_STATE: ${KAFKA_GROUP_STATE:-vehicle-state}
@@ -244,6 +208,7 @@ services:
REDIS_HOST: ${REDIS_HOST:-}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DATABASE: ${REDIS_DATABASE:-50}
REDIS_USERNAME: ${REDIS_USERNAME:-}
REDIS_PASSWORD: ${REDIS_PASSWORD:-}
ports:
- "${VEHICLE_ANALYTICS_HTTP_PORT:-20300}:20300"
@@ -259,8 +224,6 @@ networks:
volumes:
gb32960-ingest-data:
jt808-ingest-data:
yutong-mqtt-data:
xinda-push-data:
vehicle-history-archive:
vehicle-history-event-store:
vehicle-analytics-stat:

92
deploy/tools/EnsureKafkaTopics.java Normal file
View File

@@ -0,0 +1,92 @@
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.admin.NewPartitions;
import org.apache.kafka.clients.admin.NewTopic;
import org.apache.kafka.clients.admin.TopicDescription;
public final class EnsureKafkaTopics {
private static final short REPLICATION_FACTOR = 1;
private record TopicSpec(String name, int partitions) {}
private EnsureKafkaTopics() {}
public static void main(String[] args) throws Exception {
String brokers = args.length > 0 ? args[0] : "172.17.111.56:9092";
List<TopicSpec> specs =
List.of(
topic("vehicle.event.gb32960.v1", 12),
topic("vehicle.raw.gb32960.v1", 12),
topic("vehicle.dlq.gb32960.v1", 3),
topic("vehicle.event.jt808.v1", 12),
topic("vehicle.raw.jt808.v1", 12),
topic("vehicle.dlq.jt808.v1", 3),
topic("vehicle.event.mqtt-yutong.v1", 12),
topic("vehicle.raw.mqtt-yutong.v1", 12),
topic("vehicle.dlq.mqtt-yutong.v1", 3),
topic("vehicle.event.xinda.v1", 12),
topic("vehicle.raw.xinda.v1", 12),
topic("vehicle.dlq.xinda.v1", 3),
topic("vehicle.event.xinda-push.v1", 12),
topic("vehicle.raw.xinda-push.v1", 12),
topic("vehicle.dlq.xinda-push.v1", 3));
Properties props = new Properties();
props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokers);
props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, "15000");
props.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, "30000");
try (AdminClient admin = AdminClient.create(props)) {
Set<String> existing = admin.listTopics().names().get();
List<NewTopic> missing =
specs.stream()
.filter(spec -> !existing.contains(spec.name()))
.map(spec -> new NewTopic(spec.name(), spec.partitions(), REPLICATION_FACTOR))
.toList();
if (!missing.isEmpty()) {
try {
admin.createTopics(missing).all().get();
} catch (ExecutionException e) {
System.out.println("createTopics result=" + e.getCause());
}
}
Map<String, TopicSpec> byName =
specs.stream().collect(Collectors.toMap(TopicSpec::name, spec -> spec));
Map<String, TopicDescription> descriptions =
admin.describeTopics(byName.keySet()).allTopicNames().get();
Map<String, NewPartitions> increases =
descriptions.entrySet().stream()
.filter(entry -> entry.getValue().partitions().size() < byName.get(entry.getKey()).partitions())
.collect(
Collectors.toMap(
Map.Entry::getKey,
entry -> NewPartitions.increaseTo(byName.get(entry.getKey()).partitions())));
if (!increases.isEmpty()) {
admin.createPartitions(increases).all().get();
Thread.sleep(Duration.ofSeconds(2).toMillis());
descriptions = admin.describeTopics(byName.keySet()).allTopicNames().get();
}
List<String> lines = new ArrayList<>();
descriptions.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.forEach(entry -> lines.add(entry.getKey() + " partitions=" + entry.getValue().partitions().size()));
System.out.println("bootstrap=" + brokers);
lines.forEach(System.out::println);
}
}
private static TopicSpec topic(String name, int partitions) {
return new TopicSpec(name, partitions);
}
}