shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(operations): add automated reconciliation center

Browse Source
This commit is contained in:
lingniu
2026-07-16 19:14:53 +08:00
parent bbab018d55
commit d67f42b4f4
25 changed files with 1697 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
vehicle-data-platform/deploy/install-web-release.sh
View File

@@ -6,6 +6,7 @@ RELEASE_ID=${1:?release id is required}
ARCHIVE=${2:?web archive is required}
API_BINARY=${3:-}
ONEOS_SCOPE_BINARY=${4:-}
RECONCILIATION_BINARY=${5:-}
ROOT=${PLATFORM_ROOT:-/opt/lingniu-vehicle-platform}
BASE_URL=${PLATFORM_BASE_URL:-http://127.0.0.1:20300}
SERVICE=${PLATFORM_SERVICE:-lingniu-vehicle-platform}
@@ -25,6 +26,9 @@ fi
if test -n "$ONEOS_SCOPE_BINARY"; then
test -f "$ONEOS_SCOPE_BINARY" || { printf 'OneOS scope sync binary is missing: %s\n' "$ONEOS_SCOPE_BINARY" >&2; exit 1; }
fi
if test -n "$RECONCILIATION_BINARY"; then
test -f "$RECONCILIATION_BINARY" || { printf 'reconciliation evaluator binary is missing: %s\n' "$RECONCILIATION_BINARY" >&2; exit 1; }
fi
test -x "$SCRIPT_DIR/prepare-web-release-tree.sh" || { printf 'release tree helper is missing\n' >&2; exit 1; }
test -f "$SCRIPT_DIR/prune-release-history.py" || { printf 'release pruning helper is missing\n' >&2; exit 1; }
test -f "$SCRIPT_DIR/verify-customer-demo.py" || { printf 'customer demo gate is missing\n' >&2; exit 1; }
@@ -76,7 +80,7 @@ trap rollback ERR
test ! -e "$next" || { printf 'release already exists: %s\n' "$next" >&2; exit 1; }
mkdir -p "$next/web"
for runtime_file in platform-api alert-evaluator alert-stream-evaluator platform-migrate oneos-scope-sync alert-benchmark; do
for runtime_file in platform-api alert-evaluator alert-stream-evaluator platform-migrate oneos-scope-sync reconciliation-evaluator alert-benchmark; do
if test -f "$old/$runtime_file"; then
cp "$old/$runtime_file" "$next/$runtime_file"
fi
@@ -88,6 +92,9 @@ fi
if test -n "$ONEOS_SCOPE_BINARY"; then
cp "$ONEOS_SCOPE_BINARY" "$next/oneos-scope-sync"
fi
if test -n "$RECONCILIATION_BINARY"; then
cp "$RECONCILIATION_BINARY" "$next/reconciliation-evaluator"
fi
if test -f "$old/lingniu-vehicle-platform.service"; then
cp "$old/lingniu-vehicle-platform.service" "$next/lingniu-vehicle-platform.service"
fi
@@ -96,12 +103,16 @@ if test -d "$SCRIPT_DIR/migrations"; then
mkdir -p "$next/deploy/migrations"
cp "$SCRIPT_DIR"/migrations/*.sql "$next/deploy/migrations/"
fi
if test -d "$SCRIPT_DIR/systemd"; then
mkdir -p "$next/deploy/systemd"
cp "$SCRIPT_DIR"/systemd/* "$next/deploy/systemd/"
fi
cp "$SCRIPT_DIR/prepare-web-release-tree.sh" "$next/deploy/prepare-web-release-tree.sh"
cp "$SCRIPT_DIR/prune-release-history.py" "$next/deploy/prune-release-history.py"
cp "$SCRIPT_DIR/verify-customer-demo.py" "$next/deploy/verify-customer-demo.py"
cp "$0" "$next/deploy/install-web-release.sh"
chmod +x "$next/platform-api" "$next/deploy/"*.sh "$next/deploy/prune-release-history.py" "$next/deploy/verify-customer-demo.py"
for runtime_file in alert-evaluator alert-stream-evaluator platform-migrate oneos-scope-sync alert-benchmark; do
for runtime_file in alert-evaluator alert-stream-evaluator platform-migrate oneos-scope-sync reconciliation-evaluator alert-benchmark; do
test ! -f "$next/$runtime_file" || chmod +x "$next/$runtime_file"
done

7
vehicle-data-platform/deploy/install-web-release.test.sh
View File

@@ -28,23 +28,26 @@ printf 'window.__LINGNIU_APP_CONFIG__={"amapSecurityServiceHost":"/_AMapService"
tar -C "$new_web" -czf "$fixture/new.tar.gz" .
printf 'new platform API\n' > "$fixture/new-platform-api"
printf 'new OneOS scope sync\n' > "$fixture/new-oneos-scope-sync"
printf 'new reconciliation evaluator\n' > "$fixture/new-reconciliation-evaluator"
printf '#!/usr/bin/env bash\nexit 0\n' > "$fixture/bin/systemctl"
printf '#!/usr/bin/env bash\nexit 0\n' > "$fixture/bin/curl"
printf '#!/usr/bin/env bash\nprintf "mock_verify=ok\\n"\n' > "$fixture/bin/verify"
chmod +x "$fixture/bin/"*
PLATFORM_ROOT="$root" SYSTEMCTL_BIN="$fixture/bin/systemctl" CURL_BIN="$fixture/bin/curl" VERIFY_WEB_RELEASE_BIN="$fixture/bin/verify" RELEASE_HISTORY_LIMIT=3 "$SCRIPT_DIR/install-web-release.sh" new-release "$fixture/new.tar.gz" "$fixture/new-platform-api" "$fixture/new-oneos-scope-sync" > "$fixture/install.out"
PLATFORM_ROOT="$root" SYSTEMCTL_BIN="$fixture/bin/systemctl" CURL_BIN="$fixture/bin/curl" VERIFY_WEB_RELEASE_BIN="$fixture/bin/verify" RELEASE_HISTORY_LIMIT=3 "$SCRIPT_DIR/install-web-release.sh" new-release "$fixture/new.tar.gz" "$fixture/new-platform-api" "$fixture/new-oneos-scope-sync" "$fixture/new-reconciliation-evaluator" > "$fixture/install.out"
test "$(readlink -f "$root/current")" = "$(cd "$root/releases/new-release" && pwd -P)"
grep -q '^PLATFORM_RELEASE=new-release$' "$root/env/platform.env"
test "$(cat "$root/current/platform-api")" = 'new platform API'
test "$(cat "$root/current/oneos-scope-sync")" = 'new OneOS scope sync'
test "$(cat "$root/current/reconciliation-evaluator")" = 'new reconciliation evaluator'
test "$(cat "$root/current/alert-evaluator")" = 'old evaluator'
test -f "$root/current/web/assets/new.js"
test -f "$root/current/web/assets/old.js"
test -f "$root/current/web/.compatibility-manifests/1.assets"
test -x "$root/current/deploy/verify-customer-demo.py"
test -f "$root/current/deploy/migrations/016_business_scope_dimensions.sql"
test -f "$root/current/deploy/migrations/017_reconciliation_center.sql"
test -f "$root/current/deploy/systemd/lingniu-vehicle-reconciliation-evaluator.timer"
test ! -e "$root/current/lingniu-vehicle-platform.service"
test "$(find "$root/releases" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" = 3
grep -q 'web_release_install=ok release=new-release previous=old-release' "$fixture/install.out"

55
vehicle-data-platform/deploy/migrations/017_reconciliation_center.sql Normal file
View File

@@ -0,0 +1,55 @@
CREATE TABLE IF NOT EXISTS vehicle_reconciliation_run (
run_id VARCHAR(64) NOT NULL PRIMARY KEY,
status VARCHAR(24) NOT NULL,
detected_count INT UNSIGNED NOT NULL DEFAULT 0,
new_count INT UNSIGNED NOT NULL DEFAULT 0,
active_count INT UNSIGNED NOT NULL DEFAULT 0,
recovered_count INT UNSIGNED NOT NULL DEFAULT 0,
rule_counts_json JSON NOT NULL,
error_message VARCHAR(512) NOT NULL DEFAULT '',
started_at DATETIME(3) NOT NULL,
finished_at DATETIME(3) NOT NULL,
INDEX idx_reconciliation_run_finished (finished_at, status)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS vehicle_reconciliation_issue (
id VARCHAR(64) NOT NULL PRIMARY KEY,
fingerprint CHAR(64) NOT NULL,
rule_code VARCHAR(64) NOT NULL,
category VARCHAR(32) NOT NULL,
severity VARCHAR(16) NOT NULL,
status VARCHAR(32) NOT NULL,
vin VARCHAR(64) NOT NULL DEFAULT '',
plate VARCHAR(64) NOT NULL DEFAULT '',
protocol_a VARCHAR(32) NOT NULL DEFAULT '',
protocol_b VARCHAR(32) NOT NULL DEFAULT '',
title VARCHAR(160) NOT NULL,
summary VARCHAR(500) NOT NULL,
evidence_json JSON NOT NULL,
first_seen_at DATETIME(3) NOT NULL,
last_seen_at DATETIME(3) NOT NULL,
occurrence_count BIGINT UNSIGNED NOT NULL DEFAULT 1,
recovered_at DATETIME(3) NULL,
resolution_note VARCHAR(500) NOT NULL DEFAULT '',
resolved_by VARCHAR(128) NOT NULL DEFAULT '',
version INT UNSIGNED NOT NULL DEFAULT 1,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
UNIQUE KEY uk_reconciliation_fingerprint (fingerprint),
INDEX idx_reconciliation_status (status, severity, last_seen_at),
INDEX idx_reconciliation_rule (rule_code, last_seen_at),
INDEX idx_reconciliation_vehicle (vin, last_seen_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS vehicle_reconciliation_action (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
issue_id VARCHAR(64) NOT NULL,
action VARCHAR(32) NOT NULL,
from_status VARCHAR(32) NOT NULL,
to_status VARCHAR(32) NOT NULL,
actor VARCHAR(128) NOT NULL,
note VARCHAR(500) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
INDEX idx_reconciliation_action_issue (issue_id, created_at),
INDEX idx_reconciliation_action_actor (actor, created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

23
vehicle-data-platform/deploy/systemd/lingniu-vehicle-reconciliation-evaluator.service Normal file
View File

@@ -0,0 +1,23 @@
[Unit]
Description=Lingniu Vehicle Data Reconciliation Evaluator
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/opt/lingniu-vehicle-platform/current
EnvironmentFile=/opt/lingniu-go-native/env/base.env
EnvironmentFile=/opt/lingniu-vehicle-platform/env/platform.env
ExecStart=/opt/lingniu-vehicle-platform/current/reconciliation-evaluator
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=6
TimeoutStartSec=600
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
[Install]
WantedBy=multi-user.target

12
vehicle-data-platform/deploy/systemd/lingniu-vehicle-reconciliation-evaluator.timer Normal file
View File

@@ -0,0 +1,12 @@
[Unit]
Description=Run vehicle data reconciliation every day
[Timer]
OnCalendar=*-*-* 02:15:00
RandomizedDelaySec=5m
AccuracySec=1m
Persistent=true
Unit=lingniu-vehicle-reconciliation-evaluator.service
[Install]
WantedBy=timers.target