shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

449
go/vehicle-gateway/scripts/deploy-ecs-release.sh Executable file
View File

@@ -0,0 +1,449 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RELEASE="go-native-$(date +%Y%m%d%H%M%S)"
OUT_DIR=""
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
BUILD_ONLY=0
STAGE_ONLY=0
CAPACITY_RETRIES=10
CAPACITY_WARMUP_SECONDS=60
CAPACITY_RETRY_INTERVAL_SECONDS=15
CAPACITY_TIMER="lingniu-go-capacity-check.timer"
CAPACITY_SERVICE="lingniu-go-capacity-check.service"
CAPACITY_ARGS="${CAPACITY_ARGS:-}"
RELEASE_RETAIN_COUNT="${RELEASE_RETAIN_COUNT:-8}"
BINARIES=(
"gateway:./cmd/gateway"
"history-writer:./cmd/history-writer"
"stat-writer:./cmd/stat-writer"
"realtime-api:./cmd/realtime-api"
"nats-fast-writer:./cmd/nats-fast-writer"
"nats-kafka-bridge:./cmd/nats-kafka-bridge"
"fields-projector:./cmd/fields-projector"
"capacity-check:./cmd/capacity-check"
"load-sim:./cmd/load-sim"
"stats-backfill:./cmd/stats-backfill"
"identity-import:./cmd/identity-import"
"identity-writer:./cmd/identity-writer"
)
SERVICES=(
"lingniu-go-nats-kafka-bridge.service"
"lingniu-go-fields-projector.service"
"lingniu-go-nats-fast-writer.service"
"lingniu-go-history-writer.service"
"lingniu-go-stat-writer.service"
"lingniu-go-realtime-writer.service"
"lingniu-go-identity-writer.service"
"lingniu-go-realtime-api.service"
"lingniu-go-gateway.service"
)
READYZ_PORTS=(20214 20218 20215 20212 20213 20216 20217 20200 20211)
usage() {
cat <<'USAGE'
Usage:
scripts/deploy-ecs-release.sh --build-only [--release NAME] [--out-dir DIR]
scripts/deploy-ecs-release.sh --host root@115.29.187.205 [--release NAME] [--out-dir DIR]
Options:
--host HOST SSH target for ECS deployment. Omit with --build-only.
--release NAME Release directory name under /opt/lingniu-go-native/releases.
--out-dir DIR Local build output directory. Defaults to /tmp/lingniu-go-native-<release>.
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
--build-only Build and verify Linux amd64 artifacts without uploading.
--stage-only Upload and verify release, but do not switch current or restart services.
--capacity-retries N
Retry capacity-check after restart. Defaults to 10.
--capacity-warmup-seconds N
Wait after readyz before capacity-check, so recent-window metrics are not dominated by restart catch-up. Defaults to 60.
--capacity-retry-interval-seconds N
Wait between capacity-check retries. Defaults to 15.
--capacity-timer UNIT
systemd timer paused during restart. Defaults to lingniu-go-capacity-check.timer.
--capacity-args ARGS
Extra arguments passed to capacity-check during deploy verification.
Example: --capacity-args "-daily-mileage-diagnostics-max-pipeline=0"
--release-retain-count N
Keep the newest N remote releases after a successful deployment. Defaults to 8.
-h, --help Show this help.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--host)
HOST="${2:-}"
shift 2
;;
--release)
RELEASE="${2:-}"
shift 2
;;
--out-dir)
OUT_DIR="${2:-}"
shift 2
;;
--remote-root)
REMOTE_ROOT="${2:-}"
shift 2
;;
--build-only)
BUILD_ONLY=1
shift
;;
--stage-only|--skip-restart)
STAGE_ONLY=1
shift
;;
--capacity-retries)
CAPACITY_RETRIES="${2:-}"
shift 2
;;
--capacity-warmup-seconds)
CAPACITY_WARMUP_SECONDS="${2:-}"
shift 2
;;
--capacity-retry-interval-seconds)
CAPACITY_RETRY_INTERVAL_SECONDS="${2:-}"
shift 2
;;
--capacity-timer)
CAPACITY_TIMER="${2:-}"
shift 2
;;
--capacity-args)
CAPACITY_ARGS="${2:-}"
shift 2
;;
--release-retain-count)
RELEASE_RETAIN_COUNT="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="/tmp/lingniu-go-native-${RELEASE}"
fi
if [[ "$BUILD_ONLY" -eq 0 && -z "$HOST" ]]; then
echo "--host is required unless --build-only is set" >&2
usage >&2
exit 2
fi
case "$CAPACITY_RETRIES" in
''|*[!0-9]*|0)
echo "--capacity-retries must be a positive integer" >&2
exit 2
;;
esac
case "$CAPACITY_WARMUP_SECONDS" in
''|*[!0-9]*)
echo "--capacity-warmup-seconds must be a non-negative integer" >&2
exit 2
;;
esac
case "$CAPACITY_RETRY_INTERVAL_SECONDS" in
''|*[!0-9]*)
echo "--capacity-retry-interval-seconds must be a non-negative integer" >&2
exit 2
;;
esac
case "$RELEASE_RETAIN_COUNT" in
''|*[!0-9]*|0)
echo "--release-retain-count must be a positive integer" >&2
exit 2
;;
esac
assert_linux_amd64() {
local binary="$1"
local info
info="$(file "$binary")"
case "$info" in
*"ELF 64-bit"*"x86-64"*) ;;
*)
echo "binary is not Linux amd64 ELF: $binary" >&2
echo "$info" >&2
exit 1
;;
esac
}
hash_file() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$@"
else
shasum -a 256 "$@"
fi
}
build_release() {
rm -rf "$OUT_DIR"
mkdir -p "$OUT_DIR"
pushd "$ROOT_DIR" >/dev/null
local names=()
for spec in "${BINARIES[@]}"; do
local name="${spec%%:*}"
local pkg="${spec#*:}"
names+=("$name")
echo "building $name from $pkg"
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
go build -trimpath -ldflags="${GO_LDFLAGS:--s -w}" -o "$OUT_DIR/$name" "$pkg"
assert_linux_amd64 "$OUT_DIR/$name"
done
popd >/dev/null
(
cd "$OUT_DIR"
hash_file "${names[@]}" > SHA256SUMS
{
echo "release=${RELEASE}"
echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "git_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)"
echo "go_version=$(go version)"
echo "target=linux/amd64"
} > RELEASE_NAME
)
echo "build output: $OUT_DIR"
}
preflight_remote_ssh() {
echo "checking ssh connectivity to ${HOST}"
if ssh -o ConnectTimeout=10 -o ConnectionAttempts=1 "$HOST" "true"; then
return
fi
cat >&2 <<EOF
ssh preflight failed for ${HOST}.
The release was not built, uploaded, or switched. Restore SSH access first, then rerun this script.
EOF
exit 255
}
deploy_release() {
local archive="/tmp/${RELEASE}.tar.gz"
local remote_tmp="/tmp/lingniu-go-native-${RELEASE}"
local remote_bins=""
local services=""
local readyz_ports=""
local spec
for spec in "${BINARIES[@]}"; do
remote_bins="${remote_bins} ${spec%%:*}"
done
for spec in "${SERVICES[@]}"; do
services="${services} ${spec}"
done
for spec in "${READYZ_PORTS[@]}"; do
readyz_ports="${readyz_ports} ${spec}"
done
COPYFILE_DISABLE=1 tar --format ustar -czf "$archive" -C "$OUT_DIR" .
ssh "$HOST" "mkdir -p '$remote_tmp'"
scp "$archive" "$HOST:${remote_tmp}.tar.gz"
local remote_cmd
printf -v remote_cmd \
"REMOTE_ROOT=%q RELEASE=%q REMOTE_TMP=%q REMOTE_BINS=%q SERVICES=%q READYZ_PORTS=%q STAGE_ONLY=%q CAPACITY_RETRIES=%q CAPACITY_WARMUP_SECONDS=%q CAPACITY_RETRY_INTERVAL_SECONDS=%q CAPACITY_TIMER=%q CAPACITY_SERVICE=%q CAPACITY_ARGS=%q RELEASE_RETAIN_COUNT=%q bash -s" \
"$REMOTE_ROOT" "$RELEASE" "$remote_tmp" "$remote_bins" "$services" "$readyz_ports" "$STAGE_ONLY" "$CAPACITY_RETRIES" "$CAPACITY_WARMUP_SECONDS" "$CAPACITY_RETRY_INTERVAL_SECONDS" "$CAPACITY_TIMER" "$CAPACITY_SERVICE" "$CAPACITY_ARGS" "$RELEASE_RETAIN_COUNT"
local deploy_status=0
ssh "$HOST" "$remote_cmd" <<'REMOTE' || deploy_status=$?
set -euo pipefail
capacity_check_args=()
if [[ -n "${CAPACITY_ARGS:-}" ]]; then
# shellcheck disable=SC2206
capacity_check_args=(${CAPACITY_ARGS})
fi
run_capacity_check() {
if [[ "${#capacity_check_args[@]}" -gt 0 ]]; then
"${REMOTE_ROOT}/current/capacity-check" -timeout 20s "${capacity_check_args[@]}"
return
fi
"${REMOTE_ROOT}/current/capacity-check" -timeout 20s
}
release_dir="${REMOTE_ROOT}/releases/${RELEASE}"
cleanup_remote_tmp() {
rm -rf "${REMOTE_TMP}" "${REMOTE_TMP}.tar.gz"
}
rm -rf "${REMOTE_TMP}"
mkdir -p "${REMOTE_TMP}" "${release_dir}"
tar -xzf "${REMOTE_TMP}.tar.gz" -C "${REMOTE_TMP}"
for bin in ${REMOTE_BINS}; do
test -f "${REMOTE_TMP}/${bin}"
file "${REMOTE_TMP}/${bin}" | grep -Eq 'ELF 64-bit.*x86-64'
done
cp "${REMOTE_TMP}"/* "${release_dir}/"
for bin in ${REMOTE_BINS}; do
chmod 0755 "${release_dir}/${bin}"
done
if [[ "${STAGE_ONLY}" == "1" ]]; then
cleanup_remote_tmp
echo "staged release ${RELEASE} at ${release_dir}"
exit 0
fi
timer_was_active=0
timer_exists=0
if [[ -n "${CAPACITY_TIMER}" ]] && systemctl list-unit-files "${CAPACITY_TIMER}" --no-legend 2>/dev/null | grep -q "${CAPACITY_TIMER}"; then
timer_exists=1
if systemctl is-active --quiet "${CAPACITY_TIMER}"; then
timer_was_active=1
systemctl stop "${CAPACITY_TIMER}"
echo "paused ${CAPACITY_TIMER} during deploy"
fi
fi
if [[ -n "${CAPACITY_SERVICE}" ]] && systemctl list-unit-files "${CAPACITY_SERVICE}" --no-legend 2>/dev/null | grep -q "${CAPACITY_SERVICE}"; then
systemctl stop "${CAPACITY_SERVICE}" 2>/dev/null || true
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
fi
restore_capacity_timer() {
if [[ -n "${CAPACITY_SERVICE}" ]]; then
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
fi
if [[ "${timer_exists}" == "1" && "${timer_was_active}" == "1" ]]; then
if [[ -n "${CAPACITY_SERVICE}" ]]; then
if systemctl start "${CAPACITY_SERVICE}" 2>/dev/null; then
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
echo "verified ${CAPACITY_SERVICE} after deploy"
fi
fi
systemctl start "${CAPACITY_TIMER}" 2>/dev/null || true
echo "resumed ${CAPACITY_TIMER}"
if [[ -n "${CAPACITY_SERVICE}" ]]; then
for _ in 1 2 3 4 5 6 7 8 9 10; do
if ! systemctl is-active --quiet "${CAPACITY_SERVICE}"; then
break
fi
sleep 1
done
if systemctl is-failed --quiet "${CAPACITY_SERVICE}" && run_capacity_check >/dev/null 2>&1; then
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
echo "cleared transient ${CAPACITY_SERVICE} failure after timer resume"
fi
fi
fi
}
cleanup_old_releases() {
local current_dir
current_dir="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
local kept=0
while IFS= read -r release_path; do
[[ -n "${release_path}" ]] || continue
if [[ "${release_path}" == "${current_dir}" || "${kept}" -lt "${RELEASE_RETAIN_COUNT}" ]]; then
kept=$((kept + 1))
continue
fi
rm -rf -- "${release_path}"
done < <(find "${REMOTE_ROOT}/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-)
}
previous_release="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
release_switched=0
deploy_succeeded=0
on_exit() {
local status=$?
trap - EXIT
set +e
if [[ "${release_switched}" == "1" && "${deploy_succeeded}" != "1" && -n "${previous_release}" && -d "${previous_release}" ]]; then
echo "deployment failed; rolling back to ${previous_release}" >&2
ln -sfn "${previous_release}" "${REMOTE_ROOT}/current"
for service in ${SERVICES}; do
systemctl restart "${service}"
done
for service in ${SERVICES}; do
systemctl is-active "${service}"
done
fi
cleanup_remote_tmp
restore_capacity_timer
exit "${status}"
}
trap on_exit EXIT
ln -sfn "${release_dir}" "${REMOTE_ROOT}/current"
release_switched=1
for service in ${SERVICES}; do
systemctl restart "${service}"
done
for service in ${SERVICES}; do
systemctl is-active "${service}"
done
for port in ${READYZ_PORTS}; do
ok=0
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
if curl -fsS "http://127.0.0.1:${port}/readyz"; then
echo
ok=1
break
fi
sleep 2
done
if [[ "${ok}" != "1" ]]; then
echo "readyz failed for port ${port}" >&2
exit 1
fi
done
if [[ "${CAPACITY_WARMUP_SECONDS}" != "0" ]]; then
echo "waiting ${CAPACITY_WARMUP_SECONDS}s before capacity-check warmup"
sleep "${CAPACITY_WARMUP_SECONDS}"
fi
capacity_ok=0
for attempt in $(seq 1 "${CAPACITY_RETRIES}"); do
echo "capacity-check attempt ${attempt}/${CAPACITY_RETRIES}"
if run_capacity_check; then
capacity_ok=1
break
fi
if [[ "${attempt}" != "${CAPACITY_RETRIES}" && "${CAPACITY_RETRY_INTERVAL_SECONDS}" != "0" ]]; then
sleep "${CAPACITY_RETRY_INTERVAL_SECONDS}"
fi
done
if [[ "${capacity_ok}" != "1" ]]; then
exit 1
fi
deploy_succeeded=1
cleanup_old_releases
REMOTE
rm -f "$archive"
if [[ "$deploy_status" -ne 0 ]]; then
return "$deploy_status"
fi
if [[ "$STAGE_ONLY" -eq 1 ]]; then
echo "staged release ${RELEASE} at ${HOST}:${REMOTE_ROOT}/releases/${RELEASE}"
else
echo "deployed release ${RELEASE} to ${HOST}:${REMOTE_ROOT}"
fi
}
if [[ "$BUILD_ONLY" -eq 0 ]]; then
preflight_remote_ssh
fi
build_release
if [[ "$BUILD_ONLY" -eq 1 ]]; then
exit 0
fi
deploy_release

99
go/vehicle-gateway/scripts/install-fields-projector-service.sh Executable file
View File

@@ -0,0 +1,99 @@
#!/usr/bin/env bash
set -euo pipefail
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
usage() {
cat <<'USAGE'
Usage:
scripts/install-fields-projector-service.sh --host root@115.29.187.205 [--remote-root DIR]
Prepares the Kafka RAW-to-fields projector service. It does not disable the
legacy bridge projection; switch BRIDGE_DERIVE_FIELDS_FROM_RAW_ENABLED only
after the new consumer has started and its offsets are established.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--host)
HOST="${2:-}"
shift 2
;;
--remote-root)
REMOTE_ROOT="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$HOST" ]]; then
echo "--host is required" >&2
exit 2
fi
ssh "$HOST" "REMOTE_ROOT='$REMOTE_ROOT' bash -s" <<'REMOTE'
set -euo pipefail
env_dir="${REMOTE_ROOT}/env"
projector_env="${env_dir}/fields-projector.env"
unit="/etc/systemd/system/lingniu-go-fields-projector.service"
mkdir -p "${env_dir}"
cat >"${projector_env}" <<'ENV'
FIELDS_PROJECTOR_GROUP_PREFIX=vehicle-fields-projector-v1
KAFKA_START_OFFSET=last
FIELDS_PROJECTOR_WORKERS_PER_PROTOCOL=1
FIELDS_PROJECTOR_BATCH_SIZE=500
FIELDS_PROJECTOR_BATCH_WAIT_MS=20
FIELDS_PROJECTOR_OPERATION_TIMEOUT_MS=30000
FIELDS_PROJECTOR_RETRY_DELAY_MS=500
HEALTH_ADDR=127.0.0.1:20218
ENV
chmod 0640 "${projector_env}"
cat >"${unit}" <<UNIT
[Unit]
Description=Lingniu Go RAW Fields Projector
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=${REMOTE_ROOT}/current
EnvironmentFile=${env_dir}/base.env
EnvironmentFile=${projector_env}
Environment=GOMEMLIMIT=384MiB
ExecStart=${REMOTE_ROOT}/current/fields-projector
Restart=always
RestartSec=3
LimitNOFILE=1048576
MemoryLimit=512M
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable lingniu-go-fields-projector.service
if [[ -x "${REMOTE_ROOT}/current/fields-projector" ]]; then
systemd-analyze verify "${unit}"
systemctl restart lingniu-go-fields-projector.service
systemctl is-active lingniu-go-fields-projector.service
else
echo "fields-projector binary is not in current release yet; service prepared for the next deploy"
fi
echo "fields projector service prepared; bridge projection remains unchanged"
REMOTE

169
go/vehicle-gateway/scripts/install-identity-source-sync-timer.sh Executable file
View File

@@ -0,0 +1,169 @@
#!/usr/bin/env bash
set -euo pipefail
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
ON_CALENDAR="*-*-* *:*:00"
TIMEOUT="2m"
APPLY="true"
PRUNE_UNMANAGED="false"
PRUNE_MIN_AGE="1h"
RETIRE_STALE_UNMANAGED="false"
RETIRE_MIN_AGE="8h"
usage() {
cat <<'USAGE'
Usage:
scripts/install-identity-source-sync-timer.sh --host root@115.29.187.205 [options]
Options:
--host HOST SSH target for ECS installation.
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
--on-calendar SPEC systemd OnCalendar value. Defaults to "*-*-* *:*:00".
--timeout DURATION identity-import timeout. Defaults to 2m.
--dry-run Install timer without -apply.
--prune-unmanaged Also prune stale unmanaged vehicle_data_source rows.
--prune-min-age DUR Minimum age for --prune-unmanaged. Defaults to 1h.
--retire-stale-unmanaged
Disable stale unmanaged vehicle_data_source rows that may be referenced by historical mileage.
--retire-min-age DUR
Minimum age for --retire-stale-unmanaged. Defaults to 8h.
-h, --help Show this help.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--host)
HOST="${2:-}"
shift 2
;;
--remote-root)
REMOTE_ROOT="${2:-}"
shift 2
;;
--on-calendar)
ON_CALENDAR="${2:-}"
shift 2
;;
--timeout)
TIMEOUT="${2:-}"
shift 2
;;
--dry-run)
APPLY="false"
shift
;;
--prune-unmanaged)
PRUNE_UNMANAGED="true"
shift
;;
--prune-min-age)
PRUNE_MIN_AGE="${2:-}"
shift 2
;;
--retire-stale-unmanaged)
RETIRE_STALE_UNMANAGED="true"
shift
;;
--retire-min-age)
RETIRE_MIN_AGE="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$HOST" ]]; then
echo "--host is required" >&2
usage >&2
exit 2
fi
if [[ -z "$ON_CALENDAR" ]]; then
echo "--on-calendar must not be empty" >&2
exit 2
fi
if [[ -z "$TIMEOUT" ]]; then
echo "--timeout must not be empty" >&2
exit 2
fi
if [[ -z "$PRUNE_MIN_AGE" ]]; then
echo "--prune-min-age must not be empty" >&2
exit 2
fi
if [[ -z "$RETIRE_MIN_AGE" ]]; then
echo "--retire-min-age must not be empty" >&2
exit 2
fi
remote_cmd=""
printf -v remote_cmd \
"REMOTE_ROOT=%q ON_CALENDAR=%q TIMEOUT=%q APPLY=%q PRUNE_UNMANAGED=%q PRUNE_MIN_AGE=%q RETIRE_STALE_UNMANAGED=%q RETIRE_MIN_AGE=%q bash -s" \
"$REMOTE_ROOT" "$ON_CALENDAR" "$TIMEOUT" "$APPLY" "$PRUNE_UNMANAGED" "$PRUNE_MIN_AGE" "$RETIRE_STALE_UNMANAGED" "$RETIRE_MIN_AGE"
ssh "$HOST" "$remote_cmd" <<'REMOTE'
set -euo pipefail
install -d /etc/systemd/system
apply_arg=""
if [[ "${APPLY}" == "true" ]]; then
apply_arg=" -apply"
fi
prune_arg=""
if [[ "${PRUNE_UNMANAGED}" == "true" ]]; then
prune_arg=" -prune-unmanaged-data-sources -prune-data-source-min-age=\"${PRUNE_MIN_AGE}\""
fi
retire_arg=""
if [[ "${RETIRE_STALE_UNMANAGED}" == "true" ]]; then
retire_arg=" -retire-stale-unmanaged-data-sources -retire-data-source-min-age=\"${RETIRE_MIN_AGE}\""
fi
cat >/etc/systemd/system/lingniu-go-identity-source-sync.service <<SERVICE
[Unit]
Description=Lingniu Go Identity Source Sync
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=${REMOTE_ROOT}/current
EnvironmentFile=${REMOTE_ROOT}/env/base.env
ExecStart=/bin/sh -c 'exec "${REMOTE_ROOT}/current/identity-import" -sync-data-sources${prune_arg}${retire_arg}${apply_arg} -timeout="${TIMEOUT}"'
TimeoutStartSec=5min
StandardOutput=journal
StandardError=journal
SERVICE
cat >/etc/systemd/system/lingniu-go-identity-source-sync.timer <<TIMER
[Unit]
Description=Run Lingniu Go Identity Source Sync
[Timer]
OnCalendar=${ON_CALENDAR}
AccuracySec=10s
RandomizedDelaySec=10s
Persistent=false
Unit=lingniu-go-identity-source-sync.service
[Install]
WantedBy=timers.target
TIMER
systemctl daemon-reload
systemd-analyze verify /etc/systemd/system/lingniu-go-identity-source-sync.service /etc/systemd/system/lingniu-go-identity-source-sync.timer
systemctl reset-failed lingniu-go-identity-source-sync.service 2>/dev/null || true
systemctl enable --now lingniu-go-identity-source-sync.timer
systemctl start lingniu-go-identity-source-sync.service
systemctl reset-failed lingniu-go-identity-source-sync.service 2>/dev/null || true
systemctl list-timers lingniu-go-identity-source-sync.timer --no-pager
systemctl show lingniu-go-identity-source-sync.service -p Result -p ExecMainStatus -p ActiveState -p SubState --no-pager
REMOTE

113
go/vehicle-gateway/scripts/install-identity-writer-service.sh Executable file
View File

@@ -0,0 +1,113 @@
#!/usr/bin/env bash
set -euo pipefail
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
usage() {
cat <<'USAGE'
Usage:
scripts/install-identity-writer-service.sh --host root@115.29.187.205 [--remote-root DIR]
Prepares the native identity-writer service and switches Gateway registration
persistence to Kafka delegation. The existing Gateway binary ignores the new
environment switch, so this is safe to run before staging the first release
that contains identity-writer.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--host)
HOST="${2:-}"
shift 2
;;
--remote-root)
REMOTE_ROOT="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$HOST" ]]; then
echo "--host is required" >&2
exit 2
fi
ssh "$HOST" "REMOTE_ROOT='$REMOTE_ROOT' bash -s" <<'REMOTE'
set -euo pipefail
env_dir="${REMOTE_ROOT}/env"
gateway_env="${env_dir}/gateway.env"
identity_env="${env_dir}/identity-writer.env"
unit="/etc/systemd/system/lingniu-go-identity-writer.service"
mkdir -p "${env_dir}"
cat >"${identity_env}" <<'ENV'
KAFKA_GROUP=go-identity-writer
KAFKA_TOPIC=vehicle.raw.go.jt808.v1
KAFKA_START_OFFSET=last
HEALTH_ADDR=127.0.0.1:20217
MYSQL_ENSURE_SCHEMA=true
MYSQL_MAX_OPEN_CONNS=8
MYSQL_MAX_IDLE_CONNS=4
MYSQL_CONN_MAX_LIFETIME_SECONDS=300
IDENTITY_WRITER_BATCH_SIZE=500
IDENTITY_WRITER_BATCH_WAIT_MS=20
IDENTITY_WRITER_RETRY_DELAY_MS=1000
JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS=600
LOCAL_TZ=Asia/Shanghai
ENV
chmod 0640 "${identity_env}"
touch "${gateway_env}"
if grep -q '^JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=' "${gateway_env}"; then
sed -i 's/^JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=.*/JT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false/' "${gateway_env}"
else
printf '\nJT808_REGISTRATION_GATEWAY_WRITES_ENABLED=false\n' >>"${gateway_env}"
fi
cat >"${unit}" <<UNIT
[Unit]
Description=Lingniu Go JT808 Identity Writer
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
WorkingDirectory=${REMOTE_ROOT}/current
EnvironmentFile=${env_dir}/base.env
EnvironmentFile=${identity_env}
Environment=GOMEMLIMIT=192MiB
ExecStart=${REMOTE_ROOT}/current/identity-writer
Restart=always
RestartSec=3
LimitNOFILE=1048576
MemoryLimit=256M
KillSignal=SIGTERM
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable lingniu-go-identity-writer.service
if [[ -x "${REMOTE_ROOT}/current/identity-writer" ]]; then
systemd-analyze verify "${unit}"
systemctl restart lingniu-go-identity-writer.service
systemctl is-active lingniu-go-identity-writer.service
else
echo "identity-writer binary is not in current release yet; service prepared for the next deploy"
fi
echo "identity writer service prepared; gateway MySQL registration writes delegated"
REMOTE

139
go/vehicle-gateway/scripts/install-stats-backfill-timer.sh Executable file
View File

@@ -0,0 +1,139 @@
#!/usr/bin/env bash
set -euo pipefail
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
DAYS_BACK=1
WINDOW_DAYS=3
PROTOCOLS="GB32960,JT808,YUTONG_MQTT"
ON_CALENDAR="*-*-* 01:30:00"
DRY_RUN="false"
usage() {
cat <<'USAGE'
Usage:
scripts/install-stats-backfill-timer.sh --host root@115.29.187.205 [options]
Options:
--host HOST SSH target for ECS installation.
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
--days-back N Target end date relative to local day. Defaults to 1, meaning yesterday.
--window-days N Number of days to recalculate ending at days-back. Defaults to 3.
--protocols LIST Comma-separated protocols. Defaults to GB32960,JT808,YUTONG_MQTT.
--on-calendar SPEC systemd OnCalendar value. Defaults to "*-*-* 01:30:00".
--dry-run Install timer in dry-run mode.
-h, --help Show this help.
USAGE
}
while [[ $# -gt 0 ]]; do
case "$1" in
--host)
HOST="${2:-}"
shift 2
;;
--remote-root)
REMOTE_ROOT="${2:-}"
shift 2
;;
--days-back)
DAYS_BACK="${2:-}"
shift 2
;;
--window-days)
WINDOW_DAYS="${2:-}"
shift 2
;;
--protocols)
PROTOCOLS="${2:-}"
shift 2
;;
--on-calendar)
ON_CALENDAR="${2:-}"
shift 2
;;
--dry-run)
DRY_RUN="true"
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$HOST" ]]; then
echo "--host is required" >&2
usage >&2
exit 2
fi
case "$DAYS_BACK" in
''|*[!0-9]*)
echo "--days-back must be a non-negative integer" >&2
exit 2
;;
esac
case "$WINDOW_DAYS" in
''|*[!0-9]*|0)
echo "--window-days must be a positive integer" >&2
exit 2
;;
esac
remote_cmd=""
printf -v remote_cmd \
"REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \
"$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN"
ssh "$HOST" "$remote_cmd" <<'REMOTE'
set -euo pipefail
install -d /etc/systemd/system
cat >/etc/systemd/system/lingniu-go-stats-backfill.service <<SERVICE
[Unit]
Description=Lingniu Go Stats Backfill
Documentation=file:${REMOTE_ROOT}/current/docs/ops/vehicle-ingest-runbook.md
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
WorkingDirectory=${REMOTE_ROOT}/current
Environment=BACKFILL_METHOD=last_diff
Environment=BACKFILL_DRY_RUN=${DRY_RUN}
Environment=BACKFILL_DAYS_BACK=${DAYS_BACK}
Environment=BACKFILL_WINDOW_DAYS=${WINDOW_DAYS}
Environment=BACKFILL_PROTOCOLS=${PROTOCOLS}
ExecStart=${REMOTE_ROOT}/current/stats-backfill
TimeoutStartSec=45min
StandardOutput=journal
StandardError=journal
SERVICE
cat >/etc/systemd/system/lingniu-go-stats-backfill.timer <<TIMER
[Unit]
Description=Run Lingniu Go Stats Backfill daily
[Timer]
OnCalendar=${ON_CALENDAR}
AccuracySec=1min
Persistent=false
Unit=lingniu-go-stats-backfill.service
[Install]
WantedBy=timers.target
TIMER
systemctl daemon-reload
systemd-analyze verify /etc/systemd/system/lingniu-go-stats-backfill.service /etc/systemd/system/lingniu-go-stats-backfill.timer
systemctl reset-failed lingniu-go-stats-backfill.service 2>/dev/null || true
systemctl enable --now lingniu-go-stats-backfill.timer
systemctl list-timers lingniu-go-stats-backfill.timer --no-pager
REMOTE