ops(web): bound immutable release history

2026-07-16 04:18:15 +08:00
parent 8e8f3f9f5c
commit 326d03cd6d
10 changed files with 392 additions and 3 deletions
--- a/vehicle-data-platform/deploy/install-web-release.sh
+++ b/vehicle-data-platform/deploy/install-web-release.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RELEASE_ID=${1:?release id is required}
+ARCHIVE=${2:?web archive is required}
+ROOT=${PLATFORM_ROOT:-/opt/lingniu-vehicle-platform}
+BASE_URL=${PLATFORM_BASE_URL:-http://127.0.0.1:20300}
+SERVICE=${PLATFORM_SERVICE:-lingniu-vehicle-platform}
+COMPATIBILITY_GENERATIONS=${COMPATIBILITY_GENERATIONS:-3}
+RELEASE_HISTORY_LIMIT=${RELEASE_HISTORY_LIMIT:-20}
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+SYSTEMCTL_BIN=${SYSTEMCTL_BIN:-systemctl}
+CURL_BIN=${CURL_BIN:-curl}
+
+case "$RELEASE_ID" in
+  ''|*[!A-Za-z0-9._-]*) printf 'invalid release id: %s\n' "$RELEASE_ID" >&2; exit 1 ;;
+esac
+test -f "$ARCHIVE" || { printf 'web archive is missing: %s\n' "$ARCHIVE" >&2; exit 1; }
+test -x "$SCRIPT_DIR/prepare-web-release-tree.sh" || { printf 'release tree helper is missing\n' >&2; exit 1; }
+test -f "$SCRIPT_DIR/prune-release-history.py" || { printf 'release pruning helper is missing\n' >&2; exit 1; }
+
+old=$(readlink -f "$ROOT/current")
+next="$ROOT/releases/$RELEASE_ID"
+env_file="$ROOT/env/platform.env"
+old_release=$(sed -n 's/^PLATFORM_RELEASE=//p' "$env_file" | tail -1)
+switched=false
+
+set_release_env() {
+  python3 - "$env_file" "$1" <<'PY'
+from __future__ import print_function
+import io
+import sys
+
+path, release = sys.argv[1], sys.argv[2]
+with io.open(path, 'r', encoding='utf-8') as handle:
+    lines = handle.read().splitlines()
+updated = []
+found = False
+for line in lines:
+    if line.startswith('PLATFORM_RELEASE='):
+        updated.append('PLATFORM_RELEASE=' + release)
+        found = True
+    else:
+        updated.append(line)
+if not found:
+    updated.append('PLATFORM_RELEASE=' + release)
+with io.open(path, 'w', encoding='utf-8') as handle:
+    handle.write('\n'.join(updated) + '\n')
+PY
+}
+
+rollback() {
+  local status=$?
+  trap - ERR
+  set +e
+  if test "$switched" = true; then
+    ln -s "$old" "$ROOT/current.rollback"
+    python3 -c 'import os,sys; os.replace(sys.argv[1], sys.argv[2])' "$ROOT/current.rollback" "$ROOT/current"
+    set_release_env "$old_release"
+    "$SYSTEMCTL_BIN" restart "$SERVICE"
+  fi
+  printf 'web release install failed; current restored to %s\n' "$old" >&2
+  exit "$status"
+}
+trap rollback ERR
+
+test ! -e "$next" || { printf 'release already exists: %s\n' "$next" >&2; exit 1; }
+mkdir -p "$next/web"
+cp "$old/platform-api" "$next/platform-api"
+cp "$old/lingniu-vehicle-platform.service" "$next/lingniu-vehicle-platform.service"
+cp -a "$old/deploy" "$next/deploy"
+cp "$SCRIPT_DIR/prepare-web-release-tree.sh" "$next/deploy/prepare-web-release-tree.sh"
+cp "$SCRIPT_DIR/prune-release-history.py" "$next/deploy/prune-release-history.py"
+cp "$0" "$next/deploy/install-web-release.sh"
+chmod +x "$next/platform-api" "$next/deploy/"*.sh "$next/deploy/prune-release-history.py"
+
+while IFS= read -r member; do
+  case "$member" in
+    /*|../*|*/../*|*/..) printf 'web archive contains unsafe path: %s\n' "$member" >&2; exit 1 ;;
+  esac
+done < <(tar -tzf "$ARCHIVE")
+tar --no-same-owner -xzf "$ARCHIVE" -C "$next/web"
+find "$next/web" -type f -name '._*' -delete
+"$next/deploy/prepare-web-release-tree.sh" "$next/web" "$old/web" "$COMPATIBILITY_GENERATIONS"
+
+ln -s "$next" "$ROOT/current.next"
+python3 -c 'import os,sys; os.replace(sys.argv[1], sys.argv[2])' "$ROOT/current.next" "$ROOT/current"
+switched=true
+set_release_env "$RELEASE_ID"
+"$SYSTEMCTL_BIN" restart "$SERVICE"
+
+ready=false
+for _ in $(seq 1 15); do
+  if "$SYSTEMCTL_BIN" is-active --quiet "$SERVICE" && "$CURL_BIN" -fsS "$BASE_URL/" >/dev/null; then ready=true; break; fi
+  sleep 1
+done
+test "$ready" = true || { printf 'service did not become ready\n' >&2; exit 1; }
+verify_bin=${VERIFY_WEB_RELEASE_BIN:-$next/deploy/verify-web-release.sh}
+"$verify_bin" "$next/web" "$BASE_URL" "$next/web/.compatibility-assets"
+
+if test -f "$ROOT/env/access-tokens.env"; then
+  set -a
+  # shellcheck disable=SC1090
+  . "$ROOT/env/access-tokens.env"
+  set +a
+  test -n "${VIEWER_TOKEN:-}" || { printf 'viewer token is unavailable for health verification\n' >&2; exit 1; }
+  health=$("$CURL_BIN" -fsS -H "Authorization: Bearer $VIEWER_TOKEN" "$BASE_URL/api/ops/health")
+  printf '%s' "$health" | python3 -c 'import json,sys; value=(((json.load(sys.stdin).get("data") or {}).get("runtime") or {}).get("platformRelease")); sys.exit(0 if value == sys.argv[1] else 1)' "$RELEASE_ID"
+fi
+
+prune_bin=${PRUNE_RELEASE_BIN:-$next/deploy/prune-release-history.py}
+python3 "$prune_bin" "$ROOT" "$RELEASE_HISTORY_LIMIT" "$next" "$old"
+rm -f "$ARCHIVE"
+switched=false
+trap - ERR
+printf 'web_release_install=ok release=%s previous=%s\n' "$RELEASE_ID" "$(basename "$old")"
--- a/vehicle-data-platform/deploy/install-web-release.test.sh
+++ b/vehicle-data-platform/deploy/install-web-release.test.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+fixture=$(mktemp -d)
+trap 'rm -rf "$fixture"' EXIT
+root="$fixture/platform"
+old="$root/releases/old-release"
+mkdir -p "$old/web/assets" "$old/deploy" "$root/env" "$fixture/bin"
+printf '#!/usr/bin/env bash\nexit 0\n' > "$old/platform-api"
+chmod +x "$old/platform-api"
+printf '[Service]\n' > "$old/lingniu-vehicle-platform.service"
+cp "$SCRIPT_DIR/verify-web-release.sh" "$old/deploy/verify-web-release.sh"
+printf 'old.js\n' > "$old/web/.release-assets"
+printf 'old\n' > "$old/web/assets/old.js"
+ln -s "$old" "$root/current"
+printf 'PLATFORM_RELEASE=old-release\n' > "$root/env/platform.env"
+
+for index in 1 2 3 4; do mkdir -p "$root/releases/stale-$index"; touch -t "20260716010${index}" "$root/releases/stale-$index"; done
+
+new_web="$fixture/new-web"
+mkdir -p "$new_web/assets"
+printf 'new.js\n' > "$new_web/.release-assets"
+printf 'new\n' > "$new_web/assets/new.js"
+printf '<main>new</main>\n' > "$new_web/index.html"
+printf 'window.__LINGNIU_APP_CONFIG__={"amapSecurityServiceHost":"/_AMapService"};\n' > "$new_web/app-config.js"
+tar -C "$new_web" -czf "$fixture/new.tar.gz" .
+
+printf '#!/usr/bin/env bash\nexit 0\n' > "$fixture/bin/systemctl"
+printf '#!/usr/bin/env bash\nexit 0\n' > "$fixture/bin/curl"
+printf '#!/usr/bin/env bash\nprintf "mock_verify=ok\\n"\n' > "$fixture/bin/verify"
+chmod +x "$fixture/bin/"*
+
+PLATFORM_ROOT="$root" SYSTEMCTL_BIN="$fixture/bin/systemctl" CURL_BIN="$fixture/bin/curl" VERIFY_WEB_RELEASE_BIN="$fixture/bin/verify" RELEASE_HISTORY_LIMIT=3 "$SCRIPT_DIR/install-web-release.sh" new-release "$fixture/new.tar.gz" > "$fixture/install.out"
+test "$(readlink -f "$root/current")" = "$(cd "$root/releases/new-release" && pwd -P)"
+grep -q '^PLATFORM_RELEASE=new-release$' "$root/env/platform.env"
+test -f "$root/current/web/assets/new.js"
+test -f "$root/current/web/assets/old.js"
+test -f "$root/current/web/.compatibility-manifests/1.assets"
+test "$(find "$root/releases" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" = 3
+grep -q 'web_release_install=ok release=new-release previous=old-release' "$fixture/install.out"
+
+rollback_web="$fixture/rollback-web"
+mkdir -p "$rollback_web/assets"
+printf 'rollback.js\n' > "$rollback_web/.release-assets"
+printf 'rollback\n' > "$rollback_web/assets/rollback.js"
+printf '<main>rollback</main>\n' > "$rollback_web/index.html"
+printf 'window.__LINGNIU_APP_CONFIG__={"amapSecurityServiceHost":"/_AMapService"};\n' > "$rollback_web/app-config.js"
+tar -C "$rollback_web" -czf "$fixture/rollback.tar.gz" .
+printf '#!/usr/bin/env bash\nexit 1\n' > "$fixture/bin/fail-verify"
+chmod +x "$fixture/bin/fail-verify"
+if PLATFORM_ROOT="$root" SYSTEMCTL_BIN="$fixture/bin/systemctl" CURL_BIN="$fixture/bin/curl" VERIFY_WEB_RELEASE_BIN="$fixture/bin/fail-verify" RELEASE_HISTORY_LIMIT=3 "$SCRIPT_DIR/install-web-release.sh" broken-release "$fixture/rollback.tar.gz" > "$fixture/rollback.out" 2>&1; then
+  printf 'expected failed verification to roll back\n' >&2
+  exit 1
+fi
+test "$(readlink -f "$root/current")" = "$(cd "$root/releases/new-release" && pwd -P)"
+grep -q '^PLATFORM_RELEASE=new-release$' "$root/env/platform.env"
+grep -q 'current restored' "$fixture/rollback.out"
+
+printf 'install-web-release tests passed\n'
--- a/vehicle-data-platform/deploy/prepare-web-release-tree.sh
+++ b/vehicle-data-platform/deploy/prepare-web-release-tree.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+NEW_WEB=${1:?new web root is required}
+OLD_WEB=${2:?old web root is required}
+MAX_COMPATIBILITY_GENERATIONS=${3:-3}
+
+case "$MAX_COMPATIBILITY_GENERATIONS" in
+  ''|*[!0-9]*) printf 'invalid compatibility generation limit: %s\n' "$MAX_COMPATIBILITY_GENERATIONS" >&2; exit 1 ;;
+esac
+test "$MAX_COMPATIBILITY_GENERATIONS" -ge 1 || { printf 'compatibility generation limit must be positive\n' >&2; exit 1; }
+test -f "$NEW_WEB/.release-assets" || { printf 'new release manifest is missing\n' >&2; exit 1; }
+test -f "$OLD_WEB/.release-assets" || { printf 'old release manifest is missing\n' >&2; exit 1; }
+mkdir -p "$NEW_WEB/assets" "$NEW_WEB/.compatibility-manifests"
+
+validate_asset() {
+  local asset=$1
+  case "$asset" in
+    ''|/*|*/*|.|..) printf 'unsafe release asset: %s\n' "$asset" >&2; return 1 ;;
+  esac
+}
+
+while IFS= read -r asset || test -n "$asset"; do
+  asset=${asset%$'\r'}
+  test -n "$asset" || continue
+  validate_asset "$asset"
+  test -f "$NEW_WEB/assets/$asset" || { printf 'current asset is missing: %s\n' "$NEW_WEB/assets/$asset" >&2; exit 1; }
+done < "$NEW_WEB/.release-assets"
+
+copy_manifest_assets() {
+  local manifest=$1
+  local asset source destination
+  while IFS= read -r asset || test -n "$asset"; do
+    asset=${asset%$'\r'}
+    test -n "$asset" || continue
+    validate_asset "$asset"
+    source="$OLD_WEB/assets/$asset"
+    destination="$NEW_WEB/assets/$asset"
+    test -f "$source" || { printf 'compatibility asset is missing: %s\n' "$source" >&2; return 1; }
+    if test -f "$destination"; then
+      cmp -s "$source" "$destination" || { printf 'asset hash collision has different content: %s\n' "$asset" >&2; return 1; }
+    else
+      cp "$source" "$destination"
+    fi
+  done < "$manifest"
+}
+
+cp "$OLD_WEB/.release-assets" "$NEW_WEB/.compatibility-manifests/1.assets"
+generation=2
+while test "$generation" -le "$MAX_COMPATIBILITY_GENERATIONS"; do
+  previous=$((generation - 1))
+  source_manifest="$OLD_WEB/.compatibility-manifests/$previous.assets"
+  test -f "$source_manifest" || break
+  cp "$source_manifest" "$NEW_WEB/.compatibility-manifests/$generation.assets"
+  generation=$((generation + 1))
+done
+
+compatibility_manifest="$NEW_WEB/.compatibility-assets"
+: > "$compatibility_manifest"
+for manifest in "$NEW_WEB"/.compatibility-manifests/*.assets; do
+  test -f "$manifest" || continue
+  copy_manifest_assets "$manifest"
+  cat "$manifest" >> "$compatibility_manifest"
+done
+LC_ALL=C sort -u "$compatibility_manifest" -o "$compatibility_manifest"
+
+current_count=$(awk 'NF { count += 1 } END { print count + 0 }' "$NEW_WEB/.release-assets")
+compatibility_count=$(awk 'NF { count += 1 } END { print count + 0 }' "$compatibility_manifest")
+generation_count=$(find "$NEW_WEB/.compatibility-manifests" -maxdepth 1 -type f -name '*.assets' | wc -l | tr -d ' ')
+served_count=$(find "$NEW_WEB/assets" -maxdepth 1 -type f | wc -l | tr -d ' ')
+printf 'web_release_tree=ok current_assets=%s compatibility_assets=%s compatibility_generations=%s served_assets=%s\n' "$current_count" "$compatibility_count" "$generation_count" "$served_count"
--- a/vehicle-data-platform/deploy/prepare-web-release.test.sh
+++ b/vehicle-data-platform/deploy/prepare-web-release.test.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+fixture=$(mktemp -d)
+trap 'rm -rf "$fixture"' EXIT
+
+old="$fixture/old"
+new="$fixture/new"
+mkdir -p "$old/assets" "$old/.compatibility-manifests" "$new/assets"
+printf 'old-current.js\nshared.js\n' > "$old/.release-assets"
+printf 'older-one.js\n' > "$old/.compatibility-manifests/1.assets"
+printf 'too-old.js\n' > "$old/.compatibility-manifests/2.assets"
+for asset in old-current.js shared.js older-one.js too-old.js orphan.js; do printf '%s\n' "$asset" > "$old/assets/$asset"; done
+printf 'new-current.js\nshared.js\n' > "$new/.release-assets"
+printf 'new-current.js\n' > "$new/assets/new-current.js"
+cp "$old/assets/shared.js" "$new/assets/shared.js"
+
+output=$("$SCRIPT_DIR/prepare-web-release-tree.sh" "$new" "$old" 2)
+test "$output" = 'web_release_tree=ok current_assets=2 compatibility_assets=3 compatibility_generations=2 served_assets=4'
+test -f "$new/assets/old-current.js"
+test -f "$new/assets/older-one.js"
+test ! -e "$new/assets/too-old.js"
+test ! -e "$new/assets/orphan.js"
+test "$(cat "$new/.compatibility-manifests/1.assets")" = "$(cat "$old/.release-assets")"
+test "$(cat "$new/.compatibility-manifests/2.assets")" = "$(cat "$old/.compatibility-manifests/1.assets")"
+
+unsafe="$fixture/unsafe"
+cp -R "$new" "$unsafe"
+printf '../secret.js\n' > "$old/.release-assets"
+if "$SCRIPT_DIR/prepare-web-release-tree.sh" "$unsafe" "$old" 2 > "$fixture/unsafe.out" 2>&1; then
+  printf 'expected an unsafe asset to fail\n' >&2
+  exit 1
+fi
+grep -q 'unsafe release asset' "$fixture/unsafe.out"
+
+printf 'prepare-web-release tests passed\n'
--- a/vehicle-data-platform/deploy/prune-release-history.py
+++ b/vehicle-data-platform/deploy/prune-release-history.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import argparse
+import os
+import shutil
+from pathlib import Path
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Keep a bounded number of immutable platform releases.')
+    parser.add_argument('root')
+    parser.add_argument('limit', type=int)
+    parser.add_argument('protected', nargs='*')
+    args = parser.parse_args()
+
+    root = Path(args.root).resolve()
+    releases = (root / 'releases').resolve()
+    if root == Path('/') or releases.parent != root or not releases.is_dir():
+        raise SystemExit('unsafe or missing release root: {}'.format(releases))
+    if args.limit < 2:
+        raise SystemExit('release history limit must be at least 2')
+
+    protected = set()
+    for value in args.protected:
+        path = Path(value).resolve()
+        if path.parent != releases or not path.is_dir():
+            raise SystemExit('protected release is outside the release root: {}'.format(path))
+        protected.add(path)
+
+    candidates = [path.resolve() for path in releases.iterdir() if path.is_dir() and not path.is_symlink()]
+    candidates.sort(key=lambda path: path.stat().st_mtime, reverse=True)
+    keep = set(protected)
+    for path in candidates:
+        if len(keep) >= args.limit:
+            break
+        keep.add(path)
+
+    removed = 0
+    for path in candidates:
+        if path in keep:
+            continue
+        if path.parent != releases:
+            raise SystemExit('refusing to remove a nested or escaped path: {}'.format(path))
+        shutil.rmtree(str(path))
+        removed += 1
+
+    print('release_prune=ok before={} kept={} removed={}'.format(len(candidates), len(keep), removed))
+
+
+if __name__ == '__main__':
+    main()
--- a/vehicle-data-platform/deploy/prune-release-history.test.sh
+++ b/vehicle-data-platform/deploy/prune-release-history.test.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+fixture=$(mktemp -d)
+trap 'rm -rf "$fixture"' EXIT
+mkdir -p "$fixture/releases"
+for index in 1 2 3 4 5 6; do
+  mkdir "$fixture/releases/r$index"
+  touch -t "20260716010${index}" "$fixture/releases/r$index"
+done
+
+output=$(python3 "$SCRIPT_DIR/prune-release-history.py" "$fixture" 3 "$fixture/releases/r1")
+test "$output" = 'release_prune=ok before=6 kept=3 removed=3'
+test -d "$fixture/releases/r1"
+test -d "$fixture/releases/r6"
+test -d "$fixture/releases/r5"
+test ! -e "$fixture/releases/r2"
+test ! -e "$fixture/releases/r3"
+test ! -e "$fixture/releases/r4"
+
+if python3 "$SCRIPT_DIR/prune-release-history.py" / 3 > "$fixture/unsafe.out" 2>&1; then
+  printf 'expected an unsafe root to fail\n' >&2
+  exit 1
+fi
+grep -q 'unsafe or missing release root' "$fixture/unsafe.out"
+
+printf 'prune-release-history tests passed\n'