lingniu-vehicle-ingest/go/vehicle-gateway/scripts/deploy-ecs-release.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RELEASE="go-native-$(date +%Y%m%d%H%M%S)"
OUT_DIR=""
HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
BUILD_ONLY=0
STAGE_ONLY=0
CAPACITY_RETRIES=10
CAPACITY_WARMUP_SECONDS=60
CAPACITY_RETRY_INTERVAL_SECONDS=15
CAPACITY_TIMER="lingniu-go-capacity-check.timer"
CAPACITY_SERVICE="lingniu-go-capacity-check.service"
CAPACITY_ARGS="${CAPACITY_ARGS:-}"
RELEASE_RETAIN_COUNT="${RELEASE_RETAIN_COUNT:-8}"

BINARIES=(
  "gateway:./cmd/gateway"
  "history-writer:./cmd/history-writer"
  "stat-writer:./cmd/stat-writer"
  "realtime-api:./cmd/realtime-api"
  "nats-fast-writer:./cmd/nats-fast-writer"
  "nats-kafka-bridge:./cmd/nats-kafka-bridge"
  "fields-projector:./cmd/fields-projector"
  "capacity-check:./cmd/capacity-check"
  "load-sim:./cmd/load-sim"
  "stats-backfill:./cmd/stats-backfill"
  "identity-import:./cmd/identity-import"
  "identity-writer:./cmd/identity-writer"
)

SERVICES=(
  "lingniu-go-nats-kafka-bridge.service"
  "lingniu-go-fields-projector.service"
  "lingniu-go-nats-fast-writer.service"
  "lingniu-go-history-writer.service"
  "lingniu-go-stat-writer.service"
  "lingniu-go-realtime-writer.service"
  "lingniu-go-identity-writer.service"
  "lingniu-go-realtime-api.service"
  "lingniu-go-gateway.service"
)

READYZ_PORTS=(20214 20218 20215 20212 20213 20216 20217 20200 20211)

usage() {
  cat <<'USAGE'
Usage:
  scripts/deploy-ecs-release.sh --build-only [--release NAME] [--out-dir DIR]
  scripts/deploy-ecs-release.sh --host root@115.29.187.205 [--release NAME] [--out-dir DIR]

Options:
  --host HOST          SSH target for ECS deployment. Omit with --build-only.
  --release NAME      Release directory name under /opt/lingniu-go-native/releases.
  --out-dir DIR       Local build output directory. Defaults to /tmp/lingniu-go-native-<release>.
  --remote-root DIR   Remote install root. Defaults to /opt/lingniu-go-native.
  --build-only        Build and verify Linux amd64 artifacts without uploading.
  --stage-only        Upload and verify release, but do not switch current or restart services.
  --capacity-retries N
                      Retry capacity-check after restart. Defaults to 10.
  --capacity-warmup-seconds N
                      Wait after readyz before capacity-check, so recent-window metrics are not dominated by restart catch-up. Defaults to 60.
  --capacity-retry-interval-seconds N
                      Wait between capacity-check retries. Defaults to 15.
  --capacity-timer UNIT
                      systemd timer paused during restart. Defaults to lingniu-go-capacity-check.timer.
  --capacity-args ARGS
                      Extra arguments passed to capacity-check during deploy verification.
                      Example: --capacity-args "-daily-mileage-diagnostics-max-pipeline=0"
  --release-retain-count N
                      Keep the newest N remote releases after a successful deployment. Defaults to 8.
  -h, --help          Show this help.
USAGE
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --host)
      HOST="${2:-}"
      shift 2
      ;;
    --release)
      RELEASE="${2:-}"
      shift 2
      ;;
    --out-dir)
      OUT_DIR="${2:-}"
      shift 2
      ;;
    --remote-root)
      REMOTE_ROOT="${2:-}"
      shift 2
      ;;
    --build-only)
      BUILD_ONLY=1
      shift
      ;;
    --stage-only|--skip-restart)
      STAGE_ONLY=1
      shift
      ;;
    --capacity-retries)
      CAPACITY_RETRIES="${2:-}"
      shift 2
      ;;
    --capacity-warmup-seconds)
      CAPACITY_WARMUP_SECONDS="${2:-}"
      shift 2
      ;;
    --capacity-retry-interval-seconds)
      CAPACITY_RETRY_INTERVAL_SECONDS="${2:-}"
      shift 2
      ;;
    --capacity-timer)
      CAPACITY_TIMER="${2:-}"
      shift 2
      ;;
    --capacity-args)
      CAPACITY_ARGS="${2:-}"
      shift 2
      ;;
    --release-retain-count)
      RELEASE_RETAIN_COUNT="${2:-}"
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "unknown argument: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
done

if [[ -z "$OUT_DIR" ]]; then
  OUT_DIR="/tmp/lingniu-go-native-${RELEASE}"
fi

if [[ "$BUILD_ONLY" -eq 0 && -z "$HOST" ]]; then
  echo "--host is required unless --build-only is set" >&2
  usage >&2
  exit 2
fi

case "$CAPACITY_RETRIES" in
  ''|*[!0-9]*|0)
    echo "--capacity-retries must be a positive integer" >&2
    exit 2
    ;;
esac
case "$CAPACITY_WARMUP_SECONDS" in
  ''|*[!0-9]*)
    echo "--capacity-warmup-seconds must be a non-negative integer" >&2
    exit 2
    ;;
esac
case "$CAPACITY_RETRY_INTERVAL_SECONDS" in
  ''|*[!0-9]*)
    echo "--capacity-retry-interval-seconds must be a non-negative integer" >&2
    exit 2
    ;;
esac
case "$RELEASE_RETAIN_COUNT" in
  ''|*[!0-9]*|0)
    echo "--release-retain-count must be a positive integer" >&2
    exit 2
    ;;
esac

assert_linux_amd64() {
  local binary="$1"
  local info
  info="$(file "$binary")"
  case "$info" in
    *"ELF 64-bit"*"x86-64"*) ;;
    *)
      echo "binary is not Linux amd64 ELF: $binary" >&2
      echo "$info" >&2
      exit 1
      ;;
  esac
}

hash_file() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$@"
  else
    shasum -a 256 "$@"
  fi
}

build_release() {
  rm -rf "$OUT_DIR"
  mkdir -p "$OUT_DIR"
  pushd "$ROOT_DIR" >/dev/null
  local names=()
  for spec in "${BINARIES[@]}"; do
    local name="${spec%%:*}"
    local pkg="${spec#*:}"
    names+=("$name")
    echo "building $name from $pkg"
    GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
      go build -trimpath -ldflags="${GO_LDFLAGS:--s -w}" -o "$OUT_DIR/$name" "$pkg"
    assert_linux_amd64 "$OUT_DIR/$name"
  done
  popd >/dev/null

  (
    cd "$OUT_DIR"
    hash_file "${names[@]}" > SHA256SUMS
    {
      echo "release=${RELEASE}"
      echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
      echo "git_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)"
      echo "go_version=$(go version)"
      echo "target=linux/amd64"
    } > RELEASE_NAME
  )
  echo "build output: $OUT_DIR"
}

preflight_remote_ssh() {
  echo "checking ssh connectivity to ${HOST}"
  if ssh -o ConnectTimeout=10 -o ConnectionAttempts=1 "$HOST" "true"; then
    return
  fi
  cat >&2 <<EOF
ssh preflight failed for ${HOST}.
The release was not built, uploaded, or switched. Restore SSH access first, then rerun this script.
EOF
  exit 255
}

deploy_release() {
  local archive="/tmp/${RELEASE}.tar.gz"
  local remote_tmp="/tmp/lingniu-go-native-${RELEASE}"
  local remote_bins=""
  local services=""
  local readyz_ports=""
  local spec
  for spec in "${BINARIES[@]}"; do
    remote_bins="${remote_bins} ${spec%%:*}"
  done
  for spec in "${SERVICES[@]}"; do
    services="${services} ${spec}"
  done
  for spec in "${READYZ_PORTS[@]}"; do
    readyz_ports="${readyz_ports} ${spec}"
  done

  COPYFILE_DISABLE=1 tar --format ustar -czf "$archive" -C "$OUT_DIR" .
  ssh "$HOST" "mkdir -p '$remote_tmp'"
  scp "$archive" "$HOST:${remote_tmp}.tar.gz"
  local remote_cmd
  printf -v remote_cmd \
    "REMOTE_ROOT=%q RELEASE=%q REMOTE_TMP=%q REMOTE_BINS=%q SERVICES=%q READYZ_PORTS=%q STAGE_ONLY=%q CAPACITY_RETRIES=%q CAPACITY_WARMUP_SECONDS=%q CAPACITY_RETRY_INTERVAL_SECONDS=%q CAPACITY_TIMER=%q CAPACITY_SERVICE=%q CAPACITY_ARGS=%q RELEASE_RETAIN_COUNT=%q bash -s" \
    "$REMOTE_ROOT" "$RELEASE" "$remote_tmp" "$remote_bins" "$services" "$readyz_ports" "$STAGE_ONLY" "$CAPACITY_RETRIES" "$CAPACITY_WARMUP_SECONDS" "$CAPACITY_RETRY_INTERVAL_SECONDS" "$CAPACITY_TIMER" "$CAPACITY_SERVICE" "$CAPACITY_ARGS" "$RELEASE_RETAIN_COUNT"
  local deploy_status=0
  ssh "$HOST" "$remote_cmd" <<'REMOTE' || deploy_status=$?
set -euo pipefail

capacity_check_args=()
if [[ -n "${CAPACITY_ARGS:-}" ]]; then
  # shellcheck disable=SC2206
  capacity_check_args=(${CAPACITY_ARGS})
fi

run_capacity_check() {
  if [[ "${#capacity_check_args[@]}" -gt 0 ]]; then
    "${REMOTE_ROOT}/current/capacity-check" -timeout 20s "${capacity_check_args[@]}"
    return
  fi
  "${REMOTE_ROOT}/current/capacity-check" -timeout 20s
}

release_dir="${REMOTE_ROOT}/releases/${RELEASE}"
cleanup_remote_tmp() {
  rm -rf "${REMOTE_TMP}" "${REMOTE_TMP}.tar.gz"
}
rm -rf "${REMOTE_TMP}"
mkdir -p "${REMOTE_TMP}" "${release_dir}"
tar -xzf "${REMOTE_TMP}.tar.gz" -C "${REMOTE_TMP}"

for bin in ${REMOTE_BINS}; do
  test -f "${REMOTE_TMP}/${bin}"
  file "${REMOTE_TMP}/${bin}" | grep -Eq 'ELF 64-bit.*x86-64'
done

cp "${REMOTE_TMP}"/* "${release_dir}/"
for bin in ${REMOTE_BINS}; do
  chmod 0755 "${release_dir}/${bin}"
done

if [[ "${STAGE_ONLY}" == "1" ]]; then
  cleanup_remote_tmp
  echo "staged release ${RELEASE} at ${release_dir}"
  exit 0
fi

timer_was_active=0
timer_exists=0
if [[ -n "${CAPACITY_TIMER}" ]] && systemctl list-unit-files "${CAPACITY_TIMER}" --no-legend 2>/dev/null | grep -q "${CAPACITY_TIMER}"; then
  timer_exists=1
  if systemctl is-active --quiet "${CAPACITY_TIMER}"; then
    timer_was_active=1
    systemctl stop "${CAPACITY_TIMER}"
    echo "paused ${CAPACITY_TIMER} during deploy"
  fi
fi
if [[ -n "${CAPACITY_SERVICE}" ]] && systemctl list-unit-files "${CAPACITY_SERVICE}" --no-legend 2>/dev/null | grep -q "${CAPACITY_SERVICE}"; then
  systemctl stop "${CAPACITY_SERVICE}" 2>/dev/null || true
  systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
fi
restore_capacity_timer() {
  if [[ -n "${CAPACITY_SERVICE}" ]]; then
    systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
  fi
  if [[ "${timer_exists}" == "1" && "${timer_was_active}" == "1" ]]; then
    if [[ -n "${CAPACITY_SERVICE}" ]]; then
      if systemctl start "${CAPACITY_SERVICE}" 2>/dev/null; then
        systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
        echo "verified ${CAPACITY_SERVICE} after deploy"
      fi
    fi
    systemctl start "${CAPACITY_TIMER}" 2>/dev/null || true
    echo "resumed ${CAPACITY_TIMER}"
    if [[ -n "${CAPACITY_SERVICE}" ]]; then
      for _ in 1 2 3 4 5 6 7 8 9 10; do
        if ! systemctl is-active --quiet "${CAPACITY_SERVICE}"; then
          break
        fi
        sleep 1
      done
      if systemctl is-failed --quiet "${CAPACITY_SERVICE}" && run_capacity_check >/dev/null 2>&1; then
        systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
        echo "cleared transient ${CAPACITY_SERVICE} failure after timer resume"
      fi
    fi
  fi
}
cleanup_old_releases() {
  local current_dir
  current_dir="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
  local kept=0
  while IFS= read -r release_path; do
    [[ -n "${release_path}" ]] || continue
    if [[ "${release_path}" == "${current_dir}" || "${kept}" -lt "${RELEASE_RETAIN_COUNT}" ]]; then
      kept=$((kept + 1))
      continue
    fi
    rm -rf -- "${release_path}"
  done < <(find "${REMOTE_ROOT}/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-)
}
previous_release="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
release_switched=0
deploy_succeeded=0
on_exit() {
  local status=$?
  trap - EXIT
  set +e
  if [[ "${release_switched}" == "1" && "${deploy_succeeded}" != "1" && -n "${previous_release}" && -d "${previous_release}" ]]; then
    echo "deployment failed; rolling back to ${previous_release}" >&2
    ln -sfn "${previous_release}" "${REMOTE_ROOT}/current"
    for service in ${SERVICES}; do
      systemctl restart "${service}"
    done
    for service in ${SERVICES}; do
      systemctl is-active "${service}"
    done
  fi
  cleanup_remote_tmp
  restore_capacity_timer
  exit "${status}"
}
trap on_exit EXIT

ln -sfn "${release_dir}" "${REMOTE_ROOT}/current"
release_switched=1

for service in ${SERVICES}; do
  systemctl restart "${service}"
done

for service in ${SERVICES}; do
  systemctl is-active "${service}"
done

for port in ${READYZ_PORTS}; do
  ok=0
  for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
    if curl -fsS "http://127.0.0.1:${port}/readyz"; then
      echo
      ok=1
      break
    fi
    sleep 2
  done
  if [[ "${ok}" != "1" ]]; then
    echo "readyz failed for port ${port}" >&2
    exit 1
  fi
done

if [[ "${CAPACITY_WARMUP_SECONDS}" != "0" ]]; then
  echo "waiting ${CAPACITY_WARMUP_SECONDS}s before capacity-check warmup"
  sleep "${CAPACITY_WARMUP_SECONDS}"
fi

capacity_ok=0
for attempt in $(seq 1 "${CAPACITY_RETRIES}"); do
  echo "capacity-check attempt ${attempt}/${CAPACITY_RETRIES}"
  if run_capacity_check; then
    capacity_ok=1
    break
  fi
  if [[ "${attempt}" != "${CAPACITY_RETRIES}" && "${CAPACITY_RETRY_INTERVAL_SECONDS}" != "0" ]]; then
    sleep "${CAPACITY_RETRY_INTERVAL_SECONDS}"
  fi
done
if [[ "${capacity_ok}" != "1" ]]; then
  exit 1
fi
deploy_succeeded=1
cleanup_old_releases
REMOTE
  rm -f "$archive"
  if [[ "$deploy_status" -ne 0 ]]; then
    return "$deploy_status"
  fi
  if [[ "$STAGE_ONLY" -eq 1 ]]; then
    echo "staged release ${RELEASE} at ${HOST}:${REMOTE_ROOT}/releases/${RELEASE}"
  else
    echo "deployed release ${RELEASE} to ${HOST}:${REMOTE_ROOT}"
  fi
}

if [[ "$BUILD_ONLY" -eq 0 ]]; then
  preflight_remote_ssh
fi
build_release
if [[ "$BUILD_ONLY" -eq 1 ]]; then
  exit 0
fi
deploy_release