#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" RELEASE="go-native-$(date +%Y%m%d%H%M%S)" OUT_DIR="" HOST="" REMOTE_ROOT="/opt/lingniu-go-native" BUILD_ONLY=0 STAGE_ONLY=0 CAPACITY_RETRIES=10 CAPACITY_WARMUP_SECONDS=60 CAPACITY_RETRY_INTERVAL_SECONDS=15 CAPACITY_TIMER="lingniu-go-capacity-check.timer" CAPACITY_SERVICE="lingniu-go-capacity-check.service" CAPACITY_ARGS="${CAPACITY_ARGS:-}" RELEASE_RETAIN_COUNT="${RELEASE_RETAIN_COUNT:-8}" BINARIES=( "gateway:./cmd/gateway" "history-writer:./cmd/history-writer" "stat-writer:./cmd/stat-writer" "realtime-api:./cmd/realtime-api" "nats-fast-writer:./cmd/nats-fast-writer" "nats-kafka-bridge:./cmd/nats-kafka-bridge" "fields-projector:./cmd/fields-projector" "capacity-check:./cmd/capacity-check" "load-sim:./cmd/load-sim" "stats-backfill:./cmd/stats-backfill" "identity-import:./cmd/identity-import" "identity-writer:./cmd/identity-writer" ) SERVICES=( "lingniu-go-nats-kafka-bridge.service" "lingniu-go-fields-projector.service" "lingniu-go-nats-fast-writer.service" "lingniu-go-history-writer.service" "lingniu-go-stat-writer.service" "lingniu-go-realtime-writer.service" "lingniu-go-identity-writer.service" "lingniu-go-realtime-api.service" "lingniu-go-gateway.service" ) READYZ_PORTS=(20214 20218 20215 20212 20213 20216 20217 20200 20211) usage() { cat <<'USAGE' Usage: scripts/deploy-ecs-release.sh --build-only [--release NAME] [--out-dir DIR] scripts/deploy-ecs-release.sh --host root@115.29.187.205 [--release NAME] [--out-dir DIR] Options: --host HOST SSH target for ECS deployment. Omit with --build-only. --release NAME Release directory name under /opt/lingniu-go-native/releases. --out-dir DIR Local build output directory. Defaults to /tmp/lingniu-go-native-. --remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native. --build-only Build and verify Linux amd64 artifacts without uploading. --stage-only Upload and verify release, but do not switch current or restart services. --capacity-retries N Retry capacity-check after restart. Defaults to 10. --capacity-warmup-seconds N Wait after readyz before capacity-check, so recent-window metrics are not dominated by restart catch-up. Defaults to 60. --capacity-retry-interval-seconds N Wait between capacity-check retries. Defaults to 15. --capacity-timer UNIT systemd timer paused during restart. Defaults to lingniu-go-capacity-check.timer. --capacity-args ARGS Extra arguments passed to capacity-check during deploy verification. Example: --capacity-args "-daily-mileage-diagnostics-max-pipeline=0" --release-retain-count N Keep the newest N remote releases after a successful deployment. Defaults to 8. -h, --help Show this help. USAGE } while [[ $# -gt 0 ]]; do case "$1" in --host) HOST="${2:-}" shift 2 ;; --release) RELEASE="${2:-}" shift 2 ;; --out-dir) OUT_DIR="${2:-}" shift 2 ;; --remote-root) REMOTE_ROOT="${2:-}" shift 2 ;; --build-only) BUILD_ONLY=1 shift ;; --stage-only|--skip-restart) STAGE_ONLY=1 shift ;; --capacity-retries) CAPACITY_RETRIES="${2:-}" shift 2 ;; --capacity-warmup-seconds) CAPACITY_WARMUP_SECONDS="${2:-}" shift 2 ;; --capacity-retry-interval-seconds) CAPACITY_RETRY_INTERVAL_SECONDS="${2:-}" shift 2 ;; --capacity-timer) CAPACITY_TIMER="${2:-}" shift 2 ;; --capacity-args) CAPACITY_ARGS="${2:-}" shift 2 ;; --release-retain-count) RELEASE_RETAIN_COUNT="${2:-}" shift 2 ;; -h|--help) usage exit 0 ;; *) echo "unknown argument: $1" >&2 usage >&2 exit 2 ;; esac done if [[ -z "$OUT_DIR" ]]; then OUT_DIR="/tmp/lingniu-go-native-${RELEASE}" fi if [[ "$BUILD_ONLY" -eq 0 && -z "$HOST" ]]; then echo "--host is required unless --build-only is set" >&2 usage >&2 exit 2 fi case "$CAPACITY_RETRIES" in ''|*[!0-9]*|0) echo "--capacity-retries must be a positive integer" >&2 exit 2 ;; esac case "$CAPACITY_WARMUP_SECONDS" in ''|*[!0-9]*) echo "--capacity-warmup-seconds must be a non-negative integer" >&2 exit 2 ;; esac case "$CAPACITY_RETRY_INTERVAL_SECONDS" in ''|*[!0-9]*) echo "--capacity-retry-interval-seconds must be a non-negative integer" >&2 exit 2 ;; esac case "$RELEASE_RETAIN_COUNT" in ''|*[!0-9]*|0) echo "--release-retain-count must be a positive integer" >&2 exit 2 ;; esac assert_linux_amd64() { local binary="$1" local info info="$(file "$binary")" case "$info" in *"ELF 64-bit"*"x86-64"*) ;; *) echo "binary is not Linux amd64 ELF: $binary" >&2 echo "$info" >&2 exit 1 ;; esac } hash_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" else shasum -a 256 "$@" fi } build_release() { rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR" pushd "$ROOT_DIR" >/dev/null local names=() for spec in "${BINARIES[@]}"; do local name="${spec%%:*}" local pkg="${spec#*:}" names+=("$name") echo "building $name from $pkg" GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ go build -trimpath -ldflags="${GO_LDFLAGS:--s -w}" -o "$OUT_DIR/$name" "$pkg" assert_linux_amd64 "$OUT_DIR/$name" done popd >/dev/null ( cd "$OUT_DIR" hash_file "${names[@]}" > SHA256SUMS { echo "release=${RELEASE}" echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "git_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" echo "go_version=$(go version)" echo "target=linux/amd64" } > RELEASE_NAME ) echo "build output: $OUT_DIR" } preflight_remote_ssh() { echo "checking ssh connectivity to ${HOST}" if ssh -o ConnectTimeout=10 -o ConnectionAttempts=1 "$HOST" "true"; then return fi cat >&2 </dev/null | grep -q "${CAPACITY_TIMER}"; then timer_exists=1 if systemctl is-active --quiet "${CAPACITY_TIMER}"; then timer_was_active=1 systemctl stop "${CAPACITY_TIMER}" echo "paused ${CAPACITY_TIMER} during deploy" fi fi if [[ -n "${CAPACITY_SERVICE}" ]] && systemctl list-unit-files "${CAPACITY_SERVICE}" --no-legend 2>/dev/null | grep -q "${CAPACITY_SERVICE}"; then systemctl stop "${CAPACITY_SERVICE}" 2>/dev/null || true systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true fi restore_capacity_timer() { if [[ -n "${CAPACITY_SERVICE}" ]]; then systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true fi if [[ "${timer_exists}" == "1" && "${timer_was_active}" == "1" ]]; then if [[ -n "${CAPACITY_SERVICE}" ]]; then if systemctl start "${CAPACITY_SERVICE}" 2>/dev/null; then systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true echo "verified ${CAPACITY_SERVICE} after deploy" fi fi systemctl start "${CAPACITY_TIMER}" 2>/dev/null || true echo "resumed ${CAPACITY_TIMER}" if [[ -n "${CAPACITY_SERVICE}" ]]; then for _ in 1 2 3 4 5 6 7 8 9 10; do if ! systemctl is-active --quiet "${CAPACITY_SERVICE}"; then break fi sleep 1 done if systemctl is-failed --quiet "${CAPACITY_SERVICE}" && run_capacity_check >/dev/null 2>&1; then systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true echo "cleared transient ${CAPACITY_SERVICE} failure after timer resume" fi fi fi } cleanup_old_releases() { local current_dir current_dir="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)" local kept=0 while IFS= read -r release_path; do [[ -n "${release_path}" ]] || continue if [[ "${release_path}" == "${current_dir}" || "${kept}" -lt "${RELEASE_RETAIN_COUNT}" ]]; then kept=$((kept + 1)) continue fi rm -rf -- "${release_path}" done < <(find "${REMOTE_ROOT}/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-) } previous_release="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)" release_switched=0 deploy_succeeded=0 on_exit() { local status=$? trap - EXIT set +e if [[ "${release_switched}" == "1" && "${deploy_succeeded}" != "1" && -n "${previous_release}" && -d "${previous_release}" ]]; then echo "deployment failed; rolling back to ${previous_release}" >&2 ln -sfn "${previous_release}" "${REMOTE_ROOT}/current" for service in ${SERVICES}; do systemctl restart "${service}" done for service in ${SERVICES}; do systemctl is-active "${service}" done fi cleanup_remote_tmp restore_capacity_timer exit "${status}" } trap on_exit EXIT ln -sfn "${release_dir}" "${REMOTE_ROOT}/current" release_switched=1 for service in ${SERVICES}; do systemctl restart "${service}" done for service in ${SERVICES}; do systemctl is-active "${service}" done for port in ${READYZ_PORTS}; do ok=0 for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do if curl -fsS "http://127.0.0.1:${port}/readyz"; then echo ok=1 break fi sleep 2 done if [[ "${ok}" != "1" ]]; then echo "readyz failed for port ${port}" >&2 exit 1 fi done if [[ "${CAPACITY_WARMUP_SECONDS}" != "0" ]]; then echo "waiting ${CAPACITY_WARMUP_SECONDS}s before capacity-check warmup" sleep "${CAPACITY_WARMUP_SECONDS}" fi capacity_ok=0 for attempt in $(seq 1 "${CAPACITY_RETRIES}"); do echo "capacity-check attempt ${attempt}/${CAPACITY_RETRIES}" if run_capacity_check; then capacity_ok=1 break fi if [[ "${attempt}" != "${CAPACITY_RETRIES}" && "${CAPACITY_RETRY_INTERVAL_SECONDS}" != "0" ]]; then sleep "${CAPACITY_RETRY_INTERVAL_SECONDS}" fi done if [[ "${capacity_ok}" != "1" ]]; then exit 1 fi deploy_succeeded=1 cleanup_old_releases REMOTE rm -f "$archive" if [[ "$deploy_status" -ne 0 ]]; then return "$deploy_status" fi if [[ "$STAGE_ONLY" -eq 1 ]]; then echo "staged release ${RELEASE} at ${HOST}:${REMOTE_ROOT}/releases/${RELEASE}" else echo "deployed release ${RELEASE} to ${HOST}:${REMOTE_ROOT}" fi } if [[ "$BUILD_ONLY" -eq 0 ]]; then preflight_remote_ssh fi build_release if [[ "$BUILD_ONLY" -eq 1 ]]; then exit 0 fi deploy_release