feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
449
go/vehicle-gateway/scripts/deploy-ecs-release.sh
Executable file
449
go/vehicle-gateway/scripts/deploy-ecs-release.sh
Executable file
@@ -0,0 +1,449 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RELEASE="go-native-$(date +%Y%m%d%H%M%S)"
|
||||
OUT_DIR=""
|
||||
HOST=""
|
||||
REMOTE_ROOT="/opt/lingniu-go-native"
|
||||
BUILD_ONLY=0
|
||||
STAGE_ONLY=0
|
||||
CAPACITY_RETRIES=10
|
||||
CAPACITY_WARMUP_SECONDS=60
|
||||
CAPACITY_RETRY_INTERVAL_SECONDS=15
|
||||
CAPACITY_TIMER="lingniu-go-capacity-check.timer"
|
||||
CAPACITY_SERVICE="lingniu-go-capacity-check.service"
|
||||
CAPACITY_ARGS="${CAPACITY_ARGS:-}"
|
||||
RELEASE_RETAIN_COUNT="${RELEASE_RETAIN_COUNT:-8}"
|
||||
|
||||
BINARIES=(
|
||||
"gateway:./cmd/gateway"
|
||||
"history-writer:./cmd/history-writer"
|
||||
"stat-writer:./cmd/stat-writer"
|
||||
"realtime-api:./cmd/realtime-api"
|
||||
"nats-fast-writer:./cmd/nats-fast-writer"
|
||||
"nats-kafka-bridge:./cmd/nats-kafka-bridge"
|
||||
"fields-projector:./cmd/fields-projector"
|
||||
"capacity-check:./cmd/capacity-check"
|
||||
"load-sim:./cmd/load-sim"
|
||||
"stats-backfill:./cmd/stats-backfill"
|
||||
"identity-import:./cmd/identity-import"
|
||||
"identity-writer:./cmd/identity-writer"
|
||||
)
|
||||
|
||||
SERVICES=(
|
||||
"lingniu-go-nats-kafka-bridge.service"
|
||||
"lingniu-go-fields-projector.service"
|
||||
"lingniu-go-nats-fast-writer.service"
|
||||
"lingniu-go-history-writer.service"
|
||||
"lingniu-go-stat-writer.service"
|
||||
"lingniu-go-realtime-writer.service"
|
||||
"lingniu-go-identity-writer.service"
|
||||
"lingniu-go-realtime-api.service"
|
||||
"lingniu-go-gateway.service"
|
||||
)
|
||||
|
||||
READYZ_PORTS=(20214 20218 20215 20212 20213 20216 20217 20200 20211)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/deploy-ecs-release.sh --build-only [--release NAME] [--out-dir DIR]
|
||||
scripts/deploy-ecs-release.sh --host root@115.29.187.205 [--release NAME] [--out-dir DIR]
|
||||
|
||||
Options:
|
||||
--host HOST SSH target for ECS deployment. Omit with --build-only.
|
||||
--release NAME Release directory name under /opt/lingniu-go-native/releases.
|
||||
--out-dir DIR Local build output directory. Defaults to /tmp/lingniu-go-native-<release>.
|
||||
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
|
||||
--build-only Build and verify Linux amd64 artifacts without uploading.
|
||||
--stage-only Upload and verify release, but do not switch current or restart services.
|
||||
--capacity-retries N
|
||||
Retry capacity-check after restart. Defaults to 10.
|
||||
--capacity-warmup-seconds N
|
||||
Wait after readyz before capacity-check, so recent-window metrics are not dominated by restart catch-up. Defaults to 60.
|
||||
--capacity-retry-interval-seconds N
|
||||
Wait between capacity-check retries. Defaults to 15.
|
||||
--capacity-timer UNIT
|
||||
systemd timer paused during restart. Defaults to lingniu-go-capacity-check.timer.
|
||||
--capacity-args ARGS
|
||||
Extra arguments passed to capacity-check during deploy verification.
|
||||
Example: --capacity-args "-daily-mileage-diagnostics-max-pipeline=0"
|
||||
--release-retain-count N
|
||||
Keep the newest N remote releases after a successful deployment. Defaults to 8.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--host)
|
||||
HOST="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--release)
|
||||
RELEASE="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--out-dir)
|
||||
OUT_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--remote-root)
|
||||
REMOTE_ROOT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--build-only)
|
||||
BUILD_ONLY=1
|
||||
shift
|
||||
;;
|
||||
--stage-only|--skip-restart)
|
||||
STAGE_ONLY=1
|
||||
shift
|
||||
;;
|
||||
--capacity-retries)
|
||||
CAPACITY_RETRIES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--capacity-warmup-seconds)
|
||||
CAPACITY_WARMUP_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--capacity-retry-interval-seconds)
|
||||
CAPACITY_RETRY_INTERVAL_SECONDS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--capacity-timer)
|
||||
CAPACITY_TIMER="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--capacity-args)
|
||||
CAPACITY_ARGS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--release-retain-count)
|
||||
RELEASE_RETAIN_COUNT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="/tmp/lingniu-go-native-${RELEASE}"
|
||||
fi
|
||||
|
||||
if [[ "$BUILD_ONLY" -eq 0 && -z "$HOST" ]]; then
|
||||
echo "--host is required unless --build-only is set" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
case "$CAPACITY_RETRIES" in
|
||||
''|*[!0-9]*|0)
|
||||
echo "--capacity-retries must be a positive integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$CAPACITY_WARMUP_SECONDS" in
|
||||
''|*[!0-9]*)
|
||||
echo "--capacity-warmup-seconds must be a non-negative integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$CAPACITY_RETRY_INTERVAL_SECONDS" in
|
||||
''|*[!0-9]*)
|
||||
echo "--capacity-retry-interval-seconds must be a non-negative integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$RELEASE_RETAIN_COUNT" in
|
||||
''|*[!0-9]*|0)
|
||||
echo "--release-retain-count must be a positive integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
assert_linux_amd64() {
|
||||
local binary="$1"
|
||||
local info
|
||||
info="$(file "$binary")"
|
||||
case "$info" in
|
||||
*"ELF 64-bit"*"x86-64"*) ;;
|
||||
*)
|
||||
echo "binary is not Linux amd64 ELF: $binary" >&2
|
||||
echo "$info" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$@"
|
||||
else
|
||||
shasum -a 256 "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
build_release() {
|
||||
rm -rf "$OUT_DIR"
|
||||
mkdir -p "$OUT_DIR"
|
||||
pushd "$ROOT_DIR" >/dev/null
|
||||
local names=()
|
||||
for spec in "${BINARIES[@]}"; do
|
||||
local name="${spec%%:*}"
|
||||
local pkg="${spec#*:}"
|
||||
names+=("$name")
|
||||
echo "building $name from $pkg"
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \
|
||||
go build -trimpath -ldflags="${GO_LDFLAGS:--s -w}" -o "$OUT_DIR/$name" "$pkg"
|
||||
assert_linux_amd64 "$OUT_DIR/$name"
|
||||
done
|
||||
popd >/dev/null
|
||||
|
||||
(
|
||||
cd "$OUT_DIR"
|
||||
hash_file "${names[@]}" > SHA256SUMS
|
||||
{
|
||||
echo "release=${RELEASE}"
|
||||
echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "git_commit=$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||
echo "go_version=$(go version)"
|
||||
echo "target=linux/amd64"
|
||||
} > RELEASE_NAME
|
||||
)
|
||||
echo "build output: $OUT_DIR"
|
||||
}
|
||||
|
||||
preflight_remote_ssh() {
|
||||
echo "checking ssh connectivity to ${HOST}"
|
||||
if ssh -o ConnectTimeout=10 -o ConnectionAttempts=1 "$HOST" "true"; then
|
||||
return
|
||||
fi
|
||||
cat >&2 <<EOF
|
||||
ssh preflight failed for ${HOST}.
|
||||
The release was not built, uploaded, or switched. Restore SSH access first, then rerun this script.
|
||||
EOF
|
||||
exit 255
|
||||
}
|
||||
|
||||
deploy_release() {
|
||||
local archive="/tmp/${RELEASE}.tar.gz"
|
||||
local remote_tmp="/tmp/lingniu-go-native-${RELEASE}"
|
||||
local remote_bins=""
|
||||
local services=""
|
||||
local readyz_ports=""
|
||||
local spec
|
||||
for spec in "${BINARIES[@]}"; do
|
||||
remote_bins="${remote_bins} ${spec%%:*}"
|
||||
done
|
||||
for spec in "${SERVICES[@]}"; do
|
||||
services="${services} ${spec}"
|
||||
done
|
||||
for spec in "${READYZ_PORTS[@]}"; do
|
||||
readyz_ports="${readyz_ports} ${spec}"
|
||||
done
|
||||
|
||||
COPYFILE_DISABLE=1 tar --format ustar -czf "$archive" -C "$OUT_DIR" .
|
||||
ssh "$HOST" "mkdir -p '$remote_tmp'"
|
||||
scp "$archive" "$HOST:${remote_tmp}.tar.gz"
|
||||
local remote_cmd
|
||||
printf -v remote_cmd \
|
||||
"REMOTE_ROOT=%q RELEASE=%q REMOTE_TMP=%q REMOTE_BINS=%q SERVICES=%q READYZ_PORTS=%q STAGE_ONLY=%q CAPACITY_RETRIES=%q CAPACITY_WARMUP_SECONDS=%q CAPACITY_RETRY_INTERVAL_SECONDS=%q CAPACITY_TIMER=%q CAPACITY_SERVICE=%q CAPACITY_ARGS=%q RELEASE_RETAIN_COUNT=%q bash -s" \
|
||||
"$REMOTE_ROOT" "$RELEASE" "$remote_tmp" "$remote_bins" "$services" "$readyz_ports" "$STAGE_ONLY" "$CAPACITY_RETRIES" "$CAPACITY_WARMUP_SECONDS" "$CAPACITY_RETRY_INTERVAL_SECONDS" "$CAPACITY_TIMER" "$CAPACITY_SERVICE" "$CAPACITY_ARGS" "$RELEASE_RETAIN_COUNT"
|
||||
local deploy_status=0
|
||||
ssh "$HOST" "$remote_cmd" <<'REMOTE' || deploy_status=$?
|
||||
set -euo pipefail
|
||||
|
||||
capacity_check_args=()
|
||||
if [[ -n "${CAPACITY_ARGS:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
capacity_check_args=(${CAPACITY_ARGS})
|
||||
fi
|
||||
|
||||
run_capacity_check() {
|
||||
if [[ "${#capacity_check_args[@]}" -gt 0 ]]; then
|
||||
"${REMOTE_ROOT}/current/capacity-check" -timeout 20s "${capacity_check_args[@]}"
|
||||
return
|
||||
fi
|
||||
"${REMOTE_ROOT}/current/capacity-check" -timeout 20s
|
||||
}
|
||||
|
||||
release_dir="${REMOTE_ROOT}/releases/${RELEASE}"
|
||||
cleanup_remote_tmp() {
|
||||
rm -rf "${REMOTE_TMP}" "${REMOTE_TMP}.tar.gz"
|
||||
}
|
||||
rm -rf "${REMOTE_TMP}"
|
||||
mkdir -p "${REMOTE_TMP}" "${release_dir}"
|
||||
tar -xzf "${REMOTE_TMP}.tar.gz" -C "${REMOTE_TMP}"
|
||||
|
||||
for bin in ${REMOTE_BINS}; do
|
||||
test -f "${REMOTE_TMP}/${bin}"
|
||||
file "${REMOTE_TMP}/${bin}" | grep -Eq 'ELF 64-bit.*x86-64'
|
||||
done
|
||||
|
||||
cp "${REMOTE_TMP}"/* "${release_dir}/"
|
||||
for bin in ${REMOTE_BINS}; do
|
||||
chmod 0755 "${release_dir}/${bin}"
|
||||
done
|
||||
|
||||
if [[ "${STAGE_ONLY}" == "1" ]]; then
|
||||
cleanup_remote_tmp
|
||||
echo "staged release ${RELEASE} at ${release_dir}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
timer_was_active=0
|
||||
timer_exists=0
|
||||
if [[ -n "${CAPACITY_TIMER}" ]] && systemctl list-unit-files "${CAPACITY_TIMER}" --no-legend 2>/dev/null | grep -q "${CAPACITY_TIMER}"; then
|
||||
timer_exists=1
|
||||
if systemctl is-active --quiet "${CAPACITY_TIMER}"; then
|
||||
timer_was_active=1
|
||||
systemctl stop "${CAPACITY_TIMER}"
|
||||
echo "paused ${CAPACITY_TIMER} during deploy"
|
||||
fi
|
||||
fi
|
||||
if [[ -n "${CAPACITY_SERVICE}" ]] && systemctl list-unit-files "${CAPACITY_SERVICE}" --no-legend 2>/dev/null | grep -q "${CAPACITY_SERVICE}"; then
|
||||
systemctl stop "${CAPACITY_SERVICE}" 2>/dev/null || true
|
||||
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
|
||||
fi
|
||||
restore_capacity_timer() {
|
||||
if [[ -n "${CAPACITY_SERVICE}" ]]; then
|
||||
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
|
||||
fi
|
||||
if [[ "${timer_exists}" == "1" && "${timer_was_active}" == "1" ]]; then
|
||||
if [[ -n "${CAPACITY_SERVICE}" ]]; then
|
||||
if systemctl start "${CAPACITY_SERVICE}" 2>/dev/null; then
|
||||
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
|
||||
echo "verified ${CAPACITY_SERVICE} after deploy"
|
||||
fi
|
||||
fi
|
||||
systemctl start "${CAPACITY_TIMER}" 2>/dev/null || true
|
||||
echo "resumed ${CAPACITY_TIMER}"
|
||||
if [[ -n "${CAPACITY_SERVICE}" ]]; then
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||
if ! systemctl is-active --quiet "${CAPACITY_SERVICE}"; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if systemctl is-failed --quiet "${CAPACITY_SERVICE}" && run_capacity_check >/dev/null 2>&1; then
|
||||
systemctl reset-failed "${CAPACITY_SERVICE}" 2>/dev/null || true
|
||||
echo "cleared transient ${CAPACITY_SERVICE} failure after timer resume"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
cleanup_old_releases() {
|
||||
local current_dir
|
||||
current_dir="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
|
||||
local kept=0
|
||||
while IFS= read -r release_path; do
|
||||
[[ -n "${release_path}" ]] || continue
|
||||
if [[ "${release_path}" == "${current_dir}" || "${kept}" -lt "${RELEASE_RETAIN_COUNT}" ]]; then
|
||||
kept=$((kept + 1))
|
||||
continue
|
||||
fi
|
||||
rm -rf -- "${release_path}"
|
||||
done < <(find "${REMOTE_ROOT}/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2-)
|
||||
}
|
||||
previous_release="$(readlink -f "${REMOTE_ROOT}/current" 2>/dev/null || true)"
|
||||
release_switched=0
|
||||
deploy_succeeded=0
|
||||
on_exit() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
if [[ "${release_switched}" == "1" && "${deploy_succeeded}" != "1" && -n "${previous_release}" && -d "${previous_release}" ]]; then
|
||||
echo "deployment failed; rolling back to ${previous_release}" >&2
|
||||
ln -sfn "${previous_release}" "${REMOTE_ROOT}/current"
|
||||
for service in ${SERVICES}; do
|
||||
systemctl restart "${service}"
|
||||
done
|
||||
for service in ${SERVICES}; do
|
||||
systemctl is-active "${service}"
|
||||
done
|
||||
fi
|
||||
cleanup_remote_tmp
|
||||
restore_capacity_timer
|
||||
exit "${status}"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
ln -sfn "${release_dir}" "${REMOTE_ROOT}/current"
|
||||
release_switched=1
|
||||
|
||||
for service in ${SERVICES}; do
|
||||
systemctl restart "${service}"
|
||||
done
|
||||
|
||||
for service in ${SERVICES}; do
|
||||
systemctl is-active "${service}"
|
||||
done
|
||||
|
||||
for port in ${READYZ_PORTS}; do
|
||||
ok=0
|
||||
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
||||
if curl -fsS "http://127.0.0.1:${port}/readyz"; then
|
||||
echo
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [[ "${ok}" != "1" ]]; then
|
||||
echo "readyz failed for port ${port}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${CAPACITY_WARMUP_SECONDS}" != "0" ]]; then
|
||||
echo "waiting ${CAPACITY_WARMUP_SECONDS}s before capacity-check warmup"
|
||||
sleep "${CAPACITY_WARMUP_SECONDS}"
|
||||
fi
|
||||
|
||||
capacity_ok=0
|
||||
for attempt in $(seq 1 "${CAPACITY_RETRIES}"); do
|
||||
echo "capacity-check attempt ${attempt}/${CAPACITY_RETRIES}"
|
||||
if run_capacity_check; then
|
||||
capacity_ok=1
|
||||
break
|
||||
fi
|
||||
if [[ "${attempt}" != "${CAPACITY_RETRIES}" && "${CAPACITY_RETRY_INTERVAL_SECONDS}" != "0" ]]; then
|
||||
sleep "${CAPACITY_RETRY_INTERVAL_SECONDS}"
|
||||
fi
|
||||
done
|
||||
if [[ "${capacity_ok}" != "1" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
deploy_succeeded=1
|
||||
cleanup_old_releases
|
||||
REMOTE
|
||||
rm -f "$archive"
|
||||
if [[ "$deploy_status" -ne 0 ]]; then
|
||||
return "$deploy_status"
|
||||
fi
|
||||
if [[ "$STAGE_ONLY" -eq 1 ]]; then
|
||||
echo "staged release ${RELEASE} at ${HOST}:${REMOTE_ROOT}/releases/${RELEASE}"
|
||||
else
|
||||
echo "deployed release ${RELEASE} to ${HOST}:${REMOTE_ROOT}"
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "$BUILD_ONLY" -eq 0 ]]; then
|
||||
preflight_remote_ssh
|
||||
fi
|
||||
build_release
|
||||
if [[ "$BUILD_ONLY" -eq 1 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
deploy_release
|
||||
Reference in New Issue
Block a user