lingniu-vehicle-ingest/vehicle-data-platform/apps/api/cmd/gzev-mileage/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/go-sql-driver/mysql"
	"lingniu/vehicle-data-platform/apps/api/internal/gzev"
)

func main() {
	if err := run(); err != nil {
		fmt.Fprintln(os.Stderr, "gzev-mileage:", err)
		os.Exit(1)
	}
}
func run() error {
	mode := flag.String("mode", "sync", "sync (MySQL), fetch (raw daily files), probe (one complete day), status (MySQL coverage), notify-test (one ServerChan test push)")
	start := flag.String("from", env("GZEV_START_DATE", "2024-01-01"), "inclusive first day")
	end := flag.String("to", "", "inclusive last day; defaults to yesterday")
	sessionPath := flag.String("session", env("GZEV_SESSION_FILE", "/var/lib/lingniu-gzev/session.json"), "0600 JSON session file")
	out := flag.String("out", "", "fetch output directory")
	maxDays := flag.Int("max-days", 30, "maximum days fetched per run; oldest unfinished dates resume next run")
	lookback := flag.Int("lookback", 7, "re-fetch recent complete days")
	flag.Parse()
	if *mode != "sync" && *mode != "fetch" && *mode != "probe" && *mode != "status" && *mode != "notify-test" {
		return errors.New("unknown mode")
	}
	if *maxDays < 1 || *lookback < 0 {
		return errors.New("invalid day limits")
	}
	today := gzev.ShanghaiToday(time.Now())
	if *end == "" {
		*end = today.AddDate(0, 0, -1).Format("2006-01-02")
	}
	first, err := gzev.ParseDate(*start)
	if err != nil {
		return err
	}
	last, err := gzev.ParseDate(*end)
	if err != nil {
		return err
	}
	if first.After(last) || gzev.ValidateDate(*end, time.Now()) != nil {
		return errors.New("invalid range: end must be before today in Asia/Shanghai")
	}
	var session gzev.Session
	var sessionErr error
	if *mode != "status" && *mode != "notify-test" {
		session, sessionErr = gzev.ReadSession(*sessionPath)
		if sessionErr != nil && *mode != "sync" {
			return sessionErr
		}
	}
	client := gzev.NewClient(session)
	account := env("GZEV_ACCOUNT", "guangdonglingniu")
	if len(account) > 64 {
		return errors.New("account too long")
	}
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer cancel()
	if *mode == "notify-test" {
		if err := gzev.ServerChanFromEnv().SendTest(ctx); err != nil {
			return err
		}
		fmt.Println("ServerChan accepted test notification")
		return nil
	}
	if *mode == "probe" || *mode == "fetch" {
		if err := client.VerifyAccount(ctx, account); err != nil {
			return err
		}
	}
	if *mode == "probe" {
		client.PageSize = 500
		day, err := client.FetchDay(ctx, *end)
		if err != nil {
			return err
		}
		fmt.Printf("date=%s rows=%d pages=%d\n", day.Date, len(day.Rows), len(day.Pages))
		return nil
	}
	if *mode == "fetch" {
		if *out == "" {
			return errors.New("fetch requires --out")
		}
		if err := os.MkdirAll(*out, 0700); err != nil {
			return err
		}
		count := 0
		for day := first; !day.After(last) && count < *maxDays; day = day.AddDate(0, 0, 1) {
			date := day.Format("2006-01-02")
			path := filepath.Join(*out, date+".json")
			if _, err := os.Stat(path); err == nil {
				continue
			} else if !os.IsNotExist(err) {
				return err
			}
			result, err := client.FetchDay(ctx, date)
			if err != nil {
				return err
			}
			raw, err := json.Marshal(result)
			if err != nil {
				return err
			}
			tmp, err := os.CreateTemp(*out, ".gzev-*")
			if err != nil {
				return err
			}
			name := tmp.Name()
			_, writeErr := tmp.Write(raw)
			syncErr := tmp.Sync()
			closeErr := tmp.Close()
			if err := errors.Join(writeErr, syncErr, closeErr); err != nil {
				os.Remove(name)
				return err
			}
			if err := os.Rename(name, path); err != nil {
				os.Remove(name)
				return err
			}
			fmt.Printf("downloaded date=%s rows=%d pages=%d\n", date, len(result.Rows), len(result.Pages))
			count++
		}
		return nil
	}
	dsn := os.Getenv("MYSQL_DSN")
	if dsn == "" {
		return errors.New("MYSQL_DSN is required for sync")
	}
	config, err := mysql.ParseDSN(dsn)
	if err != nil {
		return errors.New("invalid MYSQL_DSN")
	}
	config.ParseTime = true
	config.Loc = time.UTC
	config.Timeout = 10 * time.Second
	config.ReadTimeout = 60 * time.Second
	config.WriteTimeout = 60 * time.Second
	db, err := sql.Open("mysql", config.FormatDSN())
	if err != nil {
		return errors.New("cannot initialize MySQL")
	}
	defer db.Close()
	if *mode == "status" {
		var days, records, missing int64
		var latest sql.NullString
		err := db.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(SUM(row_count),0),COALESCE(SUM(missing_count),0),CAST(MAX(last_success_at) AS CHAR) FROM gzev_mileage_day WHERE account=? AND stat_date BETWEEN ? AND ?", account, *start, *end).Scan(&days, &records, &missing, &latest)
		if err != nil {
			return err
		}
		return json.NewEncoder(os.Stdout).Encode(map[string]any{"account": account, "from": *start, "to": *end, "expected_days": int(last.Sub(first).Hours()/24) + 1, "fetched_days": days, "rows": records, "missing_rows": missing, "last_success_utc": latest.String})
	}
	err = gzev.WithLock(ctx, db, account, func(store *gzev.Store) error {
		if sessionErr != nil {
			return errors.Join(gzev.ErrAuth, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send))
		}
		if err := client.VerifyAccount(ctx, account); err != nil {
			if errors.Is(err, gzev.ErrAuth) {
				return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send))
			}
			return err
		}
		if err := store.ResolveAuth(ctx); err != nil {
			return err
		}
		count := 0
		var failures []error
		// First collect yesterday so a long historical backfill cannot starve daily updates.
		dates := []time.Time{}
		recentStart := today.AddDate(0, 0, -*lookback)
		for d := last; !d.Before(first) && !d.Before(recentStart); d = d.AddDate(0, 0, -1) {
			dates = append(dates, d)
		}
		for d := first; !d.After(last); d = d.AddDate(0, 0, 1) {
			if d.Before(recentStart) {
				dates = append(dates, d)
			}
		}
		for _, day := range dates {
			if count >= *maxDays {
				break
			}
			date := day.Format("2006-01-02")
			recent := !day.Before(today.AddDate(0, 0, -*lookback))
			need, err := store.NeedsDay(ctx, date, recent)
			if err != nil {
				return err
			}
			if !need {
				continue
			}
			// Read again for each day: operators can atomically replace the session during backfill.
			s, err := gzev.ReadSession(*sessionPath)
			if err != nil {
				return errors.Join(gzev.ErrAuth, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send))
			}
			changed := s.Authorization != client.Session.Authorization
			client.Session = s
			if changed {
				if err := client.VerifyAccount(ctx, account); err != nil {
					if errors.Is(err, gzev.ErrAuth) {
						return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send))
					}
					return err
				}
			}
			result, err := client.FetchDay(ctx, date)
			count++
			if errors.Is(err, gzev.ErrAuth) {
				return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send))
			}
			if err != nil {
				if ctx.Err() != nil {
					return ctx.Err()
				}
				if failureErr := store.RecordFailure(ctx, date); failureErr != nil {
					return failureErr
				}
				failures = append(failures, fmt.Errorf("%s: %w", date, err))
				continue
			}
			if err := store.ResolveAuth(ctx); err != nil {
				return err
			}
			if err := store.SaveDay(ctx, result); err != nil {
				if failureErr := store.RecordFailure(ctx, date); failureErr != nil {
					return failureErr
				}
				failures = append(failures, fmt.Errorf("%s: %w", date, err))
				continue
			}
			fmt.Printf("stored date=%s rows=%d pages=%d\n", date, len(result.Rows), len(result.Pages))
		}
		return errors.Join(failures...)
	})
	return err
}
func env(key, fallback string) string {
	if v := strings.TrimSpace(os.Getenv(key)); v != "" {
		return v
	}
	return fallback
}