package main import ( "context" "database/sql" "encoding/json" "errors" "flag" "fmt" "os" "os/signal" "path/filepath" "strings" "syscall" "time" "github.com/go-sql-driver/mysql" "lingniu/vehicle-data-platform/apps/api/internal/gzev" ) func main() { if err := run(); err != nil { fmt.Fprintln(os.Stderr, "gzev-mileage:", err) os.Exit(1) } } func run() error { mode := flag.String("mode", "sync", "sync (MySQL), fetch (raw daily files), probe (one complete day), status (MySQL coverage), notify-test (one ServerChan test push)") start := flag.String("from", env("GZEV_START_DATE", "2024-01-01"), "inclusive first day") end := flag.String("to", "", "inclusive last day; defaults to yesterday") sessionPath := flag.String("session", env("GZEV_SESSION_FILE", "/var/lib/lingniu-gzev/session.json"), "0600 JSON session file") out := flag.String("out", "", "fetch output directory") maxDays := flag.Int("max-days", 30, "maximum days fetched per run; oldest unfinished dates resume next run") lookback := flag.Int("lookback", 7, "re-fetch recent complete days") flag.Parse() if *mode != "sync" && *mode != "fetch" && *mode != "probe" && *mode != "status" && *mode != "notify-test" { return errors.New("unknown mode") } if *maxDays < 1 || *lookback < 0 { return errors.New("invalid day limits") } today := gzev.ShanghaiToday(time.Now()) if *end == "" { *end = today.AddDate(0, 0, -1).Format("2006-01-02") } first, err := gzev.ParseDate(*start) if err != nil { return err } last, err := gzev.ParseDate(*end) if err != nil { return err } if first.After(last) || gzev.ValidateDate(*end, time.Now()) != nil { return errors.New("invalid range: end must be before today in Asia/Shanghai") } var session gzev.Session var sessionErr error if *mode != "status" && *mode != "notify-test" { session, sessionErr = gzev.ReadSession(*sessionPath) if sessionErr != nil && *mode != "sync" { return sessionErr } } client := gzev.NewClient(session) account := env("GZEV_ACCOUNT", "guangdonglingniu") if len(account) > 64 { return errors.New("account too long") } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() if *mode == "notify-test" { if err := gzev.ServerChanFromEnv().SendTest(ctx); err != nil { return err } fmt.Println("ServerChan accepted test notification") return nil } if *mode == "probe" || *mode == "fetch" { if err := client.VerifyAccount(ctx, account); err != nil { return err } } if *mode == "probe" { client.PageSize = 500 day, err := client.FetchDay(ctx, *end) if err != nil { return err } fmt.Printf("date=%s rows=%d pages=%d\n", day.Date, len(day.Rows), len(day.Pages)) return nil } if *mode == "fetch" { if *out == "" { return errors.New("fetch requires --out") } if err := os.MkdirAll(*out, 0700); err != nil { return err } count := 0 for day := first; !day.After(last) && count < *maxDays; day = day.AddDate(0, 0, 1) { date := day.Format("2006-01-02") path := filepath.Join(*out, date+".json") if _, err := os.Stat(path); err == nil { continue } else if !os.IsNotExist(err) { return err } result, err := client.FetchDay(ctx, date) if err != nil { return err } raw, err := json.Marshal(result) if err != nil { return err } tmp, err := os.CreateTemp(*out, ".gzev-*") if err != nil { return err } name := tmp.Name() _, writeErr := tmp.Write(raw) syncErr := tmp.Sync() closeErr := tmp.Close() if err := errors.Join(writeErr, syncErr, closeErr); err != nil { os.Remove(name) return err } if err := os.Rename(name, path); err != nil { os.Remove(name) return err } fmt.Printf("downloaded date=%s rows=%d pages=%d\n", date, len(result.Rows), len(result.Pages)) count++ } return nil } dsn := os.Getenv("MYSQL_DSN") if dsn == "" { return errors.New("MYSQL_DSN is required for sync") } config, err := mysql.ParseDSN(dsn) if err != nil { return errors.New("invalid MYSQL_DSN") } config.ParseTime = true config.Loc = time.UTC config.Timeout = 10 * time.Second config.ReadTimeout = 60 * time.Second config.WriteTimeout = 60 * time.Second db, err := sql.Open("mysql", config.FormatDSN()) if err != nil { return errors.New("cannot initialize MySQL") } defer db.Close() if *mode == "status" { var days, records, missing int64 var latest sql.NullString err := db.QueryRowContext(ctx, "SELECT COUNT(*),COALESCE(SUM(row_count),0),COALESCE(SUM(missing_count),0),CAST(MAX(last_success_at) AS CHAR) FROM gzev_mileage_day WHERE account=? AND stat_date BETWEEN ? AND ?", account, *start, *end).Scan(&days, &records, &missing, &latest) if err != nil { return err } return json.NewEncoder(os.Stdout).Encode(map[string]any{"account": account, "from": *start, "to": *end, "expected_days": int(last.Sub(first).Hours()/24) + 1, "fetched_days": days, "rows": records, "missing_rows": missing, "last_success_utc": latest.String}) } err = gzev.WithLock(ctx, db, account, func(store *gzev.Store) error { if sessionErr != nil { return errors.Join(gzev.ErrAuth, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send)) } if err := client.VerifyAccount(ctx, account); err != nil { if errors.Is(err, gzev.ErrAuth) { return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send)) } return err } if err := store.ResolveAuth(ctx); err != nil { return err } count := 0 var failures []error // First collect yesterday so a long historical backfill cannot starve daily updates. dates := []time.Time{} recentStart := today.AddDate(0, 0, -*lookback) for d := last; !d.Before(first) && !d.Before(recentStart); d = d.AddDate(0, 0, -1) { dates = append(dates, d) } for d := first; !d.After(last); d = d.AddDate(0, 0, 1) { if d.Before(recentStart) { dates = append(dates, d) } } for _, day := range dates { if count >= *maxDays { break } date := day.Format("2006-01-02") recent := !day.Before(today.AddDate(0, 0, -*lookback)) need, err := store.NeedsDay(ctx, date, recent) if err != nil { return err } if !need { continue } // Read again for each day: operators can atomically replace the session during backfill. s, err := gzev.ReadSession(*sessionPath) if err != nil { return errors.Join(gzev.ErrAuth, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send)) } changed := s.Authorization != client.Session.Authorization client.Session = s if changed { if err := client.VerifyAccount(ctx, account); err != nil { if errors.Is(err, gzev.ErrAuth) { return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send)) } return err } } result, err := client.FetchDay(ctx, date) count++ if errors.Is(err, gzev.ErrAuth) { return errors.Join(err, store.AuthFailure(ctx, gzev.ServerChanFromEnv().Send)) } if err != nil { if ctx.Err() != nil { return ctx.Err() } if failureErr := store.RecordFailure(ctx, date); failureErr != nil { return failureErr } failures = append(failures, fmt.Errorf("%s: %w", date, err)) continue } if err := store.ResolveAuth(ctx); err != nil { return err } if err := store.SaveDay(ctx, result); err != nil { if failureErr := store.RecordFailure(ctx, date); failureErr != nil { return failureErr } failures = append(failures, fmt.Errorf("%s: %w", date, err)) continue } fmt.Printf("stored date=%s rows=%d pages=%d\n", date, len(result.Rows), len(result.Pages)) } return errors.Join(failures...) }) return err } func env(key, fallback string) string { if v := strings.TrimSpace(os.Getenv(key)); v != "" { return v } return fallback }