feat(platform): consolidate production vehicle data workflows
This commit is contained in:
lingniu
205
vehicle-data-platform/apps/api/internal/businessscope/model.go
Normal file
205
vehicle-data-platform/apps/api/internal/businessscope/model.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const SourceSystem = "oneos"
|
||||
|
||||
const (
|
||||
ReasonVehicleMissing = "VEHICLE_MISSING"
|
||||
ReasonVINMissing = "VIN_MISSING"
|
||||
ReasonCustomerMissing = "CUSTOMER_MISSING"
|
||||
ReasonCustomerProfileMissing = "CUSTOMER_PROFILE_MISSING"
|
||||
ReasonContractMissing = "CONTRACT_MISSING"
|
||||
ReasonContractProfileMissing = "CONTRACT_PROFILE_MISSING"
|
||||
ReasonContractCustomerMissing = "CONTRACT_CUSTOMER_MISSING"
|
||||
ReasonCustomerMismatch = "CUSTOMER_MISMATCH"
|
||||
ReasonDuplicateVehicle = "DUPLICATE_VEHICLE_SCOPE"
|
||||
ReasonDuplicateVIN = "DUPLICATE_VIN_SCOPE"
|
||||
)
|
||||
|
||||
type Candidate struct {
|
||||
RowNumber int
|
||||
VehicleID int64
|
||||
VehiclePresent bool
|
||||
VIN string
|
||||
PlateNumber string
|
||||
CustomerID int64
|
||||
CustomerPresent bool
|
||||
CustomerProfileExists bool
|
||||
ContractID int64
|
||||
ContractPresent bool
|
||||
ContractProfileExists bool
|
||||
EffectiveCustomerID int64
|
||||
EffectiveCustomerSet bool
|
||||
ContractCode string
|
||||
ProjectName string
|
||||
OperationStatus string
|
||||
ScopeStartAt time.Time
|
||||
SourceUpdatedAt *time.Time
|
||||
}
|
||||
|
||||
type ScopeItem struct {
|
||||
VehicleID int64
|
||||
VIN string
|
||||
PlateNumber string
|
||||
CustomerID int64
|
||||
ContractID int64
|
||||
ContractCode string
|
||||
ProjectName string
|
||||
OperationStatus string
|
||||
ScopeStartAt time.Time
|
||||
SourceUpdatedAt *time.Time
|
||||
}
|
||||
|
||||
type Rejection struct {
|
||||
RowNumber int
|
||||
VehicleID *int64
|
||||
VIN string
|
||||
CustomerID *int64
|
||||
ContractID *int64
|
||||
ReasonCode string
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
RunID string
|
||||
SourceVersion string
|
||||
Checksum string
|
||||
GeneratedAt time.Time
|
||||
Candidates int
|
||||
Items []ScopeItem
|
||||
Rejections []Rejection
|
||||
}
|
||||
|
||||
func BuildSnapshot(candidates []Candidate, generatedAt time.Time) (Snapshot, error) {
|
||||
generatedAt = generatedAt.UTC()
|
||||
reasons := make([]string, len(candidates))
|
||||
vehicleRows := map[int64][]int{}
|
||||
vinRows := map[string][]int{}
|
||||
for i, candidate := range candidates {
|
||||
candidate.VIN = normalizeVIN(candidate.VIN)
|
||||
candidates[i] = candidate
|
||||
reasons[i] = validateCandidate(candidate)
|
||||
if reasons[i] == "" {
|
||||
vehicleRows[candidate.VehicleID] = append(vehicleRows[candidate.VehicleID], i)
|
||||
vinRows[candidate.VIN] = append(vinRows[candidate.VIN], i)
|
||||
}
|
||||
}
|
||||
for _, indexes := range vehicleRows {
|
||||
if len(indexes) > 1 {
|
||||
for _, index := range indexes {
|
||||
reasons[index] = ReasonDuplicateVehicle
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, indexes := range vinRows {
|
||||
if len(indexes) > 1 {
|
||||
for _, index := range indexes {
|
||||
if reasons[index] == "" {
|
||||
reasons[index] = ReasonDuplicateVIN
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items := make([]ScopeItem, 0, len(candidates))
|
||||
rejections := make([]Rejection, 0)
|
||||
for index, candidate := range candidates {
|
||||
if reasons[index] != "" {
|
||||
rejections = append(rejections, rejectionFor(candidate, reasons[index]))
|
||||
continue
|
||||
}
|
||||
items = append(items, ScopeItem{
|
||||
VehicleID: candidate.VehicleID, VIN: candidate.VIN, PlateNumber: strings.TrimSpace(candidate.PlateNumber),
|
||||
CustomerID: candidate.CustomerID, ContractID: candidate.ContractID,
|
||||
ContractCode: strings.TrimSpace(candidate.ContractCode), ProjectName: strings.TrimSpace(candidate.ProjectName),
|
||||
OperationStatus: strings.TrimSpace(candidate.OperationStatus), ScopeStartAt: candidate.ScopeStartAt,
|
||||
SourceUpdatedAt: candidate.SourceUpdatedAt,
|
||||
})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].CustomerID != items[j].CustomerID {
|
||||
return items[i].CustomerID < items[j].CustomerID
|
||||
}
|
||||
return items[i].VIN < items[j].VIN
|
||||
})
|
||||
sort.Slice(rejections, func(i, j int) bool { return rejections[i].RowNumber < rejections[j].RowNumber })
|
||||
canonical, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return Snapshot{}, fmt.Errorf("marshal canonical scope: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(canonical)
|
||||
checksum := hex.EncodeToString(sum[:])
|
||||
runID, err := randomHex(16)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
return Snapshot{
|
||||
RunID: runID, SourceVersion: "oneos-v1:" + checksum, Checksum: checksum,
|
||||
GeneratedAt: generatedAt, Candidates: len(candidates), Items: items, Rejections: rejections,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateCandidate(candidate Candidate) string {
|
||||
if !candidate.VehiclePresent || candidate.VehicleID <= 0 {
|
||||
return ReasonVehicleMissing
|
||||
}
|
||||
if normalizeVIN(candidate.VIN) == "" {
|
||||
return ReasonVINMissing
|
||||
}
|
||||
if !candidate.CustomerPresent || candidate.CustomerID <= 0 {
|
||||
return ReasonCustomerMissing
|
||||
}
|
||||
if !candidate.CustomerProfileExists {
|
||||
return ReasonCustomerProfileMissing
|
||||
}
|
||||
if !candidate.ContractPresent || candidate.ContractID <= 0 {
|
||||
return ReasonContractMissing
|
||||
}
|
||||
if !candidate.ContractProfileExists {
|
||||
return ReasonContractProfileMissing
|
||||
}
|
||||
if !candidate.EffectiveCustomerSet || candidate.EffectiveCustomerID <= 0 {
|
||||
return ReasonContractCustomerMissing
|
||||
}
|
||||
if candidate.CustomerID != candidate.EffectiveCustomerID {
|
||||
return ReasonCustomerMismatch
|
||||
}
|
||||
if candidate.ScopeStartAt.IsZero() {
|
||||
return ReasonVehicleMissing
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func rejectionFor(candidate Candidate, reason string) Rejection {
|
||||
rejection := Rejection{RowNumber: candidate.RowNumber, VIN: normalizeVIN(candidate.VIN), ReasonCode: reason}
|
||||
if candidate.VehicleID > 0 {
|
||||
rejection.VehicleID = int64Pointer(candidate.VehicleID)
|
||||
}
|
||||
if candidate.CustomerPresent && candidate.CustomerID > 0 {
|
||||
rejection.CustomerID = int64Pointer(candidate.CustomerID)
|
||||
}
|
||||
if candidate.ContractPresent && candidate.ContractID > 0 {
|
||||
rejection.ContractID = int64Pointer(candidate.ContractID)
|
||||
}
|
||||
return rejection
|
||||
}
|
||||
|
||||
func normalizeVIN(value string) string { return strings.ToUpper(strings.TrimSpace(value)) }
|
||||
func int64Pointer(value int64) *int64 { return &value }
|
||||
|
||||
func randomHex(bytes int) (string, error) {
|
||||
buffer := make([]byte, bytes)
|
||||
if _, err := rand.Read(buffer); err != nil {
|
||||
return "", fmt.Errorf("generate run id: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(buffer), nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func validCandidate(row int, vehicleID, customerID int64, vin string) Candidate {
|
||||
return Candidate{
|
||||
RowNumber: row, VehicleID: vehicleID, VehiclePresent: true, VIN: vin, PlateNumber: "沪A00001",
|
||||
CustomerID: customerID, CustomerPresent: true, CustomerProfileExists: true,
|
||||
ContractID: vehicleID + 1000, ContractPresent: true, ContractProfileExists: true,
|
||||
EffectiveCustomerID: customerID, EffectiveCustomerSet: true,
|
||||
ContractCode: "HT-1", ProjectName: "项目", OperationStatus: "1",
|
||||
ScopeStartAt: time.Date(2026, 7, 1, 8, 0, 0, 0, time.FixedZone("CST", 8*3600)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSnapshotClassifiesAndNormalizes(t *testing.T) {
|
||||
candidates := []Candidate{
|
||||
validCandidate(1, 10, 100, " lvin0001 "),
|
||||
func() Candidate {
|
||||
value := validCandidate(2, 11, 101, "LVIN0002")
|
||||
value.CustomerProfileExists = false
|
||||
return value
|
||||
}(),
|
||||
func() Candidate {
|
||||
value := validCandidate(3, 12, 102, "LVIN0003")
|
||||
value.EffectiveCustomerID = 999
|
||||
return value
|
||||
}(),
|
||||
}
|
||||
snapshot, err := BuildSnapshot(candidates, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snapshot.Candidates != 3 || len(snapshot.Items) != 1 || len(snapshot.Rejections) != 2 {
|
||||
t.Fatalf("unexpected accounting: %#v", snapshot)
|
||||
}
|
||||
if snapshot.Items[0].VIN != "LVIN0001" {
|
||||
t.Fatalf("VIN not normalized: %q", snapshot.Items[0].VIN)
|
||||
}
|
||||
if snapshot.Rejections[0].ReasonCode != ReasonCustomerProfileMissing || snapshot.Rejections[1].ReasonCode != ReasonCustomerMismatch {
|
||||
t.Fatalf("unexpected reasons: %#v", snapshot.Rejections)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSnapshotRejectsEveryDuplicateVehicle(t *testing.T) {
|
||||
snapshot, err := BuildSnapshot([]Candidate{
|
||||
validCandidate(1, 10, 100, "LVIN0001"),
|
||||
validCandidate(2, 10, 100, "LVIN0001"),
|
||||
}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snapshot.Items) != 0 || len(snapshot.Rejections) != 2 {
|
||||
t.Fatalf("duplicates must all be quarantined: %#v", snapshot)
|
||||
}
|
||||
for _, rejection := range snapshot.Rejections {
|
||||
if rejection.ReasonCode != ReasonDuplicateVehicle {
|
||||
t.Fatalf("unexpected duplicate reason: %#v", rejection)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotVersionIsContentAddressed(t *testing.T) {
|
||||
first := []Candidate{validCandidate(1, 10, 100, "LVIN0001"), validCandidate(2, 11, 101, "LVIN0002")}
|
||||
second := []Candidate{first[1], first[0]}
|
||||
a, err := BuildSnapshot(first, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := BuildSnapshot(second, time.Now().Add(time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.SourceVersion != b.SourceVersion || a.Checksum != b.Checksum {
|
||||
t.Fatalf("same content produced different versions: %s %s", a.SourceVersion, b.SourceVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSnapshotUsesCountAndRatioGates(t *testing.T) {
|
||||
snapshot := Snapshot{Candidates: 639, Items: make([]ScopeItem, 613), Rejections: make([]Rejection, 26)}
|
||||
if err := ValidateSnapshot(snapshot, 100, 0.10); err != nil {
|
||||
t.Fatalf("production baseline should pass: %v", err)
|
||||
}
|
||||
if err := ValidateSnapshot(snapshot, 20, 0.10); err == nil {
|
||||
t.Fatal("rejection count limit should fail")
|
||||
}
|
||||
if err := ValidateSnapshot(snapshot, 100, 0.01); err == nil {
|
||||
t.Fatal("rejection ratio limit should fail")
|
||||
}
|
||||
}
|
||||
105
vehicle-data-platform/apps/api/internal/businessscope/publish.go
Normal file
105
vehicle-data-platform/apps/api/internal/businessscope/publish.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PublishResult struct {
|
||||
Changed bool
|
||||
ActiveVersion string
|
||||
}
|
||||
|
||||
func ValidateSnapshot(snapshot Snapshot, maxRejected int, maxRejectRatio float64) error {
|
||||
if snapshot.Candidates <= 0 {
|
||||
return fmt.Errorf("OneOS scope snapshot contains no candidate rows")
|
||||
}
|
||||
if len(snapshot.Items) <= 0 {
|
||||
return fmt.Errorf("OneOS scope snapshot contains no publishable rows")
|
||||
}
|
||||
if snapshot.Candidates != len(snapshot.Items)+len(snapshot.Rejections) {
|
||||
return fmt.Errorf("scope accounting mismatch: candidates=%d accepted=%d rejected=%d", snapshot.Candidates, len(snapshot.Items), len(snapshot.Rejections))
|
||||
}
|
||||
if maxRejected >= 0 && len(snapshot.Rejections) > maxRejected {
|
||||
return fmt.Errorf("scope rejected rows exceed limit: rejected=%d limit=%d", len(snapshot.Rejections), maxRejected)
|
||||
}
|
||||
ratio := float64(len(snapshot.Rejections)) / float64(snapshot.Candidates)
|
||||
if maxRejectRatio >= 0 && ratio > maxRejectRatio {
|
||||
return fmt.Errorf("scope rejected ratio exceeds limit: ratio=%.6f limit=%.6f", ratio, maxRejectRatio)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Publish(ctx context.Context, db *sql.DB, snapshot Snapshot) (PublishResult, error) {
|
||||
tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return PublishResult{}, fmt.Errorf("begin scope publish: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var activeVersion sql.NullString
|
||||
if err := tx.QueryRowContext(ctx, `SELECT active_version FROM business_scope_state WHERE id=1 FOR UPDATE`).Scan(&activeVersion); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("lock business scope state; apply migration 012_business_scope_projection.sql: %w", err)
|
||||
}
|
||||
status := "published"
|
||||
changed := !activeVersion.Valid || activeVersion.String != snapshot.SourceVersion
|
||||
if !changed {
|
||||
status = "unchanged"
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO business_scope_sync_run(
|
||||
run_id,source_system,source_version,status,candidate_count,accepted_count,rejected_count,source_checksum,started_at,finished_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?)`, snapshot.RunID, SourceSystem, snapshot.SourceVersion, status, snapshot.Candidates, len(snapshot.Items), len(snapshot.Rejections), snapshot.Checksum, snapshot.GeneratedAt, now); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("insert business scope sync run: %w", err)
|
||||
}
|
||||
if changed {
|
||||
// A source version can become active again after the source data reverts.
|
||||
// Rebuild that inactive version while holding the state lock so publishing
|
||||
// remains atomic and cannot fail on its existing primary keys.
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM business_customer_vehicle_scope WHERE source_version=?`, snapshot.SourceVersion); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("clear inactive business scope version: %w", err)
|
||||
}
|
||||
statement, err := tx.PrepareContext(ctx, `INSERT INTO business_customer_vehicle_scope(
|
||||
source_version,customer_id,vin,vehicle_id,contract_id,contract_code,plate_number,project_name,operation_status,scope_start_at,source_updated_at,published_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
return PublishResult{}, fmt.Errorf("prepare business scope insert: %w", err)
|
||||
}
|
||||
defer statement.Close()
|
||||
for _, item := range snapshot.Items {
|
||||
var contractID any
|
||||
if item.ContractID > 0 {
|
||||
contractID = item.ContractID
|
||||
}
|
||||
if _, err := statement.ExecContext(ctx, snapshot.SourceVersion, item.CustomerID, item.VIN, item.VehicleID, contractID,
|
||||
item.ContractCode, item.PlateNumber, item.ProjectName, item.OperationStatus, item.ScopeStartAt, item.SourceUpdatedAt, now); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("insert business scope item: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(snapshot.Rejections) > 0 {
|
||||
statement, err := tx.PrepareContext(ctx, `INSERT INTO business_scope_rejection(
|
||||
run_id,row_number,vehicle_id,vin,customer_id,contract_id,reason_code,created_at
|
||||
) VALUES(?,?,?,?,?,?,?,?)`)
|
||||
if err != nil {
|
||||
return PublishResult{}, fmt.Errorf("prepare scope rejection insert: %w", err)
|
||||
}
|
||||
defer statement.Close()
|
||||
for _, rejected := range snapshot.Rejections {
|
||||
if _, err := statement.ExecContext(ctx, snapshot.RunID, rejected.RowNumber, rejected.VehicleID, rejected.VIN,
|
||||
rejected.CustomerID, rejected.ContractID, rejected.ReasonCode, now); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("insert business scope rejection: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE business_scope_state SET
|
||||
active_version=?,source_checksum=?,candidate_count=?,accepted_count=?,rejected_count=?,generated_at=?,published_at=?,last_success_at=?
|
||||
WHERE id=1`, snapshot.SourceVersion, snapshot.Checksum, snapshot.Candidates, len(snapshot.Items), len(snapshot.Rejections), snapshot.GeneratedAt, now, now); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("activate business scope snapshot: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return PublishResult{}, fmt.Errorf("commit business scope snapshot: %w", err)
|
||||
}
|
||||
return PublishResult{Changed: changed, ActiveVersion: snapshot.SourceVersion}, nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
)
|
||||
|
||||
const scopeInsertSQL = `INSERT INTO business_customer_vehicle_scope(
|
||||
source_version,customer_id,vin,vehicle_id,contract_id,contract_code,plate_number,project_name,operation_status,scope_start_at,source_updated_at,published_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`
|
||||
|
||||
func publishSnapshot(version string) Snapshot {
|
||||
generatedAt := time.Date(2026, 7, 14, 8, 0, 0, 0, time.UTC)
|
||||
return Snapshot{
|
||||
RunID: "0123456789abcdef0123456789abcdef", SourceVersion: version,
|
||||
Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
GeneratedAt: generatedAt, Candidates: 1,
|
||||
Items: []ScopeItem{{
|
||||
VehicleID: 10, VIN: "LVIN0001", PlateNumber: "沪A00001", CustomerID: 100,
|
||||
ContractID: 1010, ContractCode: "HT-1", ProjectName: "项目", OperationStatus: "1",
|
||||
ScopeStartAt: generatedAt.Add(-time.Hour),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRebuildsPreviouslyStoredVersionBeforeActivation(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
snapshot := publishSnapshot("oneos-v1:new")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(regexp.QuoteMeta(`SELECT active_version FROM business_scope_state WHERE id=1 FOR UPDATE`)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"active_version"}).AddRow("oneos-v1:current"))
|
||||
mock.ExpectExec(`INSERT INTO business_scope_sync_run`).
|
||||
WithArgs(snapshot.RunID, SourceSystem, snapshot.SourceVersion, "published", 1, 1, 0, snapshot.Checksum, snapshot.GeneratedAt, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM business_customer_vehicle_scope WHERE source_version=?`)).
|
||||
WithArgs(snapshot.SourceVersion).WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
prepared := mock.ExpectPrepare(regexp.QuoteMeta(scopeInsertSQL))
|
||||
item := snapshot.Items[0]
|
||||
prepared.ExpectExec().WithArgs(
|
||||
snapshot.SourceVersion, item.CustomerID, item.VIN, item.VehicleID, item.ContractID,
|
||||
item.ContractCode, item.PlateNumber, item.ProjectName, item.OperationStatus, item.ScopeStartAt, nil, sqlmock.AnyArg(),
|
||||
).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(`UPDATE business_scope_state SET`).
|
||||
WithArgs(snapshot.SourceVersion, snapshot.Checksum, 1, 1, 0, snapshot.GeneratedAt, sqlmock.AnyArg(), sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
result, err := Publish(context.Background(), db, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Changed || result.ActiveVersion != snapshot.SourceVersion {
|
||||
t.Fatalf("unexpected publish result: %#v", result)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishUnchangedDoesNotRewriteScopeRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
snapshot := publishSnapshot("oneos-v1:same")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(regexp.QuoteMeta(`SELECT active_version FROM business_scope_state WHERE id=1 FOR UPDATE`)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"active_version"}).AddRow(snapshot.SourceVersion))
|
||||
mock.ExpectExec(`INSERT INTO business_scope_sync_run`).
|
||||
WithArgs(snapshot.RunID, SourceSystem, snapshot.SourceVersion, "unchanged", 1, 1, 0, snapshot.Checksum, snapshot.GeneratedAt, sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(`UPDATE business_scope_state SET`).
|
||||
WithArgs(snapshot.SourceVersion, snapshot.Checksum, 1, 1, 0, snapshot.GeneratedAt, sqlmock.AnyArg(), sqlmock.AnyArg()).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectCommit()
|
||||
|
||||
result, err := Publish(context.Background(), db, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Changed {
|
||||
t.Fatalf("unchanged snapshot reported as changed: %#v", result)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishRollsBackWhenVersionCannotBeCleared(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
snapshot := publishSnapshot("oneos-v1:new")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectQuery(regexp.QuoteMeta(`SELECT active_version FROM business_scope_state WHERE id=1 FOR UPDATE`)).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"active_version"}).AddRow("oneos-v1:current"))
|
||||
mock.ExpectExec(`INSERT INTO business_scope_sync_run`).WillReturnResult(sqlmock.NewResult(1, 1))
|
||||
mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM business_customer_vehicle_scope WHERE source_version=?`)).
|
||||
WithArgs(snapshot.SourceVersion).WillReturnError(errors.New("delete failed"))
|
||||
mock.ExpectRollback()
|
||||
|
||||
if _, err := Publish(context.Background(), db, snapshot); err == nil {
|
||||
t.Fatal("publish should fail when stale version cannot be rebuilt")
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
200
vehicle-data-platform/apps/api/internal/businessscope/source.go
Normal file
200
vehicle-data-platform/apps/api/internal/businessscope/source.go
Normal file
@@ -0,0 +1,200 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// candidateQuery deliberately derives the active lease lifecycle from delivery and
|
||||
// return task facts. vehicle_lease_order_record remains an independent customer
|
||||
// ownership check, but its last_return_time is not authoritative: saving a return
|
||||
// draft currently updates that aggregate field before the return is completed.
|
||||
const candidateQuery = `SELECT
|
||||
dv.vehicle_id,
|
||||
v.id,
|
||||
COALESCE(v.vin, ''),
|
||||
COALESCE(v.plate_number, dv.plate_number, r.plate_number, ''),
|
||||
r.customer_id,
|
||||
ci.id,
|
||||
dv.contract_id,
|
||||
co.id,
|
||||
COALESCE(co.other_customer_id, co.customer_id),
|
||||
COALESCE(co.contract_code, r.contract_code, ''),
|
||||
COALESCE(co.project_name, r.project_name, ''),
|
||||
COALESCE(vs.operation_status, ''),
|
||||
dv.delivery_time,
|
||||
dv.update_time
|
||||
FROM delivery_vehicle dv
|
||||
LEFT JOIN vehicle_info v ON v.id = dv.vehicle_id AND v.del_flag = '0'
|
||||
LEFT JOIN vehicle_lease_order_record r ON r.vehicle_id = dv.vehicle_id AND r.del_flag = '0'
|
||||
LEFT JOIN customer_info ci ON ci.id = r.customer_id AND ci.del_flag = '0'
|
||||
LEFT JOIN vehicle_lease_contract_info co ON co.id = dv.contract_id AND co.del_flag = '0'
|
||||
LEFT JOIN vehicle_status vs ON vs.vehicle_id = dv.vehicle_id AND vs.del_flag = '0'
|
||||
WHERE dv.del_flag = '0'
|
||||
AND dv.delivery_status IN (2, 3)
|
||||
AND dv.vehicle_id IS NOT NULL
|
||||
AND dv.delivery_time IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM delivery_vehicle newer
|
||||
WHERE newer.del_flag = '0'
|
||||
AND newer.vehicle_id = dv.vehicle_id
|
||||
AND newer.vehicle_id IS NOT NULL
|
||||
AND (
|
||||
COALESCE(newer.delivery_time, '1000-01-01') > COALESCE(dv.delivery_time, '1000-01-01')
|
||||
OR (
|
||||
COALESCE(newer.delivery_time, '1000-01-01') = COALESCE(dv.delivery_time, '1000-01-01')
|
||||
AND newer.id > dv.id
|
||||
)
|
||||
)
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM return_vehicle_task rt
|
||||
WHERE rt.delivery_vehicle_id = dv.id
|
||||
AND rt.del_flag = '0'
|
||||
AND rt.status IN (2, 3, 5)
|
||||
)
|
||||
ORDER BY dv.vehicle_id, dv.id, r.id`
|
||||
|
||||
func ReadCandidates(ctx context.Context, db *sql.DB) ([]Candidate, error) {
|
||||
connection, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acquire OneOS connection: %w", err)
|
||||
}
|
||||
defer connection.Close()
|
||||
if err := VerifyReadOnlyGrants(ctx, connection); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := connection.ExecContext(ctx, `SET SESSION TRANSACTION READ ONLY`); err != nil {
|
||||
return nil, fmt.Errorf("force OneOS session read only: %w", err)
|
||||
}
|
||||
if _, err := connection.ExecContext(ctx, `SET SESSION MAX_EXECUTION_TIME = 10000`); err != nil {
|
||||
return nil, fmt.Errorf("set OneOS query timeout: %w", err)
|
||||
}
|
||||
tx, err := connection.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("begin OneOS read-only snapshot: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, candidateQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query OneOS customer vehicle candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
candidates := make([]Candidate, 0, 1024)
|
||||
for rows.Next() {
|
||||
var vehicleID, vehicleProfileID, customerID, customerProfileID sql.NullInt64
|
||||
var contractID, contractProfileID, effectiveCustomerID sql.NullInt64
|
||||
var vin, plate, contractCode, projectName, operationStatus string
|
||||
var scopeStart time.Time
|
||||
var updatedAt sql.NullTime
|
||||
if err := rows.Scan(
|
||||
&vehicleID, &vehicleProfileID, &vin, &plate, &customerID, &customerProfileID,
|
||||
&contractID, &contractProfileID, &effectiveCustomerID, &contractCode, &projectName,
|
||||
&operationStatus, &scopeStart, &updatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan OneOS scope candidate: %w", err)
|
||||
}
|
||||
candidate := Candidate{
|
||||
RowNumber: len(candidates) + 1,
|
||||
VehicleID: vehicleID.Int64, VehiclePresent: vehicleID.Valid && vehicleProfileID.Valid,
|
||||
VIN: vin, PlateNumber: plate,
|
||||
CustomerID: customerID.Int64, CustomerPresent: customerID.Valid,
|
||||
CustomerProfileExists: customerProfileID.Valid,
|
||||
ContractID: contractID.Int64, ContractPresent: contractID.Valid,
|
||||
ContractProfileExists: contractProfileID.Valid,
|
||||
EffectiveCustomerID: effectiveCustomerID.Int64, EffectiveCustomerSet: effectiveCustomerID.Valid,
|
||||
ContractCode: contractCode, ProjectName: projectName, OperationStatus: operationStatus,
|
||||
ScopeStartAt: scopeStart,
|
||||
}
|
||||
if updatedAt.Valid {
|
||||
value := updatedAt.Time
|
||||
candidate.SourceUpdatedAt = &value
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate OneOS scope candidates: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, fmt.Errorf("finish OneOS read-only snapshot: %w", err)
|
||||
}
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
func VerifyReadOnlyGrants(ctx context.Context, connection *sql.Conn) error {
|
||||
rows, err := connection.QueryContext(ctx, `SHOW GRANTS FOR CURRENT_USER`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect OneOS database grants: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
grants := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var grant string
|
||||
if err := rows.Scan(&grant); err != nil {
|
||||
return fmt.Errorf("scan OneOS database grant: %w", err)
|
||||
}
|
||||
grants = append(grants, grant)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate OneOS database grants: %w", err)
|
||||
}
|
||||
if err := ValidateReadOnlyGrants(grants); err != nil {
|
||||
return fmt.Errorf("OneOS database account is not read-only: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateReadOnlyGrants(grants []string) error {
|
||||
if len(grants) == 0 {
|
||||
return fmt.Errorf("SHOW GRANTS returned no rows")
|
||||
}
|
||||
allowedSelectScopes := map[string]bool{
|
||||
"LN_ASSET_MANAGEMENT.VEHICLE_LEASE_ORDER_RECORD": true,
|
||||
"LN_ASSET_MANAGEMENT.DELIVERY_VEHICLE": true,
|
||||
"LN_ASSET_MANAGEMENT.RETURN_VEHICLE_TASK": true,
|
||||
"LN_ASSET_MANAGEMENT.VEHICLE_INFO": true,
|
||||
"LN_ASSET_MANAGEMENT.CUSTOMER_INFO": true,
|
||||
"LN_ASSET_MANAGEMENT.VEHICLE_LEASE_CONTRACT_INFO": true,
|
||||
"LN_ASSET_MANAGEMENT.VEHICLE_STATUS": true,
|
||||
}
|
||||
for _, raw := range grants {
|
||||
grant := strings.ToUpper(strings.TrimSpace(raw))
|
||||
if strings.HasPrefix(grant, "SET DEFAULT ROLE ") {
|
||||
return fmt.Errorf("role-based grants are not accepted; grant SELECT directly to the sync account")
|
||||
}
|
||||
if !strings.HasPrefix(grant, "GRANT ") {
|
||||
return fmt.Errorf("unsupported grant statement")
|
||||
}
|
||||
onIndex := strings.Index(grant, " ON ")
|
||||
if onIndex < 0 {
|
||||
return fmt.Errorf("unsupported role or dynamic grant")
|
||||
}
|
||||
privileges := strings.TrimSpace(strings.TrimPrefix(grant[:onIndex], "GRANT "))
|
||||
toIndex := strings.Index(grant[onIndex+4:], " TO ")
|
||||
if toIndex < 0 {
|
||||
return fmt.Errorf("unsupported grant scope")
|
||||
}
|
||||
scope := strings.ReplaceAll(strings.TrimSpace(grant[onIndex+4:onIndex+4+toIndex]), "`", "")
|
||||
for _, privilege := range strings.Split(privileges, ",") {
|
||||
privilege = strings.TrimSpace(privilege)
|
||||
switch privilege {
|
||||
case "USAGE":
|
||||
if scope != "*.*" {
|
||||
return fmt.Errorf("USAGE has unsupported scope %q", scope)
|
||||
}
|
||||
case "SELECT":
|
||||
if !allowedSelectScopes[scope] {
|
||||
return fmt.Errorf("SELECT scope %q is not required by the sync query", scope)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("disallowed privilege %q", privilege)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package businessscope
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCandidateQueryUsesCompletedLifecycleFactsInsteadOfAggregateReturnTime(t *testing.T) {
|
||||
assertions := []string{
|
||||
"FROM delivery_vehicle dv",
|
||||
"dv.delivery_status IN (2, 3)",
|
||||
"FROM return_vehicle_task rt",
|
||||
"rt.status IN (2, 3, 5)",
|
||||
"FROM delivery_vehicle newer",
|
||||
"LEFT JOIN vehicle_lease_order_record r",
|
||||
}
|
||||
for _, expected := range assertions {
|
||||
if !strings.Contains(candidateQuery, expected) {
|
||||
t.Fatalf("candidate query missing %q", expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(candidateQuery, "r.last_return_time") {
|
||||
t.Fatal("candidate query must not trust aggregate last_return_time written by return drafts")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReadOnlyGrantsAllowsOnlyRequiredTableSelects(t *testing.T) {
|
||||
grants := []string{
|
||||
"GRANT USAGE ON *.* TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`vehicle_lease_order_record` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`delivery_vehicle` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`return_vehicle_task` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`vehicle_info` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`customer_info` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`vehicle_lease_contract_info` TO `reader`@`%`",
|
||||
"GRANT SELECT ON `ln_asset_management`.`vehicle_status` TO `reader`@`%`",
|
||||
}
|
||||
if err := ValidateReadOnlyGrants(grants); err != nil {
|
||||
t.Fatalf("read-only grants rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateReadOnlyGrantsRejectsBroadOrWritePrivileges(t *testing.T) {
|
||||
cases := [][]string{
|
||||
{"GRANT ALL PRIVILEGES ON `ln_asset_management`.* TO `reader`@`%`"},
|
||||
{"GRANT SELECT, UPDATE ON `ln_asset_management`.* TO `reader`@`%`"},
|
||||
{"GRANT SELECT ON *.* TO `reader`@`%`"},
|
||||
{"GRANT SELECT ON `ln_asset_management`.* TO `reader`@`%`"},
|
||||
{"GRANT SELECT ON `ry-cloud`.`sys_user` TO `reader`@`%`"},
|
||||
{"GRANT SHOW VIEW ON `ln_asset_management`.`vehicle_info` TO `reader`@`%`"},
|
||||
{"GRANT PROCESS, REPLICATION CLIENT ON *.* TO `reader`@`%`"},
|
||||
{"GRANT `scope_reader_role`@`%` TO `reader`@`%`"},
|
||||
{"SET DEFAULT ROLE `scope_reader_role`@`%` TO `reader`@`%`"},
|
||||
}
|
||||
for _, grants := range cases {
|
||||
if err := ValidateReadOnlyGrants(grants); err == nil {
|
||||
t.Fatalf("unsafe grants accepted: %#v", grants)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user