feat: make database migration testable and backfill assets

2026-09-17 02:42:38 +08:00
parent 4607312b7a
commit 7169a7b5fd
4 changed files with 187 additions and 7 deletions
@@ -227,7 +227,7 @@ func (a *App) bootstrapAdmin() {
 	if err := a.db.Create(&u).Error; err != nil {
 		panic(err)
 	}
-	if err := a.db.Model(&Task{}).Where("user_id = 0").Update("user_id", u.ID).Error; err != nil {
+	if err := a.db.Model(&Task{}).Where("user_id = 0 OR user_id IS NULL").Update("user_id", u.ID).Error; err != nil {
 		panic(err)
 	}
 	if err := a.db.Create(&CreditAccount{UserID: u.ID}).Error; err != nil {
@@ -15,18 +15,22 @@ func main() {
 	_ = os.MkdirAll(filepath.Dir(dbPath), 0755)
 	db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
 	must(err)
-	must(db.AutoMigrate(&Task{}, &MediaAsset{}, &Setting{}, &User{}, &UserPermission{}, &CreditAccount{}, &CreditTransaction{}, &CreditReservation{}, &RedeemBatch{}, &RedeemCode{}, &RedeemRecord{}, &WebDAVConfig{}, &Session{}, &AuditLog{}))
-	// Replace legacy global task/reservation uniqueness constraints with scoped indexes.
-	must(db.Exec("DROP INDEX IF EXISTS idx_tasks_source_rid_quality").Error)
-	must(db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_user_source_rid_quality ON tasks(user_id, source, r_id, quality)").Error)
-	must(db.Exec("DROP INDEX IF EXISTS idx_credit_reservations_task_id").Error)
-	must(db.Exec("CREATE INDEX IF NOT EXISTS idx_credit_reservations_task_id ON credit_reservations(task_id)").Error)
+	must(migrateDatabase(db))

 	client := &http.Client{Timeout: 2 * 60 * 1e9}
 	cfg := loadConfig(db)
 	a := &App{db: db, cfg: cfg, queue: make(chan uint, 100), httpClient: client}
 	a.loadSettings()
 	a.bootstrapAdmin()
+	must(backfillMediaAssets(db, a.cfg.DownloadDir))
+	if os.Getenv("MIGRATION_CHECK") == "1" {
+		var users, tasks, assets int64
+		db.Model(&User{}).Count(&users)
+		db.Model(&Task{}).Count(&tasks)
+		db.Model(&MediaAsset{}).Count(&assets)
+		println("migration check ok", "users", users, "tasks", tasks, "assets", assets)
+		return
+	}
 	normalizeConfig(&a.cfg)
 	a.am = &AMAPIClient{BaseURL: a.cfg.AMAPI, HTTP: client}
 	if a.cfg.WebDAVURL != "" {
@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gorm.io/gorm"
+)
+
+// migrateDatabase applies schema changes and repairs legacy indexes. It is
+// intentionally separate from startup so migration behavior can be tested on
+// a disposable SQLite database before touching production data.
+func migrateDatabase(db *gorm.DB) error {
+	if err := db.AutoMigrate(&Task{}, &MediaAsset{}, &Setting{}, &User{}, &UserPermission{}, &CreditAccount{}, &CreditTransaction{}, &CreditReservation{}, &RedeemBatch{}, &RedeemCode{}, &RedeemRecord{}, &WebDAVConfig{}, &Session{}, &AuditLog{}); err != nil {
+		return err
+	}
+	if err := db.Exec("DROP INDEX IF EXISTS idx_tasks_source_rid_quality").Error; err != nil {
+		return err
+	}
+	if err := db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_user_source_rid_quality ON tasks(user_id, source, r_id, quality)").Error; err != nil {
+		return err
+	}
+	if err := db.Exec("DROP INDEX IF EXISTS idx_credit_reservations_task_id").Error; err != nil {
+		return err
+	}
+	return db.Exec("CREATE INDEX IF NOT EXISTS idx_credit_reservations_task_id ON credit_reservations(task_id)").Error
+}
+
+// backfillMediaAssets turns legacy completed tasks into shared media assets.
+// Existing physical files are reused; no media is moved or deleted.
+func backfillMediaAssets(db *gorm.DB, downloadDir string) error {
+	var tasks []Task
+	if err := db.Where("(asset_id = 0 OR asset_id IS NULL) AND status = ?", "done").Find(&tasks).Error; err != nil {
+		return err
+	}
+	for i := range tasks {
+		t := &tasks[i]
+		key := assetKey(t.Source, t.RID, t.Quality)
+		var asset MediaAsset
+		result := db.Where("asset_key = ?", key).Limit(1).Find(&asset)
+		if result.Error != nil {
+			return result.Error
+		}
+		if result.RowsAffected == 0 {
+			asset = MediaAsset{
+				AssetKey:      key,
+				Source:        t.Source,
+				RID:           t.RID,
+				Quality:       t.Quality,
+				LocalPath:     t.LocalPath,
+				Format:        t.Format,
+				ActualQuality: t.ActualQuality,
+				Size:          t.DownloadedBytes,
+				Status:        "done",
+			}
+			if asset.LocalPath != "" {
+				if fi, statErr := os.Stat(asset.LocalPath); statErr == nil && fi.Size() > 0 {
+					asset.Size = fi.Size()
+				}
+			} else if downloadDir != "" {
+				candidate := filepath.Join(downloadDir, safe(t.Artist)+" - "+safe(t.Title)+"."+t.Format)
+				if fi, statErr := os.Stat(candidate); statErr == nil && fi.Size() > 0 {
+					asset.LocalPath, asset.Size = candidate, fi.Size()
+				}
+			}
+			if err := db.Create(&asset).Error; err != nil {
+				return fmt.Errorf("create media asset for task %d: %w", t.ID, err)
+			}
+		}
+		if t.AssetID != asset.ID {
+			if err := db.Model(t).Update("asset_id", asset.ID).Error; err != nil {
+				return fmt.Errorf("link media asset for task %d: %w", t.ID, err)
+			}
+		}
+	}
+	return nil
+}
@@ -0,0 +1,98 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/glebarez/sqlite"
+	"gorm.io/gorm"
+)
+
+func openMigrationTestDB(t *testing.T) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "migration.db")), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db
+}
+
+func TestMigrateDatabaseLegacySchema(t *testing.T) {
+	db := openMigrationTestDB(t)
+	if err := db.Exec(`CREATE TABLE tasks (
+		id integer PRIMARY KEY AUTOINCREMENT,
+		source text, r_id text, title text, artist text, album text, pic text,
+		quality text, format text, status text, error text, local_path text,
+		remote_path text, created_at datetime, updated_at datetime, total_bytes integer,
+		downloaded_bytes integer, progress real, resolved_quality text, actual_quality text
+	)`).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Exec("CREATE UNIQUE INDEX idx_tasks_source_rid_quality ON tasks(source, r_id, quality)").Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Exec("INSERT INTO tasks(source,r_id,quality,status) VALUES('kw','legacy-1','standard','done')").Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := migrateDatabase(db); err != nil {
+		t.Fatal(err)
+	}
+
+	var task Task
+	if err := db.First(&task).Error; err != nil {
+		t.Fatal(err)
+	}
+	if task.UserID != 0 {
+		t.Fatalf("legacy task user id = %d, want 0 before bootstrap", task.UserID)
+	}
+	var indexes []struct{ Name string }
+	if err := db.Raw("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='tasks'").Scan(&indexes).Error; err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	for _, idx := range indexes {
+		if idx.Name == "idx_tasks_user_source_rid_quality" {
+			found = true
+		}
+		if idx.Name == "idx_tasks_source_rid_quality" {
+			t.Fatalf("legacy global task index still exists")
+		}
+	}
+	if !found {
+		t.Fatalf("scoped task index not created")
+	}
+}
+
+func TestBackfillMediaAssetsLinksExistingFiles(t *testing.T) {
+	db := openMigrationTestDB(t)
+	if err := migrateDatabase(db); err != nil {
+		t.Fatal(err)
+	}
+	dir := t.TempDir()
+	path := filepath.Join(dir, "Artist - Title.mp3")
+	if err := os.WriteFile(path, []byte("audio"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	task := Task{UserID: 1, Source: "kw", RID: "rid-1", Title: "Title", Artist: "Artist", Quality: "standard", Format: "mp3", Status: "done", LocalPath: path}
+	if err := db.Create(&task).Error; err != nil {
+		t.Fatal(err)
+	}
+	if err := backfillMediaAssets(db, dir); err != nil {
+		t.Fatal(err)
+	}
+	var asset MediaAsset
+	if err := db.Where("asset_key = ?", assetKey(task.Source, task.RID, task.Quality)).First(&asset).Error; err != nil {
+		t.Fatal(err)
+	}
+	if asset.Status != "done" || asset.LocalPath != path || asset.Size != int64(len("audio")) {
+		t.Fatalf("unexpected asset: %+v", asset)
+	}
+	var got Task
+	if err := db.First(&got, task.ID).Error; err != nil {
+		t.Fatal(err)
+	}
+	if got.AssetID != asset.ID {
+		t.Fatalf("task asset id = %d, want %d", got.AssetID, asset.ID)
+	}
+}