feat: add custom seed path to config (#2702)

* chore: replace SeedDataPath by DefaultSeedDataPath

* wip: add path seed matching logic

* chore: add test for utils.GetSeedFiles

* chore: wip tests mock

* chore: fix lint

* chore: show seed path

* chore: change comment

* chore: apply pr suggestions

* chore: fix lint

* chore: keep default value assignation

* chore: remove DefaultSeedPath

* chore: keep consistent WARNING message

* chore: inline get seed file path

* chore: address review comments

---------

Co-authored-by: Qiao Han <qiao@supabase.io>

cmd/db.go

@@ -276,7 +276,7 @@ func init() {
 	pushFlags := dbPushCmd.Flags()
 	pushFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.")
 	pushFlags.BoolVar(&includeRoles, "include-roles", false, "Include custom roles from "+utils.CustomRolesPath+".")
-	pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from "+utils.SeedDataPath+".")
+	pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from your config.")
 	pushFlags.BoolVar(&dryRun, "dry-run", false, "Print the migrations that would be applied, but don't actually apply them.")
 	pushFlags.String("db-url", "", "Pushes to the database specified by the connection string (must be percent-encoded).")
 	pushFlags.Bool("linked", true, "Pushes to the linked project.")

cmd/start.go

@@ -27,7 +27,7 @@ func validateExcludedContainers(excludedContainers []string) {
 		// Sort the names list so it's easier to visually spot the one you looking for
 		sort.Strings(validContainers)
 		warning := fmt.Sprintf("%s The following container names are not valid to exclude: %s\nValid containers to exclude are: %s\n",
-			utils.Yellow("Warning:"),
+			utils.Yellow("WARNING:"),
 			utils.Aqua(strings.Join(invalidContainers, ", ")),
 			utils.Aqua(strings.Join(validContainers, ", ")))
 		fmt.Fprint(os.Stderr, warning)

internal/db/push/push.go

@@ -41,7 +41,11 @@ func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles,
 		fmt.Fprintln(os.Stderr, "Would push these migrations:")
 		fmt.Fprint(os.Stderr, utils.Bold(confirmPushAll(pending)))
 		if includeSeed {
-			fmt.Fprintln(os.Stderr, "Would seed data "+utils.Bold(utils.SeedDataPath)+"...")
+			seedPaths, err := utils.GetSeedFiles(fsys)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(os.Stderr, "Would seed data %v...\n", seedPaths)
 		}
 	} else {
 		msg := fmt.Sprintf("Do you want to push these migrations to the remote database?\n%s\n", confirmPushAll(pending))

internal/db/push/push_test.go

@@ -162,7 +162,9 @@ func TestPushAll(t *testing.T) {
 
 	t.Run("throws error on seed failure", func(t *testing.T) {
 		// Setup in-memory fs
-		fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath}
+		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
+		fsys := &fstest.OpenErrorFs{DenyPath: seedPath}
+		_, _ = fsys.Create(seedPath)
 		path := filepath.Join(utils.MigrationsDir, "0_test.sql")
 		require.NoError(t, afero.WriteFile(fsys, path, []byte{}, 0644))
 		// Setup mock postgres

internal/db/reset/reset_test.go

@@ -362,7 +362,7 @@ func TestResetRemote(t *testing.T) {
 		fsys := afero.NewMemMapFs()
 		path := filepath.Join(utils.MigrationsDir, "0_schema.sql")
 		require.NoError(t, afero.WriteFile(fsys, path, nil, 0644))
-		seedPath := filepath.Join(utils.SeedDataPath)
+		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
 		// Will raise an error when seeding
 		require.NoError(t, afero.WriteFile(fsys, seedPath, []byte("INSERT INTO test_table;"), 0644))
 		// Setup mock postgres

internal/db/start/start_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/docker/docker/api/types"
@@ -60,7 +61,7 @@ func TestStartDatabase(t *testing.T) {
 		roles := "create role test"
 		require.NoError(t, afero.WriteFile(fsys, utils.CustomRolesPath, []byte(roles), 0644))
 		seed := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(seed), 0644))
+		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(seed), 0644))
 		// Setup mock docker
 		require.NoError(t, apitest.MockDocker(utils.Docker))
 		defer gock.OffAll()

internal/init/init.go

@@ -40,19 +40,14 @@ func Run(ctx context.Context, fsys afero.Fs, createVscodeSettings, createIntelli
 		return err
 	}
 
-	// 2. Create `seed.sql`.
-	if err := initSeed(fsys); err != nil {
-		return err
-	}
-
-	// 3. Append to `.gitignore`.
+	// 2. Append to `.gitignore`.
 	if utils.IsGitRepo() {
 		if err := updateGitIgnore(utils.GitIgnorePath, fsys); err != nil {
 			return err
 		}
 	}
 
-	// 4. Generate VS Code settings.
+	// 3. Generate VS Code settings.
 	if createVscodeSettings != nil {
 		if *createVscodeSettings {
 			return writeVscodeConfig(fsys)
@@ -77,15 +72,6 @@ func Run(ctx context.Context, fsys afero.Fs, createVscodeSettings, createIntelli
 	return nil
 }
 
-func initSeed(fsys afero.Fs) error {
-	f, err := fsys.OpenFile(utils.SeedDataPath, os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		return errors.Errorf("failed to create seed file: %w", err)
-	}
-	defer f.Close()
-	return nil
-}
-
 func updateGitIgnore(ignorePath string, fsys afero.Fs) error {
 	var contents []byte
 

internal/init/init_test.go

@@ -28,10 +28,6 @@ func TestInitCommand(t *testing.T) {
 		exists, err = afero.Exists(fsys, utils.GitIgnorePath)
 		assert.NoError(t, err)
 		assert.True(t, exists)
-		// Validate generated seed.sql
-		exists, err = afero.Exists(fsys, utils.SeedDataPath)
-		assert.NoError(t, err)
-		assert.True(t, exists)
 		// Validate vscode settings file isn't generated
 		exists, err = afero.Exists(fsys, settingsPath)
 		assert.NoError(t, err)
@@ -70,15 +66,6 @@ func TestInitCommand(t *testing.T) {
 		assert.Error(t, Run(context.Background(), fsys, nil, nil, utils.InitParams{}))
 	})
 
-	t.Run("throws error on seed failure", func(t *testing.T) {
-		// Setup in-memory fs
-		fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath}
-		// Run test
-		err := Run(context.Background(), fsys, nil, nil, utils.InitParams{})
-		// Check error
-		assert.ErrorIs(t, err, os.ErrPermission)
-	})
-
 	t.Run("creates vscode settings file", func(t *testing.T) {
 		// Setup in-memory fs
 		fsys := &afero.MemMapFs{}

internal/migration/apply/apply.go

@@ -27,11 +27,11 @@ func MigrateAndSeed(ctx context.Context, version string, conn *pgx.Conn, fsys af
 }
 
 func SeedDatabase(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {
-	err := migration.SeedData(ctx, []string{utils.SeedDataPath}, conn, afero.NewIOFS(fsys))
-	if errors.Is(err, os.ErrNotExist) {
-		return nil
+	seedPaths, err := utils.GetSeedFiles(fsys)
+	if err != nil {
+		return err
 	}
-	return err
+	return migration.SeedData(ctx, seedPaths, conn, afero.NewIOFS(fsys))
 }
 
 func CreateCustomRoles(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {

internal/migration/apply/apply_test.go

@@ -44,7 +44,7 @@ func TestMigrateDatabase(t *testing.T) {
 		path := filepath.Join(utils.MigrationsDir, "0_test.sql")
 		sql := "create schema public"
 		require.NoError(t, afero.WriteFile(fsys, path, []byte(sql), 0644))
-		seedPath := filepath.Join(utils.SeedDataPath)
+		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
 		// This will raise an error when seeding
 		require.NoError(t, afero.WriteFile(fsys, seedPath, []byte("INSERT INTO test_table;"), 0644))
 		// Setup mock postgres
@@ -82,7 +82,7 @@ func TestSeedDatabase(t *testing.T) {
 		fsys := afero.NewMemMapFs()
 		// Setup seed file
 		sql := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(sql), 0644))
+		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644))
 		// Setup mock postgres
 		conn := pgtest.NewConn()
 		defer conn.Close(t)
@@ -100,7 +100,9 @@ func TestSeedDatabase(t *testing.T) {
 
 	t.Run("throws error on read failure", func(t *testing.T) {
 		// Setup in-memory fs
-		fsys := &fstest.OpenErrorFs{DenyPath: utils.SeedDataPath}
+		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
+		fsys := &fstest.OpenErrorFs{DenyPath: seedPath}
+		_, _ = fsys.Create(seedPath)
 		// Run test
 		err := SeedDatabase(context.Background(), nil, fsys)
 		// Check error
@@ -112,7 +114,7 @@ func TestSeedDatabase(t *testing.T) {
 		fsys := afero.NewMemMapFs()
 		// Setup seed file
 		sql := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, utils.SeedDataPath, []byte(sql), 0644))
+		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644))
 		// Setup mock postgres
 		conn := pgtest.NewConn()
 		defer conn.Close(t)

internal/utils/misc.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"time"
 
 	"github.com/docker/docker/client"
@@ -148,7 +149,6 @@ var (
 	FallbackImportMapPath = filepath.Join(FunctionsDir, "import_map.json")
 	FallbackEnvFilePath   = filepath.Join(FunctionsDir, ".env")
 	DbTestsDir            = filepath.Join(SupabaseDirPath, "tests")
-	SeedDataPath          = filepath.Join(SupabaseDirPath, "seed.sql")
 	CustomRolesPath       = filepath.Join(SupabaseDirPath, "roles.sql")
 
 	ErrNotLinked   = errors.Errorf("Cannot find project ref. Have you run %s?", Aqua("supabase link"))
@@ -157,6 +157,26 @@ var (
 	ErrNotRunning  = errors.Errorf("%s is not running.", Aqua("supabase start"))
 )
 
+// Match the glob patterns from the config to get a deduplicated
+// array of all migrations files to apply in the declared order.
+func GetSeedFiles(fsys afero.Fs) ([]string, error) {
+	seedPaths := Config.Db.Seed.SqlPaths
+	var files []string
+	for _, pattern := range seedPaths {
+		fullPattern := filepath.Join(SupabaseDirPath, pattern)
+		matches, err := afero.Glob(fsys, fullPattern)
+		if err != nil {
+			return nil, errors.Errorf("failed to apply glob pattern for %w", err)
+		}
+		if len(matches) == 0 {
+			fmt.Fprintf(os.Stderr, "%s Your pattern %s matched 0 seed files.\n", Yellow("WARNING:"), pattern)
+		}
+		sort.Strings(matches)
+		files = append(files, matches...)
+	}
+	return RemoveDuplicates(files), nil
+}
+
 func GetCurrentTimestamp() string {
 	// Magic number: https://stackoverflow.com/q/45160822.
 	return time.Now().UTC().Format("20060102150405")

internal/utils/misc_test.go

@@ -75,3 +75,76 @@ func TestProjectRoot(t *testing.T) {
 		assert.Equal(t, cwd, path)
 	})
 }
+
+func TestGetSeedFiles(t *testing.T) {
+	t.Run("returns seed files matching patterns", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Create seed files
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644))
+		// Mock config patterns
+		Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql"}
+
+		// Run test
+		files, err := GetSeedFiles(fsys)
+
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql"}, files)
+	})
+	t.Run("returns seed files matching patterns skip duplicates", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Create seed files
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
+		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644))
+		// Mock config patterns
+		Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql", "seeds/seed*.sql"}
+
+		// Run test
+		files, err := GetSeedFiles(fsys)
+
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql", "supabase/seeds/seed3.sql"}, files)
+	})
+
+	t.Run("returns error on invalid pattern", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Mock config patterns
+		Config.Db.Seed.SqlPaths = []string{"[*!#@D#"}
+
+		// Run test
+		files, err := GetSeedFiles(fsys)
+
+		// Check error
+		assert.Nil(t, err)
+		// The resuling seed list should be empty
+		assert.ElementsMatch(t, []string{}, files)
+	})
+
+	t.Run("returns empty list if no files match", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Mock config patterns
+		Config.Db.Seed.SqlPaths = []string{"seeds/*.sql"}
+
+		// Run test
+		files, err := GetSeedFiles(fsys)
+
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.Empty(t, files)
+	})
+}

pkg/config/config.go

@@ -172,7 +172,8 @@ type (
 	}
 
 	seed struct {
-		Enabled bool `toml:"enabled"`
+		Enabled  bool     `toml:"enabled"`
+		SqlPaths []string `toml:"sql_paths"`
 	}
 
 	pooler struct {
@@ -482,7 +483,8 @@ func NewConfig(editors ...ConfigEditor) config {
 				SecretKeyBase: "EAx3IQ/wRG1v47ZD4NE4/9RzBI8Jmil3x0yhcW4V2NHBP6c2iPIzwjofi2Ep4HIG",
 			},
 			Seed: seed{
-				Enabled: true,
+				Enabled:  true,
+				SqlPaths: []string{"./seed.sql"},
 			},
 		},
 		Realtime: realtime{

pkg/config/templates/config.toml

@@ -39,6 +39,14 @@ default_pool_size = 20
 # Maximum number of client connections allowed.
 max_client_conn = 100
 
+[db.seed]
+# If enabled, seeds the database after migrations during a db reset.
+enabled = true
+# Specifies an ordered list of seed files to load during db reset.
+# Supports glob patterns relative to supabase directory. For example:
+# sql_paths = ['./seeds/*.sql', '../project-src/seeds/*-load-testing.sql']
+sql_paths = ['./seed.sql']
+
 [realtime]
 enabled = true
 # Bind realtime via either IPv4 or IPv6. (default: IPv4)

pkg/config/testdata/config.toml

@@ -39,6 +39,14 @@ default_pool_size = 20
 # Maximum number of client connections allowed.
 max_client_conn = 100
 
+[db.seed]
+# If enabled, seeds the database after migrations during a db reset.
+enabled = true
+# Specifies an ordered list of seed files to load during db reset.
+# Supports glob patterns relative to supabase directory. For example:
+# sql_paths = ['./seeds/*.sql', '../project-src/seeds/*-load-testing.sql']
+sql_paths = ['./seed.sql']
+
 [realtime]
 enabled = true
 # Bind realtime via either IPv4 or IPv6. (default: IPv6)

pkg/config/utils.go

@@ -29,7 +29,6 @@ type pathBuilder struct {
 	FallbackImportMapPath string
 	FallbackEnvFilePath   string
 	DbTestsDir            string
-	SeedDataPath          string
 	CustomRolesPath       string
 }
 
@@ -63,7 +62,6 @@ func NewPathBuilder(configPath string) pathBuilder {
 		FallbackImportMapPath: filepath.Join(base, "functions", "import_map.json"),
 		FallbackEnvFilePath:   filepath.Join(base, "functions", ".env"),
 		DbTestsDir:            filepath.Join(base, "tests"),
-		SeedDataPath:          filepath.Join(base, "seed.sql"),
 		CustomRolesPath:       filepath.Join(base, "roles.sql"),
 	}
 }

pkg/migration/seed.go

@@ -12,8 +12,7 @@ import (
 
 func SeedData(ctx context.Context, pending []string, conn *pgx.Conn, fsys fs.FS) error {
 	for _, path := range pending {
-		filename := filepath.Base(path)
-		fmt.Fprintf(os.Stderr, "Seeding data from %s...\n", filename)
+		fmt.Fprintf(os.Stderr, "Seeding data from %s...\n", path)
 		// Batch seed commands, safe to use statement cache
 		if seed, err := NewMigrationFromFile(path, fsys); err != nil {
 			return err