Skip to content

Commit

Permalink
implement Initial Migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@olmobrutall
olmobrutall committed Jun 2, 2022
1 parent 8085193 commit c57b3d6
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 38 deletions.
53 changes: 41 additions & 12 deletions Signum.Engine.Extensions/Migrations/MigrationLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,35 @@ public static void Start(SchemaBuilder sb)
return false;
};

Administrator.AvoidSimpleGenerate = () =>
{
if(SqlMigrationRunner.MigrationDirectoryIsEmpty())
{
Console.WriteLine("Your SQL Migrations Directory is empty.");
if (SafeConsole.Ask("Do you want to create the INITIAL SQL Migration instead?"))
{
SqlMigrationRunner.CreateInitialMigration();
SqlMigrationRunner.SqlMigrations();
return true;
}
}
else
{
var hasInitial = SqlMigrationRunner.ReadMigrationsDirectory(silent: true).MinBy(a => a.Version)?.Comment.Contains("Initial Migration");
Console.WriteLine("You have an Initial SQL Migration.");
if (SafeConsole.Ask("Do you want to run the SQL Migrations instead?"))
{
SqlMigrationRunner.SqlMigrations();
return true;
}
}
return false;
};
}
}

Expand All @@ -90,24 +119,24 @@ public static void EnsureMigrationTable<T>() where T : Entity
{
using (var tr = new Transaction())
{
if (Administrator.ExistsTable<T>())
return;
if (!Administrator.ExistsTable<T>())
{
var table = Schema.Current.Table<T>();
var sqlBuilder = Connector.Current.SqlBuilder;

var table = Schema.Current.Table<T>();
var sqlBuilder = Connector.Current.SqlBuilder;
if (!table.Name.Schema.IsDefault() && !Administrator.ExistSchema(table.Name.Schema))
sqlBuilder.CreateSchema(table.Name.Schema).ExecuteLeaves();

if (!table.Name.Schema.IsDefault() && !Database.View<SysSchemas>().Any(s => s.name == table.Name.Schema.Name))
sqlBuilder.CreateSchema(table.Name.Schema).ExecuteLeaves();
sqlBuilder.CreateTableSql(table).ExecuteLeaves();

sqlBuilder.CreateTableSql(table).ExecuteLeaves();
foreach (var i in table.GeneratAllIndexes().Where(i => !(i is PrimaryKeyIndex)))
{
sqlBuilder.CreateIndex(i, checkUnique: null).ExecuteLeaves();
}

foreach (var i in table.GeneratAllIndexes().Where(i => !(i is PrimaryKeyIndex)))
{
sqlBuilder.CreateIndex(i, checkUnique: null).ExecuteLeaves();
SafeConsole.WriteLineColor(ConsoleColor.White, "Table " + table.Name + " auto-generated...");
}

SafeConsole.WriteLineColor(ConsoleColor.White, "Table " + table.Name + " auto-generated...");

tr.Commit();
}
}
Expand Down
61 changes: 52 additions & 9 deletions Signum.Engine.Extensions/Migrations/SqlMigrationRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,41 @@ public static void SqlMigrations(bool autoRun)
{
List<MigrationInfo> list = ReadMigrationsDirectory();

SetExecuted(list);
if (!autoRun && !Connector.Current.HasTables() && list.Count == 0)
{
if (!SafeConsole.Ask("Create initial migration?"))
return;

if (!Prompt(list, autoRun) || autoRun)
return;
CreateInitialMigration();
}
else
{
SetExecuted(list);

if (!Prompt(list, autoRun) || autoRun)
return;
}
}
}

public static void CreateInitialMigration()
{
var script = Schema.Current.GenerationScipt(databaseNameReplacement: DatabaseNameReplacement)!;

string version = DateTime.Now.ToString("yyyy.MM.dd-HH.mm.ss");

string comment = "Initial Migration";

string fileName = version + "_" + FileNameValidatorAttribute.RemoveInvalidCharts(comment) + ".sql";

File.WriteAllText(Path.Combine(MigrationsDirectory, fileName), script.ToString(), Encoding.UTF8);
}

private static void SetExecuted(List<MigrationInfo> migrations)
{
if (!Connector.Current.HasTables())
return;

MigrationLogic.EnsureMigrationTable<SqlMigrationEntity>();

var first = migrations.FirstOrDefault();
Expand Down Expand Up @@ -60,15 +86,30 @@ private static void SetExecuted(List<MigrationInfo> migrations)
migrations.Sort(a => a.Version);
}

public static List<MigrationInfo> ReadMigrationsDirectory()
public static bool MigrationDirectoryIsEmpty()
{
Console.WriteLine();
SafeConsole.WriteLineColor(ConsoleColor.DarkGray, "Reading migrations from: " + MigrationsDirectory);
return !Directory.Exists(MigrationsDirectory) || Directory.EnumerateFiles(MigrationsDirectory).IsEmpty();
}

if (!Directory.Exists(MigrationsDirectory))
public static List<MigrationInfo> ReadMigrationsDirectory(bool silent = false)
{
if (silent)
{
Directory.CreateDirectory(MigrationsDirectory);
SafeConsole.WriteLineColor(ConsoleColor.White, "Directory " + MigrationsDirectory + " auto-generated...");
if (!Directory.Exists(MigrationsDirectory))
return new List<MigrationInfo>();
}
else
{
if (!Directory.Exists(MigrationsDirectory))
{
Directory.CreateDirectory(MigrationsDirectory);
SafeConsole.WriteLineColor(ConsoleColor.White, "Directory " + MigrationsDirectory + " auto-generated...");
}
else
{
Console.WriteLine();
SafeConsole.WriteLineColor(ConsoleColor.DarkGray, "Reading migrations from: " + MigrationsDirectory);
}
}

Regex regex = new Regex(@"(?<version>\d{4}\.\d{2}\.\d{2}\-\d{2}\.\d{2}\.\d{2})(_(?<comment>.+))?\.sql");
Expand Down Expand Up @@ -211,6 +252,8 @@ private static void Execute(MigrationInfo mi)

SqlPreCommandExtensions.ExecuteScript(title, text);

MigrationLogic.EnsureMigrationTable<SqlMigrationEntity>();

new SqlMigrationEntity
{
VersionNumber = mi.Version,
Expand Down
47 changes: 37 additions & 10 deletions Signum.Engine/Administrator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,17 @@ namespace Signum.Engine;

public static class Administrator
{
public static Func<bool>? OnTotalGeneration;

public static void TotalGeneration()
{
foreach (var db in Schema.Current.DatabaseNames())
{
Connector.Current.CleanDatabase(db);
SafeConsole.WriteColor(ConsoleColor.DarkGray, '.');
}
CleanAllDatabases();

ExecuteGenerationScript();
}

public static void ExecuteGenerationScript()
{
SqlPreCommandConcat totalScript = (SqlPreCommandConcat)Schema.Current.GenerationScipt()!;
foreach (SqlPreCommand command in totalScript.Commands)
{
Expand All @@ -27,6 +30,15 @@ public static void TotalGeneration()
}
}

private static void CleanAllDatabases()
{
foreach (var db in Schema.Current.DatabaseNames())
{
Connector.Current.CleanDatabase(db);
SafeConsole.WriteColor(ConsoleColor.DarkGray, '.');
}
}

public static string GenerateViewCodes(params string[] tableNames) => tableNames.ToString(tn => GenerateViewCode(tn), "\r\n\r\n");

public static string GenerateViewCode(string tableName) => GenerateViewCode(ObjectName.Parse(tableName, Schema.Current.Settings.IsPostgres));
Expand Down Expand Up @@ -77,12 +89,13 @@ private static string GenerateColumnCode(DiffColumn c)
return Schema.Current.GenerationScipt();
}



public static Func<bool>? AvoidSimpleGenerate;

public static void NewDatabase()
{
var databaseName = Connector.Current.DatabaseName();
if (Database.View<SysTables>().Any())
if (Connector.Current.HasTables())
{
SafeConsole.WriteLineColor(ConsoleColor.Red, $"Are you sure you want to delete all the data in the database '{databaseName}'?");
Console.Write($"Confirm by writing the name of the database:");
Expand All @@ -95,8 +108,16 @@ public static void NewDatabase()
}
}

Console.Write("Creating new database...");
Administrator.TotalGeneration();
Console.Write("Cleaning database...");
using(Connector.CommandTimeoutScope(5 * 60))
CleanAllDatabases();
Console.WriteLine("Done.");

if (AvoidSimpleGenerate?.Invoke() == true)
return;

Console.Write("Generating new database database...");
ExecuteGenerationScript();
Console.WriteLine("Done.");
}

Expand Down Expand Up @@ -262,7 +283,13 @@ join s in Database.View<SysSchemas>() on t.schema_id equals s.schema_id
}
}

public static bool ExistSchema(SchemaName name)
{
if (Schema.Current.Settings.IsPostgres)
return Database.View<PgNamespace>().Any(ns => ns.nspname == name.Name);

return Database.View<SysSchemas>().Any(s => s.name == name.Name);
}

public static List<T> TryRetrieveAll<T>(Replacements replacements)
where T : Entity
Expand Down Expand Up @@ -582,7 +609,7 @@ from ifk in targetTable.IncommingForeignKeys()
ParentColumn = parentTable.Columns().SingleEx(c => c.column_id == ifk.ForeignKeyColumns().SingleEx().parent_column_id).name,
}).ToList());

foreignKeys.ForEach(fk => sqlBuilder.AlterTableDropConstraint(fk.ParentTable!, fk.Name! /*CSBUG*/).ExecuteLeaves());
foreignKeys.ForEach(fk => sqlBuilder.AlterTableDropConstraint(fk.ParentTable!, fk.Name).ExecuteLeaves());

return new Disposable(() =>
{
Expand Down
24 changes: 19 additions & 5 deletions Signum.Engine/Engine/SchemaGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,19 @@ public static class SchemaGenerator
{
var sqlBuilder = Connector.Current.SqlBuilder;
Schema s = Schema.Current;

List<ITable> tables = s.GetDatabaseTables().Where(t => !s.IsExternalDatabase(t.Name.Schema.Database)).ToList();

SqlPreCommand? createTables = tables.Select(t => sqlBuilder.CreateTableSql(t)).Combine(Spacing.Double)?.PlainSqlCommand();

if (createTables != null)
createTables.GoAfter = true;

SqlPreCommand? foreignKeys = tables.Select(sqlBuilder.AlterTableForeignKeys).Combine(Spacing.Double)?.PlainSqlCommand();

if (foreignKeys != null)
foreignKeys.GoAfter = true;

SqlPreCommand? indices = tables.Select(t =>
{
var allIndexes = t.GeneratAllIndexes().Where(a => !(a is PrimaryKeyIndex)); ;
Expand All @@ -45,17 +52,24 @@ public static class SchemaGenerator
}).NotNull().Combine(Spacing.Double)?.PlainSqlCommand();

if (indices != null)
indices.GoAfter = true;

return SqlPreCommand.Combine(Spacing.Triple, createTables, foreignKeys, indices);
}

public static SqlPreCommand? InsertEnumValuesScript()
{
return (from t in Schema.Current.Tables.Values
let enumType = EnumEntity.Extract(t.Type)
where enumType != null
select EnumEntity.GetEntities(enumType).Select((e, i) => t.InsertSqlSync(e, suffix: t.Name.Name + i)).Combine(Spacing.Simple)
var result = (from t in Schema.Current.Tables.Values
let enumType = EnumEntity.Extract(t.Type)
where enumType != null
select EnumEntity.GetEntities(enumType).Select((e, i) => t.InsertSqlSync(e, suffix: t.Name.Name + i)).Combine(Spacing.Simple)
).Combine(Spacing.Double)?.PlainSqlCommand();

if (result != null)
result.GoAfter = true;

return result;
}

public static SqlPreCommand? PostgresExtensions()
Expand Down Expand Up @@ -90,7 +104,6 @@ select EnumEntity.GetEntities(enumType).Select((e, i) => t.InsertSqlSync(e, suff
if (!connector.AllowsSetSnapshotIsolation)
return null;


var list = connector.Schema.DatabaseNames().Select(a => a?.Name).ToList();

if (list.Contains(null))
Expand All @@ -112,6 +125,7 @@ select EnumEntity.GetEntities(enumType).Select((e, i) => t.InsertSqlSync(e, suff
).Combine(Spacing.Double);

return cmd;

}

private static bool SnapshotIsolationEnabled(DatabaseName dbName)
Expand Down
2 changes: 1 addition & 1 deletion Signum.Engine/Engine/SqlPreCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ public static void ExecuteScript(string title, string script)
{
using (Connector.CommandTimeoutScope(Timeout))
{
var regex = new Regex(@" *(GO|USE \w+|USE \[[^\]]+\]) *(\r?\n|$)", RegexOptions.IgnoreCase);
var regex = new Regex(@"^ *(GO|USE \w+|USE \[[^\]]+\]) *(\r?\n|$)", RegexOptions.IgnoreCase | RegexOptions.Multiline);

var parts = regex.Split(script);

Expand Down
5 changes: 4 additions & 1 deletion Signum.Engine/Schema/Schema.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,14 +459,17 @@ public Table View(Type viewType)
return ViewBuilder.NewView(viewType);
}



public event Func<SqlPreCommand?> Generating;
internal SqlPreCommand? GenerationScipt()
public SqlPreCommand? GenerationScipt(string? databaseNameReplacement = null)
{
OnBeforeDatabaseAccess();

if (Generating == null)
return null;

using (databaseNameReplacement == null ? null : ObjectName.OverrideOptions(new ObjectNameOptions { DatabaseNameReplacement = databaseNameReplacement }))
using (CultureInfoUtils.ChangeBothCultures(ForceCultureInfo))
using (ExecutionMode.Global())
{
Expand Down

3 comments on commit c57b3d6

@olmobrutall
Copy link
Collaborator Author

@olmobrutall olmobrutall commented on c57b3d6 Jun 13, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Initial Migration

Traditionally there are two ways to create a database compatible with a Signum Application:

  • Run GenerateEnvironment test, to create a small database with sample data that can be used as a starting point to run all the other unit tests.

  • Run SQL and C# migrations to adapt a production-like database to the latest requirements of the application.

Both alternatives, however, start very similarly: by calling Adminiatrator.CreateDatabase. This method deletes all the tables, views, indexes, foreigner keys in the database (if any) and then creates a new empty database with all the tables as required by the applications.

Unfortunately, this process was typically executed straight away, without storing the script to generate the tables in a Initial Migration.

This is ok for the GenerateEnvironment case, but not if we want to make a reproducible chain of migrations that is able to generate the current production schema from an empty database.

This new feature helps developers that do not have a production-like database at hand to create a new SQL migration in development time.

Still, creating SQL migrations on a database where most tables are empty has some risks: Creating Unique Indexes, not nullable columns or new foreign keys could be misleadingly easy in your dev machine but could not work in live.

How it works

There are two ways of creating an Initial Migration from your Terminal applications:

  • Executing SQL Migrations from an empty database.
  • When running Create New Database, just after it deletes all the tables, it will suggest creating an initial migration (if Migrations folder is empty) or execute existing migrations.

@rezanos
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a great useful improvement
Thanks 🙏

@MehdyKarimpour
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome job 👍

Please sign in to comment.