Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Take min/max batch size into account again for seeding #28955

Merged
merged 1 commit into from
Sep 7, 2022
Merged

Take min/max batch size into account again for seeding #28955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2190,17 +2190,11 @@ private IEnumerable<MigrationOperation> GetDataOperations(
yield break;
}

var commands = identityMaps.Values.SelectMany(m => m.Rows).Where(r =>
{
return r.EntityState switch
{
EntityState.Added => true,
EntityState.Modified => true,
EntityState.Unchanged => false,
EntityState.Deleted => diffContext.FindDrop(r.Table!) == null,
_ => throw new InvalidOperationException($"Unexpected entity state: {r.EntityState}")
};
});
var commands = identityMaps.Values
.SelectMany(m => m.Rows)
.Where(
r => r.EntityState is EntityState.Added or EntityState.Modified
|| (r.EntityState is EntityState.Deleted && diffContext.FindDrop(r.Table!) == null));

var commandSets = new CommandBatchPreparer(CommandBatchPreparerDependencies)
.TopologicalSort(commands);
Expand Down
9 changes: 9 additions & 0 deletions src/EFCore.Relational/Update/ICommandBatchPreparer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,13 @@ public interface ICommandBatchPreparer
/// <param name="updateAdapter">The model data.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> BatchCommands(IList<IUpdateEntry> entries, IUpdateAdapter updateAdapter);

/// <summary>
/// Given a set of modification commands, returns one more ready-to-execute batches for those commands, taking into account e.g.
/// maximum batch sizes and other batching constraints.
/// </summary>
/// <param name="commandSet">The set of commands to be organized in batches.</param>
/// <param name="moreCommandSets">Whether more command sets are expected after this one within the same save operation.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> CreateCommandBatches(IEnumerable<IReadOnlyModificationCommand> commandSet, bool moreCommandSets);
}
165 changes: 101 additions & 64 deletions src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Security.Principal;
using System.Text;
using Microsoft.EntityFrameworkCore.Metadata.Internal;

Expand Down Expand Up @@ -70,91 +69,129 @@ public virtual IEnumerable<ModificationCommandBatch> BatchCommands(

for (var commandSetIndex = 0; commandSetIndex < commandSets.Count; commandSetIndex++)
{
var commandSet = commandSets[commandSetIndex];
var batches = CreateCommandBatches(
commandSets[commandSetIndex],
commandSetIndex < commandSets.Count - 1,
assertColumnModification: true,
parameterNameGenerator);

var batch = Dependencies.ModificationCommandBatchFactory.Create();
foreach (var modificationCommand in commandSet)
foreach (var batch in batches)
{
yield return batch;
}
}
}

/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
public virtual IEnumerable<ModificationCommandBatch> CreateCommandBatches(
IEnumerable<IReadOnlyModificationCommand> commandSet,
bool moreCommandSets)
=> CreateCommandBatches(commandSet, moreCommandSets, assertColumnModification: false);

/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
private IEnumerable<ModificationCommandBatch> CreateCommandBatches(
IEnumerable<IReadOnlyModificationCommand> commandSet,
bool moreCommandSets,
bool assertColumnModification,
ParameterNameGenerator? parameterNameGenerator = null)
{
var batch = Dependencies.ModificationCommandBatchFactory.Create();

foreach (var modificationCommand in commandSet)
{
#if DEBUG
if (assertColumnModification)
{
(modificationCommand as ModificationCommand)?.AssertColumnsNotInitialized();
roji marked this conversation as resolved.
Show resolved Hide resolved
if (modificationCommand.EntityState == EntityState.Modified
&& !modificationCommand.ColumnModifications.Any(m => m.IsWrite))
{
continue;
}
}
#endif

if (modificationCommand.EntityState == EntityState.Modified
&& !modificationCommand.ColumnModifications.Any(m => m.IsWrite))
{
continue;
}

if (!batch.TryAddCommand(modificationCommand))
if (!batch.TryAddCommand(modificationCommand))
{
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
{
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
if (batch.ModificationCommands.Count > 1)
{
if (batch.ModificationCommands.Count > 1)
{
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}

batch.Complete(moreBatchesExpected: true);
batch.Complete(moreBatchesExpected: true);

yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);
yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);

foreach (var command in batch.ModificationCommands)
{
batch = StartNewBatch(parameterNameGenerator, command);
batch.Complete(moreBatchesExpected: true);
foreach (var command in batch.ModificationCommands)
{
batch = StartNewBatch(parameterNameGenerator, command);
batch.Complete(moreBatchesExpected: true);

yield return batch;
}
yield return batch;
}

batch = StartNewBatch(parameterNameGenerator, modificationCommand);
}
}

var hasMoreCommandSets = commandSetIndex < commandSets.Count - 1;
batch = StartNewBatch(parameterNameGenerator, modificationCommand);
}
}

if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
{
if (batch.ModificationCommands.Count > 1)
{
if (batch.ModificationCommands.Count > 1)
{
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}

batch.Complete(moreBatchesExpected: hasMoreCommandSets);
batch.Complete(moreBatchesExpected: moreCommandSets);

yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);
yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);

for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++)
{
var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]);
singleCommandBatch.Complete(
moreBatchesExpected: hasMoreCommandSets || commandIndex < batch.ModificationCommands.Count - 1);
for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++)
{
var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]);
singleCommandBatch.Complete(
moreBatchesExpected: moreCommandSets || commandIndex < batch.ModificationCommands.Count - 1);

yield return singleCommandBatch;
}
yield return singleCommandBatch;
}
}
}

private ModificationCommandBatch StartNewBatch(
ParameterNameGenerator parameterNameGenerator,
IReadOnlyModificationCommand modificationCommand)
{
parameterNameGenerator.Reset();
var batch = Dependencies.ModificationCommandBatchFactory.Create();
batch.TryAddCommand(modificationCommand);
return batch;
ModificationCommandBatch StartNewBatch(
ParameterNameGenerator? parameterNameGenerator,
IReadOnlyModificationCommand modificationCommand)
{
parameterNameGenerator?.Reset();
var batch = Dependencies.ModificationCommandBatchFactory.Create();
batch.TryAddCommand(modificationCommand);
return batch;
}
}

/// <summary>
Expand Down
22 changes: 14 additions & 8 deletions src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Microsoft.EntityFrameworkCore.SqlServer.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Metadata.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Update.Internal;
using Microsoft.EntityFrameworkCore.Update.Internal;

// ReSharper disable once CheckNamespace
namespace Microsoft.EntityFrameworkCore.Migrations;
Expand All @@ -33,17 +34,18 @@ public class SqlServerMigrationsSqlGenerator : MigrationsSqlGenerator
private IReadOnlyList<MigrationOperation> _operations = null!;
private int _variableCounter;

private readonly ICommandBatchPreparer _commandBatchPreparer;

/// <summary>
/// Creates a new <see cref="SqlServerMigrationsSqlGenerator" /> instance.
/// </summary>
/// <param name="dependencies">Parameter object containing dependencies for this service.</param>
/// <param name="migrationsAnnotations">Provider-specific Migrations annotations to use.</param>
/// <param name="commandBatchPreparer">The command batch preparer.</param>
public SqlServerMigrationsSqlGenerator(
MigrationsSqlGeneratorDependencies dependencies,
IRelationalAnnotationProvider migrationsAnnotations)
ICommandBatchPreparer commandBatchPreparer)
: base(dependencies)
{
}
=> _commandBatchPreparer = commandBatchPreparer;

/// <summary>
/// Generates commands from a list of operations.
Expand Down Expand Up @@ -1445,10 +1447,14 @@ protected override void Generate(
GenerateIdentityInsert(builder, operation, on: true, model);

var sqlBuilder = new StringBuilder();
((ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator).AppendBulkInsertOperation(
sqlBuilder,
GenerateModificationCommands(operation, model).ToList(),
0);

var modificationCommands = GenerateModificationCommands(operation, model).ToList();
var updateSqlGenerator = (ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator;

foreach (var batch in _commandBatchPreparer.CreateCommandBatches(modificationCommands, moreCommandSets: true))
{
updateSqlGenerator.AppendBulkInsertOperation(sqlBuilder, batch.ModificationCommands, commandPosition: 0);
}

if (Options.HasFlag(MigrationsSqlGenerationOptions.Idempotent))
{
Expand Down
92 changes: 92 additions & 0 deletions test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,85 @@ IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name'
");
}

[ConditionalFact]
public virtual void InsertDataOperation_max_batch_size_is_respected()
{
// The SQL Server max batch size is 42 by default
var values = new object[50, 1];
for (var i = 0; i < 50; i++)
{
values[i, 0] = "Foo" + i;
}

Generate(
CreateGotModel,
new InsertDataOperation
{
Table = "People",
Columns = new[] { "First Name" },
Values = values
});

AssertSql(
@"IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] ON;
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo0'),
(N'Foo1'),
(N'Foo2'),
(N'Foo3'),
(N'Foo4'),
(N'Foo5'),
(N'Foo6'),
(N'Foo7'),
(N'Foo8'),
(N'Foo9'),
(N'Foo10'),
(N'Foo11'),
(N'Foo12'),
(N'Foo13'),
(N'Foo14'),
(N'Foo15'),
(N'Foo16'),
(N'Foo17'),
(N'Foo18'),
(N'Foo19'),
(N'Foo20'),
(N'Foo21'),
(N'Foo22'),
(N'Foo23'),
(N'Foo24'),
(N'Foo25'),
(N'Foo26'),
(N'Foo27'),
(N'Foo28'),
(N'Foo29'),
(N'Foo30'),
(N'Foo31'),
(N'Foo32'),
(N'Foo33'),
(N'Foo34'),
(N'Foo35'),
(N'Foo36'),
(N'Foo37'),
(N'Foo38'),
(N'Foo39'),
(N'Foo40'),
(N'Foo41');
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo42'),
(N'Foo43'),
(N'Foo44'),
(N'Foo45'),
(N'Foo46'),
(N'Foo47'),
(N'Foo48'),
(N'Foo49');
IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] OFF;
");
}

public override void InsertDataOperation_throws_for_unsupported_column_types()
=> base.InsertDataOperation_throws_for_unsupported_column_types();

Expand Down Expand Up @@ -1045,6 +1124,19 @@ public virtual void CreateIndex_generates_exec_when_legacy_filter_and_idempotent
");
}

private static void CreateGotModel(ModelBuilder b)
=> b.HasDefaultSchema("dbo").Entity(
"Person", pb =>
{
pb.ToTable("People");
pb.Property<string>("FirstName").HasColumnName("First Name");
pb.Property<string>("LastName").HasColumnName("Last Name");
pb.Property<string>("Birthplace").HasColumnName("Birthplace");
pb.Property<string>("Allegiance").HasColumnName("House Allegiance");
pb.Property<string>("Culture").HasColumnName("Culture");
pb.HasKey("FirstName", "LastName");
});

public SqlServerMigrationsSqlGeneratorTest()
: base(
SqlServerTestHelpers.Instance,
Expand Down