Skip to content

Commit

Permalink
Fix Dataloader.Ecto.run_batches/1 function for high number of batches
Browse files Browse the repository at this point in the history
As stated by the Elixir doc, Maps’ keys do not follow any ordering. In practice is is especially true starting at 32 elements.
Therefore the batches' results were not in the same order as the batches' keys which resulted in an error when fetching a batch result.
  • Loading branch information
jueberschlag committed Jun 29, 2021
1 parent 898b6b1 commit 3638308
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 2 deletions.
6 changes: 4 additions & 2 deletions lib/dataloader/ecto.ex
Original file line number Diff line number Diff line change
Expand Up @@ -600,8 +600,10 @@ if Code.ensure_loaded?(Ecto) do
on_timeout: :kill_task
]

batches = Enum.to_list(source.batches)

results =
source.batches
batches
|> Task.async_stream(
fn batch ->
id = :erlang.unique_integer()
Expand All @@ -621,7 +623,7 @@ if Code.ensure_loaded?(Ecto) do
{:exit, reason} -> {:error, reason}
end)

source.batches
batches
|> Enum.map(fn {key, _set} -> key end)
|> Enum.zip(results)
|> Map.new()
Expand Down
30 changes: 30 additions & 0 deletions test/dataloader/ecto_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -456,4 +456,34 @@ defmodule Dataloader.EctoTest do
Dataloader.get(loader, Timeout, User, user.id)
end
end

test "when there are many batches (more than 32)" do
source =
Dataloader.Ecto.new(
Repo,
query: fn queryable, _ ->
queryable
end
)

loader =
Dataloader.new()
|> Dataloader.add_source(Test, source)

users =
for x <- 0..50 do
%User{username: "Ben Wilson #{x}"} |> Repo.insert!()
end

loader =
users
|> Enum.reduce(loader, fn user, loader ->
# Force a batch per user by adding id to batch key
Dataloader.load(loader, Test, {User, id: user.id}, user.id)
end)
|> Dataloader.run()

assert Enum.map(users, &Dataloader.get(loader, Test, {User, id: &1.id}, &1.id)) ==
users
end
end

0 comments on commit 3638308

Please sign in to comment.