Add early stopping utils

docs/src/utilities.md

@@ -102,4 +102,5 @@ Flux.nfan
 Flux.throttle
 Flux.stop
 Flux.skip
+Flux.early_stopping
 ```

src/utils.jl

@@ -742,3 +742,47 @@ modules(m) = [x for x in Functors.fcollect(m) if !isleaflike(x)]
 isleaflike(x) = Functors.isleaf(x)
 isleaflike(::Tuple{Vararg{<:Number}}) = true
 isleaflike(::Tuple{Vararg{<:AbstractArray{<:Number}}}) = true
+
+"""
+    early_stopping(f; delta=-, min_delta=0, patience=3)
+
+Return a function that evaluates the metric `f` and compares its value
+against its value on last invocation. When the difference has been less
+than `min_delta` for at least `patience` times, `true` is returned,
+otherwise `false` is returned.
+
+By default, `early_stopping` expects the metric `f` to be minimized.
+However, if you are using some increasing metric, accuracy for example,
+you can change `early_stopping`'s behaviour by customizing the `delta`
+function: `(best_score, score) -> score - best_score`.
+
+# Examples
+```jldoctest
+julia> function loss()
+       l = 0
+       return () -> l += 1
+       end # pseudo loss function that returns increasing values
+loss (generic function with 1 method)
+
+julia> es = Flux.early_stopping(loss(); patience=3);
+
+julia> Flux.@epochs 10 begin
+       es() || break
+       end
+[ Info: Epoch 1
+[ Info: Epoch 2
+[ Info: Epoch 3
+```
+"""
+function early_stopping(f; delta = -, min_delta = 0, patience = 3)
+  best_score = f()
+  count = 0
+
+  return function ()
+    score = f()
+    Δ = delta(best_score, score)
+    count = Δ < min_delta ? count + 1 : 0
+    best_score = Δ < 0 ? best_score : score
+    return count < patience
+  end
+end

test/utils.jl

@@ -418,3 +418,43 @@ end
                                 LayerNorm(8)))
   @test length(modules) == 5
 end
+
+@testset "Early stopping" begin
+  function metric(; step = 1)
+    l = 0
+    return () -> l += step
+  end
+
+  @testset "delta" begin
+    es = Flux.early_stopping(metric(); delta=(best_score, score) -> score - best_score)
+
+    n_iter = 0
+    while n_iter < 99
+      es() ? n_iter += 1 : break
+    end
+
+    @test n_iter == 99
+  end
+
+  @testset "min delta" begin
+    es = Flux.early_stopping(metric(step=-2); min_delta=1)
+
+    n_iter = 0
+    while n_iter < 99
+      es() ? n_iter += 1 : break
+    end
+
+    @test n_iter == 99
+  end
+
+  @testset "patience" begin
+    es = Flux.early_stopping(metric(); patience=10)
+
+    n_iter = 0
+    while n_iter < 99
+      es() ? n_iter += 1 : break
+    end
+
+    @test n_iter == 9
+  end
+end