From ff9fb483172ad792c47b7652eff1173044f3b9ae Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Thu, 21 Sep 2017 16:34:01 -0400
Subject: [PATCH] fix an invariance bug in `limit_type_depth`. part of #23786
 (#23800)

---
 base/inference.jl | 11 ++++++++---
 test/inference.jl |  7 +++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/base/inference.jl b/base/inference.jl
index 8383d53a45eca..ec9ac4b9536d0 100644
--- a/base/inference.jl
+++ b/base/inference.jl
@@ -837,7 +837,7 @@ function limit_type_depth(@nospecialize(t), d::Int, cov::Bool, vars::Vector{Type
             end
             ub = Any
         else
-            ub = limit_type_depth(v.ub, d - 1, true)
+            ub = limit_type_depth(v.ub, d - 1, cov, vars)
         end
         if v.lb === Bottom || type_depth(v.lb) > d
             # note: lower bounds need to be widened by making them lower
@@ -855,7 +855,8 @@ function limit_type_depth(@nospecialize(t), d::Int, cov::Bool, vars::Vector{Type
     if d < 0
         if isvarargtype(t)
             # never replace Vararg with non-Vararg
-            return Vararg{limit_type_depth(P[1], d, cov, vars), P[2]}
+            # passing depth=0 avoids putting a bare typevar here, for the diagonal rule
+            return Vararg{limit_type_depth(P[1], 0, cov, vars), P[2]}
         end
         widert = t.name.wrapper
         if !(t <: widert)
@@ -870,7 +871,11 @@ function limit_type_depth(@nospecialize(t), d::Int, cov::Bool, vars::Vector{Type
         return var
     end
     stillcov = cov && (t.name === Tuple.name)
-    Q = map(x -> limit_type_depth(x, d - 1, stillcov, vars), P)
+    newdepth = d - 1
+    if isvarargtype(t)
+        newdepth = max(newdepth, 0)
+    end
+    Q = map(x -> limit_type_depth(x, newdepth, stillcov, vars), P)
     R = t.name.wrapper{Q...}
     if cov && !stillcov
         for var in vars
diff --git a/test/inference.jl b/test/inference.jl
index daf205518b37c..6d8e5cf78ac20 100644
--- a/test/inference.jl
+++ b/test/inference.jl
@@ -1218,3 +1218,10 @@ let c(::Type{T}, x) where {T<:Array} = T,
     f() = c(Vector{Any[Int][1]}, [1])
     @test f() === Vector{Int}
 end
+
+# issue #23786
+struct T23786{D<:Tuple{Vararg{Vector{T} where T}}, N}
+end
+let t = Tuple{Type{T23786{D, N} where N where D<:Tuple{Vararg{Array{T, 1} where T, N} where N}}}
+    @test Core.Inference.limit_type_depth(t, 4) >: t
+end