Merge pull request #32973 from JuliaLang/backports-release-1.3

WIP: Backports for 1.3-RC2

Make.inc

@@ -1005,7 +1005,7 @@ USE_BINARYBUILDER ?= 0
 endif
 
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
-BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV
+BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM MBEDTLS LIBSSH2 CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)
@@ -1022,6 +1022,9 @@ endef
 $(foreach proj,$(BB_PROJECTS),$(eval $(call SET_BB_DEFAULT,$(proj))))
 
 
+# GMP and MPFR have serious performance regressions for now
+USE_BINARYBUILDER_GMP ?= 0
+USE_BINARYBUILDER_MPFR ?= 0
 
 # Use the Assertions build
 BINARYBUILDER_LLVM_ASSERTS ?= 0

NEWS.md

@@ -94,7 +94,6 @@ Standard library changes
 #### Dates
 
 * `DateTime` and `Time` formatting/parsing now supports 12-hour clocks with AM/PM via `I` and `p` codes, similar to `strftime` ([#32308]).
-* Fixed `repr` such that it displays `Time` as it would be entered in Julia ([#32103]).
 
 #### Statistics
 

base/abstractdict.jl

@@ -579,7 +579,9 @@ end
 
 function setindex!(d::IdDict{K,V}, @nospecialize(val), @nospecialize(key)) where {K, V}
     !isa(key, K) && throw(ArgumentError("$(limitrepr(key)) is not a valid key for type $K"))
-    val = convert(V, val)
+    if !(val isa V) # avoid a dynamic call
+        val = convert(V, val)
+    end
     if d.ndel >= ((3*length(d.ht))>>2)
         rehash!(d, max(length(d.ht)>>1, 32))
         d.ndel = 0

base/broadcast.jl

@@ -260,8 +260,13 @@ of the `Broadcasted` object empty (populated with [`nothing`](@ref)).
     end
     return Broadcasted{Style}(bc.f, bc.args, axes)
 end
-instantiate(bc::Broadcasted{<:Union{AbstractArrayStyle{0}, Style{Tuple}}}) = bc
-
+instantiate(bc::Broadcasted{<:AbstractArrayStyle{0}}) = bc
+# Tuples don't need axes, but when they have axes (for .= assignment), we need to check them (#33020)
+instantiate(bc::Broadcasted{Style{Tuple}, Nothing}) = bc
+function instantiate(bc::Broadcasted{Style{Tuple}})
+    check_broadcast_axes(bc.axes, bc.args...)
+    return bc
+end
 ## Flattening
 
 """
@@ -503,7 +508,7 @@ function check_broadcast_shape(shp, Ashp::Tuple)
     _bcsm(shp[1], Ashp[1]) || throw(DimensionMismatch("array could not be broadcast to match destination"))
     check_broadcast_shape(tail(shp), tail(Ashp))
 end
-check_broadcast_axes(shp, A) = check_broadcast_shape(shp, axes(A))
+@inline check_broadcast_axes(shp, A) = check_broadcast_shape(shp, axes(A))
 # comparing many inputs
 @inline function check_broadcast_axes(shp, A, As...)
     check_broadcast_axes(shp, A)
@@ -798,13 +803,13 @@ Like [`broadcast`](@ref), except in the case of a 0-dimensional result where it
 Broadcast automatically unwraps zero-dimensional results to be just the element itself,
 but in some cases it is necessary to always return a container — even in the 0-dimensional case.
 """
-function broadcast_preserving_zero_d(f, As...)
+@inline function broadcast_preserving_zero_d(f, As...)
     bc = broadcasted(f, As...)
     r = materialize(bc)
     return length(axes(bc)) == 0 ? fill!(similar(bc, typeof(r)), r) : r
 end
-broadcast_preserving_zero_d(f) = fill(f())
-broadcast_preserving_zero_d(f, as::Number...) = fill(f(as...))
+@inline broadcast_preserving_zero_d(f) = fill(f())
+@inline broadcast_preserving_zero_d(f, as::Number...) = fill(f(as...))
 
 """
     Broadcast.materialize(bc)

base/condition.jl

@@ -22,7 +22,6 @@ function trylock end
 function islocked end
 unlockall(l::AbstractLock) = unlock(l) # internal function for implementing `wait`
 relockall(l::AbstractLock, token::Nothing) = lock(l) # internal function for implementing `wait`
-assert_havelock(l::AbstractLock) = assert_havelock(l, Threads.threadid())
 assert_havelock(l::AbstractLock, tid::Integer) =
     (islocked(l) && tid == Threads.threadid()) ? nothing : concurrency_violation()
 assert_havelock(l::AbstractLock, tid::Task) =
@@ -106,7 +105,7 @@ function wait(c::GenericCondition)
     try
         return wait()
     catch
-        list_deletefirst!(c.waitq, ct)
+        ct.queue === nothing || list_deletefirst!(ct.queue, ct)
         rethrow()
     finally
         relockall(c.lock, token)

base/file.jl

@@ -582,7 +582,7 @@ function mktempdir(parent::AbstractString=tempdir();
 
     req = Libc.malloc(_sizeof_uv_fs)
     try
-        ret = ccall(:uv_fs_mkdtemp, Int32,
+        ret = ccall(:uv_fs_mkdtemp, Cint,
                     (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}),
                     C_NULL, req, tpath, C_NULL)
         if ret < 0

base/float.jl

@@ -196,7 +196,7 @@ function Float16(val::Float32)
     # NOTE: we maybe should ignore NaNs here, but the payload is
     # getting truncated anyway so "rounding" it might not matter
     nextbit = (f >> (sh-1)) & 1
-    if nextbit != 0
+    if nextbit != 0 && (h & 0x7C00) != 0x7C00
         # Round halfway to even or check lower bits
         if h&1 == 1 || (f & ((1<<(sh-1))-1)) != 0
             h += UInt16(1)

base/gcutils.jl

@@ -26,8 +26,8 @@ end
 
 function finalizer(f::Ptr{Cvoid}, o::T) where T
     @_inline_meta
-    if isimmutable(T)
-        error("objects of type ", T, " cannot be finalized")
+    if isimmutable(o)
+        error("objects of type ", typeof(o), " cannot be finalized")
     end
     ccall(:jl_gc_add_ptr_finalizer, Cvoid, (Ptr{Cvoid}, Any, Ptr{Cvoid}),
           Core.getptls(), o, f)

base/loading.jl

@@ -1220,6 +1220,18 @@ function create_expr_cache(input::String, output::String, concrete_deps::typeof(
     return io
 end
 
+function compilecache_path(pkg::PkgId)::String
+    entrypath, entryfile = cache_file_entry(pkg)
+    cachepath = joinpath(DEPOT_PATH[1], entrypath)
+    isdir(cachepath) || mkpath(cachepath)
+    if pkg.uuid === nothing
+        abspath(cachepath, entryfile) * ".ji"
+    else
+        project_precompile_slug = slug(_crc32c(something(Base.active_project(), "")), 5)
+        abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
+    end
+end
+
 """
     Base.compilecache(module::PkgId)
 
@@ -1238,19 +1250,16 @@ const MAX_NUM_PRECOMPILE_FILES = 10
 
 function compilecache(pkg::PkgId, path::String)
     # decide where to put the resulting cache file
-    entrypath, entryfile = cache_file_entry(pkg)
-    cachepath = joinpath(DEPOT_PATH[1], entrypath)
-    isdir(cachepath) || mkpath(cachepath)
-    if pkg.uuid === nothing
-        cachefile = abspath(cachepath, entryfile) * ".ji"
-    else
-        candidates = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
-        if length(candidates) >= MAX_NUM_PRECOMPILE_FILES
-            idx = findmin(mtime.(joinpath.(cachepath, candidates)))[2]
-            rm(joinpath(cachepath, candidates[idx]))
+    cachefile = compilecache_path(pkg)
+    # prune the directory with cache files
+    if pkg.uuid !== nothing
+        cachepath = dirname(cachefile)
+        entrypath, entryfile = cache_file_entry(pkg)
+        cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
+        if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES
+            idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2]
+            rm(joinpath(cachepath, cachefiles[idx]))
         end
-        project_precompile_slug = slug(_crc32c(something(Base.active_project(), "")), 5)
-        cachefile = abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
     end
     # build up the list of modules that we want the precompile process to preserve
     concrete_deps = copy(_concrete_dependencies)

base/lock.jl

@@ -16,6 +16,7 @@ mutable struct ReentrantLock <: AbstractLock
     ReentrantLock() = new(nothing, GenericCondition{Threads.SpinLock}(), 0)
 end
 
+assert_havelock(l::ReentrantLock) = assert_havelock(l, l.locked_by)
 
 """
     islocked(lock) -> Status (Boolean)

base/locks-mt.jl

@@ -30,6 +30,10 @@ struct SpinLock <: AbstractLock
     SpinLock() = new(Atomic{Int}(0))
 end
 
+# Note: this cannot assert that the lock is held by the correct thread, because we do not
+# track which thread locked it. Users beware.
+Base.assert_havelock(l::SpinLock) = islocked(l) ? nothing : concurrency_violation()
+
 function lock(l::SpinLock)
     while true
         if l.handle[] == 0

base/path.jl

@@ -337,28 +337,6 @@ abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...))
 
 if Sys.iswindows()
 
-function realpath(path::AbstractString)
-    h = ccall(:CreateFileW, stdcall, Int, (Cwstring, UInt32, UInt32, Ptr{Cvoid}, UInt32, UInt32, Int),
-                path, 0, 0x03, C_NULL, 3, 0x02000000, 0)
-    windowserror(:realpath, h == -1)
-    try
-        buf = Vector{UInt16}(undef, 256)
-        oldlen = len = length(buf)
-        while len >= oldlen
-            len = ccall(:GetFinalPathNameByHandleW, stdcall, UInt32, (Int, Ptr{UInt16}, UInt32, UInt32),
-                            h, buf, (oldlen=len)-1, 0x0)
-            windowserror(:realpath, iszero(len))
-            resize!(buf, len) # strips NUL terminator on last call
-        end
-        if 4 < len < 264 && 0x005c == buf[1] == buf[2] == buf[4] && 0x003f == buf[3]
-            Base._deletebeg!(buf, 4) # omit \\?\ prefix for paths < MAXPATH in length
-        end
-        return transcode(String, buf)
-    finally
-        windowserror(:realpath, iszero(ccall(:CloseHandle, stdcall, Cint, (Int,), h)))
-    end
-end
-
 function longpath(path::AbstractString)
     p = cwstring(path)
     buf = zeros(UInt16, length(p))
@@ -373,14 +351,6 @@ function longpath(path::AbstractString)
     end
 end
 
-else # !windows
-function realpath(path::AbstractString)
-    p = ccall(:realpath, Ptr{UInt8}, (Cstring, Ptr{UInt8}), path, C_NULL)
-    systemerror(:realpath, p == C_NULL)
-    str = unsafe_string(p)
-    Libc.free(p)
-    return str
-end
 end # os-test
 
 
@@ -393,8 +363,23 @@ filesystem's stored case for the path is returned.
 
 (This function throws an exception if `path` does not exist in the filesystem.)
 """
-realpath(path::AbstractString)
-
+function realpath(path::AbstractString)
+    req = Libc.malloc(_sizeof_uv_fs)
+    try
+        ret = ccall(:uv_fs_realpath, Cint,
+                    (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Ptr{Cvoid}),
+                    C_NULL, req, path, C_NULL)
+        if ret < 0
+            ccall(:uv_fs_req_cleanup, Cvoid, (Ptr{Cvoid},), req)
+            uv_error("realpath", ret)
+        end
+        path = unsafe_string(ccall(:jl_uv_fs_t_ptr, Cstring, (Ptr{Cvoid},), req))
+        ccall(:uv_fs_req_cleanup, Cvoid, (Ptr{Cvoid},), req)
+        return path
+    finally
+        Libc.free(req)
+    end
+end
 
 if Sys.iswindows()
 # on windows, ~ means "temporary file"

base/shell.jl

@@ -70,7 +70,14 @@ function shell_parse(str::AbstractString, interpolate::Bool=true;
             isempty(st) && error("\$ right before end of command")
             stpos, c = popfirst!(st)
             isspace(c) && error("space not allowed right after \$")
-            ex, j = Meta.parse(s,stpos,greedy=false)
+            if startswith(SubString(s, stpos), "var\"")
+                # Disallow var"#" syntax in cmd interpolations.
+                # TODO: Allow only identifiers after the $ for consistency with
+                # string interpolation syntax (see #3150)
+                ex, j = :var, stpos+3
+            else
+                ex, j = Meta.parse(s,stpos,greedy=false)
+            end
             last_parse = (stpos:prevind(s, j)) .+ s.offset
             update_arg(ex);
             s = SubString(s, j)

base/stream.jl

@@ -924,15 +924,20 @@ function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt)
     uvw = uv_write_async(s, p, n)
     ct = current_task()
     preserve_handle(ct)
+    sigatomic_begin()
     uv_req_set_data(uvw, ct)
     iolock_end()
     status = try
+        sigatomic_end()
         # wait for the last chunk to complete (or error)
         # assume that any errors would be sticky,
         # (so we don't need to monitor the error status of the intermediate writes)
         wait()::Cint
     finally
+        # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end
+        sigatomic_end()
         iolock_begin()
+        ct.queue === nothing || list_deletefirst!(ct.queue, ct)
         if uv_req_data(uvw) != C_NULL
             # uvw is still alive,
             # so make sure we won't get spurious notifications later

base/sysinfo.jl

@@ -459,6 +459,9 @@ for executable permissions only (with `.exe` and `.com` extensions added on
 Windows platforms); no searching of `PATH` is performed.
 """
 function which(program_name::String)
+    if isempty(program_name)
+       return nothing
+    end
     # Build a list of program names that we're going to try
     program_names = String[]
     base_pname = basename(program_name)
@@ -501,7 +504,7 @@ function which(program_name::String)
         for pname in program_names
             program_path = joinpath(path_dir, pname)
             # If we find something that matches our name and we can execute
-            if isexecutable(program_path)
+            if isfile(program_path) && isexecutable(program_path)
                 return realpath(program_path)
             end
         end

base/task.jl

@@ -561,7 +561,16 @@ Switch to the scheduler to allow another scheduled task to run. A task that call
 function is still runnable, and will be restarted immediately if there are no other runnable
 tasks.
 """
-yield() = (enq_work(current_task()); wait())
+function yield()
+    ct = current_task()
+    enq_work(ct)
+    try
+        wait()
+    catch
+        ct.queue === nothing || list_deletefirst!(ct.queue, ct)
+        rethrow()
+    end
+end
 
 """
     yield(t::Task, arg = nothing)

base/threadingconstructs.jl

@@ -18,9 +18,6 @@ on `threadid()`.
 """
 nthreads() = Int(unsafe_load(cglobal(:jl_n_threads, Cint)))
 
-# Only read/written by the main thread
-const in_threaded_loop = Ref(false)
-
 function _threadsfor(iter,lbody)
     lidx = iter.args[1]         # index
     range = iter.args[2]
@@ -65,15 +62,11 @@ function _threadsfor(iter,lbody)
             end
         end
         end
-        # Hack to make nested threaded loops kinda work
-        if threadid() != 1 || in_threaded_loop[]
-            # We are in a nested threaded loop
+        if threadid() != 1
+            # only thread 1 can enter/exit _threadedregion
             Base.invokelatest(threadsfor_fun, true)
         else
-            in_threaded_loop[] = true
-            # the ccall is not expected to throw
             ccall(:jl_threading_run, Cvoid, (Any,), threadsfor_fun)
-            in_threaded_loop[] = false
         end
         nothing
     end