pidlock cachefile generation for efficiency

base/loading.jl

@@ -1778,6 +1778,7 @@ function _require(pkg::PkgId, env=nothing)
 
         # attempt to load the module file via the precompile cache locations
         if JLOptions().use_compiled_modules != 0
+            @label search
             m = _require_search_from_serialized(pkg, path, UInt128(0))
             if m isa Module
                 return m
@@ -1800,8 +1801,12 @@ function _require(pkg::PkgId, env=nothing)
         if JLOptions().use_compiled_modules != 0
             if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0)
                 # spawn off a new incremental pre-compile task for recursive `require` calls
-                cachefile = compilecache(pkg, path)
-                if isa(cachefile, Exception)
+                cachefile = maybe_cachefile_lock(path) do
+                    compilecache(pkg, path)
+                end
+                if isnothing(cachefile) # maybe_cachefile_lock returns nothing if it had to wait for another process
+                    @goto search # the new cachefile will have the newest mtime so will come first in the search
+                elseif isa(cachefile, Exception)
                     if precompilableerror(cachefile)
                         verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
                         @logmsg verbosity "Skipping precompilation since __precompile__(false). Importing $pkg."
@@ -2701,6 +2706,18 @@ function show(io::IO, cf::CacheFlags)
     print(io, ", opt_level = ", cf.opt_level)
 end
 
+# Set by FileWatching.__init__()
+const cachefile_lock_hook = Ref{Function}()
+
+function maybe_cachefile_lock(f, srcpath::String)
+    if isassigned(cachefile_lock_hook)
+        cachefile_lock_hook[](f, () -> nothing, srcpath)
+    else
+        # for packages loaded before FileWatching.__init__()
+        f()
+    end
+end
+
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey
 # otherwise returns the list of dependencies to also check
 @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false)

stdlib/FileWatching/src/FileWatching.jl

@@ -18,7 +18,8 @@ export
     PollingFileWatcher,
     FDWatcher,
     # pidfile:
-    mkpidlock
+    mkpidlock,
+    PidfileLockFailure
 
 import Base: @handle_as, wait, close, eventloop, notify_error, IOError,
     _sizeof_uv_poll, _sizeof_uv_fs_poll, _sizeof_uv_fs_event, _uv_hook_close, uv_error, _UVError,
@@ -457,11 +458,16 @@ function uv_fspollcb(handle::Ptr{Cvoid}, status::Int32, prev::Ptr, curr::Ptr)
     nothing
 end
 
+cachefile_lock(f_first::Function , f_waited::Function, path::String) = mkpidlock(f_first, f_waited, string(path, ".pidlock"))
+
 function __init__()
     global uv_jl_pollcb = @cfunction(uv_pollcb, Cvoid, (Ptr{Cvoid}, Cint, Cint))
     global uv_jl_fspollcb = @cfunction(uv_fspollcb, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cvoid}))
     global uv_jl_fseventscb_file = @cfunction(uv_fseventscb_file, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32))
     global uv_jl_fseventscb_folder = @cfunction(uv_fseventscb_folder, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32))
+
+    Base.cachefile_lock_hook[] = cachefile_lock
+
     nothing
 end
 

stdlib/FileWatching/src/pidfile.jl

@@ -1,7 +1,7 @@
 module Pidfile
 
 
-export mkpidlock
+export mkpidlock, PidfileLockFailure
 
 using Base:
     IOError, UV_EEXIST, UV_ESRCH,
@@ -18,6 +18,7 @@ using Base.Sys: iswindows
 
 """
     mkpidlock([f::Function], at::String, [pid::Cint, proc::Process]; kwopts...)
+    mkpidlock(f_first::Function, f_waited::Function, at::String, [pid::Cint, proc::Process]; kwopts...)
 
 Create a pidfile lock for the path "at" for the current process
 or the process identified by pid or proc. Can take a function to execute once locked,
@@ -28,6 +29,9 @@ The lock will be released by either `close`, a `finalizer`, or shortly after `pr
 Make sure the return value is live through the end of the critical section of
 your program, so the `finalizer` does not reclaim it early.
 
+The form `mkpidlock(f_first::Function, f_waited::Function ...` allows different functions to be called depending
+on whether a lock did not exist (`f_first`) or existed already and waiting was necessary (`f_waited`).
+
 Optional keyword arguments:
  - `mode`: file access mode (modified by the process umask). Defaults to world-readable.
  - `poll_interval`: Specify the maximum time to between attempts (if `watch_file` doesn't work)
@@ -47,13 +51,14 @@ mutable struct LockMonitor
     const path::String
     const fd::File
     const update::Union{Nothing,Timer}
+    const waited::Bool
 
     global function mkpidlock(at::String, pid::Cint; stale_age::Real=0, refresh::Real=stale_age/2, kwopts...)
         local lock
         atdir, atname = splitdir(at)
         isempty(atdir) && (atdir = pwd())
         at = realpath(atdir) * path_separator * atname
-        fd = open_exclusive(at; stale_age=stale_age, kwopts...)
+        fd, waited = open_exclusive(at; stale_age=stale_age, kwopts...)
         update = nothing
         try
             write_pidfile(fd, pid)
@@ -62,7 +67,7 @@ mutable struct LockMonitor
                 # `fd` here instead of `lock`.
                 update = Timer(t -> isopen(t) && touch(fd), refresh; interval=refresh)
             end
-            lock = new(at, fd, update)
+            lock = new(at, fd, update, waited)
             finalizer(close, lock)
         catch ex
             tryrmopenfile(at)
@@ -85,6 +90,21 @@ function mkpidlock(f::Function, at::String, pid::Cint; kwopts...)
     end
 end
 
+mkpidlock(f_first::Function, f_waited::Function, at::String; kwopts...) = mkpidlock(f_first, f_waited, at, getpid(); kwopts...)
+
+function mkpidlock(f_first::Function, f_waited::Function, at::String, pid::Cint; kwopts...)
+    lock = mkpidlock(at, pid; kwopts...)
+    try
+        if lock.waited
+            return f_waited()
+        else
+            return f_first()
+        end
+    finally
+        close(lock)
+    end
+end
+
 function mkpidlock(at::String, proc::Process; kwopts...)
     lock = mkpidlock(at, getpid(proc); kwopts...)
     closer = @async begin
@@ -192,14 +212,19 @@ function tryopen_exclusive(path::String, mode::Integer = 0o444)
     return nothing
 end
 
+struct PidfileLockFailure <: Exception
+    msg::String
+end
+
 """
-    open_exclusive(path::String; mode, poll_interval, stale_age) :: File
+    open_exclusive(path::String; mode, poll_interval, stale_age) :: (File, waited::Bool)
 
 Create a new a file for read-write advisory-exclusive access.
 If `wait` is `false` then error out if the lock files exist
 otherwise block until we get the lock.
 
 For a description of the keyword arguments, see [`mkpidlock`](@ref).
+
 """
 function open_exclusive(path::String;
                         mode::Integer = 0o444 #= read-only =#,
@@ -208,7 +233,7 @@ function open_exclusive(path::String;
                         stale_age::Real = 0 #= disabled =#)
     # fast-path: just try to open it
     file = tryopen_exclusive(path, mode)
-    file === nothing || return file
+    file === nothing || return file, false
     if !wait
         if file === nothing && stale_age > 0
             if stale_age > 0 && stale_pidfile(path, stale_age)
@@ -218,9 +243,9 @@ function open_exclusive(path::String;
             file = tryopen_exclusive(path, mode)
         end
         if file === nothing
-            error("Failed to get pidfile lock for $(repr(path)).")
+            throw(PidfileLockFailure("Failed to get pidfile lock for $(repr(path))."))
         else
-            return file
+            return file, false
         end
     end
     # fall-back: wait for the lock
@@ -235,7 +260,7 @@ function open_exclusive(path::String;
         end
         # now try again to create it
         file = tryopen_exclusive(path, mode)
-        file === nothing || return file
+        file === nothing || return file, true
         Base.wait(t) # sleep for a bit before trying again
         if stale_age > 0 && stale_pidfile(path, stale_age)
             # if the file seems stale, try to remove it before attempting again