From 1893c651ab15d4c32f2b4ca39a83eaadb945f55e Mon Sep 17 00:00:00 2001 From: Ian Date: Sat, 18 Mar 2023 22:30:25 -0400 Subject: [PATCH] pidlock cachefile generation for efficiency --- base/loading.jl | 42 +++++++++++++++++++++++-- stdlib/FileWatching/docs/src/index.md | 1 + stdlib/FileWatching/src/FileWatching.jl | 10 ++++-- stdlib/FileWatching/src/pidfile.jl | 32 +++++++++++++++++-- stdlib/FileWatching/test/pidfile.jl | 8 ++--- 5 files changed, 82 insertions(+), 11 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b345293e3bafc..f5c7aa28395ef 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1902,8 +1902,17 @@ function _require(pkg::PkgId, env=nothing) @goto load_from_cache end # spawn off a new incremental pre-compile task for recursive `require` calls - cachefile = compilecache(pkg, path) - if isa(cachefile, Exception) + cachefile_or_module = maybe_cachefile_lock(pkg, path) do + # double-check now that we have lock + m = _require_search_from_serialized(pkg, path, UInt128(0)) + m isa Module && return m + compilecache(pkg, path) + end + cachefile_or_module isa Module && return cachefile_or_module::Module + cachefile = cachefile_or_module + if isnothing(cachefile) # maybe_cachefile_lock returns nothing if it had to wait for another process + @goto load_from_cache # the new cachefile will have the newest mtime so will come first in the search + elseif isa(cachefile, Exception) if precompilableerror(cachefile) verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug @logmsg verbosity "Skipping precompilation since __precompile__(false). Importing $pkg." @@ -2805,6 +2814,35 @@ function show(io::IO, cf::CacheFlags) print(io, ", opt_level = ", cf.opt_level) end +# Set by FileWatching.__init__() +global mkpidlock_hook +global trymkpidlock_hook +global parse_pidfile_hook + +# allows processes to wait if another process is precompiling a given source already +function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String) + if @isdefined(mkpidlock_hook) && @isdefined(trymkpidlock_hook) && @isdefined(parse_pidfile_hook) + pidfile = string(srcpath, ".pidlock") + cachefile = invokelatest(trymkpidlock_hook, f, pidfile) + if cachefile === false + pid, hostname, age = invokelatest(parse_pidfile_hook, pidfile) + verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug + if isempty(hostname) || hostname == gethostname() + @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg" + else + @logmsg verbosity "Waiting for another machine (hostname: $hostname, pid: $pid) to finish precompiling $pkg" + end + # wait until the lock is available, but don't actually acquire it + # returning nothing indicates a process waited for another + return invokelatest(mkpidlock_hook, Returns(nothing), pidfile) + end + return cachefile + else + # for packages loaded before FileWatching.__init__() + f() + end +end + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false) diff --git a/stdlib/FileWatching/docs/src/index.md b/stdlib/FileWatching/docs/src/index.md index 6c332511f578f..a420d49232345 100644 --- a/stdlib/FileWatching/docs/src/index.md +++ b/stdlib/FileWatching/docs/src/index.md @@ -20,6 +20,7 @@ A simple utility tool for creating advisory pidfiles (lock files). ```@docs mkpidlock +trymkpidlock close(lock::LockMonitor) ``` diff --git a/stdlib/FileWatching/src/FileWatching.jl b/stdlib/FileWatching/src/FileWatching.jl index 17ae24460db6b..2a654547ae6e3 100644 --- a/stdlib/FileWatching/src/FileWatching.jl +++ b/stdlib/FileWatching/src/FileWatching.jl @@ -18,7 +18,8 @@ export PollingFileWatcher, FDWatcher, # pidfile: - mkpidlock + mkpidlock, + trymkpidlock import Base: @handle_as, wait, close, eventloop, notify_error, IOError, _sizeof_uv_poll, _sizeof_uv_fs_poll, _sizeof_uv_fs_event, _uv_hook_close, uv_error, _UVError, @@ -462,6 +463,11 @@ function __init__() global uv_jl_fspollcb = @cfunction(uv_fspollcb, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cvoid})) global uv_jl_fseventscb_file = @cfunction(uv_fseventscb_file, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32)) global uv_jl_fseventscb_folder = @cfunction(uv_fseventscb_folder, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32)) + + Base.mkpidlock_hook = mkpidlock + Base.trymkpidlock_hook = trymkpidlock + Base.parse_pidfile_hook = Pidfile.parse_pidfile + nothing end @@ -885,6 +891,6 @@ function poll_file(s::AbstractString, interval_seconds::Real=5.007, timeout_s::R end include("pidfile.jl") -import .Pidfile: mkpidlock +import .Pidfile: mkpidlock, trymkpidlock end diff --git a/stdlib/FileWatching/src/pidfile.jl b/stdlib/FileWatching/src/pidfile.jl index b78f7ef070018..6d40414e20db2 100644 --- a/stdlib/FileWatching/src/pidfile.jl +++ b/stdlib/FileWatching/src/pidfile.jl @@ -1,7 +1,7 @@ module Pidfile -export mkpidlock +export mkpidlock, trymkpidlock using Base: IOError, UV_EEXIST, UV_ESRCH, @@ -41,6 +41,16 @@ Optional keyword arguments: """ function mkpidlock end +""" + trymkpidlock([f::Function], at::String, [pid::Cint, proc::Process]; kwopts...) + +Like `mkpidlock` except returns `false` instead of waiting if the file is already locked. + +!!! compat "Julia 1.10" + This function requires at least Julia 1.10. + +""" +function trymkpidlock end # mutable only because we want to add a finalizer mutable struct LockMonitor @@ -95,6 +105,18 @@ function mkpidlock(at::String, proc::Process; kwopts...) return lock end +function trymkpidlock(args...; kwargs...) + try + mkpidlock(args...; kwargs..., wait=false) + catch ex + if ex isa PidlockedError + return false + else + rethrow() + end + end +end + """ Base.touch(::Pidfile.LockMonitor) @@ -192,8 +214,12 @@ function tryopen_exclusive(path::String, mode::Integer = 0o444) return nothing end +struct PidlockedError <: Exception + msg::AbstractString +end + """ - open_exclusive(path::String; mode, poll_interval, stale_age) :: File + open_exclusive(path::String; mode, poll_interval, wait, stale_age) :: File Create a new a file for read-write advisory-exclusive access. If `wait` is `false` then error out if the lock files exist @@ -218,7 +244,7 @@ function open_exclusive(path::String; file = tryopen_exclusive(path, mode) end if file === nothing - error("Failed to get pidfile lock for $(repr(path)).") + throw(PidlockedError("Failed to get pidfile lock for $(repr(path)).")) else return file end diff --git a/stdlib/FileWatching/test/pidfile.jl b/stdlib/FileWatching/test/pidfile.jl index 94621f6af78e3..c2cb0c88a1b1e 100644 --- a/stdlib/FileWatching/test/pidfile.jl +++ b/stdlib/FileWatching/test/pidfile.jl @@ -180,14 +180,14 @@ end Base.errormonitor(rmtask) t1 = time() - @test_throws ErrorException open_exclusive("pidfile", wait=false) + @test_throws Pidfile.PidlockedError open_exclusive("pidfile", wait=false) @test time()-t1 ≈ 0 atol=1 sleep(1) @test !deleted t1 = time() - @test_throws ErrorException open_exclusive("pidfile", wait=false) + @test_throws Pidfile.PidlockedError open_exclusive("pidfile", wait=false) @test time()-t1 ≈ 0 atol=1 wait(rmtask) @@ -246,7 +246,7 @@ end Base.errormonitor(waittask) # mkpidlock with no waiting - t = @elapsed @test_throws ErrorException mkpidlock("pidfile", wait=false) + t = @elapsed @test_throws Pidfile.PidlockedError mkpidlock("pidfile", wait=false) @test t ≈ 0 atol=1 t = @elapsed lockf1 = mkpidlock(joinpath(dir, "pidfile")) @@ -354,7 +354,7 @@ end @test lockf.update === nothing sleep(1) - t = @elapsed @test_throws ErrorException mkpidlock("pidfile-2", wait=false, stale_age=1, poll_interval=1, refresh=0) + t = @elapsed @test_throws Pidfile.PidlockedError mkpidlock("pidfile-2", wait=false, stale_age=1, poll_interval=1, refresh=0) @test t ≈ 0 atol=1 sleep(5)