diff --git a/src/GraphType.jl b/src/GraphType.jl new file mode 100644 index 0000000000000..263fa67420e2f --- /dev/null +++ b/src/GraphType.jl @@ -0,0 +1,910 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module GraphType + +using ..Types +import ..Types.uuid_julia +import Pkg3.equalto + +export Graph, add_reqs!, add_fixed!, simplify_graph! + +# This is used to keep track of dependency relations when propagating +# requirements, so as to emit useful information in case of unsatisfiable +# conditions. +# The `why` field is a Vector which keeps track of the requirements. Each +# entry is a Tuple of two elements: +# 1) the first element is the reason, and it can be either :fixed (for +# fixed packages), :explicit_requirement (for explicitly required packages), +# or a Tuple `(:constr_prop, p, backtrace_item)` (for requirements induced +# indirectly), where `p` is the package index and `backtrace_item` is +# another ResolveBacktraceItem. +# 2) the second element is a BitVector representing the requirement as a mask +# over the possible states of the package +mutable struct ResolveBacktraceItem + why::Vector{Any} + ResolveBacktraceItem() = new(Any[]) +end + +function Base.push!(ritem::ResolveBacktraceItem, reason, versionmask) + push!(ritem.why, (reason, versionmask)) +end + +# Installation state: either a version, or uninstalled +const InstState = Union{VersionNumber,Void} + +mutable struct GraphData + # packages list + pkgs::Vector{UUID} + + # number of packages + np::Int + + # states per package: one per version + uninstalled + spp::Vector{Int} + + # pakage dict: associates an index to each package id + pdict::Dict{UUID,Int} + + # package versions: for each package, keep the list of the + # possible version numbers; this defines a + # mapping from version numbers of a package + # to indices + pvers::Vector{Vector{VersionNumber}} + + # versions dict: associates a version index to each package + # version; such that + # pvers[p0][vdict[p0][vn]] = vn + vdict::Vector{Dict{VersionNumber,Int}} + + uuid_to_name::Dict{UUID,String} + + reqs::Requires + fixed::Dict{UUID,Fixed} + + # pruned packages: during graph simplification, packages that + # only have one allowed version are pruned. + # This keeps track of them, so that they may + # be returned in the solution (unless they + # were explicitly fixed) + pruned::Dict{UUID,VersionNumber} + + # equivalence classes: for each package and each of its possible + # states, keep track of other equivalent states + eq_classes::Dict{UUID,Dict{InstState,Set{InstState}}} + + function GraphData( + versions::Dict{UUID,Set{VersionNumber}}, + deps::Dict{UUID,Dict{VersionRange,Dict{String,UUID}}}, + compat::Dict{UUID,Dict{VersionRange,Dict{String,VersionSpec}}}, + uuid_to_name::Dict{UUID,String}, + reqs::Requires, + fixed::Dict{UUID,Fixed} + ) + # generate pkgs + pkgs = sort!(collect(keys(versions))) + np = length(pkgs) + + # generate pdict + pdict = Dict{UUID,Int}(pkgs[p0] => p0 for p0 = 1:np) + + # generate spp and pvers + pvers = [sort!(collect(versions[pkgs[p0]])) for p0 = 1:np] + spp = length.(pvers) .+ 1 + + # generate vdict + vdict = [Dict{VersionNumber,Int}(vn => i for (i,vn) in enumerate(pvers[p0])) for p0 = 1:np] + + pruned = Dict{UUID,VersionNumber}() + + # equivalence classes (at the beginning each state represents just itself) + eq_vn(v0, p0) = (v0 == spp[p0] ? nothing : pvers[p0][v0]) + eq_classes = Dict(pkgs[p0] => Dict(eq_vn(v0,p0) => Set([eq_vn(v0,p0)]) for v0 = 1:spp[p0]) for p0 = 1:np) + + return new(pkgs, np, spp, pdict, pvers, vdict, uuid_to_name, reqs, fixed, pruned, eq_classes) + end +end + +@enum DepDir FORWARD BACKWARDS BIDIR NONE + +function update_depdir(dd0::DepDir, dd1::DepDir) + dd0 == dd1 && return dd0 + dd0 == NONE && return dd1 + dd1 == NONE && return dd0 + return BIDIR +end + +mutable struct Graph + # data: + # stores all the structures required to map between + # parsed items (names, UUIDS, version numbers...) and + # the numeric representation used in the main Graph data + # structure. + data::GraphData + + # adjacency matrix: + # for each package, has the list of neighbors + # indices + gadj::Vector{Vector{Int}} + + # compatibility mask: + # for each package p0 has a list of bool masks. + # Each entry in the list gmsk[p0] is relative to the + # package p1 as read from gadj[p0]. + # Each mask has dimension spp1 × spp0, where + # spp0 is the number of states of p0, and + # spp1 is the number of states of p1. + gmsk::Vector{Vector{BitMatrix}} + + # dependency direction: + # keeps track of which direction the dependency goes. + gdir::Vector{Vector{DepDir}} + + # constraints: + # a mask of allowed states for each package (e.g. to express + # requirements) + gconstr::Vector{BitVector} + + # adjacency dict: + # allows one to retrieve the indices in gadj, so that + # gadj[p0][adjdict[p1][p0]] = p1 + # ("At which index does package p1 appear in gadj[p0]?") + adjdict::Vector{Dict{Int,Int}} + + # indices of the packages that were *explicitly* required + # used to favor their versions at resolution + req_inds::Set{Int} + + # indices of the packages that were *explicitly* fixed + # used to avoid returning them in the solution + fix_inds::Set{Int} + + # states per package: same as in GraphData + spp::Vector{Int} + + # backtrace: keep track of the resolution process + bktrc::Vector{ResolveBacktraceItem} + + # number of packages (all Vectors above have this length) + np::Int + + function Graph( + versions::Dict{UUID,Set{VersionNumber}}, + deps::Dict{UUID,Dict{VersionRange,Dict{String,UUID}}}, + compat::Dict{UUID,Dict{VersionRange,Dict{String,VersionSpec}}}, + uuid_to_name::Dict{UUID,String}, + reqs::Requires = Requires(), + fixed::Dict{UUID,Fixed} = Dict{UUID,Fixed}(uuid_julia=>Fixed(VERSION)) + ) + + extra_uuids = union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...) + extra_uuids ⊆ keys(versions) || error("unknown UUID found in reqs/fixed") # TODO? + + data = GraphData(versions, deps, compat, uuid_to_name, reqs, fixed) + pkgs, np, spp, pdict, pvers, vdict = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict + + extended_deps = [[Dict{Int,BitVector}() for v0 = 1:(spp[p0]-1)] for p0 = 1:np] + for p0 = 1:np, v0 = 1:(spp[p0]-1) + n2u = Dict{String,UUID}() + vn = pvers[p0][v0] + for (vr,vrmap) in deps[pkgs[p0]] + vn ∈ vr || continue + for (name,uuid) in vrmap + # check conflicts ?? + n2u[name] = uuid + end + end + req = Dict{Int,VersionSpec}() + for (vr,vrmap) in compat[pkgs[p0]] + vn ∈ vr || continue + for (name,vs) in vrmap + haskey(n2u, name) || error("Unknown package $name found in the compatibility requirements of $(pkgID(pkgs[p0], uuid_to_name))") + uuid = n2u[name] + p1 = pdict[uuid] + # check conflicts instead of intersecting? + # (intersecting is used by fixed packages though...) + req_p1 = get!(req, p1) do; VersionSpec() end + req[p1] = req_p1 ∩ vs + end + end + # The remaining dependencies do not have compatibility constraints + for uuid in values(n2u) + get!(req, pdict[uuid]) do; VersionSpec() end + end + # Translate the requirements into bit masks + # req_msk = Dict(p1 => BitArray(pvers[p1][v1] ∈ vs for v1 = 0:(spp[p1]-1)) for (p1,vs) in req) + req_msk = Dict(p1 => (pvers[p1][1:(spp[p1]-1)] .∈ vs) for (p1,vs) in req) + extended_deps[p0][v0] = req_msk + end + + gadj = [Int[] for p0 = 1:np] + gmsk = [BitMatrix[] for p0 = 1:np] + gdir = [DepDir[] for p0 = 1:np] + gconstr = [trues(spp[p0]) for p0 = 1:np] + adjdict = [Dict{Int,Int}() for p0 = 1:np] + + for p0 = 1:np, v0 = 1:(spp[p0]-1), (p1,rmsk1) in extended_deps[p0][v0] + j0 = get(adjdict[p1], p0, length(gadj[p0]) + 1) + j1 = get(adjdict[p0], p1, length(gadj[p1]) + 1) + + @assert (j0 > length(gadj[p0]) && j1 > length(gadj[p1])) || + (j0 ≤ length(gadj[p0]) && j1 ≤ length(gadj[p1])) + + if j0 > length(gadj[p0]) + push!(gadj[p0], p1) + push!(gadj[p1], p0) + j0 = length(gadj[p0]) + j1 = length(gadj[p1]) + + adjdict[p1][p0] = j0 + adjdict[p0][p1] = j1 + + bm = trues(spp[p1], spp[p0]) + bmt = bm' + + push!(gmsk[p0], bm) + push!(gmsk[p1], bmt) + + push!(gdir[p0], FORWARD) + push!(gdir[p1], BACKWARDS) + else + bm = gmsk[p0][j0] + bmt = gmsk[p1][j1] + gdir[p0][j0] = update_depdir(gdir[p0][j0], FORWARD) + gdir[p1][j1] = update_depdir(gdir[p1][j1], BACKWARDS) + end + + for v1 = 1:(spp[p1]-1) + rmsk1[v1] && continue + bm[v1, v0] = false + bmt[v0, v1] = false + end + bm[end,v0] = false + bmt[v0,end] = false + end + + req_inds = Set{Int}() + fix_inds = Set{Int}() + + bktrc = [ResolveBacktraceItem() for p0 = 1:np] + + graph = new(data, gadj, gmsk, gdir, gconstr, adjdict, req_inds, fix_inds, spp, bktrc, np) + + _add_fixed!(graph, fixed) + _add_reqs!(graph, reqs, :explicit_requirement) + + @assert check_consistency(graph) + check_constraints(graph) + + return graph + end +end + +""" +Add explicit requirements to the graph. +""" +function add_reqs!(graph::Graph, reqs::Requires) + _add_reqs!(graph, reqs, :explicit_requirement) + check_constraints(graph) + # TODO: add reqs to graph data? + return graph +end + +function _add_reqs!(graph::Graph, reqs::Requires, reason) + gconstr = graph.gconstr + spp = graph.spp + req_inds = graph.req_inds + bktrc = graph.bktrc + pdict = graph.data.pdict + pvers = graph.data.pvers + + for (rp,rvs) in reqs + haskey(pdict, rp) || error("unknown required package $(pkgID(rp, graph))") + rp0 = pdict[rp] + new_constr = trues(spp[rp0]) + for rv0 = 1:(spp[rp0]-1) + rvn = pvers[rp0][rv0] + rvn ∈ rvs || (new_constr[rv0] = false) + end + new_constr[end] = false + old_constr = copy(gconstr[rp0]) + gconstr[rp0] .&= new_constr + reason ≡ :explicit_requirement && push!(req_inds, rp0) + old_constr ≠ gconstr[rp0] && push!(bktrc[rp0], reason, new_constr) + end + return graph +end + +"Add fixed packages to the graph, and their requirements." +function add_fixed!(graph::Graph, fixed::Dict{UUID,Fixed}) + _add_fixed!(graph, fixed) + check_constraints(graph) + # TODO: add fixed to graph data? + return graph +end + +function _add_fixed!(graph::Graph, fixed::Dict{UUID,Fixed}) + gconstr = graph.gconstr + spp = graph.spp + fix_inds = graph.fix_inds + bktrc = graph.bktrc + pdict = graph.data.pdict + vdict = graph.data.vdict + + for (fp,fx) in fixed + haskey(pdict, fp) || error("unknown fixed package $(pkgID(fp, graph))") + fp0 = pdict[fp] + fv0 = vdict[fp0][fx.version] + new_constr = falses(spp[fp0]) + new_constr[fv0] = true + gconstr[fp0] .&= new_constr + push!(fix_inds, fp0) + push!(bktrc[fp0], :fixed, new_constr) + _add_reqs!(graph, fx.requires, (:constr_prop, fp0, bktrc[fp0])) + end + return graph +end + +Types.pkgID(p::UUID, graph::Graph) = pkgID(p, graph.data.uuid_to_name) +Types.pkgID(p0::Int, graph::Graph) = pkgID(graph.data.pkgs[p0], graph) + +function check_consistency(graph::Graph) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gdir = graph.gdir + gconstr = graph.gconstr + adjdict = graph.adjdict + req_inds = graph.req_inds + fix_inds = graph.fix_inds + bktrc = graph.bktrc + data = graph.data + pkgs = data.pkgs + pdict = data.pdict + pvers = data.pvers + vdict = data.vdict + pruned = data.pruned + eq_classes = data.eq_classes + + @assert np ≥ 0 + for x in [spp, gadj, gmsk, gdir, gconstr, adjdict, bktrc, pkgs, pdict, pvers, vdict] + @assert length(x) == np + end + for p0 = 1:np + @assert pdict[pkgs[p0]] == p0 + spp0 = spp[p0] + @assert spp0 ≥ 1 + pvers0 = pvers[p0] + vdict0 = vdict[p0] + @assert length(pvers0) == spp0 - 1 + for v0 = 1:(spp0-1) + @assert vdict0[pvers0[v0]] == v0 + end + for (vn,v0) in vdict0 + @assert 1 ≤ v0 ≤ spp0-1 + @assert pvers0[v0] == vn + end + gconstr0 = gconstr[p0] + @assert length(gconstr0) == spp0 + + gadj0 = gadj[p0] + gmsk0 = gmsk[p0] + gdir0 = gdir[p0] + adjdict0 = adjdict[p0] + @assert length(gmsk0) == length(gadj0) + @assert length(adjdict0) == length(gadj0) + @assert length(gdir0) == length(gadj0) + for (j0,p1) in enumerate(gadj0) + @assert adjdict[p1][p0] == j0 + spp1 = spp[p1] + @assert size(gmsk0[j0]) == (spp1,spp0) + j1 = adjdict0[p1] + gmsk1 = gmsk[p1] + @assert gmsk1[j1] == gmsk0[j0]' + end + end + for (p,p0) in pdict + @assert 1 ≤ p0 ≤ np + @assert pkgs[p0] == p + @assert !haskey(pruned, p) + end + for p0 in req_inds + @assert 1 ≤ p0 ≤ np + @assert !gconstr[p0][end] + end + for p0 in fix_inds + @assert 1 ≤ p0 ≤ np + @assert !gconstr[p0][end] + @assert count(gconstr[p0]) == 1 + end + + for (p,eq_cl) in eq_classes, (rvn,rvs) in eq_cl + @assert rvn ∈ rvs + end + + return true +end + +"Show the resolution backtrace for some package" +function showbacktrace(io::IO, graph::Graph, p0::Int) + _show(io, graph, p0, graph.bktrc[p0], "", Set{ResolveBacktraceItem}()) +end + +# Show a recursive tree with requirements applied to a package, either directly or indirectly +function _show(io::IO, graph::Graph, p0::Int, ritem::ResolveBacktraceItem, indent::String, seen::Set{ResolveBacktraceItem}) + id0 = pkgID(p0, graph) + gconstr = graph.gconstr + pkgs = graph.data.pkgs + pvers = graph.data.pvers + + function vs_string(p0::Int, vmask::BitVector) + vns = Vector{Any}(pvers[p0][vmask[1:(end-1)]]) + vmask[end] && push!(vns, "uninstalled") + return join(string.(vns), ", ", " or ") + end + + l = length(ritem.why) + for (i,(w,vmask)) in enumerate(ritem.why) + print(io, indent, (i==l ? '└' : '├'), '─') + if w ≡ :fixed + @assert count(vmask) == 1 + println(io, "$id0 is fixed to version ", vs_string(p0, vmask)) + elseif w ≡ :explicit_requirement + @assert !vmask[end] + if any(vmask) + println(io, "an explicit requirement sets $id0 to versions: ", vs_string(p0, vmask)) + else + println(io, "an explicit requirement cannot be matched by any of the available versions of $id0") + end + else + @assert w isa Tuple{Symbol,Int,ResolveBacktraceItem} + @assert w[1] == :constr_prop + p1 = w[2] + if !is_current_julia(graph, p1) + id1 = pkgID(p1, graph) + otheritem = w[3] + if any(vmask) + println(io, "the only versions of $id0 compatible with $id1 (whose allowed versions are $(vs_string(p1, gconstr[p1])))\n", + indent, (i==l ? " " : "│ "),"are these: ", vs_string(p0, vmask)) + else + println(io, "no versions of $id0 are compatible with $id1 (whose allowed versions are $(vs_string(p1, gconstr[p1])))") + end + if otheritem ∈ seen + println(io, indent, (i==l ? " " : "│ "), "└─see above for $id1 backtrace") + continue + end + push!(seen, otheritem) + _show(io, graph, p1, otheritem, indent * (i==l ? " " : "│ "), seen) + else + if any(vmask) + println(io, "the only versions of $id0 compatible with julia v$VERSION are these: ", vs_string(p0, vmask)) + else + println(io, "no versions of $id0 are compatible with julia v$VERSION") + end + end + end + end +end + +function is_current_julia(graph::Graph, p1::Int) + gconstr = graph.gconstr + fix_inds = graph.fix_inds + pkgs = graph.data.pkgs + pvers = graph.data.pvers + + (pkgs[p1] == uuid_julia && p1 ∈ fix_inds) || return false + jconstr = gconstr[p1] + return length(jconstr) == 2 && !jconstr[2] && pvers[p1][1] == VERSION +end + +"Check for contradictions in the constraints." +function check_constraints(graph::Graph) + np = graph.np + gconstr = graph.gconstr + pkgs = graph.data.pkgs + pvers = graph.data.pvers + + id(p0::Int) = pkgID(pkgs[p0], graph) + + for p0 = 1:np + any(gconstr[p0]) && continue + err_msg = "Unsatisfiable requirements detected for package $(id(p0)):\n" + err_msg *= sprint(showbacktrace, graph, p0) + throw(PkgError(err_msg)) + end + return true +end + +""" +Propagates current constraints, determining new implicit constraints. +Throws an error in case impossible requirements are detected, printing +a backtrace. +""" +function propagate_constraints!(graph::Graph) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + bktrc = graph.bktrc + gconstr = graph.gconstr + adjdict = graph.adjdict + pkgs = graph.data.pkgs + pvers = graph.data.pvers + + id(p0::Int) = pkgID(pkgs[p0], graph) + + # packages which are not allowed to be uninstalled + staged = Set{Int}(p0 for p0 = 1:np if !gconstr[p0][end]) + + while !isempty(staged) + staged_next = Set{Int}() + for p0 in staged + gconstr0 = gconstr[p0] + for (j1,p1) in enumerate(gadj[p0]) + # we don't propagate to julia (purely to have better error messages) + is_current_julia(graph, p1) && continue + + msk = gmsk[p0][j1] + # consider the sub-mask with only allowed versions of p0 + sub_msk = msk[:,gconstr0] + # if an entire row of the sub-mask is false, that version of p1 + # is effectively forbidden + # (this is just like calling `any` row-wise) + added_constr1 = any!(BitVector(spp[p1]), sub_msk) + # apply the new constraints, checking for contradictions + # (keep the old ones for comparison) + gconstr1 = gconstr[p1] + old_gconstr1 = copy(gconstr1) + gconstr1 .&= added_constr1 + # if the new constraints are more restrictive than the + # previous ones, record it and propagate them next + if gconstr1 ≠ old_gconstr1 + push!(staged_next, p1) + push!(bktrc[p1], (:constr_prop, p0, bktrc[p0]), added_constr1) + end + if !any(gconstr1) + err_msg = "Unsatisfiable requirements detected for package $(id(p1)):\n" + err_msg *= sprint(showbacktrace, graph, p1) + throw(PkgError(err_msg)) + end + end + end + staged = staged_next + end + return graph +end + +""" +Enforce the uninstalled state on all packages that are not reachable from the required ones +or from the packages in the `sources` argument. +""" +function disable_unreachable!(graph::Graph, sources::Set{Int} = Set{Int}()) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gconstr = graph.gconstr + adjdict = graph.adjdict + pkgs = graph.data.pkgs + + # packages which are not allowed to be uninstalled + staged = union(sources, Set{Int}(p0 for p0 = 1:np if !gconstr[p0][end])) + seen = copy(staged) + + while !isempty(staged) + staged_next = Set{Int}() + for p0 in staged + gconstr0idx = find(gconstr[p0][1:(end-1)]) + for (j1,p1) in enumerate(gadj[p0]) + all(gmsk[p0][j1][end,gconstr0idx]) && continue # the package is not required by any of the allowed versions of p0 + p1 ∈ seen || push!(staged_next, p1) + end + end + union!(seen, staged_next) + staged = staged_next + end + + # Force uninstalled state for all unseen packages + for p0 = 1:np + p0 ∈ seen && continue + gconstr0 = gconstr[p0] + @assert gconstr0[end] + fill!(gconstr0, false) + gconstr0[end] = true + end + + return graph +end + +""" +Reduce the number of versions in the graph by putting all the versions of +a package that behave identically into equivalence classes, keeping only +the highest version of the class as representative. +""" +function compute_eq_classes!(graph::Graph; verbose::Bool = false) + np = graph.np + sumspp = sum(graph.spp) + for p0 = 1:np + build_eq_classes1!(graph, p0) + end + + if verbose + info(""" + EQ CLASSES STATS: + before: $(sumspp) + after: $(sum(graph.spp)) + """) + end + + # wipe out backtrace because it doesn't make sense now + # TODO: save it somehow? + graph.bktrc = [ResolveBacktraceItem() for p0 = 1:np] + + @assert check_consistency(graph) + + return graph +end + +function build_eq_classes1!(graph::Graph, p0::Int) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gconstr = graph.gconstr + adjdict = graph.adjdict + data = graph.data + pkgs = data.pkgs + pvers = data.pvers + vdict = data.vdict + eq_classes = data.eq_classes + + # concatenate all the constraints; the columns of the + # result encode the behavior of each version + cmat = vcat(BitMatrix(gconstr[p0]'), gmsk[p0]...) + cvecs = [cmat[:,v0] for v0 = 1:spp[p0]] + + # find unique behaviors + repr_vecs = unique(cvecs) + + # number of equivaent classes + neq = length(repr_vecs) + + neq == spp[p0] && return # nothing to do here + + # group versions into sets that behave identically + eq_sets = [Set{Int}(v0 for v0 in 1:spp[p0] if cvecs[v0] == rvec) for rvec in repr_vecs] + sort!(eq_sets, by=maximum) + + # each set is represented by its highest-valued member + repr_vers = map(maximum, eq_sets) + # the last representative must always be the uninstalled state + @assert repr_vers[end] == spp[p0] + + # update equivalence classes + eq_vn(v0) = (v0 == spp[p0] ? nothing : pvers[p0][v0]) + eq_classes0 = eq_classes[pkgs[p0]] + for (v0,rvs) in zip(repr_vers, eq_sets) + @assert v0 ∈ rvs + vn0 = eq_vn(v0) + for v1 in rvs + v1 == v0 && continue + vn1 = eq_vn(v1) + @assert vn1 ≢ nothing + union!(eq_classes0[vn0], eq_classes0[vn1]) + delete!(eq_classes0, vn1) + end + end + + # reduce the constraints and the interaction matrices + spp[p0] = neq + gconstr[p0] = gconstr[p0][repr_vers] + for (j1,p1) in enumerate(gadj[p0]) + gmsk[p0][j1] = gmsk[p0][j1][:,repr_vers] + + j0 = adjdict[p0][p1] + gmsk[p1][j0] = gmsk[p1][j0][repr_vers,:] + end + + # reduce/rebuild version dictionaries + pvers[p0] = pvers[p0][repr_vers[1:(end-1)]] + vdict[p0] = Dict(vn => i for (i,vn) in enumerate(pvers[p0])) + + return +end + +""" +Prune away fixed and unnecessary packages, and the +disallowed versions for the remaining packages. +""" +function prune_graph!(graph::Graph; verbose::Bool = false) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gdir = graph.gdir + gconstr = graph.gconstr + adjdict = graph.adjdict + req_inds = graph.req_inds + fix_inds = graph.fix_inds + bktrc = graph.bktrc + data = graph.data + pkgs = data.pkgs + pdict = data.pdict + pvers = data.pvers + vdict = data.vdict + pruned = data.pruned + + # We will remove all packages that only have one allowed state + # (includes fixed packages and forbidden packages) + pkg_mask = BitArray(count(gconstr[p0]) ≠ 1 for p0 = 1:np) + new_np = count(pkg_mask) + + # a map that translates the new index ∈ 1:new_np into its + # corresponding old index ∈ 1:np + old_idx = find(pkg_mask) + # the reverse of the above + new_idx = Dict{Int,Int}() + for new_p0 = 1:new_np + new_idx[old_idx[new_p0]] = new_p0 + end + + # Update requirement indices + new_req_inds = Set{Int}() + for p0 in req_inds + pkg_mask[p0] || continue + push!(new_req_inds, new_idx[p0]) + end + + # Fixed packages will all be pruned + new_fix_inds = Set{Int}() + for p0 in fix_inds + @assert !pkg_mask[p0] + end + + # Record which packages we are going to prune + for p0 in find(.~(pkg_mask)) + # Find the version + s0 = findfirst(gconstr[p0]) + # We don't record fixed packages + p0 ∈ fix_inds && (@assert s0 ≠ spp[p0]; continue) + p0 ∈ req_inds && @assert s0 ≠ spp[p0] + # We don't record packages that are not going to be installed + s0 == spp[p0] && continue + @assert !haskey(pruned, pkgs[p0]) + pruned[pkgs[p0]] = pvers[p0][s0] + end + + # Update packages records + new_pkgs = pkgs[pkg_mask] + new_pdict = Dict(new_pkgs[new_p0]=>new_p0 for new_p0 = 1:new_np) + + # For each package (unless it's going to be pruned) we will remove all + # versions that aren't allowed (but not the "uninstalled" state) + function keep_vers(new_p0) + p0 = old_idx[new_p0] + return BitArray((v0 == spp[p0]) | gconstr[p0][v0] for v0 = 1:spp[p0]) + end + vers_mask = [keep_vers(new_p0) for new_p0 = 1:new_np] + + # Update number of states per package + new_spp = Int[count(vers_mask[new_p0]) for new_p0 = 1:new_np] + + # Update versions maps + function compute_pvers(new_p0) + p0 = old_idx[new_p0] + pvers0 = pvers[p0] + vmsk0 = vers_mask[new_p0] + return pvers0[vmsk0[1:(end-1)]] + end + new_pvers = [compute_pvers(new_p0) for new_p0 = 1:new_np] + new_vdict = [Dict(vn => v0 for (v0,vn) in enumerate(new_pvers[new_p0])) for new_p0 = 1:new_np] + + # The new constraints are all going to be `true`, except possibly + # for the "uninstalled" state, which we copy over from the old + function compute_gconstr(new_p0) + p0 = old_idx[new_p0] + new_gconstr0 = trues(new_spp[new_p0]) + new_gconstr0[end] = gconstr[p0][end] + return new_gconstr0 + end + new_gconstr = [compute_gconstr(new_p0) for new_p0 = 1:new_np] + + # Recreate the graph adjacency list, skipping some packages + new_gadj = [Int[] for new_p0 = 1:new_np] + new_adjdict = [Dict{Int,Int}() for new_p0 = 1:new_np] + + for new_p0 = 1:new_np, (j1,p1) in enumerate(gadj[old_idx[new_p0]]) + pkg_mask[p1] || continue + new_p1 = new_idx[p1] + + new_j0 = get(new_adjdict[new_p1], new_p0, length(new_gadj[new_p0]) + 1) + new_j1 = get(new_adjdict[new_p0], new_p1, length(new_gadj[new_p1]) + 1) + + @assert (new_j0 > length(new_gadj[new_p0]) && new_j1 > length(new_gadj[new_p1])) || + (new_j0 ≤ length(new_gadj[new_p0]) && new_j1 ≤ length(new_gadj[new_p1])) + + new_j0 > length(new_gadj[new_p0]) || continue + push!(new_gadj[new_p0], new_p1) + push!(new_gadj[new_p1], new_p0) + new_j0 = length(new_gadj[new_p0]) + new_j1 = length(new_gadj[new_p1]) + + new_adjdict[new_p1][new_p0] = new_j0 + new_adjdict[new_p0][new_p1] = new_j1 + end + + # Recompute gdir on the new adjacency list + function compute_gdir(new_p0, new_j0) + p0 = old_idx[new_p0] + new_p1 = new_gadj[new_p0][new_j0] + p1 = old_idx[new_p1] + j0 = adjdict[p1][p0] + return gdir[p0][j0] + end + new_gdir = [[compute_gdir(new_p0, new_j0) for new_j0 = 1:length(new_gadj[new_p0])] for new_p0 = 1:new_np] + + # Recompute compatibility masks on the new adjacency list, and filtering out some versions + function compute_gmsk(new_p0, new_j0) + p0 = old_idx[new_p0] + new_p1 = new_gadj[new_p0][new_j0] + p1 = old_idx[new_p1] + j0 = adjdict[p1][p0] + return gmsk[p0][j0][vers_mask[new_p1],vers_mask[new_p0]] + end + new_gmsk = [[compute_gmsk(new_p0, new_j0) for new_j0 = 1:length(new_gadj[new_p0])] for new_p0 = 1:new_np] + + # Clear out resolution backtrace + # TODO: save it somehow? + new_bktrc = [ResolveBacktraceItem() for new_p0 = 1:new_np] + + # Done + + if verbose + info(""" + GRAPH SIMPLIFY STATS: + before: np = $np ⟨spp⟩ = $(mean(spp)) + after: np = $new_np ⟨spp⟩ = $(mean(new_spp)) + """) + end + + # Replace old data with new + data.pkgs = new_pkgs + data.np = new_np + data.spp = new_spp + data.pdict = new_pdict + data.pvers = new_pvers + data.vdict = new_vdict + # Notes: + # * uuid_to_name, reqs, fixed, eq_classes are unchanged + # * pruned was updated in-place + + # Replace old structures with new ones + graph.gadj = new_gadj + graph.gmsk = new_gmsk + graph.gdir = new_gdir + graph.gconstr = new_gconstr + graph.adjdict = new_adjdict + graph.req_inds = new_req_inds + graph.fix_inds = new_fix_inds + graph.spp = new_spp + graph.bktrc = new_bktrc + graph.np = new_np + + @assert check_consistency(graph) + + return graph +end + +""" +Simplifies the graph by propagating constraints, disabling unreachable versions, pruning +and grouping versions into equivalence classes. +""" +function simplify_graph!(graph::Graph, sources::Set{Int} = Set{Int}(); verbose::Bool = false) + propagate_constraints!(graph) + disable_unreachable!(graph, sources) + prune_graph!(graph, verbose = verbose) + compute_eq_classes!(graph, verbose = verbose) + return graph +end + +end # module diff --git a/src/Operations.jl b/src/Operations.jl index 3c5273b6e6e72..14c6e825e3a2f 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2,8 +2,9 @@ module Operations using Base.Random: UUID using Base: LibGit2 -using Pkg3: TerminalMenus, Types, Query, Resolve +using Pkg3: TerminalMenus, Types, GraphType, Resolve import Pkg3: GLOBAL_SETTINGS, depots, BinaryProvider +import Pkg3.Types: uuid_julia const SlugInt = UInt32 # max p = 4 const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" @@ -90,6 +91,8 @@ end get_or_make(::Type{T}, d::Dict{K}, k::K) where {T,K} = haskey(d, k) ? convert(T, d[k]) : T() +get_or_make!(d::Dict{K,V}, k::K) where {K,V} = get!(d, k) do; V() end + function load_versions(path::String) toml = parse_toml(path, "versions.toml") Dict(VersionNumber(ver) => SHA1(info["hash-sha1"]) for (ver, info) in toml) @@ -114,36 +117,76 @@ end load_package_data(f::Base.Callable, path::String, version::VersionNumber) = get(load_package_data(f, path, [version]), version, nothing) -function deps_graph(env::EnvCache, pkgs::Vector{PackageSpec}) - deps = DepsGraph() - uuids = [pkg.uuid for pkg in pkgs] +function load_package_data_raw(T::Type, path::String) + toml = parse_toml(path, fakeit=true) + data = Dict{VersionRange,Dict{String,T}}() + for (v, d) in toml, (key, value) in d + vr = VersionRange(v) + dict = get!(data, vr, Dict{String,T}()) + haskey(dict, key) && cmderror("$vr/$key is duplicated in $path") + dict[key] = T(value) + end + return data +end + +function deps_graph(env::EnvCache, uuid_to_name::Dict{UUID,String}, reqs::Requires, fixed::Dict{UUID,Fixed}) + uuids = union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...) seen = UUID[] + + all_versions = Dict{UUID,Set{VersionNumber}}(fp => Set([fx.version]) for (fp,fx) in fixed) + all_deps = Dict{UUID,Dict{VersionRange,Dict{String,UUID}}}(fp => Dict(VersionRange(fx.version) => Dict()) for (fp,fx) in fixed) + all_compat = Dict{UUID,Dict{VersionRange,Dict{String,VersionSpec}}}(fp => Dict(VersionRange(fx.version) => Dict()) for (fp,fx) in fixed) while true unseen = setdiff(uuids, seen) isempty(unseen) && break for uuid in unseen push!(seen, uuid) - deps[uuid] = valtype(deps)() + all_versions_u = get_or_make!(all_versions, uuid) + all_deps_u = get_or_make!(all_deps, uuid) + all_compat_u = get_or_make!(all_compat, uuid) + # make sure all versions of all packages know about julia uuid + if uuid ≠ uuid_julia + deps_u_allvers = get_or_make!(all_deps_u, VersionRange()) + deps_u_allvers["julia"] = uuid_julia + end for path in registered_paths(env, uuid) version_info = load_versions(path) versions = sort!(collect(keys(version_info))) - dependencies = load_package_data(UUID, joinpath(path, "dependencies.toml"), versions) - compatibility = load_package_data(VersionSpec, joinpath(path, "compatibility.toml"), versions) - for (v, h) in version_info - d = get_or_make(Dict{String,UUID}, dependencies, v) - r = get_or_make(Dict{String,VersionSpec}, compatibility, v) - q = Dict(u => get_or_make(VersionSpec, r, p) for (p, u) in d) - # VERSION in get_or_make(VersionSpec, r, "julia") || continue - deps[uuid][v] = q - for (p, u) in d - u in uuids || push!(uuids, u) + deps_data = load_package_data_raw(UUID, joinpath(path, "dependencies.toml")) + compatibility_data = load_package_data_raw(VersionSpec, joinpath(path, "compatibility.toml")) + + union!(all_versions_u, versions) + + for (vr,dd) in deps_data + all_deps_u_vr = get_or_make!(all_deps_u, vr) + for (name,other_uuid) in dd + # check conflicts?? + all_deps_u_vr[name] = other_uuid + other_uuid in uuids || push!(uuids, other_uuid) + end + end + for (vr,cd) in compatibility_data + all_compat_u_vr = get_or_make!(all_compat_u, vr) + for (name,vs) in cd + # check conflicts?? + all_compat_u_vr[name] = vs end end end end find_registered!(env, uuids) end - return deps + + for uuid in uuids + try + uuid_to_name[uuid] = registered_name(env, uuid) + end + info = manifest_info(env, uuid) + info ≡ nothing && continue + uuid_to_name[UUID(info["uuid"])] = info["name"] + end + + return Graph(all_versions, all_deps, all_compat, uuid_to_name, reqs, fixed) end "Resolve a set of versions given package version specs" @@ -151,7 +194,7 @@ function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID, info("Resolving package versions") # anything not mentioned is fixed uuids = UUID[pkg.uuid for pkg in pkgs] - uuid_to_name = Dict{UUID,String}() + uuid_to_name = Dict{UUID,String}(uuid_julia => "julia") for (name::String, uuid::UUID) in env.project["deps"] uuid_to_name[uuid] = name uuid in uuids && continue @@ -161,20 +204,17 @@ function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID, push!(pkgs, PackageSpec(name, uuid, ver)) end # construct data structures for resolver and call it - reqs = Requires(pkg.uuid => pkg.version for pkg in pkgs) - deps = deps_graph(env, pkgs) - for dep_uuid in keys(deps) - info = manifest_info(env, UUID(dep_uuid)) - if info != nothing - uuid_to_name[UUID(info["uuid"])] = info["name"] - end - end - deps = Query.prune_dependencies(reqs, deps, uuid_to_name) - vers = Resolve.resolve(reqs, deps, uuid_to_name) + reqs = Requires(pkg.uuid => pkg.version for pkg in pkgs if pkg.uuid ≠ uuid_julia) + fixed = Dict([uuid_julia => Fixed(VERSION)]) + graph = deps_graph(env, uuid_to_name, reqs, fixed) + + simplify_graph!(graph) + vers = resolve(graph) find_registered!(env, collect(keys(vers))) # update vector of package versions for pkg in pkgs - pkg.version = vers[pkg.uuid] + # Fixed packages are not returned by resolve (they already have their version set) + haskey(vers, pkg.uuid) && (pkg.version = vers[pkg.uuid]) end uuids = UUID[pkg.uuid for pkg in pkgs] for (uuid, ver) in vers @@ -559,6 +599,10 @@ function rm(env::EnvCache, pkgs::Vector{PackageSpec}) end function add(env::EnvCache, pkgs::Vector{PackageSpec}) + # if julia is passed as a package the solver gets tricked; + # this catches the error early on + any(pkg->(pkg.uuid == uuid_julia), pkgs) && + error("Trying to add julia as a package") # copy added name/UUIDs into project for pkg in pkgs env.project["deps"][pkg.name] = string(pkg.uuid) diff --git a/src/Pkg3.jl b/src/Pkg3.jl index 5d2bd82228da2..2dfb16e817b67 100644 --- a/src/Pkg3.jl +++ b/src/Pkg3.jl @@ -42,7 +42,7 @@ include("../ext/TOML/src/TOML.jl") include("../ext/TerminalMenus/src/TerminalMenus.jl") include("Types.jl") -include("Query.jl") +include("GraphType.jl") include("Resolve.jl") include("Display.jl") include("Operations.jl") diff --git a/src/Query.jl b/src/Query.jl deleted file mode 100644 index 6fa2d945e1f7c..0000000000000 --- a/src/Query.jl +++ /dev/null @@ -1,485 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -module Query - -using ..Types -import ..Types.uuid_julia -import Pkg3.equalto - -function init_resolve_backtrace(uuid_to_name::Dict{UUID,String}, reqs::Requires, fix::Dict{UUID,Fixed} = Dict{UUID,Fixed}()) - bktrc = ResolveBacktrace(uuid_to_name) - for (p,f) in fix - bktrc[p] = ResolveBacktraceItem(:fixed, f.version) - end - for (p,vs) in reqs - bktrcp = get!(bktrc, p) do; ResolveBacktraceItem() end - push!(bktrcp, :required, vs) - end - return bktrc -end - -function check_fixed(reqs::Requires, fix::Dict{UUID,Fixed}, deps::DepsGraph, uuid_to_name::Dict{UUID,String}) - id(p) = pkgID(p, uuid_to_name) - for (p1,f1) in fix - for p2 in keys(f1.requires) - if !(haskey(deps, p2) || haskey(fix, p2)) - throw(PkgError("unknown package $(id(p1)) required by $(id(p2))")) - end - end - if !satisfies(p1, f1.version, reqs) - warn("$(id(p1)) is fixed at $(f1.version) conflicting with top-level requirement: $(reqs[p1])") - end - for (p2,f2) in fix - if !satisfies(p1, f1.version, f2.requires) - warn("$(id(p1)) is fixed at $(f1.version) conflicting with requirement for $(id(p2)): $(f2.requires[p1])") - end - end - end -end - -function propagate_fixed!(reqs::Requires, bktrc::ResolveBacktrace, fix::Dict{UUID,Fixed}) - for (p,f) in fix - merge_requires!(reqs, f.requires) - for (rp,rvs) in f.requires - bktrc_rp = get!(bktrc, rp) do; ResolveBacktraceItem() end - push!(bktrc_rp, p=>bktrc[p], rvs) - end - end - for (p,f) in fix - delete!(reqs, p) - end - reqs -end - -# Specialized copy for the deps argument below because the deepcopy is slow -function depscopy(deps::DepsGraph) - new_deps = similar(deps) - for (p,depsp) in deps - new_depsp = similar(depsp) - for (vn,vdep) in depsp - new_depsp[vn] = copy(vdep) - end - new_deps[p] = new_depsp - end - return new_deps -end - -# Generate a reverse dependency graph (package names only) -function gen_backdeps(deps::DepsGraph) - backdeps = Dict{UUID,Set{UUID}}() - for (p,depsp) in deps, (vn,vdep) in depsp, rp in keys(vdep) - s = get!(backdeps, rp) do; Set{UUID}() end - push!(s, p) - end - return backdeps -end - -function dependencies(deps::DepsGraph, fix::Dict = Dict{UUID,Fixed}(uuid_julia=>Fixed(VERSION))) - deps = depscopy(deps) - conflicts = Dict{UUID,Set{UUID}}() - to_expunge = VersionNumber[] - emptied = UUID[] - backdeps = gen_backdeps(deps) - - for (fp,fx) in fix - delete!(deps, fp) - haskey(backdeps, fp) || continue - for p in backdeps[fp] - haskey(deps, p) || continue - depsp = deps[p] - empty!(to_expunge) - for (vn,vdep) in depsp - if satisfies(fp, fx.version, vdep) - delete!(vdep, fp) - else - conflicts_p = get!(conflicts, p) do; Set{UUID}() end - push!(conflicts_p, fp) - # don't delete vn from depsp right away so as not to screw up iteration - push!(to_expunge, vn) - end - end - for vn in to_expunge - delete!(depsp, vn) - end - isempty(depsp) && push!(emptied, p) - end - end - while !isempty(emptied) - deleted_pkgs = UUID[] - for p in emptied - delete!(deps, p) - push!(deleted_pkgs, p) - end - empty!(emptied) - - for dp in deleted_pkgs - haskey(backdeps, dp) || continue - for p in backdeps[dp] - haskey(deps, p) || continue - depsp = deps[p] - empty!(to_expunge) - for (vn,vdep) in depsp - haskey(vdep, dp) || continue - conflicts_p = get!(conflicts, p) do; Set{UUID}() end - union!(conflicts_p, conflicts[dp]) - push!(to_expunge, vn) - end - for vn in to_expunge - delete!(depsp, vn) - end - isempty(depsp) && push!(emptied, p) - end - end - end - deps, conflicts -end - -function check_requirements(reqs::Requires, deps::DepsGraph, fix::Dict{UUID,Fixed}, - uuid_to_name::Dict{UUID,String}) - id(p) = pkgID(p, uuid_to_name) - for (p,vs) in reqs - any(vn->(vn ∈ vs), keys(deps[p])) && continue - remaining_vs = VersionSpec() - err_msg = "fixed packages introduce conflicting requirements for $(id(p)): \n" - available_list = sort!(collect(keys(deps[p]))) - for (p1,f1) in fix - f1r = f1.requires - haskey(f1r, p) || continue - err_msg *= " $(id(p1)) requires versions $(f1r[p])" - if !any([vn in f1r[p] for vn in available_list]) - err_msg *= " [none of the available versions can satisfy this requirement]" - end - err_msg *= "\n" - remaining_vs = intersect(remaining_vs, f1r[p]) - end - if isempty(remaining_vs) - err_msg *= " the requirements are unsatisfiable because their intersection is empty" - else - err_msg *= " available versions are $(join(available_list, ", ", " and "))" - end - throw(PkgError(err_msg)) - end -end - -# If there are explicitly required packages, dicards all versions outside -# the allowed range. -# It also propagates requirements: when all allowed versions of a required package -# require some other package, this creates a new implicit requirement. -# The propagation is tracked so that in case a contradiction is detected the error -# message allows to determine the cause. -# This is a pre-pruning step, so it also creates some structures which are later used by pruning -function filter_versions(reqs::Requires, deps::DepsGraph, - bktrc::ResolveBacktrace, uuid_to_name::Dict{UUID,String}) - id(p) = pkgID(p, uuid_to_name) - allowed = Dict{UUID,Dict{VersionNumber,Bool}}() - staged = copy(reqs) - while !isempty(staged) - staged_next = Requires() - for (p,vs) in staged - # Parse requirements and store allowed versions. - depsp = deps[p] - if !haskey(allowed, p) - allowedp = Dict{VersionNumber,Bool}(vn=>true for vn in keys(depsp)) - allowed[p] = allowedp - seen = false - else - allowedp = allowed[p] - oldallowedp = copy(allowedp) - seen = true - end - for vn in keys(depsp) - allowedp[vn] &= vn ∈ vs - end - @assert !isempty(allowedp) - if !any(values(allowedp)) - err_msg = "Unsatisfiable requirements detected for package $(id(p)):\n" - err_msg *= sprint(showitem, bktrc, p) - err_msg *= """The intersection of the requirements is $(bktrc[p].versionreq). - None of the available versions can satisfy this requirement.""" - throw(PkgError(err_msg)) - end - - # If we've seen this package already and nothing has changed since - # the last time, we stop here. - seen && allowedp == oldallowedp && continue - - # Propagate requirements: - # if all allowed versions of a required package require some other package, - # then compute the union of the allowed versions for that other package, and - # treat that as a new requirement. - # Start by filtering out the non-allowed versions - fdepsp = Dict{VersionNumber,Requires}(vn=>depsp[vn] for vn in keys(depsp) if allowedp[vn]) - # Collect all required packages - isreq = Dict{UUID,Bool}(rp=>true for vdep in values(fdepsp) for rp in keys(vdep)) - # Compute whether a required package appears in all requirements - for rp in keys(isreq) - isreq[rp] = all(haskey(vdep, rp) for vdep in values(fdepsp)) - end - - # Create a list of candidates for new implicit requirements - staged_new = Set{UUID}() - for vdep in values(fdepsp), (rp,rvs) in vdep - # Skip packages that may not be required - isreq[rp] || continue - # Compute the union of the version sets - if haskey(staged_next, rp) - snvs = staged_next[rp] - union!(snvs, rvs) - else - snvs = copy(rvs) - staged_next[rp] = snvs - end - push!(staged_new, rp) - end - for rp in staged_new - @assert isreq[rp] - srvs = staged_next[rp] - bktrcp = get!(bktrc, rp) do; ResolveBacktraceItem(); end - push!(bktrcp, p=>bktrc[p], srvs) - if isa(bktrcp.versionreq, VersionSpec) && isempty(bktrcp.versionreq) - err_msg = "Unsatisfiable requirements detected for package $(id(rp)):\n" - err_msg *= sprint(showitem, bktrc, rp) - err_msg *= "The intersection of the requirements is empty." - throw(PkgError(err_msg)) - end - end - end - staged = staged_next - end - - filtered_deps = DepsGraph() - for (p,depsp) in deps - filtered_deps[p] = Dict{VersionNumber,Requires}() - allowedp = get(allowed, p) do; Dict{VersionNumber,Bool}() end - fdepsp = filtered_deps[p] - for (vn,vdep) in depsp - get(allowedp, vn, true) || continue - fdepsp[vn] = vdep - end - end - - return filtered_deps, allowed -end - -# Reduce the number of versions by creating equivalence classes, and retaining -# only the highest version for each equivalence class. -# Two versions are equivalent if: -# 1) They appear together as dependecies of another package (i.e. for each -# dependency relation, they are both required or both not required) -# 2) They have the same dependencies -# Preliminarily calls filter_versions. -function prune_versions(reqs::Requires, deps::DepsGraph, bktrc::ResolveBacktrace, uuid_to_name::Dict{UUID,String}) - filtered_deps, allowed = filter_versions(reqs, deps, bktrc, uuid_to_name) - if !isempty(reqs) - filtered_deps = dependencies_subset(filtered_deps, Set{UUID}(keys(reqs))) - end - - # To each version in each package, we associate a BitVector. - # It is going to hold a pattern such that all versions with - # the same pattern are equivalent. - vmask = Dict{UUID,Dict{VersionNumber,BitVector}}() - - # For each package, we examine the dependencies of its versions - # and put together those which are equal. - # While we're at it, we also collect all dependencies into alldeps - alldeps = Dict{UUID,Set{VersionSpec}}() - for (p,fdepsp) in filtered_deps - # Extract unique dependencies lists (aka classes), thereby - # assigning an index to each class. - uniqdepssets = unique(values(fdepsp)) - - # Store all dependencies seen so far for later use - for r in uniqdepssets, (rp,rvs) in r - get!(alldeps, rp) do; Set{VersionSpec}() end - push!(alldeps[rp], rvs) - end - - # If the package has just one version, it's uninteresting - length(deps[p]) == 1 && continue - - # Grow the pattern by the number of classes - luds = length(uniqdepssets) - @assert !haskey(vmask, p) - vmask[p] = Dict{VersionNumber,BitVector}() - vmaskp = vmask[p] - for vn in keys(fdepsp) - vmaskp[vn] = falses(luds) - end - for (vn,vdep) in fdepsp - vmind = findfirst(equalto(vdep), uniqdepssets) - @assert vmind > 0 - vm = vmaskp[vn] - vm[vmind] = true - end - end - - # Produce dependency patterns. - for (p,vss) in alldeps, vs in vss - # packages with just one version, or dependencies - # which do not distiguish between versions, are not - # interesting - (length(deps[p]) == 1 || vs == VersionSpec()) && continue - - # Store the dependency info in the patterns - @assert haskey(vmask, p) - for (vn,vm) in vmask[p] - push!(vm, vn in vs) - end - end - - # At this point, the vmask patterns are computed. We divide them into - # classes so that we can keep just one version for each class. - pruned_vers = Dict{UUID,Vector{VersionNumber}}() - eq_classes = Dict{UUID,Dict{VersionNumber,Vector{VersionNumber}}}() - for (p,vmaskp) in vmask - vmask0_uniq = unique(values(vmaskp)) - nc = length(vmask0_uniq) - classes = [VersionNumber[] for c0 = 1:nc] - for (vn,vm) in vmaskp - c0 = findfirst(equalto(vm), vmask0_uniq) - push!(classes[c0], vn) - end - map(sort!, classes) - - # For each nonempty class, we store only the highest version) - pruned_vers[p] = VersionNumber[] - prunedp = pruned_vers[p] - eq_classes[p] = Dict{VersionNumber,Vector{VersionNumber}}() - eqclassp = eq_classes[p] - for cl in classes - isempty(cl) && continue - vtop = maximum(cl) - push!(prunedp, vtop) - @assert !haskey(eqclassp, vtop) - eqclassp[vtop] = cl - end - sort!(prunedp) - end - # Put non-allowed versions into eq_classes - for (p,allowedp) in allowed - haskey(eq_classes, p) || continue - eqclassp = eq_classes[p] - for (vn,a) in allowedp - a && continue - eqclassp[vn] = [vn] - end - end - # Put all remaining packages into eq_classes - for (p,depsp) in deps - haskey(eq_classes, p) && continue - eq_classes[p] = Dict{VersionNumber,Vector{VersionNumber}}() - eqclassp = eq_classes[p] - for vn in keys(depsp) - eqclassp[vn] = [vn] - end - end - - - # Recompute deps. We could simplify them, but it's not worth it - new_deps = DepsGraph() - - for (p,depsp) in filtered_deps - @assert !haskey(new_deps, p) - if !haskey(pruned_vers, p) - new_deps[p] = depsp - continue - end - new_deps[p] = Dict{VersionNumber,Requires}() - pruned_versp = pruned_vers[p] - for (vn,vdep) in depsp - vn ∈ pruned_versp || continue - new_deps[p][vn] = vdep - end - end - - #println("pruning stats:") - #numvers = 0 - #numdeps = 0 - #for (p,d) in deps, (vn,vdep) in d - # numvers += 1 - # for r in vdep - # numdeps += 1 - # end - #end - #numnewvers = 0 - #numnewdeps = 0 - #for (p,d) in new_deps, (vn,vdep) in d - # numnewvers += 1 - # for r in vdep - # numnewdeps += 1 - # end - #end - #println(" before: vers=$numvers deps=$numdeps") - #println(" after: vers=$numnewvers deps=$numnewdeps") - #println() - - return new_deps, eq_classes -end -prune_versions(deps::DepsGraph, uuid_to_name::Dict{UUID,String}) = - prune_versions(Requires(), deps, ResolveBacktrace(uuid_to_name), uuid_to_name) - -# Build a graph restricted to a subset of the packages -function subdeps(deps::DepsGraph, pkgs::Set{UUID}) - sub_deps = DepsGraph() - for p in pkgs - haskey(sub_deps, p) || (sub_deps[p] = Dict{VersionNumber,Requires}()) - sub_depsp = sub_deps[p] - for (vn,vdep) in deps[p] - sub_depsp[vn] = vdep - end - end - return sub_deps -end - -# Build a subgraph incuding only the (direct and indirect) dependencies -# of a given package set -function dependencies_subset(deps::DepsGraph, pkgs::Set{UUID}) - staged::Set{UUID} = filter(p->p ∈ keys(deps), pkgs) - allpkgs = copy(staged) - while !isempty(staged) - staged_next = Set{UUID}() - for p in staged, vdep in values(get(deps, p, Dict{VersionNumber,Requires}())), rp in keys(vdep) - rp ∉ allpkgs && rp ≠ uuid_julia && push!(staged_next, rp) - end - union!(allpkgs, staged_next) - staged = staged_next - end - - return subdeps(deps, allpkgs) -end - -# Build a subgraph incuding only the (direct and indirect) dependencies and dependants -# of a given package set -function undirected_dependencies_subset(deps::DepsGraph, pkgs::Set{UUID}) - graph = Dict{UUID,Set{UUID}}() - - for (p,d) in deps - haskey(graph, p) || (graph[p] = Set{UUID}()) - for vdep in values(d), rp in keys(vdep) - push!(graph[p], rp) - haskey(graph, rp) || (graph[rp] = Set{UUID}()) - push!(graph[rp], p) - end - end - - staged = pkgs - allpkgs = copy(pkgs) - while !isempty(staged) - staged_next = Set{UUID}() - for p in staged, rp in graph[p] - rp ∉ allpkgs && push!(staged_next, rp) - end - union!(allpkgs, staged_next) - staged = staged_next - end - - return subdeps(deps, allpkgs) -end - -function prune_dependencies(reqs::Requires, deps::DepsGraph, uuid_to_name::Dict{UUID,String}, - bktrc::ResolveBacktrace = init_resolve_backtrace(uuid_to_name, reqs)) - deps, _ = prune_versions(reqs, deps, bktrc, uuid_to_name) - return deps -end - -end # module diff --git a/src/Resolve.jl b/src/Resolve.jl index d87d2c8abad07..ad856db077bf5 100644 --- a/src/Resolve.jl +++ b/src/Resolve.jl @@ -3,165 +3,358 @@ module Resolve include(joinpath("resolve", "VersionWeights.jl")) -include(joinpath("resolve", "PkgToMaxSumInterface.jl")) include(joinpath("resolve", "MaxSum.jl")) using ..Types -using ..Query, .PkgToMaxSumInterface, .MaxSum +using ..GraphType +using .MaxSum import ..Types: uuid_julia +import ..GraphType: is_current_julia export resolve, sanity_check -# Use the max-sum algorithm to resolve packages dependencies -function resolve(reqs::Requires, deps::DepsGraph, uuid_to_name::Dict{UUID,String}) - id(p) = pkgID(p, uuid_to_name) - - # init interface structures - interface = Interface(reqs, deps) +"Resolve package dependencies." +function resolve(graph::Graph; verbose::Bool = false) + id(p) = pkgID(p, graph) # attempt trivial solution first - ok, sol = greedysolver(interface) - if !ok - # trivial solution failed, use maxsum solver - graph = Graph(interface) - msgs = Messages(interface, graph) + ok, sol = greedysolver(graph) - try - sol = maxsum(graph, msgs) - catch err - isa(err, UnsatError) || rethrow(err) - p = interface.pkgs[err.info] - # TODO: build tools to analyze the problem, and suggest to use them here. - msg = - """ - resolve is unable to satisfy package requirements. - The problem was detected when trying to find a feasible version - for package $(id(p)). - However, this only means that package $(id(p)) is involved in an - unsatisfiable or difficult dependency relation, and the root of - the problem may be elsewhere. - """ - if msgs.num_nondecimated != graph.np - msg *= """ - (you may try increasing the value of the JULIA_PKGRESOLVE_ACCURACY - environment variable) - """ - end - ## info("ERROR MESSAGE:\n" * msg) - throw(PkgError(msg)) - end + ok && @goto solved - # verify solution (debug code) and enforce its optimality - @assert verify_solution(sol, interface) - enforce_optimality!(sol, interface) - @assert verify_solution(sol, interface) + verbose && info("resolve: greedy failed") + + # trivial solution failed, use maxsum solver + msgs = Messages(graph) + + try + sol = maxsum(graph, msgs) + catch err + isa(err, UnsatError) || rethrow(err) + verbose && info("resolve: maxsum failed") + p = graph.data.pkgs[err.info] + # TODO: build tools to analyze the problem, and suggest to use them here. + msg = + """ + resolve is unable to satisfy package requirements. + The problem was detected when trying to find a feasible version + for package $(id(p)). + However, this only means that package $(id(p)) is involved in an + unsatisfiable or difficult dependency relation, and the root of + the problem may be elsewhere. + """ + if msgs.num_nondecimated != graph.np + msg *= """ + (you may try increasing the value of the JULIA_PKGRESOLVE_ACCURACY + environment variable) + """ + end + ## info("ERROR MESSAGE:\n" * msg) + throw(PkgError(msg)) end - # return the solution as a Dict mapping package_name => sha1 - return compute_output_dict(sol, interface) -end + # verify solution (debug code) and enforce its optimality + @assert verify_solution(sol, graph) + enforce_optimality!(sol, graph) -# Scan dependencies for (explicit or implicit) contradictions -function sanity_check(deps::DepsGraph, uuid_to_name::Dict{UUID,String}, - pkgs::Set{UUID} = Set{UUID}()) - id(p) = pkgID(p, uuid_to_name) + @label solved - isempty(pkgs) || (deps = Query.undirected_dependencies_subset(deps, pkgs)) + verbose && info("resolve: succeeded") - deps, eq_classes = Query.prune_versions(deps, uuid_to_name) + # return the solution as a Dict mapping UUID => VersionNumber + return compute_output_dict(sol, graph) +end - ndeps = Dict{UUID,Dict{VersionNumber,Int}}() +""" +Scan the graph for (explicit or implicit) contradictions. Returns a list of problematic +(package,version) combinations. +""" +function sanity_check(graph::Graph, sources::Set{UUID} = Set{UUID}(); verbose::Bool = false) + req_inds = graph.req_inds + fix_inds = graph.fix_inds - for (p,depsp) in deps - ndeps[p] = ndepsp = Dict{VersionNumber,Int}() - for (vn,vdep) in depsp - ndepsp[vn] = length(vdep) - end + id(p) = pkgID(p, graph) + + isempty(req_inds) || warn("sanity check called on a graph with non-empty requirements") + if !any(is_current_julia(graph, fp0) for fp0 in fix_inds) + warn("sanity check called on a graph without current julia requirement, adding it") + add_fixed!(graph, Dict(uuid_julia=>Fixed(VERSION))) + end + if length(fix_inds) ≠ 1 + warn("sanity check called on a graph with extra fixed requirements (besides julia)") end - vers = [(p,vn) for (p,depsp) in deps for vn in keys(depsp)] - sort!(vers, by=pvn->(-ndeps[pvn[1]][pvn[2]])) + isources = isempty(sources) ? + Set{Int}(1:graph.np) : + Set{Int}(graph.data.pdict[p] for p in sources) + + simplify_graph!(graph, isources, verbose = verbose) + + np = graph.np + spp = graph.spp + gadj = graph.gadj + data = graph.data + pkgs = data.pkgs + pdict = data.pdict + pvers = data.pvers + eq_classes = data.eq_classes + + problematic = Tuple{String,VersionNumber}[] + + np == 0 && return problematic + + vers = [(pkgs[p0],pvers[p0][v0]) for p0 = 1:np for v0 = 1:(spp[p0]-1)] + sort!(vers, by=pv->(-length(gadj[pdict[pv[1]]]))) nv = length(vers) - svdict = Dict{Tuple{UUID,VersionNumber},Int}(vers[i][1:2]=>i for i = 1:nv) + svdict = Dict{Tuple{UUID,VersionNumber},Int}(vers[i] => i for i = 1:nv) checked = falses(nv) - problematic = Tuple{String,VersionNumber,String}[] i = 1 for (p,vn) in vers - ndeps[p][vn] == 0 && break + length(gadj[pdict[p]]) == 0 && break checked[i] && (i += 1; continue) - fixed = Dict{UUID,Fixed}(p=>Fixed(vn, deps[p][vn]), uuid_julia=>Fixed(VERSION)) - sub_reqs = Requires() - bktrc = Query.init_resolve_backtrace(uuid_to_name, sub_reqs, fixed) - Query.propagate_fixed!(sub_reqs, bktrc, fixed) - sub_deps = Query.dependencies_subset(deps, Set{UUID}([p])) - sub_deps, conflicts = Query.dependencies(sub_deps, fixed) + sub_graph = deepcopy(graph) + req = Requires(p => vn) + add_reqs!(sub_graph, req) try - for rp in keys(sub_reqs) - haskey(sub_deps, rp) && continue - if uuid_julia in conflicts[rp] - throw(PkgError("$(id(rp)) can't be installed because it has no versions that support $VERSION " * - "of julia. You may need to update METADATA by running `Pkg.update()`")) - else - sconflicts = join(map(id, conflicts[rp]), ", ", " and ") - throw(PkgError("$(id(rp)) requirements can't be satisfied because " * - "of the following fixed packages: $sconflicts")) - end - end - Query.check_requirements(sub_reqs, sub_deps, fixed, uuid_to_name) - sub_deps = Query.prune_dependencies(sub_reqs, sub_deps, uuid_to_name, bktrc) + simplify_graph!(sub_graph, verbose = verbose) catch err isa(err, PkgError) || rethrow(err) ## info("ERROR MESSAGE:\n" * err.msg) for vneq in eq_classes[p][vn] - push!(problematic, (id(p), vneq, "")) + push!(problematic, (id(p), vneq)) + end + i += 1 + continue + end + + ok, sol = greedysolver(sub_graph) + + ok && @goto solved + + msgs = Messages(sub_graph) + + try + sol = maxsum(sub_graph, msgs) + @assert verify_solution(sol, sub_graph) + catch err + isa(err, UnsatError) || rethrow(err) + for vneq in eq_classes[p][vn] + push!(problematic, (id(p), vneq)) end i += 1 continue end - interface = Interface(sub_reqs, sub_deps) - - red_pkgs = interface.pkgs - red_np = interface.np - red_spp = interface.spp - red_pvers = interface.pvers - - ok, sol = greedysolver(interface) - - if !ok - try - graph = Graph(interface) - msgs = Messages(interface, graph) - sol = maxsum(graph, msgs) - ok = verify_solution(sol, interface) - @assert ok - catch err - isa(err, UnsatError) || rethrow(err) - pp = red_pkgs[err.info] - for vneq in eq_classes[p][vn] - push!(problematic, (p, vneq, pp)) + + @label solved + + sol_dict = compute_output_dict(sol, sub_graph) + for (sp, svn) in sol_dict + j = svdict[sp,svn] + checked[j] = true + end + + i += 1 + end + + return sort!(problematic) +end + +""" +Translate the solver output (a Vector{Int} of package states) into a Dict which +associates a VersionNumber to each installed package UUID. +""" +function compute_output_dict(sol::Vector{Int}, graph::Graph) + np = graph.np + spp = graph.spp + fix_inds = graph.fix_inds + pkgs = graph.data.pkgs + pvers = graph.data.pvers + pruned = graph.data.pruned + + want = Dict{UUID,VersionNumber}() + for p0 = 1:np + p0 ∈ fix_inds && continue + p = pkgs[p0] + s0 = sol[p0] + s0 == spp[p0] && continue + vn = pvers[p0][s0] + want[p] = vn + end + for (p,vn) in pruned + @assert !haskey(want, p) + want[p] = vn + end + + return want +end + +""" +Preliminary solver attempt: tries to maximize each version; bails out as soon as +some non-trivial requirement is detected. +""" +function greedysolver(graph::Graph) + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gconstr = graph.gconstr + np = graph.np + + # initialize solution: all uninstalled + sol = [spp[p0] for p0 = 1:np] + + # packages which are not allowed to be uninstalled + # (NOTE: this is potentially a superset of graph.req_inds, + # since it may include implicit requirements) + req_inds = Set{Int}(p0 for p0 = 1:np if !gconstr[p0][end]) + + # set up required packages to their highest allowed versions + for rp0 in req_inds + # look for the highest version which satisfies the requirements + rv0 = findlast(gconstr[rp0]) + @assert rv0 ≠ 0 && rv0 ≠ spp[rp0] + sol[rp0] = rv0 + end + + # we start from required packages and explore the graph + # following dependencies + staged = req_inds + seen = copy(staged) + + while !isempty(staged) + staged_next = Set{Int}() + for p0 in staged + s0 = sol[p0] + @assert s0 < spp[p0] + + # scan dependencies + for (j1,p1) in enumerate(gadj[p0]) + msk = gmsk[p0][j1] + # look for the highest version which satisfies the requirements + v1 = findlast(msk[:,s0] .& gconstr[p1]) + v1 == spp[p1] && continue # p1 is not required by p0's current version + # if we found a version, and the package was uninstalled + # or the same version was already selected, we're ok; + # otherwise we can't be sure what the optimal configuration is + # and we bail out + if v1 > 0 && (sol[p1] == spp[p1] || sol[p1] == v1) + sol[p1] = v1 + else + return (false, Int[]) end + + p1 ∈ seen || push!(staged_next, p1) end end - if ok - for p0 = 1:red_np - s0 = sol[p0] - if s0 != red_spp[p0] - j = svdict[(red_pkgs[p0], red_pvers[p0][s0])] - checked[j] = true + union!(seen, staged_next) + staged = staged_next + end + + @assert verify_solution(sol, graph) + + return true, sol +end + +""" +Verifies that the solver solution fulfills all hard constraints +(requirements and dependencies). This is intended as debug code. +""" +function verify_solution(sol::Vector{Int}, graph::Graph) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gconstr = graph.gconstr + + # verify constraints and dependencies + for p0 = 1:np + s0 = sol[p0] + gconstr[p0][s0] || return false + for (j1,p1) in enumerate(gadj[p0]) + msk = gmsk[p0][j1] + s1 = sol[p1] + msk[s1,s0] || return false # TODO: print debug info + end + end + return true +end + +""" +Push the given solution to a local optimium if needed: keeps increasing +the states of the given solution as long as no constraints are violated. +It also removes unnecessary parts of the solution which are unconnected +to the required packages. +""" +function enforce_optimality!(sol::Vector{Int}, graph::Graph) + np = graph.np + spp = graph.spp + gadj = graph.gadj + gmsk = graph.gmsk + gdir = graph.gdir + gconstr = graph.gconstr + + restart = true + while restart + restart = false + for p0 = 1:np + s0 = sol[p0] + s0 == spp[p0] && continue # the package is not installed + + # check if bumping to the higher version would violate a constraint + gconstr[p0][s0+1] || continue + + # check if bumping to the higher version would violate a constraint + viol = false + for (j1,p1) in enumerate(gadj[p0]) + s1 = sol[p1] + msk = gmsk[p0][j1] + if !msk[s1, s0+1] + viol = true + break end end - checked[i] = true + viol && continue + + # So the solution is non-optimal: we bump it manually + sol[p0] += 1 + restart = true end - i += 1 end - return sort!(problematic) + # Finally uninstall unneeded packages: + # start from the required ones and keep only + # the packages reachable from them along the graph. + # (These should have been removed in the previous step, but in principle + # an unconnected yet self-sustaining cycle may have survived.) + uninst = trues(np) + staged = Set{Int}(p0 for p0 = 1:np if !gconstr[p0][end]) + seen = copy(staged) + + while !isempty(staged) + staged_next = Set{Int}() + for p0 in staged + s0 = sol[p0] + @assert s0 < spp[p0] + uninst[p0] = false + for (j1,p1) in enumerate(gadj[p0]) + gmsk[p0][j1][end,s0] && continue # the package is not required by p0 at version s0 + p1 ∈ seen || push!(staged_next, p1) + end + end + union!(seen, staged_next) + staged = staged_next + end + + for p0 in find(uninst) + sol[p0] = spp[p0] + end + + @assert verify_solution(sol, graph) end end # module diff --git a/src/Types.jl b/src/Types.jl index 1728fee4b606a..c34be70e85b8d 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -8,8 +8,7 @@ import Pkg3 import Pkg3: depots, logdir, iswindows export UUID, pkgID, SHA1, VersionRange, VersionSpec, empty_versionspec, - Requires, Fixed, DepsGraph, merge_requires!, satisfies, - PkgError, ResolveBacktraceItem, ResolveBacktrace, showitem, + Requires, Fixed, merge_requires!, satisfies, PkgError, PackageSpec, UpgradeLevel, EnvCache, CommandError, cmderror, has_name, has_uuid, write_env, parse_toml, find_registered!, project_resolve!, manifest_resolve!, registry_resolve!, ensure_resolved, @@ -40,7 +39,7 @@ const uuid_registry = uuid5(uuid_julia, "registry") ## user-friendly representation of package IDs ## function pkgID(p::UUID, uuid_to_name::Dict{UUID,String}) - name = haskey(uuid_to_name, p) ? uuid_to_name[p] : "UNKNOWN" + name = get(uuid_to_name, p, "UNKNOWN") uuid_short = string(p)[1:8] return "$name [$uuid_short]" end @@ -289,8 +288,6 @@ end satisfies(pkg::UUID, ver::VersionNumber, reqs::Requires) = !haskey(reqs, pkg) || in(ver, reqs[pkg]) -const DepsGraph = Dict{UUID,Dict{VersionNumber,Requires}} - struct Fixed version::VersionNumber requires::Requires @@ -327,105 +324,6 @@ function Base.showerror(io::IO, pkgerr::PkgError) end end - -const VersionReq = Union{VersionNumber,VersionSpec} -const WhyReq = Tuple{VersionReq,Any} - -# This is used to keep track of dependency relations when propagating -# requirements, so as to emit useful information in case of unsatisfiable -# conditions. -# The `versionreq` field keeps track of the remaining allowed versions, -# intersecting all requirements. -# The `why` field is a Vector which keeps track of the requirements. Each -# entry is a Tuple of two elements: -# 1) the first element is the version requirement (can be a single VersionNumber -# or a VersionSpec). -# 2) the second element can be either :fixed (for requirements induced by -# fixed packages), :required (for requirements induced by explicitly -# required packages), or a Pair p=>backtrace_item (for requirements induced -# indirectly, where `p` is the package name and `backtrace_item` is -# another ResolveBacktraceItem. -mutable struct ResolveBacktraceItem - versionreq::VersionReq - why::Vector{WhyReq} - ResolveBacktraceItem() = new(VersionSpec(), WhyReq[]) - ResolveBacktraceItem(reason, versionreq::VersionReq) = new(versionreq, WhyReq[(versionreq,reason)]) -end - -function Base.push!(ritem::ResolveBacktraceItem, reason, versionspec::VersionSpec) - if ritem.versionreq isa VersionSpec - ritem.versionreq = ritem.versionreq ∩ versionspec - elseif ritem.versionreq ∉ versionspec - ritem.versionreq = copy(empty_versionspec) - end - push!(ritem.why, (versionspec,reason)) -end - -function Base.push!(ritem::ResolveBacktraceItem, reason, version::VersionNumber) - if ritem.versionreq isa VersionSpec - if version ∈ ritem.versionreq - ritem.versionreq = version - else - ritem.versionreq = copy(empty_versionspec) - end - elseif ritem.versionreq ≠ version - ritem.versionreq = copy(empty_versionspec) - end - push!(ritem.why, (version,reason)) -end - - -struct ResolveBacktrace - uuid_to_name::Dict{UUID,String} - bktrc::Dict{UUID,ResolveBacktraceItem} - ResolveBacktrace(uuid_to_name::Dict{UUID,String}) = new(uuid_to_name, Dict{UUID,ResolveBacktraceItem}()) -end - -Base.getindex(bt::ResolveBacktrace, p) = bt.bktrc[p] -Base.setindex!(bt::ResolveBacktrace, v, p) = setindex!(bt.bktrc, v, p) -Base.haskey(bt::ResolveBacktrace, p) = haskey(bt.bktrc, p) -Base.get!(bt::ResolveBacktrace, p, def) = get!(bt.bktrc, p, def) -Base.get!(def::Base.Callable, bt::ResolveBacktrace, p) = get!(def, bt.bktrc, p) - -showitem(io::IO, bt::ResolveBacktrace, p) = _show(io, bt.uuid_to_name, bt[p], "", Set{ResolveBacktraceItem}([bt[p]])) - -function _show(io::IO, uuid_to_name::Dict{UUID,String}, ritem::ResolveBacktraceItem, indent::String, seen::Set{ResolveBacktraceItem}) - l = length(ritem.why) - for (i,(vs,w)) in enumerate(ritem.why) - print(io, indent, (i==l ? '└' : '├'), '─') - if w ≡ :fixed - @assert vs isa VersionNumber - println(io, "version $vs set by fixed requirement (package is checked out, dirty or pinned)") - elseif w ≡ :required - @assert vs isa VersionSpec - println(io, "version range $vs set by an explicit requirement") - else - @assert w isa Pair{UUID,ResolveBacktraceItem} - if vs isa VersionNumber - print(io, "version $vs ") - else - print(io, "version range $vs ") - end - id = pkgID(w[1], uuid_to_name) - otheritem = w[2] - print(io, "required by package $id, ") - if otheritem.versionreq isa VersionSpec - println(io, "whose allowed version range is $(otheritem.versionreq):") - else - println(io, "whose only allowed version is $(otheritem.versionreq):") - end - if otheritem ∈ seen - println(io, (i==l ? " " : "│ ") * indent, "└─[see above for $id backtrace]") - continue - end - push!(seen, otheritem) - _show(io, uuid_to_name, otheritem, (i==l ? " " : "│ ") * indent, seen) - end - end -end - - - ## command errors (no stacktrace) ## struct CommandError <: Exception @@ -984,12 +882,12 @@ function find_registered!( for line in eachline(io) ismatch(r"^ \s* \[ \s* packages \s* \] \s* $"x, line) && break end - # fine lines with uuid or name we're looking for + # find lines with uuid or name we're looking for for line in eachline(io) ismatch(regex, line) || continue m = match(line_re, line) m == nothing && - error("misformated registry.toml package entry: $line") + error("misformatted registry.toml package entry: $line") uuid = UUID(m.captures[1]) name = Base.unescape_string(m.captures[2]) path = abspath(registry, Base.unescape_string(m.captures[3])) diff --git a/src/resolve/MaxSum.jl b/src/resolve/MaxSum.jl index b4c6bc89e7513..584d44b1cc8f4 100644 --- a/src/resolve/MaxSum.jl +++ b/src/resolve/MaxSum.jl @@ -4,9 +4,9 @@ module MaxSum include("FieldValues.jl") -using .FieldValues, ..VersionWeights, ..PkgToMaxSumInterface +using .FieldValues, ..VersionWeights, ...Types, ...GraphType -export UnsatError, Graph, Messages, maxsum +export UnsatError, Messages, maxsum # An exception type used internally to signal that an unsatisfiable # constraint was detected @@ -34,130 +34,6 @@ mutable struct MaxSumParams end end -# aux function to make graph generation consistent across platforms -sortedpairs(d::Dict) = (k=>d[k] for k in sort!(collect(keys(d)))) - -# Graph holds the graph structure onto which max-sum is run, in -# sparse format -mutable struct Graph - # adjacency matrix: - # for each package, has the list of neighbors - # indices (both dependencies and dependants) - gadj::Vector{Vector{Int}} - - # compatibility mask: - # for each package p0 has a list of bool masks. - # Each entry in the list gmsk[p0] is relative to the - # package p1 as read from gadj[p0]. - # Each mask has dimension spp1 x spp0, where - # spp0 is the number of states of p0, and - # spp1 is the number of states of p1. - gmsk::Vector{Vector{BitMatrix}} - - # dependency direction: - # keeps track of which direction the dependency goes - # takes 3 values: - # 1 = dependant - # -1 = dependency - # 0 = both - # Used to break symmetry between dependants and - # dependencies (introduces a FieldValue at level l3). - # The "both" case is for when there are dependency - # relations which go both ways - gdir::Vector{Vector{Int}} - - # adjacency dict: - # allows one to retrieve the indices in gadj, so that - # gadj[p0][adjdict[p1][p0]] = p1 - # ("At which index does package p1 appear in gadj[p0]?") - adjdict::Vector{Dict{Int,Int}} - - # states per package: same as in Interface - spp::Vector{Int} - - # update order: shuffled at each iteration - perm::Vector{Int} - - # number of packages (all Vectors above have this length) - np::Int - - function Graph(interface::Interface) - deps = interface.deps - np = interface.np - - spp = interface.spp - pdict = interface.pdict - pvers = interface.pvers - vdict = interface.vdict - - gadj = [Int[] for i = 1:np] - gmsk = [BitMatrix[] for i = 1:np] - gdir = [Int[] for i = 1:np] - adjdict = [Dict{Int,Int}() for i = 1:np] - - for (p,depsp) in sortedpairs(deps) - p0 = pdict[p] - vdict0 = vdict[p0] - for (vn,vdep) in sortedpairs(depsp) - v0 = vdict0[vn] - for (rp, rvs) in sortedpairs(vdep) - p1 = pdict[rp] - - j0 = 1 - while j0 <= length(gadj[p0]) && gadj[p0][j0] != p1 - j0 += 1 - end - j1 = 1 - while j1 <= length(gadj[p1]) && gadj[p1][j1] != p0 - j1 += 1 - end - @assert (j0 > length(gadj[p0]) && j1 > length(gadj[p1])) || - (j0 <= length(gadj[p0]) && j1 <= length(gadj[p1])) - - if j0 > length(gadj[p0]) - push!(gadj[p0], p1) - push!(gadj[p1], p0) - j0 = length(gadj[p0]) - j1 = length(gadj[p1]) - - adjdict[p1][p0] = j0 - adjdict[p0][p1] = j1 - - bm = trues(spp[p1], spp[p0]) - bmt = bm' - - push!(gmsk[p0], bm) - push!(gmsk[p1], bmt) - - push!(gdir[p0], 1) - push!(gdir[p1], -1) - else - bm = gmsk[p0][j0] - bmt = gmsk[p1][j1] - if gdir[p0][j0] == -1 - gdir[p0][j0] = 0 - gdir[p1][j1] = 0 - end - end - - for v1 = 1:length(pvers[p1]) - if pvers[p1][v1] ∉ rvs - bm[v1, v0] = false - bmt[v0, v1] = false - end - end - bm[end,v0] = false - bmt[v0,end] = false - end - end - end - - perm = [1:np;] - - return new(gadj, gmsk, gdir, adjdict, spp, perm, np) - end -end - # Messages has the cavity messages and the total fields, and # gets updated iteratively (and occasionally decimated) until # convergence @@ -180,51 +56,52 @@ mutable struct Messages decimated::BitVector num_nondecimated::Int - function Messages(interface::Interface, graph::Graph) - reqs = interface.reqs - pkgs = interface.pkgs - np = interface.np - spp = interface.spp - pvers = interface.pvers - pdict = interface.pdict - vweight = interface.vweight + function Messages(graph::Graph) + np = graph.np + spp = graph.spp + gconstr = graph.gconstr + req_inds = graph.req_inds + pvers = graph.data.pvers + pdict = graph.data.pdict + + ## generate wveights (v0 == spp[p0] is the "uninstalled" state) + vweight = [[VersionWeight(v0 < spp[p0] ? pvers[p0][v0] : v"0") for v0 = 1:spp[p0]] for p0 = 1:np] # external fields: favor newest versions over older, and no-version over all fld = [[FieldValue(0, zero(VersionWeight), vweight[p0][v0], (v0==spp[p0]), 0) for v0 = 1:spp[p0]] for p0 = 1:np] - # enforce requirements - for (rp, rvs) in reqs - p0 = pdict[rp] - pvers0 = pvers[p0] + # enforce constraints + for p0 = 1:np fld0 = fld[p0] - for v0 = 1:spp[p0]-1 - vn = pvers0[v0] - if !in(vn, rvs) - # the state is forbidden by requirements - fld0[v0] = FieldValue(-1) - else - # the state is one of those explicitly requested: - # favor it at a higer level than normal (upgrade - # FieldValue from l2 to l1) - fld0[v0] += FieldValue(0, vweight[p0][v0], -vweight[p0][v0]) - end + gconstr0 = gconstr[p0] + for v0 = 1:spp[p0] + gconstr0[v0] || (fld0[v0] = FieldValue(-1)) end - # the uninstalled state is forbidden by requirements - fld0[spp[p0]] = FieldValue(-1) end + + # favor explicit requirements + for rp0 in req_inds + fld0 = fld[rp0] + gconstr0 = gconstr[rp0] + for v0 = 1:spp[rp0]-1 + gconstr0[v0] || continue + # the state is one of those explicitly requested: + # favor it at a higer level than normal (upgrade + # FieldValue from l2 to l1) + fld0[v0] += FieldValue(0, vweight[rp0][v0], -vweight[rp0][v0]) + end + end + # normalize fields for p0 = 1:np - m = maximum(fld[p0]) - for v0 = 1:spp[p0] - fld[p0][v0] -= m - end + fld[p0] .-= maximum(fld[p0]) end - initial_fld = deepcopy(fld) + initial_fld = [copy(f0) for f0 in fld] # initialize cavity messages to 0 gadj = graph.gadj - msg = [[zeros(FieldValue, spp[p0]) for p1 = 1:length(gadj[p0])] for p0 = 1:np] + msg = [[zeros(FieldValue, spp[p0]) for j1 = 1:length(gadj[p0])] for p0 = 1:np] return new(msg, fld, initial_fld, falses(np), np) end @@ -285,7 +162,7 @@ function update(p0::Int, graph::Graph, msgs::Messages) # compute the output cavity message p0->p1 cavmsg = fld0 - msg0[j0] - if dir1 == -1 + if dir1 == GraphType.BACKWARDS # p0 depends on p1 for v0 = 1:spp0-1 cavmsg[v0] += FieldValue(0, VersionWeight(0), VersionWeight(0), 0, v0) @@ -302,7 +179,7 @@ function update(p0::Int, graph::Graph, msgs::Messages) # through the constraint encoded in the bitmask # (nearly equivalent to: # newmsg = [maximum(cavmsg[bm1[:,v1]]) for v1 = 1:spp1] - # except for the gnrg term) + # except for the directional term) m = FieldValue(-1) for v1 = 1:spp1 for v0 = 1:spp0 @@ -310,7 +187,7 @@ function update(p0::Int, graph::Graph, msgs::Messages) newmsg[v1] = max(newmsg[v1], cavmsg[v0]) end end - if dir1 == 1 && v1 != spp1 + if dir1 == GraphType.FORWARD && v1 != spp1 # p1 depends on p0 newmsg[v1] += FieldValue(0, VersionWeight(0), VersionWeight(0), 0, v1) end @@ -343,30 +220,32 @@ function update(p0::Int, graph::Graph, msgs::Messages) end # A simple shuffling machinery for the update order in iterate() -# (woulnd't pass any random quality test but it's arguably enough) -let step = 1 -global shuffleperm, shuffleperminit -shuffleperminit() = (step = 1) -function shuffleperm(graph::Graph) - perm = graph.perm - np = graph.np - for j = np:-1:2 - k = mod(step,j) + 1 - perm[j], perm[k] = perm[k], perm[j] - step += isodd(j) ? 1 : k - end - #@assert isperm(perm) +# (wouldn't pass any random quality test but it's arguably enough) +mutable struct NodePerm + p::Vector{Int} + step::Int64 + NodePerm(np::Integer) = new(collect(1:np), 1) end + +function Base.shuffle!(perm::NodePerm) + p = perm.p + for j = length(p):-1:2 + k = perm.step % j + 1 + p[j], p[k] = p[k], p[j] + perm.step += isodd(j) ? 1 : k + end + #@assert isperm(p) end +Base.start(perm::NodePerm) = start(perm.p) +Base.next(perm::NodePerm, x) = next(perm.p, x) +Base.done(perm::NodePerm, x) = done(perm.p, x) + # Call update for all nodes (i.e. packages) in # random order -function iterate(graph::Graph, msgs::Messages) - np = graph.np - +function iterate(graph::Graph, msgs::Messages, perm::NodePerm) maxdiff = zero(FieldValue) - shuffleperm(graph) - perm = graph.perm + shuffle!(perm) for p0 in perm maxdiff0 = update(p0, graph, msgs) maxdiff = max(maxdiff, maxdiff0) @@ -487,10 +366,10 @@ function maxsum(graph::Graph, msgs::Messages) params = MaxSumParams() it = 0 - shuffleperminit() + perm = NodePerm(graph.np) while true it += 1 - maxdiff = iterate(graph, msgs) + maxdiff = iterate(graph, msgs, perm) #println("it = $it maxdiff = $maxdiff") if maxdiff == zero(FieldValue) diff --git a/src/resolve/PkgToMaxSumInterface.jl b/src/resolve/PkgToMaxSumInterface.jl deleted file mode 100644 index 627caf1071f65..0000000000000 --- a/src/resolve/PkgToMaxSumInterface.jl +++ /dev/null @@ -1,365 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -module PkgToMaxSumInterface - -using ...Types -using ...Query, ..VersionWeights - -export Interface, compute_output_dict, greedysolver, - verify_solution, enforce_optimality! - -# A collection of objects which allow interfacing external (Pkg) and -# internal (MaxSum) representation -mutable struct Interface - # requirements and dependencies, in external representation - reqs::Requires - deps::DepsGraph - - # packages list - pkgs::Vector{UUID} - - # number of packages - np::Int - - # states per package: one per version + uninstalled - spp::Vector{Int} - - # pakage dict: associates an index to each package id - pdict::Dict{UUID,Int} - - # package versions: for each package, keep the list of the - # possible version numbers; this defines a - # mapping from version numbers of a package - # to indices - pvers::Vector{Vector{VersionNumber}} - - # versions dict: associates a version index to each package - # version; such that - # pvers[p0][vdict[p0][vn]] = vn - vdict::Vector{Dict{VersionNumber,Int}} - - # version weights: the weight for each version of each package - # (versions include the uninstalled state; the - # higher the weight, the more favored the version) - vweight::Vector{Vector{VersionWeight}} - - function Interface(reqs::Requires, deps::DepsGraph) - # generate pkgs - pkgs = sort!(collect(keys(deps))) - - np = length(pkgs) - - # generate pdict - pdict = Dict{UUID,Int}(pkgs[i] => i for i = 1:np) - - # generate spp and pvers - spp = Vector{Int}(np) - - pvers = [VersionNumber[] for i = 1:np] - - for (p,depsp) in deps, vn in keys(depsp) - p0 = pdict[p] - push!(pvers[p0], vn) - end - for p0 = 1:np - sort!(pvers[p0]) - spp[p0] = length(pvers[p0]) + 1 - end - - # generate vdict - vdict = [Dict{VersionNumber,Int}() for p0 = 1:np] - for (p,depsp) in deps - p0 = pdict[p] - vdict0 = vdict[p0] - pvers0 = pvers[p0] - for vn in keys(depsp) - for v0 in 1:length(pvers0) - if pvers0[v0] == vn - vdict0[vn] = v0 - break - end - end - end - end - - ## generate wveights: - vweight = Vector{Vector{VersionWeight}}(np) - for p0 = 1:np - pvers0 = pvers[p0] - spp0 = spp[p0] - vweight0 = vweight[p0] = Vector{VersionWeight}(spp0) - for v0 = 1:spp0-1 - vweight0[v0] = VersionWeight(pvers0[v0]) - end - vweight0[spp0] = VersionWeight(v"0") # last version means uninstalled - end - - return new(reqs, deps, pkgs, np, spp, pdict, pvers, vdict, vweight) - end -end - -# The output format is a Dict which associates a VersionNumber to each installed package name -function compute_output_dict(sol::Vector{Int}, interface::Interface) - pkgs = interface.pkgs - np = interface.np - pvers = interface.pvers - spp = interface.spp - - want = Dict{UUID,VersionNumber}() - for p0 = 1:np - p = pkgs[p0] - s = sol[p0] - if s != spp[p0] - v = pvers[p0][s] - want[p] = v - end - end - - return want -end - -# Produce a trivial solution: try to maximize each version; -# bail out as soon as some non-trivial requirements are detected. -function greedysolver(interface::Interface) - reqs = interface.reqs - deps = interface.deps - spp = interface.spp - pdict = interface.pdict - pvers = interface.pvers - np = interface.np - - # initialize solution: all uninstalled - sol = [spp[p0] for p0 = 1:np] - - # set up required packages to their highest allowed versions - for (rp,rvs) in reqs - rp0 = pdict[rp] - # look for the highest version which satisfies the requirements - rv = spp[rp0] - 1 - while rv > 0 - rvn = pvers[rp0][rv] - rvn ∈ rvs && break - rv -= 1 - end - @assert rv > 0 - sol[rp0] = rv - end - - # we start from required packages and explore the graph - # following dependencies - staged = Set{UUID}(keys(reqs)) - seen = copy(staged) - - while !isempty(staged) - staged_next = Set{UUID}() - for p in staged - p0 = pdict[p] - @assert sol[p0] < spp[p0] - vn = pvers[p0][sol[p0]] - vdep = deps[p][vn] - - # scan dependencies - for (rp,rvs) in vdep - rp0 = pdict[rp] - # look for the highest version which satisfies the requirements - rv = spp[rp0] - 1 - while rv > 0 - rvn = pvers[rp0][rv] - rvn ∈ rvs && break - rv -= 1 - end - # if we found a version, and the package was uninstalled - # or the same version was already selected, we're ok; - # otherwise we can't be sure what the optimal configuration is - # and we bail out - if rv > 0 && (sol[rp0] == spp[rp0] || sol[rp0] == rv) - sol[rp0] = rv - else - return (false, Int[]) - end - - rp ∈ seen || push!(staged_next, rp) - end - end - union!(seen, staged_next) - staged = staged_next - end - - @assert verify_solution(sol, interface) - - return true, sol -end - -# verifies that the solution fulfills all hard constraints -# (requirements and dependencies) -function verify_solution(sol::Vector{Int}, interface::Interface) - reqs = interface.reqs - deps = interface.deps - spp = interface.spp - pdict = interface.pdict - pvers = interface.pvers - vdict = interface.vdict - - # verify requirements - for (p,vs) in reqs - p0 = pdict[p] - sol[p0] != spp[p0] || return false - vn = pvers[p0][sol[p0]] - vn ∈ vs || return false - end - - # verify dependencies - for (p,depsp) in deps - p0 = pdict[p] - vdict0 = vdict[p0] - for (vn,vdep) in depsp - v0 = vdict0[vn] - if sol[p0] == v0 - for (rp, rvs) in vdep - p1 = pdict[rp] - if sol[p1] == spp[p1] - println(""" - VERIFICATION ERROR: REQUIRED DEPENDENCY NOT INSTALLED - package p=$p (p0=$p0) version=$vn (v0=$v0) requires package rp=$rp in version set rvs=$rvs - but package $rp is not being installed (p1=$p1 sol[p1]=$(sol[p1]) == spp[p1]=$(spp[p1])) - """) - return false - end - vn1 = pvers[p1][sol[p1]] - if vn1 ∉ rvs - println(""" - VERIFICATION ERROR: INVALID VERSION - package p=$p (p0=$p0) version=$vn (v0=$v0) requires package rp=$rp in version set rvs=$rvs - but package $rp version is being set to $vn1 (p1=$p1 sol[p1]=$(sol[p1]) spp[p1]=$(spp[p1])) - """) - return false - end - end - end - end - end - return true -end - -# Push the given solution to a local optimium if needed -function enforce_optimality!(sol::Vector{Int}, interface::Interface) - np = interface.np - - reqs = interface.reqs - deps = interface.deps - pkgs = interface.pkgs - spp = interface.spp - pdict = interface.pdict - pvers = interface.pvers - vdict = interface.vdict - - # prepare some useful structures - # pdeps[p0][v0] has all dependencies of package p0 version v0 - pdeps = [Vector{Requires}(spp[p0]-1) for p0 = 1:np] - # prevdeps[p1][p0][v0] is the VersionSpec of package p1 which package p0 version v0 - # depends upon - prevdeps = [Dict{Int,Dict{Int,VersionSpec}}() for p0 = 1:np] - - for (p,depsp) in deps - p0 = pdict[p] - vdict0 = vdict[p0] - for (vn,vdep) in depsp - v0 = vdict0[vn] - pdeps[p0][v0] = vdep - for (rp,rvs) in vdep - p1 = pdict[rp] - if !haskey(prevdeps[p1], p0) - prevdeps[p1][p0] = Dict{Int,VersionSpec}() - end - prevdeps[p1][p0][v0] = rvs - end - end - end - - restart = true - while restart - restart = false - for p0 = 1:np - s0 = sol[p0] - if s0 >= spp[p0] - 1 - # either the package is not installed, - # or it's already at the maximum version - continue - end - viol = false - # check if the higher version has a depencency which - # would be violated by the state of the remaining packages - for (p,vs) in pdeps[p0][s0+1] - p1 = pdict[p] - if sol[p1] == spp[p1] - # the dependency is violated because - # the other package is not being installed - viol = true - break - end - vn = pvers[p1][sol[p1]] - if vn ∉ vs - # the dependency is violated because - # the other package version is invalid - viol = true - break - end - end - viol && continue - - # check if bumping the version would violate some - # dependency of another package - for (p1,d) in prevdeps[p0] - vs = get(d, sol[p1], nothing) - vs === nothing && continue - vn = pvers[p0][s0+1] - if vn ∉ vs - # bumping the version would violate - # the dependency - viol = true - break - end - end - viol && continue - # So the solution is non-optimal: we bump it manually - #warn("nonoptimal solution for package $(interface.pkgs[p0]): sol=$s0") - sol[p0] += 1 - restart = true - end - end - - # Finally uninstall unneeded packages: - # start from the required ones and keep only - # the packages reachable from them along the graph - uninst = trues(np) - staged = Set{UUID}(keys(reqs)) - seen = copy(staged) - - while !isempty(staged) - staged_next = Set{UUID}() - for p in staged - p0 = pdict[p] - uninst[p0] = false - @assert sol[p0] < spp[p0] - vn = pvers[p0][sol[p0]] - vdep = deps[p][vn] - - # scan dependencies - for (rp,rvs) in vdep - rp0 = pdict[rp] - @assert sol[rp0] < spp[rp0] && pvers[rp0][sol[rp0]] ∈ rvs - rp ∈ seen || push!(staged_next, rp) - end - end - union!(seen, staged_next) - staged = staged_next - end - - for p0 in find(uninst) - sol[p0] = spp[p0] - end - - return -end - -end diff --git a/test/resolve.jl b/test/resolve.jl index b98b773d86030..9194cbca03a4c 100644 --- a/test/resolve.jl +++ b/test/resolve.jl @@ -4,11 +4,15 @@ module ResolveTest using ..Test using Pkg3.Types -using Pkg3.Query +using Pkg3.GraphType +using Pkg3.Types: VersionBound using Pkg3.Resolve using Pkg3.Resolve.VersionWeights import Pkg3.Types: uuid5, uuid_package +# print info, stats etc. +const VERBOSE = false + # Check that VersionWeight keeps the same ordering as VersionNumber vlst = [ @@ -31,43 +35,10 @@ for v1 in vlst, v2 in vlst @test ceq == (vw1 == vw2) end -# TODO: check that these are unacceptable for VersionSpec -vlst_invalid = [ - v"1.0.0-pre", - v"1.0.0-pre1", - v"1.0.1-pre", - v"1.0.0-0.pre.2", - v"1.0.0-0.pre.3", - v"1.0.0-0.pre1.tst", - v"1.0.0-pre.1+0.1", - v"1.0.0-pre.1+0.1plus", - v"1.0.0-pre.1-+0.1plus", - v"1.0.0-pre.1-+0.1Plus", - v"1.0.0-pre.1-+0.1pLUs", - v"1.0.0-pre.1-+0.1pluS", - v"1.0.0+0.1plus", - v"1.0.0+0.1plus-", - v"1.0.0+-", - v"1.0.0-", - v"1.0.0+", - v"1.0.0--", - v"1.0.0---", - v"1.0.0--+-", - v"1.0.0+--", - v"1.0.0+-.-", - v"1.0.0+0.-", - v"1.0.0+-.0", - v"1.0.0-a+--", - v"1.0.0-a+-.-", - v"1.0.0-a+0.-", - v"1.0.0-a+-.0" - ] - - # auxiliary functions pkguuid(p::String) = uuid5(uuid_package, p) function storeuuid(p::String, uuid_to_name::Dict{UUID,String}) - uuid = pkguuid(p) + uuid = p == "julia" ? Types.uuid_julia : pkguuid(p) if haskey(uuid_to_name, uuid) @assert uuid_to_name[uuid] == p else @@ -77,60 +48,159 @@ function storeuuid(p::String, uuid_to_name::Dict{UUID,String}) end wantuuids(want_data) = Dict{UUID,VersionNumber}(pkguuid(p) => v for (p,v) in want_data) -function deps_from_data(deps_data, uuid_to_name = Dict{UUID,String}()) - deps = DepsGraph() +function load_package_data_raw(T::Type, input::String) + toml = Types.TOML.parse(input) + data = Dict{VersionRange,Dict{String,T}}() + for (v, d) in toml, (key, value) in d + vr = VersionRange(v) + dict = get!(data, vr, Dict{String,T}()) + haskey(dict, key) && cmderror("$ver/$key is duplicated in $path") + dict[key] = T(value) + end + return data +end + +function gen_versionranges(dict::Dict{K,Set{VersionNumber}}, srtvers::Vector{VersionNumber}) where {K} + vranges = Dict{K,Vector{VersionRange}}() + for (vreq,vset) in dict + vranges[vreq] = VersionRange[] + while !isempty(vset) + vn0 = minimum(vset) + i = findfirst(srtvers, vn0) + @assert i ≠ 0 + pop!(vset, vn0) + vn1 = vn0 + pushed = false + j = i + 1 + while j ≤ length(srtvers) + vn = srtvers[j] + if vn ∈ vset + pop!(vset, vn) + vn1 = vn + j += 1 + else + # vn1 = srtvers[j-1] + push!(vranges[vreq], VersionRange(VersionBound(vn0),VersionBound(vn1))) + pushed = true + break + end + end + !pushed && push!(vranges[vreq], VersionRange(VersionBound(vn0),VersionBound(vn1))) + end + end + allvranges = unique(vcat(collect(values(vranges))...)) + return vranges, allvranges +end + +function graph_from_data(deps_data, uuid_to_name = Dict{UUID,String}(Types.uuid_julia=>"julia")) uuid(p) = storeuuid(p, uuid_to_name) + # deps = DepsGraph(uuid_to_name) + fixed = Dict(Types.uuid_julia => Fixed(VERSION)) + all_versions = Dict{UUID,Set{VersionNumber}}(fp => Set([fx.version]) for (fp,fx) in fixed) + all_deps = Dict{UUID,Dict{VersionRange,Dict{String,UUID}}}(fp => Dict(VersionRange(fx.version)=>Dict()) for (fp,fx) in fixed) + all_compat = Dict{UUID,Dict{VersionRange,Dict{String,VersionSpec}}}(fp => Dict(VersionRange(fx.version)=>Dict()) for (fp,fx) in fixed) + + deps = Dict{String,Dict{VersionNumber,Dict{String,VersionSpec}}}() for d in deps_data - p, vn, r = uuid(d[1]), d[2], d[3:end] + p, vn, r = d[1], d[2], d[3:end] if !haskey(deps, p) - deps[p] = Dict{VersionNumber,Requires}() + deps[p] = Dict{VersionNumber,Dict{String,VersionSpec}}() end if !haskey(deps[p], vn) - deps[p][vn] = Dict{UUID,VersionSpec}() + deps[p][vn] = Dict{String,VersionSpec}() end isempty(r) && continue - rp = uuid(r[1]) + rp = r[1] rvs = VersionSpec(r[2:end]) deps[p][vn][rp] = rvs end - deps, uuid_to_name + for p in keys(deps) + u = uuid(p) + all_versions[u] = Set(keys(deps[p])) + srtvers = sort!(collect(keys(deps[p]))) + + deps_pkgs = Dict{String,Set{VersionNumber}}() + for (vn,vreq) in deps[p], rp in keys(vreq) + push!(get!(deps_pkgs, rp, Set{VersionNumber}()), vn) + end + vranges, allvranges = gen_versionranges(deps_pkgs, srtvers) + all_deps[u] = Dict{VersionRange,Dict{String,UUID}}(VersionRange()=>Dict{String,UUID}("julia"=>Types.uuid_julia)) + for vrng in allvranges + all_deps[u][vrng] = Dict{String,UUID}() + for (rp,vvr) in vranges + vrng ∈ vvr || continue + all_deps[u][vrng][rp] = uuid(rp) + end + end + + deps_reqs = Dict{Pair{String,VersionSpec},Set{VersionNumber}}() + for (vn,vreq) in deps[p], (rp,rvs) in vreq + push!(get!(deps_reqs, (rp=>rvs), Set{VersionNumber}()), vn) + end + vranges, allvranges = gen_versionranges(deps_reqs, srtvers) + all_compat[u] = Dict{VersionRange,Dict{String,VersionSpec}}() + for vrng in allvranges + all_compat[u][vrng] = Dict{String,VersionSpec}() + for (req,vvr) in vranges + vrng ∈ vvr || continue + rp,rvs = req + all_compat[u][vrng][rp] = rvs + end + end + end + return Graph(all_versions, all_deps, all_compat, uuid_to_name, Requires(), fixed) end -function reqs_from_data(reqs_data, uuid_to_name = Dict{UUID,String}()) +function reqs_from_data(reqs_data, graph::Graph) reqs = Dict{UUID,VersionSpec}() - uuid(p) = storeuuid(p, uuid_to_name) + function uuid_check(p) + uuid = pkguuid(p) + @assert graph.data.uuid_to_name[uuid] == p + return uuid + end for r in reqs_data - p = uuid(r[1]) + p = uuid_check(r[1]) reqs[p] = VersionSpec(r[2:end]) end - reqs, uuid_to_name + reqs end function sanity_tst(deps_data, expected_result; pkgs=[]) - deps, uuid_to_name = deps_from_data(deps_data) - id(p) = pkgID(pkguuid(p), uuid_to_name) - #println("deps=$deps") - #println() - result = sanity_check(deps, uuid_to_name, Set(pkguuid(p) for p in pkgs)) + graph = graph_from_data(deps_data) + id(p) = pkgID(pkguuid(p), graph) + if VERBOSE + println() + info("sanity check") + @show deps_data + @show pkgs + end + result = sanity_check(graph, Set(pkguuid(p) for p in pkgs), verbose = VERBOSE) + length(result) == length(expected_result) || return false expected_result_uuid = [(id(p), vn) for (p,vn) in expected_result] - for (p, vn, pp) in result - (p, vn) ∈ expected_result_uuid || return false + for r in result + r ∈ expected_result_uuid || return false end return true end sanity_tst(deps_data; kw...) = sanity_tst(deps_data, []; kw...) function resolve_tst(deps_data, reqs_data, want_data = nothing) - deps, uuid_to_name = deps_from_data(deps_data) - reqs, uuid_to_name = reqs_from_data(reqs_data, uuid_to_name) - - #println() - #println("deps=$deps") - #println("reqs=$reqs") - deps = Query.prune_dependencies(reqs, deps, uuid_to_name) - want = resolve(reqs, deps, uuid_to_name) + if VERBOSE + println() + info("resolving") + @show deps_data + @show reqs_data + end + graph = graph_from_data(deps_data) + reqs = reqs_from_data(reqs_data, graph) + add_reqs!(graph, reqs) + + simplify_graph!(graph, verbose = VERBOSE) + want = resolve(graph, verbose = VERBOSE) + return want == wantuuids(want_data) end +VERBOSE && info("SCHEME 1") ## DEPENDENCY SCHEME 1: TWO PACKAGES, DAG deps_data = Any[ ["A", v"1", "B", "1-*"], @@ -150,6 +220,7 @@ reqs_data = Any[ ] want_data = Dict("B"=>v"2") +resolve_tst(deps_data, reqs_data, want_data) @test resolve_tst(deps_data, reqs_data, want_data) # require just A: must bring in B @@ -160,6 +231,7 @@ want_data = Dict("A"=>v"2", "B"=>v"2") @test resolve_tst(deps_data, reqs_data, want_data) +VERBOSE && info("SCHEME 2") ## DEPENDENCY SCHEME 2: TWO PACKAGES, CYCLIC deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -192,6 +264,7 @@ want_data = Dict("A"=>v"1", "B"=>v"2") @test resolve_tst(deps_data, reqs_data, want_data) +VERBOSE && info("SCHEME 3") ## DEPENDENCY SCHEME 3: THREE PACKAGES, CYCLIC, TWO MUTUALLY EXCLUSIVE SOLUTIONS deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -233,6 +306,7 @@ reqs_data = Any[ @test_throws PkgError resolve_tst(deps_data, reqs_data) +VERBOSE && info("SCHEME 4") ## DEPENDENCY SCHEME 4: TWO PACKAGES, DAG, WITH TRIVIAL INCONSISTENCY deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -240,7 +314,7 @@ deps_data = Any[ ] @test sanity_tst(deps_data, [("A", v"1")]) -@test sanity_tst(deps_data, [("A", v"1")], pkgs=["B"]) +@test sanity_tst(deps_data, pkgs=["B"]) # require B (must not give errors) reqs_data = Any[ @@ -250,6 +324,7 @@ want_data = Dict("B"=>v"1") @test resolve_tst(deps_data, reqs_data, want_data) +VERBOSE && info("SCHEME 5") ## DEPENDENCY SCHEME 5: THREE PACKAGES, DAG, WITH IMPLICIT INCONSISTENCY deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -263,8 +338,8 @@ deps_data = Any[ ] @test sanity_tst(deps_data, [("A", v"2")]) -@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B"]) -@test sanity_tst(deps_data, [("A", v"2")], pkgs=["C"]) +@test sanity_tst(deps_data, pkgs=["B"]) +@test sanity_tst(deps_data, pkgs=["C"]) # require A, any version (must use the highest non-inconsistent) reqs_data = Any[ @@ -280,6 +355,7 @@ reqs_data = Any[ @test_throws PkgError resolve_tst(deps_data, reqs_data) +VERBOSE && info("SCHEME 6") ## DEPENDENCY SCHEME 6: TWO PACKAGES, CYCLIC, TOTALLY INCONSISTENT deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -304,6 +380,7 @@ reqs_data = Any[ @test_throws PkgError resolve_tst(deps_data, reqs_data) +VERBOSE && info("SCHEME 7") ## DEPENDENCY SCHEME 7: THREE PACKAGES, CYCLIC, WITH INCONSISTENCY deps_data = Any[ ["A", v"1", "B", "1"], @@ -338,6 +415,7 @@ reqs_data = Any[ @test_throws PkgError resolve_tst(deps_data, reqs_data) +VERBOSE && info("SCHEME 8") ## DEPENDENCY SCHEME 8: THREE PACKAGES, CYCLIC, TOTALLY INCONSISTENT deps_data = Any[ ["A", v"1", "B", "1"], @@ -370,6 +448,7 @@ reqs_data = Any[ ] @test_throws PkgError resolve_tst(deps_data, reqs_data) +VERBOSE && info("SCHEME 9") ## DEPENDENCY SCHEME 9: SIX PACKAGES, DAG deps_data = Any[ ["A", v"1"], @@ -433,6 +512,7 @@ want_data = Dict("A"=>v"2", "B"=>v"2", "C"=>v"1", "D"=>v"2", "E"=>v"1", "F"=>v"1") @test resolve_tst(deps_data, reqs_data, want_data) +VERBOSE && info("SCHEME 10") ## DEPENDENCY SCHEME 10: FIVE PACKAGES, SAME AS SCHEMES 5 + 1, UNCONNECTED deps_data = Any[ ["A", v"1", "B", "2-*"], @@ -450,10 +530,10 @@ deps_data = Any[ ] @test sanity_tst(deps_data, [("A", v"2")]) -@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B"]) +@test sanity_tst(deps_data, pkgs=["B"]) @test sanity_tst(deps_data, pkgs=["D"]) @test sanity_tst(deps_data, pkgs=["E"]) -@test sanity_tst(deps_data, [("A", v"2")], pkgs=["B", "D"]) +@test sanity_tst(deps_data, pkgs=["B", "D"]) # require A, any version (must use the highest non-inconsistent) reqs_data = Any[ @@ -478,6 +558,7 @@ reqs_data = Any[ want_data = Dict("A"=>v"1", "B"=>v"2", "C"=>v"2", "D"=>v"2", "E"=>v"2") @test resolve_tst(deps_data, reqs_data, want_data) +VERBOSE && info("SCHEME 11") ## DEPENDENCY SCHEME 11: A REALISTIC EXAMPLE ## ref issue #21485