diff --git a/examples/pypi_wheel_provider.py b/examples/pypi_wheel_provider.py index 888aa69..04179ba 100644 --- a/examples/pypi_wheel_provider.py +++ b/examples/pypi_wheel_provider.py @@ -14,7 +14,8 @@ from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version -from resolvelib import BaseReporter, Resolver +from resolvelib import BaseReporter +from resolvelib.resolvers.resolution import Resolver from .extras_provider import ExtrasProvider diff --git a/examples/reporter_demo.py b/examples/reporter_demo.py index 4ff0430..2fbf6e1 100644 --- a/examples/reporter_demo.py +++ b/examples/reporter_demo.py @@ -1,9 +1,11 @@ from collections import namedtuple -import resolvelib from packaging.specifiers import SpecifierSet from packaging.version import Version +import resolvelib +import resolvelib.resolvers.resolution + index = """ first 1.0.0 second == 1.0.0 @@ -123,7 +125,7 @@ def pinning(self, candidate): provider = Provider(index.splitlines()) reporter = Reporter() - resolver = resolvelib.Resolver(provider, reporter) + resolver = resolvelib.resolvers.resolution.Resolver(provider, reporter) root_reqs = [Requirement("first", SpecifierSet())] result = resolver.resolve(root_reqs) diff --git a/examples/visualization/run_pypi.py b/examples/visualization/run_pypi.py index d6a397d..e0faabb 100644 --- a/examples/visualization/run_pypi.py +++ b/examples/visualization/run_pypi.py @@ -2,7 +2,7 @@ from visualization.generate import generate_html from visualization.reporter import GraphGeneratingReporter -from resolvelib import Resolver +from resolvelib.resolvers.resolution import Resolver if __name__ == "__main__": provider = PyPIProvider() diff --git a/src/resolvelib/__init__.py b/src/resolvelib/__init__.py index fd9f249..e7f36c7 100644 --- a/src/resolvelib/__init__.py +++ b/src/resolvelib/__init__.py @@ -23,5 +23,5 @@ ResolutionError, ResolutionImpossible, ResolutionTooDeep, - Resolver, ) +from .resolvers.resolution import Resolver diff --git a/src/resolvelib/reporters.py b/src/resolvelib/reporters.py index 91afad3..2ce4626 100644 --- a/src/resolvelib/reporters.py +++ b/src/resolvelib/reporters.py @@ -1,16 +1,11 @@ from __future__ import annotations -from typing import Generic - -from .structs import ( - CT, - KT, - RT, - Collection, - Criterion, - RequirementInformation, - State, -) +from typing import TYPE_CHECKING, Collection, Generic + +from .structs import CT, KT, RT, RequirementInformation, State + +if TYPE_CHECKING: + from .resolvers.criterion import Criterion class BaseReporter(Generic[RT, CT, KT]): diff --git a/src/resolvelib/resolvers/__init__.py b/src/resolvelib/resolvers/__init__.py index 1f0ddcc..263d4ff 100644 --- a/src/resolvelib/resolvers/__init__.py +++ b/src/resolvelib/resolvers/__init__.py @@ -1,19 +1,20 @@ from ..structs import RequirementInformation from .abstract import AbstractResolver, Result -from .criterion import ( +from .exceptions import ( InconsistentCandidate, RequirementsConflicted, ResolutionError, ResolutionImpossible, ResolutionTooDeep, - Resolver, ResolverException, ) +from .resolution import Resolution, Resolver __all__ = [ "AbstractResolver", "InconsistentCandidate", "Resolver", + "Resolution", "RequirementsConflicted", "ResolutionError", "ResolutionImpossible", diff --git a/src/resolvelib/resolvers/abstract.py b/src/resolvelib/resolvers/abstract.py index 047e2fd..99fec95 100644 --- a/src/resolvelib/resolvers/abstract.py +++ b/src/resolvelib/resolvers/abstract.py @@ -3,10 +3,10 @@ import collections from typing import TYPE_CHECKING, Any, Generic, Iterable, Mapping, NamedTuple -from resolvelib.providers import AbstractProvider -from resolvelib.reporters import BaseReporter - -from ..structs import CT, KT, RT, Criterion, DirectedGraph +from ..providers import AbstractProvider +from ..reporters import BaseReporter +from ..structs import CT, KT, RT, DirectedGraph +from .criterion import Criterion if TYPE_CHECKING: diff --git a/src/resolvelib/resolvers/criterion.py b/src/resolvelib/resolvers/criterion.py index bb1c733..33d4257 100644 --- a/src/resolvelib/resolvers/criterion.py +++ b/src/resolvelib/resolvers/criterion.py @@ -1,556 +1,46 @@ -from __future__ import annotations +from typing import Collection, Generic, Iterable, Iterator -import collections -import itertools -import operator -from typing import TYPE_CHECKING, Collection, Generic, Iterable, Mapping +from ..structs import CT, RT, RequirementInformation -from ..providers import AbstractProvider -from ..reporters import BaseReporter -from ..structs import ( - CT, - KT, - RT, - Criterion, - DirectedGraph, - IterableView, - IteratorMapping, - RequirementInformation, - State, - build_iter_view, -) -from .abstract import AbstractResolver, Result -if TYPE_CHECKING: - from ..providers import Preference +class Criterion(Generic[RT, CT]): + """Representation of possible resolution results of a package. + This holds three attributes: -class ResolverException(Exception): - """A base class for all exceptions raised by this module. + * `information` is a collection of `RequirementInformation` pairs. + Each pair is a requirement contributing to this criterion, and the + candidate that provides the requirement. + * `incompatibilities` is a collection of all known not-to-work candidates + to exclude from consideration. + * `candidates` is a collection containing all possible candidates deducted + from the union of contributing requirements and known incompatibilities. + It should never be empty, except when the criterion is an attribute of a + raised `RequirementsConflicted` (in which case it is always empty). - Exceptions derived by this class should all be handled in this module. Any - bubbling pass the resolver should be treated as a bug. - """ - - -class RequirementsConflicted(ResolverException, Generic[RT, CT]): - def __init__(self, criterion: Criterion[RT, CT]) -> None: - super().__init__(criterion) - self.criterion = criterion - - def __str__(self) -> str: - return "Requirements conflict: {}".format( - ", ".join(repr(r) for r in self.criterion.iter_requirement()), - ) - - -class InconsistentCandidate(ResolverException, Generic[RT, CT]): - def __init__(self, candidate: CT, criterion: Criterion[RT, CT]): - super().__init__(candidate, criterion) - self.candidate = candidate - self.criterion = criterion - - def __str__(self) -> str: - return "Provided candidate {!r} does not satisfy {}".format( - self.candidate, - ", ".join(repr(r) for r in self.criterion.iter_requirement()), - ) - - -class ResolutionError(ResolverException): - pass - - -class ResolutionImpossible(ResolutionError, Generic[RT, CT]): - def __init__(self, causes: Collection[RequirementInformation[RT, CT]]): - super().__init__(causes) - # causes is a list of RequirementInformation objects - self.causes = causes - - -class ResolutionTooDeep(ResolutionError): - def __init__(self, round_count: int) -> None: - super().__init__(round_count) - self.round_count = round_count - - -class Resolution(Generic[RT, CT, KT]): - """Stateful resolution object. - - This is designed as a one-off object that holds information to kick start - the resolution process, and holds the results afterwards. + .. note:: + This class is intended to be externally immutable. **Do not** mutate + any of its attribute containers. """ def __init__( self, - provider: AbstractProvider[RT, CT, KT], - reporter: BaseReporter[RT, CT, KT], - ) -> None: - self._p = provider - self._r = reporter - self._states: list[State[RT, CT, KT]] = [] - - @property - def state(self) -> State[RT, CT, KT]: - try: - return self._states[-1] - except IndexError as e: - raise AttributeError("state") from e - - def _push_new_state(self) -> None: - """Push a new state into history. - - This new state will be used to hold resolution results of the next - coming round. - """ - base = self._states[-1] - state = State( - mapping=base.mapping.copy(), - criteria=base.criteria.copy(), - backtrack_causes=base.backtrack_causes[:], - ) - self._states.append(state) - - def _add_to_criteria( - self, - criteria: dict[KT, Criterion[RT, CT]], - requirement: RT, - parent: CT | None, - ) -> None: - self._r.adding_requirement(requirement=requirement, parent=parent) - - identifier = self._p.identify(requirement_or_candidate=requirement) - criterion = criteria.get(identifier) - if criterion: - incompatibilities = list(criterion.incompatibilities) - else: - incompatibilities = [] - - matches = self._p.find_matches( - identifier=identifier, - requirements=IteratorMapping( - criteria, - operator.methodcaller("iter_requirement"), - {identifier: [requirement]}, - ), - incompatibilities=IteratorMapping( - criteria, - operator.attrgetter("incompatibilities"), - {identifier: incompatibilities}, - ), - ) - - if criterion: - information = list(criterion.information) - information.append(RequirementInformation(requirement, parent)) - else: - information = [RequirementInformation(requirement, parent)] - - criterion = Criterion( - candidates=build_iter_view(matches), - information=information, - incompatibilities=incompatibilities, - ) - if not criterion.candidates: - raise RequirementsConflicted(criterion) - criteria[identifier] = criterion - - def _remove_information_from_criteria( - self, criteria: dict[KT, Criterion[RT, CT]], parents: Collection[KT] + candidates: Iterable[CT], + information: Collection[RequirementInformation[RT, CT]], + incompatibilities: Collection[CT], ) -> None: - """Remove information from parents of criteria. - - Concretely, removes all values from each criterion's ``information`` - field that have one of ``parents`` as provider of the requirement. - - :param criteria: The criteria to update. - :param parents: Identifiers for which to remove information from all criteria. - """ - if not parents: - return - for key, criterion in criteria.items(): - criteria[key] = Criterion( - criterion.candidates, - [ - information - for information in criterion.information - if ( - information.parent is None - or self._p.identify(information.parent) not in parents - ) - ], - criterion.incompatibilities, - ) + self.candidates = candidates + self.information = information + self.incompatibilities = incompatibilities - def _get_preference(self, name: KT) -> Preference: - return self._p.get_preference( - identifier=name, - resolutions=self.state.mapping, - candidates=IteratorMapping( - self.state.criteria, - operator.attrgetter("candidates"), - ), - information=IteratorMapping( - self.state.criteria, - operator.attrgetter("information"), - ), - backtrack_causes=self.state.backtrack_causes, + def __repr__(self) -> str: + requirements = ", ".join( + f"({req!r}, via={parent!r})" for req, parent in self.information ) + return f"Criterion({requirements})" - def _is_current_pin_satisfying( - self, name: KT, criterion: Criterion[RT, CT] - ) -> bool: - try: - current_pin = self.state.mapping[name] - except KeyError: - return False - return all( - self._p.is_satisfied_by(requirement=r, candidate=current_pin) - for r in criterion.iter_requirement() - ) - - def _get_updated_criteria(self, candidate: CT) -> dict[KT, Criterion[RT, CT]]: - criteria = self.state.criteria.copy() - for requirement in self._p.get_dependencies(candidate=candidate): - self._add_to_criteria(criteria, requirement, parent=candidate) - return criteria - - def _attempt_to_pin_criterion(self, name: KT) -> list[Criterion[RT, CT]]: - criterion = self.state.criteria[name] - - causes: list[Criterion[RT, CT]] = [] - for candidate in criterion.candidates: - try: - criteria = self._get_updated_criteria(candidate) - except RequirementsConflicted as e: - self._r.rejecting_candidate(e.criterion, candidate) - causes.append(e.criterion) - continue - - # Check the newly-pinned candidate actually works. This should - # always pass under normal circumstances, but in the case of a - # faulty provider, we will raise an error to notify the implementer - # to fix find_matches() and/or is_satisfied_by(). - satisfied = all( - self._p.is_satisfied_by(requirement=r, candidate=candidate) - for r in criterion.iter_requirement() - ) - if not satisfied: - raise InconsistentCandidate(candidate, criterion) - - self._r.pinning(candidate=candidate) - self.state.criteria.update(criteria) - - # Put newly-pinned candidate at the end. This is essential because - # backtracking looks at this mapping to get the last pin. - self.state.mapping.pop(name, None) - self.state.mapping[name] = candidate - - return [] - - # All candidates tried, nothing works. This criterion is a dead - # end, signal for backtracking. - return causes - - def _patch_criteria( - self, incompatibilities_from_broken: list[tuple[KT, list[CT]]] - ) -> bool: - # Create a new state from the last known-to-work one, and apply - # the previously gathered incompatibility information. - for k, incompatibilities in incompatibilities_from_broken: - if not incompatibilities: - continue - try: - criterion = self.state.criteria[k] - except KeyError: - continue - matches = self._p.find_matches( - identifier=k, - requirements=IteratorMapping( - self.state.criteria, - operator.methodcaller("iter_requirement"), - ), - incompatibilities=IteratorMapping( - self.state.criteria, - operator.attrgetter("incompatibilities"), - {k: incompatibilities}, - ), - ) - candidates: IterableView[CT] = build_iter_view(matches) - if not candidates: - return False - incompatibilities.extend(criterion.incompatibilities) - self.state.criteria[k] = Criterion( - candidates=candidates, - information=list(criterion.information), - incompatibilities=incompatibilities, - ) - return True - - def _backjump(self, causes: list[RequirementInformation[RT, CT]]) -> bool: - """Perform backjumping. - - When we enter here, the stack is like this:: - - [ state Z ] - [ state Y ] - [ state X ] - .... earlier states are irrelevant. - - 1. No pins worked for Z, so it does not have a pin. - 2. We want to reset state Y to unpinned, and pin another candidate. - 3. State X holds what state Y was before the pin, but does not - have the incompatibility information gathered in state Y. - - Each iteration of the loop will: - - 1. Identify Z. The incompatibility is not always caused by the latest - state. For example, given three requirements A, B and C, with - dependencies A1, B1 and C1, where A1 and B1 are incompatible: the - last state might be related to C, so we want to discard the - previous state. - 2. Discard Z. - 3. Discard Y but remember its incompatibility information gathered - previously, and the failure we're dealing with right now. - 4. Push a new state Y' based on X, and apply the incompatibility - information from Y to Y'. - 5a. If this causes Y' to conflict, we need to backtrack again. Make Y' - the new Z and go back to step 2. - 5b. If the incompatibilities apply cleanly, end backtracking. - """ - incompatible_reqs: Iterable[CT | RT] = itertools.chain( - (c.parent for c in causes if c.parent is not None), - (c.requirement for c in causes), - ) - incompatible_deps = {self._p.identify(r) for r in incompatible_reqs} - while len(self._states) >= 3: - # Remove the state that triggered backtracking. - del self._states[-1] - - # Optimistically backtrack to a state that caused the incompatibility - broken_state = self.state - while True: - # Retrieve the last candidate pin and known incompatibilities. - try: - broken_state = self._states.pop() - name, candidate = broken_state.mapping.popitem() - except (IndexError, KeyError): - raise ResolutionImpossible(causes) from None - - # Only backjump if the current broken state is - # an incompatible dependency - if name not in incompatible_deps: - break - - # If the current dependencies and the incompatible dependencies - # are overlapping then we have found a cause of the incompatibility - current_dependencies = { - self._p.identify(d) for d in self._p.get_dependencies(candidate) - } - if not current_dependencies.isdisjoint(incompatible_deps): - break - - # Fallback: We should not backtrack to the point where - # broken_state.mapping is empty, so stop backtracking for - # a chance for the resolution to recover - if not broken_state.mapping: - break - - incompatibilities_from_broken = [ - (k, list(v.incompatibilities)) for k, v in broken_state.criteria.items() - ] - - # Also mark the newly known incompatibility. - incompatibilities_from_broken.append((name, [candidate])) - - self._push_new_state() - success = self._patch_criteria(incompatibilities_from_broken) - - # It works! Let's work on this new state. - if success: - return True - - # State does not work after applying known incompatibilities. - # Try the still previous state. - - # No way to backtrack anymore. - return False - - def _extract_causes( - self, criteron: list[Criterion[RT, CT]] - ) -> list[RequirementInformation[RT, CT]]: - """Extract causes from list of criterion and deduplicate""" - return list({id(i): i for c in criteron for i in c.information}.values()) - - def resolve(self, requirements: Iterable[RT], max_rounds: int) -> State[RT, CT, KT]: - if self._states: - raise RuntimeError("already resolved") - - self._r.starting() - - # Initialize the root state. - self._states = [ - State( - mapping=collections.OrderedDict(), - criteria={}, - backtrack_causes=[], - ) - ] - for r in requirements: - try: - self._add_to_criteria(self.state.criteria, r, parent=None) - except RequirementsConflicted as e: - raise ResolutionImpossible(e.criterion.information) from e - - # The root state is saved as a sentinel so the first ever pin can have - # something to backtrack to if it fails. The root state is basically - # pinning the virtual "root" package in the graph. - self._push_new_state() - - for round_index in range(max_rounds): - self._r.starting_round(index=round_index) - - unsatisfied_names = [ - key - for key, criterion in self.state.criteria.items() - if not self._is_current_pin_satisfying(key, criterion) - ] - - # All criteria are accounted for. Nothing more to pin, we are done! - if not unsatisfied_names: - self._r.ending(state=self.state) - return self.state - - # keep track of satisfied names to calculate diff after pinning - satisfied_names = set(self.state.criteria.keys()) - set(unsatisfied_names) - - # Choose the most preferred unpinned criterion to try. - name = min(unsatisfied_names, key=self._get_preference) - failure_criterion = self._attempt_to_pin_criterion(name) - - if failure_criterion: - causes = self._extract_causes(failure_criterion) - # Backjump if pinning fails. The backjump process puts us in - # an unpinned state, so we can work on it in the next round. - self._r.resolving_conflicts(causes=causes) - success = self._backjump(causes) - self.state.backtrack_causes[:] = causes - - # Dead ends everywhere. Give up. - if not success: - raise ResolutionImpossible(self.state.backtrack_causes) - else: - # discard as information sources any invalidated names - # (unsatisfied names that were previously satisfied) - newly_unsatisfied_names = { - key - for key, criterion in self.state.criteria.items() - if key in satisfied_names - and not self._is_current_pin_satisfying(key, criterion) - } - self._remove_information_from_criteria( - self.state.criteria, newly_unsatisfied_names - ) - # Pinning was successful. Push a new state to do another pin. - self._push_new_state() - - self._r.ending_round(index=round_index, state=self.state) - - raise ResolutionTooDeep(max_rounds) - - -def _has_route_to_root( - criteria: Mapping[KT, Criterion[RT, CT]], - key: KT | None, - all_keys: dict[int, KT | None], - connected: set[KT | None], -) -> bool: - if key in connected: - return True - if key not in criteria: - return False - assert key is not None - for p in criteria[key].iter_parent(): - try: - pkey = all_keys[id(p)] - except KeyError: - continue - if pkey in connected: - connected.add(key) - return True - if _has_route_to_root(criteria, pkey, all_keys, connected): - connected.add(key) - return True - return False - - -def _build_result(state: State[RT, CT, KT]) -> Result[RT, CT, KT]: - mapping = state.mapping - all_keys: dict[int, KT | None] = {id(v): k for k, v in mapping.items()} - all_keys[id(None)] = None - - graph: DirectedGraph[KT | None] = DirectedGraph() - graph.add(None) # Sentinel as root dependencies' parent. - - connected: set[KT | None] = {None} - for key, criterion in state.criteria.items(): - if not _has_route_to_root(state.criteria, key, all_keys, connected): - continue - if key not in graph: - graph.add(key) - for p in criterion.iter_parent(): - try: - pkey = all_keys[id(p)] - except KeyError: - continue - if pkey not in graph: - graph.add(pkey) - graph.connect(pkey, key) - - return Result( - mapping={k: v for k, v in mapping.items() if k in connected}, - graph=graph, - criteria=state.criteria, - ) - - -class Resolver(AbstractResolver[RT, CT, KT]): - """The thing that performs the actual resolution work.""" - - base_exception = ResolverException - - def resolve( # type: ignore[override] - self, - requirements: Iterable[RT], - max_rounds: int = 100, - ) -> Result[RT, CT, KT]: - """Take a collection of constraints, spit out the resolution result. - - The return value is a representation to the final resolution result. It - is a tuple subclass with three public members: - - * `mapping`: A dict of resolved candidates. Each key is an identifier - of a requirement (as returned by the provider's `identify` method), - and the value is the resolved candidate. - * `graph`: A `DirectedGraph` instance representing the dependency tree. - The vertices are keys of `mapping`, and each edge represents *why* - a particular package is included. A special vertex `None` is - included to represent parents of user-supplied requirements. - * `criteria`: A dict of "criteria" that hold detailed information on - how edges in the graph are derived. Each key is an identifier of a - requirement, and the value is a `Criterion` instance. - - The following exceptions may be raised if a resolution cannot be found: + def iter_requirement(self) -> Iterator[RT]: + return (i.requirement for i in self.information) - * `ResolutionImpossible`: A resolution cannot be found for the given - combination of requirements. The `causes` attribute of the - exception is a list of (requirement, parent), giving the - requirements that could not be satisfied. - * `ResolutionTooDeep`: The dependency tree is too deeply nested and - the resolver gave up. This is usually caused by a circular - dependency, but you can try to resolve this by increasing the - `max_rounds` argument. - """ - resolution = Resolution(self.provider, self.reporter) - state = resolution.resolve(requirements, max_rounds=max_rounds) - return _build_result(state) + def iter_parent(self) -> Iterator[CT | None]: + return (i.parent for i in self.information) diff --git a/src/resolvelib/resolvers/exceptions.py b/src/resolvelib/resolvers/exceptions.py new file mode 100644 index 0000000..ad3611b --- /dev/null +++ b/src/resolvelib/resolvers/exceptions.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Collection, Generic + +from ..structs import CT, RT, RequirementInformation +from .criterion import Criterion + + +class ResolverException(Exception): + """A base class for all exceptions raised by this module. + + Exceptions derived by this class should all be handled in this module. Any + bubbling pass the resolver should be treated as a bug. + """ + + +class RequirementsConflicted(ResolverException, Generic[RT, CT]): + def __init__(self, criterion: Criterion[RT, CT]) -> None: + super().__init__(criterion) + self.criterion = criterion + + def __str__(self) -> str: + return "Requirements conflict: {}".format( + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + + +class InconsistentCandidate(ResolverException, Generic[RT, CT]): + def __init__(self, candidate: CT, criterion: Criterion[RT, CT]): + super().__init__(candidate, criterion) + self.candidate = candidate + self.criterion = criterion + + def __str__(self) -> str: + return "Provided candidate {!r} does not satisfy {}".format( + self.candidate, + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + + +class ResolutionError(ResolverException): + pass + + +class ResolutionImpossible(ResolutionError, Generic[RT, CT]): + def __init__(self, causes: Collection[RequirementInformation[RT, CT]]): + super().__init__(causes) + # causes is a list of RequirementInformation objects + self.causes = causes + + +class ResolutionTooDeep(ResolutionError): + def __init__(self, round_count: int) -> None: + super().__init__(round_count) + self.round_count = round_count diff --git a/src/resolvelib/resolvers/resolution.py b/src/resolvelib/resolvers/resolution.py new file mode 100644 index 0000000..c9bfb75 --- /dev/null +++ b/src/resolvelib/resolvers/resolution.py @@ -0,0 +1,511 @@ +from __future__ import annotations + +import collections +import itertools +import operator +from typing import Collection, Generic, Iterable, Mapping + +from ..providers import AbstractProvider, Preference +from ..reporters import BaseReporter +from ..resolvers.abstract import AbstractResolver, Result +from ..resolvers.exceptions import ( + InconsistentCandidate, + RequirementsConflicted, + ResolutionImpossible, + ResolutionTooDeep, + ResolverException, +) +from ..structs import ( + CT, + KT, + RT, + DirectedGraph, + IterableView, + IteratorMapping, + RequirementInformation, + State, + build_iter_view, +) +from .criterion import Criterion + + +def _build_result(state: State[RT, CT, KT]) -> Result[RT, CT, KT]: + mapping = state.mapping + all_keys: dict[int, KT | None] = {id(v): k for k, v in mapping.items()} + all_keys[id(None)] = None + + graph: DirectedGraph[KT | None] = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + + connected: set[KT | None] = {None} + for key, criterion in state.criteria.items(): + if not _has_route_to_root(state.criteria, key, all_keys, connected): + continue + if key not in graph: + graph.add(key) + for p in criterion.iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey not in graph: + graph.add(pkey) + graph.connect(pkey, key) + + return Result( + mapping={k: v for k, v in mapping.items() if k in connected}, + graph=graph, + criteria=state.criteria, + ) + + +class Resolution(Generic[RT, CT, KT]): + """Stateful resolution object. + + This is designed as a one-off object that holds information to kick start + the resolution process, and holds the results afterwards. + """ + + def __init__( + self, + provider: AbstractProvider[RT, CT, KT], + reporter: BaseReporter[RT, CT, KT], + ) -> None: + self._p = provider + self._r = reporter + self._states: list[State[RT, CT, KT]] = [] + + @property + def state(self) -> State[RT, CT, KT]: + try: + return self._states[-1] + except IndexError as e: + raise AttributeError("state") from e + + def _push_new_state(self) -> None: + """Push a new state into history. + + This new state will be used to hold resolution results of the next + coming round. + """ + base = self._states[-1] + state = State( + mapping=base.mapping.copy(), + criteria=base.criteria.copy(), + backtrack_causes=base.backtrack_causes[:], + ) + self._states.append(state) + + def _add_to_criteria( + self, + criteria: dict[KT, Criterion[RT, CT]], + requirement: RT, + parent: CT | None, + ) -> None: + self._r.adding_requirement(requirement=requirement, parent=parent) + + identifier = self._p.identify(requirement_or_candidate=requirement) + criterion = criteria.get(identifier) + if criterion: + incompatibilities = list(criterion.incompatibilities) + else: + incompatibilities = [] + + matches = self._p.find_matches( + identifier=identifier, + requirements=IteratorMapping( + criteria, + operator.methodcaller("iter_requirement"), + {identifier: [requirement]}, + ), + incompatibilities=IteratorMapping( + criteria, + operator.attrgetter("incompatibilities"), + {identifier: incompatibilities}, + ), + ) + + if criterion: + information = list(criterion.information) + information.append(RequirementInformation(requirement, parent)) + else: + information = [RequirementInformation(requirement, parent)] + + criterion = Criterion( + candidates=build_iter_view(matches), + information=information, + incompatibilities=incompatibilities, + ) + if not criterion.candidates: + raise RequirementsConflicted(criterion) + criteria[identifier] = criterion + + def _remove_information_from_criteria( + self, criteria: dict[KT, Criterion[RT, CT]], parents: Collection[KT] + ) -> None: + """Remove information from parents of criteria. + + Concretely, removes all values from each criterion's ``information`` + field that have one of ``parents`` as provider of the requirement. + + :param criteria: The criteria to update. + :param parents: Identifiers for which to remove information from all criteria. + """ + if not parents: + return + for key, criterion in criteria.items(): + criteria[key] = Criterion( + criterion.candidates, + [ + information + for information in criterion.information + if ( + information.parent is None + or self._p.identify(information.parent) not in parents + ) + ], + criterion.incompatibilities, + ) + + def _get_preference(self, name: KT) -> Preference: + return self._p.get_preference( + identifier=name, + resolutions=self.state.mapping, + candidates=IteratorMapping( + self.state.criteria, + operator.attrgetter("candidates"), + ), + information=IteratorMapping( + self.state.criteria, + operator.attrgetter("information"), + ), + backtrack_causes=self.state.backtrack_causes, + ) + + def _is_current_pin_satisfying( + self, name: KT, criterion: Criterion[RT, CT] + ) -> bool: + try: + current_pin = self.state.mapping[name] + except KeyError: + return False + return all( + self._p.is_satisfied_by(requirement=r, candidate=current_pin) + for r in criterion.iter_requirement() + ) + + def _get_updated_criteria(self, candidate: CT) -> dict[KT, Criterion[RT, CT]]: + criteria = self.state.criteria.copy() + for requirement in self._p.get_dependencies(candidate=candidate): + self._add_to_criteria(criteria, requirement, parent=candidate) + return criteria + + def _attempt_to_pin_criterion(self, name: KT) -> list[Criterion[RT, CT]]: + criterion = self.state.criteria[name] + + causes: list[Criterion[RT, CT]] = [] + for candidate in criterion.candidates: + try: + criteria = self._get_updated_criteria(candidate) + except RequirementsConflicted as e: + self._r.rejecting_candidate(e.criterion, candidate) + causes.append(e.criterion) + continue + + # Check the newly-pinned candidate actually works. This should + # always pass under normal circumstances, but in the case of a + # faulty provider, we will raise an error to notify the implementer + # to fix find_matches() and/or is_satisfied_by(). + satisfied = all( + self._p.is_satisfied_by(requirement=r, candidate=candidate) + for r in criterion.iter_requirement() + ) + if not satisfied: + raise InconsistentCandidate(candidate, criterion) + + self._r.pinning(candidate=candidate) + self.state.criteria.update(criteria) + + # Put newly-pinned candidate at the end. This is essential because + # backtracking looks at this mapping to get the last pin. + self.state.mapping.pop(name, None) + self.state.mapping[name] = candidate + + return [] + + # All candidates tried, nothing works. This criterion is a dead + # end, signal for backtracking. + return causes + + def _patch_criteria( + self, incompatibilities_from_broken: list[tuple[KT, list[CT]]] + ) -> bool: + # Create a new state from the last known-to-work one, and apply + # the previously gathered incompatibility information. + for k, incompatibilities in incompatibilities_from_broken: + if not incompatibilities: + continue + try: + criterion = self.state.criteria[k] + except KeyError: + continue + matches = self._p.find_matches( + identifier=k, + requirements=IteratorMapping( + self.state.criteria, + operator.methodcaller("iter_requirement"), + ), + incompatibilities=IteratorMapping( + self.state.criteria, + operator.attrgetter("incompatibilities"), + {k: incompatibilities}, + ), + ) + candidates: IterableView[CT] = build_iter_view(matches) + if not candidates: + return False + incompatibilities.extend(criterion.incompatibilities) + self.state.criteria[k] = Criterion( + candidates=candidates, + information=list(criterion.information), + incompatibilities=incompatibilities, + ) + return True + + def _backjump(self, causes: list[RequirementInformation[RT, CT]]) -> bool: + """Perform backjumping. + + When we enter here, the stack is like this:: + + [ state Z ] + [ state Y ] + [ state X ] + .... earlier states are irrelevant. + + 1. No pins worked for Z, so it does not have a pin. + 2. We want to reset state Y to unpinned, and pin another candidate. + 3. State X holds what state Y was before the pin, but does not + have the incompatibility information gathered in state Y. + + Each iteration of the loop will: + + 1. Identify Z. The incompatibility is not always caused by the latest + state. For example, given three requirements A, B and C, with + dependencies A1, B1 and C1, where A1 and B1 are incompatible: the + last state might be related to C, so we want to discard the + previous state. + 2. Discard Z. + 3. Discard Y but remember its incompatibility information gathered + previously, and the failure we're dealing with right now. + 4. Push a new state Y' based on X, and apply the incompatibility + information from Y to Y'. + 5a. If this causes Y' to conflict, we need to backtrack again. Make Y' + the new Z and go back to step 2. + 5b. If the incompatibilities apply cleanly, end backtracking. + """ + incompatible_reqs: Iterable[CT | RT] = itertools.chain( + (c.parent for c in causes if c.parent is not None), + (c.requirement for c in causes), + ) + incompatible_deps = {self._p.identify(r) for r in incompatible_reqs} + while len(self._states) >= 3: + # Remove the state that triggered backtracking. + del self._states[-1] + + # Optimistically backtrack to a state that caused the incompatibility + broken_state = self.state + while True: + # Retrieve the last candidate pin and known incompatibilities. + try: + broken_state = self._states.pop() + name, candidate = broken_state.mapping.popitem() + except (IndexError, KeyError): + raise ResolutionImpossible(causes) from None + + # Only backjump if the current broken state is + # an incompatible dependency + if name not in incompatible_deps: + break + + # If the current dependencies and the incompatible dependencies + # are overlapping then we have found a cause of the incompatibility + current_dependencies = { + self._p.identify(d) for d in self._p.get_dependencies(candidate) + } + if not current_dependencies.isdisjoint(incompatible_deps): + break + + # Fallback: We should not backtrack to the point where + # broken_state.mapping is empty, so stop backtracking for + # a chance for the resolution to recover + if not broken_state.mapping: + break + + incompatibilities_from_broken = [ + (k, list(v.incompatibilities)) for k, v in broken_state.criteria.items() + ] + + # Also mark the newly known incompatibility. + incompatibilities_from_broken.append((name, [candidate])) + + self._push_new_state() + success = self._patch_criteria(incompatibilities_from_broken) + + # It works! Let's work on this new state. + if success: + return True + + # State does not work after applying known incompatibilities. + # Try the still previous state. + + # No way to backtrack anymore. + return False + + def _extract_causes( + self, criteron: list[Criterion[RT, CT]] + ) -> list[RequirementInformation[RT, CT]]: + """Extract causes from list of criterion and deduplicate""" + return list({id(i): i for c in criteron for i in c.information}.values()) + + def resolve(self, requirements: Iterable[RT], max_rounds: int) -> State[RT, CT, KT]: + if self._states: + raise RuntimeError("already resolved") + + self._r.starting() + + # Initialize the root state. + self._states = [ + State( + mapping=collections.OrderedDict(), + criteria={}, + backtrack_causes=[], + ) + ] + for r in requirements: + try: + self._add_to_criteria(self.state.criteria, r, parent=None) + except RequirementsConflicted as e: + raise ResolutionImpossible(e.criterion.information) from e + + # The root state is saved as a sentinel so the first ever pin can have + # something to backtrack to if it fails. The root state is basically + # pinning the virtual "root" package in the graph. + self._push_new_state() + + for round_index in range(max_rounds): + self._r.starting_round(index=round_index) + + unsatisfied_names = [ + key + for key, criterion in self.state.criteria.items() + if not self._is_current_pin_satisfying(key, criterion) + ] + + # All criteria are accounted for. Nothing more to pin, we are done! + if not unsatisfied_names: + self._r.ending(state=self.state) + return self.state + + # keep track of satisfied names to calculate diff after pinning + satisfied_names = set(self.state.criteria.keys()) - set(unsatisfied_names) + + # Choose the most preferred unpinned criterion to try. + name = min(unsatisfied_names, key=self._get_preference) + failure_criterion = self._attempt_to_pin_criterion(name) + + if failure_criterion: + causes = self._extract_causes(failure_criterion) + # Backjump if pinning fails. The backjump process puts us in + # an unpinned state, so we can work on it in the next round. + self._r.resolving_conflicts(causes=causes) + success = self._backjump(causes) + self.state.backtrack_causes[:] = causes + + # Dead ends everywhere. Give up. + if not success: + raise ResolutionImpossible(self.state.backtrack_causes) + else: + # discard as information sources any invalidated names + # (unsatisfied names that were previously satisfied) + newly_unsatisfied_names = { + key + for key, criterion in self.state.criteria.items() + if key in satisfied_names + and not self._is_current_pin_satisfying(key, criterion) + } + self._remove_information_from_criteria( + self.state.criteria, newly_unsatisfied_names + ) + # Pinning was successful. Push a new state to do another pin. + self._push_new_state() + + self._r.ending_round(index=round_index, state=self.state) + + raise ResolutionTooDeep(max_rounds) + + +class Resolver(AbstractResolver[RT, CT, KT]): + """The thing that performs the actual resolution work.""" + + base_exception = ResolverException + + def resolve( # type: ignore[override] + self, + requirements: Iterable[RT], + max_rounds: int = 100, + ) -> Result[RT, CT, KT]: + """Take a collection of constraints, spit out the resolution result. + + The return value is a representation to the final resolution result. It + is a tuple subclass with three public members: + + * `mapping`: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * `graph`: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + * `criteria`: A dict of "criteria" that hold detailed information on + how edges in the graph are derived. Each key is an identifier of a + requirement, and the value is a `Criterion` instance. + + The following exceptions may be raised if a resolution cannot be found: + + * `ResolutionImpossible`: A resolution cannot be found for the given + combination of requirements. The `causes` attribute of the + exception is a list of (requirement, parent), giving the + requirements that could not be satisfied. + * `ResolutionTooDeep`: The dependency tree is too deeply nested and + the resolver gave up. This is usually caused by a circular + dependency, but you can try to resolve this by increasing the + `max_rounds` argument. + """ + resolution = Resolution(self.provider, self.reporter) + state = resolution.resolve(requirements, max_rounds=max_rounds) + return _build_result(state) + + +def _has_route_to_root( + criteria: Mapping[KT, Criterion[RT, CT]], + key: KT | None, + all_keys: dict[int, KT | None], + connected: set[KT | None], +) -> bool: + if key in connected: + return True + if key not in criteria: + return False + assert key is not None + for p in criteria[key].iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey in connected: + connected.add(key) + return True + if _has_route_to_root(criteria, pkey, all_keys, connected): + connected.add(key) + return True + return False diff --git a/src/resolvelib/structs.py b/src/resolvelib/structs.py index 930767d..18c74d4 100644 --- a/src/resolvelib/structs.py +++ b/src/resolvelib/structs.py @@ -5,7 +5,6 @@ from typing import ( TYPE_CHECKING, Callable, - Collection, Generic, Iterable, Iterator, @@ -23,6 +22,7 @@ Matches = Union[Iterable[CT], Callable[[], Iterable[CT]]] if TYPE_CHECKING: + from .resolvers.criterion import Criterion class RequirementInformation(NamedTuple, Generic[RT, CT]): requirement: RT @@ -207,46 +207,3 @@ def build_iter_view(matches: Matches[CT]) -> Iterable[CT]: IterableView = Iterable - - -class Criterion(Generic[RT, CT]): - """Representation of possible resolution results of a package. - - This holds three attributes: - - * `information` is a collection of `RequirementInformation` pairs. - Each pair is a requirement contributing to this criterion, and the - candidate that provides the requirement. - * `incompatibilities` is a collection of all known not-to-work candidates - to exclude from consideration. - * `candidates` is a collection containing all possible candidates deducted - from the union of contributing requirements and known incompatibilities. - It should never be empty, except when the criterion is an attribute of a - raised `RequirementsConflicted` (in which case it is always empty). - - .. note:: - This class is intended to be externally immutable. **Do not** mutate - any of its attribute containers. - """ - - def __init__( - self, - candidates: Iterable[CT], - information: Collection[RequirementInformation[RT, CT]], - incompatibilities: Collection[CT], - ) -> None: - self.candidates = candidates - self.information = information - self.incompatibilities = incompatibilities - - def __repr__(self) -> str: - requirements = ", ".join( - f"({req!r}, via={parent!r})" for req, parent in self.information - ) - return f"Criterion({requirements})" - - def iter_requirement(self) -> Iterator[RT]: - return (i.requirement for i in self.information) - - def iter_parent(self) -> Iterator[CT | None]: - return (i.parent for i in self.information) diff --git a/tests/functional/cocoapods/test_resolvers_cocoapods.py b/tests/functional/cocoapods/test_resolvers_cocoapods.py index c1312cb..cfce99b 100644 --- a/tests/functional/cocoapods/test_resolvers_cocoapods.py +++ b/tests/functional/cocoapods/test_resolvers_cocoapods.py @@ -7,7 +7,8 @@ import pytest -from resolvelib import AbstractProvider, ResolutionImpossible, Resolver +from resolvelib import AbstractProvider, ResolutionImpossible +from resolvelib.resolvers.resolution import Resolver Requirement = collections.namedtuple("Requirement", "name spec") Candidate = collections.namedtuple("Candidate", "name ver deps") diff --git a/tests/functional/python/test_resolvers_python.py b/tests/functional/python/test_resolvers_python.py index 18c1550..2c6496e 100644 --- a/tests/functional/python/test_resolvers_python.py +++ b/tests/functional/python/test_resolvers_python.py @@ -11,7 +11,8 @@ import packaging.version import pytest -from resolvelib import AbstractProvider, ResolutionImpossible, Resolver +from resolvelib import AbstractProvider, ResolutionImpossible +from resolvelib.resolvers.resolution import Resolver Candidate = collections.namedtuple("Candidate", "name version extras") diff --git a/tests/functional/swift-package-manager/test_resolvers_swift.py b/tests/functional/swift-package-manager/test_resolvers_swift.py index ce9b73e..66d45a1 100644 --- a/tests/functional/swift-package-manager/test_resolvers_swift.py +++ b/tests/functional/swift-package-manager/test_resolvers_swift.py @@ -5,8 +5,8 @@ import pytest -from resolvelib import Resolver from resolvelib.providers import AbstractProvider +from resolvelib.resolvers.resolution import Resolver Requirement = collections.namedtuple("Requirement", "container constraint") Candidate = collections.namedtuple("Candidate", "container version") diff --git a/tests/test_resolvers.py b/tests/test_resolvers.py index 3c0ff5c..af13475 100644 --- a/tests/test_resolvers.py +++ b/tests/test_resolvers.py @@ -12,15 +12,14 @@ BaseReporter, InconsistentCandidate, ResolutionImpossible, - Resolver, ) -from resolvelib.resolvers.criterion import Resolution +from resolvelib.resolvers.criterion import Criterion +from resolvelib.resolvers.resolution import Resolution, Resolver if TYPE_CHECKING: from typing import Iterable, Mapping - from resolvelib.resolvers.criterion import ( - Criterion, + from resolvelib.resolvers.exceptions import ( RequirementInformation, RequirementsConflicted, )