-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
backend/ninja: use a two step process for dependency scanning
This splits the scanner into two discrete steps, one that scans the source files, and one that that reads in the dependency information and produces a dyndep. The scanner uses the JSON format from https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html, which is the same format the MSVC and Clang use for C++ modules scanning. This will allow us to more easily move to using MSVC and clang-scan-deps when possible. As an added bonus, this correctly tracks dependencies across TU and Target boundaries, unlike the previous implementation, which assumed that if it couldn't find a provider that everything was good, but could run into issues. Because of that limitation Fortran code had to fully depend on all of it's dependencies, transitive or not. Now, when using the dep scanner, we can remove that restriction, allowing more parallelism.
- Loading branch information
Showing
3 changed files
with
254 additions
and
66 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # Copyright © 2021 Intel Corporation | ||
|
|
||
| """Accumulator for p1689r5 module dependencies. | ||
| See: https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
| import json | ||
| import re | ||
| import textwrap | ||
| import typing as T | ||
|
|
||
| if T.TYPE_CHECKING: | ||
| from .depscan import Description, Rule | ||
|
|
||
| # The quoting logic has been copied from the ninjabackend to avoid having to | ||
| # import half of Meson just to quote outputs, which is a performance problem | ||
| _QUOTE_PAT = re.compile(r"[$ :\n]") | ||
|
|
||
|
|
||
| def quote(text: str) -> str: | ||
| # Fast path for when no quoting is necessary | ||
| if not _QUOTE_PAT.search(text): | ||
| return text | ||
| if '\n' in text: | ||
| errmsg = textwrap.dedent(f'''\ | ||
| Ninja does not support newlines in rules. The content was: | ||
| {text} | ||
| Please report this error with a test case to the Meson bug tracker.''') | ||
| raise RuntimeError(errmsg) | ||
| return _QUOTE_PAT.sub(r'$\g<0>', text) | ||
|
|
||
|
|
||
| _PROVIDER_CACHE: T.Dict[str, str] = {} | ||
|
|
||
|
|
||
| def get_provider(rules: T.List[Rule], name: str) -> T.Optional[str]: | ||
| """Get the object that a module from another Target provides | ||
| We must rely on the object file here instead of the module itself, because | ||
| the object rule is part of the generated build.ninja, while the module is | ||
| only declared inside a dyndep. This creates for the dyndep generator to | ||
| depend on previous dyndeps as order deps. Since the module | ||
| interface file will be generated when the object is generated we can rely on | ||
| that in proxy and simplify generation. | ||
| :param rules: The list of rules to check | ||
| :param name: The logical-name to look for | ||
| :raises RuntimeError: If no provider can be found | ||
| :return: The object file of the rule providing the module | ||
| """ | ||
| # Cache the result for performance reasons | ||
| if name in _PROVIDER_CACHE: | ||
| return _PROVIDER_CACHE[name] | ||
|
|
||
| for r in rules: | ||
| for p in r.get('provides', []): | ||
| if p['logical-name'] == name: | ||
| obj = r['primary-output'] | ||
| _PROVIDER_CACHE[name] = obj | ||
| return obj | ||
| return None | ||
|
|
||
|
|
||
| def process_rules(rules: T.List[Rule], | ||
| extra_rules: T.List[Rule], | ||
| ) -> T.Iterable[T.Tuple[str, T.Optional[T.List[str]], T.Optional[T.List[str]]]]: | ||
| """Process the rules for this Target | ||
| :param rules: the rules for this target | ||
| :param extra_rules: the rules for all of the targets this one links with, to use their provides | ||
| :yield: A tuple of the output, the exported modules, and the consumed modules | ||
| """ | ||
| for rule in rules: | ||
| prov: T.Optional[T.List[str]] = None | ||
| req: T.Optional[T.List[str]] = None | ||
| if 'provides' in rule: | ||
| prov = [p['compiled-module-path'] for p in rule['provides']] | ||
| if 'requires' in rule: | ||
| req = [] | ||
| for p in rule['requires']: | ||
| modfile = p.get('compiled-module-path') | ||
| if modfile is not None: | ||
| req.append(modfile) | ||
| else: | ||
| # We can't error if this is not found because of compiler | ||
| # provided modules | ||
| found = get_provider(extra_rules, p['logical-name']) | ||
| if found: | ||
| req.append(found) | ||
| yield rule['primary-output'], prov, req | ||
|
|
||
|
|
||
| def format(files: T.Optional[T.List[str]]) -> str: | ||
| if files: | ||
| fmt = " ".join(quote(f) for f in files) | ||
| return f'| {fmt}' | ||
| return '' | ||
|
|
||
|
|
||
| def gen(outfile: str, desc: Description, extra_rules: T.List[Rule]) -> int: | ||
| with open(outfile, 'w', encoding='utf-8') as f: | ||
| f.write('ninja_dyndep_version = 1\n\n') | ||
|
|
||
| for obj, provides, requires in process_rules(desc['rules'], extra_rules): | ||
| ins = format(requires) | ||
| out = format(provides) | ||
| f.write(f'build {quote(obj)} {out}: dyndep {ins}\n\n') | ||
|
|
||
| return 0 | ||
|
|
||
|
|
||
| def run(args: T.List[str]) -> int: | ||
| assert len(args) >= 2, 'got wrong number of arguments!' | ||
| outfile, jsonfile, *jsondeps = args | ||
| with open(jsonfile, 'r', encoding='utf-8') as f: | ||
| desc: Description = json.load(f) | ||
|
|
||
| # All rules, necessary for fulfilling across TU and target boundaries | ||
| rules = desc['rules'].copy() | ||
| for dep in jsondeps: | ||
| with open(dep, encoding='utf-8') as f: | ||
| d: Description = json.load(f) | ||
| rules.extend(d['rules']) | ||
|
|
||
| return gen(outfile, desc, rules) |
Oops, something went wrong.