New approach to squashfs images

bin/cefs

@@ -0,0 +1 @@
+scripts/python_module_shim

bin/lib/ce_install.py

@@ -16,10 +16,13 @@
 import yaml
 
 from lib.amazon_properties import get_properties_compilers_and_libraries
+from lib.cefs.config import CefsConfig
+from lib.cefs.root import CefsFsRoot
+from lib.cefs.squash import SquashFsCreator
 from lib.config_safe_loader import ConfigSafeLoader
+from lib.installable.installable import Installable
 from lib.installation import installers_for
 from lib.installation_context import InstallationContext
-from lib.installable.installable import Installable
 from lib.library_yaml import LibraryYaml
 
 _LOGGER = logging.getLogger(__name__)
@@ -206,7 +209,7 @@ def cli(
         file_handler.setFormatter(formatter)
         root_logger.addHandler(file_handler)
     if not log or log_to_console:
-        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler = logging.StreamHandler(sys.stderr)
         console_handler.setFormatter(formatter)
         root_logger.addHandler(console_handler)
     context = InstallationContext(
@@ -228,12 +231,15 @@ def cli(
 
 @cli.command(name="list")
 @click.pass_obj
+@click.option("--json", is_flag=True, help="Output in JSON format")
+@click.option("--installed-only", is_flag=True, help="Only output installed targets")
 @click.argument("filter_", metavar="FILTER", nargs=-1)
-def list_cmd(context: CliContext, filter_: List[str]):
+def list_cmd(context: CliContext, filter_: List[str], json: bool, installed_only: bool):
     """List installation targets matching FILTER."""
-    print("Installation candidates:")
     for installable in context.get_installables(filter_):
-        print(installable.name)
+        if installed_only and not installable.is_installed():
+            continue
+        print(installable.to_json() if json else installable.name)
         _LOGGER.debug(installable)
 
 
@@ -341,6 +347,78 @@ def squash(context: CliContext, filter_: List[str], force: bool, image_dir: Path
             installable.squash_to(destination)
 
 
+CEFS_ROOT = Path("/cefs")
+
+
+@cli.command()
+@click.pass_obj
+@click.option("--force", is_flag=True, help="Force even if would otherwise skip")
+@click.option(
+    "--cefs-mountpoint",
+    default=Path("/cefs"),
+    metavar="MOUNTPOINT",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Install or assume cefs is to use MOUNTPOINT",
+    show_default=True,
+)
+@click.option(
+    "--squash-image-root",
+    default=Path("/opt/cefs-images"),
+    metavar="IMAGE_DIR",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Store or look for squashfs images in IMAGE_DIR",
+    show_default=True,
+)
+@click.argument("filter_", metavar="FILTER", nargs=-1)
+def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image_root: Path, cefs_mountpoint: Path):
+    """Squash all things matching to a single layer."""
+
+    installation_context = context.installation_context
+    cefs_config = CefsConfig(mountpoint=cefs_mountpoint, image_root=squash_image_root)
+    fs_root = CefsFsRoot(fs_root=installation_context.destination, config=cefs_config)
+
+    current_image = fs_root.read_image()
+    to_install = []
+    for installable in context.get_installables(filter_):
+        if force or installable.should_install():
+            to_install.append(installable)
+            dest_path = installation_context.destination / installable.install_path
+            if dest_path.exists() and not dest_path.is_symlink():
+                _LOGGER.error("Found an installable that wasn't a symlink: %s", dest_path)
+                sys.exit(1)
+
+    install_creator = SquashFsCreator(config=cefs_config)
+    with install_creator.creation_path() as tmp_path:
+        _LOGGER.info("Installing everything to a temp dir: %s", tmp_path)
+        installation_context.set_temp_destination(tmp_path)
+        num_installed = 0
+        for installable in to_install:
+            installable.install()
+            if not installable.is_installed():
+                _LOGGER.error("%s installed OK, but doesn't appear as installed after", installable.name)
+                sys.exit(1)
+            current_image.add_metadata(f"Installing {installable.install_path} from {installable}")
+            num_installed += 1
+
+    if not num_installed:
+        click.echo("No changes: not updating base image")
+        return
+
+    new_squashfs_cefs = install_creator.cefs_path
+    for installable in to_install:
+        current_image.link_path(Path(installable.install_path), new_squashfs_cefs / installable.install_path)
+
+    _LOGGER.info("Building new root fs")
+    root_creator = SquashFsCreator(config=cefs_config)
+    with root_creator.creation_path() as tmp_path:
+        current_image.render_to(tmp_path)
+
+    new_squashfs_cefs = root_creator.cefs_path
+
+    click.echo(f"Built to {new_squashfs_cefs}")
+    fs_root.update(new_squashfs_cefs)
+
+
 @cli.command()
 @click.pass_obj
 @click.option(

bin/lib/cefs.md

@@ -0,0 +1,61 @@
+## CEFS
+
+The Compiler Explorer File System manages a read-only, layered, hierarchy of squashfs images.
+It's designed for compilers and libraries in Compiler Explorer, but any read-only content would work. It's not really a file system, either. So it's a terrible name. But hey!
+
+### Why!
+
+We use immutable squashfs images on NFS for the compilers and libraries. That's great, fast performance, great caching behaviour. But, mounting each compiler squashfs individually is slow for system startup (~1m for ~1000). We ideally want fewer squashfs mounts, but with the ability to "overlay" changes (like daily builds) without totally rebuilding everything every time.
+
+### Overview
+
+There is an autofs mount at `/cefs` which automounts squashfs images of the form `/cefs/ID` to `/opt/cefs-images/ID.sqfs`. The filesystem root is a symlink to a squashfs image in cefs. The root itself is a number of symlinks - one per managed directory - to other locations in cefs.
+
+As an example:
+
+- `/opt/compiler-explorer` would be the root, and is a symlink to `/cefs/SOME-ROOT-ID`.
+- `/cefs/SOME_ROOT_ID` maps to a squashfs image `/opt/cefs-images/SOME_ROOT_ID.sqfs`.
+- `/opt/cefs-images/SOME_ROOT_ID.sqfs` contains symlinks for each compiler maybe something like:
+  - `gcc-12.1.0` -> `/cefs/SOME_INSTALL_ID/gcc-12.1.0`
+  - `gcc-12.2.0` -> `/cefs/SOME_INSTALL_ID/gcc-12.2.0`
+  - `gcc-trunk` -> `/cefs/DIFFERENT_ID/gcc-trunk`
+- Here we see the advantage of cefs: we minimize the number of actual squashfs images by relying on several installations being in the "same" squashfs image. We can consolidate images over time too.
+
+The expectation is there's one big squashfs install image holding all the stable compilers, regenerated monthly. And then several overlaying images, some for ad hoc builds of new compilers (e.g. new releases of GCC), and some for the daily builds.
+
+## Using
+
+To set up you'll need to run `sudo cefs install`. (it needs root to fiddle with autofs)
+
+I locally link `/opt/compiler-explorer` to `~/ce` in this instance so I can update it quickly as a mortal user without making `/opt` writable to my user.
+
+To make an empty root like this:
+
+```
+$ sudo ln -sfT ~/ce /opt/compiler-explorer  # one-time setup only as root
+$ cefs create-root ~/ce
+...lots of output...
+Fresh new cefs root created at /cefs/662a3bd6f57ac05b28032dbb5dd2bb489e9761c7
+$ ls /opt/compiler-explorer
+metadata.txt
+$ cat /opt/compiler-explorer/metadata.txt
+Initial empty image created at 2022-09-05 22:56:03.530590 by mgodbolt
+```
+
+Now you have an empty base layer.  After that you can layer on things using `ce_install buildroot` (terrible name needs changing):
+
+```
+$ ce_install buildroot gcc # literally all the GCCs...
+[...lots of work...]
+$ ls -l /opt/compiler-explorer/
+$ ls -l /opt/compiler-explorer/
+lrwxrwxrwx 1 root root  57 Sep  5 18:02 gcc-12.1.0 -> /cefs/fad719bc481ff3dbaffc7f5ef804c4b5ffc281d1/gcc-12.1.0/
+lrwxrwxrwx 1 root root  57 Sep  5 18:02 gcc-12.2.0 -> /cefs/fad719bc481ff3dbaffc7f5ef804c4b5ffc281d1/gcc-12.2.0/
+...etc
+```
+
+## TODO
+
+- cleanup
+- consolidation
+- everything, really