Merge branch 'cut-over'

.aliases/verify-default-nix

@@ -9,24 +9,24 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$SCRIPTDIR/.."
 
 FLAKE_NIXPKGS_REV=$(nix flake metadata --json | jq -r .locks.nodes.nixpkgs.locked.rev)
-FLAKE_NIXPKGS_UNSTABLE_REV=$(nix flake metadata --json | jq -r '.locks.nodes["nixpkgs-unstable"].locked.rev')
+FLAKE_NIXPKGS_MASTER_REV=$(nix flake metadata --json | jq -r '.locks.nodes["nixpkgs-master"].locked.rev')
 
 echo "FLAKE_NIXPKGS_REV: $FLAKE_NIXPKGS_REV"
-echo "FLAKE_NIXPKGS_UNSTABLE_REV: $FLAKE_NIXPKGS_UNSTABLE_REV"
+echo "FLAKE_NIXPKGS_MASTER_REV: $FLAKE_NIXPKGS_MASTER_REV"
 
 DEFAULT_NIXPKGS_REV=$(cat default.nix | grep nixpkgs-rev | cut -d'"' -f2)
-DEFAULT_NIXPKGS_UNSTABLE_REV=$(cat default.nix | grep nixpkgs-unstable-rev | cut -d'"' -f2)
+DEFAULT_NIXPKGS_MASTER_REV=$(cat default.nix | grep nixpkgs-master-rev | cut -d'"' -f2)
 
 echo "DEFAULT_NIXPKGS_REV: $DEFAULT_NIXPKGS_REV"
-echo "DEFAULT_NIXPKGS_UNSTABLE_REV: $DEFAULT_NIXPKGS_UNSTABLE_REV"
+echo "DEFAULT_NIXPKGS_MASTER_REV: $DEFAULT_NIXPKGS_MASTER_REV"
 
 
 if [ "$FLAKE_NIXPKGS_REV" != "$DEFAULT_NIXPKGS_REV" ]; then
     echo "nixpkgs rev differed between flake and default.nix"
     exit 1
 fi
 
-if [ "$FLAKE_NIXPKGS_UNSTABLE_REV" != "$DEFAULT_NIXPKGS_UNSTABLE_REV" ]; then
-    echo "nixpkgs-unstable rev differed between flake and default.nix"
+if [ "$FLAKE_NIXPKGS_MASTER_REV" != "$DEFAULT_NIXPKGS_MASTER_REV" ]; then
+    echo "nixpkgs-master rev differed between flake and default.nix"
     exit 1
 fi

.github/workflows/ci.yml

@@ -16,6 +16,28 @@ jobs:
       if: "!contains(github.event.head_commit.message, 'noci')"
       run: ./.aliases/verify-default-nix
 
+  build-ui-metadata:
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build .#ui_metadata_farm
+      if: "!contains(github.event.head_commit.message, 'noci')"
+      run: |
+        nix build .#ui_metadata_farm
+
+  build-sample-environments:
+    runs-on: self-hosted
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build .#sample_environments_farm
+      if: "!contains(github.event.head_commit.message, 'noci')"
+      run: |
+        nix build .#sample_environments_farm
+
   test:
     name: ${{matrix.suite.name}} (${{matrix.suite.flags}})
     runs-on: self-hosted
@@ -37,13 +59,15 @@ jobs:
         - name: haskell
           flags: "-f haskell-ghc98"
         - name: julia
+          flags: "-f julia_19"
+        - name: julia
+          flags: "-f julia_110"
         - name: octave
         - name: postgres
         - name: python
         - name: r
         - name: ruby
         - name: rust
-        - name: sample-environments
         - name: searchers
         - name: zsh
         - name: spellchecker
@@ -58,7 +82,7 @@ jobs:
       run: |
         cd tests
 
-        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsUnstablePath)
+        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsPath)
 
         export PATH=$(nix build --no-link .#stack.x86_64-linux --json | jq -r '.[0].outputs.out')/bin:$PATH
         echo "Got path: $PATH"
@@ -74,7 +98,7 @@ jobs:
         echo "Got TEST_ROOT: $TEST_ROOT"
         echo TEST_ROOT="$TEST_ROOT" >> $GITHUB_ENV
 
-        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsUnstablePath)
+        export NIX_PATH=nixpkgs=$(nix run .#nixpkgsPath)
         export PATH=$(nix build --no-link .#stack.x86_64-linux --json | jq -r '.[0].outputs.out')/bin:$PATH
 
         stack run --nix --no-nix-pure -- \

codedown.nix

@@ -1,99 +1,68 @@
 { pkgsStable
-, pkgsUnstable
 , pkgsMaster
 , requiredPackages ? []
 , system ? "x86_64-linux"
 }:
 
 let
-  common = pkgsStable.callPackage ./languages/common.nix {};
+  common = pkgsStable.callPackage ./modules/languages/common.nix {};
 
   callPackage = pkgsStable.callPackage;
 
-  # Languages
-  # First argument controls whether attributes get filtered to the valid ones.
-  # This can be expensive to evaluate for languages like Haskell where there are tons of
-  # Stackage snapshots and one nix file for each. So, we don't bother with that when evaluating
-  # the languages attrset normally--only when building the languagesSearcher.
-  languagesFn = filterToValid: pkgsStable.lib.zipAttrsWith (n: v: pkgsStable.lib.head v) [
-    (callPackage ./languages/bash {})
-    (callPackage ./languages/clojure {})
-    (callPackage ./languages/coq {})
-    (pkgsMaster.callPackage ./languages/cpp {})
-    (callPackage ./languages/go {})
-    (pkgsMaster.callPackage ./languages/haskell {})
-    (callPackage ./languages/julia {})
-    (callPackage ./languages/octave {})
-    (callPackage ./languages/postgres {})
-    (callPackage ./languages/python {
-      poetry2nix = import (pkgsStable.fetchFromGitHub {
-        owner = "nix-community";
-        repo = "poetry2nix";
-        rev = "78fc8882411c29c8eb5f162b09fcafe08b8b03a3";
-        sha256 = "1dfgm286c48ac6yrk16xz41d0rsg6bv08122ngy420b0z88la9nj";
-      }) {
-        pkgs = pkgsStable;
-      };
-    })
-    (callPackage ./languages/r {})
-    (pkgsMaster.callPackage ./languages/ruby {})
-    (pkgsMaster.callPackage ./languages/rust {})
-  ];
-
   lib = pkgsStable.lib;
 
 in
 
 rec {
-  spellchecker = pkgsUnstable.callPackage ./language_servers/markdown-spellcheck-lsp {};
-
-  languages = languagesFn false;
-
-  shells = {
-    zsh = callPackage ./shells/zsh {};
-    fish = callPackage ./shells/fish {};
-    bash = callPackage ./shells/bash {};
-  };
-
-  exporters = {
-    nbconvert-small = pkgsMaster.callPackage ./exporters/nbconvert.nix { texliveScheme = pkgsStable.texlive.combined.scheme-small; };
-    nbconvert-large = pkgsMaster.callPackage ./exporters/nbconvert.nix { texliveScheme = pkgsStable.texlive.combined.scheme-full; };
-  };
+  spellchecker = pkgsMaster.callPackage ./modules/language_servers/markdown-spellcheck-lsp {};
 
   testing = {
-    builds-forever = pkgsMaster.callPackage ./misc/builds-forever.nix {};
+    builds-forever = pkgsMaster.callPackage ./modules/testing/builds-forever.nix {};
   };
 
   # Exported so clients can build searchers for other package sets, like "codedown.searcher nixpkgs"
   searcher = common.searcher;
 
-  codedownSearcher = common.searcher' {
-    # Note that we deliberately don't include "testing" packages in the searcher
-    packages = languagesFn true
-      // (lib.mapAttrs' (n: v: lib.nameValuePair ("shells." + n) v) shells)
-      // (lib.mapAttrs' (n: v: lib.nameValuePair ("exporters." + n) v) exporters)
-      // { inherit spellchecker; };
-  };
+  settingsSchemas = lib.mapAttrs (attr: value: value.meta.settingsSchema or []) languages;
 
-  languagesIcons = common.searcherIcons' {
-    packages = languagesFn true;
-    packageMustBeDerivation = false;
+  evaluateConfig = callPackage ./nix/evaluate-config.nix {
+    inherit pkgsStable pkgsMaster;
   };
 
-  settingsSchemas = lib.mapAttrs (attr: value:
-    common.safeEval (lib.attrByPath ["meta" "settingsSchema"] [] value)
-  ) languages;
+  everythingConfig = let
+    base = evaluateConfig {};
+    kernelNames = builtins.attrNames base.options.kernels;
+    shellNames = builtins.attrNames base.options.shells;
+    exporterNames = builtins.attrNames base.options.exporters;
+  in
+    builtins.foldl' lib.recursiveUpdate {} (
+      (map (n: { kernels.${n}.enable = true; }) kernelNames)
+      ++ (map (n: { shells.${n}.enable = true; }) shellNames)
+      ++ (map (n: { exporters.${n}.enable = true; }) exporterNames)
+    );
 
-  mkCodeDownEnvironment = callPackage ./codedown/mkCodeDownEnvironment.nix {
-    inherit requiredPackages languages;
+  everythingEnv = evaluateConfig everythingConfig;
+
+  codedownSearcher = common.searcher' {
+    # Note that we deliberately don't include "testing" packages in the searcher
+    packages = everythingEnv.config.builtKernels
+      // (lib.mapAttrs' (n: v: lib.nameValuePair ("shells." + n) v) everythingEnv.config.builtShells)
+      // (lib.mapAttrs' (n: v: lib.nameValuePair ("exporters." + n) v) everythingEnv.config.builtExporters)
+      // { "language-servers.spellchecker" = spellchecker; }
+    ;
   };
 
-  makeEnvironment = callPackage ./codedown/makeEnvironment.nix {
-    inherit pkgsStable pkgsUnstable pkgsMaster;
+  languages = everythingEnv.config.builtKernels;
+
+  makeEnvironment = callPackage ./nix/makeEnvironment.nix {
+    inherit pkgsStable pkgsMaster;
   };
 
-  validateCodeDownEnvironment = callPackage ./codedown/validateCodeDownEnvironment.nix {};
+  validateEnvironment = callPackage ./nix/validateEnvironment.nix {};
+
+  # Exposed for consumers to pin and use to gather metadata from other channels like Nixpkgs
+  inherit (callPackage ./nix/uiMetadata.nix {}) chooseInterestingMeta;
 
   # Exposed so it's easier to compute build dependencies in the presence of IFD
-  inherit pkgsStable pkgsUnstable requiredPackages;
+  inherit pkgsStable pkgsMaster requiredPackages;
 }