Fix #9815: fix caching for quick-jobs CI (XDG, cache keys) (backport #…

…9845) (#10041) * Fix #9815: switch quick-jobs CI to XDG Fix #9815: - Cache `~/.local/state/cabal` instead of `~/.cabal/store` - `~/.local/bin` is used instead of `~/.cabal/bin` and is already in the PATH (verify this by calling `alex` after installing it) As I am passing by: - bump cache action to v4 - double-quote `$USER` to keep actionlint happy - move `if` from shell-level to job-level - allow newest `alex` (cherry picked from commit e916cb5) * CI quick-jobs: use preinstalled GHC and Cabal (cherry picked from commit c209a82) * Makefile: remove dead target 'lexer', use '.PHONY' systematically The `lexer` target was removed in #8980 (cherry picked from commit e600087) * CI "Meta checks": correct cache key (cherry picked from commit 56426e4) * CI "Meta checks": print Haskell versions (cherry picked from commit 9a311bd) * CI "Doctest Cabal": daily refresh of cache (cherry picked from commit ba6f6ff) * CI "Check Field Syntax Reference": correct cache key (cherry picked from commit 5949e3f) * Update generated Cabal/src/Distribution/Simple/Build/Macros/Z.hs Not sure why this was not up to date on master and still CI passed. Maybe the content of this file is dependent on the GHC version we are using to build the `get-cabal-macros` tool? (cherry picked from commit 947860a) * CI quick-jobs: entirely wipe ghcup directory rights workaround (cherry picked from commit 5aa8afd) * !fixup --------- Co-authored-by: Andreas Abel <andreas.abel@ifi.lmu.de> Co-authored-by: Artem Pelenitsyn <a.pelenitsyn@gmail.com>
haskell · May 23, 2024 · f2999b3 · f2999b3
1 parent ae8776e
commit f2999b3
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 26 deletions.
diff --git a/.github/workflows/quick-jobs.yml b/.github/workflows/quick-jobs.yml
@@ -21,47 +21,73 @@ jobs:
   meta:
     name: Meta checks
     runs-on: ubuntu-latest
+    env:
+      cabal_build: >-
+        cabal build --builddir=dist-newstyle-meta --project-file=cabal.project.meta
+        gen-cabal-macros
+        gen-paths-module
+        gen-spdx
+        gen-spdx-exc
     # This job is not run in a container, any recent GHC should be fine
     steps:
-      - name: Set PATH
-        # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#adding-a-system-path
-        run: |
-          echo "$HOME/.cabal/bin" >> $GITHUB_PATH
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cabal/store
-          key: linux-store-meta
       - name: ghcup
         run: |
           ghcup --version
           ghcup config set cache true
           ghcup install ghc $GHC_FOR_QUICK_JOBS
           ghcup set ghc $GHC_FOR_QUICK_JOBS
+      - name: Haskell versions
+        run: |
+          ghc --version
+          cabal --version
       - name: Update Hackage index
         run: cabal v2-update
-      - name: Install alex
-        run: cabal v2-install alex --constraint='alex ==3.2.7.3'
       - uses: actions/checkout@v4
+      - name: Generate build plan for correct cache key
+        run: ${{ env.cabal_build }} --dry-run
+      - name: Restore cached dependencies
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ~/.local/state/cabal
+          key: linux-store-meta-${{ hashfiles('dist-newstyle-meta/cache/plan.json') }}
+          restore-keys: linux-store-meta-
+      - name: Build tools
+        run: ${{ env.cabal_build }}
       - name: Regenerate files
         run: |
-          make -B lexer
           make -B spdx
           make -B templates
       - name: Check that diff is clean
         run: |
           git status > /dev/null
           git diff-files -p --exit-code
+      - name: Cache dependencies
+        uses: actions/cache/save@v4
+        if: always() && steps.cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/.local/state/cabal
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+
   doctest:
     name: Doctest Cabal
     runs-on: ubuntu-latest
     steps:
-      - name: Set PATH
+      # It is complicated to get a proper cache key for the dependencies of a package
+      # (here: doctest) that we just `cabal install`.
+      # So, as a heuristics we update the cache once per day.
+      # Updating it with each run would be an alternative, but we a short of cache space,
+      # and this would generate too many new caches.
+      - name: Use date as cache key
         run: |
-          echo "$HOME/.cabal/bin" >> $GITHUB_PATH
-      - uses: actions/cache@v3
+          echo "DATE=$(date +'%Y-%m-%d')" >> "${GITHUB_ENV}"
+      - name: Restore cached dependencies
+        uses: actions/cache/restore@v4
+        id: cache
         with:
-          path: ~/.cabal/store
-          key: linux-store-doctest
+          path: ~/.local/state/cabal
+          key: linux-store-doctest-${{ env.DATE }}
+          restore-keys: linux-store-doctest
       - name: ghcup
         run: |
           ghcup --version
@@ -79,9 +105,18 @@ jobs:
         run: make doctest-install
       - name: Doctest
         run: make doctest
+      - name: Cache dependencies
+        if: always() && steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.local/state/cabal
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+
   buildinfo:
     name: Check Field Syntax Reference
     runs-on: ubuntu-latest
+    env:
+      cabal_build: cabal build buildinfo-reference-generator
     steps:
       - name: Set PATH
         run: |
@@ -103,8 +138,26 @@ jobs:
       - name: Update Hackage index
         run: cabal v2-update
       - uses: actions/checkout@v4
+      - name: Generate build plan for correct cache key
+        run: ${{ env.cabal_build }} --dry-run
+      - name: Restore cached dependencies
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ~/.local/state/cabal
+          key: linux-store-buildinfo-doc-diff-${{ hashfiles('dist-newstyle/cache/plan.json') }}
+          restore-keys: linux-store-buildinfo-doc-diff
+      - name: Build buildinfo-reference-generator
+        run: ${{ env.cabal_build }}
       - name: Are buildinfo docs up to date?
         run: make doc/buildinfo-fields-reference.rst
+      - name: Cache dependencies
+        uses: actions/cache/save@v4
+        if: always() && steps.cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/.local/state/cabal
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+
   release-project:
     name: Check Release Project
     runs-on: ubuntu-latest
@@ -129,4 +182,3 @@ jobs:
         run: cabal build all --dry-run --project-file=cabal.project.release
       - name: Check Release with Latest Hackage
         run: cabal build all --dry-run --project-file=cabal.project.release --index-state="hackage.haskell.org HEAD"
-
diff --git a/Cabal/src/Distribution/Simple/Build/Macros/Z.hs b/Cabal/src/Distribution/Simple/Build/Macros/Z.hs
@@ -3,8 +3,8 @@
 module Distribution.Simple.Build.Macros.Z (render, Z(..), ZPackage (..), ZTool (..)) where
 import Distribution.ZinzaPrelude
 data Z
-    = Z {zPackages :: ([ZPackage]),
-         zTools :: ([ZTool]),
+    = Z {zPackages :: [ZPackage],
+         zTools :: [ZTool],
          zPackageKey :: String,
          zComponentId :: String,
          zPackageVersion :: Version,

diff --git a/Makefile b/Makefile
@@ -1,5 +1,6 @@
-.PHONY : all lexer sdpx lib exe doctest
-.PHONY : phony
+.PHONY: phony
+  # Adding dependency "phony" is like declaring a target as ".PHONY":
+  # See https://www.gnu.org/software/make/manual/html_node/Force-Targets.html
 
 CABALBUILD := cabal build
 CABALRUN   := cabal run
@@ -13,25 +14,32 @@ DOCTEST := cabal repl --with-ghc=doctest --repl-options="-w" --ghc-options="-Wwa
 
 # default rules
 
+.PHONY: all
 all : exe lib
 
-lib : $(LEXER_HS)
+.PHONY: lib
+lib :
 	$(CABALBUILD) Cabal:libs
 
-exe : $(LEXER_HS)
+.PHONY: exe
+exe :
 	$(CABALBUILD) cabal-install:exes
 
+.PHONY: init
 init: ## Set up git hooks and ignored revisions
 	@git config core.hooksPath .githooks
 	## TODO
 
+.PHONY: style
 style: ## Run the code styler
 	@fourmolu -q -i Cabal Cabal-syntax cabal-install
 
+.PHONY: style-modified
 style-modified: ## Run the code styler on modified files
 	@git ls-files --modified Cabal Cabal-syntax cabal-install \
 		| grep '.hs$$' | xargs -P $(PROCS) -I {} fourmolu -q -i {}
 
+.PHONY: style-commit
 style-commit: ## Run the code styler on the previous commit
 	@git diff --name-only HEAD $(COMMIT) Cabal Cabal-syntax cabal-install \
 		| grep '.hs$$' | xargs -P $(PROCS) -I {} fourmolu -q -i {}
@@ -41,6 +49,10 @@ style-commit: ## Run the code styler on the previous commit
 SPDX_LICENSE_HS:=Cabal-syntax/src/Distribution/SPDX/LicenseId.hs
 SPDX_EXCEPTION_HS:=Cabal-syntax/src/Distribution/SPDX/LicenseExceptionId.hs
 
+# Note: the 'spdx' goal is used in .github/workflows/quick-jobs.yml.
+# Any changes to this goal need to be reconciled with this workflow.
+#
+.PHONY: spdx
 spdx : $(SPDX_LICENSE_HS) $(SPDX_EXCEPTION_HS)
 
 SPDX_LICENSE_VERSIONS:=3.0 3.2 3.6 3.9 3.10 3.16 3.23
@@ -56,7 +68,11 @@ $(SPDX_EXCEPTION_HS) : templates/SPDX.LicenseExceptionId.template.hs cabal-dev-s
 TEMPLATE_MACROS:=Cabal/src/Distribution/Simple/Build/Macros/Z.hs
 TEMPLATE_PATHS:=Cabal/src/Distribution/Simple/Build/PathsModule/Z.hs
 
-templates : phony $(TEMPLATE_MACROS) $(TEMPLATE_PATHS)
+# Note: the 'templates' goal is used in .github/workflows/quick-jobs.yml.
+# Any changes to this goal need to be reconciled with this workflow.
+#
+.PHONY: templates
+templates : $(TEMPLATE_MACROS) $(TEMPLATE_PATHS)
 
 $(TEMPLATE_MACROS) : templates/cabal_macros.template.h cabal-dev-scripts/src/GenCabalMacros.hs
 	cabal run --builddir=dist-newstyle-meta --project-file=cabal.project.meta gen-cabal-macros -- $< $@
@@ -75,18 +91,21 @@ doc/buildinfo-fields-reference.rst : \
 	cabal run buildinfo-reference-generator buildinfo-reference-generator/template.zinza | tee $@
 	git diff --exit-code $@
 
-# analyse-imports
-analyse-imports : phony
+.PHONY: analyse-imports
+analyse-imports :
 	find Cabal-syntax/src Cabal/src cabal-install/src -type f -name '*.hs' | xargs cabal run --builddir=dist-newstyle-meta --project-file=cabal.project.meta analyse-imports --
 
 # ghcid
 
+.PHONY: ghcid-lib
 ghcid-lib :
 	ghcid -c 'cabal repl Cabal'
 
+.PHONY: ghcid-cli
 ghcid-cli :
 	ghcid -c 'cabal repl cabal-install'
 
+.PHONY: doctest
 doctest :
 	$(DOCTEST) Cabal-syntax
 	$(DOCTEST) Cabal-described
@@ -95,46 +114,57 @@ doctest :
 	$(DOCTEST) cabal-install
 
 # This is not run as part of validate.sh (we need hackage-security, which is tricky to get).
+.PHONY: doctest-cli
 doctest-cli :
 	doctest -D__DOCTEST__ --fast cabal-install/src cabal-install-solver/src cabal-install-solver/src-assertion
 
+.PHONY: doctest-install
 doctest-install:
 	cabal install doctest --overwrite-policy=always --ignore-project
 
 # tests
 
+.PHONY: check-tests
 check-tests :
 	$(CABALRUN) check-tests -- --cwd Cabal-tests ${TEST}
 
+.PHONY: parser-tests
 parser-tests :
 	$(CABALRUN) parser-tests -- --cwd Cabal-tests ${TEST}
 
+.PHONY: parser-tests-accept
 parser-tests-accept :
 	$(CABALRUN) parser-tests -- --cwd Cabal-tests --accept ${TEST}
 
+.PHONY: custom-setup-tests
 custom-setup-tests :
 	$(CABALRUN) custom-setup-tests --
 
+.PHONY: hackage-parsec-tests
 hackage-parsec-tests :
 	$(CABALRUN) hackage-tests -- parsec +RTS -s -qg -I0 -A64M -N${THREADS} -RTS ${TEST}
 
+.PHONY: hackage-rountrip-tests
 hackage-roundtrip-tests :
 	$(CABALRUN) hackage-tests -- roundtrip +RTS -s -qg -I0 -A64M -N${THREADS} -RTS ${TEST}
 
+.PHONY: cabal-install-test
 cabal-install-test:
 	$(CABALBUILD) -j3 cabal-tests cabal
 	rm -rf .ghc.environment.*
 	cd cabal-testsuite && `cabal list-bin cabal-tests` --with-cabal=`cabal list-bin cabal` --hide-successes -j3 ${TEST}
 
 # hackage-benchmarks (solver)
 
+.PHONY: hackage-benchmarks-run
 hackage-benchmarks-run:
 	$(CABALBUILD) -j3 hackage-benchmark cabal
 	rm -rf .ghc.environment.*
 	$$(cabal list-bin hackage-benchmark) --cabal1=cabal --cabal2=$$(cabal list-bin cabal) --packages="hakyll servant-auth-server" --print-trials --concurrently
 
 
 # This doesn't run build, as you first need to test with cabal-install-test :)
+.PHONY: cabal-install-test-accept
 cabal-install-test-accept:
 	rm -rf .ghc.environment.*
 	cd cabal-testsuite && `cabal list-bin cabal-tests` --with-cabal=`cabal list-bin cabal` --hide-successes -j3 --accept ${TEST}
@@ -145,12 +175,14 @@ cabal-install-test-accept:
 #
 #   make validate-via-docker-all -j4 -O
 #
+.PHONY: validate-via-docker-all
 validate-via-docker-all : validate-via-docker-8.2.2
 validate-via-docker-all : validate-via-docker-8.4.4
 validate-via-docker-all : validate-via-docker-8.6.5
 validate-via-docker-all : validate-via-docker-8.8.4
 validate-via-docker-all : validate-via-docker-8.10.4
 
+.PHONY: validate-dockerfiles
 validate-dockerfiles : .docker/validate-8.10.4.dockerfile
 validate-dockerfiles : .docker/validate-8.8.4.dockerfile
 validate-dockerfiles : .docker/validate-8.6.5.dockerfile
@@ -164,21 +196,27 @@ validate-dockerfiles : .docker/validate-8.2.2.dockerfile
 # and we have a test relying on this limit being sufficiently small
 DOCKERARGS:=--ulimit nofile=1024:1024
 
+.PHONY: validate-via-docker-8.2.2
 validate-via-docker-8.2.2:
 	docker build $(DOCKERARGS) -t cabal-validate:8.2.2 -f .docker/validate-8.2.2.dockerfile .
 
+.PHONY: validate-via-docker-8.4.4
 validate-via-docker-8.4.4:
 	docker build $(DOCKERARGS) -t cabal-validate:8.4.4 -f .docker/validate-8.4.4.dockerfile .
 
+.PHONY: validate-via-docker-8.6.5
 validate-via-docker-8.6.5:
 	docker build $(DOCKERARGS) -t cabal-validate:8.6.5 -f .docker/validate-8.6.5.dockerfile .
 
+.PHONY: validate-via-docker-8.8.4
 validate-via-docker-8.8.4:
 	docker build $(DOCKERARGS) -t cabal-validate:8.8.4 -f .docker/validate-8.8.4.dockerfile .
 
+.PHONY: validate-via-docker-8.10.4
 validate-via-docker-8.10.4:
 	docker build $(DOCKERARGS) -t cabal-validate:8.10.4 -f .docker/validate-8.10.4.dockerfile .
 
+.PHONY: validate-via-docker-old
 validate-via-docker-old:
 	docker build $(DOCKERARGS) -t cabal-validate:older -f .docker/validate-old.dockerfile .
 
@@ -199,6 +237,7 @@ bootstrap-json-%: phony
 
 BOOTSTRAP_GHC_VERSIONS := 9.0.2 9.2.8 9.4.8 9.6.4 9.8.2
 
+.PHONY: bootstrap-jsons
 bootstrap-jsons: $(BOOTSTRAP_GHC_VERSIONS:%=bootstrap-json-%)
 
 # documentation