diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 540e1eb157e2f..caf97abf78d9e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -361,8 +361,8 @@ jobs:
             os: macos-13
           - name: dist-aarch64-apple
             env:
-              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin"
-              RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin"
+              SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
+              RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               SELECT_XCODE: /Applications/Xcode_13.4.1.app
               USE_XCODE_CLANG: 1
@@ -372,20 +372,8 @@ jobs:
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
-            os: macos-13-xlarge
-          - name: aarch64-apple
-            env:
-              SCRIPT: "./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin"
-              RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_13.4.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            os: macos-13-xlarge
+              JEMALLOC_SYS_WITH_LG_PAGE: 14
+            os: macos-13
           - name: x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 3af370bf006ad..68a3afc910f22 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -553,14 +553,17 @@ jobs:
           # This target only needs to support 11.0 and up as nothing else supports the hardware
           - name: dist-aarch64-apple
             env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
+              SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2
               RUST_CONFIGURE_ARGS: >-
+                --build=x86_64-apple-darwin
+                --host=aarch64-apple-darwin
+                --target=aarch64-apple-darwin
                 --enable-full-tools
                 --enable-sanitizers
                 --enable-profiler
+                --disable-docs
                 --set rust.jemalloc
                 --set llvm.ninja=false
-                --set rust.lto=thin
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               SELECT_XCODE: /Applications/Xcode_13.4.1.app
               USE_XCODE_CLANG: 1
@@ -570,26 +573,15 @@ jobs:
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-macos-m1
-
-          # This target only needs to support 11.0 and up as nothing else supports the hardware
-          - name: aarch64-apple
-            env:
-              SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
-              RUST_CONFIGURE_ARGS: >-
-                --enable-sanitizers
-                --enable-profiler
-                --set rust.jemalloc
-                --set llvm.ninja=false
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_13.4.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-m1
+              # Corresponds to 16K page size
+              #
+              # Shouldn't be needed if jemalloc-sys is updated to
+              # handle this platform like iOS or if we build on
+              # aarch64-apple-darwin itself.
+              #
+              # https://github.com/gnzlbg/jemallocator/blob/c27a859e98e3cb790dc269773d9da71a1e918458/jemalloc-sys/build.rs#L237
+              JEMALLOC_SYS_WITH_LG_PAGE: 14
+            <<: *job-macos-xl
 
           ######################
           #  Windows Builders  #