diff --git a/Cabal-tests/lib/Test/Utils/TempTestDir.hs b/Cabal-tests/lib/Test/Utils/TempTestDir.hs
index a4b0b08cba4..7d1ca76e04f 100644
--- a/Cabal-tests/lib/Test/Utils/TempTestDir.hs
+++ b/Cabal-tests/lib/Test/Utils/TempTestDir.hs
@@ -2,11 +2,12 @@
 
 module Test.Utils.TempTestDir
   ( withTestDir
+  , withTestDir'
   , removeDirectoryRecursiveHack
   ) where
 
 import Distribution.Compat.Internal.TempFile (createTempDirectory)
-import Distribution.Simple.Utils (warn)
+import Distribution.Simple.Utils (warn, TempFileOptions (..), defaultTempFileOptions)
 import Distribution.Verbosity
 
 import Control.Concurrent (threadDelay)
@@ -23,12 +24,26 @@ import qualified System.Info (os)
 -- | Much like 'withTemporaryDirectory' but with a number of hacks to make
 -- sure on windows that we can clean up the directory at the end.
 withTestDir :: (MonadIO m, MonadMask m) => Verbosity -> String -> (FilePath -> m a) -> m a
-withTestDir verbosity template action = do
-  systmpdir <- liftIO getTemporaryDirectory
+withTestDir verbosity template action = withTestDir' verbosity defaultTempFileOptions template action
+
+withTestDir' :: (MonadIO m, MonadMask m) => Verbosity -> TempFileOptions -> String -> (FilePath -> m a) -> m a
+withTestDir' verbosity tempFileOpts template action = do
+  systmpdir <-
+    -- MacOS returns /var/folders/... which is a symlink (/var -> /private/var),
+    -- so the test-suite struggles to make the build cwd-agnostic in particular
+    -- for the ShowBuildInfo tests. This canonicalizePath call makes it
+    -- /private/var/folders/... which will work.
+    liftIO $ canonicalizePath =<< getTemporaryDirectory
   bracket
     ( do { tmpRelDir <- liftIO $ createTempDirectory systmpdir template
          ; return $ systmpdir </> tmpRelDir } )
-    (liftIO . removeDirectoryRecursiveHack verbosity)
+    (liftIO
+      -- This ensures that the temp files are not deleted at the end of the test.
+      -- It replicates the behavior of @withTempDirectoryEx@.
+      . when (not (optKeepTempFiles tempFileOpts))
+      -- This is the bit that helps with Windows deleting all files.
+      . removeDirectoryRecursiveHack verbosity
+      )
     action
 
 -- | On Windows, file locks held by programs we run (in this case VCSs)
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.test.hs
index 6a05d9f0e96..e2c42fe43fd 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.test.hs
@@ -2,7 +2,7 @@ import Test.Cabal.Prelude
 
 main = cabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
+    expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
         cabal "v2-build" ["exe"]
         withPlan $ do
             r <- runPlanExe' "I" "exe" []
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs
index 3e4577aecfa..87ab998b3b0 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs
@@ -1,8 +1,7 @@
 import Test.Cabal.Prelude
 main = setupAndCabalTest $ do
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $ do
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $ do
     withPackageDb $ do
       withDirectory "mylib" $ setup_install_with_docs ["--ipid", "mylib-0.1.0.0"]
       withDirectory "mysql" $ setup_install_with_docs ["--ipid", "mysql-0.1.0.0"]
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs
index b6034011a14..80990b1136c 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs
@@ -2,8 +2,7 @@ import Test.Cabal.Prelude
 main = setupTest $ do
   -- No cabal test because per-component is broken with it
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
     withPackageDb $
       withDirectory "Includes2" $ do
         let setup_install' args = setup_install_with_docs args
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.test.hs
index 45a4546819c..a2431cdf389 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.test.hs
@@ -1,8 +1,8 @@
 import Test.Cabal.Prelude
 
 main = cabalTest $ do
-    ghcVer <- isGhcVersion ">= 9.10"
     skipUnlessGhcVersion ">= 8.1"
+    ghcVer <- isGhcVersion ">= 9.10"
     skipIf "Windows + 9.10.1 (#10191)" (isWindows && ghcVer)
     withProjectFile "cabal.external.project" $ do
         cabal "v2-build" ["exe"]
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.test.hs
index cca3ecf2954..b0d3e21688f 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.test.hs
@@ -2,7 +2,7 @@ import Test.Cabal.Prelude
 
 main = cabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
+    expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
         cabal "v2-build" ["exe"]
         withPlan $ do
             r <- runPlanExe' "I" "exe" []
diff --git a/cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs b/cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs
index d7ae9a1921d..a905e873cac 100644
--- a/cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs
@@ -3,8 +3,7 @@ import Data.List
 import qualified Data.Char as Char
 main = setupAndCabalTest $ do
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
     withPackageDb $ do
       containers_id <- getIPID "containers"
       withDirectory "repo/sigs-0.1.0.0" $ setup_install_with_docs ["--ipid", "sigs-0.1.0.0"]
@@ -21,4 +20,3 @@ main = setupAndCabalTest $ do
       withDirectory "repo/exe-0.1.0.0" $ do
         setup_install []
         runExe' "exe" [] >>= assertOutputContains "fromList [(0,2),(2,4)]"
-
diff --git a/cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs b/cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs
index 97dae2b597c..4931cd33c99 100644
--- a/cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 main = setupAndCabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    skipUnless "no profiling libs" =<< hasProfiledLibraries
+    skipIfNoProfiledLibraries
     setup "configure" ["--enable-profiling"]
     setup "build" []
diff --git a/cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs b/cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs
index 930a61bb54e..d05327839da 100644
--- a/cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 main =
-  cabalTest $ expectBrokenIf isWindows 10191 $ withShorterPathForNewBuildStore $ do
+  cabalTest $ expectBrokenIfWindows 10191 $ withShorterPathForNewBuildStore $ do
     skipUnlessGhcVersion ">= 8.1"
     withRepo "repo" $ do
       cabal "v2-build" ["T6385"]
diff --git a/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc-clang.bat b/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc-clang.bat
index a2e60a9c592..63221f54528 100644
--- a/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc-clang.bat
+++ b/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc-clang.bat
@@ -1,11 +1,7 @@
 @echo OFF
 
-where /q clang.exe
-
-IF %ERRORLEVEL% EQU 0 (
-   call clang.exe -DNOERROR6 %*
-   EXIT /B %ERRORLEVEL%
-)
-
-ECHO "Cannot find C compiler"
-EXIT /B 1
+REM replace the libdir with the path to the compiler
+FOR /f "delims=" %%A in ('call ghc.exe --print-libdir') do set "var=%%A"
+setlocal EnableDelayedExpansion
+CALL !var:lib=mingw\bin\clang.exe! -DNOERROR6 %*
+EXIT /B %ERRORLEVEL%
diff --git a/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc.bat b/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc.bat
index 504f6b800cd..560845174d8 100644
--- a/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc.bat
+++ b/cabal-testsuite/PackageTests/CCompilerOverride/custom-cc.bat
@@ -1,11 +1,7 @@
 @echo OFF
 
-where /q gcc.exe
-
-IF %ERRORLEVEL% EQU 0 (
-   call gcc.exe -DNOERROR6 %*
-   EXIT /B %ERRORLEVEL%
-)
-
-ECHO "Cannot find C compiler"
-EXIT /B 1
+REM replace the libdir with the path to the compiler
+FOR /f "delims=" %%A in ('call ghc.exe --print-libdir') do set "var=%%A"
+setlocal EnableDelayedExpansion
+CALL !var:lib=mingw\bin\gcc.exe! -DNOERROR6 %*
+EXIT /B %ERRORLEVEL%
diff --git a/cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs b/cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs
index 800a540696a..d0d9a11057c 100644
--- a/cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs
+++ b/cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs
@@ -1,7 +1,7 @@
 import Test.Cabal.Prelude
 
 main = setupTest $ do
-    skipIf "ghc < 7.8" =<< isGhcVersion "< 7.8"
+    skipIfGhcVersion "< 7.8"
     setup "configure" []
     res <- setup' "build" []
     assertOutputContains "= Post common block elimination =" res
diff --git a/cabal-testsuite/PackageTests/CustomDep/cabal.project b/cabal-testsuite/PackageTests/CustomDep/cabal.project
deleted file mode 100644
index d4198c181d9..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/cabal.project
+++ /dev/null
@@ -1 +0,0 @@
-packages: client custom
diff --git a/cabal-testsuite/PackageTests/CustomDep/cabal.test.hs b/cabal-testsuite/PackageTests/CustomDep/cabal.test.hs
deleted file mode 100644
index 9058afe19c0..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/cabal.test.hs
+++ /dev/null
@@ -1,10 +0,0 @@
-import Test.Cabal.Prelude
-main = cabalTest $ do
-    -- implicit setup-depends conflict with GHC >= 8.2; c.f. #415
-    skipUnlessGhcVersion "< 8.2"
-    -- This test depends heavily on what packages are in the global
-    -- database, don't record the output
-    recordMode DoNotRecord $ do
-        -- TODO: Hack, delete me
-        withEnvFilter (`notElem` ["HOME", "CABAL_DIR"]) $ do
-            cabal "v2-build" ["all"]
diff --git a/cabal-testsuite/PackageTests/CustomDep/client/B.hs b/cabal-testsuite/PackageTests/CustomDep/client/B.hs
deleted file mode 100644
index af119169669..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/client/B.hs
+++ /dev/null
@@ -1,2 +0,0 @@
-module B where
-import A
diff --git a/cabal-testsuite/PackageTests/CustomDep/client/Setup.hs b/cabal-testsuite/PackageTests/CustomDep/client/Setup.hs
deleted file mode 100644
index 9a994af677b..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/client/Setup.hs
+++ /dev/null
@@ -1,2 +0,0 @@
-import Distribution.Simple
-main = defaultMain
diff --git a/cabal-testsuite/PackageTests/CustomDep/client/client.cabal b/cabal-testsuite/PackageTests/CustomDep/client/client.cabal
deleted file mode 100644
index 2bb74b34594..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/client/client.cabal
+++ /dev/null
@@ -1,12 +0,0 @@
-name:                client
-version:             0.1.0.0
-license:             BSD3
-author:              Edward Z. Yang
-maintainer:          ezyang@cs.stanford.edu
-build-type:          Simple
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     B
-  build-depends:       base, custom
-  default-language:    Haskell2010
diff --git a/cabal-testsuite/PackageTests/CustomDep/custom/A.hs b/cabal-testsuite/PackageTests/CustomDep/custom/A.hs
deleted file mode 100644
index d843c00b782..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/custom/A.hs
+++ /dev/null
@@ -1 +0,0 @@
-module A where
diff --git a/cabal-testsuite/PackageTests/CustomDep/custom/Setup.hs b/cabal-testsuite/PackageTests/CustomDep/custom/Setup.hs
deleted file mode 100644
index 9a994af677b..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/custom/Setup.hs
+++ /dev/null
@@ -1,2 +0,0 @@
-import Distribution.Simple
-main = defaultMain
diff --git a/cabal-testsuite/PackageTests/CustomDep/custom/custom.cabal b/cabal-testsuite/PackageTests/CustomDep/custom/custom.cabal
deleted file mode 100644
index 4f327080098..00000000000
--- a/cabal-testsuite/PackageTests/CustomDep/custom/custom.cabal
+++ /dev/null
@@ -1,15 +0,0 @@
-name:                custom
-version:             0.1.0.0
-license:             BSD3
-author:              Edward Z. Yang
-maintainer:          ezyang@cs.stanford.edu
-build-type:          Custom
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     A
-  build-depends:       base
-  default-language:    Haskell2010
-
-custom-setup
-  setup-depends: base, Cabal
diff --git a/cabal-testsuite/PackageTests/CustomPlain/A.hs b/cabal-testsuite/PackageTests/CustomPlain/A.hs
deleted file mode 100644
index d843c00b782..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/A.hs
+++ /dev/null
@@ -1 +0,0 @@
-module A where
diff --git a/cabal-testsuite/PackageTests/CustomPlain/Setup.hs b/cabal-testsuite/PackageTests/CustomPlain/Setup.hs
deleted file mode 100644
index 20b960ede90..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/Setup.hs
+++ /dev/null
@@ -1,3 +0,0 @@
-import Distribution.Simple
-import System.IO
-main = hPutStrLn stderr "ThisIsCustomYeah" >> defaultMain
diff --git a/cabal-testsuite/PackageTests/CustomPlain/cabal.project b/cabal-testsuite/PackageTests/CustomPlain/cabal.project
deleted file mode 100644
index e6fdbadb439..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/cabal.project
+++ /dev/null
@@ -1 +0,0 @@
-packages: .
diff --git a/cabal-testsuite/PackageTests/CustomPlain/cabal.test.hs b/cabal-testsuite/PackageTests/CustomPlain/cabal.test.hs
deleted file mode 100644
index 42c64595594..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/cabal.test.hs
+++ /dev/null
@@ -1,11 +0,0 @@
-import Test.Cabal.Prelude
-main = cabalTest $ do
-    -- implicit setup-depends conflict with GHC >= 8.2; c.f. #415
-    skipUnlessGhcVersion "< 8.2"
-    -- Regression test for #4393
-    recordMode DoNotRecord $ do
-        -- TODO: Hack; see also CustomDep/cabal.test.hs
-        withEnvFilter (`notElem` ["HOME", "CABAL_DIR"]) $ do
-            -- On -v2, we don't have vQuiet set, which suppressed
-            -- the error
-            cabal "v2-build" ["-v1"]
diff --git a/cabal-testsuite/PackageTests/CustomPlain/plain.cabal b/cabal-testsuite/PackageTests/CustomPlain/plain.cabal
deleted file mode 100644
index d0666a10fd8..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/plain.cabal
+++ /dev/null
@@ -1,11 +0,0 @@
-name:                plain
-version:             0.1.0.0
-license:             BSD3
-author:              Edward Z. Yang
-maintainer:          ezyang@cs.stanford.edu
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     A
-  build-depends:       base
-  default-language:    Haskell2010
diff --git a/cabal-testsuite/PackageTests/CustomPlain/setup.cabal.out b/cabal-testsuite/PackageTests/CustomPlain/setup.cabal.out
deleted file mode 100644
index 2d3ff0f9451..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/setup.cabal.out
+++ /dev/null
@@ -1,7 +0,0 @@
-# Setup configure
-Resolving dependencies...
-Configuring plain-0.1.0.0...
-Warning: No 'build-type' specified. If you do not need a custom Setup.hs or ./configure script then use 'build-type: Simple'.
-# Setup build
-Preprocessing library for plain-0.1.0.0...
-Building library for plain-0.1.0.0...
diff --git a/cabal-testsuite/PackageTests/CustomPlain/setup.out b/cabal-testsuite/PackageTests/CustomPlain/setup.out
deleted file mode 100644
index 3fdfe2b7f77..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/setup.out
+++ /dev/null
@@ -1,6 +0,0 @@
-# Setup configure
-Configuring plain-0.1.0.0...
-Warning: [no-build-type] No 'build-type' specified. If you do not need a custom Setup.hs or ./configure script then use 'build-type: Simple'.
-# Setup build
-Preprocessing library for plain-0.1.0.0...
-Building library for plain-0.1.0.0...
diff --git a/cabal-testsuite/PackageTests/CustomPlain/setup.test.hs b/cabal-testsuite/PackageTests/CustomPlain/setup.test.hs
deleted file mode 100644
index abf668397b8..00000000000
--- a/cabal-testsuite/PackageTests/CustomPlain/setup.test.hs
+++ /dev/null
@@ -1,4 +0,0 @@
-import Test.Cabal.Prelude
-main = setupTest $ do
-    setup' "configure" [] >>= assertOutputContains "ThisIsCustomYeah"
-    setup' "build"     [] >>= assertOutputContains "ThisIsCustomYeah"
diff --git a/cabal-testsuite/PackageTests/GHCJS/BuildRunner/cabal.test.hs b/cabal-testsuite/PackageTests/GHCJS/BuildRunner/cabal.test.hs
index 18fa0d364fe..245901c8bef 100644
--- a/cabal-testsuite/PackageTests/GHCJS/BuildRunner/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/GHCJS/BuildRunner/cabal.test.hs
@@ -1,7 +1,7 @@
 import Test.Cabal.Prelude
 
 main = do
-  cabalTest . expectBrokenIf isWindows 10179 . recordMode DoNotRecord $ do
+  cabalTest . expectBrokenIfWindows 10179 . recordMode DoNotRecord $ do
     cwd <- fmap testCurrentDir getTestEnv
     testInvokedWithBuildRunner cwd "test" []
     testInvokedWithBuildRunner cwd "run" ["ghcjs-exe"]
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectory/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectory/setup.test.hs
index 0f0e86a3c60..86a7cf804ea 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectory/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectory/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
       env <- getTestEnv
       let cwd = testCurrentDir env
       ghc_path <- programPathM ghcProgram
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryGhcVersion/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryGhcVersion/setup.test.hs
index 20a661437bf..0b0a37413be 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryGhcVersion/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryGhcVersion/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
       env <- getTestEnv
       let cwd = testCurrentDir env
       ghc_path <- programPathM ghcProgram
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryVersion/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryVersion/setup.test.hs
index 20a661437bf..0b0a37413be 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryVersion/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/SameDirectoryVersion/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
       env <- getTestEnv
       let cwd = testCurrentDir env
       ghc_path <- programPathM ghcProgram
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/Symlink/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/Symlink/setup.test.hs
index 18cb32829b1..0b80a76a952 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/Symlink/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/Symlink/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
     withSymlink "bin/ghc" "ghc" $ do
         env <- getTestEnv
         let cwd = testCurrentDir env
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkGhcVersion/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkGhcVersion/setup.test.hs
index 7837f5f9f3b..150f2bc8739 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkGhcVersion/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkGhcVersion/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
     withSymlink "bin/ghc-7.10" "ghc" $ do
         env <- getTestEnv
         let cwd = testCurrentDir env
diff --git a/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkVersion/setup.test.hs b/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkVersion/setup.test.hs
index 7837f5f9f3b..150f2bc8739 100644
--- a/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkVersion/setup.test.hs
+++ b/cabal-testsuite/PackageTests/GhcPkgGuess/SymlinkVersion/setup.test.hs
@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 
-main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
+main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
     withSymlink "bin/ghc-7.10" "ghc" $ do
         env <- getTestEnv
         let cwd = testCurrentDir env
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Foo.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Foo.hs
deleted file mode 100644
index efbf93bbde8..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Foo.hs
+++ /dev/null
@@ -1 +0,0 @@
-module Foo where
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Main.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Main.hs
deleted file mode 100644
index d82a4bd93b7..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/Main.hs
+++ /dev/null
@@ -1,4 +0,0 @@
-module Main where
-
-main :: IO ()
-main = return ()
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/build-depends-extra-version.cabal b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/build-depends-extra-version.cabal
deleted file mode 100644
index 01c7b3b5972..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/build-depends-extra-version.cabal
+++ /dev/null
@@ -1,17 +0,0 @@
-name:                build-depends-extra-version
-version:             0.1.0.0
-synopsis:            Checks build-depends warns for extraneous version
-license:             BSD3
-category:            Testing
-build-type:          Simple
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     Foo
-  default-language:    Haskell2010
-
-executable bar
-  main-is:             Main.hs
-  build-depends:       build-depends-extra-version >=0.0.0.1
-  -- ^ internal dependency, extraneous version
-  default-language:    Haskell2010
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.cabal.out b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.cabal.out
deleted file mode 100644
index 61f9d97a4ba..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.cabal.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-depends-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal library: build-depends-extra-version >=0.0.0.1. This version range includes the current package but isn't needed as the current package's library will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal library: build-depends-extra-version >=0.0.0.1. This version range includes the current package but isn't needed as the current package's library will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-depends-extra-version-0.1.0.0...
-Source tarball created: setup.cabal.dist/work/dist/build-depends-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.out b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.out
deleted file mode 100644
index 2b9987c8544..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-depends-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal library: build-depends-extra-version >=0.0.0.1. This version range includes the current package but isn't needed as the current package's library will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal library: build-depends-extra-version >=0.0.0.1. This version range includes the current package but isn't needed as the current package's library will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-depends-extra-version-0.1.0.0...
-Source tarball created: setup.dist/work/dist/build-depends-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.test.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.test.hs
deleted file mode 100644
index c006efe6f72..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildDependsExtra/setup.test.hs
+++ /dev/null
@@ -1,6 +0,0 @@
-import Test.Cabal.Prelude
--- Test unneed version bound on internal build-tools deps
-main = setupAndCabalTest . expectBroken 7470 $ do
-    setup' "configure" []
-    assertOutputContains "extraneous version range"
-        =<< setup' "sdist" []
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Foo.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Foo.hs
deleted file mode 100644
index efbf93bbde8..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Foo.hs
+++ /dev/null
@@ -1 +0,0 @@
-module Foo where
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Main.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Main.hs
deleted file mode 100644
index d82a4bd93b7..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/Main.hs
+++ /dev/null
@@ -1,4 +0,0 @@
-module Main where
-
-main :: IO ()
-main = return ()
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/build-tool-depends-extra-version.cabal b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/build-tool-depends-extra-version.cabal
deleted file mode 100644
index af43e0cdd02..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/build-tool-depends-extra-version.cabal
+++ /dev/null
@@ -1,18 +0,0 @@
-name:                build-tool-depends-extra-version
-version:             0.1.0.0
-synopsis:            Checks build-tool-depends warns for extraneous version
-license:             BSD3
-category:            Testing
-build-type:          Simple
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     Foo
-  build-tool-depends:  build-tool-depends-extra-version:hello-world >=0.0.0.1
-  -- ^ internal dependency, extraneous version
-  default-language:    Haskell2010
-
-executable             hello-world
-  main-is:             Main.hs
-  build-depends:       base
-  default-language:    Haskell2010
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.cabal.out b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.cabal.out
deleted file mode 100644
index c247d90f995..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.cabal.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-tool-depends-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal executable: build-tool-depends-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal executable: build-tool-depends-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-tool-depends-extra-version-0.1.0.0...
-Source tarball created: setup.cabal.dist/work/dist/build-tool-depends-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.out b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.out
deleted file mode 100644
index cd0ba1f5e0a..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-tool-depends-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal executable: build-tool-depends-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal executable: build-tool-depends-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-tool-depends-extra-version-0.1.0.0...
-Source tarball created: setup.dist/work/dist/build-tool-depends-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.test.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.test.hs
deleted file mode 100644
index c006efe6f72..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolDependsExtra/setup.test.hs
+++ /dev/null
@@ -1,6 +0,0 @@
-import Test.Cabal.Prelude
--- Test unneed version bound on internal build-tools deps
-main = setupAndCabalTest . expectBroken 7470 $ do
-    setup' "configure" []
-    assertOutputContains "extraneous version range"
-        =<< setup' "sdist" []
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Foo.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Foo.hs
deleted file mode 100644
index efbf93bbde8..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Foo.hs
+++ /dev/null
@@ -1 +0,0 @@
-module Foo where
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Main.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Main.hs
deleted file mode 100644
index d82a4bd93b7..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/Main.hs
+++ /dev/null
@@ -1,4 +0,0 @@
-module Main where
-
-main :: IO ()
-main = return ()
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/build-tools-extra-version.cabal b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/build-tools-extra-version.cabal
deleted file mode 100644
index c5371bf45ee..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/build-tools-extra-version.cabal
+++ /dev/null
@@ -1,18 +0,0 @@
-name:                build-tools-extra-version
-version:             0.1.0.0
-synopsis:            Checks build-tools warns for extraneous version
-license:             BSD3
-category:            Testing
-build-type:          Simple
-cabal-version:       >=1.10
-
-library
-  exposed-modules:     Foo
-  build-tools:         hello-world >=0.0.0.1
-  -- ^ internal dependency, extraneous version
-  default-language:    Haskell2010
-
-executable             hello-world
-  main-is:             Main.hs
-  build-depends:       base
-  default-language:    Haskell2010
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.cabal.out b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.cabal.out
deleted file mode 100644
index 58d3172cb43..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.cabal.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-tools-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal executable: build-tools-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal executable: build-tools-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-tools-extra-version-0.1.0.0...
-Source tarball created: setup.cabal.dist/work/dist/build-tools-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.out b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.out
deleted file mode 100644
index 28059d26ccd..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.out
+++ /dev/null
@@ -1,13 +0,0 @@
-# Setup configure
-Configuring build-tools-extra-version-0.1.0.0...
-Warning: The package has an extraneous version range for a dependency on an internal executable: build-tools-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-# Setup sdist
-Distribution quality errors:
-The package has an extraneous version range for a dependency on an internal executable: build-tools-extra-version:hello-world >=0.0.0.1. This version range includes the current package but isn't needed as the current package's executable will always be used.
-Distribution quality warnings:
-No 'maintainer' field.
-No 'description' field.
-A 'license-file' is not specified.
-Note: the public hackage server would reject this package.
-Building source dist for build-tools-extra-version-0.1.0.0...
-Source tarball created: setup.dist/work/dist/build-tools-extra-version-0.1.0.0.tar.gz
diff --git a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.test.hs b/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.test.hs
deleted file mode 100644
index c006efe6f72..00000000000
--- a/cabal-testsuite/PackageTests/InternalVersions/BuildToolsExtra/setup.test.hs
+++ /dev/null
@@ -1,6 +0,0 @@
-import Test.Cabal.Prelude
--- Test unneed version bound on internal build-tools deps
-main = setupAndCabalTest . expectBroken 7470 $ do
-    setup' "configure" []
-    assertOutputContains "extraneous version range"
-        =<< setup' "sdist" []
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/A.hs b/cabal-testsuite/PackageTests/NewBuild/T4375/A.hs
deleted file mode 100644
index d843c00b782..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/A.hs
+++ /dev/null
@@ -1 +0,0 @@
-module A where
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/Setup.hs b/cabal-testsuite/PackageTests/NewBuild/T4375/Setup.hs
deleted file mode 100644
index 9a994af677b..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/Setup.hs
+++ /dev/null
@@ -1,2 +0,0 @@
-import Distribution.Simple
-main = defaultMain
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/a.cabal b/cabal-testsuite/PackageTests/NewBuild/T4375/a.cabal
deleted file mode 100644
index e3c9ed6bca1..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/a.cabal
+++ /dev/null
@@ -1,11 +0,0 @@
-name: a
-version: 0.1
-build-type: Custom
-cabal-version: >= 1.10
-
--- no explicit setup deps
-
-library
-  exposed-modules: A
-  build-depends: base
-  default-language: Haskell2010
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.out b/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.out
deleted file mode 100644
index ebbf63746ef..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.out
+++ /dev/null
@@ -1,7 +0,0 @@
-# cabal v2-update
-Downloading the latest package list from test-local-repo
-# cabal v2-build
-Resolving dependencies...
-Build profile: -w ghc-<GHCVER> -O1
-In order, the following will be built:
- - a-0.1 (lib:a) (first run)
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.project b/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.project
deleted file mode 100644
index e6fdbadb439..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.project
+++ /dev/null
@@ -1 +0,0 @@
-packages: .
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.test.hs b/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.test.hs
deleted file mode 100644
index b42f3f28c7a..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/cabal.test.hs
+++ /dev/null
@@ -1,13 +0,0 @@
-import Test.Cabal.Prelude
-main =
-  -- TODO: is this test ever run?
-  cabalTest $ withShorterPathForNewBuildStore $ do
-    -- Don't run this test unless the GHC is sufficiently recent
-    -- to not ship boot old-time/old-locale
-    skipUnlessGhcVersion ">= 7.11"
-    -- Don't run this test on GHC 8.2, which ships with Cabal 2.0,
-    -- which is not eligible for old-style Custom setup (if
-    -- we had the full Hackage index, we'd try it.)
-    skipUnlessGhcVersion "< 8.1"
-    withRepo "repo" $ do
-        cabal "v2-build" ["a"]
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/System/Locale.hs b/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/System/Locale.hs
deleted file mode 100644
index 83af47b0ef4..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/System/Locale.hs
+++ /dev/null
@@ -1 +0,0 @@
-module System.Locale where
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/old-locale.cabal b/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/old-locale.cabal
deleted file mode 100644
index f6e3dadf6d5..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-locale-1.0.0.7/old-locale.cabal
+++ /dev/null
@@ -1,9 +0,0 @@
-name: old-locale
-version: 1.0.0.7
-build-type: Simple
-cabal-version: >= 1.10
-
-library
-    exposed-modules: System.Locale
-    build-depends: base
-    default-language: Haskell2010
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/System/Time.hs b/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/System/Time.hs
deleted file mode 100644
index f4185b6527a..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/System/Time.hs
+++ /dev/null
@@ -1,2 +0,0 @@
-module System.Time where
-import System.Locale
diff --git a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/old-time.cabal b/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/old-time.cabal
deleted file mode 100644
index 11ac6cccc91..00000000000
--- a/cabal-testsuite/PackageTests/NewBuild/T4375/repo/old-time-1.1.0.3/old-time.cabal
+++ /dev/null
@@ -1,9 +0,0 @@
-name: old-time
-version: 1.1.0.3
-build-type: Simple
-cabal-version: >= 1.10
-
-library
-    exposed-modules: System.Time
-    build-depends: base, old-locale == 1.0.*
-    default-language: Haskell2010
diff --git a/cabal-testsuite/PackageTests/NewSdist/DeterministicTrivial/deterministic.test.hs b/cabal-testsuite/PackageTests/NewSdist/DeterministicTrivial/deterministic.test.hs
index 67c2944b39c..8b260c9e1ba 100644
--- a/cabal-testsuite/PackageTests/NewSdist/DeterministicTrivial/deterministic.test.hs
+++ b/cabal-testsuite/PackageTests/NewSdist/DeterministicTrivial/deterministic.test.hs
@@ -24,6 +24,5 @@ main = cabalTest $ do
     known <- liftIO (BS.readFile knownSdist)
     unknown <- liftIO (BS.readFile mySdist)
 
-    skipIf "#8356" True -- bogus, just to indicate that the test is skipped
     assertEqual "hashes didn't match for sdist" True True
     -- assertEqual "hashes didn't match for sdist" (BS16.encode $ SHA256.hash known) (BS16.encode $ SHA256.hash unknown)
diff --git a/cabal-testsuite/PackageTests/NewUpdate/RejectFutureIndexStates/cabal.test.hs b/cabal-testsuite/PackageTests/NewUpdate/RejectFutureIndexStates/cabal.test.hs
index 475a093360d..442d3508849 100644
--- a/cabal-testsuite/PackageTests/NewUpdate/RejectFutureIndexStates/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/NewUpdate/RejectFutureIndexStates/cabal.test.hs
@@ -1,13 +1,7 @@
 import Test.Cabal.Prelude
 import Data.List (isPrefixOf)
 
-main = cabalTest $ do
-
-  skip "Flaky test failing in `curl`, see #9530"
-
-  testBody
-
-testBody = withProjectFile "cabal.project" $ withRemoteRepo "repo" $ do
+main = skipIfCIAndWindows 10230 >> cabalTest (flakyIfCI 9530 $ withProjectFile "cabal.project" $ withRemoteRepo "repo" $ do
 
   output <- last
           . words
@@ -18,9 +12,9 @@ testBody = withProjectFile "cabal.project" $ withRemoteRepo "repo" $ do
         <$> recordMode DoNotRecord (cabal' "update" [])
   -- update golden output with actual timestamp
   shell "cp" ["cabal.out.in", "cabal.out"]
-  shell "sed" ["-i''", "-e", "s/REPLACEME/" <> output <> "/g", "cabal.out"]
+  shell "sed" [ "-i" ++ if not isWindows then "''" else "", "-e", "s/REPLACEME/" <> output <> "/g", "cabal.out"]
   -- This shall fail with an error message as specified in `cabal.out`
   fails $ cabal "build" ["--index-state=4000-01-01T00:00:00Z", "fake-pkg"]
   -- This shall fail by not finding the package, what indicates that it
   -- accepted an older index-state.
-  fails $ cabal "build" ["--index-state=2023-01-01T00:00:00Z", "fake-pkg"]
+  fails $ cabal "build" ["--index-state=2023-01-01T00:00:00Z", "fake-pkg"])
diff --git a/cabal-testsuite/PackageTests/NewUpdate/UpdateIndexState/update-index-state.test.hs b/cabal-testsuite/PackageTests/NewUpdate/UpdateIndexState/update-index-state.test.hs
index e6485d51f71..eb3bc8daff7 100644
--- a/cabal-testsuite/PackageTests/NewUpdate/UpdateIndexState/update-index-state.test.hs
+++ b/cabal-testsuite/PackageTests/NewUpdate/UpdateIndexState/update-index-state.test.hs
@@ -1,19 +1,13 @@
 import Test.Cabal.Prelude
 
-main = cabalTest $ do
-
-  skip "Flaky test failing in `curl`, see #9530"
-
-  testBody
-
-testBody = withRemoteRepo "repo" $ do
+main = skipIfCIAndWindows 10230 >> cabalTest (flakyIfCI 9530 $ withRemoteRepo "repo" $ do
 
   -- The _first_ update call causes a warning about missing mirrors, the warning
   -- is platform-dependent and it's not part of the test expectations, so we
   -- check the output manually.
   res <- recordMode DoNotRecord $
-           cabal' "update" ["repository.localhost,2022-01-28T02:36:41Z"]
+          cabal' "update" ["repository.localhost,2022-01-28T02:36:41Z"]
   assertOutputContains "The index-state is set to 2022-01-28T02:36:41Z" res
   assertOutputDoesNotContain "revert" res
   cabal "update" ["repository.localhost,2016-09-24T17:47:48Z"]
-  cabal "update" ["repository.localhost,2022-01-28T02:36:41Z"]
+  cabal "update" ["repository.localhost,2022-01-28T02:36:41Z"])
diff --git a/cabal-testsuite/PackageTests/PkgConfigParse/setup.test.hs b/cabal-testsuite/PackageTests/PkgConfigParse/setup.test.hs
index 7ccdfca1655..2528e7459e5 100644
--- a/cabal-testsuite/PackageTests/PkgConfigParse/setup.test.hs
+++ b/cabal-testsuite/PackageTests/PkgConfigParse/setup.test.hs
@@ -1,7 +1,7 @@
 import Test.Cabal.Prelude
 
 -- Test that invalid unicode in pkg-config output doesn't trip up cabal very much
-main = cabalTest $ expectBrokenIf isWindows 10179 $ do
+main = cabalTest $ expectBrokenIfWindows 10179 $ do
   cdir <- testCurrentDir `fmap` getTestEnv
   res <- cabal' "v2-build" ["--extra-prog-path="++cdir, "-v2"]
   assertOutputContains "Some pkg-config packages have names containing invalid unicode: or" res
diff --git a/cabal-testsuite/PackageTests/Regression/T4025/setup.test.hs b/cabal-testsuite/PackageTests/Regression/T4025/setup.test.hs
index d6034c7c3f2..b26afa66701 100644
--- a/cabal-testsuite/PackageTests/Regression/T4025/setup.test.hs
+++ b/cabal-testsuite/PackageTests/Regression/T4025/setup.test.hs
@@ -1,11 +1,9 @@
 import Test.Cabal.Prelude
 -- Test that we don't accidentally add the inplace directory to
 -- an executable RPATH.
-main = do
-  skipIfWindows "doesn't support RPATH"
-  setupAndCabalTest $ do
-    ghc <- isGhcVersion ">= 8.10.7"
-    expectBrokenIf (isOSX && ghc) 7610 $ do -- see also issue #7988
+main = setupAndCabalTest $ do
+    skipIfNoSharedLibraries
+    expectBrokenIfOSXAndGhc ">= 8.10.7" 7610 $ do -- see also issue #7988
       setup "configure" ["--enable-executable-dynamic"]
       setup "build" []
       -- This should fail as it we should NOT be able to find the
diff --git a/cabal-testsuite/PackageTests/Regression/T4270/setup.test.hs b/cabal-testsuite/PackageTests/Regression/T4270/setup.test.hs
index bd227b75f3e..6dbe57b8ede 100644
--- a/cabal-testsuite/PackageTests/Regression/T4270/setup.test.hs
+++ b/cabal-testsuite/PackageTests/Regression/T4270/setup.test.hs
@@ -6,7 +6,6 @@ main = setupAndCabalTest $ do
   skipIfAllCabalVersion "< 2.2"
   skipIfNoSharedLibraries
   skipUnless "no shared Cabal"  =<< hasCabalShared
-  ghc <- isGhcVersion "== 8.0.2"
-  expectBrokenIf (isOSX && ghc) 8028 $ do
+  expectBrokenIfOSXAndGhc "== 8.0.2" 8028 $ do
     setup_build ["--enable-tests", "--enable-executable-dynamic"]
     setup "test" []
diff --git a/cabal-testsuite/PackageTests/Regression/T5309/cabal.test.hs b/cabal-testsuite/PackageTests/Regression/T5309/cabal.test.hs
index e64f2e36951..4700f191578 100644
--- a/cabal-testsuite/PackageTests/Regression/T5309/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/Regression/T5309/cabal.test.hs
@@ -1,8 +1,6 @@
 import Test.Cabal.Prelude
-main = do
-  cabalTest $ do
-    ghcVer <- isGhcVersion ">= 9.4"
-    expectBrokenIf (isWindows && ghcVer) 10189 $ do
+main = cabalTest $ do
+    expectBrokenIfWindowsCIAndGhc ">= 9.4" 10189 $ do
       cabal "v2-build" ["all"]
       cabal "v2-test"  ["all"]
       cabal "v2-bench" ["all"]
diff --git a/cabal-testsuite/PackageTests/Regression/T6906/cabal.test.hs b/cabal-testsuite/PackageTests/Regression/T6906/cabal.test.hs
index fabfcbdbede..9f71b845736 100644
--- a/cabal-testsuite/PackageTests/Regression/T6906/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/Regression/T6906/cabal.test.hs
@@ -1,8 +1,7 @@
 import Test.Cabal.Prelude
 
 main = cabalTest $ do
-   ghcsWithMaxPathIssue <- isGhcVersion "< 8.6.5"
-   expectBrokenIf (isWindows && ghcsWithMaxPathIssue) 6271 $ do
+   expectBrokenIfWindowsAndGhc "< 8.6.5" 6271 $ do
       res <- recordMode DoNotRecord $ cabalG' ["--config=cabal.config"] "v2-install" ["-v3"]
       assertOutputContains "creating file with the inputs used to compute the package hash:" res
       assertOutputContains "extra-lib-dirs: bar" res
diff --git a/cabal-testsuite/PackageTests/TestSuiteTests/ExeV10/cabal-with-hpc.multitest.hs b/cabal-testsuite/PackageTests/TestSuiteTests/ExeV10/cabal-with-hpc.multitest.hs
index c62f83385a3..172b48d8c80 100644
--- a/cabal-testsuite/PackageTests/TestSuiteTests/ExeV10/cabal-with-hpc.multitest.hs
+++ b/cabal-testsuite/PackageTests/TestSuiteTests/ExeV10/cabal-with-hpc.multitest.hs
@@ -8,10 +8,9 @@ import qualified Distribution.Verbosity as Verbosity
 
 import Test.Cabal.Prelude
 
-main = cabalTest $ do
-    skipIf "osx" isOSX -- TODO: re-enable this once the macOS CI
-                       -- issues are resolved, see discussion in #4902.
-
+main = do
+  skipIfCIAndOSX 4902
+  cabalTest $ do
     hasShared   <- hasSharedLibraries
     hasProfiled <- hasProfiledLibraries
     hpcOk       <- correctHpcVersion
diff --git a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-copy.test.hs b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-copy.test.hs
index e05f52953f5..353caa5b5c8 100644
--- a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-copy.test.hs
+++ b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-copy.test.hs
@@ -1,8 +1,6 @@
 import Test.Cabal.Prelude
 
-main = do
-    skipIfWindows "#10180"
-    cabalTest $ withShorterPathForNewBuildStore $ do
-        storeDir <- testStoreDir <$> getTestEnv
-        let options = ["--installdir=" ++ storeDir]
-        cabalG options "v2-install" ["--install-method=copy"]
+main = cabalTest $ expectBrokenIfWindows 10180 $ withShorterPathForNewBuildStore $ do
+    storeDir <- testStoreDir <$> getTestEnv
+    let options = ["--installdir=" ++ storeDir]
+    cabalG options "v2-install" ["--install-method=copy"]
diff --git a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-symlink.test.hs b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-symlink.test.hs
index d25ecb0465e..62f9305d4ee 100644
--- a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-symlink.test.hs
+++ b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/clean-install-by-symlink.test.hs
@@ -1,8 +1,6 @@
 import Test.Cabal.Prelude
 
-main = do
-    skipIfWindows "#10180"
-    cabalTest $ withShorterPathForNewBuildStore $ do
-      storeDir <- testStoreDir <$> getTestEnv
-      let options = ["--installdir=" ++ storeDir]
-      cabalG options "v2-install" []
+main = cabalTest $ expectBrokenIfWindows 10180 $ withShorterPathForNewBuildStore $ do
+    storeDir <- testStoreDir <$> getTestEnv
+    let options = ["--installdir=" ++ storeDir]
+    cabalG options "v2-install" []
diff --git a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/dirty-install.test.hs b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/dirty-install.test.hs
index 232d526f983..17008de3f7c 100644
--- a/cabal-testsuite/PackageTests/WarnEarlyOverwrite/dirty-install.test.hs
+++ b/cabal-testsuite/PackageTests/WarnEarlyOverwrite/dirty-install.test.hs
@@ -2,13 +2,11 @@ import Test.Cabal.Prelude
 
 import System.FilePath
 
-main = do
-    skipIfWindows "#10180"
-    cabalTest $ withShorterPathForNewBuildStore $ do
-        storeDir <- testStoreDir <$> getTestEnv
-        let options = ["--installdir=" ++ storeDir]
-        -- Touch the target to see if the warning is made early before the build.
-        _ <- runM "touch" [ (if isWindows then (<.> "exe") else id)
-                          $ storeDir </> "warn-early-overwrite" ] Nothing
-        fails $ cabalG options "v2-install" []
-        cabalG options "v2-install" ["--overwrite-policy=always"]
+main = cabalTest $ expectBrokenIfWindows 10180 $ withShorterPathForNewBuildStore $ do
+    storeDir <- testStoreDir <$> getTestEnv
+    let options = ["--installdir=" ++ storeDir]
+    -- Touch the target to see if the warning is made early before the build.
+    _ <- runM "touch" [ (if isWindows then (<.> "exe") else id)
+                        $ storeDir </> "warn-early-overwrite" ] Nothing
+    fails $ cabalG options "v2-install" []
+    cabalG options "v2-install" ["--overwrite-policy=always"]
diff --git a/cabal-testsuite/PackageTests/postCheckoutCommand/cabal.test.hs b/cabal-testsuite/PackageTests/postCheckoutCommand/cabal.test.hs
index 65fddd48be5..50e45454ff3 100644
--- a/cabal-testsuite/PackageTests/postCheckoutCommand/cabal.test.hs
+++ b/cabal-testsuite/PackageTests/postCheckoutCommand/cabal.test.hs
@@ -1,8 +1,6 @@
 import Test.Cabal.Prelude
 
-main = do
-  skipIfWindows "see #10182"
-  cabalTest $ do
+main = cabalTest $ do
     withProjectFile "cabal.positive.project" $ do
         cabal "v2-build" ["-v0"]
     withProjectFile "cabal.negative.project" $ do
diff --git a/cabal-testsuite/README.md b/cabal-testsuite/README.md
index 79b9185fecb..73b39d56801 100644
--- a/cabal-testsuite/README.md
+++ b/cabal-testsuite/README.md
@@ -200,17 +200,29 @@ and stderr.
 these with include `hasSharedLibraries`, `hasProfiledLibraries`,
 `hasCabalShared`, `isGhcVersion`, `isWindows`, `isLinux`, `isOSX`.
 
+There are some pre-defined versions of those combinators like `skipIfWindows`
+or `skipIfCI`. If possible try to use those as the error message will be uniform
+with other tests, allowing for `grep`ing the output more easily.
+
+Make sure that you only skip tests which cannot be run by fundamental reasons,
+like the OS or the capabilities of the GHC version. If a test is failing do not
+skip it, mark it as broken instead (see next question).
+
+**How do I mark a test as broken?**  Use `expectBroken`, which takes
+the ticket number as its first argument.
+
+**How do I mark a flaky test?** If a test passes only sometimes for unknown
+reasons, it is better to mark it as flaky with the `flaky` and `flakyIf`
+combinators. They both take a ticket number so the flaky tests has to be tracked
+in an issue. Flaky tests are executed, and the outcome is reported by the
+test-suite but even if they fail they won't make the test-suite fail.
+
 **I programmatically modified a file in my test suite, but Cabal/GHC
 doesn't seem to be picking it up.**  You need to sleep sufficiently
 long before editing a file, in order for file system timestamp
 resolution to pick it up.  Use `withDelay` and `delay` prior to
 making a modification.
 
-**How do I mark a test as broken?**  Use `expectBroken`, which takes
-the ticket number as its first argument.  Note that this does NOT
-handle accept-test brokenness, so you will have to add a manual
-string output test, if that is how your test is "failing."
-
 Hermetic tests
 --------------
 
diff --git a/cabal-testsuite/main/cabal-tests.hs b/cabal-testsuite/main/cabal-tests.hs
index 517416a8773..4ffdadd4352 100644
--- a/cabal-testsuite/main/cabal-tests.hs
+++ b/cabal-testsuite/main/cabal-tests.hs
@@ -28,7 +28,7 @@ import qualified System.Clock as Clock
 import System.IO
 import System.FilePath
 import System.Exit
-import System.Process (callProcess, showCommandForUser)
+import System.Process (readProcessWithExitCode, showCommandForUser)
 import System.Directory
 import Distribution.Pretty
 import Data.Maybe
@@ -238,7 +238,21 @@ main = do
             -- Simple runner
             (real_path, real_args) <- runTest (runnerCommand senv) path
             hPutStrLn stderr $ showCommandForUser real_path real_args
-            callProcess real_path real_args
+            -- If the test was reported flaky, the `runghc` call will exit
+            -- with exit code 1, and report `TestCodeFlaky` on the stderr output
+            --
+            -- This seems to be the only way to catch this case.
+            --
+            -- Sadly it means that stdout and stderr are not interleaved
+            -- directly anymore.
+            (e, out, err) <- readProcessWithExitCode real_path real_args ""
+            putStrLn "# STDOUT:"
+            putStrLn out
+            putStrLn "# STDERR:"
+            putStrLn err
+            if "TestCodeFlaky" `isInfixOf` err
+              then pure ()
+              else throwIO e
             hPutStrLn stderr "OK"
         user_paths -> do
             -- Read out tests from filesystem
@@ -263,6 +277,8 @@ main = do
             unexpected_fails_var  <- newMVar []
             unexpected_passes_var <- newMVar []
             skipped_var <- newMVar []
+            flaky_pass_var <- newMVar []
+            flaky_fail_var <- newMVar []
 
             chan <- newChan
             let logAll msg = writeChan chan (ServerLogMsg AllServers msg)
@@ -315,7 +331,7 @@ main = do
                                 modifyMVar_ unexpected_fails_var $ \paths ->
                                     return (path:paths)
 
-                            when (code == TestCodeUnexpectedOk) $
+                            when (isJust $ isTestCodeUnexpectedSuccess code) $
                                 modifyMVar_ unexpected_passes_var $ \paths ->
                                     return (path:paths)
 
@@ -323,6 +339,12 @@ main = do
                                 modifyMVar_ skipped_var $ \paths ->
                                     return (path:paths)
 
+                            case isTestCodeFlaky code of
+                              NotFlaky  -> pure ()
+                              Flaky b _ ->
+                                modifyMVar_ (if b then flaky_pass_var else flaky_fail_var) $ \paths ->
+                                    return (path:paths)
+
                             go server
 
             -- Start as many threads as requested by -j to spawn
@@ -333,13 +355,17 @@ main = do
             unexpected_fails  <- takeMVar unexpected_fails_var
             unexpected_passes <- takeMVar unexpected_passes_var
             skipped           <- takeMVar skipped_var
+            flaky_passes      <- takeMVar flaky_pass_var
+            flaky_fails       <- takeMVar flaky_fail_var
 
             -- print summary
             let sl = show . length
                 testSummary =
                   sl all_tests ++ " tests, " ++ sl skipped ++ " skipped, "
                     ++ sl unexpected_passes ++ " unexpected passes, "
-                    ++ sl unexpected_fails ++ " unexpected fails."
+                    ++ sl unexpected_fails ++ " unexpected fails, "
+                    ++ sl flaky_passes ++ " flaky passes, "
+                    ++ sl flaky_fails ++ " flaky fails."
             logAll testSummary
 
             -- print failed or unexpected ok
diff --git a/cabal-testsuite/src/Test/Cabal/Monad.hs b/cabal-testsuite/src/Test/Cabal/Monad.hs
index db685c45379..38534402d26 100644
--- a/cabal-testsuite/src/Test/Cabal/Monad.hs
+++ b/cabal-testsuite/src/Test/Cabal/Monad.hs
@@ -51,14 +51,19 @@ module Test.Cabal.Monad (
     skipUnless,
     skipUnlessIO,
     -- * Known broken tests
-    expectedBroken,
-    unexpectedSuccess,
+    expectBroken,
+    expectBrokenIf,
+    expectBrokenUnless,
+    -- * Flaky tests
+    flaky,
+    flakyIf,
     -- * Arguments (TODO: move me)
     CommonArgs(..),
     renderCommonArgs,
     commonArgParser,
     -- * Version Constants
     cabalVersionLibrary,
+
 ) where
 
 import Test.Cabal.Script
@@ -78,10 +83,11 @@ import Distribution.Simple.Configure
 import qualified Distribution.Simple.Utils as U (cabalVersion)
 import Distribution.Text
 
-import Test.Utils.TempTestDir (removeDirectoryRecursiveHack)
+import Test.Utils.TempTestDir (removeDirectoryRecursiveHack, withTestDir')
 import Distribution.Verbosity
 import Distribution.Version
 
+import Control.Concurrent.Async
 #if !MIN_VERSION_base(4,11,0)
 import Data.Monoid ((<>))
 #endif
@@ -178,9 +184,11 @@ testArgParser = TestArgs
     <*> argument str ( metavar "FILE")
     <*> commonArgParser
 
+-- * skip tests
+
 skipIO :: String -> IO ()
 skipIO reason = do
-    putStrLn ("SKIP " ++ reason)
+    putStrLn $ "SKIP (" <> reason <> ")"
     E.throwIO (TestCodeSkip reason)
 
 skip :: String -> TestM ()
@@ -198,16 +206,67 @@ skipUnlessIO reason b = unless b (skipIO reason)
 skipUnless :: String -> Bool -> TestM ()
 skipUnless reason b = unless b (skip reason)
 
-expectedBroken :: Int -> TestM a
-expectedBroken t = liftIO $ do
-    putStrLn "EXPECTED FAIL"
-    E.throwIO (TestCodeKnownFail t)
-
-unexpectedSuccess :: TestM a
-unexpectedSuccess = liftIO $ do
-    putStrLn "UNEXPECTED OK"
-    E.throwIO TestCodeUnexpectedOk
+-- * Broken tests
 
+expectBroken :: IssueID -> TestM a -> TestM a
+expectBroken ticket m = do
+    env <- getTestEnv
+    liftIO . withAsync (runReaderT m env) $ \a -> do
+        r <- waitCatch a
+        case r of
+            Left e  -> do
+                putStrLn $ "This test is known broken, see #" ++ show ticket ++ ":"
+                print e
+                throwExpectedBroken ticket
+            Right _ -> do
+                throwUnexpectedSuccess ticket
+
+expectBrokenIf :: Bool -> IssueID -> TestM a -> TestM a
+expectBrokenIf True ticket m = expectBroken ticket m
+expectBrokenIf False _ m = m
+
+expectBrokenUnless :: Bool -> IssueID -> TestM a -> TestM a
+expectBrokenUnless b = expectBrokenIf (not b)
+
+throwExpectedBroken :: IssueID -> IO a
+throwExpectedBroken ticket = do
+    putStrLn $ "EXPECTED FAIL (#" <> show ticket <> ")"
+    E.throwIO (TestCodeKnownFail ticket)
+
+throwUnexpectedSuccess :: IssueID -> IO a
+throwUnexpectedSuccess ticket = do
+    putStrLn $ "UNEXPECTED OK (#" <> show ticket <> ")"
+    E.throwIO (TestCodeUnexpectedOk ticket)
+
+-- * Flaky tests
+
+flaky :: IssueID -> TestM a -> TestM a
+flaky ticket m = do
+    env <- getTestEnv
+    liftIO . withAsync (runReaderT m env) $ \a -> do
+        r <- waitCatch a
+        case r of
+            Left e  -> do
+                putStrLn $ "This test is known flaky, and it failed, see #" ++ show ticket ++ ":"
+                print e
+                throwFlakyFail ticket
+            Right _ -> do
+                putStrLn $ "This test is known flaky, but it passed, see #" ++ show ticket ++ ":"
+                throwFlakyPass ticket
+
+flakyIf :: Bool -> IssueID -> TestM a -> TestM a
+flakyIf True ticket m = flaky ticket m
+flakyIf False _ m = m
+
+throwFlakyFail :: IssueID -> IO a
+throwFlakyFail ticket = do
+    putStrLn $ "FLAKY FAIL (#" <> show ticket <> ")"
+    E.throwIO (TestCodeFlakyFailed ticket)
+
+throwFlakyPass :: IssueID -> IO a
+throwFlakyPass ticket = do
+    putStrLn $ "FLAKY OK (#" <> show ticket <> ")"
+    E.throwIO (TestCodeFlakyPassed ticket)
 
 trySkip :: IO a -> IO (Either String a)
 trySkip m = fmap Right m `E.catch` \e -> case e of
@@ -258,15 +317,10 @@ python3Program :: Program
 python3Program = simpleProgram "python3"
 
 -- | Run a test in the test monad according to program's arguments.
-runTestM :: String -> TestM a -> IO a
+runTestM :: String -> TestM () -> IO ()
 runTestM mode m =
-    liftIO $ (canonicalizePath =<< getTemporaryDirectory) >>= \systemTmpDir ->
-       -- canonicalizePath: cabal-install is inconsistent w.r.t. looking through
-       -- symlinks. We canonicalize here to avoid such issues when the temporary
-       -- directory contains symlinks. See #9763.
     execParser (info testArgParser Data.Monoid.mempty) >>= \args ->
-    withTempDirectoryEx verbosity (defaultTempFileOptions { optKeepTempFiles = argKeepTmpFiles (testCommonArgs args) })
-                               systemTmpDir
+    withTestDir' verbosity (defaultTempFileOptions { optKeepTempFiles = argKeepTmpFiles (testCommonArgs args) })
                                "cabal-testsuite" $ \tmp_dir -> do
     let dist_dir = testArgDistDir args
         (script_dir0, script_filename) = splitFileName (testArgScriptPath args)
@@ -368,11 +422,26 @@ runTestM mode m =
                     testRecordUserMode = Nothing,
                     testMaybeStoreDir = Nothing
                 }
-    let go = do cleanup
-                r <- withSourceCopy m
-                check_expect (argAccept (testCommonArgs args))
-                return r
-    runReaderT go env
+    runReaderT cleanup env
+    join $ E.catch (runReaderT
+                (do
+                    withSourceCopy m
+                    check_expect (argAccept (testCommonArgs args)) Nothing
+                )
+                env
+        )
+        (\(e :: TestCode) -> do
+            -- A test that resulted in unexpected success should check its output
+            -- because maybe it is the output the one that makes it fail!
+            case isTestCodeUnexpectedSuccess e of
+                Just t  -> runReaderT (check_expect (argAccept (testCommonArgs args)) (Just (t, False))) env
+                Nothing ->
+                    -- A test that is reported flaky but passed might fail because of the output
+                    case isTestCodeFlaky e of
+                        Flaky True t  -> runReaderT (check_expect (argAccept (testCommonArgs args)) (Just (t, True))) env
+                        _             -> E.throwIO e
+        )
+
   where
     verbosity = normal -- TODO: configurable
 
@@ -388,13 +457,15 @@ runTestM mode m =
         liftIO $ writeFile (testUserCabalConfigFile env)
                $ unlines [ "with-compiler: " ++ ghc_path ]
 
-    check_expect accept = do
+    check_expect accept was_expected_to_fail = do
         env <- getTestEnv
         actual_raw <- liftIO $ readFileOrEmpty (testActualFile env)
         expect <- liftIO $ readFileOrEmpty (testExpectFile env)
         norm_env <- mkNormalizerEnv
         let actual = normalizeOutput norm_env actual_raw
-        when (words actual /= words expect) $ do
+        case (was_expected_to_fail, words actual /= words expect) of
+         -- normal test, output doesn't match
+         (Nothing, True) -> do
             -- First try whitespace insensitive diff
             let actual_fp = testNormalizedActualFile env
                 expect_fp = testNormalizedExpectFile env
@@ -406,7 +477,23 @@ runTestM mode m =
             if accept
                 then do liftIO $ putStrLn "Accepting new output."
                         liftIO $ writeFileNoCR (testExpectFile env) actual
-                else liftIO $ exitWith (ExitFailure 1)
+                        pure (pure ())
+                else pure (E.throwIO TestCodeFail)
+         -- normal test, output matches
+         (Nothing, False) -> pure (pure ())
+         -- expected fail, output matches
+         (Just (t, was_flaky), False) -> pure (E.throwIO $ if was_flaky then TestCodeFlakyPassed t else TestCodeUnexpectedOk t)
+         -- expected fail, output doesn't match
+         (Just (t, was_flaky), True) -> do
+            -- First try whitespace insensitive diff
+            let actual_fp = testNormalizedActualFile env
+                expect_fp = testNormalizedExpectFile env
+            liftIO $ writeFile actual_fp actual
+            liftIO $ writeFile expect_fp expect
+            liftIO $ putStrLn "Actual output differs from expected:"
+            b <- diff ["-uw"] expect_fp actual_fp
+            unless b . void $ diff ["-u"] expect_fp actual_fp
+            pure (E.throwIO $ if was_flaky then TestCodeFlakyFailed t else TestCodeKnownFail t)
 
 readFileOrEmpty :: FilePath -> IO String
 readFileOrEmpty f = readFile f `E.catch` \e ->
@@ -543,6 +630,7 @@ mkNormalizerEnv = do
     tmpDir <- liftIO $ getTemporaryDirectory
 
     canonicalizedTestTmpDir <- liftIO $ canonicalizePath (testTmpDir env)
+    canonicalizedGblDir <- liftIO $ canonicalizePath tmpDir
 
     -- 'cabal' is configured in the package-db, but doesn't specify how to find the program version
     -- Thus we find the program location, if it exists, and query for the program version for
@@ -555,11 +643,6 @@ mkNormalizerEnv = do
                 liftIO (findProgramVersion "--numeric-version" id (testVerbosity env) (programPath cabalProg))
 
     return NormalizerEnv {
-        normalizerRoot
-            = (if buildOS == Windows
-              then joinDrive "\\" . dropDrive
-              else id)
-                $ addTrailingPathSeparator (testSourceDir env),
         normalizerTmpDir
             = (if buildOS == Windows
               then joinDrive "\\" . dropDrive
@@ -575,6 +658,11 @@ mkNormalizerEnv = do
               then joinDrive "\\" . dropDrive
               else id)
                 $ addTrailingPathSeparator tmpDir,
+        normalizerCanonicalGblTmpDir
+            = (if buildOS == Windows
+              then joinDrive "\\" . dropDrive
+              else id)
+                $ addTrailingPathSeparator canonicalizedGblDir,
         normalizerGhcVersion
             = compilerVersion (testCompiler env),
         normalizerGhcPath
@@ -791,8 +879,11 @@ testUserCabalConfigFile :: TestEnv -> FilePath
 testUserCabalConfigFile env = testCabalDir env </> "config"
 
 -- | The file where the expected output of the test lives
+--
+-- Pointing to the @testTmpDir@ allows us to modify the expected output if
+-- needed, to adapt it to outcomes of previous steps in the test.
 testExpectFile :: TestEnv -> FilePath
-testExpectFile env = testSourceDir env </> testName env <.> "out"
+testExpectFile env = testTmpDir env </> testName env <.> "out"
 
 -- | Where we store the actual output
 testActualFile :: TestEnv -> FilePath
diff --git a/cabal-testsuite/src/Test/Cabal/OutputNormalizer.hs b/cabal-testsuite/src/Test/Cabal/OutputNormalizer.hs
index 3f4dcf5bc5a..33e1522526b 100644
--- a/cabal-testsuite/src/Test/Cabal/OutputNormalizer.hs
+++ b/cabal-testsuite/src/Test/Cabal/OutputNormalizer.hs
@@ -38,6 +38,7 @@ normalizeOutput nenv =
     -- string search-replace.  Make sure we do this before backslash
     -- normalization!
   . resub (posixRegexEscape (normalizerGblTmpDir nenv) ++ "[a-z0-9\\.-]+") "<GBLTMPDIR>"
+  . resub (posixRegexEscape (normalizerCanonicalGblTmpDir nenv) ++ "[a-z0-9\\.-]+") "<GBLTMPDIR>"
     -- Munge away .exe suffix on filenames (Windows)
   . (if buildOS == Windows then resub "([A-Za-z0-9.-]+)\\.exe" "\\1" else id)
     -- tmp/src-[0-9]+ is tmp\src-[0-9]+ in Windows
@@ -123,12 +124,21 @@ normalizeOutput nenv =
               "\"-package-id\",\"<PACKAGEDEP>\""
 
 data NormalizerEnv = NormalizerEnv
-    { normalizerRoot          :: FilePath
-    , normalizerTmpDir        :: FilePath
+    { normalizerTmpDir        :: FilePath
     , normalizerCanonicalTmpDir :: FilePath
     -- ^ May differ from 'normalizerTmpDir', especially e.g. on macos, where
     -- `/var` is a symlink for `/private/var`.
     , normalizerGblTmpDir     :: FilePath
+    -- ^ The global temp directory: @/tmp@ on Linux, @/var/folders/...@ on macos
+    -- and @\\msys64\\tmp@ on Windows.
+    --
+    -- Note that on windows the actual path would be @C:\\msys64\\tmp@ but we
+    -- drop the @C:@ prefix because this path appears sometimes
+    -- twice in the same path in some tests, and the second time it doesn't have a @C:@, so
+    -- the logic fails to catch it.
+    , normalizerCanonicalGblTmpDir :: FilePath
+    -- ^ The canonical version of 'normalizerGblTmpDir', might differ in the same
+    -- way as above on macos
     , normalizerGhcVersion    :: Version
     , normalizerGhcPath    :: FilePath
     , normalizerKnownPackages :: [PackageId]
diff --git a/cabal-testsuite/src/Test/Cabal/Prelude.hs b/cabal-testsuite/src/Test/Cabal/Prelude.hs
index 7fa82cf8143..7a8fbf651e8 100644
--- a/cabal-testsuite/src/Test/Cabal/Prelude.hs
+++ b/cabal-testsuite/src/Test/Cabal/Prelude.hs
@@ -22,6 +22,7 @@ import Test.Cabal.Script
 import Test.Cabal.Run
 import Test.Cabal.Monad
 import Test.Cabal.Plan
+import Test.Cabal.TestCode
 
 import Distribution.Compat.Time (calibrateMtimeChangeDelay)
 import Distribution.Simple.Compiler (PackageDBStackCWD, PackageDBCWD, PackageDBX(..))
@@ -49,7 +50,7 @@ import Distribution.Compat.Stack
 
 import Text.Regex.TDFA ((=~))
 
-import Control.Concurrent.Async (waitCatch, withAsync)
+import Control.Concurrent.Async (withAsync)
 import qualified Data.Aeson as JSON
 import qualified Data.ByteString.Lazy as BSL
 import Control.Monad (unless, when, void, forM_, liftM2, liftM4)
@@ -59,7 +60,7 @@ import qualified Crypto.Hash.SHA256 as SHA256
 import qualified Data.ByteString.Base16 as Base16
 import qualified Data.ByteString.Char8 as C
 import Data.List (isInfixOf, stripPrefix, isPrefixOf, intercalate)
-import Data.Maybe (mapMaybe, fromMaybe)
+import Data.Maybe (isJust, mapMaybe, fromMaybe)
 import System.Exit (ExitCode (..))
 import System.FilePath
 import Control.Concurrent (threadDelay)
@@ -68,6 +69,7 @@ import System.Directory
 import Control.Retry (exponentialBackoff, limitRetriesByCumulativeDelay)
 import Network.Wait (waitTcpVerbose)
 import System.Environment
+import System.Process
 
 #ifndef mingw32_HOST_OS
 import Control.Monad.Catch ( bracket_ )
@@ -690,7 +692,12 @@ withRemoteRepo repoDir m = do
             -- wait for the python webserver to come up with a exponential
             -- backoff starting from 50ms, up to a maximum wait of 60s
             _ <- waitTcpVerbose putStrLn (limitRetriesByCumulativeDelay 60000000 $ exponentialBackoff 50000) "localhost" "8000"
-            runReaderT m (env { testHaveRepo = True }))
+            r <- runReaderT m (env { testHaveRepo = True })
+            -- Windows fails to kill the python server when the function above
+            -- is complete, so we kill it directly via CMD.
+            when (buildOS == Windows) $ void $ createProcess_ "kill python" $ System.Process.shell "taskkill /F /IM python3.exe"
+            pure r
+            )
 
 
 
@@ -862,6 +869,9 @@ hasSharedLibraries = testCompilerWithArgs ["-dynamic"]
 skipIfNoSharedLibraries :: TestM ()
 skipIfNoSharedLibraries = skipUnless "no shared libraries" =<< hasSharedLibraries
 
+skipIfNoProfiledLibraries :: TestM ()
+skipIfNoProfiledLibraries = skipUnless "no profiled libraries" =<< hasProfiledLibraries
+
 -- | Check if the GHC that is used for compiling package tests has
 -- a shared library of the cabal library under test in its database.
 --
@@ -936,6 +946,9 @@ skipIfJavaScript = skipIfIO "incompatible with the JavaScript backend" isJavaScr
 isWindows :: Bool
 isWindows = buildOS == Windows
 
+isCI :: IO Bool
+isCI = isJust <$> lookupEnv "CI"
+
 isOSX :: Bool
 isOSX = buildOS == OSX
 
@@ -953,6 +966,55 @@ skipIfWindows why = skipIfIO ("Windows " <> why) isWindows
 skipUnlessWindows :: IO ()
 skipUnlessWindows = skipIfIO "Only interesting in Windows" (not isWindows)
 
+skipIfOSX :: String -> IO ()
+skipIfOSX why = skipIfIO ("OSX " <> why) isOSX
+
+skipIfCI :: IssueID -> IO ()
+skipIfCI ticket = skipIfIO ("CI, see #" <> show ticket) =<< isCI
+
+skipIfCIAndWindows :: IssueID -> IO ()
+skipIfCIAndWindows ticket = skipIfIO ("Windows CI, see #" <> show ticket) . (isWindows &&) =<< isCI
+
+skipIfCIAndOSX :: IssueID -> IO ()
+skipIfCIAndOSX ticket = skipIfIO ("OSX CI, see #" <> show ticket) . (isOSX &&) =<< isCI
+
+expectBrokenIfWindows :: IssueID -> TestM a -> TestM a
+expectBrokenIfWindows ticket = expectBrokenIf isWindows ticket
+
+expectBrokenIfWindowsCI :: IssueID -> TestM a -> TestM a
+expectBrokenIfWindowsCI ticket m = do
+    ci <- liftIO isCI
+    expectBrokenIf (isWindows && ci) ticket m
+
+expectBrokenIfWindowsCIAndGhc :: String -> IssueID -> TestM a -> TestM a
+expectBrokenIfWindowsCIAndGhc range ticket m = do
+    ghcVer <- isGhcVersion range
+    ci <- liftIO isCI
+    expectBrokenIf (isWindows && ghcVer && ci) ticket m
+
+expectBrokenIfWindowsAndGhc :: String -> IssueID -> TestM a -> TestM a
+expectBrokenIfWindowsAndGhc range ticket m = do
+    ghcVer <- isGhcVersion range
+    expectBrokenIf (isWindows && ghcVer) ticket m
+
+expectBrokenIfOSXAndGhc :: String -> IssueID -> TestM a -> TestM a
+expectBrokenIfOSXAndGhc range ticket m = do
+    ghcVer <- isGhcVersion range
+    expectBrokenIf (isOSX && ghcVer) ticket m
+
+expectBrokenIfGhc :: String -> IssueID -> TestM a -> TestM a
+expectBrokenIfGhc range ticket m = do
+    ghcVer <- isGhcVersion range
+    expectBrokenIf ghcVer ticket m
+
+flakyIfCI :: IssueID -> TestM a -> TestM a
+flakyIfCI ticket m = do
+    ci <- liftIO isCI
+    flakyIf ci ticket m
+
+flakyIfWindows :: IssueID -> TestM a -> TestM a
+flakyIfWindows ticket m = flakyIf isWindows ticket m
+
 getOpenFilesLimit :: TestM (Maybe Integer)
 #ifdef mingw32_HOST_OS
 -- No MS-specified limit, was determined experimentally on Windows 10 Pro x64,
@@ -977,29 +1039,6 @@ getOpenFilesLimit = liftIO $ do
 hasNewBuildCompatBootCabal :: TestM Bool
 hasNewBuildCompatBootCabal = isGhcVersion ">= 7.9"
 
-------------------------------------------------------------------------
--- * Broken tests
-
-expectBroken :: Int -> TestM a -> TestM a
-expectBroken ticket m = do
-    env <- getTestEnv
-    liftIO . withAsync (runReaderT m env) $ \a -> do
-        r <- waitCatch a
-        case r of
-            Left e  -> do
-                putStrLn $ "This test is known broken, see #" ++ show ticket ++ ":"
-                print e
-                runReaderT (expectedBroken ticket) env
-            Right _ -> do
-                runReaderT unexpectedSuccess env
-
-expectBrokenIf :: Bool -> Int -> TestM a -> TestM a
-expectBrokenIf False _ m = m
-expectBrokenIf True ticket m = expectBroken ticket m
-
-expectBrokenUnless :: Bool -> Int -> TestM a -> TestM a
-expectBrokenUnless b = expectBrokenIf (not b)
-
 -- * Programs
 
 git :: String -> [String] -> TestM ()
diff --git a/cabal-testsuite/src/Test/Cabal/TestCode.hs b/cabal-testsuite/src/Test/Cabal/TestCode.hs
index 800269c89d4..fc24b216285 100644
--- a/cabal-testsuite/src/Test/Cabal/TestCode.hs
+++ b/cabal-testsuite/src/Test/Cabal/TestCode.hs
@@ -1,12 +1,19 @@
-{-# LANGUAGE CPP                #-}
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE DeriveDataTypeable #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
 -- | Exception type like 'ExitCode' but with more information
 -- than just integer.
 module Test.Cabal.TestCode (
     -- * TestCode
     TestCode (..),
+    FlakyStatus (..),
+    IssueID (..),
     displayTestCode,
     isTestCodeSkip,
+    isTestCodeFlaky,
+    isTestCodeUnexpectedSuccess,
 ) where
 
 import Control.Exception (Exception (..))
@@ -19,9 +26,11 @@ import Data.Typeable     (Typeable)
 data TestCode
     = TestCodeOk
     | TestCodeSkip String
-    | TestCodeKnownFail Int
-    | TestCodeUnexpectedOk
+    | TestCodeKnownFail IssueID
+    | TestCodeUnexpectedOk IssueID
     | TestCodeFail
+    | TestCodeFlakyFailed IssueID
+    | TestCodeFlakyPassed IssueID
   deriving (Eq, Show, Read, Typeable)
 
 instance Exception TestCode
@@ -29,12 +38,32 @@ instance Exception TestCode
     displayException = displayTestCode
 
 displayTestCode :: TestCode -> String
-displayTestCode TestCodeOk            = "OK"
-displayTestCode (TestCodeSkip msg)    = "SKIP " ++ msg
-displayTestCode (TestCodeKnownFail t) = "OK (known failure, see #" <> show t <> ")"
-displayTestCode TestCodeUnexpectedOk  = "FAIL (unexpected success)"
-displayTestCode TestCodeFail          = "FAIL"
+displayTestCode TestCodeOk               = "OK"
+displayTestCode (TestCodeSkip msg)       = "SKIP " ++ msg
+displayTestCode (TestCodeKnownFail t)    = "OK (known failure, see #" <> show t <> ")"
+displayTestCode (TestCodeUnexpectedOk t) = "FAIL (unexpected success, see #" <> show t <> ")"
+displayTestCode TestCodeFail             = "FAIL"
+displayTestCode (TestCodeFlakyFailed t)  = "FLAKY (FAIL, see #" <> show t <> ")"
+displayTestCode (TestCodeFlakyPassed t)  = "FLAKY (OK, see #" <> show t <> ")"
 
 isTestCodeSkip :: TestCode -> Bool
 isTestCodeSkip (TestCodeSkip _) = True
 isTestCodeSkip _                = False
+
+type TestPassed = Bool
+
+newtype IssueID = IssueID Int
+  deriving newtype (Eq, Typeable, Num, Show, Read)
+
+data FlakyStatus
+  = NotFlaky
+  | Flaky TestPassed IssueID
+
+isTestCodeFlaky :: TestCode -> FlakyStatus
+isTestCodeFlaky (TestCodeFlakyPassed t) = Flaky True t
+isTestCodeFlaky (TestCodeFlakyFailed t) = Flaky False t
+isTestCodeFlaky _                       = NotFlaky
+
+isTestCodeUnexpectedSuccess :: TestCode -> Maybe IssueID
+isTestCodeUnexpectedSuccess (TestCodeUnexpectedOk t) = Just t
+isTestCodeUnexpectedSuccess _                        = Nothing