diff --git a/src/renderer/dx/CustomTextRenderer.cpp b/src/renderer/dx/CustomTextRenderer.cpp
index dcc69364c8d..be589662507 100644
--- a/src/renderer/dx/CustomTextRenderer.cpp
+++ b/src/renderer/dx/CustomTextRenderer.cpp
@@ -273,7 +273,12 @@ using namespace Microsoft::Console::Render;
     rect.right = std::accumulate(advancesSpan.cbegin(), advancesSpan.cend(), rect.right);
 
     // Clip all drawing in this glyph run to where we expect.
-    d2dContext->PushAxisAlignedClip(rect, D2D1_ANTIALIAS_MODE_PER_PRIMITIVE);
+    // We need the AntialiasMode here to be Aliased to ensure
+    //  that background boxes line up with each other and don't leave behind
+    //  stray colors.
+    // See GH#3626 for more details.
+    d2dContext->PushAxisAlignedClip(rect, D2D1_ANTIALIAS_MODE_ALIASED);
+
     // Ensure we pop it on the way out
     auto popclip = wil::scope_exit([&d2dContext]() noexcept {
         d2dContext->PopAxisAlignedClip();
diff --git a/src/renderer/dx/DxRenderer.cpp b/src/renderer/dx/DxRenderer.cpp
index d5fed132ab5..77c5bffb755 100644
--- a/src/renderer/dx/DxRenderer.cpp
+++ b/src/renderer/dx/DxRenderer.cpp
@@ -533,6 +533,11 @@ void DxEngine::_ComputePixelShaderSettings() noexcept
                                                                     &props,
                                                                     &_d2dRenderTarget));
 
+        // We need the AntialiasMode for non-text object to be Aliased to ensure
+        //  that background boxes line up with each other and don't leave behind
+        //  stray colors.
+        // See GH#3626 for more details.
+        _d2dRenderTarget->SetAntialiasMode(D2D1_ANTIALIAS_MODE_ALIASED);
         _d2dRenderTarget->SetTextAntialiasMode(_antialiasingMode);
 
         RETURN_IF_FAILED(_d2dRenderTarget->CreateSolidColorBrush(D2D1::ColorF(D2D1::ColorF::DarkRed),