Use estimated formatted lengths to optimize performance of VT rendering

src/host/ut_host/VtRendererTests.cpp

@@ -113,6 +113,8 @@ class Microsoft::Console::Render::VtRendererTest
     TEST_METHOD(WinTelnetTestColors);
     TEST_METHOD(WinTelnetTestCursor);
 
+    TEST_METHOD(FormattedString);
+
     TEST_METHOD(TestWrapping);
 
     TEST_METHOD(TestResize);
@@ -1472,3 +1474,35 @@ void VtRendererTest::TestCursorVisibility()
     VERIFY_IS_FALSE(engine->_nextCursorIsVisible);
     VERIFY_IS_FALSE(engine->_needToDisableCursor);
 }
+
+void VtRendererTest::FormattedString()
+{
+    // This test works with a static cache variable that
+    // can be affected by other tests
+    BEGIN_TEST_METHOD_PROPERTIES()
+        TEST_METHOD_PROPERTY(L"IsolationLevel", L"Method")
+    END_TEST_METHOD_PROPERTIES();
+
+    static const std::string format("\x1b[%dm");
+    const auto value = 12;
+
+    Viewport view = SetUpViewport();
+    wil::unique_hfile hFile = wil::unique_hfile(INVALID_HANDLE_VALUE);
+    auto engine = std::make_unique<Xterm256Engine>(std::move(hFile), p, view, g_ColorTable, static_cast<WORD>(COLOR_TABLE_SIZE));
+    auto pfn = std::bind(&VtRendererTest::WriteCallback, this, std::placeholders::_1, std::placeholders::_2);
+    engine->SetTestCallback(pfn);
+
+    Log::Comment(L"1.) Write it once. It should resize itself.");
+    qExpectedInput.push_back("\x1b[12m");
+    VERIFY_SUCCEEDED(engine->_WriteFormattedString(&format, value));
+
+    Log::Comment(L"2.) Write the same thing again, should be fine.");
+    qExpectedInput.push_back("\x1b[12m");
+    VERIFY_SUCCEEDED(engine->_WriteFormattedString(&format, value));
+
+    Log::Comment(L"3.) Now write something huge. Should resize itself and still be fine.");
+    static const std::string bigFormat("\x1b[28;3;%d;%d;%dm");
+    const auto bigValue = 500;
+    qExpectedInput.push_back("\x1b[28;3;500;500;500m");
+    VERIFY_SUCCEEDED(engine->_WriteFormattedString(&bigFormat, bigValue, bigValue, bigValue));
+}

src/renderer/vt/VtSequences.cpp

@@ -82,7 +82,6 @@ using namespace Microsoft::Console::Render;
 [[nodiscard]] HRESULT VtEngine::_EraseCharacter(const short chars) noexcept
 {
     static const std::string format = "\x1b[%dX";
-
     return _WriteFormattedString(&format, chars);
 }
 
@@ -95,7 +94,6 @@ using namespace Microsoft::Console::Render;
 [[nodiscard]] HRESULT VtEngine::_CursorForward(const short chars) noexcept
 {
     static const std::string format = "\x1b[%dC";
-
     return _WriteFormattedString(&format, chars);
 }
 

src/renderer/vt/state.cpp

@@ -187,35 +187,66 @@ VtEngine::VtEngine(_In_ wil::unique_hfile pipe,
 // - Helper for calling _Write with a string for formatting a sequence. Used
 //      extensively by VtSequences.cpp
 // Arguments:
-// - pFormat: the pointer to the string to write to the pipe.
+// - pFormat: pointer to format string to write to the pipe
 // - ...: a va_list of args to format the string with.
 // Return Value:
 // - S_OK, E_INVALIDARG for a invalid format string, or suitable HRESULT error
 //      from writing pipe.
 [[nodiscard]] HRESULT VtEngine::_WriteFormattedString(const std::string* const pFormat, ...) noexcept
+try
 {
+    va_list args;
+    va_start(args, pFormat);
+
+    // NOTE: pFormat is a pointer because varargs refuses to operate with a ref in that position
+    // NOTE: We're not using string_view because it doesn't guarantee null (which will be needed
+    //       later in the formatting method).
+
     HRESULT hr = E_FAIL;
-    va_list argList;
-    va_start(argList, pFormat);
 
-    int cchNeeded = _scprintf(pFormat->c_str(), argList);
+    // We're going to hold onto our format string space across calls because
+    // the VT renderer will be formatting a LOT of strings and alloc/freeing them
+    // over and over is going to be way worse for perf than just holding some extra
+    // memory for formatting purposes.
+    // See _formatBuffer for its location.
+
+    // First, plow ahead using our pre-reserved string space.
+    LPSTR destEnd = nullptr;
+    size_t destRemaining = 0;
+    if (SUCCEEDED(StringCchVPrintfExA(_formatBuffer.data(),
+                                      _formatBuffer.size(),
+                                      &destEnd,
+                                      &destRemaining,
+                                      STRSAFE_NO_TRUNCATION,
+                                      pFormat->c_str(),
+                                      args)))
+    {
+        return _Write({ _formatBuffer.data(), _formatBuffer.size() - destRemaining });
+    }
+
+    // If we didn't succeed at filling/using the existing space, then
+    // we're going to take the long way by counting the space required and resizing up to that
+    // space and formatting.
+
+    const auto needed = _scprintf(pFormat->c_str(), args);
     // -1 is the _scprintf error case https://msdn.microsoft.com/en-us/library/t32cf9tb.aspx
-    if (cchNeeded > -1)
+    if (needed > -1)
     {
-        wistd::unique_ptr<char[]> psz = wil::make_unique_nothrow<char[]>(cchNeeded + 1);
-        RETURN_IF_NULL_ALLOC(psz);
+        _formatBuffer.resize(static_cast<size_t>(needed) + 1);
 
-        int cchWritten = _vsnprintf_s(psz.get(), cchNeeded + 1, cchNeeded, pFormat->c_str(), argList);
-        hr = _Write({ psz.get(), gsl::narrow<size_t>(cchWritten) });
+        const auto written = _vsnprintf_s(_formatBuffer.data(), _formatBuffer.size(), needed, pFormat->c_str(), args);
+        hr = _Write({ _formatBuffer.data(), gsl::narrow<size_t>(written) });
     }
     else
     {
         hr = E_INVALIDARG;
     }
 
-    va_end(argList);
+    va_end(args);
+
     return hr;
 }
+CATCH_RETURN();
 
 // Method Description:
 // - This method will update the active font on the current device context

src/renderer/vt/vtrenderer.hpp

@@ -110,6 +110,8 @@ namespace Microsoft::Console::Render
         wil::unique_hfile _hFile;
         std::string _buffer;
 
+        std::string _formatBuffer;
+
         const Microsoft::Console::IDefaultColorProvider& _colorProvider;
 
         COLORREF _LastFG;