Prototype for Windows Terminal: Use notifications instead of monitoring for new text

source/NVDAObjects/UIA/__init__.py

@@ -41,6 +41,7 @@
 	iterUIARangeByUnit,
 	UIAMixedAttributeError,
 	UIATextRangeFromElement,
+	_shouldUseWindowsTerminalNotifications,
 )
 from NVDAObjects.window import Window
 from NVDAObjects import (
@@ -1211,7 +1212,10 @@ def findOverlayClasses(self,clsList):
 			winConsoleUIA.findExtraOverlayClasses(self, clsList)
 		elif UIAClassName in _all_wt_UIAClassNames:
 			from . import winConsoleUIA
-			clsList.append(winConsoleUIA.WinTerminalUIA)
+			if _shouldUseWindowsTerminalNotifications():
+				clsList.append(winConsoleUIA._NotificationsBasedWinTerminalUIA)
+			else:
+				clsList.append(winConsoleUIA._DiffBasedWinTerminalUIA)
 
 		# Add editableText support if UIA supports a text pattern
 		if self.TextInfo==UIATextInfo:
@@ -2100,7 +2104,13 @@ def event_UIA_systemAlert(self):
 		# Ideally, we wouldn't use getPropertiesBraille directly.
 		braille.handler.message(braille.getPropertiesBraille(name=self.name, role=self.role))
 
-	def event_UIA_notification(self, notificationKind=None, notificationProcessing=UIAHandler.NotificationProcessing_CurrentThenMostRecent, displayString=None, activityId=None):
+	def event_UIA_notification(
+			self,
+			notificationKind: Optional[int] = None,
+			notificationProcessing: Optional[int] = UIAHandler.NotificationProcessing_CurrentThenMostRecent,
+			displayString: Optional[str] = None,
+			activityId: Optional[str] = None
+	):
 		"""
 		Introduced in Windows 10 Fall Creators Update (build 16299).
 		This base implementation announces all notifications from the UIA element.

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -1,20 +1,29 @@
 # A part of NonVisual Desktop Access (NVDA)
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
-# Copyright (C) 2019-2022 Bill Dengler
+# Copyright (C) 2019-2022 Bill Dengler, Leonard de Ruijter
 
+import api
+import config
+import controlTypes
 import ctypes
 import NVDAHelper
+import NVDAState
+import speech
 import textInfos
 import textUtils
 import UIAHandler
 
 from comtypes import COMError
 from diffHandler import prefer_difflib
 from logHandler import log
-from UIAHandler.utils import _getConhostAPILevel
+from typing import (
+	Any,
+	Optional,
+)
+from UIAHandler.utils import _getConhostAPILevel, _shouldUseWindowsTerminalNotifications
 from UIAHandler.constants import WinConsoleAPILevel
-from . import UIATextInfo
+from . import UIA, UIATextInfo
 from ..behaviors import EnhancedTermTypedCharSupport, KeyboardHandlerBasedTypedCharSupport
 from ..window import Window
 
@@ -419,14 +428,58 @@ def findExtraOverlayClasses(obj, clsList):
 		clsList.append(consoleUIAWindow)
 
 
-class WinTerminalUIA(EnhancedTermTypedCharSupport):
+class _DiffBasedWinTerminalUIA(EnhancedTermTypedCharSupport):
+	"""
+	An overlay class for Windows Terminal (wt.exe) that uses diffing to speak
+	new text.
+	"""
+
 	def event_UIA_notification(self, **kwargs):
-		"""
-		In an upcoming terminal release, UIA notification events will be sent
-		to announce new text. Block these for now to avoid double-reporting of
-		text changes.
-		@note: In the longer term, NVDA should leverage these events in place
-		of the current LiveText strategy, as performance will likely be
-		significantly improved and #11002 can be completely mitigated.
-		"""
+		"Block notification events when diffing to prevent double reporting."
 		log.debugWarning(f"Notification event blocked to avoid double-report: {kwargs}")
+
+
+class _NotificationsBasedWinTerminalUIA(UIA):
+	"""
+	An overlay class for Windows Terminal (wt.exe) that uses UIA notification
+	events provided by the application to speak new text.
+	"""
+
+	#: Override the role, which is controlTypes.Role.STATICTEXT by default.
+	role = controlTypes.Role.TERMINAL
+	#: New line text is announced using UIA notification events
+	announceNewLineText = False
+
+	def event_UIA_notification(
+			self,
+			notificationKind: Optional[int] = None,
+			notificationProcessing: Optional[int] = UIAHandler.NotificationProcessing_CurrentThenMostRecent,
+			displayString: Optional[str] = None,
+			activityId: Optional[str] = None
+	):
+		# Do not announce output from background terminals.
+		if self.appModule != api.getFocusObject().appModule:
+			return
+		# microsoft/terminal#12358: Automatic reading of terminal output
+		# is provided by UIA notifications. If the user does not want
+		# automatic reporting of dynamic output, suppress this notification.
+		if not config.conf["presentation"]["reportDynamicContentChanges"]:
+			return
+		for line in displayString.splitlines():
+			if line and not line.isspace():  # Don't say "blank" during autoread
+				speech.speakText(line)
+
+
+def __getattr__(attrName: str) -> Any:
+	"""Module level `__getattr__` used to preserve backward compatibility."""
+	if attrName == "WinTerminalUIA" and NVDAState._allowDeprecatedAPI():
+		log.warning(
+			"WinTerminalUIA is deprecated. "
+			"Instead use _DiffBasedWinTerminalUIA or _NotificationsBasedWinTerminalUIA"
+		)
+		return (
+			_NotificationsBasedWinTerminalUIA
+			if _shouldUseWindowsTerminalNotifications()
+			else _DiffBasedWinTerminalUIA
+		)
+	raise AttributeError(f"module {repr(__name__)} has no attribute {repr(attrName)}")

source/UIAHandler/__init__.py

@@ -119,6 +119,9 @@
 
 textChangeUIAClassNames = (
 	"_WwG",  # Microsoft Word
+)
+
+windowsTerminalUIAClassNames = (
 	"TermControl",
 	"TermControl2",
 	"WPFTermControl",
@@ -648,6 +651,10 @@ def func():
 				if (
 					element.currentClassName in textChangeUIAClassNames
 					or element.CachedAutomationID in textChangeUIAAutomationIDs
+					or (
+						not utils._shouldUseWindowsTerminalNotifications()
+						and element.currentClassName in windowsTerminalUIAClassNames
+					)
 				):
 					group = self.localEventHandlerGroupWithTextChanges
 					logPrefix = "Explicitly"
@@ -704,6 +711,10 @@ def IUIAutomationEventHandler_HandleAutomationEvent(self,sender,eventID):
 			if (
 				sender.currentClassName in textChangeUIAClassNames
 				or sender.CachedAutomationID in textChangeUIAAutomationIDs
+				or (
+					not utils._shouldUseWindowsTerminalNotifications()
+					and sender.currentClassName in windowsTerminalUIAClassNames
+				)
 			):
 				NVDAEventName = "textChange"
 			else:

source/UIAHandler/utils.py

@@ -6,6 +6,7 @@
 import operator
 from comtypes import COMError
 import config
+from config.featureFlagEnums import WindowsTerminalStrategyFlag
 import ctypes
 import UIAHandler
 import weakref
@@ -362,3 +363,8 @@ def _shouldSelectivelyRegister() -> bool:
 		return False
 	else:
 		return winVersion.getWinVer() >= winVersion.WIN11_22H2
+
+
+def _shouldUseWindowsTerminalNotifications() -> bool:
+	"Determines whether to use notifications for new text reporting in Windows Terminal."
+	return config.conf["terminals"]["wtStrategy"] == WindowsTerminalStrategyFlag.NOTIFICATIONS

source/config/configSpec.py

@@ -249,6 +249,7 @@
 	speakPasswords = boolean(default=false)
 	keyboardSupportInLegacy = boolean(default=True)
 	diffAlgo = option("auto", "dmp", "difflib", default="auto")
+	wtStrategy = featureFlag(optionsEnum="WindowsTerminalStrategyFlag", behaviorOfDefault="diffing")
 
 [update]
 	autoCheck = boolean(default=true)

source/config/featureFlagEnums.py

@@ -1,5 +1,5 @@
 # A part of NonVisual Desktop Access (NVDA)
-# Copyright (C) 2022 NV Access Limited
+# Copyright (C) 2022 NV Access Limited, Bill Dengler
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
 
@@ -67,6 +67,26 @@ def __bool__(self):
 		return self == BoolFlag.ENABLED
 
 
+class WindowsTerminalStrategyFlag(DisplayStringEnum):
+	"""
+	A feature flag for defining how new text is calculated in Windows Terminal
+	(wt.exe).
+	"""
+
+	@property
+	def _displayStringLabels(self):
+		return {
+			# Translators: Label for an option in NVDA settings.
+			self.DIFFING: _("Diffing"),
+			# Translators: Label for an option in NVDA settings.
+			self.NOTIFICATIONS: _("UIA notifications"),
+		}
+
+	DEFAULT = enum.auto()
+	DIFFING = enum.auto()
+	NOTIFICATIONS = enum.auto()
+
+
 def getAvailableEnums() -> typing.Generator[typing.Tuple[str, FlagValueEnum], None, None]:
 	for name, value in globals().items():
 		if (

source/gui/settingsDialogs.py

@@ -2908,6 +2908,17 @@ def __init__(self, parent):
 			self._getDefaultValue(["terminals", "diffAlgo"])
 		)
 
+		self.wtStrategyCombo: nvdaControls.FeatureFlagCombo = terminalsGroup.addLabeledControl(
+			labelText=_(
+				# Translators: This is the label for a combo-box in the Advanced settings panel.
+				"Speak new text in Windows Terminal via:"
+			),
+			wxCtrlClass=nvdaControls.FeatureFlagCombo,
+			keyPath=["terminals", "wtStrategy"],
+			conf=config.conf,
+		)
+		self.bindHelpEvent("WtStrategy", self.wtStrategyCombo)
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Speech")
@@ -3082,6 +3093,7 @@ def haveConfigDefaultsBeenRestored(self):
 			and self.keyboardSupportInLegacyCheckBox.IsChecked() == self.keyboardSupportInLegacyCheckBox.defaultValue
 			and self.winConsoleSpeakPasswordsCheckBox.IsChecked() == self.winConsoleSpeakPasswordsCheckBox.defaultValue
 			and self.diffAlgoCombo.GetSelection() == self.diffAlgoCombo.defaultValue
+			and self.wtStrategyCombo.isValueConfigSpecDefault()
 			and self.cancelExpiredFocusSpeechCombo.GetSelection() == self.cancelExpiredFocusSpeechCombo.defaultValue
 			and self.loadChromeVBufWhenBusyCombo.isValueConfigSpecDefault()
 			and self.caretMoveTimeoutSpinControl.GetValue() == self.caretMoveTimeoutSpinControl.defaultValue
@@ -3106,6 +3118,7 @@ def restoreToDefaults(self):
 		self.winConsoleSpeakPasswordsCheckBox.SetValue(self.winConsoleSpeakPasswordsCheckBox.defaultValue)
 		self.keyboardSupportInLegacyCheckBox.SetValue(self.keyboardSupportInLegacyCheckBox.defaultValue)
 		self.diffAlgoCombo.SetSelection(self.diffAlgoCombo.defaultValue == 'auto')
+		self.wtStrategyCombo.resetToConfigSpecDefault()
 		self.cancelExpiredFocusSpeechCombo.SetSelection(self.cancelExpiredFocusSpeechCombo.defaultValue)
 		self.loadChromeVBufWhenBusyCombo.resetToConfigSpecDefault()
 		self.caretMoveTimeoutSpinControl.SetValue(self.caretMoveTimeoutSpinControl.defaultValue)
@@ -3135,6 +3148,7 @@ def onSave(self):
 		config.conf['terminals']['diffAlgo'] = (
 			self.diffAlgoVals[diffAlgoChoice]
 		)
+		self.wtStrategyCombo.saveCurrentValueToConf()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		config.conf["documentFormatting"]["reportTransparentColor"] = (
 			self.reportTransparentColorCheckBox.IsChecked()

user_docs/en/changes.t2t

@@ -11,6 +11,9 @@ What's New in NVDA
  - This is not referring to screen reader specific headers via named ranges, which is currently not supported via UI automation.
  -
 - An unassigned script has been added to toggle delayed character descriptions. (#14267)
+- Added an experimental option to leverage the UIA notification support in Windows Terminal to report new or changed text in the terminal, resulting in improved stability and responsivity. (#13781)
+ - Consult the user guide for limitations of this experimental option.
+ - 
 -
 
 
@@ -51,6 +54,8 @@ Please open a GitHub issue if your Add-on has an issue with updating to the new
 
 
 === Deprecations ===
+- ``NVDAObjects.UIA.winConsoleUIA.WinTerminalUIA`` is deprecated and usage is dscouraged. (#14047)
+-
 
 
 = 2022.4 =

user_docs/en/userGuide.t2t

@@ -2231,6 +2231,25 @@ This setting may stabilize reading of incoming text in some applications.
 However, in terminals, when inserting or deleting a character in the middle of a line, the text after the caret will be read out.
 -
 
+==== Speak new text in Windows Terminal via ====[WtStrategy]
+: Default
+  Diffing
+: Options
+  Diffing, UIA notifications
+:
+
+This option selects how NVDA determines what text is "new" (and thus what to speak when "report dynamic content changes" is enabled) in Windows Terminal and the WPF Windows Terminal control used in Visual Studio 2022.
+It does not affect the Windows Console (``conhost.exe``).
+The Speak new text in Windows Terminal combo box has three options:
+- Default: This option is currently equivalent to "diffing", but it is anticipated to change once support for UIA notifications is further developed.
+- Diffing: This option uses the selected diff algorithm to calculate changes each time the terminal renders new text.
+This is identical to NVDA's behaviour in versions 2022.4 and earlier.
+- UIA notifications: This option defers the responsibility of determining what text to speak to Windows Terminal itself, meanning that NVDA no longer has to determine what text currently on-screen is "new".
+This should markedly improve performance and stability of Windows Terminal, but this feature is not yet complete.
+In particular, typed characters that are not displayed on-screen, such as passwords, are reported when this option is selected.
+Additionally, contiguous spans of output of over 1,000 characters may not be reported accurately.
+-
+
 ==== Attempt to cancel speech for expired focus events ====[CancelExpiredFocusSpeech]
 This option enables behaviour which attempts to cancel speech for expired focus events.
 In particular moving quickly through messages in Gmail with Chrome can cause NVDA to speak outdated information.