From afdf289a780a4f9805b1df52fdcdc41cc752b8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= <3130044+MrAnno@users.noreply.github.com> Date: Wed, 9 Nov 2022 21:14:55 +0100 Subject: [PATCH] Advanced audio processing settings (#8759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Šimon Brandner Fixes https://github.com/vector-im/element-web/issues/6278 Fixes undefined --- src/MediaDeviceHandler.ts | 37 +++++++++ .../tabs/user/VoiceUserSettingsTab.tsx | 75 ++++++++++++++++--- src/i18n/strings/en_EN.json | 14 +++- src/settings/Settings.tsx | 31 ++++++-- test/MediaDeviceHandler-test.ts | 65 ++++++++++++++++ .../tabs/user/VoiceUserSettingsTab-test.tsx | 56 ++++++++++++++ test/test-utils/test-utils.ts | 1 + 7 files changed, 257 insertions(+), 22 deletions(-) create mode 100644 test/MediaDeviceHandler-test.ts create mode 100644 test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.tsx diff --git a/src/MediaDeviceHandler.ts b/src/MediaDeviceHandler.ts index 0e6d2b98bc7..6d60bc72f0d 100644 --- a/src/MediaDeviceHandler.ts +++ b/src/MediaDeviceHandler.ts @@ -88,6 +88,16 @@ export default class MediaDeviceHandler extends EventEmitter { await MatrixClientPeg.get().getMediaHandler().setAudioInput(audioDeviceId); await MatrixClientPeg.get().getMediaHandler().setVideoInput(videoDeviceId); + + await MediaDeviceHandler.updateAudioSettings(); + } + + private static async updateAudioSettings(): Promise { + await MatrixClientPeg.get().getMediaHandler().setAudioSettings({ + autoGainControl: MediaDeviceHandler.getAudioAutoGainControl(), + echoCancellation: MediaDeviceHandler.getAudioEchoCancellation(), + noiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression(), + }); } public setAudioOutput(deviceId: string): void { @@ -123,6 +133,21 @@ export default class MediaDeviceHandler extends EventEmitter { } } + public static async setAudioAutoGainControl(value: boolean): Promise { + await SettingsStore.setValue("webrtc_audio_autoGainControl", null, SettingLevel.DEVICE, value); + await MediaDeviceHandler.updateAudioSettings(); + } + + public static async setAudioEchoCancellation(value: boolean): Promise { + await SettingsStore.setValue("webrtc_audio_echoCancellation", null, SettingLevel.DEVICE, value); + await MediaDeviceHandler.updateAudioSettings(); + } + + public static async setAudioNoiseSuppression(value: boolean): Promise { + await SettingsStore.setValue("webrtc_audio_noiseSuppression", null, SettingLevel.DEVICE, value); + await MediaDeviceHandler.updateAudioSettings(); + } + public static getAudioOutput(): string { return SettingsStore.getValueAt(SettingLevel.DEVICE, "webrtc_audiooutput"); } @@ -135,6 +160,18 @@ export default class MediaDeviceHandler extends EventEmitter { return SettingsStore.getValueAt(SettingLevel.DEVICE, "webrtc_videoinput"); } + public static getAudioAutoGainControl(): boolean { + return SettingsStore.getValue("webrtc_audio_autoGainControl"); + } + + public static getAudioEchoCancellation(): boolean { + return SettingsStore.getValue("webrtc_audio_echoCancellation"); + } + + public static getAudioNoiseSuppression(): boolean { + return SettingsStore.getValue("webrtc_audio_noiseSuppression"); + } + /** * Returns the current set deviceId for a device kind * @param {MediaDeviceKindEnum} kind of the device that will be returned diff --git a/src/components/views/settings/tabs/user/VoiceUserSettingsTab.tsx b/src/components/views/settings/tabs/user/VoiceUserSettingsTab.tsx index dbd22deb4bd..f447158ccc1 100644 --- a/src/components/views/settings/tabs/user/VoiceUserSettingsTab.tsx +++ b/src/components/views/settings/tabs/user/VoiceUserSettingsTab.tsx @@ -27,6 +27,7 @@ import { MatrixClientPeg } from "../../../../../MatrixClientPeg"; import Modal from "../../../../../Modal"; import { SettingLevel } from "../../../../../settings/SettingLevel"; import SettingsFlag from '../../../elements/SettingsFlag'; +import LabelledToggleSwitch from "../../../elements/LabelledToggleSwitch"; import ErrorDialog from '../../../dialogs/ErrorDialog'; const getDefaultDevice = (devices: Array>) => { @@ -41,8 +42,14 @@ const getDefaultDevice = (devices: Array>) => { } }; -interface IState extends Record { +interface IState { mediaDevices: IMediaDevices; + [MediaDeviceKindEnum.AudioOutput]: string; + [MediaDeviceKindEnum.AudioInput]: string; + [MediaDeviceKindEnum.VideoInput]: string; + audioAutoGainControl: boolean; + audioEchoCancellation: boolean; + audioNoiseSuppression: boolean; } export default class VoiceUserSettingsTab extends React.Component<{}, IState> { @@ -54,6 +61,9 @@ export default class VoiceUserSettingsTab extends React.Component<{}, IState> { [MediaDeviceKindEnum.AudioOutput]: null, [MediaDeviceKindEnum.AudioInput]: null, [MediaDeviceKindEnum.VideoInput]: null, + audioAutoGainControl: MediaDeviceHandler.getAudioAutoGainControl(), + audioEchoCancellation: MediaDeviceHandler.getAudioEchoCancellation(), + audioNoiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression(), }; } @@ -183,22 +193,63 @@ export default class VoiceUserSettingsTab extends React.Component<{}, IState> { return (
{ _t("Voice & Video") }
+ { requestButton }
- { requestButton } + { _t("Voice settings") } { speakerDropdown } { microphoneDropdown } + { + await MediaDeviceHandler.setAudioAutoGainControl(v); + this.setState({ audioAutoGainControl: MediaDeviceHandler.getAudioAutoGainControl() }); + }} + label={_t("Automatically adjust the microphone volume")} + data-testid='voice-auto-gain' + /> +
+
+ { _t("Video settings") } { webcamDropdown } - - +
+ +
{ _t("Advanced") }
+
+ { _t("Voice processing") } +
+ { + await MediaDeviceHandler.setAudioNoiseSuppression(v); + this.setState({ audioNoiseSuppression: MediaDeviceHandler.getAudioNoiseSuppression() }); + }} + label={_t("Noise suppression")} + data-testid='voice-noise-suppression' + /> + { + await MediaDeviceHandler.setAudioEchoCancellation(v); + this.setState({ audioEchoCancellation: MediaDeviceHandler.getAudioEchoCancellation() }); + }} + label={_t("Echo cancellation")} + data-testid='voice-echo-cancellation' + /> +
+
+ { _t("Connection") } + + +
); diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json index 3c2bfea41d8..260ebfc1116 100644 --- a/src/i18n/strings/en_EN.json +++ b/src/i18n/strings/en_EN.json @@ -976,7 +976,11 @@ "Match system theme": "Match system theme", "Use a system font": "Use a system font", "System font name": "System font name", - "Allow Peer-to-Peer for 1:1 calls (if you enable this, the other party might be able to see your IP address)": "Allow Peer-to-Peer for 1:1 calls (if you enable this, the other party might be able to see your IP address)", + "Allow Peer-to-Peer for 1:1 calls": "Allow Peer-to-Peer for 1:1 calls", + "When enabled, the other party might be able to see your IP address": "When enabled, the other party might be able to see your IP address", + "Automatic gain control": "Automatic gain control", + "Echo cancellation": "Echo cancellation", + "Noise suppression": "Noise suppression", "Send analytics data": "Send analytics data", "Record the client name, version, and url to recognise sessions more easily in session manager": "Record the client name, version, and url to recognise sessions more easily in session manager", "Never send encrypted messages to unverified sessions from this session": "Never send encrypted messages to unverified sessions from this session", @@ -992,7 +996,8 @@ "Show shortcut to welcome checklist above the room list": "Show shortcut to welcome checklist above the room list", "Show hidden events in timeline": "Show hidden events in timeline", "Low bandwidth mode (requires compatible homeserver)": "Low bandwidth mode (requires compatible homeserver)", - "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)": "Allow fallback call assist server turn.matrix.org when your homeserver does not offer one (your IP address would be shared during a call)", + "Allow fallback call assist server (turn.matrix.org)": "Allow fallback call assist server (turn.matrix.org)", + "Only applies if your homeserver does not offer one. Your IP address would be shared during a call.": "Only applies if your homeserver does not offer one. Your IP address would be shared during a call.", "Show previews/thumbnails for images": "Show previews/thumbnails for images", "Enable message search in encrypted rooms": "Enable message search in encrypted rooms", "How fast should messages be downloaded.": "How fast should messages be downloaded.", @@ -1619,6 +1624,11 @@ "No Microphones detected": "No Microphones detected", "Camera": "Camera", "No Webcams detected": "No Webcams detected", + "Voice settings": "Voice settings", + "Automatically adjust the microphone volume": "Automatically adjust the microphone volume", + "Video settings": "Video settings", + "Voice processing": "Voice processing", + "Connection": "Connection", "This room is not accessible by remote Matrix servers": "This room is not accessible by remote Matrix servers", "Warning: Upgrading a room will not automatically migrate room members to the new version of the room. We'll post a link to the new room in the old version of the room - room members will have to click this link to join the new room.": "Warning: Upgrading a room will not automatically migrate room members to the new version of the room. We'll post a link to the new room in the old version of the room - room members will have to click this link to join the new room.", "Upgrade this space to the recommended room version": "Upgrade this space to the recommended room version", diff --git a/src/settings/Settings.tsx b/src/settings/Settings.tsx index 723b789ab01..81856cc9f3b 100644 --- a/src/settings/Settings.tsx +++ b/src/settings/Settings.tsx @@ -127,7 +127,8 @@ export type SettingValueType = boolean | string | number[] | string[] | - Record; + Record | + null; export interface IBaseSetting { isFeature?: false | undefined; @@ -712,10 +713,8 @@ export const SETTINGS: {[setting: string]: ISetting} = { }, "webRtcAllowPeerToPeer": { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS_WITH_CONFIG, - displayName: _td( - "Allow Peer-to-Peer for 1:1 calls " + - "(if you enable this, the other party might be able to see your IP address)", - ), + displayName: _td("Allow Peer-to-Peer for 1:1 calls"), + description: _td("When enabled, the other party might be able to see your IP address"), default: true, invertedSettingName: 'webRtcForceTURN', }, @@ -731,6 +730,21 @@ export const SETTINGS: {[setting: string]: ISetting} = { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, default: "default", }, + "webrtc_audio_autoGainControl": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + displayName: _td("Automatic gain control"), + default: true, + }, + "webrtc_audio_echoCancellation": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + displayName: _td("Echo cancellation"), + default: true, + }, + "webrtc_audio_noiseSuppression": { + supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, + displayName: _td("Noise suppression"), + default: true, + }, "language": { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS_WITH_CONFIG, default: "en", @@ -902,9 +916,10 @@ export const SETTINGS: {[setting: string]: ISetting} = { }, "fallbackICEServerAllowed": { supportedLevels: LEVELS_DEVICE_ONLY_SETTINGS, - displayName: _td( - "Allow fallback call assist server turn.matrix.org when your homeserver " + - "does not offer one (your IP address would be shared during a call)", + displayName: _td("Allow fallback call assist server (turn.matrix.org)"), + description: _td( + "Only applies if your homeserver does not offer one. " + + "Your IP address would be shared during a call.", ), // This is a tri-state value, where `null` means "prompt the user". default: null, diff --git a/test/MediaDeviceHandler-test.ts b/test/MediaDeviceHandler-test.ts new file mode 100644 index 00000000000..359ba5fc53d --- /dev/null +++ b/test/MediaDeviceHandler-test.ts @@ -0,0 +1,65 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked } from 'jest-mock'; + +import { SettingLevel } from "../src/settings/SettingLevel"; +import { MatrixClientPeg } from '../src/MatrixClientPeg'; +import { stubClient } from "./test-utils"; +import MediaDeviceHandler from "../src/MediaDeviceHandler"; +import SettingsStore from '../src/settings/SettingsStore'; + +jest.mock("../src/settings/SettingsStore"); + +const SettingsStoreMock = mocked(SettingsStore); + +describe("MediaDeviceHandler", () => { + beforeEach(() => { + stubClient(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it("sets audio settings", async () => { + const expectedAudioSettings = new Map([ + ["webrtc_audio_autoGainControl", false], + ["webrtc_audio_echoCancellation", true], + ["webrtc_audio_noiseSuppression", false], + ]); + + SettingsStoreMock.getValue.mockImplementation((settingName): any => { + return expectedAudioSettings.get(settingName); + }); + + await MediaDeviceHandler.setAudioAutoGainControl(false); + await MediaDeviceHandler.setAudioEchoCancellation(true); + await MediaDeviceHandler.setAudioNoiseSuppression(false); + + expectedAudioSettings.forEach((value, key) => { + expect(SettingsStoreMock.setValue).toHaveBeenCalledWith( + key, null, SettingLevel.DEVICE, value, + ); + }); + + expect(MatrixClientPeg.get().getMediaHandler().setAudioSettings).toHaveBeenCalledWith({ + autoGainControl: false, + echoCancellation: true, + noiseSuppression: false, + }); + }); +}); diff --git a/test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.tsx b/test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.tsx new file mode 100644 index 00000000000..c303efb8a75 --- /dev/null +++ b/test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.tsx @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import React from 'react'; +import { mocked } from 'jest-mock'; +import { render } from '@testing-library/react'; + +import VoiceUserSettingsTab from '../../../../../../src/components/views/settings/tabs/user/VoiceUserSettingsTab'; +import MediaDeviceHandler from "../../../../../../src/MediaDeviceHandler"; + +jest.mock("../../../../../../src/MediaDeviceHandler"); +const MediaDeviceHandlerMock = mocked(MediaDeviceHandler); + +describe('', () => { + const getComponent = (): React.ReactElement => (); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders audio processing settings', () => { + const { getByTestId } = render(getComponent()); + expect(getByTestId('voice-auto-gain')).toBeTruthy(); + expect(getByTestId('voice-noise-suppression')).toBeTruthy(); + expect(getByTestId('voice-echo-cancellation')).toBeTruthy(); + }); + + it('sets and displays audio processing settings', () => { + MediaDeviceHandlerMock.getAudioAutoGainControl.mockReturnValue(false); + MediaDeviceHandlerMock.getAudioEchoCancellation.mockReturnValue(true); + MediaDeviceHandlerMock.getAudioNoiseSuppression.mockReturnValue(false); + + const { getByRole } = render(getComponent()); + + getByRole("switch", { name: "Automatically adjust the microphone volume" }).click(); + getByRole("switch", { name: "Noise suppression" }).click(); + getByRole("switch", { name: "Echo cancellation" }).click(); + + expect(MediaDeviceHandler.setAudioAutoGainControl).toHaveBeenCalledWith(true); + expect(MediaDeviceHandler.setAudioEchoCancellation).toHaveBeenCalledWith(false); + expect(MediaDeviceHandler.setAudioNoiseSuppression).toHaveBeenCalledWith(true); + }); +}); diff --git a/test/test-utils/test-utils.ts b/test/test-utils/test-utils.ts index 1a8792b810a..ef95d6d5a76 100644 --- a/test/test-utils/test-utils.ts +++ b/test/test-utils/test-utils.ts @@ -185,6 +185,7 @@ export function createTestClient(): MatrixClient { getMediaHandler: jest.fn().mockReturnValue({ setVideoInput: jest.fn(), setAudioInput: jest.fn(), + setAudioSettings: jest.fn(), } as unknown as MediaHandler), uploadContent: jest.fn(), getEventMapper: () => (opts) => new MatrixEvent(opts),