Feat: Voice activity threshold

spec/unit/webrtc/callFeed.spec.ts

@@ -85,5 +85,125 @@ describe("CallFeed", () => {
                 expect(feed.isVideoMuted()).toBeTruthy();
             });
         });
+
+        describe("voice activity detection", () => {
+            it("If voice activity is disabled we should not mute/unmute with it", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set threshold to infinity, this ensures we never hit the threshold.
+                feed.setVoiceActivityThreshold(Infinity);
+                // the number doesn't matter, anything below infinity is ok.
+                feed.speakingVolumeSamples = [-60];
+                // we expect no mute/unmute behavior from vad from here on.
+                feed.VADEnabled = false;
+
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(true);
+                }, 1000);
+            });
+
+            it("enables track when volume is above threshold", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set the threshold and the samples to ensure the user is unmuted at the start.
+                feed.setVoiceActivityThreshold(-80);
+                feed.speakingVolumeSamples = [-60];
+
+                // user has -40db which is louder than -50db, so the user should be unmuted.
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(true);
+                }, 1000);
+            });
+
+            it("disables track when volume is below threshold", () => {
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set the threshold and the samples to ensure the user is muted at the start.
+                feed.setVoiceActivityThreshold(-80);
+                feed.speakingVolumeSamples = [-90];
+
+                // the user is too quiet, user should be muted.
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(false);
+                }, 1000);
+            });
+
+            it("should not disable audio track after a few milliseconds", async () => {
+                // Someone speaks
+                // Stops speaking for a few milliseconds
+                // -> Is not muted before cooldown -> (VAD_COOLDOWN)
+
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set the threshold and the samples to ensure the user is unmuted at the start.
+                feed.setVoiceActivityThreshold(-80);
+                feed.speakingVolumeSamples = [-60];
+
+                // pretend the user is silent after 100ms
+                setTimeout(() => {
+                    feed.speakingVolumeSamples = [-Infinity];
+                }, 100);
+
+                // the user should still be unmuted after another 50ms.
+                // Cooldown is 200ms, so this is within the range.
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(true);
+                }, 150);
+            });
+
+            it("should disable audio track after cooldown", async () => {
+                // Someone speaks
+                // Stops speaking
+                // -> Is muted after cooldown -> (VAD_COOLDOWN)
+
+                //@ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set the threshold and the samples to ensure the user is unmuted at the start.
+                feed.setVoiceActivityThreshold(-80);
+                feed.speakingVolumeSamples = [-60];
+
+                // pretend the user is silent after 100ms
+                setTimeout(() => {
+                    feed.speakingVolumeSamples = [-Infinity];
+                }, 100);
+
+                // The user should be muted after another 210ms.
+                // 200ms is the cooldown, so we are outside of that range.
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(false);
+                }, 310);
+            });
+
+            it("cooldown should be reset when speaking", async () => {
+                // Cooldown is reset after speaking again
+
+                // @ts-ignore Mock
+                feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true));
+
+                // set the threshold and the samples to ensure the user is unmuted at the start.
+                feed.setVoiceActivityThreshold(-80);
+                feed.speakingVolumeSamples = [-60];
+
+                // pretend the user is silent after 100ms
+                setTimeout(() => {
+                    feed.speakingVolumeSamples = [-Infinity];
+                }, 100);
+
+                // pretend the user starts speaking again after another 100ms
+                setTimeout(() => {
+                    feed.speakingVolumeSamples = [-60];
+                }, 200);
+
+                // after yet another 100ms check if the user is still unmuted.
+                setTimeout(() => {
+                    expect(feed.stream.getAudioTracks()[0].enabled).toBe(true);
+                }, 310);
+            });
+        });
     });
 });

src/webrtc/callFeed.ts

@@ -21,9 +21,12 @@ import { RoomMember } from "../models/room-member";
 import { logger } from "../logger";
 import { TypedEventEmitter } from "../models/typed-event-emitter";
 
-const POLLING_INTERVAL = 200; // ms
+const POLLING_INTERVAL = 1; // ms
 export const SPEAKING_THRESHOLD = -60; // dB
+const VAD_THRESHOLD = -100; //dB
 const SPEAKING_SAMPLE_COUNT = 8; // samples
+const VAD_COOLDOWN = 200; // ms
+const VAD_AUDIO_DELAY = 0.001; // ms
 
 export interface ICallFeedOpts {
     client: MatrixClient;
@@ -39,6 +42,11 @@ export interface ICallFeedOpts {
      * Whether or not the remote SDPStreamMetadata says video is muted
      */
     videoMuted: boolean;
+    /**
+     * set the Tracks to muted when volume threshold has not been reached
+     * This does not show the user as muted.
+     */
+    VADEnabled?: boolean;
 }
 
 export enum CallFeedEvent {
@@ -47,6 +55,8 @@ export enum CallFeedEvent {
     LocalVolumeChanged = "local_volume_changed",
     VolumeChanged = "volume_changed",
     Speaking = "speaking",
+    VoiceActivityThresholdChanged = "voice_activity_threshold_changed",
+    VADMuteStateChanged = "vad_mute_state_changed",
     Disposed = "disposed",
 }
 
@@ -56,19 +66,25 @@ type EventHandlerMap = {
     [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void;
     [CallFeedEvent.VolumeChanged]: (volume: number) => void;
     [CallFeedEvent.Speaking]: (speaking: boolean) => void;
+    [CallFeedEvent.VoiceActivityThresholdChanged]: (threshold: number) => void;
+    [CallFeedEvent.VADMuteStateChanged]: (VADMuted: boolean) => void;
     [CallFeedEvent.Disposed]: () => void;
 };
-
 export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap> {
     public stream: MediaStream;
+    public volumeLooperStream: MediaStream;
     public sdpMetadataStreamId: string;
     public userId: string;
     public purpose: SDPStreamMetadataPurpose;
     public speakingVolumeSamples: number[];
+    public voiceActivityThreshold: number;
+    public VADEnabled = false;
+    public maxVolume = -Infinity;
 
     private client: MatrixClient;
     private roomId: string;
     private audioMuted: boolean;
+    private vadAudioMuted: boolean;
     private videoMuted: boolean;
     private localVolume = 1;
     private measuringVolumeActivity = false;
@@ -77,7 +93,15 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
     private frequencyBinCount: Float32Array;
     private speakingThreshold = SPEAKING_THRESHOLD;
     private speaking = false;
+    private vadSpeaking = false;
+    private audioDelay: DelayNode;
+    private gainNode: GainNode;
     private volumeLooperTimeout: ReturnType<typeof setTimeout>;
+    /**
+     * Cooldown for voice activity detection, so that we don't mute immediately when the user stops speaking
+     * But when he has been silent 200ms
+     */
+    private VADCooldownStarted = new Date();
     private _disposed = false;
 
     constructor(opts: ICallFeedOpts) {
@@ -91,14 +115,26 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
         this.videoMuted = opts.videoMuted;
         this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity);
         this.sdpMetadataStreamId = opts.stream.id;
+        this.voiceActivityThreshold = VAD_THRESHOLD;
 
         this.updateStream(null, opts.stream);
 
+        this.VADEnabled = opts.VADEnabled ?? false;
+
         if (this.hasAudioTrack) {
             this.initVolumeMeasuring();
         }
     }
 
+    public setVoiceActivityThreshold(threshold: number): void {
+        this.voiceActivityThreshold = threshold;
+        if (threshold === -100) {
+            this.VADEnabled = false;
+        } else {
+            this.VADEnabled = true;
+        }
+    }
+
     private get hasAudioTrack(): boolean {
         return this.stream.getAudioTracks().length > 0;
     }
@@ -132,9 +168,23 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
         this.analyser.fftSize = 512;
         this.analyser.smoothingTimeConstant = 0.1;
 
-        const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
+        this.volumeLooperStream = this.stream.clone();
+        const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.volumeLooperStream);
         mediaStreamAudioSourceNode.connect(this.analyser);
 
+        // If we handle audio manually then we need a gainnode, streamnode and a destination
+        // in this case we also add a delaynode to ensure we enable / disable vad at the right time.
+        // The delaynode needs to be created in the class to ensure functionality.
+        // Without the gainnode, the audio doesn't appear to work correctly
+        const streamNode = this.audioContext.createMediaStreamSource(this.stream);
+        this.audioDelay = this.audioContext.createDelay(VAD_AUDIO_DELAY);
+        this.gainNode = this.audioContext.createGain();
+        streamNode.connect(this.gainNode);
+        this.gainNode.connect(this.audioDelay);
+        const destination = this.audioContext.createMediaStreamDestination();
+        this.audioDelay.connect(destination);
+        this.stream = destination.stream;
+
         this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
     }
 
@@ -207,6 +257,13 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
         }
         if (videoMuted !== null) this.videoMuted = videoMuted;
         this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
+        this.VADEnabled = !audioMuted;
+    }
+
+    public setVoiceActivityDetectionMuteLocal(audioMuted: boolean | null): void {
+        if (audioMuted !== null) {
+            this.vadAudioMuted = audioMuted;
+        }
     }
 
     /**
@@ -237,17 +294,39 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
 
         this.analyser.getFloatFrequencyData(this.frequencyBinCount);
 
-        let maxVolume = -Infinity;
+        let maxCurrentVolume = -Infinity;
         for (let i = 0; i < this.frequencyBinCount.length; i++) {
-            if (this.frequencyBinCount[i] > maxVolume) {
-                maxVolume = this.frequencyBinCount[i];
+            if (this.frequencyBinCount[i] > maxCurrentVolume) {
+                maxCurrentVolume = this.frequencyBinCount[i];
             }
         }
-
+        this.maxVolume = maxCurrentVolume;
         this.speakingVolumeSamples.shift();
-        this.speakingVolumeSamples.push(maxVolume);
-
-        this.emit(CallFeedEvent.VolumeChanged, maxVolume);
+        this.speakingVolumeSamples.push(this.maxVolume);
+
+        this.emit(CallFeedEvent.VolumeChanged, this.maxVolume);
+
+        // Handle voice activity detection only if it is enabled and user has not manually muted themselves
+        if (this.VADEnabled) {
+            // If the user is speaking
+            if (this.maxVolume > this.voiceActivityThreshold) {
+                this.vadSpeaking = true;
+
+                if (this.vadAudioMuted) {
+                    this.emit(CallFeedEvent.VADMuteStateChanged, false);
+                }
+            } else if (!this.vadAudioMuted) {
+                // User stops speaking
+
+                if (this.vadSpeaking) {
+                    this.VADCooldownStarted = new Date();
+                    this.vadSpeaking = false;
+                } else if (!this.isVADInCooldown()) {
+                    // user has been silent for X milliseconds
+                    this.emit(CallFeedEvent.VADMuteStateChanged, true);
+                }
+            }
+        }
 
         let newSpeaking = false;
 
@@ -268,6 +347,11 @@ export class CallFeed extends TypedEventEmitter<CallFeedEvent, EventHandlerMap>
         this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL);
     };
 
+    private isVADInCooldown(): boolean {
+        return (
+            new Date().getTime() - this.VADCooldownStarted.getTime() < VAD_COOLDOWN);
+    }
+
     public clone(): CallFeed {
         const mediaHandler = this.client.getMediaHandler();
         const stream = this.stream.clone();