From a367e10ed9890098d5be0e7d873c3beac96d7dbb Mon Sep 17 00:00:00 2001 From: "fabio.lenherr" Date: Thu, 25 Aug 2022 15:09:05 +0200 Subject: [PATCH 1/5] fix formatting --- src/webrtc/callFeed.ts | 4 +- src/webrtc/groupCall.ts | 241 ++++++++++++---------------------------- 2 files changed, 73 insertions(+), 172 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index dbe105f8f66..0b1a194a7a4 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -324,9 +324,7 @@ export class CallFeed extends TypedEventEmitter private isVADinCooldown(): boolean { return ( - new Date().getTime() - this.VADCooldownStarted.getTime() < - this.VADCooldownMs - ); + new Date().getTime() - this.VADCooldownStarted.getTime() < this.VADCooldownMs); } public clone(): CallFeed { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cdc933b63df..0c74b074130 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -655,11 +655,7 @@ export class GroupCall extends TypedEventEmitter< return false; } } else { - await Promise.all( - this.calls.map((call) => - call.removeLocalFeed(call.localScreensharingFeed), - ), - ); + await Promise.all(this.calls.map((call) =>call.removeLocalFeed(call.localScreensharingFeed))); this.client .getMediaHandler() .stopScreensharingStream(this.localScreenshareFeed.stream); @@ -702,9 +698,8 @@ export class GroupCall extends TypedEventEmitter< } if (!newCall.groupCallId || newCall.groupCallId !== this.groupCallId) { - logger.log( - `Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn't match the current group call`, - ); + logger.log(`Incoming call with groupCallId ${ + newCall.groupCallId} ignored because it doesn't match the current group call`); newCall.reject(); return; } @@ -728,9 +723,7 @@ export class GroupCall extends TypedEventEmitter< this.addCall(newCall); } - newCall.answerWithCallFeeds( - this.getLocalFeeds().map((feed) => feed.clone()), - ); + newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); }; /** @@ -739,38 +732,31 @@ export class GroupCall extends TypedEventEmitter< private getMemberStateEvents(): MatrixEvent[]; private getMemberStateEvents(userId: string): MatrixEvent | null; - private getMemberStateEvents( - userId?: string, - ): MatrixEvent[] | MatrixEvent | null { + private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { if (userId != null) { - const event = this.room.currentState.getStateEvents( - EventType.GroupCallMemberPrefix, - userId, - ); + const event = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix,userId); return callMemberStateIsExpired(event) ? null : event; } else { - return this.room.currentState - .getStateEvents(EventType.GroupCallMemberPrefix) + return this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) .filter((event) => !callMemberStateIsExpired(event)); } } private async sendMemberStateEvent(): Promise { - const send = () => - this.updateMemberCallState({ - "m.call_id": this.groupCallId, - "m.devices": [ - { - device_id: this.client.getDeviceId(), - session_id: this.client.getSessionId(), - feeds: this.getLocalFeeds().map((feed) => ({ - purpose: feed.purpose, - })), - // TODO: Add data channels - }, - ], - // TODO "m.foci" - }); + const send = () =>this.updateMemberCallState({ + "m.call_id": this.groupCallId, + "m.devices": [ + { + device_id: this.client.getDeviceId(), + session_id: this.client.getSessionId(), + feeds: this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + })), + // TODO: Add data channels + }, + ], + // TODO "m.foci" + }); const res = await send(); @@ -778,7 +764,7 @@ export class GroupCall extends TypedEventEmitter< this.resendMemberStateTimer = setInterval(async () => { logger.log("Resending call member state"); await send(); - }, (CALL_MEMBER_STATE_TIMEOUT * 3) / 4); + }, CALL_MEMBER_STATE_TIMEOUT * 3 / 4); return res; } @@ -789,15 +775,10 @@ export class GroupCall extends TypedEventEmitter< return await this.updateMemberCallState(undefined); } - private async updateMemberCallState( - memberCallState?: IGroupCallRoomMemberCallState, - ): Promise { + private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { const localUserId = this.client.getUserId(); - const memberState = - this.getMemberStateEvents( - localUserId, - )?.getContent(); + const memberState =this.getMemberStateEvents(localUserId,)?.getContent(); let calls: IGroupCallRoomMemberCallState[] = []; @@ -806,9 +787,7 @@ export class GroupCall extends TypedEventEmitter< calls = memberState["m.calls"].filter((call) => !!call); } - const existingCallIndex = calls.findIndex( - (call) => call && call["m.call_id"] === this.groupCallId, - ); + const existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); if (existingCallIndex !== -1) { if (memberCallState) { @@ -825,12 +804,7 @@ export class GroupCall extends TypedEventEmitter< "m.expires_ts": Date.now() + CALL_MEMBER_STATE_TIMEOUT, }; - return this.client.sendStateEvent( - this.room.roomId, - EventType.GroupCallMemberPrefix, - content, - localUserId, - ); + return this.client.sendStateEvent(this.room.roomId,EventType.GroupCallMemberPrefix,content,localUserId); } public onMemberStateChanged = async (event: MatrixEvent) => { @@ -847,16 +821,12 @@ export class GroupCall extends TypedEventEmitter< }; const content = event.getContent(); - const callsState = - !callMemberStateIsExpired(event) && - Array.isArray(content["m.calls"]) + const callsState =!callMemberStateIsExpired(event) &&Array.isArray(content["m.calls"]) ? content["m.calls"].filter((call) => call) : []; // Ignore expired device data if (callsState.length === 0) { - logger.log( - `Ignoring member state from ${member.userId} member not in any calls.`, - ); + logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); ignore(); return; } @@ -866,9 +836,7 @@ export class GroupCall extends TypedEventEmitter< const callId = callState["m.call_id"]; if (!callId) { - logger.warn( - `Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`, - ); + logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); ignore(); return; } @@ -884,15 +852,10 @@ export class GroupCall extends TypedEventEmitter< this.addParticipant(member); clearTimeout(this.memberStateExpirationTimers.get(member.userId)); - this.memberStateExpirationTimers.set( - member.userId, - setTimeout(() => { - logger.warn( - `Call member state for ${member.userId} has expired`, - ); + this.memberStateExpirationTimers.set(member.userId,setTimeout(() => { + logger.warn(`Call member state for ${member.userId} has expired`); this.removeParticipant(member); - }, content["m.expires_ts"] - Date.now()), - ); + }, content["m.expires_ts"] - Date.now())); // Don't process your own member. const localUserId = this.client.getUserId(); @@ -915,9 +878,7 @@ export class GroupCall extends TypedEventEmitter< const opponentDevice = this.getDeviceForMember(member.userId); if (!opponentDevice) { - logger.warn( - `No opponent device found for ${member.userId}, ignoring.`, - ); + logger.warn(`No opponent device found for ${member.userId}, ignoring.`); this.emit( GroupCallEvent.Error, new GroupCallUnknownDeviceError(member.userId), @@ -934,22 +895,25 @@ export class GroupCall extends TypedEventEmitter< return; } - const newCall = createNewMatrixCall(this.client, this.room.roomId, { - invitee: member.userId, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - }); + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: member.userId, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + }, + ); newCall.isPtt = this.isPtt; const requestScreenshareFeed = opponentDevice.feeds.some( - (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare, - ); + (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); try { await newCall.placeCallWithCallFeeds( - this.getLocalFeeds().map((feed) => feed.clone()), + this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed, ); } catch (e) { @@ -986,11 +950,9 @@ export class GroupCall extends TypedEventEmitter< return undefined; } - const memberState = - memberStateEvent.getContent(); + const memberState = memberStateEvent.getContent(); const memberGroupCallState = memberState["m.calls"]?.find( - (call) => call && call["m.call_id"] === this.groupCallId, - ); + (call) => call && call["m.call_id"] === this.groupCallId); if (!memberGroupCallState) { return undefined; @@ -1009,9 +971,7 @@ export class GroupCall extends TypedEventEmitter< private onRetryCallLoop = () => { for (const event of this.getMemberStateEvents()) { const memberId = event.getStateKey(); - const existingCall = this.calls.find( - (call) => getCallUserId(call) === memberId, - ); + const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); const retryCallCount = this.retryCallCounts.get(memberId) || 0; if (!existingCall && retryCallCount < 3) { @@ -1020,10 +980,7 @@ export class GroupCall extends TypedEventEmitter< } } - this.retryCallLoopTimeout = setTimeout( - this.onRetryCallLoop, - this.retryCallInterval, - ); + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop,this.retryCallInterval); }; /** @@ -1040,11 +997,7 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.CallsChanged, this.calls); } - private replaceCall( - existingCall: MatrixCall, - replacementCall: MatrixCall, - hangupReason = CallErrorCode.Replaced, - ) { + private replaceCall(existingCall: MatrixCall,replacementCall: MatrixCall,hangupReason = CallErrorCode.Replaced) { const existingCallIndex = this.calls.indexOf(existingCall); if (existingCallIndex === -1) { @@ -1081,11 +1034,10 @@ export class GroupCall extends TypedEventEmitter< } const onCallFeedsChanged = () => this.onCallFeedsChanged(call); - const onCallStateChanged = (state: CallState, oldState: CallState) => - this.onCallStateChanged(call, state, oldState); + const onCallStateChanged = + (state: CallState, oldState: CallState) => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; - const onCallReplaced = (newCall: MatrixCall) => - this.replaceCall(call, newCall); + const onCallReplaced = (newCall: MatrixCall) => this.replaceCall(call, newCall); this.callHandlers.set(opponentMemberId, { onCallFeedsChanged, @@ -1139,8 +1091,7 @@ export class GroupCall extends TypedEventEmitter< this.removeUserMediaFeed(usermediaFeed); } - const screenshareFeed = - this.getScreenshareFeedByUserId(opponentMemberId); + const screenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); if (screenshareFeed) { this.removeScreenshareFeed(screenshareFeed); @@ -1154,8 +1105,7 @@ export class GroupCall extends TypedEventEmitter< throw new Error("Cannot change call feeds without user id"); } - const currentUserMediaFeed = - this.getUserMediaFeedByUserId(opponentMemberId); + const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); const remoteUsermediaFeed = call.remoteUsermediaFeed; const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; @@ -1163,40 +1113,28 @@ export class GroupCall extends TypedEventEmitter< if (!currentUserMediaFeed && remoteUsermediaFeed) { this.addUserMediaFeed(remoteUsermediaFeed); } else if (currentUserMediaFeed && remoteUsermediaFeed) { - this.replaceUserMediaFeed( - currentUserMediaFeed, - remoteUsermediaFeed, - ); + this.replaceUserMediaFeed(currentUserMediaFeed,remoteUsermediaFeed); } else if (currentUserMediaFeed && !remoteUsermediaFeed) { this.removeUserMediaFeed(currentUserMediaFeed); } } - const currentScreenshareFeed = - this.getScreenshareFeedByUserId(opponentMemberId); + const currentScreenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); const remoteScreensharingFeed = call.remoteScreensharingFeed; - const remoteScreenshareFeedChanged = - remoteScreensharingFeed !== currentScreenshareFeed; + const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; if (remoteScreenshareFeedChanged) { if (!currentScreenshareFeed && remoteScreensharingFeed) { this.addScreenshareFeed(remoteScreensharingFeed); } else if (currentScreenshareFeed && remoteScreensharingFeed) { - this.replaceScreenshareFeed( - currentScreenshareFeed, - remoteScreensharingFeed, - ); + this.replaceScreenshareFeed(currentScreenshareFeed,remoteScreensharingFeed); } else if (currentScreenshareFeed && !remoteScreensharingFeed) { this.removeScreenshareFeed(currentScreenshareFeed); } } }; - private onCallStateChanged = ( - call: MatrixCall, - state: CallState, - _oldState: CallState, - ) => { + private onCallStateChanged = (call: MatrixCall,state: CallState,_oldState: CallState) => { const audioMuted = this.localCallFeed.isAudioMuted(); if ( @@ -1242,10 +1180,7 @@ export class GroupCall extends TypedEventEmitter< this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } - private replaceUserMediaFeed( - existingFeed: CallFeed, - replacementFeed: CallFeed, - ) { + private replaceUserMediaFeed(existingFeed: CallFeed,replacementFeed: CallFeed) { const feedIndex = this.userMediaFeeds.findIndex( (feed) => feed.userId === existingFeed.userId, ); @@ -1262,9 +1197,7 @@ export class GroupCall extends TypedEventEmitter< } private removeUserMediaFeed(callFeed: CallFeed) { - const feedIndex = this.userMediaFeeds.findIndex( - (feed) => feed.userId === callFeed.userId, - ); + const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === callFeed.userId); if (feedIndex === -1) { throw new Error("Couldn't find user media feed to remove"); @@ -1289,10 +1222,7 @@ export class GroupCall extends TypedEventEmitter< let nextActiveSpeaker: string; for (const callFeed of this.userMediaFeeds) { - if ( - callFeed.userId === this.client.getUserId() && - this.userMediaFeeds.length > 1 - ) { + if (callFeed.userId === this.client.getUserId() && this.userMediaFeeds.length > 1) { continue; } @@ -1311,11 +1241,7 @@ export class GroupCall extends TypedEventEmitter< } } - if ( - nextActiveSpeaker && - this.activeSpeaker !== nextActiveSpeaker && - topAvg > SPEAKING_THRESHOLD - ) { + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { this.activeSpeaker = nextActiveSpeaker; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } @@ -1336,19 +1262,11 @@ export class GroupCall extends TypedEventEmitter< private addScreenshareFeed(callFeed: CallFeed) { this.screenshareFeeds.push(callFeed); - this.emit( - GroupCallEvent.ScreenshareFeedsChanged, - this.screenshareFeeds, - ); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } - private replaceScreenshareFeed( - existingFeed: CallFeed, - replacementFeed: CallFeed, - ) { - const feedIndex = this.screenshareFeeds.findIndex( - (feed) => feed.userId === existingFeed.userId, - ); + private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === existingFeed.userId); if (feedIndex === -1) { throw new Error("Couldn't find screenshare feed to replace"); @@ -1364,9 +1282,7 @@ export class GroupCall extends TypedEventEmitter< } private removeScreenshareFeed(callFeed: CallFeed) { - const feedIndex = this.screenshareFeeds.findIndex( - (feed) => feed.userId === callFeed.userId, - ); + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); if (feedIndex === -1) { throw new Error("Couldn't find screenshare feed to remove"); @@ -1375,10 +1291,7 @@ export class GroupCall extends TypedEventEmitter< this.screenshareFeeds.splice(feedIndex, 1); callFeed.dispose(); - this.emit( - GroupCallEvent.ScreenshareFeedsChanged, - this.screenshareFeeds, - ); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } /** @@ -1393,17 +1306,11 @@ export class GroupCall extends TypedEventEmitter< this.participants.push(member); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); - this.client.emit( - GroupCallEventHandlerEvent.Participants, - this.participants, - this, - ); + this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); } private removeParticipant(member: RoomMember) { - const index = this.participants.findIndex( - (m) => m.userId === member.userId, - ); + const index = this.participants.findIndex((m) => m.userId === member.userId); if (index === -1) { return; @@ -1412,10 +1319,6 @@ export class GroupCall extends TypedEventEmitter< this.participants.splice(index, 1); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); - this.client.emit( - GroupCallEventHandlerEvent.Participants, - this.participants, - this, - ); + this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); } } From 0a98fbeb075bbaffec467c5d5c01590984567472 Mon Sep 17 00:00:00 2001 From: "fabio.lenherr" Date: Thu, 25 Aug 2022 17:59:45 +0200 Subject: [PATCH 2/5] fix VAD --- src/webrtc/callFeed.ts | 26 +++++++++++++++++--------- yarn.lock | 5 +++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 0b1a194a7a4..9d96448512f 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -73,7 +73,7 @@ export class CallFeed extends TypedEventEmitter public voiceActivityThreshold: number; public setVADMute: (muted: boolean) => void; public VADEnabled = true; - public maxCurrentVolume = -Infinity; + public maxVolume = -Infinity; private client: MatrixClient; private roomId: string; @@ -118,6 +118,12 @@ export class CallFeed extends TypedEventEmitter public setVoiceActivityThreshold(threshold: number): void { this.voiceActivityThreshold = threshold; + if(threshold === -100) { + this.VADEnabled = false; + this.setVADMute?.(false); + } else { + this.VADEnabled = true; + } } private get hasAudioTrack(): boolean { @@ -154,7 +160,7 @@ export class CallFeed extends TypedEventEmitter this.analyser.smoothingTimeConstant = 0.1; this.secondStream = this.stream.clone(); - const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.secondStream); mediaStreamAudioSourceNode.connect(this.analyser); this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); @@ -230,6 +236,7 @@ export class CallFeed extends TypedEventEmitter } if (videoMuted !== null) this.videoMuted = videoMuted; this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); + this.VADEnabled = !audioMuted; } public setVadMuted(audioMuted: boolean | null, videoMuted: boolean | null): void { @@ -265,17 +272,18 @@ export class CallFeed extends TypedEventEmitter this.analyser.getFloatFrequencyData(this.frequencyBinCount); - this.maxCurrentVolume = -Infinity; + let maxCurrentVolume = -Infinity; for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > this.maxCurrentVolume) { - this.maxCurrentVolume = this.frequencyBinCount[i]; + if (this.frequencyBinCount[i] > maxCurrentVolume) { + maxCurrentVolume = this.frequencyBinCount[i]; } + this.maxVolume = maxCurrentVolume; } this.speakingVolumeSamples.shift(); - this.speakingVolumeSamples.push(this.maxCurrentVolume); + this.speakingVolumeSamples.push(this.maxVolume); - this.emit(CallFeedEvent.VolumeChanged, this.maxCurrentVolume); + this.emit(CallFeedEvent.VolumeChanged, this.maxVolume); let newSpeaking = false; @@ -298,9 +306,9 @@ export class CallFeed extends TypedEventEmitter } // Handle voice activity detection only if it is enabled and user has not manually muted themselves - if (this.VADEnabled && !this.audioMuted) { + if (this.VADEnabled) { // If the user is speaking - if (this.maxCurrentVolume > this.voiceActivityThreshold) { + if (this.maxVolume > this.voiceActivityThreshold) { this.VADCooldownStarted = new Date(); if (this.vadAudioMuted) { diff --git a/yarn.lock b/yarn.lock index d20923e47b9..856ba07e5cf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6827,6 +6827,11 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== +universalify@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + update-browserslist-db@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz#be06a5eedd62f107b7c19eb5bcefb194411abf38" From 512cf6811dd55276d2061fd21f11c6b240cbe005 Mon Sep 17 00:00:00 2001 From: Dashie Date: Thu, 25 Aug 2022 18:02:29 +0200 Subject: [PATCH 3/5] Update yarn.lock --- yarn.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 856ba07e5cf..d20923e47b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6827,11 +6827,6 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== -universalify@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== - update-browserslist-db@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz#be06a5eedd62f107b7c19eb5bcefb194411abf38" From 23310a524fd0832bb8f7f6b94b8aaa9a71afc167 Mon Sep 17 00:00:00 2001 From: "fabio.lenherr" Date: Thu, 25 Aug 2022 19:55:04 +0200 Subject: [PATCH 4/5] fix VAD --- src/webrtc/callFeed.ts | 46 +++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 9d96448512f..f7d7be5ca08 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -21,7 +21,7 @@ import { RoomMember } from "../models/room-member"; import { logger } from "../logger"; import { TypedEventEmitter } from "../models/typed-event-emitter"; -const POLLING_INTERVAL = 10; // ms +const POLLING_INTERVAL = 20; // ms export const SPEAKING_THRESHOLD = -60; // dB const SPEAKING_SAMPLE_COUNT = 8; // samples @@ -266,7 +266,7 @@ export class CallFeed extends TypedEventEmitter this.speakingThreshold = threshold; } - private volumeLooper = async () => { + private volumeLooper = () => { if (!this.analyser) return; if (!this.measuringVolumeActivity) return; @@ -285,11 +285,26 @@ export class CallFeed extends TypedEventEmitter this.emit(CallFeedEvent.VolumeChanged, this.maxVolume); - let newSpeaking = false; + // Handle voice activity detection only if it is enabled and user has not manually muted themselves + if (this.VADEnabled) { + // If the user is speaking + if (this.maxVolume > this.voiceActivityThreshold) { + this.VADCooldownStarted = new Date(); + + if (this.vadAudioMuted) { + this.setVADMute?.(false); + } + } else if (!this.vadAudioMuted) { + // User stops speaking + + if (!this.isVADinCooldown()) { + // user has been silent for X milliseconds + this.setVADMute?.(true); + } + } + } - const gainNode = this.audioContext.createGain(); - gainNode.gain.value = 0.1; - gainNode.connect(this.audioContext.destination); + let newSpeaking = false; for (let i = 0; i < this.speakingVolumeSamples.length; i++) { const volume = this.speakingVolumeSamples[i]; @@ -305,25 +320,6 @@ export class CallFeed extends TypedEventEmitter this.emit(CallFeedEvent.Speaking, this.speaking); } - // Handle voice activity detection only if it is enabled and user has not manually muted themselves - if (this.VADEnabled) { - // If the user is speaking - if (this.maxVolume > this.voiceActivityThreshold) { - this.VADCooldownStarted = new Date(); - - if (this.vadAudioMuted) { - this.setVADMute?.(false); - } - } else if (!this.vadAudioMuted) { - // User stops speaking - - if (!this.isVADinCooldown()) { - // user has been silent for X milliseconds - this.setVADMute?.(true); - } - } - } - this.volumeLooperTimeout = setTimeout( this.volumeLooper, POLLING_INTERVAL, From a68e52589769191d805454ef56861b075f0dc2f3 Mon Sep 17 00:00:00 2001 From: Dashie Date: Thu, 25 Aug 2022 20:02:45 +0200 Subject: [PATCH 5/5] Update src/webrtc/groupCall.ts Co-authored-by: Hugo Hutri <55588133+hugohutri@users.noreply.github.com> --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0c74b074130..4f6fe1c8948 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -743,7 +743,7 @@ export class GroupCall extends TypedEventEmitter< } private async sendMemberStateEvent(): Promise { - const send = () =>this.updateMemberCallState({ + const send = () => this.updateMemberCallState({ "m.call_id": this.groupCallId, "m.devices": [ {