From 67772a8bffc855a727fa88ffdcf3535de0f52432 Mon Sep 17 00:00:00 2001 From: Guoen Yong Date: Thu, 11 Nov 2021 16:30:33 +0800 Subject: [PATCH 1/3] chore: Polish the track group generate logic of multiple muxed audio tracks. --- .../exoplayer2/source/hls/HlsMediaPeriod.java | 71 +++++++++++++++---- .../source/hls/HlsSampleStreamWrapper.java | 31 ++++---- .../hls/playlist/HlsPlaylistParser.java | 10 +-- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java index a4db3d9c528..024bb688972 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaPeriod.java @@ -620,12 +620,13 @@ private void buildAndPrepareMainSampleStreamWrapper( numberOfAudioCodecs <= 1 && numberOfVideoCodecs <= 1 && numberOfAudioCodecs + numberOfVideoCodecs > 0; + @Nullable List muxedAudioFormats = getMuxedAudioFormats(masterPlaylist); HlsSampleStreamWrapper sampleStreamWrapper = buildSampleStreamWrapper( C.TRACK_TYPE_DEFAULT, selectedPlaylistUrls, selectedPlaylistFormats, - masterPlaylist.muxedAudioFormat, + muxedAudioFormats, masterPlaylist.muxedCaptionFormats, overridingDrmInitData, positionUs); @@ -633,6 +634,8 @@ private void buildAndPrepareMainSampleStreamWrapper( manifestUrlIndicesPerWrapper.add(selectedVariantIndices); if (allowChunklessPreparation && codecsStringAllowsChunklessPreparation) { List muxedTrackGroups = new ArrayList<>(); + int muxedAudioFormatCount = (muxedAudioFormats == null ? 0 : muxedAudioFormats.size()); + Format firstMuxedAudioFormat = (muxedAudioFormatCount > 0 ? muxedAudioFormats.get(0) : null); if (numberOfVideoCodecs > 0) { Format[] videoFormats = new Format[selectedVariantsCount]; for (int i = 0; i < videoFormats.length; i++) { @@ -641,13 +644,14 @@ private void buildAndPrepareMainSampleStreamWrapper( muxedTrackGroups.add(new TrackGroup(videoFormats)); if (numberOfAudioCodecs > 0 - && (masterPlaylist.muxedAudioFormat != null || masterPlaylist.audios.isEmpty())) { - muxedTrackGroups.add( - new TrackGroup( - deriveAudioFormat( - selectedPlaylistFormats[0], - masterPlaylist.muxedAudioFormat, - /* isPrimaryTrackInVariant= */ false))); + && (muxedAudioFormats != null || masterPlaylist.audios.isEmpty())) { + Format variantFormat = selectedPlaylistFormats[0]; + Format[] formats = new Format[muxedAudioFormatCount > 0 ? muxedAudioFormatCount : 1]; + for (int i = 0; i < formats.length; i++) { + Format mediaTagFormat = (i < muxedAudioFormatCount ? muxedAudioFormats.get(i) : firstMuxedAudioFormat); + formats[i] = deriveAudioFormat(variantFormat, mediaTagFormat, false); + } + muxedTrackGroups.add(new TrackGroup(formats)); } List ccFormats = masterPlaylist.muxedCaptionFormats; if (ccFormats != null) { @@ -659,10 +663,14 @@ private void buildAndPrepareMainSampleStreamWrapper( // Variants only contain audio. Format[] audioFormats = new Format[selectedVariantsCount]; for (int i = 0; i < audioFormats.length; i++) { + // TODO: We should change to find the matched muxedAudioFormat via audio group of + // variantFormat, But the format have not the group information, we will polish it here + // when supporting audio group feature later. + Format mediaTagFormat = (i < muxedAudioFormatCount ? muxedAudioFormats.get(i) : firstMuxedAudioFormat); audioFormats[i] = deriveAudioFormat( /* variantFormat= */ selectedPlaylistFormats[i], - masterPlaylist.muxedAudioFormat, + mediaTagFormat, /* isPrimaryTrackInVariant= */ true); } muxedTrackGroups.add(new TrackGroup(audioFormats)); @@ -700,8 +708,9 @@ private void buildAndPrepareAudioSampleStreamWrappers( renditionByNameIndex < audioRenditions.size(); renditionByNameIndex++) { String name = audioRenditions.get(renditionByNameIndex).name; - if (!alreadyGroupedNames.add(name)) { - // This name already has a corresponding group. + Uri url = audioRenditions.get(renditionByNameIndex).url; + if (!alreadyGroupedNames.add(name) || url == null) { + // This name already has a corresponding group. Or it is one muxed audio rendition. continue; } @@ -745,7 +754,7 @@ private HlsSampleStreamWrapper buildSampleStreamWrapper( int trackType, Uri[] playlistUrls, Format[] playlistFormats, - @Nullable Format muxedAudioFormat, + @Nullable List muxedAudioFormats, @Nullable List muxedCaptionFormats, Map overridingDrmInitData, long positionUs) { @@ -766,7 +775,7 @@ private HlsSampleStreamWrapper buildSampleStreamWrapper( overridingDrmInitData, allocator, positionUs, - muxedAudioFormat, + muxedAudioFormats, drmSessionManager, drmEventDispatcher, loadErrorHandlingPolicy, @@ -774,6 +783,42 @@ private HlsSampleStreamWrapper buildSampleStreamWrapper( metadataType); } + @Nullable + private static List getMuxedAudioFormats(HlsMasterPlaylist masterPlaylist) { + List muxedAudioFormats = new ArrayList<>(); + for (int i = 0; i < masterPlaylist.variants.size(); i++) { + Format muxedAudioFormat = getMuxedAudioFormat(masterPlaylist.audios, masterPlaylist.variants.get(i)); + if (muxedAudioFormat == null) { + continue; + } + boolean exist = false; + for (int j = 0; j < muxedAudioFormats.size(); j++) { + if (muxedAudioFormats.get(j) == muxedAudioFormat) { + exist = true; + break; + } + } + if (!exist) { + muxedAudioFormats.add(muxedAudioFormat); + } + } + return muxedAudioFormats.size() > 0 ? muxedAudioFormats : null; + } + + @Nullable + private static Format getMuxedAudioFormat(List audioRenditions, HlsMasterPlaylist.Variant variant) { + if (variant.audioGroupId == null) { + return null; + } + for (int i = 0; i < audioRenditions.size(); i++) { + HlsMasterPlaylist.Rendition audio = audioRenditions.get(i); + if (audio.url == null && variant.audioGroupId.equals(audio.groupId)) { + return audio.format; + } + } + return null; + } + private static Map deriveOverridingDrmInitData( List sessionKeyDrmInitData) { ArrayList mutableSessionKeyDrmInitData = new ArrayList<>(sessionKeyDrmInitData); diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java index bfa430ca45b..3d89c8f8154 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java @@ -124,7 +124,7 @@ public interface Callback extends SequenceableLoader.Callback muxedAudioFormats; private final DrmSessionManager drmSessionManager; private final DrmSessionEventListener.EventDispatcher drmEventDispatcher; private final LoadErrorHandlingPolicy loadErrorHandlingPolicy; @@ -189,7 +189,7 @@ public interface Callback extends SequenceableLoader.Callback overridingDrmInitData, Allocator allocator, long positionUs, - @Nullable Format muxedAudioFormat, + @Nullable List muxedAudioFormats, DrmSessionManager drmSessionManager, DrmSessionEventListener.EventDispatcher drmEventDispatcher, LoadErrorHandlingPolicy loadErrorHandlingPolicy, @@ -215,7 +215,7 @@ public HlsSampleStreamWrapper( this.chunkSource = chunkSource; this.overridingDrmInitData = overridingDrmInitData; this.allocator = allocator; - this.muxedAudioFormat = muxedAudioFormat; + this.muxedAudioFormats = muxedAudioFormats; this.drmSessionManager = drmSessionManager; this.drmEventDispatcher = drmEventDispatcher; this.loadErrorHandlingPolicy = loadErrorHandlingPolicy; @@ -1344,13 +1344,9 @@ private void buildTracksFromSampleStreams() { int chunkSourceTrackCount = chunkSourceTrackGroup.length; // Workaround for audio-only stream, https://github.com/google/ExoPlayer/issues/9608 - if (primaryExtractorTrackType == C.TRACK_TYPE_AUDIO && chunkSourceTrackCount > 0) { - Format trackFormat = chunkSourceTrackGroup.getFormat(0); - boolean isAudioOnlyChunk = (trackFormat.bitrate > 0); - if (isAudioOnlyChunk) { - primaryExtractorTrackType = C.TRACK_TYPE_VIDEO; - primaryExtractorTrackIndex = C.INDEX_UNSET; - } + if (primaryExtractorTrackType == C.TRACK_TYPE_AUDIO && muxedAudioFormats != null) { + primaryExtractorTrackType = C.TRACK_TYPE_VIDEO; + primaryExtractorTrackIndex = C.INDEX_UNSET; } // Instantiate the necessary internal data-structures. @@ -1377,12 +1373,19 @@ private void buildTracksFromSampleStreams() { primaryTrackGroupIndex = i; } else { @Nullable - Format trackFormat = + List muxedAudioFormats = primaryExtractorTrackType == C.TRACK_TYPE_VIDEO && MimeTypes.isAudio(sampleFormat.sampleMimeType) - ? muxedAudioFormat + ? this.muxedAudioFormats : null; - trackGroups[i] = new TrackGroup(deriveFormat(trackFormat, sampleFormat, false)); + int muxedAudioFormatCount = (muxedAudioFormats == null ? 0 : muxedAudioFormats.size()); + Format firstMuxedAudioFormat = (muxedAudioFormatCount > 0 ? muxedAudioFormats.get(0) : null); + Format[] formats = new Format[muxedAudioFormatCount > 0 ? muxedAudioFormatCount : 1]; + for (int j = 0; j < formats.length; j++) { + Format playlistFormat = (j < muxedAudioFormatCount ? muxedAudioFormats.get(j) : firstMuxedAudioFormat); + formats[j] = deriveFormat(playlistFormat, sampleFormat, false); + } + trackGroups[i] = new TrackGroup(formats); } } this.trackGroups = createTrackGroupArrayWithDrmInfo(trackGroups); diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java index 314f8e7d87f..1304d1dd787 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/playlist/HlsPlaylistParser.java @@ -511,13 +511,9 @@ private static HlsMasterPlaylist parseMasterPlaylist(LineIterator iterator, Stri } } formatBuilder.setSampleMimeType(sampleMimeType); - if (uri != null) { - formatBuilder.setMetadata(metadata); - audios.add(new Rendition(uri, formatBuilder.build(), groupId, name)); - } else if (variant != null) { - // TODO: Remove muxedAudioFormat and add a Rendition with a null uri to audios. - muxedAudioFormat = formatBuilder.build(); - } + // Remove muxedAudioFormat and add a Rendition with a null uri to audios. + formatBuilder.setMetadata(metadata); + audios.add(new Rendition(uri, formatBuilder.build(), groupId, name)); break; case TYPE_SUBTITLES: sampleMimeType = null; From 36d016fc9f8f2f38ae3ea354a2e26c84e9497261 Mon Sep 17 00:00:00 2001 From: Guoen Yong Date: Thu, 18 Nov 2021 12:32:32 +0800 Subject: [PATCH 2/3] chore: Apply the google Exoplayer changes for issues/9608, https://github.com/google/ExoPlayer/commit/45857b50dc251e67f7b0d609ba276a5aebf11677 --- .../source/hls/HlsSampleStreamWrapper.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java index 3d89c8f8154..5375946c5a2 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java @@ -1343,12 +1343,6 @@ private void buildTracksFromSampleStreams() { TrackGroup chunkSourceTrackGroup = chunkSource.getTrackGroup(); int chunkSourceTrackCount = chunkSourceTrackGroup.length; - // Workaround for audio-only stream, https://github.com/google/ExoPlayer/issues/9608 - if (primaryExtractorTrackType == C.TRACK_TYPE_AUDIO && muxedAudioFormats != null) { - primaryExtractorTrackType = C.TRACK_TYPE_VIDEO; - primaryExtractorTrackIndex = C.INDEX_UNSET; - } - // Instantiate the necessary internal data-structures. primaryTrackGroupIndex = C.INDEX_UNSET; trackGroupToSampleQueueIndex = new int[extractorTrackCount]; @@ -1361,13 +1355,28 @@ private void buildTracksFromSampleStreams() { for (int i = 0; i < extractorTrackCount; i++) { Format sampleFormat = Assertions.checkStateNotNull(sampleQueues[i].getUpstreamFormat()); if (i == primaryExtractorTrackIndex) { + @Nullable List muxedAudioFormats = this.muxedAudioFormats; + int muxedAudioFormatCount = (muxedAudioFormats == null ? 0 : muxedAudioFormats.size()); + Format firstMuxedAudioFormat = (muxedAudioFormatCount > 0 ? muxedAudioFormats.get(0) : null); Format[] formats = new Format[chunkSourceTrackCount]; - if (chunkSourceTrackCount == 1) { - formats[0] = sampleFormat.withManifestFormatInfo(chunkSourceTrackGroup.getFormat(0)); - } else { - for (int j = 0; j < chunkSourceTrackCount; j++) { - formats[j] = deriveFormat(chunkSourceTrackGroup.getFormat(j), sampleFormat, true); + for (int j = 0; j < chunkSourceTrackCount; j++) { + Format playlistFormat = chunkSourceTrackGroup.getFormat(j); + if (primaryExtractorTrackType == C.TRACK_TYPE_AUDIO && muxedAudioFormatCount > 0) { + // TODO: We should change to find the matched muxedAudioFormat via audio group of + // variantFormat, But the format have not the group information, we will polish it here + // when supporting audio group feature later. + Format muxedAudioFormat = (j < muxedAudioFormatCount ? muxedAudioFormats.get(j) : firstMuxedAudioFormat); + // The format.language may be miss if use playlistFormat.withManifestFormatInfo(muxedAudioFormat) + // because of the playlistFormat.sampleMimeType is null. + playlistFormat = muxedAudioFormat.withManifestFormatInfo(playlistFormat); } + // If there's only a single variant (chunkSourceTrackCount == 1) then we can safely + // retain all fields from sampleFormat. Else we need to use deriveFormat to retain only + // the fields that will be the same for all variants. + formats[j] = + chunkSourceTrackCount == 1 + ? sampleFormat.withManifestFormatInfo(playlistFormat) + : deriveFormat(playlistFormat, sampleFormat, /* propagateBitrates= */ true); } trackGroups[i] = new TrackGroup(formats); primaryTrackGroupIndex = i; From e5e58e878f537bed2ee48924e6a0b4e006755f3f Mon Sep 17 00:00:00 2001 From: Guoen Yong Date: Thu, 18 Nov 2021 13:58:50 +0800 Subject: [PATCH 3/3] chore: polish the format merge for audio only stream. --- .../android/exoplayer2/source/hls/HlsSampleStreamWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java index 5375946c5a2..56efbc53a93 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsSampleStreamWrapper.java @@ -1368,7 +1368,7 @@ private void buildTracksFromSampleStreams() { Format muxedAudioFormat = (j < muxedAudioFormatCount ? muxedAudioFormats.get(j) : firstMuxedAudioFormat); // The format.language may be miss if use playlistFormat.withManifestFormatInfo(muxedAudioFormat) // because of the playlistFormat.sampleMimeType is null. - playlistFormat = muxedAudioFormat.withManifestFormatInfo(playlistFormat); + playlistFormat = muxedAudioFormat; } // If there's only a single variant (chunkSourceTrackCount == 1) then we can safely // retain all fields from sampleFormat. Else we need to use deriveFormat to retain only