Skip to content

Commit

Permalink
Improve Format propagation within the MediaCodecRenderer.
Browse files Browse the repository at this point in the history 
For example, fix handling of pixel aspect ratio changes in
playlists where video resolution does not change.

Issue:#6646
PiperOrigin-RevId: 281276023
  • Loading branch information
@Samrobbo @ojw28
Samrobbo authored and ojw28 committed Nov 22, 2019
1 parent dbd7e05 commit e26a61b
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 54 deletions.
7 changes: 7 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@
[embedded in Matroska streams](https://matroska.org/technical/specs/subtitles/index.html).
* Use `ExoMediaDrm.Provider` in `OfflineLicenseHelper` to avoid `ExoMediaDrm`
leaks ([#4721](https://github.com/google/ExoPlayer/issues/4721)).
* Improve `Format` propagation within the `MediaCodecRenderer` and subclasses.
For example, fix handling of pixel aspect ratio changes in playlists where
video resolution does not change.
([#6646](https://github.com/google/ExoPlayer/issues/6646)).
* Rename `MediaCodecRenderer.onOutputFormatChanged` to
`MediaCodecRenderer.onOutputMediaFormatChanged`, further
clarifying the distinction between `Format` and `MediaFormat`.
* Fix byte order of HDR10+ static metadata to match CTA-861.3.
* Reconfigure audio sink when PCM encoding changes
([#6601](https://github.com/google/ExoPlayer/issues/6601)).
Expand Down
10 changes: 5 additions & 5 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,10 +475,10 @@ protected void configureCodec(
protected @KeepCodecResult int canKeepCodec(
MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) {
// TODO: We currently rely on recreating the codec when encoder delay or padding is non-zero.
// Re-creating the codec is necessary to guarantee that onOutputFormatChanged is called, which
// is where encoder delay and padding are propagated to the sink. We should find a better way to
// propagate these values, and then allow the codec to be re-used in cases where this would
// otherwise be possible.
// Re-creating the codec is necessary to guarantee that onOutputMediaFormatChanged is called,
// which is where encoder delay and padding are propagated to the sink. We should find a better
// way to propagate these values, and then allow the codec to be re-used in cases where this
// would otherwise be possible.
if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize
|| oldFormat.encoderDelay != 0
|| oldFormat.encoderPadding != 0
Expand Down Expand Up @@ -558,7 +558,7 @@ protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybac
}

@Override
protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
throws ExoPlaybackException {
@C.Encoding int encoding;
MediaFormat mediaFormat;
Expand Down
35 changes: 23 additions & 12 deletions library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@ protected boolean shouldInitCodec(MediaCodecInfo codecInfo) {
return true;
}

/**
/*
* Returns whether the codec needs the renderer to propagate the end-of-stream signal directly,
* rather than by using an end-of-stream buffer queued to the codec.
*/
Expand All @@ -574,17 +574,17 @@ protected boolean getCodecNeedsEosPropagation() {
}

/**
* Polls the pending output format queue for a given buffer timestamp. If a format is present, it
* is removed and returned. Otherwise returns {@code null}. Subclasses should only call this
* method if they are taking over responsibility for output format propagation (e.g., when using
* video tunneling).
* Polls the pending output format queue for a given buffer timestamp. If a format is present,
* {@link #onOutputFormatChanged(Format)} is called. Subclasses should only call this method if
* they are taking over responsibility for output format propagation (e.g., when using video
* tunneling).
*/
protected final @Nullable Format updateOutputFormatForTime(long presentationTimeUs) {
Format format = formatQueue.pollFloor(presentationTimeUs);
protected final void updateOutputFormatForTime(long presentationTimeUs) {
@Nullable Format format = formatQueue.pollFloor(presentationTimeUs);
if (format != null) {
outputFormat = format;
onOutputFormatChanged(outputFormat);
}
return format;
}

protected final MediaCodec getCodec() {
Expand Down Expand Up @@ -1305,11 +1305,22 @@ protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybac
* @param outputMediaFormat The new output {@link MediaFormat}.
* @throws ExoPlaybackException Thrown if an error occurs handling the new output media format.
*/
protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
throws ExoPlaybackException {
// Do nothing.
}

/**
* Called when the output {@link Format} changes from the format queue.
*
* <p>The default implementation is a no-op.
*
* @param outputFormat The new output {@link Format}.
*/
protected void onOutputFormatChanged(Format outputFormat) {
// Do nothing.
}

/**
* Handles supplemental data associated with an input buffer.
*
Expand Down Expand Up @@ -1504,7 +1515,7 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)

if (outputIndex < 0) {
if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
processOutputFormat();
processOutputMediaFormat();
return true;
} else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
processOutputBuffersChanged();
Expand Down Expand Up @@ -1596,7 +1607,7 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
}

/** Processes a new output {@link MediaFormat}. */
private void processOutputFormat() throws ExoPlaybackException {
private void processOutputMediaFormat() throws ExoPlaybackException {
MediaFormat mediaFormat = codec.getOutputFormat();
if (codecAdaptationWorkaroundMode != ADAPTATION_WORKAROUND_MODE_NEVER
&& mediaFormat.getInteger(MediaFormat.KEY_WIDTH) == ADAPTATION_WORKAROUND_SLICE_WIDTH_HEIGHT
Expand All @@ -1609,7 +1620,7 @@ private void processOutputFormat() throws ExoPlaybackException {
if (codecNeedsMonoChannelCountWorkaround) {
mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
}
onOutputFormatChanged(codec, mediaFormat);
onOutputMediaFormatChanged(codec, mediaFormat);
}

/**
Expand Down
77 changes: 40 additions & 37 deletions library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,9 @@ public VideoDecoderException(
private int buffersInCodecCount;
private long lastRenderTimeUs;

private int pendingRotationDegrees;
private float pendingPixelWidthHeightRatio;
@Nullable private MediaFormat currentMediaFormat;
private int mediaFormatWidth;
private int mediaFormatHeight;
private int currentWidth;
private int currentHeight;
private int currentUnappliedRotationDegrees;
Expand Down Expand Up @@ -353,8 +353,9 @@ public MediaCodecVideoRenderer(
joiningDeadlineMs = C.TIME_UNSET;
currentWidth = Format.NO_VALUE;
currentHeight = Format.NO_VALUE;
mediaFormatWidth = Format.NO_VALUE;
mediaFormatHeight = Format.NO_VALUE;
currentPixelWidthHeightRatio = Format.NO_VALUE;
pendingPixelWidthHeightRatio = Format.NO_VALUE;
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
clearReportedVideoSize();
}
Expand Down Expand Up @@ -749,10 +750,7 @@ protected void onCodecInitialized(String name, long initializedTimestampMs,
@Override
protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybackException {
super.onInputFormatChanged(formatHolder);
Format newFormat = formatHolder.format;
eventDispatcher.inputFormatChanged(newFormat);
pendingPixelWidthHeightRatio = newFormat.pixelWidthHeightRatio;
pendingRotationDegrees = newFormat.rotationDegrees;
eventDispatcher.inputFormatChanged(formatHolder.format);
}

/**
Expand All @@ -773,26 +771,56 @@ protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
}

@Override
protected void onOutputFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat) {
protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat) {
currentMediaFormat = outputMediaFormat;

boolean hasCrop =
outputMediaFormat.containsKey(KEY_CROP_RIGHT)
&& outputMediaFormat.containsKey(KEY_CROP_LEFT)
&& outputMediaFormat.containsKey(KEY_CROP_BOTTOM)
&& outputMediaFormat.containsKey(KEY_CROP_TOP);
int width =
mediaFormatWidth =
hasCrop
? outputMediaFormat.getInteger(KEY_CROP_RIGHT)
- outputMediaFormat.getInteger(KEY_CROP_LEFT)
+ 1
: outputMediaFormat.getInteger(MediaFormat.KEY_WIDTH);
int height =
mediaFormatHeight =
hasCrop
? outputMediaFormat.getInteger(KEY_CROP_BOTTOM)
- outputMediaFormat.getInteger(KEY_CROP_TOP)
+ 1
: outputMediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
processOutputFormat(codec, width, height);

// Must be applied each time the output MediaFormat changes.
codec.setVideoScalingMode(scalingMode);
}

@Override
protected void onOutputFormatChanged(Format outputFormat) {
if (tunneling) {
currentWidth = outputFormat.width;
currentHeight = outputFormat.height;
} else {
currentWidth = mediaFormatWidth;
currentHeight = mediaFormatHeight;
}

currentPixelWidthHeightRatio = outputFormat.pixelWidthHeightRatio;
if (Util.SDK_INT >= 21) {
// On API level 21 and above the decoder applies the rotation when rendering to the surface.
// Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
// to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
if (outputFormat.rotationDegrees == 90 || outputFormat.rotationDegrees == 270) {
int rotatedHeight = currentWidth;
currentWidth = currentHeight;
currentHeight = rotatedHeight;
currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
}
} else {
// On API level 20 and below the decoder does not apply the rotation.
currentUnappliedRotationDegrees = outputFormat.rotationDegrees;
}
}

@Override
Expand Down Expand Up @@ -945,28 +973,6 @@ && maybeDropBuffersToKeyframe(
return false;
}

private void processOutputFormat(MediaCodec codec, int width, int height) {
currentWidth = width;
currentHeight = height;
currentPixelWidthHeightRatio = pendingPixelWidthHeightRatio;
if (Util.SDK_INT >= 21) {
// On API level 21 and above the decoder applies the rotation when rendering to the surface.
// Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
// to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
if (pendingRotationDegrees == 90 || pendingRotationDegrees == 270) {
int rotatedHeight = currentWidth;
currentWidth = currentHeight;
currentHeight = rotatedHeight;
currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
}
} else {
// On API level 20 and below the decoder does not apply the rotation.
currentUnappliedRotationDegrees = pendingRotationDegrees;
}
// Must be applied each time the output MediaFormat changes.
codec.setVideoScalingMode(scalingMode);
}

private void notifyFrameMetadataListener(
long presentationTimeUs, long releaseTimeNs, Format format, MediaFormat mediaFormat) {
if (frameMetadataListener != null) {
Expand All @@ -986,10 +992,7 @@ protected long getOutputStreamOffsetUs() {

/** Called when a buffer was processed in tunneling mode. */
protected void onProcessedTunneledBuffer(long presentationTimeUs) {
@Nullable Format format = updateOutputFormatForTime(presentationTimeUs);
if (format != null) {
processOutputFormat(getCodec(), format.width, format.height);
}
updateOutputFormatForTime(presentationTimeUs);
maybeNotifyVideoSizeChanged();
maybeNotifyRenderedFirstFrame();
onProcessedOutputBuffer(presentationTimeUs);
Expand Down

0 comments on commit e26a61b

Please sign in to comment.