From cdd924503a7b5ed9d467d205c2dfd5378dce5e6e Mon Sep 17 00:00:00 2001 From: tofunmi Date: Tue, 14 Feb 2023 11:23:29 +0000 Subject: [PATCH] Fix InternalTextureManager so that all frames are sent downstream PiperOrigin-RevId: 509478455 --- .../GlEffectsFrameProcessorPixelTest.java | 3 +- .../effect/InternalTextureManager.java | 128 ++++++++++-------- 2 files changed, 77 insertions(+), 54 deletions(-) diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java index 01747da3922..59c951a2653 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java @@ -145,7 +145,8 @@ public void wrappedCrop_withImageInput_matchesGoldenFile() throws Exception { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } // TODO(b/262693274): Once texture deletion is added to InternalTextureManager.java, add a test - // queuing multiple input bitmaps to ensure successfully completion without errors. + // queuing multiple input bitmaps to ensure successfully completion without errors, ensuring the + // correct number of frames haas been queued. @Test public void noEffects_withFrameCache_matchesGoldenFile() throws Exception { diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java index d1b09caf692..148db71bd1f 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java @@ -16,7 +16,7 @@ package com.google.android.exoplayer2.effect; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; -import static java.lang.Math.round; +import static java.lang.Math.floor; import android.graphics.Bitmap; import android.opengl.GLES20; @@ -29,19 +29,31 @@ import java.util.Queue; import java.util.concurrent.LinkedBlockingQueue; -/** Forwards a frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for consumption. */ +/** + * Forwards a frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for consumption + * + *

Methods in this class can be called from any thread. + */ /* package */ class InternalTextureManager implements GlShaderProgram.InputListener { private final GlShaderProgram shaderProgram; private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; + // The queue holds all bitmaps with one or more frames pending to be sent downstream. private final Queue pendingBitmaps; private int downstreamShaderProgramCapacity; - private int availableFrameCount; - + private int framesToQueueForCurrentBitmap; private long currentPresentationTimeUs; - private long totalDurationUs; private boolean inputEnded; + private boolean outputEnded; + /** + * Creates a new instance. + * + * @param shaderProgram The {@link GlShaderProgram} for which this {@code InternalTextureManager} + * will be set as the {@link GlShaderProgram.InputListener}. + * @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor} that the methods of + * this class run on. + */ public InternalTextureManager( GlShaderProgram shaderProgram, FrameProcessingTaskExecutor frameProcessingTaskExecutor) { this.shaderProgram = shaderProgram; @@ -51,6 +63,10 @@ public InternalTextureManager( @Override public void onReadyToAcceptInputFrame() { + // TODO(b/262693274): Delete texture when last duplicate of the frame comes back from the shader + // program and change to only allocate one texId at a time. A change to the + // onInputFrameProcessed() method signature to include presentationTimeUs will probably be + // needed to do this. frameProcessingTaskExecutor.submit( () -> { downstreamShaderProgramCapacity++; @@ -58,84 +74,90 @@ public void onReadyToAcceptInputFrame() { }); } - @Override - public void onInputFrameProcessed(TextureInfo inputTexture) { - // TODO(b/262693274): Delete texture when last duplicate of the frame comes back from the shader - // program and change to only allocate one texId at a time. A change to method signature to - // include presentationTimeUs will probably be needed to do this. - frameProcessingTaskExecutor.submit( - () -> { - if (availableFrameCount == 0) { - signalEndOfInput(); - } - }); - } - + /** + * Provides an input {@link Bitmap} to put into the video frames. + * + * @see FrameProcessor#queueInputBitmap + */ public void queueInputBitmap( Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { frameProcessingTaskExecutor.submit( () -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr)); } + /** + * Signals the end of the input. + * + * @see FrameProcessor#signalEndOfInput() + */ + public void signalEndOfInput() { + frameProcessingTaskExecutor.submit( + () -> { + inputEnded = true; + signalEndOfOutput(); + }); + } + @WorkerThread private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr) throws FrameProcessingException { + if (inputEnded) { return; } + int bitmapTexId; try { - int bitmapTexId = + bitmapTexId = GlUtil.createTexture( bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ useHdr); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0); GlUtil.checkGlError(); - - TextureInfo textureInfo = - new TextureInfo( - bitmapTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight()); - int timeIncrementUs = round(C.MICROS_PER_SECOND / frameRate); - availableFrameCount += round((frameRate * durationUs) / C.MICROS_PER_SECOND); - totalDurationUs += durationUs; - pendingBitmaps.add( - new BitmapFrameSequenceInfo(textureInfo, timeIncrementUs, totalDurationUs)); } catch (GlUtil.GlException e) { throw FrameProcessingException.from(e); } + TextureInfo textureInfo = + new TextureInfo( + bitmapTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight()); + int framesToAdd = (int) floor(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); + long frameDurationUs = (long) floor(C.MICROS_PER_SECOND / frameRate); + pendingBitmaps.add(new BitmapFrameSequenceInfo(textureInfo, frameDurationUs, framesToAdd)); + maybeQueueToShaderProgram(); } @WorkerThread private void maybeQueueToShaderProgram() { - if (inputEnded || availableFrameCount == 0 || downstreamShaderProgramCapacity == 0) { + if (pendingBitmaps.isEmpty() || downstreamShaderProgramCapacity == 0) { return; } - availableFrameCount--; + + BitmapFrameSequenceInfo currentBitmap = checkNotNull(pendingBitmaps.peek()); + if (framesToQueueForCurrentBitmap == 0) { + framesToQueueForCurrentBitmap = currentBitmap.numberOfFrames; + } + + framesToQueueForCurrentBitmap--; downstreamShaderProgramCapacity--; - BitmapFrameSequenceInfo currentFrame = checkNotNull(pendingBitmaps.peek()); - shaderProgram.queueInputFrame(currentFrame.textureInfo, currentPresentationTimeUs); + shaderProgram.queueInputFrame(currentBitmap.textureInfo, currentPresentationTimeUs); - currentPresentationTimeUs += currentFrame.timeIncrementUs; - if (currentPresentationTimeUs >= currentFrame.endPresentationTimeUs) { + currentPresentationTimeUs += currentBitmap.frameDurationUs; + if (framesToQueueForCurrentBitmap == 0) { pendingBitmaps.remove(); + signalEndOfOutput(); } } - /** - * Signals the end of the input. - * - * @see FrameProcessor#signalEndOfInput() - */ - public void signalEndOfInput() { - frameProcessingTaskExecutor.submit( - () -> { - if (inputEnded) { - return; - } - inputEnded = true; - shaderProgram.signalEndOfCurrentInputStream(); - }); + @WorkerThread + private void signalEndOfOutput() { + if (framesToQueueForCurrentBitmap == 0 + && pendingBitmaps.isEmpty() + && inputEnded + && !outputEnded) { + shaderProgram.signalEndOfCurrentInputStream(); + outputEnded = true; + } } /** @@ -144,14 +166,14 @@ public void signalEndOfInput() { */ private static final class BitmapFrameSequenceInfo { public final TextureInfo textureInfo; - public final long timeIncrementUs; - public final long endPresentationTimeUs; + public final long frameDurationUs; + public final int numberOfFrames; public BitmapFrameSequenceInfo( - TextureInfo textureInfo, long timeIncrementUs, long endPresentationTimeUs) { + TextureInfo textureInfo, long frameDurationUs, int numberOfFrames) { this.textureInfo = textureInfo; - this.timeIncrementUs = timeIncrementUs; - this.endPresentationTimeUs = endPresentationTimeUs; + this.frameDurationUs = frameDurationUs; + this.numberOfFrames = numberOfFrames; } } }