Skip to content

Commit

Permalink
Fix InternalTextureManager so that all frames are sent downstream
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 509478455
  • Loading branch information
@tof-tof @christosts
tof-tof authored and christosts committed Feb 14, 2023
1 parent 08cf6db commit cdd9245
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 54 deletions.
3 changes: 2 additions & 1 deletion ...droidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ public void wrappedCrop_withImageInput_matchesGoldenFile() throws Exception {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
// TODO(b/262693274): Once texture deletion is added to InternalTextureManager.java, add a test
// queuing multiple input bitmaps to ensure successfully completion without errors.
// queuing multiple input bitmaps to ensure successfully completion without errors, ensuring the
// correct number of frames haas been queued.

@Test
public void noEffects_withFrameCache_matchesGoldenFile() throws Exception {
Expand Down
128 changes: 75 additions & 53 deletions ...ary/effect/src/main/java/com/google/android/exoplayer2/effect/InternalTextureManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
package com.google.android.exoplayer2.effect;

import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static java.lang.Math.round;
import static java.lang.Math.floor;

import android.graphics.Bitmap;
import android.opengl.GLES20;
Expand All @@ -29,19 +29,31 @@
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;

/** Forwards a frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for consumption. */
/**
* Forwards a frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for consumption
*
* <p>Methods in this class can be called from any thread.
*/
/* package */ class InternalTextureManager implements GlShaderProgram.InputListener {
private final GlShaderProgram shaderProgram;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;

private int downstreamShaderProgramCapacity;
private int availableFrameCount;

private int framesToQueueForCurrentBitmap;
private long currentPresentationTimeUs;
private long totalDurationUs;
private boolean inputEnded;
private boolean outputEnded;

/**
* Creates a new instance.
*
* @param shaderProgram The {@link GlShaderProgram} for which this {@code InternalTextureManager}
* will be set as the {@link GlShaderProgram.InputListener}.
* @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor} that the methods of
* this class run on.
*/
public InternalTextureManager(
GlShaderProgram shaderProgram, FrameProcessingTaskExecutor frameProcessingTaskExecutor) {
this.shaderProgram = shaderProgram;
Expand All @@ -51,91 +63,101 @@ public InternalTextureManager(

@Override
public void onReadyToAcceptInputFrame() {
// TODO(b/262693274): Delete texture when last duplicate of the frame comes back from the shader
// program and change to only allocate one texId at a time. A change to the
// onInputFrameProcessed() method signature to include presentationTimeUs will probably be
// needed to do this.
frameProcessingTaskExecutor.submit(
() -> {
downstreamShaderProgramCapacity++;
maybeQueueToShaderProgram();
});
}

@Override
public void onInputFrameProcessed(TextureInfo inputTexture) {
// TODO(b/262693274): Delete texture when last duplicate of the frame comes back from the shader
// program and change to only allocate one texId at a time. A change to method signature to
// include presentationTimeUs will probably be needed to do this.
frameProcessingTaskExecutor.submit(
() -> {
if (availableFrameCount == 0) {
signalEndOfInput();
}
});
}

/**
* Provides an input {@link Bitmap} to put into the video frames.
*
* @see FrameProcessor#queueInputBitmap
*/
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
frameProcessingTaskExecutor.submit(
() -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
}

/**
* Signals the end of the input.
*
* @see FrameProcessor#signalEndOfInput()
*/
public void signalEndOfInput() {
frameProcessingTaskExecutor.submit(
() -> {
inputEnded = true;
signalEndOfOutput();
});
}

@WorkerThread
private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr)
throws FrameProcessingException {

if (inputEnded) {
return;
}
int bitmapTexId;
try {
int bitmapTexId =
bitmapTexId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ useHdr);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
GlUtil.checkGlError();

TextureInfo textureInfo =
new TextureInfo(
bitmapTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight());
int timeIncrementUs = round(C.MICROS_PER_SECOND / frameRate);
availableFrameCount += round((frameRate * durationUs) / C.MICROS_PER_SECOND);
totalDurationUs += durationUs;
pendingBitmaps.add(
new BitmapFrameSequenceInfo(textureInfo, timeIncrementUs, totalDurationUs));
} catch (GlUtil.GlException e) {
throw FrameProcessingException.from(e);
}
TextureInfo textureInfo =
new TextureInfo(
bitmapTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight());
int framesToAdd = (int) floor(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
long frameDurationUs = (long) floor(C.MICROS_PER_SECOND / frameRate);
pendingBitmaps.add(new BitmapFrameSequenceInfo(textureInfo, frameDurationUs, framesToAdd));

maybeQueueToShaderProgram();
}

@WorkerThread
private void maybeQueueToShaderProgram() {
if (inputEnded || availableFrameCount == 0 || downstreamShaderProgramCapacity == 0) {
if (pendingBitmaps.isEmpty() || downstreamShaderProgramCapacity == 0) {
return;
}
availableFrameCount--;

BitmapFrameSequenceInfo currentBitmap = checkNotNull(pendingBitmaps.peek());
if (framesToQueueForCurrentBitmap == 0) {
framesToQueueForCurrentBitmap = currentBitmap.numberOfFrames;
}

framesToQueueForCurrentBitmap--;
downstreamShaderProgramCapacity--;

BitmapFrameSequenceInfo currentFrame = checkNotNull(pendingBitmaps.peek());
shaderProgram.queueInputFrame(currentFrame.textureInfo, currentPresentationTimeUs);
shaderProgram.queueInputFrame(currentBitmap.textureInfo, currentPresentationTimeUs);

currentPresentationTimeUs += currentFrame.timeIncrementUs;
if (currentPresentationTimeUs >= currentFrame.endPresentationTimeUs) {
currentPresentationTimeUs += currentBitmap.frameDurationUs;
if (framesToQueueForCurrentBitmap == 0) {
pendingBitmaps.remove();
signalEndOfOutput();
}
}

/**
* Signals the end of the input.
*
* @see FrameProcessor#signalEndOfInput()
*/
public void signalEndOfInput() {
frameProcessingTaskExecutor.submit(
() -> {
if (inputEnded) {
return;
}
inputEnded = true;
shaderProgram.signalEndOfCurrentInputStream();
});
@WorkerThread
private void signalEndOfOutput() {
if (framesToQueueForCurrentBitmap == 0
&& pendingBitmaps.isEmpty()
&& inputEnded
&& !outputEnded) {
shaderProgram.signalEndOfCurrentInputStream();
outputEnded = true;
}
}

/**
Expand All @@ -144,14 +166,14 @@ public void signalEndOfInput() {
*/
private static final class BitmapFrameSequenceInfo {
public final TextureInfo textureInfo;
public final long timeIncrementUs;
public final long endPresentationTimeUs;
public final long frameDurationUs;
public final int numberOfFrames;

public BitmapFrameSequenceInfo(
TextureInfo textureInfo, long timeIncrementUs, long endPresentationTimeUs) {
TextureInfo textureInfo, long frameDurationUs, int numberOfFrames) {
this.textureInfo = textureInfo;
this.timeIncrementUs = timeIncrementUs;
this.endPresentationTimeUs = endPresentationTimeUs;
this.frameDurationUs = frameDurationUs;
this.numberOfFrames = numberOfFrames;
}
}
}

0 comments on commit cdd9245

Please sign in to comment.