Skip to content

Commit

Permalink
Add support for customizing overlay anchor point.
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 496956035
  • Loading branch information
@tof-tof @marcbaechinger
tof-tof authored and marcbaechinger committed Jan 4, 2023
1 parent 890fd0a commit 4d1283f
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 12 deletions.
29 changes: 29 additions & 0 deletions ...droidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ public class OverlayTextureProcessorPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String OVERLAY_BITMAP_DEFAULT =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png";
public static final String OVERLAY_BITMAP_ANCHORED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png";
public static final String OVERLAY_BITMAP_SCALED =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png";
public static final String OVERLAY_BITMAP_TRANSLUCENT =
Expand Down Expand Up @@ -173,6 +175,33 @@ public void drawFrame_scaledBitmapOverlay_blendsBitmapIntoFrame() throws Excepti
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}

@Test
public void drawFrame_anchoredBitmapOverlay_blendsBitmapIntoTopLeftOfFrame() throws Exception {
String testId = "drawFrame_anchoredBitmapOverlay";
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] translateMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ -1f, /* y= */ 1f, /* z= */ 1);
OverlaySettings overlaySettings =
new OverlaySettings.Builder().setMatrix(translateMatrix).setAnchor(-1f, 1f).build();
BitmapOverlay staticBitmapOverlay =
BitmapOverlay.createStaticBitmapOverlay(overlayBitmap, overlaySettings);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(staticBitmapOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_BITMAP_ANCHORED);

overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());

maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}

@Test
public void drawFrame_translucentBitmapOverlay_blendsBitmapIntoFrame() throws Exception {
String testId = "drawFrame_translucentBitmapOverlay";
Expand Down
33 changes: 31 additions & 2 deletions library/effect/src/main/java/com/google/android/exoplayer2/effect/OverlaySettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
import static com.google.android.exoplayer2.util.Assertions.checkArgument;

import android.util.Pair;
import androidx.annotation.FloatRange;
import com.google.android.exoplayer2.util.GlUtil;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
Expand All @@ -25,22 +26,26 @@ public final class OverlaySettings {
public final boolean useHdr;
public final float alpha;
public final float[] matrix;
public final Pair<Float, Float> anchor;

private OverlaySettings(boolean useHdr, float alpha, float[] matrix) {
private OverlaySettings(boolean useHdr, float alpha, float[] matrix, Pair<Float, Float> anchor) {
this.useHdr = useHdr;
this.alpha = alpha;
this.matrix = matrix;
this.anchor = anchor;
}

/** A builder for {@link OverlaySettings} instances. */
public static final class Builder {
private boolean useHdr;
private float alpha = 1;
private float[] matrix;
private Pair<Float, Float> anchor;

/** Creates a new {@link Builder}. */
public Builder() {
matrix = GlUtil.create4x4IdentityMatrix();
anchor = Pair.create(0f, 0f);
}

/**
Expand Down Expand Up @@ -81,9 +86,33 @@ public Builder setAlpha(@FloatRange(from = 0, to = 1) float alpha) {
return this;
}

/**
* Sets the coordinates for the anchor point of the overlay.
*
* <p>The anchor point is the point inside the overlay that the overlay is positioned from.
*
* <p>The coordinates are specified in Normalised Device Coordinates (NDCs). Set to always
* return {@code (0,0)} (the center) by default.
*
* @param x the NDC x-coordinate.
* @param y the NDC y-coordinate.
*/
@CanIgnoreReturnValue
public Builder setAnchor(
@FloatRange(from = -1, to = 1) float x, @FloatRange(from = -1, to = 1) float y) {
checkArgument(
-1 <= x && x <= 1,
"x needs to be specified in terms of NDCs which lie in the interval [-1, 1].");
checkArgument(
-1 <= y && y <= 1,
"y needs to be specified in terms of NDCs which lie in the interval [-1, 1].");
this.anchor = Pair.create(x, y);
return this;
}

/** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
public OverlaySettings build() {
return new OverlaySettings(useHdr, alpha, matrix);
return new OverlaySettings(useHdr, alpha, matrix, anchor);
}
}
}
48 changes: 38 additions & 10 deletions ...ry/effect/src/main/java/com/google/android/exoplayer2/effect/OverlayTextureProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import android.content.Context;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Pair;
import com.google.android.exoplayer2.util.FrameProcessingException;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
Expand All @@ -36,6 +37,8 @@
private final ImmutableList<TextureOverlay> overlays;
private final float[] aspectRatioMatrix;
private final float[] overlayMatrix;
private final float[] anchorMatrix;
private final float[] transformationMatrix;

private int videoWidth;
private int videoHeight;
Expand All @@ -61,7 +64,8 @@ public OverlayTextureProcessor(
this.overlays = overlays;
aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
overlayMatrix = GlUtil.create4x4IdentityMatrix();

anchorMatrix = GlUtil.create4x4IdentityMatrix();
transformationMatrix = GlUtil.create4x4IdentityMatrix();
try {
glProgram =
new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size()));
Expand Down Expand Up @@ -89,26 +93,53 @@ public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProce
if (!overlays.isEmpty()) {
for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
TextureOverlay overlay = overlays.get(texUnitIndex - 1);

glProgram.setSamplerTexIdUniform(
Util.formatInvariant("uOverlayTexSampler%d", texUnitIndex),
overlay.getTextureId(presentationTimeUs),
texUnitIndex);

GlUtil.setToIdentity(aspectRatioMatrix);
Matrix.scaleM(
aspectRatioMatrix,
MATRIX_OFFSET,
videoWidth / (float) overlay.getTextureSize(presentationTimeUs).getWidth(),
videoHeight / (float) overlay.getTextureSize(presentationTimeUs).getHeight(),
/* z= */ 1);
glProgram.setFloatsUniform(
Util.formatInvariant("uAspectRatioMatrix%d", texUnitIndex), aspectRatioMatrix);
Matrix.invertM(
overlayMatrix,
MATRIX_OFFSET,
overlay.getOverlaySettings(presentationTimeUs).matrix,
MATRIX_OFFSET);
Pair<Float, Float> overlayAnchor = overlay.getOverlaySettings(presentationTimeUs).anchor;
GlUtil.setToIdentity(anchorMatrix);
Matrix.translateM(
anchorMatrix,
/* mOffset= */ 0,
overlayAnchor.first
* overlay.getTextureSize(presentationTimeUs).getWidth()
/ videoWidth,
overlayAnchor.second
* overlay.getTextureSize(presentationTimeUs).getHeight()
/ videoHeight,
/* z= */ 1);
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
overlayMatrix,
MATRIX_OFFSET,
anchorMatrix,
MATRIX_OFFSET);
Matrix.multiplyMM(
transformationMatrix,
MATRIX_OFFSET,
aspectRatioMatrix,
MATRIX_OFFSET,
transformationMatrix,
MATRIX_OFFSET);
glProgram.setFloatsUniform(
Util.formatInvariant("uOverlayMatrix%d", texUnitIndex), overlayMatrix);
Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), transformationMatrix);

glProgram.setFloatUniform(
Util.formatInvariant("uOverlayAlpha%d", texUnitIndex),
overlay.getOverlaySettings(presentationTimeUs).alpha);
Expand Down Expand Up @@ -143,9 +174,8 @@ private static String createVertexShader(int numOverlays) {

for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
shader
.append(Util.formatInvariant("uniform mat4 uAspectRatioMatrix%d;\n", texUnitIndex))
.append(Util.formatInvariant("uniform mat4 uOverlayMatrix%d;\n", texUnitIndex))
.append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex));
.append(Util.formatInvariant("uniform mat4 uTransformationMatrix%s;\n", texUnitIndex))
.append(Util.formatInvariant("varying vec2 vOverlayTexSamplingCoord%s;\n", texUnitIndex));
}

shader
Expand All @@ -160,9 +190,7 @@ private static String createVertexShader(int numOverlays) {
shader
.append(Util.formatInvariant(" vec4 aOverlayPosition%d = \n", texUnitIndex))
.append(
Util.formatInvariant(
" uAspectRatioMatrix%d * uOverlayMatrix%d * aFramePosition;\n",
texUnitIndex, texUnitIndex))
Util.formatInvariant(" uTransformationMatrix%s * aFramePosition;\n", texUnitIndex))
.append(
Util.formatInvariant(
" vOverlayTexSamplingCoord%d = getTexSamplingCoord(aOverlayPosition%d.xy);\n",
Expand Down
Binary file added ...dia/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_anchored.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 4d1283f

Please sign in to comment.