Skip to content

Commit

Permalink
Add OETF and EOTF ES2 fragment shaders for non-HDR frames.
Browse files Browse the repository at this point in the history
* Transform the intermediate color space to linear SDR by applying the SMPTE 170M EOTF and OETF.
* Use linear colors for the color filter pixel tests and update all golden bitmaps.

PiperOrigin-RevId: 476124592
  • Loading branch information
leonwind authored and marcbaechinger committed Oct 19, 2022
1 parent 331d4d6 commit afd829e
Show file tree
Hide file tree
Showing 58 changed files with 201 additions and 87 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@
@RunWith(AndroidJUnit4.class)
public class ContrastPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String INCREASE_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_contrast.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/increase_contrast.png";
public static final String DECREASE_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/decrease_contrast.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/decrease_contrast.png";
public static final String MAXIMUM_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/maximum_contrast.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/maximum_contrast.png";

// OpenGL uses floats in [0, 1] and maps 0.5f to 128 = 256 / 2.
private static final int OPENGL_NEUTRAL_RGB_VALUE = 128;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@
@RunWith(AndroidJUnit4.class)
public final class CropPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String CROP_SMALLER_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_smaller.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/crop_smaller.png";
public static final String CROP_LARGER_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_larger.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/crop_larger.png";

private final Context context = getApplicationContext();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,27 +58,27 @@
@RunWith(AndroidJUnit4.class)
public final class GlEffectsFrameProcessorPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String SCALE_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/scale_wide.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/scale_wide.png";
public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_right.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/translate_right.png";
public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate_then_translate.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/rotate_then_translate.png";
public static final String ROTATE_THEN_SCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/rotate45_then_scale2w.png";
public static final String TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_then_rotate.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/translate_then_rotate.png";
public static final String REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/request_output_height.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/request_output_height.png";
public static final String CROP_THEN_ASPECT_RATIO_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_then_aspect_ratio.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/crop_then_aspect_ratio.png";
public static final String ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate_45_scale_to_fit.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/rotate_45_scale_to_fit.png";
public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_brightness.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/increase_brightness.png";
public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png";

/** Input video of which we only use the first frame. */
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,13 @@
@RunWith(AndroidJUnit4.class)
public final class MatrixTextureProcessorPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_right.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/translate_right.png";
public static final String SCALE_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/scale_narrow.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/scale_narrow.png";
public static final String ROTATE_90_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate90.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/rotate90.png";

private final Context context = getApplicationContext();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,19 @@
@RunWith(AndroidJUnit4.class)
public final class PresentationPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_narrow.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_narrow.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_wide.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_wide.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_with_crop_narrow.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_with_crop_narrow.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_with_crop_wide.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_with_crop_wide.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_stretch_to_fit_narrow.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_narrow.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_stretch_to_fit_wide.png";
"media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_wide.png";

private final Context context = getApplicationContext();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,13 @@
@RunWith(AndroidJUnit4.class)
public final class RgbAdjustmentPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String ONLY_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/only_red_channel.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/only_red_channel.png";
public static final String INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_red_channel.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/increase_red_channel.png";
public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_brightness.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/increase_brightness.png";

private final Context context = getApplicationContext();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@
@RunWith(AndroidJUnit4.class)
public final class RgbFilterPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String GRAYSCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/grayscale.png";
public static final String INVERT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/invert.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/invert.png";

private final Context context = getApplicationContext();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@
@RunWith(AndroidJUnit4.class)
public class SingleColorLutPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String LUT_MAP_WHITE_TO_GREEN_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/lut_map_white_to_green.png";
public static final String GRAYSCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/grayscale.png";
public static final String INVERT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/invert.png";
"media/bitmap/sample_mp4_first_frame/linear_colors/invert.png";
public static final String VERTICAL_HALD_IDENTITY_LUT = "media/bitmap/lut/identity.png";
public static final String VERTICAL_HALD_GRAYSCALE_LUT = "media/bitmap/lut/grayscale.png";
public static final String VERTICAL_HALD_INVERTED_LUT = "media/bitmap/lut/inverted.png";
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -104,4 +104,5 @@ void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
vec3 rgb = yuvToRgb(srcYuv);
outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0);
// TODO(b/241902517): Transform optical to electrical colors.
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


// ES 2 fragment shader that:
// 1. Samples from an external texture with uTexSampler copying from this
// texture to the current output.
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Transforms the optical colors back to electrical ones if uApplyOetf == 1
// using the SMPTE 170M OETF.

#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
uniform int uApplyOetf;

const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;

// Transforms a single channel from electrical to optical SDR.
float sdrEotfSingleChannel(float electricalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return electricalChannel < 0.0812
? electricalChannel / 4.500
: pow((electricalChannel + 0.099) / 1.099, gamma);
}

// Transforms electronical to optical SDR using the SMPTE 170M EOTF.
vec3 sdrEotf(vec3 electricalColor) {
return vec3(
sdrEotfSingleChannel(electricalColor.r),
sdrEotfSingleChannel(electricalColor.g),
sdrEotfSingleChannel(electricalColor.b));
}

// Transforms a single channel from optical to electrical SDR.
float sdrOetfSingleChannel(float opticalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return opticalChannel < 0.018
? opticalChannel * 4.500
: 1.099 * pow(opticalChannel, inverseGamma) - 0.099;
}

// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 sdrOetf(vec3 opticalColor) {
return uApplyOetf == 1
? vec3(
sdrOetfSingleChannel(opticalColor.r),
sdrOetfSingleChannel(opticalColor.g),
sdrOetfSingleChannel(opticalColor.b))
: opticalColor;
}

void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
vec3 linearInputColor = sdrEotf(inputColor.rgb);

vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);

gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// ES 2 fragment shader that:
// 1. Samples from uTexSampler, copying from this texture to the current
// output.
// 2. Applies a 4x4 RGB color matrix to change the pixel colors.
// 3. Transforms the optical colors to electrical colors using the SMPTE
// 170M OETF.

precision mediump float;
uniform sampler2D uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;

const float inverseGamma = 0.4500;

// Transforms a single channel from optical to electrical SDR.
float sdrOetfSingleChannel(float opticalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return opticalChannel < 0.018
? opticalChannel * 4.500
: 1.099 * pow(opticalChannel, inverseGamma) - 0.099;
}

// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 sdrOetf(vec3 opticalColor) {
return vec3(
sdrOetfSingleChannel(opticalColor.r),
sdrOetfSingleChannel(opticalColor.g),
sdrOetfSingleChannel(opticalColor.b));
}

void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1);

gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
}
Loading

0 comments on commit afd829e

Please sign in to comment.