Skip to content

Commit

Permalink
Resample float audio to 16-bit by default to enable audio processing
Browse files Browse the repository at this point in the history 
This is less confusing than having audio processing functionality (e.g., playback
speed adjustment) just "not work" for some pieces of media.

If this change is merged, I will update #6749 to also track making DefaultAudioSink
intelligently enable/disable float output depending on how the audio processors are
configured.

Issue: #7134
PiperOrigin-RevId: 302871568
  • Loading branch information
@ojw28
ojw28 committed Mar 25, 2020
1 parent f794a79 commit 2555fb3
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 55 deletions.
6 changes: 6 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@
* `SimpleDecoderVideoRenderer` and `SimpleDecoderAudioRenderer` renamed to
`DecoderVideoRenderer` and `DecoderAudioRenderer` respectively, and
generalized to work with `Decoder` rather than `SimpleDecoder`.
* Audio:
* Enable playback speed adjustment and silence skipping for floating point
PCM audio, via resampling to 16-bit integer PCM. To output the original
floating point audio without adjustment, pass `enableFloatOutput=true`
to the `DefaultAudioSink` constructor
([#7134](https://github.com/google/ExoPlayer/issues/7134)).
* Text:
* Parse `<ruby>` and `<rt>` tags in WebVTT subtitles (rendering is coming
later).
Expand Down
10 changes: 6 additions & 4 deletions library/common/src/main/java/com/google/android/exoplayer2/util/Util.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1385,13 +1385,15 @@ public static boolean isEncodingLinearPcm(@C.Encoding int encoding) {
}

/**
* Returns whether {@code encoding} is high resolution (&gt; 16-bit) integer PCM.
* Returns whether {@code encoding} is high resolution (&gt; 16-bit) PCM.
*
* @param encoding The encoding of the audio data.
* @return Whether the encoding is high resolution integer PCM.
* @return Whether the encoding is high resolution PCM.
*/
public static boolean isEncodingHighResolutionIntegerPcm(@C.PcmEncoding int encoding) {
return encoding == C.ENCODING_PCM_24BIT || encoding == C.ENCODING_PCM_32BIT;
public static boolean isEncodingHighResolutionPcm(@C.PcmEncoding int encoding) {
return encoding == C.ENCODING_PCM_24BIT
|| encoding == C.ENCODING_PCM_32BIT
|| encoding == C.ENCODING_PCM_FLOAT;
}

/**
Expand Down
45 changes: 20 additions & 25 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ public long getSkippedOutputFrameCount() {

@Nullable private final AudioCapabilities audioCapabilities;
private final AudioProcessorChain audioProcessorChain;
private final boolean enableConvertHighResIntPcmToFloat;
private final boolean enableFloatOutput;
private final ChannelMappingAudioProcessor channelMappingAudioProcessor;
private final TrimmingAudioProcessor trimmingAudioProcessor;
private final AudioProcessor[] toIntPcmAvailableAudioProcessors;
Expand Down Expand Up @@ -316,7 +316,7 @@ public long getSkippedOutputFrameCount() {
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) {
this(audioCapabilities, audioProcessors, /* enableConvertHighResIntPcmToFloat= */ false);
this(audioCapabilities, audioProcessors, /* enableFloatOutput= */ false);
}

/**
Expand All @@ -326,19 +326,16 @@ public DefaultAudioSink(
* default capabilities (no encoded audio passthrough support) should be assumed.
* @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before
* output. May be empty.
* @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution
* integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer
* audio processing (for example, speed adjustment) will not be available when float output is
* in use.
* @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float
* output will be used if the input is 32-bit float, and also if the input is high resolution
* (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not
* be available when float output is in use.
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities,
AudioProcessor[] audioProcessors,
boolean enableConvertHighResIntPcmToFloat) {
this(
audioCapabilities,
new DefaultAudioProcessorChain(audioProcessors),
enableConvertHighResIntPcmToFloat);
boolean enableFloatOutput) {
this(audioCapabilities, new DefaultAudioProcessorChain(audioProcessors), enableFloatOutput);
}

/**
Expand All @@ -349,18 +346,18 @@ public DefaultAudioSink(
* default capabilities (no encoded audio passthrough support) should be assumed.
* @param audioProcessorChain An {@link AudioProcessorChain} which is used to apply playback
* parameters adjustments. The instance passed in must not be reused in other sinks.
* @param enableConvertHighResIntPcmToFloat Whether to enable conversion of high resolution
* integer PCM to 32-bit float for output, if possible. Functionality that uses 16-bit integer
* audio processing (for example, speed adjustment) will not be available when float output is
* in use.
* @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float
* output will be used if the input is 32-bit float, and also if the input is high resolution
* (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not
* be available when float output is in use.
*/
public DefaultAudioSink(
@Nullable AudioCapabilities audioCapabilities,
AudioProcessorChain audioProcessorChain,
boolean enableConvertHighResIntPcmToFloat) {
boolean enableFloatOutput) {
this.audioCapabilities = audioCapabilities;
this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain);
this.enableConvertHighResIntPcmToFloat = enableConvertHighResIntPcmToFloat;
this.enableFloatOutput = enableFloatOutput;
releasingConditionVariable = new ConditionVariable(true);
audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener());
channelMappingAudioProcessor = new ChannelMappingAudioProcessor();
Expand Down Expand Up @@ -443,18 +440,16 @@ public void configure(
}

boolean isInputPcm = Util.isEncodingLinearPcm(inputEncoding);
boolean processingEnabled = isInputPcm && inputEncoding != C.ENCODING_PCM_FLOAT;
boolean processingEnabled = isInputPcm;
int sampleRate = inputSampleRate;
int channelCount = inputChannelCount;
@C.Encoding int encoding = inputEncoding;
boolean shouldConvertHighResIntPcmToFloat =
enableConvertHighResIntPcmToFloat
boolean useFloatOutput =
enableFloatOutput
&& supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT)
&& Util.isEncodingHighResolutionIntegerPcm(inputEncoding);
&& Util.isEncodingHighResolutionPcm(inputEncoding);
AudioProcessor[] availableAudioProcessors =
shouldConvertHighResIntPcmToFloat
? toFloatPcmAvailableAudioProcessors
: toIntPcmAvailableAudioProcessors;
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) {
trimmingAudioProcessor.setTrimFrameCount(trimStartFrames, trimEndFrames);
channelMappingAudioProcessor.setChannelMap(outputChannels);
Expand Down Expand Up @@ -484,7 +479,7 @@ && supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT)
isInputPcm ? Util.getPcmFrameSize(inputEncoding, inputChannelCount) : C.LENGTH_UNSET;
int outputPcmFrameSize =
isInputPcm ? Util.getPcmFrameSize(encoding, channelCount) : C.LENGTH_UNSET;
boolean canApplyPlaybackParameters = processingEnabled && !shouldConvertHighResIntPcmToFloat;
boolean canApplyPlaybackParameters = processingEnabled && !useFloatOutput;
Configuration pendingConfiguration =
new Configuration(
isInputPcm,
Expand Down
70 changes: 44 additions & 26 deletions ...core/src/main/java/com/google/android/exoplayer2/audio/FloatResamplingAudioProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,19 @@
package com.google.android.exoplayer2.audio;

import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.util.Util;
import java.nio.ByteBuffer;

/**
* An {@link AudioProcessor} that converts 24-bit and 32-bit integer PCM audio to 32-bit float PCM
* audio.
* An {@link AudioProcessor} that converts high resolution PCM audio to 32-bit float. The following
* encodings are supported as input:
*
* <ul>
* <li>{@link C#ENCODING_PCM_24BIT}
* <li>{@link C#ENCODING_PCM_32BIT}
* <li>{@link C#ENCODING_PCM_FLOAT} ({@link #isActive()} will return {@code false})
* </ul>
*/
/* package */ final class FloatResamplingAudioProcessor extends BaseAudioProcessor {

Expand All @@ -32,42 +38,54 @@
@Override
public AudioFormat onConfigure(AudioFormat inputAudioFormat)
throws UnhandledAudioFormatException {
if (!Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)) {
@C.PcmEncoding int encoding = inputAudioFormat.encoding;
if (!Util.isEncodingHighResolutionPcm(encoding)) {
throw new UnhandledAudioFormatException(inputAudioFormat);
}
return Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding)
return encoding != C.ENCODING_PCM_FLOAT
? new AudioFormat(
inputAudioFormat.sampleRate, inputAudioFormat.channelCount, C.ENCODING_PCM_FLOAT)
: AudioFormat.NOT_SET;
}

@Override
public void queueInput(ByteBuffer inputBuffer) {
Assertions.checkState(Util.isEncodingHighResolutionIntegerPcm(inputAudioFormat.encoding));
boolean isInput32Bit = inputAudioFormat.encoding == C.ENCODING_PCM_32BIT;
int position = inputBuffer.position();
int limit = inputBuffer.limit();
int size = limit - position;

int resampledSize = isInput32Bit ? size : (size / 3) * 4;
ByteBuffer buffer = replaceOutputBuffer(resampledSize);
if (isInput32Bit) {
for (int i = position; i < limit; i += 4) {
int pcm32BitInteger =
(inputBuffer.get(i) & 0xFF)
| ((inputBuffer.get(i + 1) & 0xFF) << 8)
| ((inputBuffer.get(i + 2) & 0xFF) << 16)
| ((inputBuffer.get(i + 3) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
} else { // Input is 24-bit PCM.
for (int i = position; i < limit; i += 3) {
int pcm32BitInteger =
((inputBuffer.get(i) & 0xFF) << 8)
| ((inputBuffer.get(i + 1) & 0xFF) << 16)
| ((inputBuffer.get(i + 2) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
ByteBuffer buffer;
switch (inputAudioFormat.encoding) {
case C.ENCODING_PCM_24BIT:
buffer = replaceOutputBuffer((size / 3) * 4);
for (int i = position; i < limit; i += 3) {
int pcm32BitInteger =
((inputBuffer.get(i) & 0xFF) << 8)
| ((inputBuffer.get(i + 1) & 0xFF) << 16)
| ((inputBuffer.get(i + 2) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
break;
case C.ENCODING_PCM_32BIT:
buffer = replaceOutputBuffer(size);
for (int i = position; i < limit; i += 4) {
int pcm32BitInteger =
(inputBuffer.get(i) & 0xFF)
| ((inputBuffer.get(i + 1) & 0xFF) << 8)
| ((inputBuffer.get(i + 2) & 0xFF) << 16)
| ((inputBuffer.get(i + 3) & 0xFF) << 24);
writePcm32BitFloat(pcm32BitInteger, buffer);
}
break;
case C.ENCODING_PCM_8BIT:
case C.ENCODING_PCM_16BIT:
case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
case C.ENCODING_PCM_FLOAT:
case C.ENCODING_INVALID:
case Format.NO_VALUE:
default:
// Never happens.
throw new IllegalStateException();
}

inputBuffer.position(inputBuffer.limit());
Expand Down

0 comments on commit 2555fb3

Please sign in to comment.