Added support for Thumbnails in DASH

library/common/src/main/java/com/google/android/exoplayer2/Format.java

@@ -108,6 +108,13 @@
  * <ul>
  *   <li>{@link #accessibilityChannel}
  * </ul>
+ *
+ * <h2 id="image-formats">Fields relevant to image formats</h2>
+ *
+ * <ul>
+ *   <li>{@link #tileCountHorizontal}
+ *   <li>{@link #tileCountVertical}
+ * </ul>
  */
 public final class Format implements Bundleable {
 
@@ -167,6 +174,11 @@ public static final class Builder {
 
     private int accessibilityChannel;
 
+    // Image specific
+
+    private int tileCountHorizontal;
+    private int tileCountVertical;
+
     // Provided by the source.
 
     private @C.CryptoType int cryptoType;
@@ -190,6 +202,9 @@ public Builder() {
       pcmEncoding = NO_VALUE;
       // Text specific.
       accessibilityChannel = NO_VALUE;
+      // Image specific.
+      tileCountHorizontal = NO_VALUE;
+      tileCountVertical = NO_VALUE;
       // Provided by the source.
       cryptoType = C.CRYPTO_TYPE_NONE;
     }
@@ -234,6 +249,9 @@ private Builder(Format format) {
       this.encoderPadding = format.encoderPadding;
       // Text specific.
       this.accessibilityChannel = format.accessibilityChannel;
+      // Image specific.
+      this.tileCountHorizontal = format.tileCountHorizontal;
+      this.tileCountVertical = format.tileCountVertical;
       // Provided by the source.
       this.cryptoType = format.cryptoType;
     }
@@ -609,6 +627,30 @@ public Builder setAccessibilityChannel(int accessibilityChannel) {
       return this;
     }
 
+    // Image specific.
+
+    /**
+     * Sets {@link Format#tileCountHorizontal}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountHorizontal The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    public Builder setTileCountHorizontal(int tileCountHorizontal) {
+      this.tileCountHorizontal = tileCountHorizontal;
+      return this;
+    }
+
+    /**
+     * Sets {@link Format#tileCountVertical}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountVertical The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    public Builder setTileCountVertical(int tileCountVertical) {
+      this.tileCountVertical = tileCountVertical;
+      return this;
+    }
+
     // Provided by source.
 
     /**
@@ -781,6 +823,12 @@ public Format build() {
   /** The Accessibility channel, or {@link #NO_VALUE} if not known or applicable. */
   public final int accessibilityChannel;
 
+  // Image specific.
+
+  /** Thumbnail tile count horizontal and vertical, or {@link #NO_VALUE} if not known or applicable. */
+  public final int tileCountHorizontal;
+  public final int tileCountVertical;
+
   // Provided by source.
 
   /**
@@ -1004,6 +1052,9 @@ private Format(Builder builder) {
     encoderPadding = builder.encoderPadding == NO_VALUE ? 0 : builder.encoderPadding;
     // Text specific.
     accessibilityChannel = builder.accessibilityChannel;
+    // Image specific.
+    tileCountHorizontal = builder.tileCountHorizontal;
+    tileCountVertical = builder.tileCountVertical;
     // Provided by source.
     if (builder.cryptoType == C.CRYPTO_TYPE_NONE && drmInitData != null) {
       // Encrypted content cannot use CRYPTO_TYPE_NONE.
@@ -1250,6 +1301,9 @@ public int hashCode() {
       result = 31 * result + encoderPadding;
       // Text specific.
       result = 31 * result + accessibilityChannel;
+      // Image specific.
+      result = 31 * result + tileCountHorizontal;
+      result = 31 * result + tileCountVertical;
       // Provided by the source.
       result = 31 * result + cryptoType;
       hashCode = result;
@@ -1286,6 +1340,8 @@ public boolean equals(@Nullable Object obj) {
         && encoderDelay == other.encoderDelay
         && encoderPadding == other.encoderPadding
         && accessibilityChannel == other.accessibilityChannel
+        && tileCountHorizontal == other.tileCountHorizontal
+        && tileCountVertical == other.tileCountVertical
         && cryptoType == other.cryptoType
         && Float.compare(frameRate, other.frameRate) == 0
         && Float.compare(pixelWidthHeightRatio, other.pixelWidthHeightRatio) == 0
@@ -1480,6 +1536,8 @@ public static String toLogString(@Nullable Format format) {
   private static final String FIELD_ENCODER_PADDING = Util.intToStringMaxRadix(27);
   private static final String FIELD_ACCESSIBILITY_CHANNEL = Util.intToStringMaxRadix(28);
   private static final String FIELD_CRYPTO_TYPE = Util.intToStringMaxRadix(29);
+  private static final String FIELD_TILE_COUNT_HORIZONTAL = Util.intToStringMaxRadix(30);
+  private static final String FIELD_TILE_COUNT_VERTICAL = Util.intToStringMaxRadix(31);
 
   @Override
   public Bundle toBundle() {
@@ -1537,6 +1595,9 @@ public Bundle toBundle(boolean excludeMetadata) {
     bundle.putInt(FIELD_ACCESSIBILITY_CHANNEL, accessibilityChannel);
     // Source specific.
     bundle.putInt(FIELD_CRYPTO_TYPE, cryptoType);
+    // Image specific.
+    bundle.putInt(FIELD_TILE_COUNT_HORIZONTAL, tileCountHorizontal);
+    bundle.putInt(FIELD_TILE_COUNT_VERTICAL, tileCountVertical);
     return bundle;
   }
 
@@ -1597,8 +1658,10 @@ private static Format fromBundle(Bundle bundle) {
         .setEncoderDelay(bundle.getInt(FIELD_ENCODER_DELAY, DEFAULT.encoderDelay))
         .setEncoderPadding(bundle.getInt(FIELD_ENCODER_PADDING, DEFAULT.encoderPadding))
         // Text specific.
-        .setAccessibilityChannel(
-            bundle.getInt(FIELD_ACCESSIBILITY_CHANNEL, DEFAULT.accessibilityChannel))
+        .setAccessibilityChannel(bundle.getInt(FIELD_ACCESSIBILITY_CHANNEL, DEFAULT.accessibilityChannel))
+        // Image specific.
+        .setTileCountHorizontal(bundle.getInt(FIELD_TILE_COUNT_HORIZONTAL, DEFAULT.tileCountHorizontal))
+        .setTileCountVertical(bundle.getInt(FIELD_TILE_COUNT_VERTICAL, DEFAULT.tileCountVertical))
         // Source specific.
         .setCryptoType(bundle.getInt(FIELD_CRYPTO_TYPE, DEFAULT.cryptoType));
 

library/core/src/main/java/com/google/android/exoplayer2/thumbnail/ThumbnailDescription.java

@@ -0,0 +1,67 @@
+package com.google.android.exoplayer2.thumbnail;
+
+import android.net.Uri;
+import androidx.annotation.NonNull;
+
+public class ThumbnailDescription {
+
+  @NonNull private final String id;
+  @NonNull private final Uri uri;
+  private final int bitrate;
+  private final int tileCountHorizontal;
+  private final int tileCountVertical;
+  private final long startTimeMs;
+  private final long durationMs;
+  private final int imageWidth;            // Image width (Pixel)
+  private final int imageHeight;           // Image height (Pixel)
+
+  public ThumbnailDescription(@NonNull String id, @NonNull Uri uri, int bitrate, int tileCountHorizontal, int tileCountVertical, long startTimeMs, long durationMs, int imageWidth, int imageHeight) {
+    this.id = id;
+    this.uri = uri;
+    this.bitrate = bitrate;
+    this.tileCountHorizontal = tileCountHorizontal;
+    this.tileCountVertical = tileCountVertical;
+    this.startTimeMs = startTimeMs;
+    this.durationMs = durationMs;
+    this.imageWidth = imageWidth;
+    this.imageHeight = imageHeight;
+  }
+
+  @NonNull
+  public String getId() {
+    return id;
+  }
+
+  @NonNull
+  public Uri getUri() {
+    return uri;
+  }
+
+  public int getBitrate() {
+    return bitrate;
+  }
+
+  public int getTileCountHorizontal() {
+    return tileCountHorizontal;
+  }
+
+  public int getTileCountVertical() {
+    return tileCountVertical;
+  }
+
+  public long getStartTimeMs() {
+    return startTimeMs;
+  }
+
+  public long getDurationMs() {
+    return durationMs;
+  }
+
+  public int getImageWidth() {
+    return imageWidth;
+  }
+
+  public int getImageHeight() {
+    return imageHeight;
+  }
+}

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaSource.java

@@ -1056,9 +1056,9 @@ private static long getAvailableStartTimeInManifestUs(
     for (int i = 0; i < period.adaptationSets.size(); i++) {
       AdaptationSet adaptationSet = period.adaptationSets.get(i);
       List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude text and image adaptation sets from duration calculations, if we have at least one audio
       // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      if ((haveAudioVideoAdaptationSets && (adaptationSet.type == C.TRACK_TYPE_TEXT || adaptationSet.type == C.TRACK_TYPE_IMAGE))
           || representations.isEmpty()) {
         continue;
       }
@@ -1088,9 +1088,9 @@ private static long getAvailableEndTimeInManifestUs(
     for (int i = 0; i < period.adaptationSets.size(); i++) {
       AdaptationSet adaptationSet = period.adaptationSets.get(i);
       List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude text and image adaptation sets from duration calculations, if we have at least one audio
       // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      if ((haveAudioVideoAdaptationSets && (adaptationSet.type == C.TRACK_TYPE_TEXT || adaptationSet.type == C.TRACK_TYPE_IMAGE))
           || representations.isEmpty()) {
         continue;
       }

library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifest.java

@@ -15,12 +15,20 @@
  */
 package com.google.android.exoplayer2.source.dash.manifest;
 
+import static com.google.android.exoplayer2.util.Util.castNonNull;
+
 import android.net.Uri;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.offline.FilterableManifest;
 import com.google.android.exoplayer2.offline.StreamKey;
+import com.google.android.exoplayer2.source.dash.BaseUrlExclusionList;
+import com.google.android.exoplayer2.source.dash.DashSegmentIndex;
+import com.google.android.exoplayer2.source.dash.DashUtil;
+import com.google.android.exoplayer2.thumbnail.ThumbnailDescription;
+import com.google.android.exoplayer2.upstream.DataSpec;
 import com.google.android.exoplayer2.util.Util;
+import com.google.common.base.Ascii;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
@@ -136,6 +144,80 @@ public final long getPeriodDurationUs(int index) {
     return Util.msToUs(getPeriodDurationMs(index));
   }
 
+  /**
+   * Returns a List of ThumbnailDescription for a given periodPosition,
+   * or null if no AdaptionSet of type C.TRACK_TYPE_IMAGE is available.
+   * @param periodPositionMs the period position to get ThumbnailDescription for, e.g. current player position.
+   * @return List of ThumbnailDescription from all Representations, or null if Thumbnails are not available in the DashManifest.
+   */
+  @Nullable
+  public List<ThumbnailDescription> getThumbnailDescriptions(long periodPositionMs) {
+    ArrayList<ThumbnailDescription> thumbnailDescriptions = new ArrayList<>();
+
+    long periodPositionUs = Util.msToUs(periodPositionMs);
+    BaseUrlExclusionList baseUrlExclusionList = new BaseUrlExclusionList();
+
+    boolean isTrackTypeImageAvailable = false;
+    for (int i = 0; i < getPeriodCount(); i++) {
+      Period period = getPeriod(i);
+      long periodStartUs = Util.msToUs(period.startMs);
+      long periodDurationUs = getPeriodDurationUs(i);
+
+      List<AdaptationSet> adaptationSets = period.adaptationSets;
+      for (int j = 0; j < adaptationSets.size(); j++) {
+        AdaptationSet adaptationSet = adaptationSets.get(j);
+        if (adaptationSet.type != C.TRACK_TYPE_IMAGE) {
+          continue;
+        }
+        isTrackTypeImageAvailable = true;
+
+        // thumbnails found
+        List<Representation> representations = adaptationSet.representations;
+        for (int k = 0; k < representations.size(); k++) {
+
+          Representation representation = representations.get(k);
+          DashSegmentIndex index = representation.getIndex();
+          if (index == null) {
+            continue;
+          }
+
+          String id = representation.format.id;
+          if (id == null) {
+            continue;
+          }
+          int bitrate = representation.format.bitrate;
+          int imageWidth = representation.format.width;
+          int imageHeight = representation.format.height;
+          // get size XxY, e.g. 10x20, where 10 is column count and 20 is row count
+          int tileCountHorizontal = representation.format.tileCountHorizontal;
+          int tileCountVertical = representation.format.tileCountVertical;
+
+          long now = Util.getNowUnixTimeMs(C.TIME_UNSET);
+          String baseUrl = castNonNull(baseUrlExclusionList.selectBaseUrl(representation.baseUrls)).url;
+
+          // calculate the correct positionUs, which is FirstAvailableSegment.time + playerPosition, use that to get the correct segment
+          long firstSegmentNum = index.getFirstAvailableSegmentNum(periodDurationUs, Util.msToUs(now));
+          long firstStartTimeUs = index.getTimeUs(firstSegmentNum);
+          long positionUs = firstStartTimeUs + periodPositionUs;
+          long segmentNumber = index.getSegmentNum(positionUs, periodDurationUs);
+
+          long segmentStartTimeUs = periodStartUs + index.getTimeUs(segmentNumber);
+          long segmentDurationUs = index.getDurationUs(segmentNumber, periodDurationUs);
+
+          RangedUri rangedUri = index.getSegmentUrl(segmentNumber);
+          DataSpec dataSpec = DashUtil.buildDataSpec(representation, baseUrl, rangedUri, /* flags= */ 0);
+          Uri uri = dataSpec.uri;
+          ThumbnailDescription thumbnailDescription = new ThumbnailDescription(id, uri, bitrate, tileCountHorizontal, tileCountVertical, Util.usToMs(segmentStartTimeUs - (dynamic ? firstStartTimeUs : 0)), Util.usToMs(segmentDurationUs), imageWidth, imageHeight);
+          thumbnailDescriptions.add(thumbnailDescription);
+        }
+      }
+    }
+    if (isTrackTypeImageAvailable) {
+      return thumbnailDescriptions;
+    }
+    return null;
+  }
+
   @Override
   public final DashManifest copy(List<StreamKey> streamKeys) {
     LinkedList<StreamKey> keys = new LinkedList<>(streamKeys);