Refactor text tracks (#3641)

* Add support for fragmented text tracks without init segments (for example fragmented vtt): - use segment info/template from manifest to determine if track is fragmented or not (do not rely on codecs property) - use TextSourceBuffer also for "vtt" codec - do not request for init segments if none * clean code * Refactor text tracks management: - keep only type "text" for all text tracks - rely on TextTrackInfo.isFragmented and isEmbedded to get type of text tracks - unify text tracks metadata returned by MediaPlayer.getTracksFor() and in TEXT_TRACKS_ADDED event (TextTrackInfo extends MediaInfo) - rely on segments templates to know if a text track is fragmented or not * Fix unit tests * fix functional tests * Update offline module * Do not create StreamProcessor for embedded text tracks * Init MediaInfo.isEmbdedded to false by default * Do not create StreamProcessor for embedded text tracks * minor changes * Fix support for fragented vtt * Fix support for fragmented vtt
Dash-Industry-Forum · Jun 11, 2021 · 77b96ef · 77b96ef
1 parent cc348de
commit 77b96ef
Show file tree

Hide file tree

Showing 46 changed files with 305 additions and 340 deletions.
diff --git a/index.d.ts b/index.d.ts
@@ -1328,17 +1328,14 @@ declare namespace dashjs {
         executedRequests: any[];
     }
 
-    export class TextTrackInfo {
+    export class TextTrackInfo extends MediaInfo {
         captionData: CaptionData[] | null;
         label: string | null;
-        lang: string | null;
-        index: number;
-        isTTML: boolean;
         defaultTrack: boolean;
         kind: string;
-        roles: string[] | null;
         isFragmented: boolean;
         isEmbedded: boolean;
+        isTTML: boolean;
     }
 
     export interface CaptionData {

diff --git a/samples/captioning/multi-track-captions.html b/samples/captioning/multi-track-captions.html
@@ -38,7 +38,7 @@
             player.initialize(videoElement);
             player.attachTTMLRenderingDiv(TTMLRenderingDiv);
             player.on('currentTrackChanged', function(e) {
-                if (e.newMediaInfo.type === 'fragmentedText') {
+                if (e.newMediaInfo.type === 'text') {
                     setTimeout(showDomStorage, 0);
                 }
             });
@@ -49,7 +49,7 @@
         }
 
         function showDomStorage() {
-            document.getElementById("domStorage").innerHTML = localStorage.getItem('dashjs_fragmentedText_settings');
+            document.getElementById("domStorage").innerHTML = localStorage.getItem('dashjs_text_settings');
         }
 
         function play() {
@@ -75,7 +75,7 @@
                     lang = values[0];
                     role = values[1];
                 }
-                player.setInitialMediaSettingsFor('fragmentedText', {
+                player.setInitialMediaSettingsFor('text', {
                     lang: lang,
                     role: role
                 });
@@ -92,7 +92,7 @@
         }
 
         function clearDomStorage() {
-            localStorage.removeItem('dashjs_fragmentedText_settings');
+            localStorage.removeItem('dashjs_text_settings');
             showDomStorage();
         }
 

diff --git a/samples/chromecast/receiver/js/DashPlayerService.js b/samples/chromecast/receiver/js/DashPlayerService.js
@@ -138,7 +138,7 @@ angular.module('DashCastReceiverApp.services', [])
         let allTracks = [];
         if (player && initialized) {
           let audioTracks = player.getTracksFor('audio');
-          let textTracks = player.getTracksFor('fragmentedText');
+          let textTracks = player.getTracksFor('text');
           audioTracks.forEach(function (track) {
               allTracks.push(_convertTrack(track, cast.receiver.media.TrackType.AUDIO));
           });
@@ -156,7 +156,7 @@ angular.module('DashCastReceiverApp.services', [])
           if (currentAudioTrack) {
             trackIds.push(currentAudioTrack.index);
           }
-          let currentTextTrack = player.getCurrentTrackFor('fragmentedText');
+          let currentTextTrack = player.getCurrentTrackFor('text');
           if (currentTextTrack) {
             trackIds.push(currentTextTrack.index);
           }
@@ -192,7 +192,7 @@ angular.module('DashCastReceiverApp.services', [])
               if (audioTrack) {
                 player.setCurrentTrack(audioTrack);
               } else {
-                let textTracks = player.getTracksFor('fragmentedText');
+                let textTracks = player.getTracksFor('text');
                 textTrack = textTracks.find(function (track) { return track.index === activeTrackIds[i]; });
                 if (textTrack) {
                   player.enableText(true);

diff --git a/samples/dash-if-reference-player/app/main.js b/samples/dash-if-reference-player/app/main.js
@@ -341,6 +341,7 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
     reqConfig.send();
 
     $scope.player.on(dashjs.MediaPlayer.events.ERROR, function (e) { /* jshint ignore:line */
+        console.log(e);
         if (!e.event) {
             $scope.$apply(function () {
                 $scope.error = e.error.message;
@@ -865,12 +866,12 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         }
         if ($scope.initialSettings.text) {
             if ($scope.initialSettings.textRole) {
-                $scope.player.setInitialMediaSettingsFor('fragmentedText', {
+                $scope.player.setInitialMediaSettingsFor('text', {
                     role: $scope.initialSettings.textRole,
                     lang: $scope.initialSettings.text
                 });
             } else {
-                $scope.player.setInitialMediaSettingsFor('fragmentedText', {
+                $scope.player.setInitialMediaSettingsFor('text', {
                     lang: $scope.initialSettings.text
                 });
             }

diff --git a/src/dash/DashAdapter.js b/src/dash/DashAdapter.js
@@ -207,18 +207,7 @@ function DashAdapter() {
         return (sameId && sameCodec && sameViewpoint && sameLang && sameRoles && sameAccessibility && sameAudioChannelConfiguration);
     }
 
-    /**
-     * Returns all the mediaInfos for a given mediaType and the corresponding streamInfo.
-     * @param {object} streamInfo
-     * @param {MediaType} type
-     * @param {object} externalManifest Set to null or undefined if no external manifest is to be used
-     * @returns {Array} mediaArr
-     * @memberOf module:DashAdapter
-     * @instance
-     */
-    function getAllMediaInfoForType(streamInfo, type, externalManifest) {
-        let voLocalPeriods = voPeriods;
-        let manifest = externalManifest;
+    function _getAllMediaInfo(manifest, period, streamInfo, adaptations, type, embeddedText) {
         let mediaArr = [];
         let data,
             media,
@@ -227,31 +216,18 @@ function DashAdapter() {
             j,
             ln;
 
-        if (manifest) {
-            checkConfig();
-
-            voLocalPeriods = getRegularPeriods(manifest);
-        } else {
-            if (voPeriods.length > 0) {
-                manifest = voPeriods[0].mpd.manifest;
-            } else {
-                return mediaArr;
-            }
+        if (!adaptations || adaptations.length === 0) {
+            return [];
         }
 
-        const selectedVoPeriod = getPeriodForStreamInfo(streamInfo, voLocalPeriods);
-        const adaptationsForType = dashManifestModel.getAdaptationsForType(manifest, streamInfo ? streamInfo.index : null, type !== constants.EMBEDDED_TEXT ? type : constants.VIDEO);
-
-        if (!adaptationsForType || adaptationsForType.length === 0) return mediaArr;
-
-        const voAdaptations = dashManifestModel.getAdaptationsForPeriod(selectedVoPeriod);
+        const voAdaptations = dashManifestModel.getAdaptationsForPeriod(period);
 
-        for (i = 0, ln = adaptationsForType.length; i < ln; i++) {
-            data = adaptationsForType[i];
+        for (i = 0, ln = adaptations.length; i < ln; i++) {
+            data = adaptations[i];
             idx = dashManifestModel.getIndexForAdaptation(data, manifest, streamInfo.index);
             media = convertAdaptationToMediaInfo(voAdaptations[idx]);
 
-            if (type === constants.EMBEDDED_TEXT) {
+            if (embeddedText) {
                 let accessibilityLength = media.accessibility.length;
                 for (j = 0; j < accessibilityLength; j++) {
                     if (!media) {
@@ -298,6 +274,46 @@ function DashAdapter() {
         return mediaArr;
     }
 
+    /**
+     * Returns all the mediaInfos for a given mediaType and the corresponding streamInfo.
+     * @param {object} streamInfo
+     * @param {MediaType} type
+     * @param {object} externalManifest Set to null or undefined if no external manifest is to be used
+     * @returns {Array} mediaArr
+     * @memberOf module:DashAdapter
+     * @instance
+     */
+    function getAllMediaInfoForType(streamInfo, type, externalManifest) {
+        let voLocalPeriods = voPeriods;
+        let manifest = externalManifest;
+        let mediaArr = [];
+
+        if (manifest) {
+            checkConfig();
+
+            voLocalPeriods = getRegularPeriods(manifest);
+        } else {
+            if (voPeriods.length > 0) {
+                manifest = voPeriods[0].mpd.manifest;
+            } else {
+                return mediaArr;
+            }
+        }
+
+        const selectedVoPeriod = getPeriodForStreamInfo(streamInfo, voLocalPeriods);
+        let adaptationsForType = dashManifestModel.getAdaptationsForType(manifest, streamInfo ? streamInfo.index : null, type);
+
+        mediaArr = _getAllMediaInfo(manifest, selectedVoPeriod, streamInfo, adaptationsForType, type);
+
+        // Search for embedded text in video track
+        if (type === constants.TEXT) {
+            adaptationsForType = dashManifestModel.getAdaptationsForType(manifest, streamInfo ? streamInfo.index : null, constants.VIDEO);
+            mediaArr = mediaArr.concat(_getAllMediaInfo(manifest, selectedVoPeriod, streamInfo, adaptationsForType, type, true));
+        }
+
+        return mediaArr;
+    }
+
     /**
      * Update the internal voPeriods array with the information from the new manifest
      * @param {object} newManifest
@@ -507,14 +523,14 @@ function DashAdapter() {
 
     /**
      * Check if the given type is a text track
-     * @param {String} type
+     * @param {object} adaptation
      * @returns {boolean}
      * @memberOf module:DashAdapter
      * @instance
      * @ignore
      */
-    function getIsTextTrack(type) {
-        return dashManifestModel.getIsTextTrack(type);
+    function getIsTextTrack(adaptation) {
+        return dashManifestModel.getIsText(adaptation);
     }
 
     /**
@@ -1014,18 +1030,20 @@ function DashAdapter() {
             });
         }
 
-        mediaInfo.isText = dashManifestModel.getIsTextTrack(mediaInfo.mimeType);
+        mediaInfo.isText = dashManifestModel.getIsText(realAdaptation);
         mediaInfo.supplementalProperties = dashManifestModel.getSupplementalProperties(realAdaptation);
 
+        mediaInfo.isFragmented = dashManifestModel.getIsFragmented(realAdaptation);
+        mediaInfo.isEmbedded = false;
+
         return mediaInfo;
     }
 
     function convertVideoInfoToEmbeddedTextInfo(mediaInfo, channel, lang) {
         mediaInfo.id = channel; // CC1, CC2, CC3, or CC4
         mediaInfo.index = 100 + parseInt(channel.substring(2, 3));
-        mediaInfo.type = constants.EMBEDDED_TEXT;
+        mediaInfo.type = constants.TEXT;
         mediaInfo.codec = 'cea-608-in-SEI';
-        mediaInfo.isText = true;
         mediaInfo.isEmbedded = true;
         mediaInfo.lang = lang;
         mediaInfo.roles = ['caption'];

diff --git a/src/dash/controllers/RepresentationController.js b/src/dash/controllers/RepresentationController.js
@@ -98,7 +98,7 @@ function RepresentationController(config) {
         resetInitialSettings();
     }
 
-    function updateData(newRealAdaptation, availableRepresentations, type, quality) {
+    function updateData(newRealAdaptation, availableRepresentations, type, isFragmented, quality) {
         checkConfig();
 
         updating = true;
@@ -108,7 +108,7 @@ function RepresentationController(config) {
         currentVoRepresentation = getRepresentationForQuality(quality);
         realAdaptation = newRealAdaptation;
 
-        if (type !== Constants.VIDEO && type !== Constants.AUDIO && type !== Constants.FRAGMENTED_TEXT) {
+        if (type !== Constants.VIDEO && type !== Constants.AUDIO && (type !== Constants.TEXT || !isFragmented)) {
             endDataUpdate();
             return Promise.resolve();
         }