Fix/as switch with different codecs (#4524)

* WiP: Fix codec switch for AS switching

* Put logic for representation switches in dedicated functions inside the BufferController.js and use changeType if possible

* Add missing isSupported if MediaCapabilities API is used with content without a EssentialProperty

* Add unit tests for checking codec support

* Uncomment Media Capabilities API codec check as it seems to fail the CI/CD pipeline

* Remove unwanted files

* WiP: Improve quality selection algorithm

* Add unit tests

* Replace invalid sample stream

* Additional changes to fix the functional tests

* Changes to the Karma execution on Lambdatest

* Remove chrome from lambdatest as it fails with timeout

* Change Lambdatest config

* Run single and smoke tests in same job

* Lambdatest config change

* Lambdatest config change

* Lambdatest config change

.circleci/config.yml

@@ -185,6 +185,18 @@ jobs:
                     configfile: lambdatest-smoke
             - process_test_results
 
+    functional-tests-single-and-smoke:
+        executor: dashjs-executor
+        steps:
+            - functional_steps
+            -   run_test_suite:
+                    streamsfile: single
+                    configfile: lambdatest
+            -   run_test_suite:
+                    streamsfile: smoke
+                    configfile: lambdatest-smoke
+            - process_test_results
+
     functional-tests-full-part-1:
         executor: dashjs-executor
         steps:
@@ -222,14 +234,7 @@ workflows:
                         branches:
                             ignore:
                                 - development # skiping redundant job if already on development
-            -   functional-tests-single:
-                    filters:
-                        branches:
-                            ignore: # as creds are available only for non-forked branches
-                                - /pull\/[0-9]+/
-            -   functional-tests-smoke:
-                    requires:
-                        - functional-tests-single
+            -   functional-tests-single-and-smoke:
                     filters:
                         branches:
                             ignore: # as creds are available only for non-forked branches

contrib/akamai/controlbar/ControlBar.js

@@ -537,6 +537,7 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) {
                 contentFunc = function (element, index) {
                     var result = isNaN(index) ? ' Auto Switch' : Math.floor(element.bitrateInKbit) + ' kbps';
                     result += element && element.width && element.height ? ' (' + element.width + 'x' + element.height + ')' : '';
+                    result += element && element.codecs ? ' (' + element.codecs + ')' : '';
                     return result;
                 };
 

samples/dash-if-reference-player/app/sources.json

@@ -307,8 +307,8 @@
                     "provider": "aws"
                 },
                 {
-                    "name": "LiveSIM Caminandes 02, Gran Dillama (25fps, 25gop, 2sec, multi MOOF/MDAT, 1080p, KID=1236) v2",
-                    "url": "http://refapp.hbbtv.org/livesim2/02_llamav2/manifest.mpd",
+                    "name": "Main (3 video bitrates, 2 audio languages,3 subtitle languages) + Advertisement, Multiperiod",
+                    "url": "https://refapp.hbbtv.org/videos/multiperiod_v8.php?drm=0&advert=1&emsg=0&video=v1,v2,v3&audiolang=eng,fin&sublang=eng,fin,swe&mup=2&spd=8",
                     "provider": "hbbtv"
                 },
                 {

src/core/Settings.js

@@ -416,7 +416,7 @@ import Events from './events/Events.js';
  * That buffered range is likely to have been enqueued for playback. Pruning it causes a flush and reenqueue in WPE and WebKitGTK based browsers. This stresses the video decoder and can cause stuttering on embedded platforms.
  * @property {boolean} [useChangeTypeForTrackSwitch=true]
  * If this flag is set to true then dash.js will use the MSE v.2 API call "changeType()" before switching to a different track.
- * Note that some platforms might not implement the changeType functio. dash.js is checking for the availability before trying to call it.
+ * Note that some platforms might not implement the changeType function. dash.js is checking for the availability before trying to call it.
  * @property {boolean} [mediaSourceDurationInfinity=true]
  * If this flag is set to true then dash.js will allow `Infinity` to be set as the MediaSource duration otherwise the duration will be set to `Math.pow(2,32)` instead of `Infinity` to allow appending segments indefinitely.
  * Some platforms such as WebOS 4.x have issues with seeking when duration is set to `Infinity`, setting this flag to false resolve this.
@@ -860,7 +860,7 @@ import Events from './events/Events.js';
  * This value is used to specify the desired CMCD parameters. Parameters not included in this list are not reported.
  * @property {Array.<string>} [includeInRequests]
  * Specifies which HTTP GET requests shall carry parameters.
- * 
+ *
  * If not specified this value defaults to ['segment'].
  */
 
@@ -1221,7 +1221,7 @@ function Settings() {
                         }
                     },
                     droppedFramesRule: {
-                        active: true,
+                        active: false,
                         parameters: {
                             minimumSampleSize: 375,
                             droppedFramesPercentageThreshold: 0.15

src/dash/models/DashManifestModel.js

@@ -664,7 +664,20 @@ function DashManifestModel() {
                     voRepresentation.scanType = realRepresentation.scanType;
                 }
                 if (realRepresentation.hasOwnProperty(DashConstants.FRAMERATE)) {
-                    voRepresentation.frameRate = realRepresentation[DashConstants.FRAMERATE];
+                    const frameRate = realRepresentation[DashConstants.FRAMERATE];
+                    if (isNaN(frameRate) && frameRate.includes('/')) {
+                        const parts = frameRate.split('/');
+                        if (parts.length === 2) {
+                            const numerator = parseFloat(parts[0]);
+                            const denominator = parseFloat(parts[1]);
+
+                            if (!isNaN(numerator) && !isNaN(denominator) && denominator !== 0) {
+                                voRepresentation.frameRate = numerator / denominator;
+                            }
+                        }
+                    } else {
+                        voRepresentation.frameRate = frameRate
+                    }
                 }
                 if (realRepresentation.hasOwnProperty(DashConstants.QUALITY_RANKING)) {
                     voRepresentation.qualityRanking = realRepresentation[DashConstants.QUALITY_RANKING];
@@ -763,8 +776,18 @@ function DashManifestModel() {
                     }
                 }
 
+                voRepresentation.essentialProperties = getEssentialPropertiesForRepresentation(realRepresentation);
+                voRepresentation.supplementalProperties = getSupplementalPropertiesForRepresentation(realRepresentation);
                 voRepresentation.mseTimeOffset = calcMseTimeOffset(voRepresentation);
                 voRepresentation.path = [voAdaptation.period.index, voAdaptation.index, i];
+
+                if (!isNaN(voRepresentation.width) && !isNaN(voRepresentation.height) && !isNaN(voRepresentation.frameRate)) {
+                    voRepresentation.pixelsPerSecond = Math.max(1, voRepresentation.width * voRepresentation.height * voRepresentation.frameRate)
+                    if (!isNaN(voRepresentation.bandwidth)) {
+                        voRepresentation.bitsPerPixel = voRepresentation.bandwidth / voRepresentation.pixelsPerSecond
+                    }
+                }
+
                 voRepresentations.push(voRepresentation);
             }
         }

src/dash/vo/Representation.js

@@ -44,8 +44,10 @@ class Representation {
         this.availabilityTimeOffset = 0;
         this.bandwidth = NaN;
         this.bitrateInKbit = NaN;
+        this.bitsPerPixel = NaN;
         this.codecPrivateData = null;
         this.codecs = null;
+        this.essentialProperties = [];
         this.fragmentDuration = null;
         this.frameRate = null;
         this.height = NaN;
@@ -57,13 +59,15 @@ class Representation {
         this.mediaInfo = null;
         this.mimeType = null;
         this.mseTimeOffset = NaN;
+        this.pixelsPerSecond = NaN;
         this.presentationTimeOffset = 0;
         this.qualityRanking = NaN;
         this.range = null;
         this.scanType = null;
         this.segments = null;
         this.segmentDuration = NaN;
         this.segmentInfoType = null;
+        this.supplementalProperties = [];
         this.startNumber = 1;
         this.timescale = 1;
         this.width = NaN;

src/streaming/MediaPlayer.js

@@ -2653,7 +2653,7 @@ function MediaPlayer() {
         if (value.lang) {
             output.lang = value.lang;
         }
-        if (value.index) {
+        if (!isNaN(value.index)) {
             output.index = value.index;
         }
         if (value.viewpoint) {

src/streaming/StreamProcessor.js

@@ -635,6 +635,7 @@ function StreamProcessor(config) {
     /**
      * Called once the StreamProcessor is initialized and when the track is switched. We only have one StreamProcessor per media type. So we need to adjust the mediaInfo once we switch/select a track.
      * @param {object} newMediaInfo
+     * @param targetRepresentation
      */
     function selectMediaInfo(newMediaInfo, targetRepresentation = null) {
         return new Promise((resolve) => {
@@ -734,6 +735,7 @@ function StreamProcessor(config) {
     }
 
     function _handleDifferentSwitchTypes(e, newRepresentation) {
+
         // If the switch should occur immediately we need to replace existing stuff in the buffer
         if (e.reason && e.reason.forceReplace) {
             _prepareForForceReplacementQualitySwitch(newRepresentation);
@@ -761,7 +763,7 @@ function StreamProcessor(config) {
     function _prepareForForceReplacementQualitySwitch(voRepresentation) {
 
         // Abort the current request to avoid inconsistencies and in case a rule such as AbandonRequestRule has forced a quality switch. A quality switch can also be triggered manually by the application.
-        // If we update the buffer values now, or initialize a request to the new init segment, the currently downloading media segment might "work" with wrong values.
+        // If we update the buffer values now, or initialize a request to the new init segment, the currently downloading media segment might use wrong values.
         // Everything that is already in the buffer queue is ok and will be handled by the corresponding function below depending on the switch mode.
         fragmentModel.abortRequests();
 
@@ -772,6 +774,7 @@ function StreamProcessor(config) {
         }, { mediaType: type, streamId: streamInfo.id });
 
         scheduleController.setCheckPlaybackQuality(false);
+
         // Abort appending segments to the buffer. Also adjust the appendWindow as we might have been in the progress of prebuffering stuff.
         bufferController.prepareForForceReplacementQualitySwitch(voRepresentation)
             .then(() => {
@@ -786,6 +789,20 @@ function StreamProcessor(config) {
             });
     }
 
+    function _prepareForAbandonQualitySwitch(voRepresentation) {
+        bufferController.prepareForAbandonQualitySwitch(voRepresentation)
+            .then(() => {
+                fragmentModel.abortRequests();
+                shouldRepeatRequest = true;
+                scheduleController.setCheckPlaybackQuality(false);
+                scheduleController.startScheduleTimer();
+                qualityChangeInProgress = false;
+            })
+            .catch(() => {
+                qualityChangeInProgress = false;
+            })
+    }
+
     function _prepareForFastQualitySwitch(voRepresentation) {
         // if we switch up in quality and need to replace existing parts in the buffer we need to adjust the buffer target
         const time = playbackController.getTime();
@@ -802,7 +819,7 @@ function StreamProcessor(config) {
 
             // The new quality is higher than the one we originally requested
             if (request.bandwidth < voRepresentation.bandwidth && bufferLevel >= safeBufferLevel && abandonmentState === MetricsConstants.ALLOW_LOAD) {
-                bufferController.updateBufferTimestampOffset(voRepresentation)
+                bufferController.prepareForFastQualitySwitch(voRepresentation)
                     .then(() => {
                         // Abort the current request to avoid inconsistencies. A quality switch can also be triggered manually by the application.
                         // If we update the buffer values now, or initialize a request to the new init segment, the currently downloading media segment might "work" with wrong values.
@@ -819,7 +836,7 @@ function StreamProcessor(config) {
                     })
             }
 
-            // If we have buffered a higher quality we do not replace anything. We might cancel the current request due to abandon request rule
+            // If we have buffered a higher quality we do not replace anything.
             else {
                 _prepareForDefaultQualitySwitch(voRepresentation);
             }
@@ -838,7 +855,7 @@ function StreamProcessor(config) {
         }
 
 
-        bufferController.updateBufferTimestampOffset(voRepresentation)
+        bufferController.prepareForDefaultQualitySwitch(voRepresentation)
             .then(() => {
                 scheduleController.setCheckPlaybackQuality(false);
                 if (currentMediaInfo.segmentAlignment || currentMediaInfo.subSegmentAlignment) {
@@ -855,19 +872,6 @@ function StreamProcessor(config) {
             })
     }
 
-    function _prepareForAbandonQualitySwitch(voRepresentation) {
-        bufferController.updateBufferTimestampOffset(voRepresentation)
-            .then(() => {
-                fragmentModel.abortRequests();
-                shouldRepeatRequest = true;
-                scheduleController.setCheckPlaybackQuality(false);
-                scheduleController.startScheduleTimer();
-                qualityChangeInProgress = false;
-            })
-            .catch(() => {
-                qualityChangeInProgress = false;
-            })
-    }
 
     /**
      * We have canceled the download of a fragment and need to adjust the buffer time or reload an init segment

src/streaming/controllers/AbrController.js

@@ -225,10 +225,11 @@ function AbrController() {
         }
 
         // If bitrate should be as small as possible return the Representation with the lowest bitrate
+        const smallestRepresentation = possibleVoRepresentations.reduce((a, b) => {
+            return a.bandwidth < b.bandwidth ? a : b;
+        })
         if (bitrate <= 0) {
-            return possibleVoRepresentations.sort((a, b) => {
-                return a.bandwidth - b.bandwidth;
-            })[0]
+            return smallestRepresentation
         }
 
         // Get all Representations that have lower or equal bitrate than our target bitrate
@@ -237,10 +238,13 @@ function AbrController() {
         });
 
         if (!targetRepresentations || targetRepresentations.length === 0) {
-            return possibleVoRepresentations[0];
+            return smallestRepresentation
         }
 
-        return targetRepresentations[targetRepresentations.length - 1];
+        return targetRepresentations.reduce((max, curr) => {
+            return (curr.absoluteIndex > max.absoluteIndex) ? curr : max;
+        })
+
     }
 
     function getRepresentationByAbsoluteIndex(absoluteIndex, mediaInfo, includeCompatibleMediaInfos = true) {
@@ -284,7 +288,7 @@ function AbrController() {
         })
 
         // Now sort by quality (usually simply by bitrate)
-        voRepresentations = _sortByCalculatedQualityRank(voRepresentations);
+        voRepresentations = _sortRepresentationsByQuality(voRepresentations);
 
         // Add an absolute index
         voRepresentations.forEach((rep, index) => {
@@ -443,15 +447,17 @@ function AbrController() {
         }
     }
 
-    /**
-     * Calculate a quality rank based on bandwidth, codec and qualityRanking. Lower value means better quality.
-     * @param voRepresentations
-     * @private
-     */
-    function _sortByCalculatedQualityRank(voRepresentations) {
+    function _sortRepresentationsByQuality(voRepresentations) {
+        if (_shouldSortByQualityRankingAttribute(voRepresentations)) {
+            voRepresentations = _sortByQualityRankingAttribute(voRepresentations)
+        } else {
+            voRepresentations = _sortByDefaultParameters(voRepresentations)
+        }
 
-        // All Representations must have a qualityRanking otherwise we ignore it
-        // QualityRanking only applies to Representations within one AS. If we merged multiple AS based on the adaptation-set-switching-2016 supplemental property we can not apply this logic
+        return voRepresentations
+    }
+
+    function _shouldSortByQualityRankingAttribute(voRepresentations) {
         let firstMediaInfo = null;
         const filteredRepresentations = voRepresentations.filter((rep) => {
             if (!firstMediaInfo) {
@@ -460,19 +466,46 @@ function AbrController() {
             return !isNaN(rep.qualityRanking) && adapter.areMediaInfosEqual(firstMediaInfo, rep.mediaInfo);
         })
 
-        if (filteredRepresentations.length === voRepresentations.length) {
-            voRepresentations.sort((a, b) => {
-                return b.qualityRanking - a.qualityRanking;
-            })
-        } else {
-            voRepresentations.sort((a, b) => {
-                return a.bandwidth - b.bandwidth;
-            })
-        }
+        return filteredRepresentations.length === voRepresentations.length
+    }
+
+    function _sortByQualityRankingAttribute(voRepresentations) {
+        voRepresentations.sort((a, b) => {
+            return b.qualityRanking - a.qualityRanking;
+        })
+
+        return voRepresentations
+    }
+
+
+    function _sortByDefaultParameters(voRepresentations) {
+        voRepresentations.sort((a, b) => {
+
+            // In case both Representations are coming from the same MediaInfo then choose the one with the highest resolution and highest bitrate
+            if (adapter.areMediaInfosEqual(a.mediaInfo, b.mediaInfo)) {
+                if (!isNaN(a.pixelsPerSecond) && !isNaN(b.pixelsPerSecond) && a.pixelsPerSecond !== b.pixelsPerSecond) {
+                    return a.pixelsPerSecond - b.pixelsPerSecond
+                } else {
+                    return a.bandwidth - b.bandwidth
+                }
+            }
+
+            // In case the Representations are coming from different MediaInfos they might have different codecs. The bandwidth is not a good indicator, use bits per pixel instead
+            else {
+                if (!isNaN(a.pixelsPerSecond) && !isNaN(b.pixelsPerSecond) && a.pixelsPerSecond !== b.pixelsPerSecond) {
+                    return a.pixelsPerSecond - b.pixelsPerSecond
+                } else if (!isNaN(a.bitsPerPixel) && !isNaN(b.bitsPerPixel)) {
+                    return b.bitsPerPixel - a.bitsPerPixel
+                } else {
+                    return a.bandwidth - b.bandwidth
+                }
+            }
+        })
 
         return voRepresentations
     }
 
+
     /**
      * While fragment loading is in progress we check if we might need to abort the request
      * @param {object} e
@@ -496,6 +529,7 @@ function AbrController() {
             streamProcessor,
             currentRequest: e.request,
             throughputController,
+            adapter,
             videoModel
         });
         const switchRequest = abrRulesCollection.shouldAbandonFragment(rulesContext);
@@ -618,6 +652,7 @@ function AbrController() {
                 switchRequestHistory,
                 droppedFramesHistory,
                 streamProcessor,
+                adapter,
                 videoModel
             });
             const switchRequest = abrRulesCollection.getBestPossibleSwitchRequest(rulesContext);