Merged PR 16175: [5.0] Throw on invalid payload length in WebSockets

**Description:**
Avoid integer overflow to prevent infinite loop in reading from WebSocket. (also complies better with WebSocket RFC)
MSRC 65273 - Prevents DoS attack by sending frames with invalid payload length.

**Risk:**
Low

**Impacted assemblies:**
System.Net.WebSockets.dll
System.Net.WebSockets.WebSocketProtocol.dll

src/libraries/Common/src/System/Net/WebSockets/ManagedWebSocket.cs

@@ -1020,6 +1020,14 @@ private async ValueTask CloseWithReceiveErrorAndThrowAsync(
                 return SR.net_Websockets_ReservedBitsSet;
             }
 
+            if (header.PayloadLength < 0)
+            {
+                // as per RFC, if payload length is a 64-bit integer, the most significant bit MUST be 0
+                // frame-payload-length-63 = %x0000000000000000-7FFFFFFFFFFFFFFF; 64 bits in length
+                resultHeader = default;
+                return SR.net_Websockets_InvalidPayloadLength;
+            }
+
             if (masked)
             {
                 if (!_isServer)

src/libraries/Common/tests/System/Net/WebSockets/WebSocketCreateTest.cs

@@ -147,6 +147,36 @@ public async Task ReceiveAsync_InvalidFrameHeader_AbortsAndThrowsException(byte
             }
         }
 
+        [Theory]
+        [InlineData(new byte[] { 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }, false)] // max allowed value
+        [InlineData(new byte[] { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, true)]
+        public async Task ReceiveAsync_InvalidPayloadLength_AbortsAndThrowsException(byte[] lenBytes, bool shouldFail)
+        {
+            var frame = new byte[11];
+            frame[0] = 0b1_000_0010; // FIN, RSV, OPCODE
+            frame[1] = 0b0_1111111; // MASK, PAYLOAD_LEN
+            Array.Copy(lenBytes, 0, frame, 2, lenBytes.Length); // EXTENDED_PAYLOAD_LEN
+            frame[10] = (byte)'a';
+
+            using var stream = new MemoryStream(frame, writable: true);
+            using WebSocket websocket = CreateFromStream(stream, false, null, Timeout.InfiniteTimeSpan);
+
+            var buffer = new byte[1];
+            Task<WebSocketReceiveResult> t = websocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
+            if (shouldFail)
+            {
+                var exc = await Assert.ThrowsAsync<WebSocketException>(() => t);
+                Assert.Equal(WebSocketState.Aborted, websocket.State);
+            }
+            else
+            {
+                WebSocketReceiveResult result = await t;
+                Assert.False(result.EndOfMessage);
+                Assert.Equal(1, result.Count);
+                Assert.Equal('a', (char)buffer[0]);
+            }
+        }
+
         [Fact]
         [PlatformSpecific(~TestPlatforms.Browser)] // System.Net.Sockets is not supported on this platform.
         [ActiveIssue("https://github.com/dotnet/runtime/issues/34690", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)]

src/libraries/System.Net.WebSockets.WebSocketProtocol/src/Resources/Strings.resx

@@ -111,4 +111,7 @@
   <data name="net_Websockets_UnknownOpcode" xml:space="preserve">
     <value>The WebSocket received a frame with an unknown opcode: '0x{0}'.</value>
   </data>
+  <data name="net_Websockets_InvalidPayloadLength" xml:space="preserve">
+    <value>The WebSocket received a frame with an invalid payload length.</value>
+  </data>
 </root>

src/libraries/System.Net.WebSockets/src/Resources/Strings.resx

@@ -138,4 +138,7 @@
   <data name="NotWriteableStream" xml:space="preserve">
     <value>The base stream is not writeable.</value>
   </data>
+  <data name="net_Websockets_InvalidPayloadLength" xml:space="preserve">
+    <value>The WebSocket received a frame with an invalid payload length.</value>
+  </data>
 </root>