JsonDocument Working PR

src/System.Text.Json/System.Text.Json.sln

@@ -7,6 +7,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.Tests", "t
 		{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4} = {79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4}
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json.Performance.Tests", "tests\Performance\System.Text.Json.Performance.Tests.csproj", "{FB3EA273-567D-414F-B36D-3698BE8D198B}"
+	ProjectSection(ProjectDependencies) = postProject
+		{DF73E985-E143-4BF5-9FA4-E199E7D36235} = {DF73E985-E143-4BF5-9FA4-E199E7D36235}
+	EndProjectSection
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Text.Json", "src\System.Text.Json.csproj", "{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4}"
 	ProjectSection(ProjectDependencies) = postProject
 		{6371299B-8F39-4A0A-A9CD-70F80FF205F6} = {6371299B-8F39-4A0A-A9CD-70F80FF205F6}
@@ -30,6 +35,10 @@ Global
 		{5F553243-042C-45C0-8E49-C739131E11C3}.Debug|Any CPU.Build.0 = netcoreapp-Debug|Any CPU
 		{5F553243-042C-45C0-8E49-C739131E11C3}.Release|Any CPU.ActiveCfg = netcoreapp-Release|Any CPU
 		{5F553243-042C-45C0-8E49-C739131E11C3}.Release|Any CPU.Build.0 = netcoreapp-Release|Any CPU
+		{FB3EA273-567D-414F-B36D-3698BE8D198B}.Debug|Any CPU.ActiveCfg = netcoreapp-Debug|Any CPU
+		{FB3EA273-567D-414F-B36D-3698BE8D198B}.Debug|Any CPU.Build.0 = netcoreapp-Debug|Any CPU
+		{FB3EA273-567D-414F-B36D-3698BE8D198B}.Release|Any CPU.ActiveCfg = netcoreapp-Release|Any CPU
+		{FB3EA273-567D-414F-B36D-3698BE8D198B}.Release|Any CPU.Build.0 = netcoreapp-Release|Any CPU
 		{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4}.Debug|Any CPU.ActiveCfg = netcoreapp-Windows_NT-Debug|Any CPU
 		{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4}.Debug|Any CPU.Build.0 = netcoreapp-Windows_NT-Debug|Any CPU
 		{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4}.Release|Any CPU.ActiveCfg = netcoreapp-Windows_NT-Release|Any CPU
@@ -44,6 +53,7 @@ Global
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{5F553243-042C-45C0-8E49-C739131E11C3} = {1A2F9F4A-A032-433E-B914-ADD5992BB178}
+		{FB3EA273-567D-414F-B36D-3698BE8D198B} = {1A2F9F4A-A032-433E-B914-ADD5992BB178}
 		{79E7EE4E-E8DF-4D67-B103-6930DAAF6EF4} = {E107E9C1-E893-4E87-987E-04EF0DCEAEFD}
 		{6371299B-8F39-4A0A-A9CD-70F80FF205F6} = {2E666815-2EDB-464B-9DF6-380BF4789AD4}
 	EndGlobalSection

src/System.Text.Json/ref/System.Text.Json.cs

@@ -13,6 +13,49 @@ public enum JsonCommentHandling : byte
         Disallow = (byte)0,
         Skip = (byte)2,
     }
+    public sealed partial class JsonDocument : System.IDisposable
+    {
+        internal JsonDocument() { }
+        public System.Text.Json.JsonElement RootElement { get { throw null; } }
+        public void Dispose() { }
+        public static System.Text.Json.JsonDocument Parse(System.ReadOnlyMemory<byte> utf8Json, System.Text.Json.JsonReaderOptions readerOptions) { throw null; }
+    }
+    public partial struct JsonElement
+    {
+        private object _dummy;
+        public System.Text.Json.JsonElement this[int index] { get { throw null; } }
+        public System.Text.Json.JsonElement this[System.ReadOnlySpan<byte> utf8PropertyName] { get { throw null; } }
+        public System.Text.Json.JsonElement this[System.ReadOnlySpan<char> propertyName] { get { throw null; } }
+        public System.Text.Json.JsonElement this[string propertyName] { get { throw null; } }
+        public System.Text.Json.JsonTokenType Type { get { throw null; } }
+        public System.Text.Json.JsonElement.ChildEnumerator EnumerateChildren() { throw null; }
+        public int GetArrayLength() { throw null; }
+        public bool GetBoolean() { throw null; }
+        public double GetDouble() { throw null; }
+        public int GetInt32() { throw null; }
+        public long GetInt64() { throw null; }
+        public string GetString() { throw null; }
+        [System.CLSCompliantAttribute(false)]
+        public ulong GetUInt64() { throw null; }
+        public override string ToString() { throw null; }
+        public bool TryCopyRawValue(System.Span<byte> destination, out int bytesWritten) { throw null; }
+        public bool TryGetProperty(System.ReadOnlySpan<byte> utf8PropertyName, out System.Text.Json.JsonElement value) { throw null; }
+        public bool TryGetProperty(System.ReadOnlySpan<char> propertyName, out System.Text.Json.JsonElement value) { throw null; }
+        public bool TryGetProperty(string propertyName, out System.Text.Json.JsonElement value) { throw null; }
+        public bool TryGetRawValue(out System.ReadOnlyMemory<byte> rawValue) { throw null; }
+        public bool TryGetValue(out double value) { throw null; }
+        public bool TryGetValue(out int value) { throw null; }
+        public bool TryGetValue(out long value) { throw null; }
+        [System.CLSCompliantAttribute(false)]
+        public bool TryGetValue(out ulong value) { throw null; }
+        public partial struct ChildEnumerator
+        {
+            private object _dummy;
+            public System.Text.Json.JsonElement Current { get { throw null; } }
+            public System.Text.Json.JsonElement.ChildEnumerator GetEnumerator() { throw null; }
+            public bool MoveNext() { throw null; }
+        }
+    }
     public sealed partial class JsonReaderException : System.Exception
     {
         public JsonReaderException(string message, long lineNumber, long bytePositionInLine) { }

src/System.Text.Json/src/System.Text.Json.csproj

@@ -12,6 +12,13 @@
     <Compile Include="System\Text\Json\ConsumeTokenResult.cs" />
     <Compile Include="System\Text\Json\JsonCommentHandling.cs" />
     <Compile Include="System\Text\Json\JsonConstants.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.CustomDb.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.CustomStack.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.DbRow.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.StackRow.cs" />
+    <Compile Include="System\Text\Json\JsonDocument.TryGetProperty.cs" />
+    <Compile Include="System\Text\Json\JsonElement.cs" />
     <Compile Include="System\Text\Json\JsonReaderException.cs" />
     <Compile Include="System\Text\Json\JsonReaderHelper.cs" />
     <Compile Include="System\Text\Json\JsonReaderOptions.cs" />

src/System.Text.Json/src/System/Text/Json/BitStack.cs

@@ -114,15 +114,17 @@ public bool Pop()
             }
             else
             {
-                inObject = PopFromArray();
+                inObject = ReadFromArray(_currentDepth);
             }
             return inObject;
         }
 
         [MethodImpl(MethodImplOptions.NoInlining)]
-        private bool PopFromArray()
+        private bool ReadFromArray(int currentDepth)
         {
-            int index = _currentDepth - AllocationFreeMaxDepth - 1;
+            Debug.Assert(currentDepth == _currentDepth || currentDepth == _currentDepth - 1);
+
+            int index = currentDepth - AllocationFreeMaxDepth - 1;
             Debug.Assert(_array != null);
             Debug.Assert(index >= 0, $"Get - Negative - index: {index}, arrayLength: {_array.Length}");
 
@@ -133,6 +135,28 @@ private bool PopFromArray()
             return (_array[elementIndex] & (1 << extraBits)) != 0;
         }
 
+
+        internal bool Peek()
+        {
+            int depth = _currentDepth - 1;
+
+            if (depth < AllocationFreeMaxDepth)
+            {
+                // Pop would shift, then return the least significant bit.
+                // So Peek needs to check the unshifted position.
+                return (_allocationFreeContainer & 0b0010) != 0;
+            }
+
+            if (depth == AllocationFreeMaxDepth)
+            {
+                // Pop would not shift, but just return the least significant bit.
+                // So do that.
+                return (_allocationFreeContainer & 1) != 0;
+            }
+
+            return ReadFromArray(depth);
+        }
+
         private void DoubleArray(int minSize)
         {
             Debug.Assert(_array.Length < int.MaxValue / 2, $"Array too large - arrayLength: {_array.Length}");

src/System.Text.Json/src/System/Text/Json/JsonDocument.CustomDb.cs

@@ -0,0 +1,203 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Buffers;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+namespace System.Text.Json
+{
+    partial class JsonDocument
+    {
+        private struct CustomDb : IDisposable
+        {
+            private const int SizeOrLengthOffset = 4;
+            private const int NumberOfRowsOffset = 8;
+
+            internal int Length;
+            private byte[] _rentedBuffer;
+
+            internal CustomDb(int initialSize)
+            {
+                _rentedBuffer = ArrayPool<byte>.Shared.Rent(initialSize);
+                Length = 0;
+            }
+
+            public void Dispose()
+            {
+                if (_rentedBuffer == null)
+                {
+                    return;
+                }
+
+                ArrayPool<byte>.Shared.Return(_rentedBuffer);
+                _rentedBuffer = null;
+                Length = 0;
+            }
+
+            internal void TrimExcess()
+            {
+                if (Length <= _rentedBuffer.Length / 2)
+                {
+                    byte[] newRent = ArrayPool<byte>.Shared.Rent(Length);
+                    byte[] returnBuf = newRent;
+
+                    if (newRent.Length < _rentedBuffer.Length)
+                    {
+                        Buffer.BlockCopy(_rentedBuffer, 0, newRent, 0, Length);
+                        returnBuf = _rentedBuffer;
+                        _rentedBuffer = newRent;
+                    }
+
+                    ArrayPool<byte>.Shared.Return(returnBuf);
+                }
+            }
+
+            internal void Append(JsonTokenType tokenType, int startLocation, int length, bool isPropertyValue)
+            {
+                // StartArray or StartObject should have length -1, otherwise the length should not be -1.
+                Debug.Assert(
+                    (tokenType == JsonTokenType.StartArray || tokenType == JsonTokenType.StartObject) ==
+                    (length == DbRow.UnknownSize));
+
+                if (Length >= _rentedBuffer.Length - DbRow.Size)
+                {
+                    Enlarge();
+                }
+
+                DbRow row = new DbRow(tokenType, startLocation, length, isPropertyValue);
+                MemoryMarshal.Write(_rentedBuffer.AsSpan(Length), ref row);
+                Length += DbRow.Size;
+            }
+
+            private void Enlarge()
+            {
+                int size = _rentedBuffer.Length * 2;
+                byte[] newArray = ArrayPool<byte>.Shared.Rent(size);
+                Buffer.BlockCopy(_rentedBuffer, 0, newArray, 0, Length);
+                ArrayPool<byte>.Shared.Return(_rentedBuffer);
+                _rentedBuffer = newArray;
+            }
+
+            [Conditional("DEBUG")]
+            private void AssertValidIndex(int index)
+            {
+                Debug.Assert(index >= 0);
+                Debug.Assert(index <= Length - DbRow.Size, $"index {index} is out of bounds");
+                Debug.Assert(index % DbRow.Size == 0, $"index {index} is not at a record start position");
+            }
+
+            internal void SetLength(int index, int length)
+            {
+                AssertValidIndex(index);
+                Debug.Assert(length >= 0);
+                Span<byte> destination = _rentedBuffer.AsSpan(index + 4);
+                int cur = MemoryMarshal.Read<int>(destination);
+
+                // Persist the most significant bit
+                length |= (cur & unchecked((int)0x80000000));
+                MemoryMarshal.Write(destination, ref length);
+            }
+
+            internal void SetNumberOfRows(int index, int numberOfRows)
+            {
+                AssertValidIndex(index);
+                Debug.Assert(numberOfRows >= 1 && numberOfRows <= 0x0FFFFFFF);
+
+                Span<byte> dataPos = _rentedBuffer.AsSpan(index + NumberOfRowsOffset);
+                int current = MemoryMarshal.Read<int>(dataPos);
+
+                // Persist the most significant nybble
+                int value = (current & unchecked((int)0xF0000000)) | numberOfRows;
+                MemoryMarshal.Write(dataPos, ref value);
+            }
+
+            internal void SetHasComplexChildren(int index)
+            {
+                AssertValidIndex(index);
+
+                // The HasComplexChildren bit is the most significant bit of "SizeOrLength"
+                Span<byte> dataPos = _rentedBuffer.AsSpan(index + SizeOrLengthOffset);
+                int current = MemoryMarshal.Read<int>(dataPos);
+
+                int value = current | unchecked((int)0x80000000);
+                MemoryMarshal.Write(dataPos, ref value);
+            }
+
+            internal int FindIndexOfFirstUnsetSizeOrLength(JsonTokenType lookupType)
+            {
+                Debug.Assert(lookupType == JsonTokenType.StartObject || lookupType == JsonTokenType.StartArray);
+                return FindOpenElement(lookupType);
+            }
+
+            private int FindOpenElement(JsonTokenType lookupType)
+            {
+                Span<byte> data = _rentedBuffer.AsSpan(0, Length);
+
+                for (int i = Length - DbRow.Size; i >= 0; i -= DbRow.Size)
+                {
+                    DbRow row = MemoryMarshal.Read<DbRow>(data.Slice(i));
+
+                    if (row.IsUnknownSize && row.TokenType == lookupType)
+                    {
+                        return i;
+                    }
+                }
+
+                // We should never reach here.
+                Debug.Fail($"Unable to find expected {lookupType} token");
+                return -1;
+            }
+
+            internal void Get(int index, out DbRow row)
+            {
+                AssertValidIndex(index);
+                row = MemoryMarshal.Read<DbRow>(_rentedBuffer.AsSpan(index));
+            }
+
+            internal int GetLocation() => MemoryMarshal.Read<int>(_rentedBuffer.AsSpan());
+
+            internal int GetLocation(int index)
+            {
+                AssertValidIndex(index);
+                return MemoryMarshal.Read<int>(_rentedBuffer.AsSpan()) & int.MaxValue;
+            }
+
+            internal int GetSizeOrLength(int index)
+            {
+                AssertValidIndex(index);
+                return MemoryMarshal.Read<int>(_rentedBuffer.AsSpan(index + SizeOrLengthOffset)) & int.MaxValue;
+            }
+
+            internal JsonTokenType GetJsonTokenType(int index = 0)
+            {
+                AssertValidIndex(index);
+                uint union = MemoryMarshal.Read<uint>(_rentedBuffer.AsSpan(index + NumberOfRowsOffset));
+
+                return (JsonTokenType)(union >> 28);
+            }
+
+#if DIAGNOSTIC
+            internal string PrintDatabase()
+            {
+                StringBuilder sb = new StringBuilder();
+                sb.AppendLine("Index  Offset  SizeOrLen  TokenType    Child IPV   NumRows");
+                Span<byte> data = _rentedBuffer;
+
+                for (int i = 0; i < Length; i += DbRow.Size)
+                {
+                    DbRow record = MemoryMarshal.Read<DbRow>(data.Slice(i));
+                    sb.Append($"{i:D6} {record.Location:D7} {record.SizeOrLength:D10} ");
+                    sb.Append(record.TokenType.ToString().PadRight(13));
+                    sb.Append(record.HasChildren.ToString().PadRight(6));
+                    sb.Append(record.IsPropertyValue.ToString().PadRight(6));
+                    sb.AppendLine("" + record.NumberOfRows);
+                }
+
+                return sb.ToString();
+            }
+#endif
+        }
+    }
+}