ADT: Fix the issue with non-default object serializers in Query APIs

eng/Packages.Data.props

@@ -153,6 +153,7 @@
     <PackageReference Update="Microsoft.AspNetCore.Server.Kestrel" Version="2.2.0" />
     <PackageReference Update="Microsoft.AspNetCore.Server.WebListener" Version="1.0.2" />
     <PackageReference Update="Microsoft.Azure.Core.Spatial" Version="1.0.0" />
+    <PackageReference Update="Microsoft.Azure.Core.NewtonsoftJson" Version="1.0.0" />
     <PackageReference Update="Microsoft.Azure.Devices" Version="1.19.0" />
     <PackageReference Update="Microsoft.Azure.Devices.Client" Version="1.27.0" />
     <PackageReference Update="Microsoft.Azure.Graph.RBAC" Version="2.2.2-preview" />

sdk/digitaltwins/Azure.DigitalTwins.Core/samples/DigitalTwinsClientSample/Program.cs

@@ -16,7 +16,7 @@ public class Program
         /// </summary>
         public static async Task Main(string[] args)
         {
-            // Parse and validate paramters
+            // Parse and validate parameters
             Options options = null;
             ParserResult<Options> result = Parser.Default.ParseArguments<Options>(args)
                 .WithParsed(parsedOptions =>

sdk/digitaltwins/Azure.DigitalTwins.Core/src/Customized/QueryRestClient.cs

@@ -16,6 +16,7 @@ internal async Task<ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders>>
             QuerySpecification querySpecification,
             QueryOptions queryTwinsOptions = null,
             ObjectSerializer objectSerializer = null,
+            ObjectSerializer defaultObjectSerializer = null,
             CancellationToken cancellationToken = default)
         {
             if (querySpecification == null)
@@ -36,7 +37,7 @@ internal async Task<ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders>>
                 case 200:
                     {
                         using JsonDocument document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);
-                        QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer);
+                        QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer, defaultObjectSerializer);
                         return ResponseWithHeaders.FromValue(value, headers, message.Response);
                     }
                 default:
@@ -48,6 +49,7 @@ internal ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders> QueryTwins<
             QuerySpecification querySpecification,
             QueryOptions queryTwinsOptions = null,
             ObjectSerializer objectSerializer = null,
+            ObjectSerializer defaultObjectSerializer = null,
             CancellationToken cancellationToken = default)
         {
             if (querySpecification == null)
@@ -68,7 +70,7 @@ internal ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders> QueryTwins<
                 case 200:
                     {
                         using var document = JsonDocument.Parse(message.Response.ContentStream);
-                        QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer);
+                        QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer, defaultObjectSerializer);
                         return ResponseWithHeaders.FromValue(value, headers, message.Response);
                     }
                 default:

sdk/digitaltwins/Azure.DigitalTwins.Core/src/DigitalTwinsClient.cs

@@ -33,6 +33,12 @@ public class DigitalTwinsClient
 
         private readonly ObjectSerializer _objectSerializer;
 
+        /// <summary>
+        /// In order to serialize/deserialize JsonElements into and out of a stream, we have to use the out of the box ObjectSerializer (JsonObjectSerializer).
+        /// If the user specifies a different type of serializer to instantiate the client than the default one, the SDK will instantiate a new JsonObjectSerializer of its own.
+        /// </summary>
+        private readonly ObjectSerializer _defaultObjectSerializer;
+
         private readonly DigitalTwinsRestClient _dtRestClient;
         private readonly DigitalTwinModelsRestClient _dtModelsRestClient;
         private readonly EventRoutesRestClient _eventRoutesRestClient;
@@ -93,6 +99,17 @@ public DigitalTwinsClient(Uri endpoint, TokenCredential credential, DigitalTwins
 
             _objectSerializer = options.Serializer ?? new JsonObjectSerializer();
 
+            // If the objectSerializer is of type JsonObjectSerializer, we will re-use the same object and set it as the defaultObjectSerializer.
+            if (_objectSerializer is JsonObjectSerializer)
+            {
+                _defaultObjectSerializer = _objectSerializer;
+            }
+            // Otherwise, we will instantiate it and re-use it in the future.
+            else
+            {
+                _defaultObjectSerializer = new JsonObjectSerializer();
+            }
+
             options.AddPolicy(new BearerTokenAuthenticationPolicy(credential, GetAuthorizationScopes()), HttpPipelinePosition.PerCall);
             _httpPipeline = HttpPipelineBuilder.Build(options);
 
@@ -1722,7 +1739,7 @@ public virtual Response<DigitalTwinsModelData> GetModel(string modelId, Cancella
         /// }
         /// catch (RequestFailedException ex)
         /// {
-        ///     FatalError($&quot;Failed to decommision model &apos;{sampleModelId}&apos; due to:\n{ex}&quot;);
+        ///     FatalError($&quot;Failed to decommission model &apos;{sampleModelId}&apos; due to:\n{ex}&quot;);
         /// }
         /// </code>
         /// </example>
@@ -2040,7 +2057,13 @@ async Task<Page<T>> FirstPageFunc(int? pageSizeHint)
                             MaxItemsPerPage = pageSizeHint
                         };
 
-                        Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(querySpecification, options, _objectSerializer, cancellationToken).ConfigureAwait(false);
+                        Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(
+                            querySpecification,
+                            options,
+                            _objectSerializer,
+                            _defaultObjectSerializer,
+                            cancellationToken).ConfigureAwait(false);
+
                         return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
                     }
                     catch (Exception ex)
@@ -2067,7 +2090,13 @@ async Task<Page<T>> NextPageFunc(string nextLink, int? pageSizeHint)
                             MaxItemsPerPage = pageSizeHint
                         };
 
-                        Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(querySpecification, options, _objectSerializer, cancellationToken).ConfigureAwait(false);
+                        Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(
+                            querySpecification,
+                            options,
+                            _objectSerializer,
+                            _defaultObjectSerializer,
+                            cancellationToken).ConfigureAwait(false);
+
                         return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
                     }
                     catch (Exception ex)
@@ -2135,7 +2164,13 @@ Page<T> FirstPageFunc(int? pageSizeHint)
                             MaxItemsPerPage = pageSizeHint
                         };
 
-                        Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(querySpecification, options, _objectSerializer, cancellationToken);
+                        Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(
+                            querySpecification,
+                            options,
+                            _objectSerializer,
+                            _defaultObjectSerializer,
+                            cancellationToken);
+
                         return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
                     }
                     catch (Exception ex)
@@ -2162,7 +2197,12 @@ Page<T> NextPageFunc(string nextLink, int? pageSizeHint)
                             MaxItemsPerPage = pageSizeHint
                         };
 
-                        Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(querySpecification, options, _objectSerializer, cancellationToken);
+                        Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(
+                            querySpecification,
+                            options,
+                            _objectSerializer,
+                            _defaultObjectSerializer,
+                            cancellationToken);
                         return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
                     }
                     catch (Exception ex)

sdk/digitaltwins/Azure.DigitalTwins.Core/src/Models/QueryResult.cs

@@ -4,7 +4,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Text.Json;
-using Azure.Core;
 using Azure.Core.Serialization;
 
 namespace Azure.DigitalTwins.Core
@@ -25,14 +24,6 @@ internal partial class QueryResult<T>
         /// </summary>
         internal string ContinuationToken { get; }
 
-        /// <summary>
-        /// Initializes a new instance of QueryResult.
-        /// </summary>
-        internal QueryResult()
-        {
-            Value = new ChangeTrackingList<T>();
-        }
-
         /// <summary>
         /// Initializes a new instance of QueryResult.
         /// </summary>
@@ -49,11 +40,13 @@ internal QueryResult(IReadOnlyList<T> value, string continuationToken)
         /// </summary>
         /// <param name="element">The JSON element to be deserialized into a QueryResult.</param>
         /// <param name="objectSerializer">The object serializer instance used to deserialize the items in the collection.</param>
+        /// <param name="defaultObjectSerializer">The out of the box object serializer to interact with the JsonElement (serialize/deserialize into and out of streams).</param>
         /// <returns>A collection of query results deserialized into type <typeparamref name="T"/>.</returns>
-        internal static QueryResult<T> DeserializeQueryResult(JsonElement element, ObjectSerializer objectSerializer)
+        internal static QueryResult<T> DeserializeQueryResult(JsonElement element, ObjectSerializer objectSerializer, ObjectSerializer defaultObjectSerializer)
         {
             IReadOnlyList<T> items = default;
             string continuationToken = default;
+
             foreach (JsonProperty property in element.EnumerateObject())
             {
                 if (property.NameEquals("value"))
@@ -62,13 +55,21 @@ internal static QueryResult<T> DeserializeQueryResult(JsonElement element, Objec
                     {
                         continue;
                     }
+
                     var array = new List<T>();
+
                     foreach (JsonElement item in property.Value.EnumerateArray())
                     {
-                        using MemoryStream streamedObject = StreamHelper.WriteToStream(item, objectSerializer, default);
+                        // defaultObjectSerializer of type JsonObjectSerializer needs to be used to serialize the JsonElement into a stream.
+                        // Using any other ObjectSerializer (ex: NewtonsoftJsonObjectSerializer) won't be able to deserialize the JsonElement into
+                        // a MemoryStream correctly.
+                        using MemoryStream streamedObject = StreamHelper.WriteToStream(item, defaultObjectSerializer, default);
+
+                        // To deserialize the stream object into the generic type of T, the provided ObjectSerializer will be used.
                         T obj = (T)objectSerializer.Deserialize(streamedObject, typeof(T), default);
                         array.Add(obj);
                     }
+
                     items = array;
                     continue;
                 }

sdk/digitaltwins/Azure.DigitalTwins.Core/src/StreamHelper.cs

@@ -42,7 +42,6 @@ internal static MemoryStream WriteToStream<T>(T obj, ObjectSerializer objectSeri
             var memoryStream = new MemoryStream();
 
             objectSerializer.Serialize(memoryStream, obj, typeof(T), cancellationToken);
-
             memoryStream.Seek(0, SeekOrigin.Begin);
 
             return memoryStream;

sdk/digitaltwins/Azure.DigitalTwins.Core/tests/Azure.DigitalTwins.Core.Tests.csproj

@@ -6,11 +6,13 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Azure.Core.NewtonsoftJson" />
     <PackageReference Include="Microsoft.Extensions.Configuration" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="FluentAssertions" />
+    <PackageReference Update="Newtonsoft.Json" />
   </ItemGroup>
 
   <ItemGroup>

sdk/digitaltwins/Azure.DigitalTwins.Core/tests/NewtonsoftObjectSerializerTests.cs

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Azure.Core.Serialization;
+using FluentAssertions;
+using NUnit.Framework;
+
+namespace Azure.DigitalTwins.Core.Tests
+{
+    /// <summary>
+    /// Tests for custom ObjectSerializer.
+    /// Users can specify their own serializer/deserializer and not go with the default JsonObjectSerializer.
+    /// SDK needs to make sure it can properly use different serializers and the behavior is seamless.
+    /// </summary>
+    public class NewtonsoftObjectSerializerTests : E2eTestBase
+    {
+        public NewtonsoftObjectSerializerTests(bool isAsync)
+            : base(isAsync)
+        {
+        }
+
+        [Test]
+        public async Task TestNewtonsoftObjectSerializerWithDigitalTwins()
+        {
+            DigitalTwinsClient defaultClient = GetClient();
+
+            string roomTwinId = await GetUniqueTwinIdAsync(defaultClient, TestAssetDefaults.RoomTwinIdPrefix).ConfigureAwait(false);
+            string floorModelId = await GetUniqueModelIdAsync(defaultClient, TestAssetDefaults.FloorModelIdPrefix).ConfigureAwait(false);
+            string roomModelId = await GetUniqueModelIdAsync(defaultClient, TestAssetDefaults.RoomModelIdPrefix).ConfigureAwait(false);
+
+            try
+            {
+                // arrange
+
+                // create room model
+                string roomModel = TestAssetsHelper.GetRoomModelPayload(roomModelId, floorModelId);
+                await CreateAndListModelsAsync(defaultClient, new List<string> { roomModel }).ConfigureAwait(false);
+
+                // act
+
+                // create room twin
+                BasicDigitalTwin roomTwin = TestAssetsHelper.GetRoomTwinPayload(roomModelId);
+                await defaultClient.CreateOrReplaceDigitalTwinAsync<BasicDigitalTwin>(roomTwinId, roomTwin).ConfigureAwait(false);
+
+                // Create a client with NewtonsoftJsonObjectSerializer configured as the serializer.
+                DigitalTwinsClient testClient = GetClient(
+                    new DigitalTwinsClientOptions
+                    {
+                        Serializer = new NewtonsoftJsonObjectSerializer()
+                    });
+
+                // Get digital twin using the simple DigitalTwin model annotated with Newtonsoft attributes
+                SimpleNewtonsoftDtModel getResponse = await testClient.GetDigitalTwinAsync<SimpleNewtonsoftDtModel>(roomTwinId).ConfigureAwait(false);
+
+                getResponse.Id.Should().NotBeNullOrEmpty("Digital twin ID should not be null or empty");
+
+                // Query DigitalTwins using the simple DigitalTwin model annotated with Newtonsoft attributes
+                AsyncPageable<SimpleNewtonsoftDtModel> queryResponse = testClient.QueryAsync<SimpleNewtonsoftDtModel>("SELECT * FROM DIGITALTWINS");
+
+                await foreach (SimpleNewtonsoftDtModel twin in queryResponse)
+                {
+                    twin.Id.Should().NotBeNullOrEmpty("Digital twin Id should not be null or empty");
+                }
+            }
+            finally
+            {
+                // cleanup
+                try
+                {
+                    // delete twin
+                    if (!string.IsNullOrWhiteSpace(roomTwinId))
+                    {
+                        await defaultClient.DeleteDigitalTwinAsync(roomTwinId).ConfigureAwait(false);
+                    }
+
+                    // delete models
+                    if (!string.IsNullOrWhiteSpace(roomModelId))
+                    {
+                        await defaultClient.DeleteModelAsync(roomModelId).ConfigureAwait(false);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Assert.Fail($"Test clean up failed: {ex.Message}");
+                }
+            }
+        }
+    }
+}