Skip to content

Commit

Permalink
ADT: Fix the issue with non-default object serializers in Query APIs (#…
Browse files Browse the repository at this point in the history
…18379)

* Fix the bug with non-default ObjectSerializer in QueryClient
  • Loading branch information
azabbasi authored Feb 3, 2021
1 parent 0146cf2 commit 2b8903b
Show file tree
Hide file tree
Showing 11 changed files with 4,460 additions and 19 deletions.
1 change: 1 addition & 0 deletions eng/Packages.Data.props
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@
<PackageReference Update="Microsoft.AspNetCore.Server.Kestrel" Version="2.2.0" />
<PackageReference Update="Microsoft.AspNetCore.Server.WebListener" Version="1.0.2" />
<PackageReference Update="Microsoft.Azure.Core.Spatial" Version="1.0.0" />
<PackageReference Update="Microsoft.Azure.Core.NewtonsoftJson" Version="1.0.0" />
<PackageReference Update="Microsoft.Azure.Devices" Version="1.19.0" />
<PackageReference Update="Microsoft.Azure.Devices.Client" Version="1.27.0" />
<PackageReference Update="Microsoft.Azure.Graph.RBAC" Version="2.2.2-preview" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public class Program
/// </summary>
public static async Task Main(string[] args)
{
// Parse and validate paramters
// Parse and validate parameters
Options options = null;
ParserResult<Options> result = Parser.Default.ParseArguments<Options>(args)
.WithParsed(parsedOptions =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ internal async Task<ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders>>
QuerySpecification querySpecification,
QueryOptions queryTwinsOptions = null,
ObjectSerializer objectSerializer = null,
ObjectSerializer defaultObjectSerializer = null,
CancellationToken cancellationToken = default)
{
if (querySpecification == null)
Expand All @@ -36,7 +37,7 @@ internal async Task<ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders>>
case 200:
{
using JsonDocument document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);
QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer);
QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer, defaultObjectSerializer);
return ResponseWithHeaders.FromValue(value, headers, message.Response);
}
default:
Expand All @@ -48,6 +49,7 @@ internal ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders> QueryTwins<
QuerySpecification querySpecification,
QueryOptions queryTwinsOptions = null,
ObjectSerializer objectSerializer = null,
ObjectSerializer defaultObjectSerializer = null,
CancellationToken cancellationToken = default)
{
if (querySpecification == null)
Expand All @@ -68,7 +70,7 @@ internal ResponseWithHeaders<QueryResult<T>, QueryQueryTwinsHeaders> QueryTwins<
case 200:
{
using var document = JsonDocument.Parse(message.Response.ContentStream);
QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer);
QueryResult<T> value = QueryResult<T>.DeserializeQueryResult(document.RootElement, objectSerializer, defaultObjectSerializer);
return ResponseWithHeaders.FromValue(value, headers, message.Response);
}
default:
Expand Down
51 changes: 47 additions & 4 deletions sdk/digitaltwins/Azure.DigitalTwins.Core/src/DigitalTwinsClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ public class DigitalTwinsClient

private readonly ObjectSerializer _objectSerializer;

/// <summary>
/// In order to serialize/deserialize JsonElements into and out of a stream, we have to use the out of the box ObjectSerializer (JsonObjectSerializer).
/// If the user specifies a different type of serializer to instantiate the client than the default one, the SDK will instantiate a new JsonObjectSerializer of its own.
/// </summary>
private readonly ObjectSerializer _defaultObjectSerializer;

private readonly DigitalTwinsRestClient _dtRestClient;
private readonly DigitalTwinModelsRestClient _dtModelsRestClient;
private readonly EventRoutesRestClient _eventRoutesRestClient;
Expand Down Expand Up @@ -93,6 +99,13 @@ public DigitalTwinsClient(Uri endpoint, TokenCredential credential, DigitalTwins

_objectSerializer = options.Serializer ?? new JsonObjectSerializer();

// If the objectSerializer is of type JsonObjectSerializer, we will re-use the same object and set it as the defaultObjectSerializer.
// Otherwise, we will instantiate it and re-use it in the future.
_defaultObjectSerializer =
(_objectSerializer is JsonObjectSerializer)
? _objectSerializer
: new JsonObjectSerializer();

options.AddPolicy(new BearerTokenAuthenticationPolicy(credential, GetAuthorizationScopes()), HttpPipelinePosition.PerCall);
_httpPipeline = HttpPipelineBuilder.Build(options);

Expand Down Expand Up @@ -2040,7 +2053,15 @@ async Task<Page<T>> FirstPageFunc(int? pageSizeHint)
MaxItemsPerPage = pageSizeHint
};

Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(querySpecification, options, _objectSerializer, cancellationToken).ConfigureAwait(false);
Response<QueryResult<T>> response = await _queryClient
.QueryTwinsAsync<T>(
querySpecification,
options,
_objectSerializer,
_defaultObjectSerializer,
cancellationToken)
.ConfigureAwait(false);

return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
}
catch (Exception ex)
Expand All @@ -2067,7 +2088,15 @@ async Task<Page<T>> NextPageFunc(string nextLink, int? pageSizeHint)
MaxItemsPerPage = pageSizeHint
};

Response<QueryResult<T>> response = await _queryClient.QueryTwinsAsync<T>(querySpecification, options, _objectSerializer, cancellationToken).ConfigureAwait(false);
Response<QueryResult<T>> response = await _queryClient
.QueryTwinsAsync<T>(
querySpecification,
options,
_objectSerializer,
_defaultObjectSerializer,
cancellationToken)
.ConfigureAwait(false);

return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
}
catch (Exception ex)
Expand Down Expand Up @@ -2135,7 +2164,14 @@ Page<T> FirstPageFunc(int? pageSizeHint)
MaxItemsPerPage = pageSizeHint
};

Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(querySpecification, options, _objectSerializer, cancellationToken);
Response<QueryResult<T>> response = _queryClient
.QueryTwins<T>(
querySpecification,
options,
_objectSerializer,
_defaultObjectSerializer,
cancellationToken);

return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
}
catch (Exception ex)
Expand All @@ -2162,7 +2198,14 @@ Page<T> NextPageFunc(string nextLink, int? pageSizeHint)
MaxItemsPerPage = pageSizeHint
};

Response<QueryResult<T>> response = _queryClient.QueryTwins<T>(querySpecification, options, _objectSerializer, cancellationToken);
Response<QueryResult<T>> response = _queryClient
.QueryTwins<T>(
querySpecification,
options,
_objectSerializer,
_defaultObjectSerializer,
cancellationToken);

return Page.FromValues(response.Value.Value, response.Value.ContinuationToken, response.GetRawResponse());
}
catch (Exception ex)
Expand Down
23 changes: 12 additions & 11 deletions sdk/digitaltwins/Azure.DigitalTwins.Core/src/Models/QueryResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System.Collections.Generic;
using System.IO;
using System.Text.Json;
using Azure.Core;
using Azure.Core.Serialization;

namespace Azure.DigitalTwins.Core
Expand All @@ -25,14 +24,6 @@ internal partial class QueryResult<T>
/// </summary>
internal string ContinuationToken { get; }

/// <summary>
/// Initializes a new instance of QueryResult.
/// </summary>
internal QueryResult()
{
Value = new ChangeTrackingList<T>();
}

/// <summary>
/// Initializes a new instance of QueryResult.
/// </summary>
Expand All @@ -49,11 +40,13 @@ internal QueryResult(IReadOnlyList<T> value, string continuationToken)
/// </summary>
/// <param name="element">The JSON element to be deserialized into a QueryResult.</param>
/// <param name="objectSerializer">The object serializer instance used to deserialize the items in the collection.</param>
/// <param name="defaultObjectSerializer">The out of the box object serializer to interact with the JsonElement (serialize/deserialize into and out of streams).</param>
/// <returns>A collection of query results deserialized into type <typeparamref name="T"/>.</returns>
internal static QueryResult<T> DeserializeQueryResult(JsonElement element, ObjectSerializer objectSerializer)
internal static QueryResult<T> DeserializeQueryResult(JsonElement element, ObjectSerializer objectSerializer, ObjectSerializer defaultObjectSerializer)
{
IReadOnlyList<T> items = default;
string continuationToken = default;

foreach (JsonProperty property in element.EnumerateObject())
{
if (property.NameEquals("value"))
Expand All @@ -62,13 +55,21 @@ internal static QueryResult<T> DeserializeQueryResult(JsonElement element, Objec
{
continue;
}

var array = new List<T>();

foreach (JsonElement item in property.Value.EnumerateArray())
{
using MemoryStream streamedObject = StreamHelper.WriteToStream(item, objectSerializer, default);
// defaultObjectSerializer of type JsonObjectSerializer needs to be used to serialize the JsonElement into a stream.
// Using any other ObjectSerializer (e.g. NewtonsoftJsonObjectSerializer) won't be able to deserialize the JsonElement into
// a MemoryStream correctly.
using MemoryStream streamedObject = StreamHelper.WriteToStream(item, defaultObjectSerializer, default);

// To deserialize the stream object into the generic type of T, the provided ObjectSerializer will be used.
T obj = (T)objectSerializer.Deserialize(streamedObject, typeof(T), default);
array.Add(obj);
}

items = array;
continue;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ internal static MemoryStream WriteToStream<T>(T obj, ObjectSerializer objectSeri
var memoryStream = new MemoryStream();

objectSerializer.Serialize(memoryStream, obj, typeof(T), cancellationToken);

memoryStream.Seek(0, SeekOrigin.Begin);

return memoryStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Azure.Core.NewtonsoftJson" />
<PackageReference Include="Microsoft.Extensions.Configuration" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="FluentAssertions" />
<PackageReference Update="Newtonsoft.Json" />
</ItemGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Azure.Core.Serialization;
using FluentAssertions;
using NUnit.Framework;

namespace Azure.DigitalTwins.Core.Tests
{
/// <summary>
/// Tests for custom ObjectSerializer.
/// Users can specify their own serializer/deserializer and not go with the default JsonObjectSerializer.
/// SDK needs to make sure it can properly use different serializers and the behavior is seamless.
/// Specifically, we have a work around in the query code which requires use of a System.Text.Json serializer because it works on a JsonElement.
/// When the user initializes with a non-default serializer, we must not use that to do this work.
/// </summary>
public class NewtonsoftObjectSerializerTests : E2eTestBase
{
public NewtonsoftObjectSerializerTests(bool isAsync)
: base(isAsync)
{
}

[Test]
public async Task TestNewtonsoftObjectSerializerWithDigitalTwins()
{
DigitalTwinsClient defaultClient = GetClient();

string roomTwinId = await GetUniqueTwinIdAsync(defaultClient, TestAssetDefaults.RoomTwinIdPrefix).ConfigureAwait(false);
string floorModelId = await GetUniqueModelIdAsync(defaultClient, TestAssetDefaults.FloorModelIdPrefix).ConfigureAwait(false);
string roomModelId = await GetUniqueModelIdAsync(defaultClient, TestAssetDefaults.RoomModelIdPrefix).ConfigureAwait(false);

try
{
// arrange

// create room model
string roomModel = TestAssetsHelper.GetRoomModelPayload(roomModelId, floorModelId);
await CreateAndListModelsAsync(defaultClient, new List<string> { roomModel }).ConfigureAwait(false);

// act

// create room twin
BasicDigitalTwin roomTwin = TestAssetsHelper.GetRoomTwinPayload(roomModelId);
await defaultClient.CreateOrReplaceDigitalTwinAsync<BasicDigitalTwin>(roomTwinId, roomTwin).ConfigureAwait(false);

// Create a client with NewtonsoftJsonObjectSerializer configured as the serializer.
DigitalTwinsClient testClient = GetClient(
new DigitalTwinsClientOptions
{
Serializer = new NewtonsoftJsonObjectSerializer()
});

// Get digital twin using the simple DigitalTwin model annotated with Newtonsoft attributes
SimpleNewtonsoftDtModel getResponse = await testClient.GetDigitalTwinAsync<SimpleNewtonsoftDtModel>(roomTwinId).ConfigureAwait(false);

getResponse.Id.Should().NotBeNullOrEmpty("Digital twin ID should not be null or empty");

// Query DigitalTwins using the simple DigitalTwin model annotated with Newtonsoft attributes
AsyncPageable<SimpleNewtonsoftDtModel> queryResponse = testClient.QueryAsync<SimpleNewtonsoftDtModel>("SELECT * FROM DIGITALTWINS");

await foreach (SimpleNewtonsoftDtModel twin in queryResponse)
{
twin.Id.Should().NotBeNullOrEmpty("Digital twin Id should not be null or empty");
}
}
finally
{
// cleanup
try
{
// delete twin
if (!string.IsNullOrWhiteSpace(roomTwinId))
{
await defaultClient.DeleteDigitalTwinAsync(roomTwinId).ConfigureAwait(false);
}

// delete models
if (!string.IsNullOrWhiteSpace(roomModelId))
{
await defaultClient.DeleteModelAsync(roomModelId).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Assert.Fail($"Test clean up failed: {ex.Message}");
}
}
}
}
}
Loading

0 comments on commit 2b8903b

Please sign in to comment.