From e19701b63b4a41dd90012dca4e8ee4af784084a3 Mon Sep 17 00:00:00 2001 From: John Bowen Date: Fri, 29 Mar 2024 10:23:02 -0700 Subject: [PATCH 1/2] Adding custom serializer to Cosmos client to enable reading $type properties on child objects --- .../CosmosDictionaryDataItem.cs | 7 ++- .../CosmosExtensionServices.cs | 1 + .../RawJsonCosmosSerializer.cs | 57 +++++++++++++++++++ .../JsonSourceTests.cs | 50 ++++++++++++++++ .../DataItemExtensions.cs | 13 ++++- 5 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs index a73f331..d150818 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs @@ -1,4 +1,5 @@ using Cosmos.DataTransfer.Interfaces; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace Cosmos.DataTransfer.CosmosExtension @@ -31,11 +32,13 @@ public IEnumerable GetFieldNames() { if (value is JObject element) { - return new CosmosDictionaryDataItem(element.ToObject>().ToDictionary(k => k.Key, v => v.Value)); + return new CosmosDictionaryDataItem(element.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.DefaultSettings)) + .ToDictionary(k => k.Key, v => v.Value)); } if (value is JArray array) { - return array.ToObject>().Select(GetChildObject).ToList(); + return array.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.DefaultSettings)) + .Select(GetChildObject).ToList(); } return value; diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index dd6766e..e098c18 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -21,6 +21,7 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp ApplicationName = userAgentString, AllowBulkExecution = true, EnableContentResponseOnWrite = false, + Serializer = new RawJsonCosmosSerializer(), }; CosmosClient? cosmosClient; diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs new file mode 100644 index 0000000..00929b6 --- /dev/null +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs @@ -0,0 +1,57 @@ +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; +using System.Text; + +namespace Cosmos.DataTransfer.CosmosExtension; + +/// +/// Serializer for Cosmos allowing access to internal JsonSerializer settings. +/// +/// +/// Defaults to disabling metadata handling to allow passthrough of recognized properties like "$type". +/// +public class RawJsonCosmosSerializer : CosmosSerializer +{ + public static readonly JsonSerializerSettings DefaultSettings = new() + { + DateParseHandling = DateParseHandling.None, + MetadataPropertyHandling = MetadataPropertyHandling.Ignore + }; + + public JsonSerializerSettings SerializerSettings { get; set; } = DefaultSettings; + + public override T FromStream(Stream stream) + { + using (stream) + { + if (typeof(Stream).IsAssignableFrom(typeof(T))) + { + return (T)(object)stream; + } + + using var streamReader = new StreamReader(stream); + using var jsonReader = new JsonTextReader(streamReader); + var serializer = JsonSerializer.Create(SerializerSettings); + return serializer.Deserialize(jsonReader); + } + } + + public override Stream ToStream(T input) + { + var memoryStream = new MemoryStream(); + using (var streamWriter = new StreamWriter(memoryStream, Encoding.UTF8, leaveOpen: true)) + { + using (var jsonWriter = new JsonTextWriter(streamWriter)) + { + jsonWriter.Formatting = Formatting.None; + var serializer = JsonSerializer.Create(SerializerSettings); + serializer.Serialize(jsonWriter, input); + jsonWriter.Flush(); + streamWriter.Flush(); + } + } + + memoryStream.Position = 0; + return memoryStream; + } +} \ No newline at end of file diff --git a/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs b/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs index b7eb8dd..dce7a87 100644 --- a/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs +++ b/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs @@ -154,6 +154,56 @@ public async Task ReadAsync_WithFlatObjects_ReadsValuesFromUrl() } } + [TestMethod] + public async Task ReadAsync_WithTypeHintFields_IncludesAllInOutput() + { + var json = @"[ +{ + ""id"": 1, + ""name"": ""One"", + ""$type"": ""Number"", + ""data"": { + ""$type"": ""Object"", + ""name"": ""A"" + } +}, +{ + ""id"": 2, + ""name"": ""Two"", + ""$type"": ""Digit"", + ""data"": { + ""$type"": ""String"", + ""name"": ""B"" + } +} +]"; + var filePath = Path.Combine(Path.GetTempPath(), "TypeHintFields.json"); + await File.WriteAllTextAsync(filePath, json); + + var extension = new JsonFileSource(); + var config = TestHelpers.CreateConfig(new Dictionary + { + { "FilePath", filePath } + }); + + int counter = 0; + await foreach (var dataItem in extension.ReadAsync(config, NullLogger.Instance)) + { + counter++; + var fields = dataItem.GetFieldNames().ToArray(); + CollectionAssert.AreEquivalent(new[] { "id", "name", "$type", "data" }, fields); + Assert.IsNotNull(dataItem.GetValue("id")); + Assert.IsNotNull(dataItem.GetValue("name")); + Assert.IsNotNull(dataItem.GetValue("$type")); + var child = dataItem.GetValue("data") as JsonDictionaryDataItem; + Assert.IsNotNull(child); + CollectionAssert.AreEquivalent(new[] { "$type", "name" }, child.GetFieldNames().ToArray()); + Assert.IsNotNull(child.GetValue("$type")); + Assert.IsNotNull(child.GetValue("name")); + } + + Assert.AreEqual(2, counter); + } } } \ No newline at end of file diff --git a/Interfaces/Cosmos.DataTransfer.Interfaces/DataItemExtensions.cs b/Interfaces/Cosmos.DataTransfer.Interfaces/DataItemExtensions.cs index 93032cc..49099ea 100644 --- a/Interfaces/Cosmos.DataTransfer.Interfaces/DataItemExtensions.cs +++ b/Interfaces/Cosmos.DataTransfer.Interfaces/DataItemExtensions.cs @@ -36,10 +36,17 @@ public static class DataItemExtensions { object? value = source.GetValue(field); var fieldName = field; - if (string.Equals(field, "id", StringComparison.CurrentCultureIgnoreCase) && requireStringId && !containsLowercaseIdField) + if (string.Equals(field, "id", StringComparison.CurrentCultureIgnoreCase) && requireStringId) { - value = value?.ToString(); - fieldName = "id"; + if (!containsLowercaseIdField) + { + value = value?.ToString(); + fieldName = "id"; + } + else + { + value = value?.ToString(); + } } else if (value is IDataItem child) { From 447b05871dd2ed6b14f8249d2e50ac933f7a920b Mon Sep 17 00:00:00 2001 From: John Bowen Date: Thu, 11 Apr 2024 15:44:32 -0700 Subject: [PATCH 2/2] Resolving merge issues with serialization settings --- .../CosmosDictionaryDataItem.cs | 4 ++-- .../CosmosExtensionServices.cs | 18 +++++++++--------- .../RawJsonCosmosSerializer.cs | 19 ++++++++++++------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs index d150818..1324c37 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDictionaryDataItem.cs @@ -32,12 +32,12 @@ public IEnumerable GetFieldNames() { if (value is JObject element) { - return new CosmosDictionaryDataItem(element.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.DefaultSettings)) + return new CosmosDictionaryDataItem(element.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.GetDefaultSettings())) .ToDictionary(k => k.Key, v => v.Value)); } if (value is JArray array) { - return array.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.DefaultSettings)) + return array.ToObject>(JsonSerializer.Create(RawJsonCosmosSerializer.GetDefaultSettings())) .Select(GetChildObject).ToList(); } diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index c6afb73..664029e 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -15,23 +15,23 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp { string userAgentString = CreateUserAgentString(displayName, sourceDisplayName); + var cosmosSerializer = new RawJsonCosmosSerializer(); + if (settings is CosmosSinkSettings sinkSettings) + { + cosmosSerializer.SerializerSettings.NullValueHandling = sinkSettings.IgnoreNullValues + ? Newtonsoft.Json.NullValueHandling.Ignore + : Newtonsoft.Json.NullValueHandling.Include; + } + var clientOptions = new CosmosClientOptions { ConnectionMode = settings.ConnectionMode, ApplicationName = userAgentString, AllowBulkExecution = true, EnableContentResponseOnWrite = false, - Serializer = new RawJsonCosmosSerializer(), + Serializer = cosmosSerializer, }; - if (settings is CosmosSinkSettings sinkSettings) - { - clientOptions.SerializerOptions = new CosmosSerializationOptions - { - IgnoreNullValues = sinkSettings.IgnoreNullValues - }; - } - CosmosClient? cosmosClient; if (settings.UseRbacAuth) { diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs index 00929b6..f375393 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/RawJsonCosmosSerializer.cs @@ -12,13 +12,18 @@ namespace Cosmos.DataTransfer.CosmosExtension; /// public class RawJsonCosmosSerializer : CosmosSerializer { - public static readonly JsonSerializerSettings DefaultSettings = new() - { - DateParseHandling = DateParseHandling.None, - MetadataPropertyHandling = MetadataPropertyHandling.Ignore - }; + private static readonly Encoding DefaultEncoding = new UTF8Encoding(false, true); + + public static JsonSerializerSettings GetDefaultSettings() => + new() + { + DateParseHandling = DateParseHandling.None, + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + ContractResolver = null, + MaxDepth = 64, + }; - public JsonSerializerSettings SerializerSettings { get; set; } = DefaultSettings; + public JsonSerializerSettings SerializerSettings { get; } = GetDefaultSettings(); public override T FromStream(Stream stream) { @@ -39,7 +44,7 @@ public override T FromStream(Stream stream) public override Stream ToStream(T input) { var memoryStream = new MemoryStream(); - using (var streamWriter = new StreamWriter(memoryStream, Encoding.UTF8, leaveOpen: true)) + using (var streamWriter = new StreamWriter(memoryStream, DefaultEncoding, bufferSize: 1024, leaveOpen: true)) { using (var jsonWriter = new JsonTextWriter(streamWriter)) {