Merge branch 'main' into fix_agent_streaming_and_assistant_options

dotnet/nuget/nuget-package.props

@@ -1,7 +1,7 @@
 <Project>
   <PropertyGroup>
     <!-- Central version prefix - applies to all nuget packages. -->
-    <VersionPrefix>1.19.0</VersionPrefix>
+    <VersionPrefix>1.20.0</VersionPrefix>
     <PackageVersion Condition="'$(VersionSuffix)' != ''">$(VersionPrefix)-$(VersionSuffix)</PackageVersion>
     <PackageVersion Condition="'$(VersionSuffix)' == ''">$(VersionPrefix)</PackageVersion>
 

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using Microsoft.SemanticKernel.Data;
+using MongoDB.Bson.Serialization.Attributes;
 
 namespace SemanticKernel.Connectors.AzureCosmosDBMongoDB.UnitTests;
 
@@ -25,7 +26,8 @@ public class AzureCosmosDBMongoDBHotelModel(string hotelId)
     public float? HotelRating { get; set; }
 
     /// <summary>A bool metadata field.</summary>
-    [VectorStoreRecordData(StoragePropertyName = "parking_is_included")]
+    [BsonElement("parking_is_included")]
+    [VectorStoreRecordData]
     public bool ParkingIncluded { get; set; }
 
     /// <summary>An array metadata field.</summary>

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs

@@ -422,7 +422,7 @@ public async Task UpsertWithModelWorksCorrectlyAsync()
             }
         };
 
-        await this.TestUpsertWithModeAsync<TestModel>(
+        await this.TestUpsertWithModelAsync<TestModel>(
             dataModel: new TestModel { Id = "key", HotelName = "Test Name" },
             expectedPropertyName: "HotelName",
             definition: definition);
@@ -431,9 +431,9 @@ await this.TestUpsertWithModeAsync<TestModel>(
     [Fact]
     public async Task UpsertWithVectorStoreModelWorksCorrectlyAsync()
     {
-        await this.TestUpsertWithModeAsync<VectorStoreTestModel>(
+        await this.TestUpsertWithModelAsync<VectorStoreTestModel>(
             dataModel: new VectorStoreTestModel { Id = "key", HotelName = "Test Name" },
-            expectedPropertyName: "hotel_name");
+            expectedPropertyName: "HotelName");
     }
 
     [Fact]
@@ -448,7 +448,7 @@ public async Task UpsertWithBsonModelWorksCorrectlyAsync()
             }
         };
 
-        await this.TestUpsertWithModeAsync<BsonTestModel>(
+        await this.TestUpsertWithModelAsync<BsonTestModel>(
             dataModel: new BsonTestModel { Id = "key", HotelName = "Test Name" },
             expectedPropertyName: "hotel_name",
             definition: definition);
@@ -457,15 +457,15 @@ await this.TestUpsertWithModeAsync<BsonTestModel>(
     [Fact]
     public async Task UpsertWithBsonVectorStoreModelWorksCorrectlyAsync()
     {
-        await this.TestUpsertWithModeAsync<BsonVectorStoreTestModel>(
+        await this.TestUpsertWithModelAsync<BsonVectorStoreTestModel>(
             dataModel: new BsonVectorStoreTestModel { Id = "key", HotelName = "Test Name" },
             expectedPropertyName: "hotel_name");
     }
 
     [Fact]
     public async Task UpsertWithBsonVectorStoreWithNameModelWorksCorrectlyAsync()
     {
-        await this.TestUpsertWithModeAsync<BsonVectorStoreWithNameTestModel>(
+        await this.TestUpsertWithModelAsync<BsonVectorStoreWithNameTestModel>(
             dataModel: new BsonVectorStoreWithNameTestModel { Id = "key", HotelName = "Test Name" },
             expectedPropertyName: "bson_hotel_name");
     }
@@ -560,7 +560,7 @@ public async Task GetWithCustomMapperWorksCorrectlyAsync()
 
     #region private
 
-    private async Task TestUpsertWithModeAsync<TDataModel>(
+    private async Task TestUpsertWithModelAsync<TDataModel>(
         TDataModel dataModel,
         string expectedPropertyName,
         VectorStoreRecordDefinition? definition = null)

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordMapperTests.cs

@@ -18,36 +18,32 @@ public sealed class AzureCosmosDBMongoDBVectorStoreRecordMapperTests
 
     public AzureCosmosDBMongoDBVectorStoreRecordMapperTests()
     {
+        var keyProperty = new VectorStoreRecordKeyProperty("HotelId", typeof(string));
+
         var definition = new VectorStoreRecordDefinition
         {
             Properties =
             [
-                new VectorStoreRecordKeyProperty("HotelId", typeof(string)),
+                keyProperty,
                 new VectorStoreRecordDataProperty("HotelName", typeof(string)),
                 new VectorStoreRecordDataProperty("Tags", typeof(List<string>)),
-                new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory<float>?)) { StoragePropertyName = "description_embedding " }
+                new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)),
+                new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory<float>?))
             ]
         };
 
-        var storagePropertyNames = new Dictionary<string, string>
-        {
-            ["HotelId"] = "HotelId",
-            ["HotelName"] = "HotelName",
-            ["Tags"] = "Tags",
-            ["DescriptionEmbedding"] = "description_embedding",
-        };
-
-        this._sut = new(definition, storagePropertyNames);
+        this._sut = new(definition, keyProperty.DataModelPropertyName);
     }
 
-    [Fact(Skip = "Failing when running PR validation.")]
+    [Fact]
     public void MapFromDataToStorageModelReturnsValidObject()
     {
         // Arrange
         var hotel = new AzureCosmosDBMongoDBHotelModel("key")
         {
             HotelName = "Test Name",
             Tags = ["tag1", "tag2"],
+            ParkingIncluded = true,
             DescriptionEmbedding = new ReadOnlyMemory<float>([1f, 2f, 3f])
         };
 
@@ -60,10 +56,11 @@ public void MapFromDataToStorageModelReturnsValidObject()
         Assert.Equal("key", document["_id"]);
         Assert.Equal("Test Name", document["HotelName"]);
         Assert.Equal(["tag1", "tag2"], document["Tags"].AsBsonArray);
-        Assert.Equal([1f, 2f, 3f], document["description_embedding"].AsBsonArray);
+        Assert.True(document["parking_is_included"].AsBoolean);
+        Assert.Equal([1f, 2f, 3f], document["DescriptionEmbedding"].AsBsonArray);
     }
 
-    [Fact(Skip = "Failing when running PR validation.")]
+    [Fact]
     public void MapFromStorageToDataModelReturnsValidObject()
     {
         // Arrange
@@ -72,7 +69,8 @@ public void MapFromStorageToDataModelReturnsValidObject()
             ["_id"] = "key",
             ["HotelName"] = "Test Name",
             ["Tags"] = BsonArray.Create(new List<string> { "tag1", "tag2" }),
-            ["description_embedding"] = BsonArray.Create(new List<float> { 1f, 2f, 3f })
+            ["parking_is_included"] = BsonValue.Create(true),
+            ["DescriptionEmbedding"] = BsonArray.Create(new List<float> { 1f, 2f, 3f })
         };
 
         // Act
@@ -84,6 +82,7 @@ public void MapFromStorageToDataModelReturnsValidObject()
         Assert.Equal("key", hotel.HotelId);
         Assert.Equal("Test Name", hotel.HotelName);
         Assert.Equal(["tag1", "tag2"], hotel.Tags);
+        Assert.True(hotel.ParkingIncluded);
         Assert.True(new ReadOnlyMemory<float>([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span));
     }
 }

dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBConstants.cs

@@ -9,4 +9,7 @@ internal static class AzureCosmosDBMongoDBConstants
 {
     /// <summary>Reserved key property name in Azure CosmosDB MongoDB.</summary>
     internal const string MongoReservedKeyPropertyName = "_id";
+
+    /// <summary>Reserved key property name in data model.</summary>
+    internal const string DataModelReservedKeyPropertyName = "Id";
 }

dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs

@@ -81,11 +81,15 @@ public AzureCosmosDBMongoDBVectorStoreRecordCollection(
             requiresAtLeastOneVector: false);
 
         this._storagePropertyNames = GetStoragePropertyNames(properties, typeof(TRecord));
+
+        // Use Mongo reserved key property name as storage key property name
+        this._storagePropertyNames[properties.KeyProperty.DataModelPropertyName] = AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName;
+
         this._vectorProperties = properties.VectorProperties;
         this._vectorStoragePropertyNames = this._vectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList();
 
         this._mapper = this._options.BsonDocumentCustomMapper ??
-            new AzureCosmosDBMongoDBVectorStoreRecordMapper<TRecord>(this._vectorStoreRecordDefinition, this._storagePropertyNames);
+            new AzureCosmosDBMongoDBVectorStoreRecordMapper<TRecord>(this._vectorStoreRecordDefinition, properties.KeyProperty.DataModelPropertyName);
     }
 
     /// <inheritdoc />
@@ -404,7 +408,7 @@ private static Dictionary<string, string> GetStoragePropertyNames(
         (VectorStoreRecordKeyProperty KeyProperty, List<VectorStoreRecordDataProperty> DataProperties, List<VectorStoreRecordVectorProperty> VectorProperties) properties,
         Type dataModel)
     {
-        var storagePropertyNames = VectorStoreRecordPropertyReader.BuildPropertyNameToStorageNameMap(properties);
+        var storagePropertyNames = new Dictionary<string, string>();
 
         var allProperties = new List<VectorStoreRecordProperty>([properties.KeyProperty])
             .Concat(properties.DataProperties)
@@ -417,10 +421,8 @@ private static Dictionary<string, string> GetStoragePropertyNames(
             if (propertyInfo != null)
             {
                 var bsonElementAttribute = propertyInfo.GetCustomAttribute<BsonElementAttribute>();
-                if (bsonElementAttribute is not null)
-                {
-                    storagePropertyNames[property.DataModelPropertyName] = bsonElementAttribute.ElementName;
-                }
+
+                storagePropertyNames[property.DataModelPropertyName] = bsonElementAttribute?.ElementName ?? property.DataModelPropertyName;
             }
         }
 

dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordMapper.cs

@@ -2,10 +2,11 @@
 
 using System;
 using System.Collections.Generic;
+using System.Reflection;
 using Microsoft.SemanticKernel.Data;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization;
-using MongoDB.Bson.Serialization.Conventions;
+using MongoDB.Bson.Serialization.Attributes;
 
 namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB;
 
@@ -45,42 +46,59 @@ internal sealed class AzureCosmosDBMongoDBVectorStoreRecordMapper<TRecord> : IVe
         typeof(ReadOnlyMemory<double>?)
     ];
 
-    /// <summary>A dictionary that maps from a property name to the storage name.</summary>
-    private readonly Dictionary<string, string> _storagePropertyNames;
+    /// <summary>A key property info of the data model.</summary>
+    private readonly PropertyInfo _keyProperty;
+
+    /// <summary>A key property name of the data model.</summary>
+    private readonly string _keyPropertyName;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="AzureCosmosDBMongoDBVectorStoreRecordMapper{TRecord}"/> class.
     /// </summary>
     /// <param name="vectorStoreRecordDefinition">The record definition that defines the schema of the record type.</param>
-    /// <param name="storagePropertyNames">A dictionary that maps from a property name to the configured name that should be used when storing it.</param>
-    public AzureCosmosDBMongoDBVectorStoreRecordMapper(VectorStoreRecordDefinition vectorStoreRecordDefinition, Dictionary<string, string> storagePropertyNames)
+    /// <param name="keyPropertyName">A key property name of the data model.</param>
+    public AzureCosmosDBMongoDBVectorStoreRecordMapper(VectorStoreRecordDefinition vectorStoreRecordDefinition, string keyPropertyName)
     {
         var (keyProperty, dataProperties, vectorProperties) = VectorStoreRecordPropertyReader.FindProperties(typeof(TRecord), vectorStoreRecordDefinition, supportsMultipleVectors: true);
 
         VectorStoreRecordPropertyReader.VerifyPropertyTypes([keyProperty], s_supportedKeyTypes, "Key");
         VectorStoreRecordPropertyReader.VerifyPropertyTypes(dataProperties, s_supportedDataTypes, "Data", supportEnumerable: true);
         VectorStoreRecordPropertyReader.VerifyPropertyTypes(vectorProperties, s_supportedVectorTypes, "Vector");
 
-        this._storagePropertyNames = storagePropertyNames;
+        this._keyPropertyName = keyPropertyName;
+        this._keyProperty = keyProperty;
+    }
 
-        // Use Mongo reserved key property name as storage key property name
-        this._storagePropertyNames[keyProperty.Name] = AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName;
+    public BsonDocument MapFromDataToStorageModel(TRecord dataModel)
+    {
+        var document = dataModel.ToBsonDocument();
 
-        var conventionPack = new ConventionPack
+        // Handle key property mapping due to reserved key name in Mongo.
+        if (!document.Contains(AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName))
         {
-            new IgnoreExtraElementsConvention(ignoreExtraElements: true),
-            new AzureCosmosDBMongoDBNamingConvention(this._storagePropertyNames)
-        };
-
-        ConventionRegistry.Register(
-            nameof(AzureCosmosDBMongoDBVectorStoreRecordMapper<TRecord>),
-            conventionPack,
-            type => type == typeof(TRecord));
-    }
+            var value = document[this._keyPropertyName];
 
-    public BsonDocument MapFromDataToStorageModel(TRecord dataModel)
-        => dataModel.ToBsonDocument();
+            document.Remove(this._keyPropertyName);
+
+            document[AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName] = value;
+        }
+
+        return document;
+    }
 
     public TRecord MapFromStorageToDataModel(BsonDocument storageModel, StorageToDataModelMapperOptions options)
-        => BsonSerializer.Deserialize<TRecord>(storageModel);
+    {
+        // Handle key property mapping due to reserved key name in Mongo.
+        if (!this._keyPropertyName.Equals(AzureCosmosDBMongoDBConstants.DataModelReservedKeyPropertyName, StringComparison.OrdinalIgnoreCase) &&
+            this._keyProperty.GetCustomAttribute<BsonIdAttribute>() is null)
+        {
+            var value = storageModel[AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName];
+
+            storageModel.Remove(AzureCosmosDBMongoDBConstants.MongoReservedKeyPropertyName);
+
+            storageModel[this._keyPropertyName] = value;
+        }
+
+        return BsonSerializer.Deserialize<TRecord>(storageModel);
+    }
 }

dotnet/src/IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_AutoFunctionChoiceBehaviorTests.cs

@@ -208,7 +208,7 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
         Assert.Contains("GetCurrentDate", invokedFunctions);
     }
 
-    [Fact]
+    [Fact(Skip = "Temporarily disabled to unblock PR pipeline.")]
     public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuallyForStreamingAsync()
     {
         // Arrange

dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs

@@ -236,19 +236,19 @@ public async Task UpsertWithModelWorksCorrectlyAsync()
     public async Task UpsertWithVectorStoreModelWorksCorrectlyAsync()
     {
         // Arrange
-        var model = new VectorStoreTestModel { Id = "key", HotelName = "Test Name" };
+        var model = new VectorStoreTestModel { HotelId = "key", HotelName = "Test Name" };
 
         var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection<VectorStoreTestModel>(fixture.MongoDatabase, fixture.TestCollection);
 
         // Act
         var upsertResult = await sut.UpsertAsync(model);
-        var getResult = await sut.GetAsync(model.Id);
+        var getResult = await sut.GetAsync(model.HotelId);
 
         // Assert
         Assert.Equal("key", upsertResult);
 
         Assert.NotNull(getResult);
-        Assert.Equal("key", getResult.Id);
+        Assert.Equal("key", getResult.HotelId);
         Assert.Equal("Test Name", getResult.HotelName);
     }
 
@@ -288,19 +288,19 @@ public async Task UpsertWithBsonModelWorksCorrectlyAsync()
     public async Task UpsertWithBsonVectorStoreModelWorksCorrectlyAsync()
     {
         // Arrange
-        var model = new BsonVectorStoreTestModel { Id = "key", HotelName = "Test Name" };
+        var model = new BsonVectorStoreTestModel { HotelId = "key", HotelName = "Test Name" };
 
         var sut = new AzureCosmosDBMongoDBVectorStoreRecordCollection<BsonVectorStoreTestModel>(fixture.MongoDatabase, fixture.TestCollection);
 
         // Act
         var upsertResult = await sut.UpsertAsync(model);
-        var getResult = await sut.GetAsync(model.Id);
+        var getResult = await sut.GetAsync(model.HotelId);
 
         // Assert
         Assert.Equal("key", upsertResult);
 
         Assert.NotNull(getResult);
-        Assert.Equal("key", getResult.Id);
+        Assert.Equal("key", getResult.HotelId);
         Assert.Equal("Test Name", getResult.HotelName);
     }
 
@@ -351,7 +351,7 @@ private sealed class TestModel
     private sealed class VectorStoreTestModel
     {
         [VectorStoreRecordKey]
-        public string? Id { get; set; }
+        public string? HotelId { get; set; }
 
         [VectorStoreRecordData(StoragePropertyName = "hotel_name")]
         public string? HotelName { get; set; }
@@ -370,7 +370,7 @@ private sealed class BsonVectorStoreTestModel
     {
         [BsonId]
         [VectorStoreRecordKey]
-        public string? Id { get; set; }
+        public string? HotelId { get; set; }
 
         [BsonElement("hotel_name")]
         [VectorStoreRecordData]