From cc50c2121f756d46660f661293c4a489c834e20c Mon Sep 17 00:00:00 2001 From: Chris <66376200+crickman@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:33:11 -0800 Subject: [PATCH] .Net - Add support for instruction templating on Agents (#4486) ### Motivation and Context 1. Agent instructions support template parameters 2. Agent yaml identical / interchangeable with prompt config yaml ### Description Templating agents introduces much more dynamic capabilities, beyond the standard assistant API. As part of this, the agent template and kernel-function template have converged. ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --------- Co-authored-by: Gil LaHaye --- .../KernelSyntaxExamples/Example70_Agents.cs | 12 ++++-- .../Resources/Agents/ParrotAgent.yaml | 9 +++- .../Resources/Agents/ToolAgent.yaml | 3 +- .../Agents/AgentBuilder.Static.cs | 2 +- .../src/Experimental/Agents/AgentBuilder.cs | 23 ++++++---- dotnet/src/Experimental/Agents/AgentPlugin.cs | 19 ++++++++- .../Agents/Experimental.Agents.csproj | 3 +- dotnet/src/Experimental/Agents/IAgent.cs | 5 +++ .../Experimental/Agents/IAgentExtensions.cs | 8 ++-- .../src/Experimental/Agents/IAgentThread.cs | 6 ++- .../src/Experimental/Agents/Internal/Agent.cs | 42 ++++++++++++++++--- .../Agents/Internal/ChatThread.cs | 14 +++++-- .../Agents/Models/AgentConfigurationModel.cs | 30 ------------- .../Functions.Yaml/KernelFunctionYaml.cs | 21 +++++++--- 14 files changed, 128 insertions(+), 69 deletions(-) delete mode 100644 dotnet/src/Experimental/Agents/Models/AgentConfigurationModel.cs diff --git a/dotnet/samples/KernelSyntaxExamples/Example70_Agents.cs b/dotnet/samples/KernelSyntaxExamples/Example70_Agents.cs index 1e3aae72ffc7..8a17164744bf 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example70_Agents.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example70_Agents.cs @@ -58,6 +58,7 @@ private static async Task RunSimpleChatAsync() await ChatAsync( "Agents.ParrotAgent.yaml", // Defined under ./Resources/Agents plugin: null, // No plugin + arguments: new KernelArguments { { "count", 3 } }, "Fortune favors the bold.", "I came, I saw, I conquered.", "Practice makes perfect."); @@ -77,6 +78,7 @@ private static async Task RunWithMethodFunctionsAsync() await ChatAsync( "Agents.ToolAgent.yaml", // Defined under ./Resources/Agents plugin, + arguments: null, "Hello", "What is the special soup?", "What is the special drink?", @@ -95,14 +97,15 @@ private static async Task RunWithPromptFunctionsAsync() var function = KernelFunctionFactory.CreateFromPrompt( "Correct any misspelling or gramatical errors provided in input: {{$input}}", functionName: "spellChecker", - description: "Correct the spelling for the user input." - ); + description: "Correct the spelling for the user input."); + var plugin = KernelPluginFactory.CreateFromFunctions("spelling", "Spelling functions", new[] { function }); // Call the common chat-loop await ChatAsync( "Agents.ToolAgent.yaml", // Defined under ./Resources/Agents plugin, + arguments: null, "Hello", "Is this spelled correctly: exercize", "What is the special soup?", @@ -126,7 +129,7 @@ private static async Task RunAsFunctionAsync() try { // Invoke agent plugin. - var response = await agent.AsPlugin().InvokeAsync("Practice makes perfect."); + var response = await agent.AsPlugin().InvokeAsync("Practice makes perfect.", new KernelArguments { { "count", 2 } }); // Display result. Console.WriteLine(response ?? $"No response from agent: {agent.Id}"); @@ -149,6 +152,7 @@ private static async Task RunAsFunctionAsync() private static async Task ChatAsync( string resourcePath, KernelPlugin? plugin = null, + KernelArguments? arguments = null, params string[] messages) { // Read agent resource @@ -170,7 +174,7 @@ private static async Task ChatAsync( Console.WriteLine($"[{agent.Id}]"); // Process each user message and agent response. - foreach (var response in messages.Select(m => thread.InvokeAsync(agent, m))) + foreach (var response in messages.Select(m => thread.InvokeAsync(agent, m, arguments))) { await foreach (var message in response) { diff --git a/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ParrotAgent.yaml b/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ParrotAgent.yaml index 32daa9ef6124..26a07cf04cf3 100644 --- a/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ParrotAgent.yaml +++ b/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ParrotAgent.yaml @@ -1,4 +1,9 @@ name: Parrot -instructions: | - Repeat the user message in the voice of a pirate and then end with a parrot sound. +template_format: semantic-kernel +template: | + Repeat the user message in the voice of a pirate and then end with {{$count}} parrot sounds. description: A fun chat bot that repeats the user message in the voice of a pirate. +input_variables: + - name: count + description: The number of parrot sounds. + is_required: true diff --git a/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ToolAgent.yaml b/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ToolAgent.yaml index 0e294c38d449..474fd86a46ad 100644 --- a/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ToolAgent.yaml +++ b/dotnet/samples/KernelSyntaxExamples/Resources/Agents/ToolAgent.yaml @@ -1,5 +1,6 @@ name: ToolRunner -instructions: | +template_format: semantic-kernel +template: | Respond to the user using the single best tool. If no tool is appropriate, let the user know you only provide responses using tools. When reporting a tool result, start with, "The tool I used informed me that" diff --git a/dotnet/src/Experimental/Agents/AgentBuilder.Static.cs b/dotnet/src/Experimental/Agents/AgentBuilder.Static.cs index 557bc2de6632..652e6b4e9759 100644 --- a/dotnet/src/Experimental/Agents/AgentBuilder.Static.cs +++ b/dotnet/src/Experimental/Agents/AgentBuilder.Static.cs @@ -54,6 +54,6 @@ public static async Task GetAgentAsync( var restContext = new OpenAIRestContext(apiKey); var resultModel = await restContext.GetAssistantModelAsync(agentId, cancellationToken).ConfigureAwait(false); - return new Agent(resultModel, restContext, plugins); + return new Agent(resultModel, null, restContext, plugins); } } diff --git a/dotnet/src/Experimental/Agents/AgentBuilder.cs b/dotnet/src/Experimental/Agents/AgentBuilder.cs index d809d5c1e145..38a81e779a7a 100644 --- a/dotnet/src/Experimental/Agents/AgentBuilder.cs +++ b/dotnet/src/Experimental/Agents/AgentBuilder.cs @@ -9,7 +9,6 @@ using Microsoft.SemanticKernel.Experimental.Agents.Exceptions; using Microsoft.SemanticKernel.Experimental.Agents.Internal; using Microsoft.SemanticKernel.Experimental.Agents.Models; -using YamlDotNet.Serialization; namespace Microsoft.SemanticKernel.Experimental.Agents; @@ -23,6 +22,7 @@ public partial class AgentBuilder private string? _apiKey; private Func? _httpClientProvider; + private PromptTemplateConfig? _config; /// /// Initializes a new instance of the class. @@ -54,6 +54,7 @@ public async Task BuildAsync(CancellationToken cancellationToken = defau await Agent.CreateAsync( new OpenAIRestContext(this._apiKey!, this._httpClientProvider), this._model, + this._config, this._plugins, cancellationToken).ConfigureAwait(false); } @@ -77,15 +78,21 @@ public AgentBuilder WithOpenAIChatCompletion(string model, string apiKey) /// instance for fluid expression. public AgentBuilder FromTemplate(string template) { - var deserializer = new DeserializerBuilder().Build(); + this._config = KernelFunctionYaml.ToPromptTemplateConfig(template); - var agentKernelModel = deserializer.Deserialize(template); + this.WithInstructions(this._config.Template.Trim()); - return - this - .WithInstructions(agentKernelModel.Instructions.Trim()) - .WithName(agentKernelModel.Name.Trim()) - .WithDescription(agentKernelModel.Description.Trim()); + if (!string.IsNullOrWhiteSpace(this._config.Name)) + { + this.WithName(this._config.Name?.Trim()); + } + + if (!string.IsNullOrWhiteSpace(this._config.Description)) + { + this.WithDescription(this._config.Description?.Trim()); + } + + return this; } /// diff --git a/dotnet/src/Experimental/Agents/AgentPlugin.cs b/dotnet/src/Experimental/Agents/AgentPlugin.cs index ae84500d9563..b11deeccab6c 100644 --- a/dotnet/src/Experimental/Agents/AgentPlugin.cs +++ b/dotnet/src/Experimental/Agents/AgentPlugin.cs @@ -29,8 +29,23 @@ protected AgentPlugin(string name, string? description = null) /// The agent response public async Task InvokeAsync(string input, CancellationToken cancellationToken = default) { - var args = new KernelArguments { { "input", input } }; - var result = await this.First().InvokeAsync(this.Agent.Kernel, args, cancellationToken).ConfigureAwait(false); + return await this.InvokeAsync(input, arguments: null, cancellationToken).ConfigureAwait(false); + } + + /// + /// Invoke plugin with user input + /// + /// The user input + /// The arguments + /// A cancel token + /// The agent response + public async Task InvokeAsync(string input, KernelArguments? arguments, CancellationToken cancellationToken = default) + { + arguments ??= new KernelArguments(); + + arguments["input"] = input; + + var result = await this.First().InvokeAsync(this.Agent.Kernel, arguments, cancellationToken).ConfigureAwait(false); var response = result.GetValue()!; return response.Message; diff --git a/dotnet/src/Experimental/Agents/Experimental.Agents.csproj b/dotnet/src/Experimental/Agents/Experimental.Agents.csproj index e6ce4983d8cb..3496b3afaf5c 100644 --- a/dotnet/src/Experimental/Agents/Experimental.Agents.csproj +++ b/dotnet/src/Experimental/Agents/Experimental.Agents.csproj @@ -19,10 +19,11 @@ - + + diff --git a/dotnet/src/Experimental/Agents/IAgent.cs b/dotnet/src/Experimental/Agents/IAgent.cs index 3c72958415e8..698f944d1ad3 100644 --- a/dotnet/src/Experimental/Agents/IAgent.cs +++ b/dotnet/src/Experimental/Agents/IAgent.cs @@ -64,6 +64,11 @@ public interface IAgent /// public AgentPlugin AsPlugin(); + /// + /// Expose the agent internally as a prompt-template + /// + internal IPromptTemplate AsPromptTemplate(); + /// /// Creates a new agent chat thread. /// diff --git a/dotnet/src/Experimental/Agents/IAgentExtensions.cs b/dotnet/src/Experimental/Agents/IAgentExtensions.cs index d939813d23e0..5488fc91f663 100644 --- a/dotnet/src/Experimental/Agents/IAgentExtensions.cs +++ b/dotnet/src/Experimental/Agents/IAgentExtensions.cs @@ -16,17 +16,19 @@ public static class IAgentExtensions /// /// the agent /// the user input - /// a cancel token - /// chat messages + /// Optional arguments for parameterized instructions + /// Optional cancellation token + /// Chat messages public static async IAsyncEnumerable InvokeAsync( this IAgent agent, string input, + KernelArguments? arguments = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IAgentThread thread = await agent.NewThreadAsync(cancellationToken).ConfigureAwait(false); try { - await foreach (var message in thread.InvokeAsync(agent, input, cancellationToken)) + await foreach (var message in thread.InvokeAsync(agent, input, arguments, cancellationToken)) { yield return message; } diff --git a/dotnet/src/Experimental/Agents/IAgentThread.cs b/dotnet/src/Experimental/Agents/IAgentThread.cs index f0188093e06f..a61b46e62b8f 100644 --- a/dotnet/src/Experimental/Agents/IAgentThread.cs +++ b/dotnet/src/Experimental/Agents/IAgentThread.cs @@ -28,18 +28,20 @@ public interface IAgentThread /// Advance the thread with the specified agent. /// /// An agent instance. + /// Optional arguments for parameterized instructions /// A cancellation token /// The resulting agent message(s) - IAsyncEnumerable InvokeAsync(IAgent agent, CancellationToken cancellationToken = default); + IAsyncEnumerable InvokeAsync(IAgent agent, KernelArguments? arguments = null, CancellationToken cancellationToken = default); /// /// Advance the thread with the specified agent. /// /// An agent instance. /// The user message + /// Optional arguments for parameterized instructions /// A cancellation token /// The resulting agent message(s) - IAsyncEnumerable InvokeAsync(IAgent agent, string userMessage, CancellationToken cancellationToken = default); + IAsyncEnumerable InvokeAsync(IAgent agent, string userMessage, KernelArguments? arguments = null, CancellationToken cancellationToken = default); /// /// Delete current thread. Terminal state - Unable to perform any diff --git a/dotnet/src/Experimental/Agents/Internal/Agent.cs b/dotnet/src/Experimental/Agents/Internal/Agent.cs index 8ef8c776aa01..40c65649041a 100644 --- a/dotnet/src/Experimental/Agents/Internal/Agent.cs +++ b/dotnet/src/Experimental/Agents/Internal/Agent.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Experimental.Agents.Exceptions; using Microsoft.SemanticKernel.Experimental.Agents.Models; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; namespace Microsoft.SemanticKernel.Experimental.Agents.Internal; @@ -50,9 +51,16 @@ internal sealed class Agent : IAgent public string Instructions => this._model.Instructions; private static readonly Regex s_removeInvalidCharsRegex = new("[^0-9A-Za-z-]"); + private static readonly Dictionary s_templateFactories = + new(StringComparer.OrdinalIgnoreCase) + { + { PromptTemplateConfig.SemanticKernelTemplateFormat, new KernelPromptTemplateFactory() }, + { HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, new HandlebarsPromptTemplateFactory() }, + }; private readonly OpenAIRestContext _restContext; private readonly AssistantModel _model; + private readonly IPromptTemplate _promptTemplate; private AgentPlugin? _agentPlugin; private bool _isDeleted; @@ -62,18 +70,20 @@ internal sealed class Agent : IAgent /// /// A context for accessing OpenAI REST endpoint /// The assistant definition + /// The template config /// Plugins to initialize as agent tools /// A cancellation token /// An initialized instance. public static async Task CreateAsync( OpenAIRestContext restContext, AssistantModel assistantModel, + PromptTemplateConfig? config, IEnumerable? plugins = null, CancellationToken cancellationToken = default) { var resultModel = await restContext.CreateAssistantModelAsync(assistantModel, cancellationToken).ConfigureAwait(false); - return new Agent(resultModel, restContext, plugins); + return new Agent(resultModel, config, restContext, plugins); } /// @@ -81,14 +91,24 @@ public static async Task CreateAsync( /// internal Agent( AssistantModel assistantModel, + PromptTemplateConfig? config, OpenAIRestContext restContext, IEnumerable? plugins = null) { + config ??= + new PromptTemplateConfig + { + Name = assistantModel.Name, + Description = assistantModel.Description, + Template = assistantModel.Instructions, + }; + this._model = assistantModel; this._restContext = restContext; + this._promptTemplate = this.DefinePromptTemplate(config); IKernelBuilder builder = Kernel.CreateBuilder(); - ; + this.Kernel = Kernel .CreateBuilder() @@ -103,6 +123,8 @@ internal Agent( public AgentPlugin AsPlugin() => this._agentPlugin ??= this.DefinePlugin(); + public IPromptTemplate AsPromptTemplate() => this._promptTemplate; + /// public Task NewThreadAsync(CancellationToken cancellationToken = default) { @@ -146,19 +168,19 @@ public async Task DeleteAsync(CancellationToken cancellationToken = default) /// Marshal thread run through interface. /// /// The user input + /// Arguments for parameterized instructions /// A cancellation token. /// An agent response ( private async Task AskAsync( [Description("The user message provided to the agent.")] string input, + KernelArguments arguments, CancellationToken cancellationToken = default) { var thread = await this.NewThreadAsync(cancellationToken).ConfigureAwait(false); try { - await thread.AddUserMessageAsync(input, cancellationToken).ConfigureAwait(false); - - var messages = await thread.InvokeAsync(this, cancellationToken).ToArrayAsync(cancellationToken).ConfigureAwait(false); + var messages = await thread.InvokeAsync(this, input, arguments, cancellationToken).ToArrayAsync(cancellationToken).ConfigureAwait(false); var response = new AgentResponse { @@ -181,6 +203,16 @@ private AgentPluginImpl DefinePlugin() return new AgentPluginImpl(this, functionAsk); } + private IPromptTemplate DefinePromptTemplate(PromptTemplateConfig config) + { + if (!s_templateFactories.TryGetValue(config.TemplateFormat, out var factory)) + { + factory = new KernelPromptTemplateFactory(); + } + + return factory.Create(config); + } + private void ThrowIfDeleted() { if (this._isDeleted) diff --git a/dotnet/src/Experimental/Agents/Internal/ChatThread.cs b/dotnet/src/Experimental/Agents/Internal/ChatThread.cs index 97694ddbaec9..fda2749ee0bf 100644 --- a/dotnet/src/Experimental/Agents/Internal/ChatThread.cs +++ b/dotnet/src/Experimental/Agents/Internal/ChatThread.cs @@ -65,13 +65,13 @@ await this._restContext.CreateUserTextMessageAsync( } /// - public IAsyncEnumerable InvokeAsync(IAgent agent, CancellationToken cancellationToken) + public IAsyncEnumerable InvokeAsync(IAgent agent, KernelArguments? arguments = null, CancellationToken cancellationToken = default) { - return this.InvokeAsync(agent, string.Empty, cancellationToken); + return this.InvokeAsync(agent, string.Empty, arguments, cancellationToken); } /// - public async IAsyncEnumerable InvokeAsync(IAgent agent, string userMessage, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable InvokeAsync(IAgent agent, string userMessage, KernelArguments? arguments = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { this.ThrowIfDeleted(); @@ -80,9 +80,15 @@ public async IAsyncEnumerable InvokeAsync(IAgent agent, string use yield return await this.AddUserMessageAsync(userMessage, cancellationToken).ConfigureAwait(false); } + // Define tools as part of the run definition, since there's no enforcement that an agent + // is initialized with the same tools every time. var tools = agent.Plugins.SelectMany(p => p.Select(f => f.ToToolModel(p.Name))); - var runModel = await this._restContext.CreateRunAsync(this.Id, agent.Id, agent.Instructions, tools, cancellationToken).ConfigureAwait(false); + // Finalize prompt / agent instructions using provided parameters. + var instructions = await agent.AsPromptTemplate().RenderAsync(agent.Kernel, arguments, cancellationToken).ConfigureAwait(false); + + // Create run using templated prompt + var runModel = await this._restContext.CreateRunAsync(this.Id, agent.Id, instructions, tools, cancellationToken).ConfigureAwait(false); var run = new ChatRun(runModel, agent.Kernel, this._restContext); var results = await run.GetResultAsync(cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Experimental/Agents/Models/AgentConfigurationModel.cs b/dotnet/src/Experimental/Agents/Models/AgentConfigurationModel.cs deleted file mode 100644 index 412904ca1c03..000000000000 --- a/dotnet/src/Experimental/Agents/Models/AgentConfigurationModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -#pragma warning disable CA1812 - -using YamlDotNet.Serialization; - -namespace Microsoft.SemanticKernel.Experimental.Agents.Models; - -/// -/// Represents a yaml configuration file for an agent. -/// -internal sealed class AgentConfigurationModel -{ - /// - /// The agent name - /// - [YamlMember(Alias = "name")] - public string Name { get; set; } = string.Empty; - - /// - /// The agent description - /// - [YamlMember(Alias = "description")] - public string Description { get; set; } = string.Empty; - - /// - /// The agent instructions template - /// - [YamlMember(Alias = "instructions")] - public string Instructions { get; set; } = string.Empty; -} diff --git a/dotnet/src/Functions/Functions.Yaml/KernelFunctionYaml.cs b/dotnet/src/Functions/Functions.Yaml/KernelFunctionYaml.cs index 60b3802770c8..0c7039c5530f 100644 --- a/dotnet/src/Functions/Functions.Yaml/KernelFunctionYaml.cs +++ b/dotnet/src/Functions/Functions.Yaml/KernelFunctionYaml.cs @@ -27,12 +27,7 @@ public static KernelFunction FromPromptYaml( IPromptTemplateFactory? promptTemplateFactory = null, ILoggerFactory? loggerFactory = null) { - var deserializer = new DeserializerBuilder() - .WithNamingConvention(UnderscoredNamingConvention.Instance) - .WithNodeDeserializer(new PromptExecutionSettingsNodeDeserializer()) - .Build(); - - var promptTemplateConfig = deserializer.Deserialize(text); + PromptTemplateConfig promptTemplateConfig = ToPromptTemplateConfig(text); // Prevent the default value from being any type other than a string. // It's a temporary limitation that helps shape the public API surface @@ -53,4 +48,18 @@ public static KernelFunction FromPromptYaml( promptTemplateFactory, loggerFactory); } + + /// + /// Convert the given YAML text to a model. + /// + /// YAML representation of the to use to create the prompt function. + public static PromptTemplateConfig ToPromptTemplateConfig(string text) + { + var deserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .WithNodeDeserializer(new PromptExecutionSettingsNodeDeserializer()) + .Build(); + + return deserializer.Deserialize(text); + } }