.Net: Tuning HB Planner - CreatePlan prompt, template extraction, and type handling

dotnet/Directory.Packages.props

@@ -9,7 +9,7 @@
     <PackageVersion Include="Azure.Identity" Version="1.10.4" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.2.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
-    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.1.3" />
+    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.1.4" />
     <PackageVersion Include="Markdig" Version="0.34.0" />
     <PackageVersion Include="Handlebars.Net" Version="2.1.4" />
     <PackageVersion Include="JsonSchema.Net.Generation" Version="3.5.1" />

dotnet/src/Planners/Planners.Handlebars.UnitTests/Handlebars/HandlebarsPlannerTests.cs

@@ -193,6 +193,41 @@ public async Task ItOverridesPromptAsync()
         Assert.DoesNotContain("## Tips and reminders", plan.Prompt, StringComparison.CurrentCulture);
     }
 
+    [Fact]
+    public async Task ItThrowsIfStrictlyOnePlanCantBeIdentifiedAsync()
+    {
+        // Arrange
+        var ResponseWithMultipleHbTemplates =
+    @"```handlebars
+{{!-- Step 1: Call Summarize function --}}  
+{{set ""summary"" (SummarizePlugin-Summarize)}}  
+```
+
+```handlebars
+{{!-- Step 2: Call Translate function with the language set to French --}}  
+{{set ""translatedSummary"" (WriterPlugin-Translate language=""French"" input=(get ""summary""))}}  
+```
+
+```handlebars
+{{!-- Step 3: Call GetEmailAddress function with input set to John Doe --}}  
+{{set ""emailAddress"" (email-GetEmailAddress input=""John Doe"")}}  
+
+{{!-- Step 4: Call SendEmail function with input set to the translated summary and email_address set to the retrieved email address --}}  
+{{email-SendEmail input=(get ""translatedSummary"") email_address=(get ""emailAddress"")}}
+```
+
+```handlebars
+{{!-- Step 4: Call SendEmail function with input set to the translated summary and email_address set to the retrieved email address --}}  
+{{email-SendEmail input=(get ""translatedSummary"") email_address=(get ""emailAddress"")}}
+```";
+        var kernel = this.CreateKernelWithMockCompletionResult(ResponseWithMultipleHbTemplates);
+        var planner = new HandlebarsPlanner();
+
+        // Act & Assert
+        var exception = await Assert.ThrowsAsync<KernelException>(async () => await planner.CreatePlanAsync(kernel, "goal"));
+        Assert.True(exception?.Message?.Contains("Identified multiple Handlebars templates in model response", StringComparison.InvariantCulture));
+    }
+
     private Kernel CreateKernelWithMockCompletionResult(string testPlanString, KernelPluginCollection? plugins = null)
     {
         plugins ??= new KernelPluginCollection();

dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/Helpers/CustomHelpers/CustomHelpers.handlebars

@@ -12,9 +12,9 @@ Description: {{Description}}
 Inputs:
   {{#each Parameters}}
     - {{Name}}:
-    {{~#if ParameterType}} {{ParameterType.Name}} -
+    {{~#if Schema}} {{getSchemaTypeName this}} -
     {{~else}}
-        {{~#if Schema}} {{getSchemaTypeName this}} -{{/if}}
+        {{~#if ParameterType}} {{ParameterType.Name}} -{{/if}}
     {{~/if}}
     {{~#if Description}} {{Description}}{{/if}}
     {{~#if IsRequired}} (required){{else}} (optional){{/if}}

dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars

@@ -16,6 +16,7 @@ Follow these steps to create one Handlebars template to achieve the goal:
 1. Choose the Right Helpers:
   - Use the provided helpers to manipulate the variables you've created. Start with the basic helpers and only use custom helpers if necessary to accomplish the goal.
   - Always reference a custom helper by its full name.
+  - Be careful with syntax, i.e., Always reference a custom helper by its full name and remember to use a `#` for all block helpers.
 2. Don't Create or Assume Unlisted Helpers:
   - Only use the helpers provided. Any helper not listed is considered hallucinated and must not be used.
   - Do not invent or assume the existence of any functions not explicitly defined above.
@@ -27,7 +28,7 @@ Follow these steps to create one Handlebars template to achieve the goal:
 5. No Nested Helpers:
   - Do not nest helpers or conditionals inside other helpers. This can cause errors in the template.
 6. Output the Result:
-  - Once you have completed the necessary steps to reach the goal, use the `\{{json}}` helper to output the final result.
+  - Once you have completed the necessary steps to reach the goal, use the `\{{json}}` helper and print only your final template.
   - Ensure your template and all steps are enclosed in a ``` handlebars block.
 
 Remember, the objective is not to use all the helpers available, but to use the correct ones to achieve the desired outcome with a clear and concise template.

dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs

@@ -89,13 +89,21 @@ private async Task<HandlebarsPlan> CreatePlanCoreAsync(Kernel kernel, string goa
             var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
             modelResults = await chatCompletionService.GetChatMessageContentAsync(chatMessages, executionSettings: this._options.ExecutionSettings, cancellationToken: cancellationToken).ConfigureAwait(false);
 
-            Match match = Regex.Match(modelResults.Content, @"```\s*(handlebars)?\s*(.*)\s*```", RegexOptions.Singleline);
-            if (!match.Success)
+            // Regex breakdown:
+            // (```\s*handlebars){1}\s*: Opening backticks, starting boundary for HB template
+            // ((([^`]|`(?!``))+): Any non-backtick character or one backtick character not followed by 2 more consecutive backticks
+            // (\s*```){1}: Closing backticks, closing boundary for HB template
+            MatchCollection matches = Regex.Matches(modelResults.Content, @"(```\s*handlebars){1}\s*(([^`]|`(?!``))+)(\s*```){1}", RegexOptions.Multiline);
+            if (matches.Count < 1)
             {
-                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results.");
+                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results. Additional helpers or input may be required.\n\nPlanner output:\n{modelResults.Content}");
+            }
+            else if (matches.Count > 1)
+            {
+                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Identified multiple Handlebars templates in model response. Please try again.\n\nPlanner output:\n{modelResults.Content}");
             }
 
-            var planTemplate = match.Groups[2].Value.Trim();
+            var planTemplate = matches[0].Groups[2].Value.Trim();
             planTemplate = MinifyHandlebarsTemplate(planTemplate);
 
             return new HandlebarsPlan(planTemplate, createPlanPrompt);
@@ -153,7 +161,26 @@ private KernelParameterMetadata SetComplexTypeDefinition(
         HashSet<HandlebarsParameterTypeMetadata> complexParameterTypes,
         Dictionary<string, string> complexParameterSchemas)
     {
-        // TODO (@teresaqhoang): Handle case when schema and ParameterType can exist i.e., when ParameterType = RestApiResponse
+        if (parameter.Schema is not null)
+        {
+            // Class types will have a defined schema, but we want to handle those as built-in complex types below
+            if (parameter.ParameterType is not null && parameter.ParameterType!.IsClass)
+            {
+                parameter = new(parameter) { Schema = null };
+            }
+            else
+            {
+                // Parse the schema to extract any primitive types and set in ParameterType property instead
+                var parsedParameter = parameter.ParseJsonSchema();
+                if (parsedParameter.Schema is not null)
+                {
+                    complexParameterSchemas[parameter.GetSchemaTypeName()] = parameter.Schema.RootElement.ToJsonString();
+                }
+
+                return parsedParameter;
+            }
+        }
+
         if (parameter.ParameterType is not null)
         {
             // Async return type - need to extract the actual return type and override ParameterType property
@@ -165,17 +192,6 @@ private KernelParameterMetadata SetComplexTypeDefinition(
 
             complexParameterTypes.UnionWith(parameter.ParameterType!.ToHandlebarsParameterTypeMetadata());
         }
-        else if (parameter.Schema is not null)
-        {
-            // Parse the schema to extract any primitive types and set in ParameterType property instead
-            var parsedParameter = parameter.ParseJsonSchema();
-            if (parsedParameter.Schema is not null)
-            {
-                complexParameterSchemas[parameter.GetSchemaTypeName()] = parameter.Schema.RootElement.ToJsonString();
-            }
-
-            parameter = parsedParameter;
-        }
 
         return parameter;
     }