From 484b39d144c472f4aa23e310ed3fae6d01f58689 Mon Sep 17 00:00:00 2001 From: Joowon Date: Fri, 28 Jul 2023 09:08:31 +0900 Subject: [PATCH] Python: Add sequential planner to python (#1916) ### Motivation and Context fixes #1915 ### Description ### Contribution Checklist - [ ] The code builds clean without any errors or warnings - [ ] The PR follows SK Contribution Guidelines (https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) - [ ] The code follows the .NET coding conventions (https://learn.microsoft.com/dotnet/csharp/fundamentals/coding-style/coding-conventions) verified with `dotnet format` - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone :smile: --------- Co-authored-by: Abby Harrison Co-authored-by: Abby Harrison <54643756+awharrison-28@users.noreply.github.com> Co-authored-by: Shawn Callegari <36091529+shawncal@users.noreply.github.com> --- .vscode/launch.json | 4 +- .../sequential_planner.py | 47 +++ .../semantic_kernel/core_skills/math_skill.py | 1 + .../memory/semantic_text_memory_base.py | 1 + python/semantic_kernel/planning/__init__.py | 2 + python/semantic_kernel/planning/plan.py | 63 ++-- .../planning/planning_exception.py | 46 +++ .../Skills/SequentialPlanning/config.json | 27 ++ .../Skills/SequentialPlanning/skprompt.txt | 55 +++ .../planning/sequential_planner/__init__.py | 7 + .../sequential_planner/sequential_planner.py | 159 ++++++++ .../sequential_planner_config.py | 27 ++ .../sequential_planner_extensions.py | 230 ++++++++++++ .../sequential_planner_parser.py | 135 +++++++ .../integration/fakes/email_skill_fake.py | 25 ++ .../tests/integration/fakes/fun_skill_fake.py | 15 + .../integration/fakes/summarize_skill_fake.py | 16 + .../integration/fakes/writer_skill_fake.py | 24 ++ .../test_sequential_plan_parser.py | 72 ++++ .../test_sequential_planner.py | 147 ++++++++ .../test_sequential_planner.py | 148 ++++++++ .../test_sequential_planner_extensions.py | 254 +++++++++++++ .../test_sequential_planner_parser.py | 349 ++++++++++++++++++ 23 files changed, 1829 insertions(+), 25 deletions(-) create mode 100644 python/samples/kernel-syntax-examples/sequential_planner.py create mode 100644 python/semantic_kernel/planning/planning_exception.py create mode 100644 python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json create mode 100644 python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt create mode 100644 python/semantic_kernel/planning/sequential_planner/__init__.py create mode 100644 python/semantic_kernel/planning/sequential_planner/sequential_planner.py create mode 100644 python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py create mode 100644 python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py create mode 100644 python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py create mode 100644 python/tests/integration/fakes/email_skill_fake.py create mode 100644 python/tests/integration/fakes/fun_skill_fake.py create mode 100644 python/tests/integration/fakes/summarize_skill_fake.py create mode 100644 python/tests/integration/fakes/writer_skill_fake.py create mode 100644 python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py create mode 100644 python/tests/integration/planning/sequential_planner/test_sequential_planner.py create mode 100644 python/tests/unit/planning/sequential_planner/test_sequential_planner.py create mode 100644 python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py create mode 100644 python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 83c63b4f5199..c9bdc67c69b6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -36,12 +36,12 @@ "request": "attach" }, { - "cwd":"${workspaceFolder}/python", + "cwd": "${workspaceFolder}/python", "name": "Python: Test Module", "type": "python", "request": "launch", "module": "pytest", - "args": ["${file}"], + "args": ["${file}"] } ] } diff --git a/python/samples/kernel-syntax-examples/sequential_planner.py b/python/samples/kernel-syntax-examples/sequential_planner.py new file mode 100644 index 000000000000..58a5a0c9dbde --- /dev/null +++ b/python/samples/kernel-syntax-examples/sequential_planner.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft. All rights reserved. + +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion +from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill +from semantic_kernel.planning import SequentialPlanner + + +async def main(): + kernel = sk.Kernel() + api_key, org_id = sk.openai_settings_from_dot_env() + + kernel.add_chat_service( + "gpt-3.5", OpenAIChatCompletion("gpt-3.5-turbo", api_key, org_id) + ) + kernel.import_skill(MathSkill(), "math") + kernel.import_skill(FileIOSkill(), "fileIO") + kernel.import_skill(TimeSkill(), "time") + kernel.import_skill(TextSkill(), "text") + + # create an instance of sequential planner. + planner = SequentialPlanner(kernel) + + # the ask for which the sequential planner is going to find a relevant function. + ask = "What day of the week is today, all uppercase?" + + # ask the sequential planner to identify a suitable function from the list of functions available. + plan = await planner.create_plan_async(goal=ask) + + # ask the sequential planner to execute the identified function. + result = await plan.invoke_async() + + for step in plan._steps: + print(step.description, ":", step._state.__dict__) + + print("Expected Answer:") + print(result) + """ + Output: + 1100 + """ + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/python/semantic_kernel/core_skills/math_skill.py b/python/semantic_kernel/core_skills/math_skill.py index 9c17ddaa9347..6b8de44342ed 100644 --- a/python/semantic_kernel/core_skills/math_skill.py +++ b/python/semantic_kernel/core_skills/math_skill.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. + from semantic_kernel.orchestration.sk_context import SKContext from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter diff --git a/python/semantic_kernel/memory/semantic_text_memory_base.py b/python/semantic_kernel/memory/semantic_text_memory_base.py index 31b7393f62f0..cdb6a41d8ed8 100644 --- a/python/semantic_kernel/memory/semantic_text_memory_base.py +++ b/python/semantic_kernel/memory/semantic_text_memory_base.py @@ -59,6 +59,7 @@ async def get_async( self, collection: str, query: str, + # TODO: with_embedding: bool, ) -> Optional[MemoryQueryResult]: """Get information from the memory (calls the memory store's get method). diff --git a/python/semantic_kernel/planning/__init__.py b/python/semantic_kernel/planning/__init__.py index 17add29ee104..79ed95e2467f 100644 --- a/python/semantic_kernel/planning/__init__.py +++ b/python/semantic_kernel/planning/__init__.py @@ -1,7 +1,9 @@ from semantic_kernel.planning.basic_planner import BasicPlanner from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.sequential_planner import SequentialPlanner __all__ = [ + "SequentialPlanner", "BasicPlanner", "Plan", ] diff --git a/python/semantic_kernel/planning/plan.py b/python/semantic_kernel/planning/plan.py index d20217b4809d..5016cddfddb9 100644 --- a/python/semantic_kernel/planning/plan.py +++ b/python/semantic_kernel/planning/plan.py @@ -4,7 +4,7 @@ import re import threading from logging import Logger -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List, Optional, Union from semantic_kernel import Kernel from semantic_kernel.connectors.ai import CompleteRequestSettings @@ -20,6 +20,7 @@ from semantic_kernel.skill_definition.read_only_skill_collection_base import ( ReadOnlySkillCollectionBase, ) +from semantic_kernel.utils.null_logger import NullLogger class Plan(SKFunctionBase): @@ -112,6 +113,16 @@ def __init__( if function is not None: self.set_function(function) + @classmethod + def from_goal(cls, goal: str) -> "Plan": + return cls(description=goal, skill_name=cls.__name__) + + @classmethod + def from_function(cls, function: SKFunctionBase) -> "Plan": + plan = cls() + plan.set_function(function) + return plan + async def invoke_async( self, input: Optional[str] = None, @@ -121,7 +132,7 @@ async def invoke_async( logger: Optional[Logger] = None, # TODO: cancellation_token: CancellationToken, ) -> SKContext: - if input is not None: + if input is not None and input != "": self._state.update(input) if context is None: @@ -129,7 +140,7 @@ async def invoke_async( variables=self._state, skill_collection=None, memory=memory, - logger=logger, + logger=logger if logger is not None else NullLogger(), ) if self._function is not None: @@ -138,8 +149,8 @@ async def invoke_async( ) if result.error_occurred: result.log.error( - msg="Something went wrong in plan step {0}.{1}:'{2}'".format( - self._skill_name, self._name, context.last_error_description + "Something went wrong in plan step {0}.{1}:'{2}'".format( + self._skill_name, self._name, result.last_error_description ) ) return result @@ -162,7 +173,7 @@ def invoke( memory: Optional[SemanticTextMemoryBase] = None, logger: Optional[Logger] = None, ) -> SKContext: - if input is not None: + if input is not None and input != "": self._state.update(input) if context is None: @@ -244,7 +255,7 @@ def set_available_functions(self, plan: "Plan", context: SKContext) -> "Plan": return plan - def add_steps(self, steps: Optional[List[SKFunctionBase]]) -> None: + def add_steps(self, steps: Union[List["Plan"], List[SKFunctionBase]]) -> None: for step in steps: if type(step) is Plan: self._steps.append(step) @@ -344,7 +355,7 @@ def update_context_with_outputs(self, context: SKContext) -> None: context.variables.update(result_string) for item in self._steps[self._next_step_index - 1]._outputs: - if item in self._state: + if self._state.contains_key(item): context.variables.set(item, self._state[item]) else: context.variables.set(item, result_string) @@ -361,17 +372,18 @@ def get_next_step_variables( # - Empty if sending to another plan # - Plan.Description input_string = "" - if step._parameters["input"] is not None: - input_string = self.expand_from_variables( - variables, step._parameters["input"] - ) - elif variables["input"] is not None: - input_string = variables["input"] - elif self._state["input"] is not None: - input_string = self._state["input"] + step_input_exists, step_input_value = step._parameters.get("input") + variables_input_exists, variables_input_value = variables.get("input") + state_input_exists, state_input_value = self._state.get("input") + if step_input_exists and step_input_value != "": + input_string = self.expand_from_variables(variables, step_input_value) + elif variables_input_exists and variables_input_value != "": + input_string = variables_input_value + elif state_input_exists and state_input_value != "": + input_string = state_input_value elif len(step._steps) > 0: input_string = "" - elif self._description is not None: + elif self._description is not None and self._description != "": input_string = self._description step_variables = ContextVariables(input_string) @@ -379,15 +391,16 @@ def get_next_step_variables( # Priority for remaining stepVariables is: # - Function Parameters (pull from variables or state by a key value) # - Step Parameters (pull from variables or state by a key value) + # - All other variables. These are carried over in case the function wants access to the ambient content. function_params = step.describe() for param in function_params._parameters: - if param.name.lower == "input": + if param.name.lower() == variables._main_key.lower(): continue - if step_variables.contains_key(param.name): + + if variables.contains_key(param.name): step_variables.set(param.name, variables[param.name]) - elif ( - self._state.contains_key(param.name) - and self._state[param.name] is not None + elif self._state.contains_key(param.name) and ( + self._state[param.name] is not None and self._state[param.name] != "" ): step_variables.set(param.name, self._state[param.name]) @@ -405,6 +418,10 @@ def get_next_step_variables( else: step_variables.set(param_var, expanded_value) + for item in variables._variables: + if not step_variables.contains_key(item): + step_variables.set(item, variables[item]) + return step_variables def expand_from_variables( @@ -412,7 +429,7 @@ def expand_from_variables( ) -> str: result = input_string variables_regex = r"\$(?P\w+)" - matches = re.findall(variables_regex, input_string) + matches = [m for m in re.finditer(variables_regex, input_string)] ordered_matches = sorted( matches, key=lambda m: len(m.group("var")), reverse=True ) diff --git a/python/semantic_kernel/planning/planning_exception.py b/python/semantic_kernel/planning/planning_exception.py new file mode 100644 index 000000000000..7cf72a10f356 --- /dev/null +++ b/python/semantic_kernel/planning/planning_exception.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft. All rights reserved. + +from enum import Enum +from typing import Optional + + +class PlanningException(Exception): + class ErrorCodes(Enum): + # Unknown error. + UnknownError = -1 + # Invalid goal. + InvalidGoal = 0 + # Invalid plan. + InvalidPlan = 1 + # Invalid configuration. + InvalidConfiguration = 2 + # Create plan error. + CreatePlanError = 3 + + # The error code. + _error_code: ErrorCodes + + def __init__( + self, + error_code: ErrorCodes, + message: str, + inner_exception: Optional[Exception] = None, + ) -> None: + """Initializes a new instance of the PlanningError class. + + Arguments: + error_code {ErrorCodes} -- The error code. + message {str} -- The error message. + inner_exception {Exception} -- The inner exception. + """ + super().__init__(error_code, message, inner_exception) + self._error_code = error_code + + @property + def error_code(self) -> ErrorCodes: + """Gets the error code. + + Returns: + ErrorCodes -- The error code. + """ + return self._error_code diff --git a/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json new file mode 100644 index 000000000000..1309f85b5a1a --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/config.json @@ -0,0 +1,27 @@ +{ + "schema": 1, + "description": "Given a request or command or goal generate a step by step plan to fulfill the request using functions. This ability is also known as decision making and function flow", + "type": "completion", + "completion": { + "max_tokens": 1024, + "temperature": 0, + "top_p": 0, + "presence_penalty": 0, + "frequency_penalty": 0, + "stop_sequences": [""] + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "The question to answer", + "defaultValue": "" + }, + { + "name": "available_functions", + "description": "The list of the agent's available_functions", + "defaultValue": "" + } + ] + } +} diff --git a/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt new file mode 100644 index 000000000000..4e325abfe8bf --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/Skills/SequentialPlanning/skprompt.txt @@ -0,0 +1,55 @@ +Create an XML plan step by step, to satisfy the goal given, with the available functions. + +[AVAILABLE FUNCTIONS] + +{{$available_functions}} + +[END AVAILABLE FUNCTIONS] + +To create a plan, follow these steps: +0. The plan should be as short as possible. +1. From a create a as a series of . +2. A plan has 'INPUT' available in context variables by default. +3. Before using any function in a plan, check that it is present in the [AVAILABLE FUNCTIONS] list. If it is not, do not use it. +4. Only use functions that are required for the given goal. +5. Append an "END" XML comment at the end of the plan after the final closing tag. +6. Always output valid XML that can be parsed by an XML parser. +7. If a plan cannot be created with the [AVAILABLE FUNCTIONS], return . + +All plans take the form of: + + + + + + + + (... etc ...) + + + +To call a function, follow these steps: +1. A function has one or more named parameters and a single 'output' which are all strings. Parameter values should be xml escaped. +2. To save an 'output' from a , to pass into a future , use +3. To save an 'output' from a , to return as part of a plan result, use +4. Use a '$' to reference a context variable in a parameter, e.g. when `INPUT='world'` the parameter 'Hello $INPUT' will evaluate to `Hello world`. +5. Functions do not have access to the context variables of other functions. Do not attempt to use context variables as arrays or objects. Instead, use available functions to extract specific elements or properties from context variables. + +DO NOT DO THIS, THE PARAMETER VALUE IS NOT XML ESCAPED: + + +DO NOT DO THIS, THE PARAMETER VALUE IS ATTEMPTING TO USE A CONTEXT VARIABLE AS AN ARRAY/OBJECT: + + +Here is a valid example of how to call a function "_Function_.Name" with a single input and save its output: + + +Here is a valid example of how to call a function "FunctionName2" with a single input and return its output as part of the plan result: + + +Here is a valid example of how to call a function "Name3" with multiple inputs: + + +Begin! + +{{$input}} diff --git a/python/semantic_kernel/planning/sequential_planner/__init__.py b/python/semantic_kernel/planning/sequential_planner/__init__.py new file mode 100644 index 000000000000..1c06b014bcae --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/__init__.py @@ -0,0 +1,7 @@ +from semantic_kernel.planning.sequential_planner.sequential_planner import ( + SequentialPlanner, +) + +__all__ = [ + "SequentialPlanner", +] diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner.py new file mode 100644 index 000000000000..edb850f88fdc --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os +from typing import TYPE_CHECKING + +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_extensions import ( + SequentialPlannerSKContextExtension as SKContextExtension, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from semantic_kernel.semantic_functions.prompt_template import PromptTemplate +from semantic_kernel.semantic_functions.prompt_template_config import ( + PromptTemplateConfig, +) +from semantic_kernel.semantic_functions.semantic_function_config import ( + SemanticFunctionConfig, +) + +if TYPE_CHECKING: + from semantic_kernel.orchestration.sk_context import SKContext + from semantic_kernel.orchestration.sk_function_base import SKFunctionBase + +SEQUENTIAL_PLANNER_DEFAULT_DESCRIPTION = ( + "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow" +) + +CUR_DIR = os.path.dirname(os.path.realpath(__file__)) +PROMPT_CONFIG_FILE_PATH = os.path.join(CUR_DIR, "Skills/SequentialPlanning/config.json") +PROMPT_TEMPLATE_FILE_PATH = os.path.join( + CUR_DIR, "Skills/SequentialPlanning/skprompt.txt" +) + + +def read_file(file_path: str) -> str: + with open(file_path, "r") as file: + return file.read() + + +class SequentialPlanner: + RESTRICTED_SKILL_NAME = "SequentialPlanner_Excluded" + + config: SequentialPlannerConfig + _context: "SKContext" + _function_flow_function: "SKFunctionBase" + + def __init__( + self, kernel: Kernel, config: SequentialPlannerConfig = None, prompt: str = None + ): + assert isinstance(kernel, Kernel) + self.config = config or SequentialPlannerConfig() + + self.config.excluded_skills.append(self.RESTRICTED_SKILL_NAME) + + self._function_flow_function = self._init_flow_function(prompt, kernel) + + self._context = kernel.create_new_context() + + def _init_flow_function(self, prompt: str, kernel: Kernel): + prompt_config = PromptTemplateConfig.from_json( + read_file(PROMPT_CONFIG_FILE_PATH) + ) + prompt_template = prompt or read_file(PROMPT_TEMPLATE_FILE_PATH) + prompt_config.completion.max_tokens = self.config.max_tokens + + prompt_template = PromptTemplate( + template=prompt_template, + template_engine=kernel.prompt_template_engine, + prompt_config=prompt_config, + ) + function_config = SemanticFunctionConfig(prompt_config, prompt_template) + + return kernel.register_semantic_function( + skill_name=self.RESTRICTED_SKILL_NAME, + function_name=self.RESTRICTED_SKILL_NAME, + function_config=function_config, + ) + + async def create_plan_async(self, goal: str) -> Plan: + if len(goal) == 0: + raise PlanningException( + PlanningException.ErrorCodes.InvalidGoal, "The goal specified is empty" + ) + + relevant_function_manual = await SKContextExtension.get_functions_manual_async( + self._context, goal, self.config + ) + self._context.variables.set("available_functions", relevant_function_manual) + + self._context.variables.update(goal) + + plan_result = await self._function_flow_function.invoke_async( + context=self._context + ) + + if plan_result.error_occurred: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + f"Error creating plan for goal: {plan_result.last_error_description}", + plan_result.last_exception, + ) + + plan_result_string = plan_result.result.strip() + + try: + get_skill_function = ( + self.config.get_skill_function + or SequentialPlanParser.get_skill_function(self._context) + ) + plan = SequentialPlanParser.to_plan_from_xml( + plan_result_string, + goal, + get_skill_function, + self.config.allow_missing_functions, + ) + + if len(plan._steps) == 0: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + ( + "Not possible to create plan for goal with available functions.\n", + f"Goal:{goal}\nFunctions:\n{relevant_function_manual}", + ), + ) + + return plan + + except PlanningException as e: + if e.error_code == PlanningException.ErrorCodes.CreatePlanError: + raise e + elif e.error_code in [ + PlanningException.ErrorCodes.InvalidPlan, + PlanningException.ErrorCodes.InvalidGoal, + ]: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "Unable to create plan", + e, + ) + else: + raise PlanningException( + PlanningException.ErrorCodes.CreatePlanError, + "Unable to create plan", + e, + ) + + except Exception as e: + raise PlanningException( + PlanningException.ErrorCodes.UnknownError, + "Unknown error creating plan", + e, + ) diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py new file mode 100644 index 000000000000..5a223c4bdb8f --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_config.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import Callable, List, Optional + + +class SequentialPlannerConfig: + def __init__( + self, + relevancy_threshold: Optional[float] = None, + max_relevant_functions: int = 100, + excluded_skills: List[str] = None, + excluded_functions: List[str] = None, + included_functions: List[str] = None, + max_tokens: int = 1024, + allow_missing_functions: bool = False, + get_available_functions_async: Callable = None, + get_skill_function: Callable = None, + ): + self.relevancy_threshold: float = relevancy_threshold + self.max_relevant_functions: int = max_relevant_functions + self.excluded_skills: List[str] = excluded_skills or [] + self.excluded_functions: List[str] = excluded_functions or [] + self.included_functions: List[str] = included_functions or [] + self.max_tokens: int = max_tokens + self.allow_missing_functions: bool = allow_missing_functions + self.get_available_functions_async = get_available_functions_async + self.get_skill_function = get_skill_function diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py new file mode 100644 index 000000000000..c0460663ec18 --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_extensions.py @@ -0,0 +1,230 @@ +# Copyright (c) Microsoft. All rights reserved. + +import itertools +from typing import AsyncIterable, List + +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.memory.memory_query_result import MemoryQueryResult +from semantic_kernel.memory.null_memory import NullMemory +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.skill_definition.function_view import FunctionView + + +class SequentialPlannerFunctionViewExtension: + @staticmethod + def to_manual_string(function: FunctionView): + inputs = [ + f" - {parameter.name}: {parameter.description}" + + ( + f" (default value: {parameter.default_value})" + if parameter.default_value + else "" + ) + for parameter in function.parameters + ] + + inputs = "\n".join(inputs) + qualified_name = SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function + ) + + return f"{qualified_name}:\n description: {function.description}\n inputs:\n {inputs}" + + @staticmethod + def to_fully_qualified_name(function: FunctionView): + return f"{function.skill_name}.{function.name}" + + @staticmethod + def to_embedding_string(function: FunctionView): + inputs = "\n".join( + [ + f" - {parameter.name}: {parameter.description}" + for parameter in function.parameters + ] + ) + return f"{function.name}:\n description: {function.description}\n inputs:\n{inputs}" + + +class SequentialPlannerSKContextExtension: + PLANNER_MEMORY_COLLECTION_NAME = " Planning.SKFunctionManual" + PLAN_SK_FUNCTIONS_ARE_REMEMBERED = "Planning.SKFunctionsAreRemembered" + + @staticmethod + async def get_functions_manual_async( + context: SKContext, + semantic_query: str = None, + config: SequentialPlannerConfig = None, + ) -> str: + config = config or SequentialPlannerConfig() + + if config.get_available_functions_async is None: + functions = ( + await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + ) + else: + functions = await config.get_available_functions_async( + config, semantic_query + ) + + return "\n\n".join( + [ + SequentialPlannerFunctionViewExtension.to_manual_string(func) + for func in functions + ] + ) + + @staticmethod + async def get_available_functions_async( + context: SKContext, + config: SequentialPlannerConfig, + semantic_query: str = None, + ): + excluded_skills = config.excluded_skills or [] + excluded_functions = config.excluded_functions or [] + included_functions = config.included_functions or [] + + if context.skills is None: + raise KernelException( + KernelException.ErrorCodes.SkillCollectionNotSet, + "Skill collection not found in the context", + ) + + functions_view = context.skills.get_functions_view() + + available_functions: List[FunctionView] = [ + *functions_view._semantic_functions.values(), + *functions_view._native_functions.values(), + ] + available_functions = itertools.chain.from_iterable(available_functions) + + available_functions = [ + func + for func in available_functions + if ( + func.skill_name not in excluded_skills + and func.name not in excluded_functions + ) + ] + + if ( + semantic_query is None + or isinstance(context.memory, NullMemory) + or config.relevancy_threshold is None + ): + # If no semantic query is provided, return all available functions. + # If a Memory provider has not been registered, return all available functions. + return available_functions + + # Remember functions in memory so that they can be searched. + await SequentialPlannerSKContextExtension.remember_functions_async( + context, available_functions + ) + + # Search for functions that match the semantic query. + memories = await context.memory.search_async( + SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + semantic_query, + config.max_relevant_functions, + config.relevancy_threshold, + ) + + # Add functions that were found in the search results. + relevant_functions = ( + await SequentialPlannerSKContextExtension.get_relevant_functions_async( + context, available_functions, memories + ) + ) + + # Add any missing functions that were included but not found in the search results. + missing_functions = [ + func + for func in included_functions + if func not in [func.name for func in relevant_functions] + ] + + relevant_functions += [ + func for func in available_functions if func.name in missing_functions + ] + + return sorted(relevant_functions, key=lambda x: (x.skill_name, x.name)) + + @staticmethod + async def get_relevant_functions_async( + context: SKContext, + available_functions: List[FunctionView], + memories: AsyncIterable[MemoryQueryResult], + ) -> List[FunctionView]: + relevant_functions = [] + # TODO: cancellation + async for memory_entry in memories: + function = next( + ( + func + for func in available_functions + if SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + func + ) + == memory_entry.id + ), + None, + ) + if function is not None: + context.log.debug( + "Found relevant function. Relevance Score: {0}, Function: {1}".format( + memory_entry.relevance, + SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function + ), + ) + ) + relevant_functions.append(function) + + return relevant_functions + + @staticmethod + async def remember_functions_async( + context: SKContext, available_functions: List[FunctionView] + ): + # Check if the functions have already been saved to memory. + if context.variables.contains_key( + SequentialPlannerSKContextExtension.PLAN_SK_FUNCTIONS_ARE_REMEMBERED + ): + return + + for function in available_functions: + function_name = ( + SequentialPlannerFunctionViewExtension.to_fully_qualified_name(function) + ) + key = function_name + description = function.description or function_name + text_to_embed = SequentialPlannerFunctionViewExtension.to_embedding_string( + function + ) + + # It'd be nice if there were a saveIfNotExists method on the memory interface + memory_entry = await context.memory.get_async( + collection=SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + key=key, + with_embedding=False, + ) + if memory_entry is None: + # TODO It'd be nice if the minRelevanceScore could be a parameter for each item that was saved to memory + # As folks may want to tune their functions to be more or less relevant. + # Memory now supports these such strategies. + await context.memory.save_information_async( + collection=SequentialPlannerSKContextExtension.PLANNER_MEMORY_COLLECTION_NAME, + text=text_to_embed, + id=key, + description=description, + additional_metadata="", + ) + + # Set a flag to indicate that the functions have been saved to memory. + context.variables.set( + SequentialPlannerSKContextExtension.PLAN_SK_FUNCTIONS_ARE_REMEMBERED, "true" + ) diff --git a/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py b/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py new file mode 100644 index 000000000000..cea7747521f3 --- /dev/null +++ b/python/semantic_kernel/planning/sequential_planner/sequential_planner_parser.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft. All rights reserved. + +import re +from typing import Callable, Optional, Tuple +from xml.etree import ElementTree as ET + +from semantic_kernel.kernel_exception import KernelException +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.plan import Plan +from semantic_kernel.planning.planning_exception import PlanningException + +# Constants +GOAL_TAG = "goal" +SOLUTION_TAG = "plan" +FUNCTION_TAG = "function." +SET_CONTEXT_VARIABLE_TAG = "setContextVariable" +APPEND_TO_RESULT_TAG = "appendToResult" + + +class SequentialPlanParser: + @staticmethod + def get_skill_function( + context: SKContext, + ) -> Callable[[str, str], Optional[SKFunctionBase]]: + def function(skill_name: str, function_name: str) -> Optional[SKFunctionBase]: + try: + return context.skills.get_function(skill_name, function_name) + except KernelException: + return None + + return function + + @staticmethod + def to_plan_from_xml( + xml_string: str, + goal: str, + get_skill_function: Callable[[str, str], Optional[SKFunctionBase]], + allow_missing_functions: bool = False, + ): + xml_string = "" + xml_string + "" + try: + xml_doc = ET.fromstring(xml_string) + except ET.ParseError: + # Attempt to parse out of it + plan_regex = re.compile(r"]*>(.*?)", re.DOTALL) + match = plan_regex.search(xml_string) + + if match: + plan_xml = match.group(0) + try: + xml_doc = ET.fromstring("" + plan_xml + "") + except ET.ParseError: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to parse plan xml strings: '{xml_string}' or '{plan_xml}'", + ) + else: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to parse plan xml string: '{xml_string}'", + ) + + solution = xml_doc.findall(".//" + SOLUTION_TAG) + + plan = Plan.from_goal(goal) + for solution_node in solution: + for child_node in solution_node: + if child_node.tag == "#text" or child_node.tag == "#comment": + continue + + if child_node.tag.startswith(FUNCTION_TAG): + skill_function_name = child_node.tag.split(FUNCTION_TAG)[1] + ( + skill_name, + function_name, + ) = SequentialPlanParser.get_skill_function_names( + skill_function_name + ) + + if function_name: + skill_function = get_skill_function(skill_name, function_name) + + if skill_function is not None: + plan_step = Plan.from_function(skill_function) + + function_variables = ContextVariables() + function_outputs = [] + function_results = [] + + view = skill_function.describe() + for p in view.parameters: + function_variables.set(p.name, p.default_value) + + for attr in child_node.attrib: + if attr == SET_CONTEXT_VARIABLE_TAG: + function_outputs.append(child_node.attrib[attr]) + elif attr == APPEND_TO_RESULT_TAG: + function_outputs.append(child_node.attrib[attr]) + function_results.append(child_node.attrib[attr]) + else: + function_variables.set( + attr, child_node.attrib[attr] + ) + + plan_step._outputs = function_outputs + plan_step._parameters = function_variables + + for result in function_results: + plan._outputs.append(result) + + plan.add_steps([plan_step]) + elif allow_missing_functions: + plan.add_steps([Plan.from_goal(skill_function_name)]) + else: + raise PlanningException( + PlanningException.ErrorCodes.InvalidPlan, + f"Failed to find function '{skill_function_name}' in skill '{skill_name}'.", + ) + + return plan + + @staticmethod + def get_skill_function_names(skill_function_name: str) -> Tuple[str, str]: + skill_function_name_parts = skill_function_name.split(".") + skill_name = ( + skill_function_name_parts[0] if len(skill_function_name_parts) > 0 else "" + ) + function_name = ( + skill_function_name_parts[1] + if len(skill_function_name_parts) > 1 + else skill_function_name + ) + return skill_name, function_name diff --git a/python/tests/integration/fakes/email_skill_fake.py b/python/tests/integration/fakes/email_skill_fake.py new file mode 100644 index 000000000000..02ac2dc9a0bc --- /dev/null +++ b/python/tests/integration/fakes/email_skill_fake.py @@ -0,0 +1,25 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + + +class EmailSkillFake: + @sk_function( + description="Given an email address and message body, send an email", + name="SendEmail", + ) + def send_email(self, input: str) -> str: + return f"Sent email to: . Body: {input}" + + @sk_function( + description="Lookup an email address for a person given a name", + name="GetEmailAddress", + ) + def get_email_address(self, input: str) -> str: + if input == "": + return "johndoe1234@example.com" + return f"{input}@example.com" + + @sk_function(description="Write a short poem for an e-mail", name="WritePoem") + def write_poem(self, input: str) -> str: + return f"Roses are red, violets are blue, {input} is hard, so is this test." diff --git a/python/tests/integration/fakes/fun_skill_fake.py b/python/tests/integration/fakes/fun_skill_fake.py new file mode 100644 index 000000000000..034c4a0b2923 --- /dev/null +++ b/python/tests/integration/fakes/fun_skill_fake.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` +class FunSkillFake: + @sk_function( + description="Write a joke", + name="WriteJoke", + ) + def write_joke(self) -> str: + return "WriteJoke" diff --git a/python/tests/integration/fakes/summarize_skill_fake.py b/python/tests/integration/fakes/summarize_skill_fake.py new file mode 100644 index 000000000000..0d0857a55be4 --- /dev/null +++ b/python/tests/integration/fakes/summarize_skill_fake.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` + + +class SummarizeSkillFake: + @sk_function( + description="Summarize", + name="Summarize", + ) + def translate(self) -> str: + return "Summarize" diff --git a/python/tests/integration/fakes/writer_skill_fake.py b/python/tests/integration/fakes/writer_skill_fake.py new file mode 100644 index 000000000000..e370c80f4a78 --- /dev/null +++ b/python/tests/integration/fakes/writer_skill_fake.py @@ -0,0 +1,24 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.skill_definition.sk_function_decorator import sk_function + +# TODO: this fake skill is temporal usage. +# C# supports import skill from samples dir by using test helper and python should do the same +# `semantic-kernel/dotnet/src/IntegrationTests/TestHelpers.cs` + + +class WriterSkillFake: + @sk_function( + description="Translate", + name="Translate", + ) + def translate(self, language: str) -> str: + return f"Translate: {language}" + + @sk_function( + description="Write an outline for a novel", + name="NovelOutline", + input_default_value="", + ) + def write_novel_outline(self, input: str) -> str: + return f"Novel outline: {input}" diff --git a/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py b/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py new file mode 100644 index 000000000000..9f1c0371f178 --- /dev/null +++ b/python/tests/integration/planning/sequential_planner/test_sequential_plan_parser.py @@ -0,0 +1,72 @@ +# Copyright (c) Microsoft. All rights reserved. + +import pytest + +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from tests.integration.fakes.email_skill_fake import EmailSkillFake +from tests.integration.fakes.summarize_skill_fake import SummarizeSkillFake +from tests.integration.fakes.writer_skill_fake import WriterSkillFake + + +@pytest.mark.asyncio +async def test_can_call_to_plan_from_xml(get_aoai_config): + deployment_name, api_key, endpoint = get_aoai_config + + kernel = Kernel() + # Configure LLM service + kernel.add_text_completion_service( + "text_completion", + sk_oai.AzureChatCompletion(deployment_name, endpoint, api_key), + ) + kernel.import_skill(EmailSkillFake(), "email") + kernel.import_skill(SummarizeSkillFake(), "SummarizeSkill") + kernel.import_skill(WriterSkillFake(), "WriterSkill") + + plan_string = """ + + + + + + +""" + goal = "Summarize an input, translate to french, and e-mail to John Doe" + + plan = SequentialPlanParser.to_plan_from_xml( + plan_string, + goal, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + assert plan is not None + assert ( + plan.description + == "Summarize an input, translate to french, and e-mail to John Doe" + ) + + assert len(plan._steps) == 4 + step = plan._steps[0] + assert step.skill_name == "SummarizeSkill" + assert step.name == "Summarize" + + step = plan._steps[1] + assert step.skill_name == "WriterSkill" + assert step.name == "Translate" + assert step.parameters["language"] == "French" + assert "TRANSLATED_SUMMARY" in step._outputs + + step = plan._steps[2] + assert step.skill_name == "email" + assert step.name == "GetEmailAddress" + assert step.parameters["input"] == "John Doe" + assert "EMAIL_ADDRESS" in step._outputs + + step = plan._steps[3] + assert step.skill_name == "email" + assert step.name == "SendEmail" + assert step.parameters["input"] == "$TRANSLATED_SUMMARY" + assert step.parameters["email_address"] == "$EMAIL_ADDRESS" diff --git a/python/tests/integration/planning/sequential_planner/test_sequential_planner.py b/python/tests/integration/planning/sequential_planner/test_sequential_planner.py new file mode 100644 index 000000000000..eb10010a6eb8 --- /dev/null +++ b/python/tests/integration/planning/sequential_planner/test_sequential_planner.py @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft. All rights reserved. + +import pytest + +import semantic_kernel.connectors.ai.open_ai as sk_oai +from semantic_kernel.kernel import Kernel +from semantic_kernel.planning import SequentialPlanner +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from tests.integration.fakes.email_skill_fake import EmailSkillFake +from tests.integration.fakes.fun_skill_fake import FunSkillFake +from tests.integration.fakes.writer_skill_fake import WriterSkillFake + + +def initialize_kernel(get_aoai_config, use_embeddings=False, use_chat_model=False): + _, api_key, endpoint = get_aoai_config + + kernel = Kernel() + if use_chat_model: + kernel.add_chat_service( + "chat_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + else: + kernel.add_text_completion_service( + "text_completion", + sk_oai.AzureChatCompletion("gpt-35-turbo", endpoint, api_key), + ) + + if use_embeddings: + kernel.add_text_embedding_generation_service( + "text_embedding", + sk_oai.AzureTextEmbedding("text-embedding-ada-002", endpoint, api_key), + ) + return kernel + + +@pytest.mark.parametrize( + "use_chat_model, prompt, expected_function, expected_skill", + [ + ( + False, + "Write a joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ), + ( + True, + "Write a joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ), + ], +) +@pytest.mark.asyncio +async def test_create_plan_function_flow_async( + get_aoai_config, use_chat_model, prompt, expected_function, expected_skill +): + # Arrange + kernel = initialize_kernel(get_aoai_config, False, use_chat_model) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(FunSkillFake()) + + planner = SequentialPlanner(kernel) + + # Act + plan = await planner.create_plan_async(prompt) + + # Assert + assert any( + step.name == expected_function and step.skill_name == expected_skill + for step in plan._steps + ) + + +@pytest.mark.parametrize( + "prompt, expected_function, expected_skill, expected_default", + [ + ( + "Write a novel outline.", + "NovelOutline", + "WriterSkill", + "", + ) + ], +) +@pytest.mark.asyncio +async def test_create_plan_with_defaults_async( + get_aoai_config, prompt, expected_function, expected_skill, expected_default +): + # Arrange + kernel = initialize_kernel(get_aoai_config) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(WriterSkillFake(), "WriterSkill") + + planner = SequentialPlanner(kernel) + + # Act + plan = await planner.create_plan_async(prompt) + + # Assert + assert any( + step.name == expected_function + and step.skill_name == expected_skill + and step.parameters["input"] == expected_default + # TODO: current sk_function decorator only support default values ["input"] key + # TODO: current version of fake skills used inline sk_function but actually most of them already in samples dir. + # add test helper for python to import skills from samples dir. C# already has it. + # and step.parameters["endMarker"] == expected_default + for step in plan._steps + ) + + +@pytest.mark.parametrize( + "prompt, expected_function, expected_skill", + [ + ( + "Write a poem or joke and send it in an e-mail to Kai.", + "SendEmail", + "_GLOBAL_FUNCTIONS_", + ) + ], +) +@pytest.mark.asyncio +async def test_create_plan_goal_relevant_async( + get_aoai_config, prompt, expected_function, expected_skill +): + # Arrange + kernel = initialize_kernel(get_aoai_config, use_embeddings=True) + kernel.import_skill(EmailSkillFake()) + kernel.import_skill(FunSkillFake()) + kernel.import_skill(WriterSkillFake()) + + planner = SequentialPlanner( + kernel, + SequentialPlannerConfig(relevancy_threshold=0.65, max_relevant_functions=30), + ) + + # Act + plan = await planner.create_plan_async(prompt) + + # Assert + assert any( + step.name == expected_function and step.skill_name == expected_skill + for step in plan._steps + ) diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner.py new file mode 100644 index 000000000000..33e2ae043480 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner.py @@ -0,0 +1,148 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.kernel import Kernel +from semantic_kernel.memory.semantic_text_memory import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner import ( + SequentialPlanner, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.skill_collection_base import SkillCollectionBase + + +def create_mock_function(function_view: FunctionView): + mock_function = Mock(spec=SKFunctionBase) + mock_function.describe.return_value = function_view + mock_function.name = function_view.name + mock_function.skill_name = function_view.skill_name + return mock_function + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "goal", ["Write a poem or joke and send it in an e-mail to Kai."] +) +async def test_it_can_create_plan_async(goal): + # Arrange + kernel = Mock(spec=Kernel) + + memory = Mock(spec=SemanticTextMemoryBase) + + input = [ + ("SendEmail", "email", "Send an e-mail", False), + ("GetEmailAddress", "email", "Get an e-mail address", False), + ("Translate", "WriterSkill", "Translate something", True), + ("Summarize", "SummarizeSkill", "Summarize something", True), + ] + + functionsView = FunctionsView() + skills = Mock(spec=SkillCollectionBase) + mock_functions = [] + for name, skillName, description, isSemantic in input: + function_view = FunctionView(name, skillName, description, [], isSemantic, True) + mock_function = create_mock_function(function_view) + functionsView.add_function(function_view) + + context = SKContext(ContextVariables(), memory, skills, Mock()) + context.variables.update("MOCK FUNCTION CALLED") + mock_function.invoke_async.return_value = context + mock_functions.append(mock_function) + + skills.get_function.side_effect = lambda skill_name, function_name: next( + ( + func + for func in mock_functions + if func.skill_name == skill_name and func.name == function_name + ), + None, + ) + skills.get_functions_view.return_value = functionsView + + expected_functions = [x[0] for x in input] + expected_skills = [x[1] for x in input] + + context = SKContext(ContextVariables(), memory, skills, Mock()) + + return_context = SKContext(ContextVariables(), memory, skills, Mock()) + plan_string = """ + + + + + +""" + + return_context.variables.update(plan_string) + + mock_function_flow_function = Mock(spec=SKFunctionBase) + mock_function_flow_function.invoke_async.return_value = return_context + + kernel.skills = skills + kernel.create_new_context.return_value = context + kernel.register_semantic_function.return_value = mock_function_flow_function + + planner = SequentialPlanner(kernel) + + # Act + plan = await planner.create_plan_async(goal) + + # Assert + assert plan.description == goal + assert any( + step.name in expected_functions and step.skill_name in expected_skills + for step in plan._steps + ) + for expected_function in expected_functions: + assert any(step.name == expected_function for step in plan._steps) + for expectedSkill in expected_skills: + assert any(step.skill_name == expectedSkill for step in plan._steps) + + +@pytest.mark.asyncio +async def test_empty_goal_throws_async(): + # Arrange + kernel = Mock(spec=Kernel) + planner = SequentialPlanner(kernel) + + # Act & Assert + with pytest.raises(PlanningException): + await planner.create_plan_async("") + + +@pytest.mark.asyncio +async def test_invalid_xml_throws_async(): + # Arrange + kernel = Mock(spec=Kernel) + memory = Mock(spec=SemanticTextMemoryBase) + skills = Mock(spec=SkillCollectionBase) + + functionsView = FunctionsView() + skills.get_functions_view.return_value = functionsView + + plan_string = "notvalid<" + return_context = SKContext( + ContextVariables(plan_string), memory, skills, logger=Mock() + ) + + context = SKContext(ContextVariables(), memory, skills, logger=Mock()) + + mock_function_flow_function = Mock(spec=SKFunctionBase) + mock_function_flow_function.invoke_async.return_value = return_context + + kernel.skills = skills + kernel.create_new_context.return_value = context + kernel.register_semantic_function.return_value = mock_function_flow_function + + planner = SequentialPlanner(kernel) + + # Act & Assert + with pytest.raises(PlanningException): + await planner.create_plan_async("goal") diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py new file mode 100644 index 000000000000..64136ea00f17 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner_extensions.py @@ -0,0 +1,254 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.memory.memory_query_result import MemoryQueryResult +from semantic_kernel.memory.semantic_text_memory_base import SemanticTextMemoryBase +from semantic_kernel.orchestration.context_variables import ContextVariables +from semantic_kernel.orchestration.sk_context import SKContext +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.sequential_planner.sequential_planner_config import ( + SequentialPlannerConfig, +) +from semantic_kernel.planning.sequential_planner.sequential_planner_extensions import ( + SequentialPlannerFunctionViewExtension, + SequentialPlannerSKContextExtension, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView +from semantic_kernel.skill_definition.read_only_skill_collection_base import ( + ReadOnlySkillCollectionBase, +) +from semantic_kernel.skill_definition.skill_collection import SkillCollection + + +async def _async_generator(query_result): + yield query_result + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_no_functions_async(): + variables = ContextVariables() + skills = SkillCollection() + + memory = Mock(spec=SemanticTextMemoryBase) + memory_query_result = MemoryQueryResult( + is_reference=False, + id="id", + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + + async_enumerable = _async_generator(memory_query_result) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext(variables, memory, skills.read_only_skill_collection, Mock()) + config = SequentialPlannerConfig() + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + memory.search_async.assert_not_called() + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_functions_async(): + variables = ContextVariables() + + function_mock = Mock(spec=SKFunctionBase) + functions_view = FunctionsView() + function_view = FunctionView( + "functionName", + "skillName", + "description", + [], + is_semantic=True, + is_asynchronous=False, + ) + native_function_view = FunctionView( + "nativeFunctionName", + "skillName", + "description", + [], + is_semantic=False, + is_asynchronous=False, + ) + functions_view.add_function(function_view) + functions_view.add_function(native_function_view) + + skills = Mock(spec=ReadOnlySkillCollectionBase) + skills.get_function.return_value = function_mock + skills.get_functions_view.return_value = functions_view + + memory_query_result = MemoryQueryResult( + is_reference=False, + id=SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function_view + ), + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + + async_enumerable = _async_generator(memory_query_result) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext(variables, memory, skills, Mock()) + config = SequentialPlannerConfig() + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 + assert result[0] == function_view + + # Arrange update IncludedFunctions + config.included_functions.append(["nativeFunctionName"]) + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 # IncludedFunctions should be added to the result + assert result[0] == function_view + assert result[1] == native_function_view + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_with_functions_and_relevancy_async(): + # Arrange + variables = ContextVariables() + + # Arrange FunctionView + function_mock = Mock(spec=SKFunctionBase) + functions_view = FunctionsView() + function_view = FunctionView( + "functionName", + "skillName", + "description", + [], + is_semantic=True, + is_asynchronous=False, + ) + native_function_view = FunctionView( + "nativeFunctionName", + "skillName", + "description", + [], + is_semantic=False, + is_asynchronous=False, + ) + functions_view.add_function(function_view) + functions_view.add_function(native_function_view) + + # Arrange Mock Memory and Result + memory_query_result = MemoryQueryResult( + is_reference=False, + id=SequentialPlannerFunctionViewExtension.to_fully_qualified_name( + function_view + ), + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = _async_generator(memory_query_result) + + skills = Mock(spec=ReadOnlySkillCollectionBase) + skills.get_function.return_value = function_mock + skills.get_functions_view.return_value = functions_view + skills.read_only_skill_collection = skills + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext(variables, memory, skills, Mock()) + config = SequentialPlannerConfig(relevancy_threshold=0.78) + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 1 + assert result[0] == function_view + + # Arrange update IncludedFunctions + config.included_functions.append("nativeFunctionName") + memory.search_async.return_value = _async_generator(memory_query_result) + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + assert len(result) == 2 # IncludedFunctions should be added to the result + assert result[0] == function_view + assert result[1] == native_function_view + + +@pytest.mark.asyncio +async def test_can_call_get_available_functions_async_with_default_relevancy_async(): + # Arrange + variables = ContextVariables() + skills = SkillCollection() + + # Arrange Mock Memory and Result + memory_query_result = MemoryQueryResult( + is_reference=False, + id="id", + text="text", + description="description", + external_source_name="sourceName", + additional_metadata="value", + relevance=0.8, + embedding=None, + ) + async_enumerable = _async_generator(memory_query_result) + memory = Mock(spec=SemanticTextMemoryBase) + memory.search_async.return_value = async_enumerable + + # Arrange GetAvailableFunctionsAsync parameters + context = SKContext(variables, memory, skills, Mock()) + config = SequentialPlannerConfig(relevancy_threshold=0.78) + semantic_query = "test" + + # Act + result = await SequentialPlannerSKContextExtension.get_available_functions_async( + context, config, semantic_query + ) + + # Assert + assert result is not None + memory.search_async.assert_called_once() diff --git a/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py b/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py new file mode 100644 index 000000000000..d39e226201a3 --- /dev/null +++ b/python/tests/unit/planning/sequential_planner/test_sequential_planner_parser.py @@ -0,0 +1,349 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import Mock + +import pytest + +from semantic_kernel.kernel import Kernel +from semantic_kernel.orchestration.sk_function_base import SKFunctionBase +from semantic_kernel.planning.planning_exception import PlanningException +from semantic_kernel.planning.sequential_planner.sequential_planner_parser import ( + SequentialPlanParser, +) +from semantic_kernel.skill_definition.function_view import FunctionView +from semantic_kernel.skill_definition.functions_view import FunctionsView + + +def create_mock_function(function_view: FunctionView) -> SKFunctionBase: + mock_function = Mock(spec=SKFunctionBase) + mock_function.describe.return_value = function_view + mock_function.name = function_view.name + mock_function.skill_name = function_view.skill_name + mock_function.description = function_view.description + return mock_function + + +def create_kernel_and_functions_mock(functions) -> Kernel: + kernel = Kernel() + functions_view = FunctionsView() + for name, skill_name, description, is_semantic, result_string in functions: + function_view = FunctionView( + name, skill_name, description, [], is_semantic, True + ) + functions_view.add_function(function_view) + mock_function = create_mock_function(function_view) + + result = kernel.create_new_context() + result.variables.update(result_string) + mock_function.invoke_async.return_value = result + kernel._skill_collection.add_semantic_function(mock_function) + + return kernel + + +def test_can_call_to_plan_from_xml(): + functions = [ + ( + "Summarize", + "SummarizeSkill", + "Summarize an input", + True, + "This is the summary.", + ), + ("Translate", "WriterSkill", "Translate to french", True, "Bonjour!"), + ( + "GetEmailAddressAsync", + "email", + "Get email address", + False, + "johndoe@email.com", + ), + ("SendEmailAsync", "email", "Send email", False, "Email sent."), + ] + kernel = create_kernel_and_functions_mock(functions) + + plan_string = """ + + + + +""" + goal = "Summarize an input, translate to french, and e-mail to John Doe" + + plan = SequentialPlanParser.to_plan_from_xml( + plan_string, + goal, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + assert plan is not None + assert ( + plan.description + == "Summarize an input, translate to french, and e-mail to John Doe" + ) + + assert len(plan._steps) == 4 + assert plan._steps[0].skill_name == "SummarizeSkill" + assert plan._steps[0].name == "Summarize" + assert plan._steps[1].skill_name == "WriterSkill" + assert plan._steps[1].name == "Translate" + assert plan._steps[1].parameters["language"] == "French" + assert "TRANSLATED_SUMMARY" in plan._steps[1]._outputs + + assert plan._steps[2].skill_name == "email" + assert plan._steps[2].name == "GetEmailAddressAsync" + assert plan._steps[2].parameters["input"] == "John Doe" + assert "EMAIL_ADDRESS" in plan._steps[2]._outputs + + assert plan._steps[3].skill_name == "email" + assert plan._steps[3].name == "SendEmailAsync" + assert "$TRANSLATED_SUMMARY" in plan._steps[3].parameters["input"] + assert "$EMAIL_ADDRESS" in plan._steps[3].parameters["email_address"] + + +def test_invalid_plan_execute_plan_returns_invalid_result(): + # Arrange + kernel = create_kernel_and_functions_mock([]) + + # Act and Assert + with pytest.raises(PlanningException): + SequentialPlanParser.to_plan_from_xml( + "", + "Solve the equation x^2 = 2.", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + +def test_can_create_plan_with_text_nodes(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text = """ + Test the functionFlowRunner + + + This is some text + """ + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert plan.description == goal_text + assert len(plan._steps) == 1 + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + + +@pytest.mark.parametrize( + "plan_text, allow_missing_functions", + [ + ( + """ + + + + """, + True, + ), + ( + """ + + + + """, + False, + ), + ], +) +def test_can_create_plan_with_invalid_function_nodes( + plan_text, allow_missing_functions +): + # Arrange + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + # Act and Assert + if allow_missing_functions: + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + allow_missing_functions, + ) + + # Assert + assert plan is not None + assert len(plan._steps) == 2 + + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + assert plan._steps[0].description == "Echo an input" + + assert plan._steps[1].skill_name == plan.__class__.__name__ + assert plan._steps[1].name == "" + assert plan._steps[1].description == "MockSkill.DoesNotExist" + else: + with pytest.raises(PlanningException): + SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + allow_missing_functions, + ) + + +def test_can_create_plan_with_other_text(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text1 = """Possible result: Test the functionFlowRunner + + + This is some text + """ + plan_text2 = """ + + + This is some text + + + plan end""" + plan_text3 = """ + + + This is some text + + + plan end""" + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan1 = SequentialPlanParser.to_plan_from_xml( + plan_text1, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + plan2 = SequentialPlanParser.to_plan_from_xml( + plan_text2, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + plan3 = SequentialPlanParser.to_plan_from_xml( + plan_text3, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan1 is not None + assert plan1.description == goal_text + assert len(plan1._steps) == 1 + assert plan1._steps[0].skill_name == "MockSkill" + assert plan1._steps[0].name == "Echo" + + assert plan2 is not None + assert plan2.description == goal_text + assert len(plan2._steps) == 1 + assert plan2._steps[0].skill_name == "MockSkill" + assert plan2._steps[0].name == "Echo" + + assert plan3 is not None + assert plan3.description == goal_text + assert len(plan3._steps) == 1 + assert plan3._steps[0].skill_name == "MockSkill" + assert plan3._steps[0].name == "Echo" + + +@pytest.mark.parametrize( + "plan_text", + [ + """ """, + """ + +""", + """ + +""", + ], +) +def test_can_create_plan_with_open_api_plugin(plan_text): + # Arrange + functions = [ + ( + "codesearchresults_post", + "CodeSearch", + "Echo an input", + True, + "Mock Echo Result", + ), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + "", + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert len(plan._steps) == 1 + assert plan._steps[0].skill_name == "CodeSearch" + assert plan._steps[0].name == "codesearchresults_post" + + +def test_can_create_plan_with_ignored_nodes(): + # Arrange + goal_text = "Test the functionFlowRunner" + plan_text = """ + + Some other tag + + """ + functions = [ + ("Echo", "MockSkill", "Echo an input", True, "Mock Echo Result"), + ] + kernel = create_kernel_and_functions_mock(functions) + + # Act + plan = SequentialPlanParser.to_plan_from_xml( + plan_text, + goal_text, + SequentialPlanParser.get_skill_function(kernel.create_new_context()), + ) + + # Assert + assert plan is not None + assert plan.description == goal_text + assert len(plan._steps) == 2 + assert plan._steps[0].skill_name == "MockSkill" + assert plan._steps[0].name == "Echo" + assert len(plan._steps[1]._steps) == 0 + assert plan._steps[1].skill_name == "MockSkill" + assert plan._steps[1].name == "Echo"