Python: support Jinja2 templating (#5483)

### Motivation and Context

Jinja2 is a heavily supported Python templating language and we're
introducing it to SK Python.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

This PR brings in the ability to specify Jinja2 templates for use within
SK Python:
- Closes #5343 

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

python/notebooks/05-using-the-planner.ipynb

@@ -23,7 +23,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!python -m pip install -U semantic-kernel "
+    "!python -m pip install -U semantic-kernel"
    ]
   },
   {

python/notebooks/06-memory-and-embeddings.ipynb

@@ -349,18 +349,18 @@
    "outputs": [],
    "source": [
     "github_files = {}\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"\n",
-    "] = \"README: Installation, getting started, and how to contribute\"\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"] = (\n",
+    "    \"README: Installation, getting started, and how to contribute\"\n",
+    ")\n",
     "github_files[\n",
     "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"\n",
     "] = \"Jupyter notebook describing how to pass prompts from a file to a semantic plugin or function\"\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"\n",
-    "] = \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"\n",
-    "] = \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\"\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"] = (\n",
+    "    \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n",
+    ")\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"] = (\n",
+    "    \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\"\n",
+    ")\n",
     "github_files[\n",
     "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/Volatile/VolatileMemoryStore.cs\"\n",
     "] = \"C# class that defines a volatile embedding store\""

python/notebooks/third_party/weaviate-persistent-memory.ipynb

@@ -422,18 +422,18 @@
    "outputs": [],
    "source": [
     "github_files = {}\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"\n",
-    "] = \"README: Installation, getting started, and how to contribute\"\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/README.md\"] = (\n",
+    "    \"README: Installation, getting started, and how to contribute\"\n",
+    ")\n",
     "github_files[\n",
     "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/02-running-prompts-from-file.ipynb\"\n",
     "] = \"Jupyter notebook describing how to pass prompts from a file to a semantic plugin or function\"\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"\n",
-    "] = \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n",
-    "github_files[\n",
-    "    \"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"\n",
-    "] = \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\"\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/notebooks/00-getting-started.ipynb\"] = (\n",
+    "    \"Jupyter notebook describing how to get started with the Semantic Kernel\"\n",
+    ")\n",
+    "github_files[\"https://github.com/microsoft/semantic-kernel/tree/main/samples/plugins/ChatPlugin/ChatGPT\"] = (\n",
+    "    \"Sample demonstrating how to create a chat plugin interfacing with ChatGPT\"\n",
+    ")\n",
     "github_files[\n",
     "    \"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/Volatile/VolatileMemoryStore.cs\"\n",
     "] = \"C# class that defines a volatile embedding store\""

python/pyproject.toml

@@ -32,6 +32,7 @@ pydantic = "^2"
 motor = "^3.3.2"
 defusedxml = "^0.7.1"
 pybars4 = "^0.9.13"
+jinja2 = "^3.1.3"
 
 # Optional dependencies
 ipykernel = { version = "^6.21.1", optional = true}

python/samples/kernel-syntax-examples/azure_chat_gpt_api_jinja2.py

@@ -0,0 +1,94 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+import logging
+
+import semantic_kernel as sk
+import semantic_kernel.connectors.ai.open_ai as sk_oai
+from semantic_kernel.contents.chat_history import ChatHistory
+from semantic_kernel.functions.kernel_arguments import KernelArguments
+from semantic_kernel.utils.settings import azure_openai_settings_from_dot_env_as_dict
+
+logging.basicConfig(level=logging.WARNING)
+
+system_message = """
+You are a chat bot. Your name is Mosscap and
+you have one goal: figure out what people need.
+Your full name, should you need to know it, is
+Splendid Speckled Mosscap. You communicate
+effectively, but you tend to answer with long
+flowery prose.
+"""
+
+kernel = sk.Kernel()
+
+service_id = "chat-gpt"
+chat_service = sk_oai.AzureChatCompletion(
+    service_id=service_id, **azure_openai_settings_from_dot_env_as_dict(include_api_version=True)
+)
+kernel.add_service(chat_service)
+
+req_settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id)
+req_settings.max_tokens = 2000
+req_settings.temperature = 0.7
+req_settings.top_p = 0.8
+req_settings.auto_invoke_kernel_functions = False
+
+
+chat_function = kernel.create_function_from_prompt(
+    prompt="""{{system_message}}{% for item in chat_history %}{{ message(item) }}{% endfor %}""",
+    function_name="chat",
+    plugin_name="chat",
+    template_format="jinja2",
+    prompt_execution_settings=req_settings,
+)
+
+chat_history = ChatHistory()
+chat_history.add_user_message("Hi there, who are you?")
+chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.")
+
+
+async def chat() -> bool:
+    try:
+        user_input = input("User:> ")
+    except KeyboardInterrupt:
+        print("\n\nExiting chat...")
+        return False
+    except EOFError:
+        print("\n\nExiting chat...")
+        return False
+
+    if user_input == "exit":
+        print("\n\nExiting chat...")
+        return False
+    chat_history.add_user_message(user_input)
+    arguments = KernelArguments(system_message=system_message, chat_history=chat_history)
+
+    stream = True
+    if stream:
+        answer = kernel.invoke_stream(
+            chat_function,
+            arguments=arguments,
+        )
+        print("Mosscap:> ", end="")
+        async for message in answer:
+            print(str(message[0]), end="")
+        print("\n")
+        return True
+    answer = await kernel.invoke(
+        chat_function,
+        arguments=arguments,
+    )
+    print(f"Mosscap:> {answer}")
+    chat_history.add_assistant_message(str(answer))
+    return True
+
+
+async def main() -> None:
+    chatting = True
+    while chatting:
+        chatting = await chat()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/semantic_kernel/exceptions/template_engine_exceptions.py

@@ -87,6 +87,14 @@ class HandlebarsTemplateRenderException(BlockRenderException):
     pass
 
 
+class Jinja2TemplateSyntaxError(BlockSyntaxError):
+    pass
+
+
+class Jinja2TemplateRenderException(BlockRenderException):
+    pass
+
+
 __all__ = [
     "BlockException",
     "BlockSyntaxError",
@@ -103,4 +111,6 @@ class HandlebarsTemplateRenderException(BlockRenderException):
     "TemplateRenderException",
     "HandlebarsTemplateSyntaxError",
     "HandlebarsTemplateRenderException",
+    "Jinja2TemplateSyntaxError",
+    "Jinja2TemplateRenderException",
 ]

python/semantic_kernel/functions/kernel_function.py

@@ -12,10 +12,12 @@
 from semantic_kernel.kernel_pydantic import KernelBaseModel
 from semantic_kernel.prompt_template.const import (
     HANDLEBARS_TEMPLATE_FORMAT_NAME,
+    JINJA2_TEMPLATE_FORMAT_NAME,
     KERNEL_TEMPLATE_FORMAT_NAME,
     TEMPLATE_FORMAT_TYPES,
 )
 from semantic_kernel.prompt_template.handlebars_prompt_template import HandlebarsPromptTemplate
+from semantic_kernel.prompt_template.jinja2_prompt_template import Jinja2PromptTemplate
 from semantic_kernel.prompt_template.kernel_prompt_template import KernelPromptTemplate
 
 if TYPE_CHECKING:
@@ -32,6 +34,7 @@
 TEMPLATE_FORMAT_MAP = {
     KERNEL_TEMPLATE_FORMAT_NAME: KernelPromptTemplate,
     HANDLEBARS_TEMPLATE_FORMAT_NAME: HandlebarsPromptTemplate,
+    JINJA2_TEMPLATE_FORMAT_NAME: Jinja2PromptTemplate,
 }
 
 

python/semantic_kernel/kernel.py

@@ -40,7 +40,6 @@
 from semantic_kernel.functions.kernel_plugin_collection import KernelPluginCollection
 from semantic_kernel.kernel_pydantic import KernelBaseModel
 from semantic_kernel.prompt_template.const import (
-    HANDLEBARS_TEMPLATE_FORMAT_NAME,
     KERNEL_TEMPLATE_FORMAT_NAME,
     TEMPLATE_FORMAT_TYPES,
 )
@@ -376,7 +375,9 @@ async def invoke_prompt(
         prompt: str,
         arguments: Optional[KernelArguments] = None,
         template_format: Literal[
-            KERNEL_TEMPLATE_FORMAT_NAME, HANDLEBARS_TEMPLATE_FORMAT_NAME
+            "semantic-kernel",
+            "handlebars",
+            "jinja2",
         ] = KERNEL_TEMPLATE_FORMAT_NAME,
         **kwargs: Any,
     ) -> Optional[Union[FunctionResult, List[FunctionResult]]]:

python/semantic_kernel/prompt_template/__init__.py

@@ -2,12 +2,14 @@
 
 from semantic_kernel.prompt_template.handlebars_prompt_template import HandlebarsPromptTemplate
 from semantic_kernel.prompt_template.input_variable import InputVariable
+from semantic_kernel.prompt_template.jinja2_prompt_template import Jinja2PromptTemplate
 from semantic_kernel.prompt_template.kernel_prompt_template import KernelPromptTemplate
 from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
 
 __all__ = [
     "KernelPromptTemplate",
     "HandlebarsPromptTemplate",
+    "Jinja2PromptTemplate",
     "InputVariable",
     "PromptTemplateConfig",
 ]

python/semantic_kernel/prompt_template/const.py

@@ -4,5 +4,8 @@
 
 KERNEL_TEMPLATE_FORMAT_NAME: Literal["semantic-kernel"] = "semantic-kernel"
 HANDLEBARS_TEMPLATE_FORMAT_NAME: Literal["handlebars"] = "handlebars"
+JINJA2_TEMPLATE_FORMAT_NAME: Literal["jinja2"] = "jinja2"
 
-TEMPLATE_FORMAT_TYPES = Union[type(KERNEL_TEMPLATE_FORMAT_NAME), type(HANDLEBARS_TEMPLATE_FORMAT_NAME)]
+TEMPLATE_FORMAT_TYPES = Union[
+    type(KERNEL_TEMPLATE_FORMAT_NAME), type(HANDLEBARS_TEMPLATE_FORMAT_NAME), type(JINJA2_TEMPLATE_FORMAT_NAME)
+]

python/semantic_kernel/prompt_template/handlebars_prompt_template.py

@@ -10,7 +10,7 @@
 from semantic_kernel.functions.kernel_arguments import KernelArguments
 from semantic_kernel.prompt_template.const import HANDLEBARS_TEMPLATE_FORMAT_NAME
 from semantic_kernel.prompt_template.prompt_template_base import PromptTemplateBase
-from semantic_kernel.prompt_template.utils import HANDLEBAR_SYSTEM_HELPERS, create_helper_from_function
+from semantic_kernel.prompt_template.utils import HANDLEBAR_SYSTEM_HELPERS, create_template_helper_from_function
 
 if TYPE_CHECKING:
     from semantic_kernel.kernel import Kernel
@@ -78,7 +78,9 @@ async def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"]
         for plugin in kernel.plugins:
             helpers.update(
                 {
-                    function.fully_qualified_name: create_helper_from_function(function, kernel, arguments)
+                    function.fully_qualified_name: create_template_helper_from_function(
+                        function, kernel, arguments, self.prompt_template_config.template_format
+                    )
                     for function in plugin.functions.values()
                 }
             )
@@ -90,5 +92,6 @@ async def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"]
                 f"Error rendering prompt template: {self.prompt_template_config.template} with arguments: {arguments}"
             )
             raise HandlebarsTemplateRenderException(
-                f"Error rendering prompt template: {self.prompt_template_config.template} with arguments: {arguments}: error: {exc}"  # noqa: E501
+                f"Error rendering prompt template: {self.prompt_template_config.template} "
+                f"with arguments: {arguments}: error: {exc}"
             ) from exc