Introducing Online Inference with Hidden Markov Models (HMMs)

Bonsai.ML.sln

@@ -20,6 +20,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Directory.Build.props = Directory.Build.props
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.HiddenMarkovModels", "src\Bonsai.ML.HiddenMarkovModels\Bonsai.ML.HiddenMarkovModels.csproj", "{BAD0A733-8EFB-4EAF-9648-9851656AF7FF}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Python", "src\Bonsai.ML.Python\Bonsai.ML.Python.csproj", "{39A4414F-52B1-42D7-82FA-E65DAD885264}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Data", "src\Bonsai.ML.Data\Bonsai.ML.Data.csproj", "{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -42,6 +48,18 @@ Global
 		{81DB65B3-EA65-4947-8CF1-0E777324C082}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81DB65B3-EA65-4947-8CF1-0E777324C082}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{81DB65B3-EA65-4947-8CF1-0E777324C082}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -51,6 +69,9 @@ Global
 		{17AABD18-E275-4409-9E33-3D755B809FF6} = {12312384-8828-4786-AE19-EFCEDF968290}
 		{196AA5C7-AE8A-477B-B01A-B94676EC60EE} = {12312384-8828-4786-AE19-EFCEDF968290}
 		{81DB65B3-EA65-4947-8CF1-0E777324C082} = {461FE3E2-21C4-47F9-8405-DF72326AAB2B}
+		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{39A4414F-52B1-42D7-82FA-E65DAD885264} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13} = {12312384-8828-4786-AE19-EFCEDF968290}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {B6468F13-97CD-45E0-9E1E-C122D7F1E09F}

Directory.Build.props

@@ -14,14 +14,16 @@
     <PackageIcon>icon.png</PackageIcon>
     <IncludeSymbols>true</IncludeSymbols>
     <RepositoryType>git</RepositoryType>
-    <VersionPrefix>0.2.0</VersionPrefix>
+    <VersionPrefix>0.3.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
-    <LangVersion>9.0</LangVersion>
+    <LangVersion>12.0</LangVersion>
   </PropertyGroup>
-
   <ItemGroup>
     <Content Include="..\..\LICENSE" PackagePath="/" />
     <Content Include="..\..\icon.png" PackagePath="/" />
     <None Include="..\..\README.md" Pack="True" PackagePath="/" />
+    <EmbeddedResource Include="..\..\elementIcon.svg" >
+      <LogicalName>$(RootNamespace).$(MSBuildProjectName).svg</LogicalName>
+    </EmbeddedResource>
   </ItemGroup>
 </Project>

README.md

@@ -1,16 +1,44 @@
 # Bonsai - Machine Learning
 
-The Bonsai.ML project is a collection of packages with reactive infrastructure for adding machine learning algorithms in Bonsai. Below you will find the list of packages (and the included subpackages) currently available within the Bonsai.ML collection.
+The **Bonsai.ML** project is a collection of packages designed to integrate machine learning algorithms with Bonsai. This document provides an overview of the available packages and their functionalities.
 
-* Bonsai.ML - provides core functionality across all Bonsai.ML packages.
-* Bonsai.ML.LinearDynamicalSystems - package for performing inference of linear dynamical systems. Interfaces with the [lds_python](https://github.com/joacorapela/lds_python) package.
-  - *Bonsai.ML.LinearDynamicalSystems.Kinematics* - subpackage included in the LinearDynamicalSystems package which supports using the Kalman Filter to infer kinematic data.
-  - *Bonsai.ML.LinearDynamicalSystems.LinearRegression* - subpackage included in the LinearDynamicalSystems package which supports using the Kalman Filter to perform Bayesian linear regression.
-* Bonsai.ML.Visualizers - provides a set of visualizers for dynamic graphing/plotting.
+## Core Packages
+
+- **Bonsai.ML**
+    Provides common tools and functionality.
+
+- **Bonsai.ML.Data**
+    Provides common tools and functionality for working with data.
+
+- **Bonsai.ML.Python**
+    Provides common tools and functionality for C# packages to interface with Python.
+
+## Available Packages
+
+### Bonsai.ML.LinearDynamicalSystems 
+Facilitates inference using linear dynamical systems (LDS). It interfaces with the [lds_python](https://github.com/joacorapela/lds_python) package using the [Bonsai - Python Scripting](https://github.com/bonsai-rx/python-scripting) library.
+
+- **Bonsai.ML.LinearDynamicalSystems.Kinematics**
+    Supports the use of the Kalman Filter for inferring kinematic data.
+
+- **Bonsai.ML.LinearDynamicalSystems.LinearRegression** 
+    Utilizes the Kalman Filter to perform online Bayesian linear regression.
+
+### Bonsai.ML.HiddenMarkovModels
+Facilitates inference using Hidden Markov Models (HMMs). It interfaces with the [ssm](https://github.com/lindermanlab/ssm) package using the [Bonsai - Python Scripting](https://github.com/bonsai-rx/python-scripting) library.
+
+- **Bonsai.ML.HiddenMarkovModels.Observations**
+    Provides functionality for specifying different types of observations.
+
+- **Bonsai.ML.HiddenMarkovModels.Transitions**
+    Provides functionality for specifying different types of transition models.
+
+### Bonsai.ML.Visualizers
+Graphing and plotting library for visualizing data.
 
 > [!NOTE]
-> Bonsai.ML packages are installed through Bonsai's integrated package manager and are typically available for use immediately. However, certain packages may require additional steps for installation. See the dedicated package section for specific guides and documentation.
+> Bonsai.ML packages can be installed through Bonsai's integrated package manager and are generally ready for immediate use. However, some packages may require additional installation steps. Refer to the specific package section for detailed installation guides and documentation.
 
 ## Acknowledgments
 
-Development of this package was supported by funding from the Biotechnology and Biological Sciences Research Council [grant number BB/W019132/1].
+Development of the Bonsai.ML package is supported by the Biotechnology and Biological Sciences Research Council [grant number BB/W019132/1].

docs/articles/HiddenMarkovModels/hmm-getting-started.md

@@ -0,0 +1,47 @@
+# Getting Started
+
+The workflow starts with creating a python runtime, followed by loading the ssm package, referred to as the HMM module. After this, you can instantiate the HMM model and pass it observations of data to perform inference. Since this package relies on communication between Bonsai and Python, the observations that the model uses must be formatted into a valid string representation of a Python data type, namely a list of numbers.
+
+## Workflow
+
+```mermaid
+
+flowchart LR
+
+    A(["Create Python Runtime"])
+    B(["Load HMM Module"])
+    C(["Create HMM"])
+    D(["Generate Observations"])
+    E(["Infer Hidden State"])
+
+    A --> B
+    B --> C
+    C --> D
+    D --> E
+
+```
+
+> [!NOTE]
+> Due to the way Bonsai.ML interacts with Python, it is necessary for the first two steps to complete before instantiating the model. It is important to know that the initialization of the Python runtime, loading the module, and creating the model takes time to complete, and that only once the model has been created can inference be performed.
+
+## Implementation
+
+Below is a simplified Bonsai workflow that implements the core logic of the package.
+
+:::workflow
+![HMM Implementation](~/workflows/HMMImplementation.bonsai)
+:::
+
+A `CreateRuntime` node is used to initialize a python runtime engine, which gets passed to a `BehaviorSubject` called `PythonEngine`. Bonsai's `CreateRuntime` node should automatically detect the python virtual environment if it was activated in the same terminal that was used to launch Bonsai, otherwise the path to the virtual environment can be specified in the `CreateRuntime` node by setting the `PythonHome` property.
+
+Next, the `PythonEngine` node is passed to a `LoadHMMModule` node which will load the ssm package into the python environment.
+
+Once the HMM module has been initialized, the `CreateHMM` node instantiates a python instance of the HMM model. Here, you can specify the initialization parameters of the model and provide a `ModelName` parameter that gets used to reference the model in other parts of the Bonsai workflow.
+
+It is crucial that the `Data` are formatted into a string that the model can use, namely a string representing a Python list. For example, if you pass a Tuple with 2 items as your data, then the formatter should look something like `"[" + Item1.ToString() + Item2.ToString() + "]"`. The output of this should be used as your observations into the model, so connect your data source to a `Subject` named `Data` and modify the `FormatToPython` node to fit with your data.
+
+`Observations` are then passed to an `InferState` node, which will use the specified model (given by the `ModelName` property) to infer the latent state of the model and outputs the `StateProbabilities`, or probabilities of being in each state given the observation.
+
+### Further Examples
+
+For further examples and demonstrations for how this package works, see the [Bonsai - Machine Learning Examples](~/examples/README.md) section.

docs/articles/HiddenMarkovModels/hmm-overview.md

@@ -0,0 +1,134 @@
+# Bonsai.ML.HiddenMarkovModels Overview
+
+The HiddenMarkovModels package provides a Bonsai interface to interact with the [ssm](https://github.com/lindermanlab/ssm) package.
+
+## General Guide
+
+Since the package relies on both Bonsai and Python, installation steps for both are required. Detailed instructions are provided for installing the package in a new environment, integrating it with existing workflows, and running examples from the example folder.
+
+- To install the package for integrating with existing workflows, see the [Installation Guide](#installation-guide).
+- To get started with integrating the package into workflows, see the [Getting Started](hmm-getting-started.md) section.
+- To test the specific examples provided, check out the [Examples](~/examples/README.md) tab.
+
+## Installation Guide
+
+### Dependencies
+
+To get started, you must install the following tools:
+
+- [Python (v3.10)](https://www.python.org/downloads/) 
+- [dotnet-sdk (v8)](https://dotnet.microsoft.com/en-us/download)
+- [Git](https://git-scm.com/downloads)
+- [Bonsai-Rx Templates tool](https://www.nuget.org/packages/Bonsai.Templates)
+
+  > [!TIP]
+  > Install Python through the standard installer and add to the system PATH.
+
+### Installation Guide - Windows
+
+#### Creating New Project Environment
+
+1. Open the terminal and create a project folder:
+    ```cmd
+    cd ~\Desktop
+    mkdir HiddenMarkovModels
+    cd .\HiddenMarkovModels
+    ```
+
+2. Create a Python virtual environment:
+    ```cmd
+    python -m venv .venv
+    ```
+
+3. Create a Bonsai environment:
+    ```cmd
+    dotnet new bonsaienv
+    ```
+
+#### Python Environment Setup
+
+1. Activate the Python environment:
+    ```cmd
+    .\.venv\Scripts\activate
+    ```
+
+2. Install the ssm package:
+    ```cmd
+    pip install numpy cython
+    pip install ssm@git+https://github.com/lindermanlab/ssm@6c856ad3967941d176eb348bcd490cfaaa08ba60
+    ```
+
+3. Verify installation:
+    ```python
+    import ssm
+    ```
+
+#### Bonsai Environment Setup
+
+1. Launch Bonsai:
+    ```cmd
+    .bonsai\Bonsai.exe
+    ```
+
+2. Install the `Bonsai.ML.HiddenMarkovModels` package from the Package Manager.
+    > [!TIP]
+    > You can quickly search for the package by entering `Bonsai.ML.HiddenMarkovModels` into the search bar.
+
+### Installation Guide - Linux
+
+#### Creating New Project Environment
+
+1. Create a project folder:
+    ```cmd
+    cd ~/Desktop
+    mkdir HiddenMarkovModels
+    cd HiddenMarkovModels
+    ```
+
+2. Create a Python virtual environment:
+    ```cmd
+    python3 -m venv .venv
+    ```
+    > [!TIP]
+    > Install the virtual environment package if needed:
+    > ```cmd
+    > sudo apt install python3.10-venv
+    > ```
+
+3. Create a Bonsai environment:
+    ```cmd
+    dotnet new bonsaienv
+    ```
+    > [!NOTE]
+    > This step uses the [Bonsai Linux Environment Template tool](https://github.com/ncguilbeault/bonsai-linux-environment-template) for easy creation of bonsai environments on Linux.
+    > See [this discussion](https://github.com/orgs/bonsai-rx/discussions/1101) for more information on getting Bonsai running on Linux.
+
+#### Python Environment Setup
+
+1. Activate the Python environment:
+    ```cmd
+    source .venv/bin/activate
+    ```
+
+2. Install the ssm package:
+    ```cmd
+    pip install numpy cython
+    pip install ssm@git+https://github.com/lindermanlab/ssm@6c856ad3967941d176eb348bcd490cfaaa08ba60
+    ```
+
+3. Verify installation:
+    ```python
+    import ssm
+    ```
+
+#### Bonsai Environment Setup
+
+1. Activate and launch Bonsai:
+    ```cmd
+    source .bonsai/activate
+    bonsai
+    ```
+
+2. Install the `Bonsai.ML.HiddenMarkovModels` package from the Package Manager.
+    > [!TIP]
+    > You can quickly search for the package by entering `Bonsai.ML.HiddenMarkovModels` into the search bar.

docs/articles/LinearDynamicalSystems/lds-installation-guide-linux.md

@@ -46,7 +46,7 @@ python3 -m venv .venv
 dotnet new bonsaienvl
 ```
 
-When prompted, enter yes to run the powershell setup script.
+When prompted, enter yes to run the setup script.
 
 ### Python Environment Setup Guide
 

docs/articles/LinearDynamicalSystems/lds-installation-guide-windows.md

@@ -10,7 +10,6 @@ To get started, you must install the following tools:
 - [dotnet-sdk (v8)](https://dotnet.microsoft.com/en-us/download)
 - [Git](https://git-scm.com/downloads)
 - [Bonsai-Rx Templates tool](https://www.nuget.org/packages/Bonsai.Templates)
-- [Microsoft Visual C++ Redistributable](https://aka.ms/vs/16/release/vc_redist.x64.exe)
 
 > [!WARNING]
 > Be sure to check the specific python version and dotnet-sdk version you have installed, as different version than the ones we recommend may or may not work with this guide.
@@ -32,7 +31,7 @@ python -m venv .venv
 ```
 
 > [!TIP]
-> If receive an error that says, `python cannot be found`, check to ensure that python is available on the system path. If you just installed python, it may be necessary to restart the terminal.
+> If you receive an error that says, `python cannot be found`, check to ensure that python is available on the system path. If you just installed python, it may be necessary to restart the terminal.
 
 3. Create a bonsai environment. When prompted, enter yes to run the powershell setup script.
 

docs/articles/toc.yml

@@ -8,4 +8,9 @@
 - name: Installing on Linux
   href: LinearDynamicalSystems/lds-installation-guide-linux.md
 - name: Getting Started
-  href: LinearDynamicalSystems/lds-getting-started.md
+  href: LinearDynamicalSystems/lds-getting-started.md
+- name: HiddenMarkovModels
+- name: Overview
+  href: HiddenMarkovModels/hmm-overview.md
+- name: Getting Started
+  href: HiddenMarkovModels/hmm-getting-started.md

docs/workflows/HMMImplementation.bonsai

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="utf-8"?>
+<WorkflowBuilder Version="2.8.5"
+                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                 xmlns:py="clr-namespace:Bonsai.Scripting.Python;assembly=Bonsai.Scripting.Python"
+                 xmlns:rx="clr-namespace:Bonsai.Reactive;assembly=Bonsai.Core"
+                 xmlns:p1="clr-namespace:Bonsai.ML.HiddenMarkovModels;assembly=Bonsai.ML.HiddenMarkovModels"
+                 xmlns="https://bonsai-rx.org/2018/workflow">
+  <Workflow>
+    <Nodes>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="py:CreateRuntime" />
+      </Expression>
+      <Expression xsi:type="rx:BehaviorSubject">
+        <Name>PythonEngine</Name>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>PythonEngine</Name>
+      </Expression>
+      <Expression xsi:type="IncludeWorkflow" Path="Bonsai.ML.HiddenMarkovModels:LoadHMMModule.bonsai" />
+      <Expression xsi:type="IncludeWorkflow" Path="Bonsai.ML.HiddenMarkovModels:CreateHMM.bonsai">
+        <Name>hmm</Name>
+        <NumStates>2</NumStates>
+        <Dimensions>2</Dimensions>
+        <ObservationsType>Gaussian</ObservationsType>
+        <TransitionsType>Stationary</TransitionsType>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>Data</Name>
+      </Expression>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="p1:FormatToPythonList" />
+      </Expression>
+      <Expression xsi:type="rx:BehaviorSubject">
+        <Name>Observation</Name>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>Observation</Name>
+      </Expression>
+      <Expression xsi:type="IncludeWorkflow" Path="Bonsai.ML.HiddenMarkovModels:InferState.bonsai">
+        <Name>hmm</Name>
+      </Expression>
+      <Expression xsi:type="rx:BehaviorSubject">
+        <Name>InferredState</Name>
+      </Expression>
+    </Nodes>
+    <Edges>
+      <Edge From="0" To="1" Label="Source1" />
+      <Edge From="2" To="3" Label="Source1" />
+      <Edge From="3" To="4" Label="Source1" />
+      <Edge From="5" To="6" Label="Source1" />
+      <Edge From="6" To="7" Label="Source1" />
+      <Edge From="8" To="9" Label="Source1" />
+      <Edge From="9" To="10" Label="Source1" />
+    </Edges>
+  </Workflow>
+</WorkflowBuilder>