Implement MathsProvider pattern for STL and XSIMD backends (#115)

* Everything working with STL

* XSIMD working as well

* Fixing includes

* Apply clang-format

* Skipping approx tests

* Fixing example

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/bench.yml

@@ -75,11 +75,6 @@ jobs:
         ./build/rtneural_layer_bench tanh 5 4 4
         ./build/rtneural_layer_bench tanh 5 16 16
 
-    - name: Benchmark FastTanh
-      run: |
-        ./build/rtneural_layer_bench fast_tanh 5 4 4
-        ./build/rtneural_layer_bench fast_tanh 5 16 16
-
     - name: Benchmark ReLu
       run: |
         ./build/rtneural_layer_bench relu 5 4 4

RTNeural/activation/activation.h

@@ -45,21 +45,22 @@ class Activation : public Layer<T>
 
 #else
 #include "../common.h"
+#include "../maths/maths_stl.h"
 #include <cmath>
 
 namespace RTNeural
 {
 
 /** Dynamic implementation of a tanh activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class TanhActivation final : public Activation<T>
 {
 public:
     /** Constructs a tanh activation layer for a given size. */
     explicit TanhActivation(int size)
         : Activation<T>(
             size, [](T x)
-            { return std::tanh(x); },
+            { return MathsProvider::tanh(x); },
             "tanh")
     {
     }
@@ -73,12 +74,12 @@ class TanhActivation final : public Activation<T>
     inline void forward(const T* input, T* out) noexcept override
     {
         for(int i = 0; i < Layer<T>::out_size; ++i)
-            out[i] = std::tanh(input[i]);
+            out[i] = MathsProvider::tanh(input[i]);
     }
 };
 
 /** Static implementation of a tanh activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class TanhActivationT
 {
 public:
@@ -99,62 +100,7 @@ class TanhActivationT
     inline void forward(const T (&ins)[size]) noexcept
     {
         for(int i = 0; i < size; ++i)
-            outs[i] = std::tanh(ins[i]);
-    }
-
-    T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size];
-};
-
-/** Dynamic implementation of an approximate tanh activation layer. */
-template <typename T>
-class FastTanh final : public Activation<T>
-{
-public:
-    /** Constructs a tanh activation layer for a given size. */
-    explicit FastTanh(int size)
-        : Activation<T>(
-            size, [](T x)
-            { return tanh_approx(x); },
-            "tanh")
-    {
-    }
-
-    FastTanh(std::initializer_list<int> sizes)
-        : FastTanh(*sizes.begin())
-    {
-    }
-
-    /** Performs forward propagation for tanh activation. */
-    inline void forward(const T* input, T* out) noexcept override
-    {
-        for(int i = 0; i < Layer<T>::out_size; ++i)
-            out[i] = tanh_approx(input[i]);
-    }
-};
-
-/** Static implementation of an approximate tanh activation layer. */
-template <typename T, int size>
-class FastTanhT
-{
-public:
-    static constexpr auto in_size = size;
-    static constexpr auto out_size = size;
-
-    FastTanhT() = default;
-
-    /** Returns the name of this layer. */
-    std::string getName() const noexcept { return "tanh"; }
-
-    /** Returns true since this layer is an activation layer. */
-    constexpr bool isActivation() const noexcept { return true; }
-
-    void reset() { }
-
-    /** Performs forward propagation for tanh activation. */
-    inline void forward(const T (&ins)[size]) noexcept
-    {
-        for(int i = 0; i < size; ++i)
-            outs[i] = tanh_approx(ins[i]);
+            outs[i] = MathsProvider::tanh(ins[i]);
     }
 
     T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size];
@@ -209,15 +155,15 @@ class ReLuActivationT
 };
 
 /** Dynamic implementation of a sigmoid activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class SigmoidActivation final : public Activation<T>
 {
 public:
     /** Constructs a sigmoid activation layer for a given size. */
     explicit SigmoidActivation(int size)
         : Activation<T>(
             size, [](T x)
-            { return sigmoid(x); },
+            { return MathsProvider::sigmoid(x); },
             "sigmoid")
     {
     }
@@ -229,7 +175,7 @@ class SigmoidActivation final : public Activation<T>
 };
 
 /** Static implementation of a sigmoid activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class SigmoidActivationT
 {
 public:
@@ -250,14 +196,14 @@ class SigmoidActivationT
     inline void forward(const T (&ins)[size]) noexcept
     {
         for(int i = 0; i < size; ++i)
-            outs[i] = sigmoid(ins[i]);
+            outs[i] = MathsProvider::sigmoid(ins[i]);
     }
 
     T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size];
 };
 
 /** Dynamic implementation of a softmax activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class SoftmaxActivation final : public Activation<T>
 {
 public:
@@ -278,12 +224,23 @@ class SoftmaxActivation final : public Activation<T>
     /** Performs forward propagation for softmax activation. */
     inline void forward(const T* input, T* out) noexcept override
     {
-        softmax(input, out, Layer<T>::out_size);
+        T exp_sum = 0;
+        for(int i = 0; i < Layer<T>::out_size; ++i)
+        {
+            out[i] = MathsProvider::exp(input[i]);
+            exp_sum += out[i];
+        }
+
+        const auto exp_sum_recip = (T)1 / exp_sum;
+        for(int i = 0; i < Layer<T>::out_size; ++i)
+        {
+            out[i] *= exp_sum_recip;
+        }
     }
 };
 
 /** Static implementation of a softmax activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class SoftmaxActivationT
 {
 public:
@@ -306,7 +263,7 @@ class SoftmaxActivationT
         T exp_sum = 0;
         for(int i = 0; i < size; ++i)
         {
-            outs[i] = std::exp(ins[i]);
+            outs[i] = MathsProvider::exp(ins[i]);
             exp_sum += outs[i];
         }
 
@@ -321,15 +278,15 @@ class SoftmaxActivationT
 };
 
 /** Dynamic implementation of a elu activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class ELuActivation final : public Activation<T>
 {
 public:
     /** Constructs a softmax activation layer for a given size. */
     explicit ELuActivation(int size)
         : Activation<T>(
             size, [this](T x)
-            { return x > (T)0 ? x : (alpha * (std::exp(x) - (T)1)); },
+            { return x > (T)0 ? x : (alpha * (MathsProvider::exp(x) - (T)1)); },
             "elu")
     {
     }
@@ -347,7 +304,7 @@ class ELuActivation final : public Activation<T>
 };
 
 /** Static implementation of a elu activation layer. */
-template <typename T, int size, int AlphaNumerator = 1, int AlphaDenominator = 1>
+template <typename T, int size, int AlphaNumerator = 1, int AlphaDenominator = 1, typename MathsProvider = DefaultMathsProvider>
 class ELuActivationT
 {
 public:
@@ -370,7 +327,7 @@ class ELuActivationT
     forward(const T (&ins)[size]) noexcept
     {
         for(int i = 0; i < size; ++i)
-            outs[i] = ins[i] > (T)0 ? ins[i] : (std::exp(ins[i]) - (T)1);
+            outs[i] = ins[i] > (T)0 ? ins[i] : (MathsProvider::exp(ins[i]) - (T)1);
     }
 
     /** Performs forward propagation for elu activation (with custom alpha parameter). */
@@ -380,7 +337,7 @@ class ELuActivationT
     {
         static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator;
         for(int i = 0; i < size; ++i)
-            outs[i] = ins[i] > (T)0 ? ins[i] : (alpha * (std::exp(ins[i]) - (T)1));
+            outs[i] = ins[i] > (T)0 ? ins[i] : (alpha * (MathsProvider::exp(ins[i]) - (T)1));
     }
 
     T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size];