Merge dd24568 into 3f80a27

jatinchowdhury18 · Nov 17, 2023 · a689564 · a689564
2 parents 3f80a27 + dd24568
commit a689564
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 218 deletions.
diff --git a/RTNeural/activation/activation_eigen.h b/RTNeural/activation/activation_eigen.h
@@ -2,12 +2,13 @@
 #define ACTIVATIONEIGEN_H_INCLUDED
 
 #include "../common.h"
+#include "../maths/maths_eigen.h"
 
 namespace RTNeural
 {
 
 /** Dynamic implementation of a tanh activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class TanhActivation : public Activation<T>
 {
 public:
@@ -29,7 +30,7 @@ class TanhActivation : public Activation<T>
     {
         inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(
             input, Layer<T>::in_size, 1);
-        outVec = inVec.array().tanh();
+        outVec = MathsProvider::tanh(inVec);
 
         std::copy(outVec.data(), outVec.data() + Layer<T>::in_size, out);
     }
@@ -39,7 +40,7 @@ class TanhActivation : public Activation<T>
 };
 
 /** Static implementation of a tanh activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class TanhActivationT
 {
     using v_type = Eigen::Matrix<T, size, 1>;
@@ -65,74 +66,7 @@ class TanhActivationT
     /** Performs forward propagation for tanh activation. */
     inline void forward(const v_type& ins) noexcept
     {
-        outs = ins.array().tanh();
-    }
-
-    Eigen::Map<v_type, RTNeuralEigenAlignment> outs;
-
-private:
-    T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size];
-};
-/** Dynamic implementation of an approximate tanh activation layer. */
-template <typename T>
-class FastTanh : public Activation<T>
-{
-public:
-    /** Constructs a tanh activation layer for a given size. */
-    explicit FastTanh(int size)
-        : Activation<T>(size, {}, "tanh")
-    {
-        inVec = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>::Zero(size, 1);
-        outVec = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>::Zero(size, 1);
-    }
-
-    FastTanh(std::initializer_list<int> sizes)
-        : FastTanh(*sizes.begin())
-    {
-    }
-
-    /** Performs forward propagation for tanh activation. */
-    inline void forward(const T* input, T* out) noexcept override
-    {
-        inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(
-            input, Layer<T>::in_size, 1);
-        outVec = fast_tanh<T>(inVec);
-
-        std::copy(outVec.data(), outVec.data() + Layer<T>::in_size, out);
-    }
-
-    Eigen::Matrix<T, Eigen::Dynamic, 1> inVec;
-    Eigen::Matrix<T, Eigen::Dynamic, 1> outVec;
-};
-
-/** Static implementation of an approximate tanh activation layer. */
-template <typename T, int size>
-class FastTanhT
-{
-    using v_type = Eigen::Matrix<T, size, 1>;
-
-public:
-    static constexpr auto in_size = size;
-    static constexpr auto out_size = size;
-
-    FastTanhT()
-        : outs(outs_internal)
-    {
-        outs = v_type::Zero();
-    }
-
-    /** Returns the name of this layer. */
-    std::string getName() const noexcept { return "tanh"; }
-
-    /** Returns true if this layer is an activation layer. */
-    constexpr bool isActivation() const noexcept { return true; }
-
-    void reset() { }
-
-    /** Performs forward propagation for tanh activation. */
-    inline void forward(const v_type& ins) noexcept
-    {
-        outs = fast_tanh<T>(ins);
+        outs = MathsProvider::tanh(ins);
     }
 
     Eigen::Map<v_type, RTNeuralEigenAlignment> outs;
@@ -211,7 +145,7 @@ class ReLuActivationT
 
 /** Dynamic implementation of a sigmoid activation layer. */
 
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class SigmoidActivation : public Activation<T>
 {
 public:
@@ -233,8 +167,7 @@ class SigmoidActivation : public Activation<T>
     {
         inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(
             input, Layer<T>::in_size, 1);
-        outVec = inVec.array();
-        sigmoid(outVec);
+        outVec = MathsProvider::sigmoid(inVec);
 
         std::copy(outVec.data(), outVec.data() + Layer<T>::in_size, out);
     }
@@ -244,7 +177,7 @@ class SigmoidActivation : public Activation<T>
 };
 
 /** Static implementation of a sigmoid activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class SigmoidActivationT
 {
     using v_type = Eigen::Matrix<T, size, 1>;
@@ -270,7 +203,7 @@ class SigmoidActivationT
     /** Performs forward propagation for sigmoid activation. */
     inline void forward(const v_type& ins) noexcept
     {
-        outs = (T)1 / (((T)-1 * ins.array()).array().exp() + (T)1);
+        outs = MathsProvider::sigmoid(ins);
     }
 
     Eigen::Map<v_type, RTNeuralEigenAlignment> outs;
@@ -280,7 +213,7 @@ class SigmoidActivationT
 };
 
 /** Dynamic implementation of a softmax activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class SoftmaxActivation : public Activation<T>
 {
 public:
@@ -302,8 +235,8 @@ class SoftmaxActivation : public Activation<T>
     {
         inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(
             input, Layer<T>::in_size, 1);
-        outVec = inVec.array();
-        softmax(outVec);
+        outVec = MathsProvider::exp (inVec);
+        outVec = outVec / outVec.sum();
 
         std::copy(outVec.data(), outVec.data() + Layer<T>::in_size, out);
     }
@@ -313,7 +246,7 @@ class SoftmaxActivation : public Activation<T>
 };
 
 /** Static implementation of a softmax activation layer. */
-template <typename T, int size>
+template <typename T, int size, typename MathsProvider = DefaultMathsProvider>
 class SoftmaxActivationT
 {
     using v_type = Eigen::Matrix<T, size, 1>;
@@ -339,7 +272,7 @@ class SoftmaxActivationT
     /** Performs forward propagation for softmax activation. */
     inline void forward(const v_type& ins) noexcept
     {
-        outs = ins.array().exp();
+        outs = MathsProvider::exp(ins);
         outs = outs / outs.sum();
     }
 
@@ -350,7 +283,7 @@ class SoftmaxActivationT
 };
 
 /** Dynamic implementation of a elu activation layer. */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class ELuActivation : public Activation<T>
 {
 public:
@@ -374,7 +307,7 @@ class ELuActivation : public Activation<T>
         inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(
             input, Layer<T>::in_size, 1);
 
-        outVec = (inVec.array() > (T)0).select(inVec, alpha * (inVec.array().exp() - ones.array()));
+        outVec = (inVec.array() > (T)0).select(inVec, alpha * (MathsProvider::exp(inVec) - ones.array()));
         std::copy(outVec.data(), outVec.data() + Layer<T>::in_size, out);
     }
 
@@ -390,7 +323,7 @@ class ELuActivation : public Activation<T>
 };
 
 /** Static implementation of a elu activation layer. */
-template <typename T, int size, int AlphaNumerator = 1, int AlphaDenominator = 1>
+template <typename T, int size, int AlphaNumerator = 1, int AlphaDenominator = 1, typename MathsProvider = DefaultMathsProvider>
 class ELuActivationT
 {
     using v_type = Eigen::Matrix<T, size, 1>;
@@ -418,7 +351,7 @@ class ELuActivationT
     inline typename std::enable_if<A_N == 1 && A_D == 1, void>::type
     forward(const v_type& ins) noexcept
     {
-        outs = (ins.array() > (T)0).select(ins, ins.array().exp() - ones.array());
+        outs = (ins.array() > (T)0).select(ins, MathsProvider::exp(ins) - ones.array());
     }
 
     /** Performs forward propagation for elu activation (with custom alpha parameter). */
@@ -427,7 +360,7 @@ class ELuActivationT
     forward(const v_type& ins) noexcept
     {
         static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator;
-        outs = (ins.array() > (T)0).select(ins, alpha * (ins.array().exp() - ones.array()));
+        outs = (ins.array() > (T)0).select(ins, alpha * (MathsProvider::exp(ins) - ones.array()));
     }
 
     Eigen::Map<v_type, RTNeuralEigenAlignment> outs;

diff --git a/RTNeural/common.h b/RTNeural/common.h
@@ -41,33 +41,6 @@ constexpr auto RTNeuralEigenAlignment = Eigen::Aligned32;
 #else
 constexpr auto RTNeuralEigenAlignment = Eigen::Aligned16;
 #endif
-
-template <typename T>
-static inline void
-sigmoid(Eigen::Matrix<T, Eigen::Dynamic, 1>& vector) noexcept
-{
-    vector = (T)1 / (((T)-1 * vector.array()).array().exp() + (T)1);
-}
-
-template <typename T>
-static inline void
-softmax(Eigen::Matrix<T, Eigen::Dynamic, 1>& vector) noexcept
-{
-    vector = vector.array().exp();
-    vector = vector / vector.sum();
-}
-
-template <typename T, typename MatType>
-static inline auto fast_tanh(const MatType& in) noexcept
-{
-    constexpr auto clamp = (T)5.7;
-    auto xc = in.cwiseMin(clamp).cwiseMax(-clamp); // clamp to range [-clamp, clamp]
-
-    auto x2 = xc.array().square();
-    auto numerator = xc.array().cwiseProduct(((T)2027025 + x2.cwiseProduct((T)270270 + x2.cwiseProduct((T)6930 + (T)36 * x2.array()).array()).array()));
-    auto denominator = (T)2027025 + x2.cwiseProduct((T)945945 + x2.cwiseProduct((T)51975 + x2.cwiseProduct((T)630 + x2.array()).array()).array()).array();
-    return numerator.cwiseProduct(denominator.inverse());
-}
 } // namespace RTNeural
 
 #elif RTNEURAL_USE_XSIMD

diff --git a/RTNeural/gru/gru_eigen.h b/RTNeural/gru/gru_eigen.h
@@ -3,6 +3,7 @@
 
 #include "../Layer.h"
 #include "../common.h"
+#include "../maths/maths_eigen.h"
 
 namespace RTNeural
 {
@@ -15,7 +16,7 @@ namespace RTNeural
  * please make sure to call `reset()` before your first call to
  * the `forward()` method.
  */
-template <typename T>
+template <typename T, typename MathsProvider = DefaultMathsProvider>
 class GRULayer : public Layer<T>
 {
 public:
@@ -61,14 +62,14 @@ class GRULayer : public Layer<T>
          *                  | r | )
          */
         gammaVec.noalias() = alphaVec.segment(0, 2 * Layer<T>::out_size) + betaVec.segment(0, 2 * Layer<T>::out_size);
-        sigmoid(gammaVec);
+        gammaVec = MathsProvider::sigmoid(gammaVec);
 
         /**
          * c = tanh( alpha[2*out_sizet : 3*out_sizet] + r.cwiseProduct(beta[2*out_sizet : 3*out_sizet] )
          * i.e. c = tanh( Wc * input + bc[0] + r.cwiseProduct(Uc * h(t-1) + bc[1]) )
          */
         cVec.noalias() = alphaVec.segment(2 * Layer<T>::out_size, Layer<T>::out_size) + gammaVec.segment(Layer<T>::out_size, Layer<T>::out_size).cwiseProduct(betaVec.segment(2 * Layer<T>::out_size, Layer<T>::out_size));
-        cVec = cVec.array().tanh();
+        cVec = MathsProvider::tanh(cVec);
 
         /**
          * h(t-1) = (1 - z).cwiseProduct(c) + z.cwiseProduct(h(t-1))
@@ -151,7 +152,9 @@ class GRULayer : public Layer<T>
  * please make sure to call `reset()` before your first call to
  * the `forward()` method.
  */
-template <typename T, int in_sizet, int out_sizet, SampleRateCorrectionMode sampleRateCorr = SampleRateCorrectionMode::None>
+template <typename T, int in_sizet, int out_sizet,
+    SampleRateCorrectionMode sampleRateCorr = SampleRateCorrectionMode::None,
+    typename MathsProvider = DefaultMathsProvider>
 class GRULayerT
 {
     using in_type = Eigen::Matrix<T, in_sizet, 1>;
@@ -214,14 +217,14 @@ class GRULayerT
          * gamma = sigmoid( | z |   = sigmoid(alpha[0 : 2*out_sizet] + beta[0 : 2*out_sizet])
          *                  | r | )
          */
-        gammaVec = sigmoid(alphaVec.segment(0, 2 * out_sizet) + betaVec.segment(0, 2 * out_sizet));
+        gammaVec = MathsProvider::sigmoid(alphaVec.segment(0, 2 * out_sizet) + betaVec.segment(0, 2 * out_sizet));
 
         /**
          * c = tanh( alpha[2*out_sizet : 3*out_sizet] + r.cwiseProduct(beta[2*out_sizet : 3*out_sizet] )
          * i.e. c = tanh( Wc * input + bc[0] + r.cwiseProduct(Uc * h(t-1) + bc[1]) )
          */
         cVec.noalias() = alphaVec.segment(2 * out_sizet, out_sizet) + gammaVec.segment(out_sizet, out_sizet).cwiseProduct(betaVec.segment(2 * out_sizet, out_sizet));
-        cVec = cVec.array().tanh();
+        cVec = MathsProvider::tanh(cVec);
 
         /**
          * h(t-1) = (1 - z).cwiseProduct(c) + z.cwiseProduct(h(t-1))
@@ -306,12 +309,6 @@ class GRULayerT
             delayVec[j] = delayVec[j + 1];
     }
 
-    template <typename Vector>
-    static inline auto sigmoid(const Vector& x) noexcept
-    {
-        return (T)1 / (((T)-1 * x.array()).array().exp() + (T)1);
-    }
-
     // kernel weights
     w_k_type wCombinedWeights;
     u_k_type uCombinedWeights;