From 96053aef80f5a9dba3458e129e18ddc14ff211a5 Mon Sep 17 00:00:00 2001 From: jatinchowdhury18 Date: Mon, 13 Nov 2023 21:51:10 -0800 Subject: [PATCH] Implement MathsProvider pattern for STL and XSIMD backends (#115) * Everything working with STL * XSIMD working as well * Fixing includes * Apply clang-format * Skipping approx tests * Fixing example --------- Co-authored-by: github-actions[bot] --- .github/workflows/bench.yml | 5 - RTNeural/activation/activation.h | 103 +++++---------- RTNeural/activation/activation_xsimd.h | 94 +++----------- RTNeural/common.h | 106 ++-------------- RTNeural/gru/gru.h | 25 ++-- RTNeural/gru/gru.tpp | 98 +++++++-------- RTNeural/gru/gru_xsimd.h | 30 +++-- RTNeural/gru/gru_xsimd.tpp | 98 +++++++-------- RTNeural/lstm/lstm.h | 37 +++--- RTNeural/lstm/lstm.tpp | 78 ++++++------ RTNeural/lstm/lstm_xsimd.h | 37 +++--- RTNeural/lstm/lstm_xsimd.tpp | 78 ++++++------ RTNeural/maths/maths_stl.h | 27 ++++ RTNeural/maths/maths_xsimd.h | 33 +++++ bench/layer_creator.hpp | 5 - .../custom_layer_model/GatedActivation_STL.h | 7 +- tests/CMakeLists.txt | 1 - tests/approx_tests.hpp | 119 ------------------ tests/tests.cpp | 8 -- 19 files changed, 370 insertions(+), 619 deletions(-) create mode 100644 RTNeural/maths/maths_stl.h create mode 100644 RTNeural/maths/maths_xsimd.h delete mode 100644 tests/approx_tests.hpp diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b47afa91..1595471f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -75,11 +75,6 @@ jobs: ./build/rtneural_layer_bench tanh 5 4 4 ./build/rtneural_layer_bench tanh 5 16 16 - - name: Benchmark FastTanh - run: | - ./build/rtneural_layer_bench fast_tanh 5 4 4 - ./build/rtneural_layer_bench fast_tanh 5 16 16 - - name: Benchmark ReLu run: | ./build/rtneural_layer_bench relu 5 4 4 diff --git a/RTNeural/activation/activation.h b/RTNeural/activation/activation.h index 91fb69d6..c591dd54 100644 --- a/RTNeural/activation/activation.h +++ b/RTNeural/activation/activation.h @@ -45,13 +45,14 @@ class Activation : public Layer #else #include "../common.h" +#include "../maths/maths_stl.h" #include namespace RTNeural { /** Dynamic implementation of a tanh activation layer. */ -template +template class TanhActivation final : public Activation { public: @@ -59,7 +60,7 @@ class TanhActivation final : public Activation explicit TanhActivation(int size) : Activation( size, [](T x) - { return std::tanh(x); }, + { return MathsProvider::tanh(x); }, "tanh") { } @@ -73,12 +74,12 @@ class TanhActivation final : public Activation inline void forward(const T* input, T* out) noexcept override { for(int i = 0; i < Layer::out_size; ++i) - out[i] = std::tanh(input[i]); + out[i] = MathsProvider::tanh(input[i]); } }; /** Static implementation of a tanh activation layer. */ -template +template class TanhActivationT { public: @@ -99,62 +100,7 @@ class TanhActivationT inline void forward(const T (&ins)[size]) noexcept { for(int i = 0; i < size; ++i) - outs[i] = std::tanh(ins[i]); - } - - T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; -}; - -/** Dynamic implementation of an approximate tanh activation layer. */ -template -class FastTanh final : public Activation -{ -public: - /** Constructs a tanh activation layer for a given size. */ - explicit FastTanh(int size) - : Activation( - size, [](T x) - { return tanh_approx(x); }, - "tanh") - { - } - - FastTanh(std::initializer_list sizes) - : FastTanh(*sizes.begin()) - { - } - - /** Performs forward propagation for tanh activation. */ - inline void forward(const T* input, T* out) noexcept override - { - for(int i = 0; i < Layer::out_size; ++i) - out[i] = tanh_approx(input[i]); - } -}; - -/** Static implementation of an approximate tanh activation layer. */ -template -class FastTanhT -{ -public: - static constexpr auto in_size = size; - static constexpr auto out_size = size; - - FastTanhT() = default; - - /** Returns the name of this layer. */ - std::string getName() const noexcept { return "tanh"; } - - /** Returns true since this layer is an activation layer. */ - constexpr bool isActivation() const noexcept { return true; } - - void reset() { } - - /** Performs forward propagation for tanh activation. */ - inline void forward(const T (&ins)[size]) noexcept - { - for(int i = 0; i < size; ++i) - outs[i] = tanh_approx(ins[i]); + outs[i] = MathsProvider::tanh(ins[i]); } T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; @@ -209,7 +155,7 @@ class ReLuActivationT }; /** Dynamic implementation of a sigmoid activation layer. */ -template +template class SigmoidActivation final : public Activation { public: @@ -217,7 +163,7 @@ class SigmoidActivation final : public Activation explicit SigmoidActivation(int size) : Activation( size, [](T x) - { return sigmoid(x); }, + { return MathsProvider::sigmoid(x); }, "sigmoid") { } @@ -229,7 +175,7 @@ class SigmoidActivation final : public Activation }; /** Static implementation of a sigmoid activation layer. */ -template +template class SigmoidActivationT { public: @@ -250,14 +196,14 @@ class SigmoidActivationT inline void forward(const T (&ins)[size]) noexcept { for(int i = 0; i < size; ++i) - outs[i] = sigmoid(ins[i]); + outs[i] = MathsProvider::sigmoid(ins[i]); } T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; }; /** Dynamic implementation of a softmax activation layer. */ -template +template class SoftmaxActivation final : public Activation { public: @@ -278,12 +224,23 @@ class SoftmaxActivation final : public Activation /** Performs forward propagation for softmax activation. */ inline void forward(const T* input, T* out) noexcept override { - softmax(input, out, Layer::out_size); + T exp_sum = 0; + for(int i = 0; i < Layer::out_size; ++i) + { + out[i] = MathsProvider::exp(input[i]); + exp_sum += out[i]; + } + + const auto exp_sum_recip = (T)1 / exp_sum; + for(int i = 0; i < Layer::out_size; ++i) + { + out[i] *= exp_sum_recip; + } } }; /** Static implementation of a softmax activation layer. */ -template +template class SoftmaxActivationT { public: @@ -306,7 +263,7 @@ class SoftmaxActivationT T exp_sum = 0; for(int i = 0; i < size; ++i) { - outs[i] = std::exp(ins[i]); + outs[i] = MathsProvider::exp(ins[i]); exp_sum += outs[i]; } @@ -321,7 +278,7 @@ class SoftmaxActivationT }; /** Dynamic implementation of a elu activation layer. */ -template +template class ELuActivation final : public Activation { public: @@ -329,7 +286,7 @@ class ELuActivation final : public Activation explicit ELuActivation(int size) : Activation( size, [this](T x) - { return x > (T)0 ? x : (alpha * (std::exp(x) - (T)1)); }, + { return x > (T)0 ? x : (alpha * (MathsProvider::exp(x) - (T)1)); }, "elu") { } @@ -347,7 +304,7 @@ class ELuActivation final : public Activation }; /** Static implementation of a elu activation layer. */ -template +template class ELuActivationT { public: @@ -370,7 +327,7 @@ class ELuActivationT forward(const T (&ins)[size]) noexcept { for(int i = 0; i < size; ++i) - outs[i] = ins[i] > (T)0 ? ins[i] : (std::exp(ins[i]) - (T)1); + outs[i] = ins[i] > (T)0 ? ins[i] : (MathsProvider::exp(ins[i]) - (T)1); } /** Performs forward propagation for elu activation (with custom alpha parameter). */ @@ -380,7 +337,7 @@ class ELuActivationT { static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator; for(int i = 0; i < size; ++i) - outs[i] = ins[i] > (T)0 ? ins[i] : (alpha * (std::exp(ins[i]) - (T)1)); + outs[i] = ins[i] > (T)0 ? ins[i] : (alpha * (MathsProvider::exp(ins[i]) - (T)1)); } T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; diff --git a/RTNeural/activation/activation_xsimd.h b/RTNeural/activation/activation_xsimd.h index 4f5e328c..d84cff17 100644 --- a/RTNeural/activation/activation_xsimd.h +++ b/RTNeural/activation/activation_xsimd.h @@ -2,12 +2,13 @@ #define ACTIVATIONXSIMD_H_INCLUDED #include "../common.h" +#include "../maths/maths_xsimd.h" namespace RTNeural { /** Dynamic implementation of a tanh activation layer. */ -template +template class TanhActivation : public Activation { public: @@ -25,12 +26,12 @@ class TanhActivation : public Activation /** Performs forward propagation for tanh activation. */ inline void forward(const T* input, T* out) noexcept override { - tanh(input, out, Layer::in_size); + tanh(input, out, Layer::in_size); } }; /** Static implementation of a tanh activation layer. */ -template +template class TanhActivationT { using v_type = xsimd::simd_type; @@ -59,66 +60,7 @@ class TanhActivationT inline void forward(const v_type (&ins)[v_io_size]) noexcept { for(int i = 0; i < v_io_size; ++i) - outs[i] = xsimd::tanh(ins[i]); - } - - v_type outs[v_io_size]; -}; - -/** Dynamic implementation of an approximate tanh activation layer. */ -template -class FastTanh : public Activation -{ -public: - /** Constructs a tanh activation layer for a given size. */ - explicit FastTanh(int size) - : Activation(size, {}, "tanh") - { - } - - FastTanh(std::initializer_list sizes) - : FastTanh(*sizes.begin()) - { - } - - /** Performs forward propagation for tanh activation. */ - inline void forward(const T* input, T* out) noexcept override - { - fast_tanh(input, out, Layer::in_size); - } -}; - -/** Static implementation of an approximate tanh activation layer. */ -template -class FastTanhT -{ - using v_type = xsimd::simd_type; - static constexpr auto v_size = (int)v_type::size; - static constexpr auto v_io_size = ceil_div(size, v_size); - -public: - static constexpr auto in_size = size; - static constexpr auto out_size = size; - - FastTanhT() - { - for(int i = 0; i < v_io_size; ++i) - outs[i] = v_type((T)0); - } - - /** Returns the name of this layer. */ - std::string getName() const noexcept { return "tanh"; } - - /** Returns true since this layer is an activation layer. */ - constexpr bool isActivation() const noexcept { return true; } - - void reset() { } - - /** Performs forward propagation for tanh activation. */ - inline void forward(const v_type (&ins)[v_io_size]) noexcept - { - for(int i = 0; i < v_io_size; ++i) - outs[i] = fast_tanh(ins[i]); + outs[i] = MathsProvider::tanh(ins[i]); } v_type outs[v_io_size]; @@ -190,7 +132,7 @@ class ReLuActivationT }; /** Dynamic implementation of a sigmoid activation layer. */ -template +template class SigmoidActivation : public Activation { public: @@ -208,12 +150,12 @@ class SigmoidActivation : public Activation /** Performs forward propagation for sigmoid activation. */ inline void forward(const T* input, T* out) noexcept override { - sigmoid(input, out, Layer::in_size); + sigmoid(input, out, Layer::in_size); } }; /** Static implementation of a sigmoid activation layer. */ -template +template class SigmoidActivationT { using v_type = xsimd::simd_type; @@ -242,14 +184,14 @@ class SigmoidActivationT inline void forward(const v_type (&ins)[v_io_size]) noexcept { for(int i = 0; i < v_io_size; ++i) - outs[i] = (T)1.0 / ((T)1.0 + xsimd::exp(-ins[i])); + outs[i] = MathsProvider::sigmoid(ins[i]); } v_type outs[v_io_size]; }; /** Dynamic implementation of a softmax activation layer. */ -template +template class SoftmaxActivation : public Activation { public: @@ -267,12 +209,12 @@ class SoftmaxActivation : public Activation /** Performs forward propagation for softmax activation. */ inline void forward(const T* input, T* out) noexcept override { - softmax(input, out, Layer::in_size); + softmax(input, out, Layer::in_size); } }; /** Static implementation of a softmax activation layer. */ -template +template class SoftmaxActivationT { using v_type = xsimd::simd_type; @@ -303,7 +245,7 @@ class SoftmaxActivationT v_type exp_sum {}; for(int i = 0; i < v_io_size; ++i) { - outs[i] = xsimd::exp(ins[i]); + outs[i] = MathsProvider::exp(ins[i]); exp_sum += outs[i]; } @@ -316,7 +258,7 @@ class SoftmaxActivationT }; /** Dynamic implementation of a elu activation layer. */ -template +template class ELuActivation final : public Activation { public: @@ -334,7 +276,7 @@ class ELuActivation final : public Activation /** Performs forward propagation for softmax activation. */ inline void forward(const T* input, T* out) noexcept override { - elu(input, out, Layer::in_size, alpha); + elu(input, out, Layer::in_size, alpha); } /** Sets a custom value for the layer's "alpha" parameter. */ @@ -345,7 +287,7 @@ class ELuActivation final : public Activation }; /** Static implementation of a elu activation layer. */ -template +template class ELuActivationT { using v_type = xsimd::simd_type; @@ -372,7 +314,7 @@ class ELuActivationT forward(const v_type (&ins)[v_io_size]) noexcept { for(int i = 0; i < v_io_size; ++i) - outs[i] = xsimd::select(ins[i] > (T)0, ins[i], xsimd::exp(ins[i]) - (T)1); + outs[i] = xsimd::select(ins[i] > (T)0, ins[i], MathsProvider::exp(ins[i]) - (T)1); } /** Performs forward propagation for elu activation (with custom alpha parameter). */ @@ -382,7 +324,7 @@ class ELuActivationT { static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator; for(int i = 0; i < v_io_size; ++i) - outs[i] = xsimd::select(ins[i] > (T)0, ins[i], alpha * (xsimd::exp(ins[i]) - (T)1)); + outs[i] = xsimd::select(ins[i] > (T)0, ins[i], alpha * (MathsProvider::exp(ins[i]) - (T)1)); } v_type outs[v_io_size]; diff --git a/RTNeural/common.h b/RTNeural/common.h index 431d493c..2d0d02a5 100644 --- a/RTNeural/common.h +++ b/RTNeural/common.h @@ -29,19 +29,6 @@ constexpr T ceil_div(T num, T den) { return (num + den - 1) / den; } - -/** Pade approximation of std::tanh() */ -template -static inline T tanh_approx(T x) noexcept -{ - constexpr auto clamp = (T)5.7; - x = x > clamp ? clamp : (x < -clamp ? -clamp : x); // clamp to range [-clamp, clamp] - - auto x2 = x * x; - auto numerator = x * ((T)2027025 + x2 * ((T)270270 + x2 * ((T)6930 + (T)36 * x2))); - auto denominator = (T)2027025 + x2 * ((T)945945 + x2 * ((T)51975 + x2 * ((T)630 + x2))); - return numerator / denominator; -} } // namespace RTNeural #if RTNEURAL_USE_EIGEN @@ -175,7 +162,7 @@ static inline void vCopy(const T* in, T* out, int dim) noexcept out[i] = in[i]; } -template +template static inline void sigmoid(const T* in, T* out, int dim) noexcept { using b_type = xsimd::simd_type; @@ -186,16 +173,16 @@ static inline void sigmoid(const T* in, T* out, int dim) noexcept for(int i = 0; i < vec_size; i += inc) { b_type x_vec = xsimd::load_aligned(&in[i]); - b_type y_vec = (T)1.0 / ((T)1.0 + xsimd::exp(-x_vec)); + b_type y_vec = MathsProvider::sigmoid(x_vec); xsimd::store_aligned(&out[i], y_vec); } // Remaining part that cannot be vectorize for(auto i = vec_size; i < dim; ++i) - out[i] = 1.0 / (1.0 + std::exp(-in[i])); + out[i] = MathsProvider::sigmoid(in[i]); } -template +template static inline void softmax(const T* in, T* out, int dim) noexcept { using b_type = xsimd::simd_type; @@ -208,7 +195,7 @@ static inline void softmax(const T* in, T* out, int dim) noexcept for(int i = 0; i < vec_size; i += inc) { b_type x_vec = xsimd::load_aligned(&in[i]); - b_type y_vec = xsimd::exp(x_vec); + b_type y_vec = MathsProvider::exp(x_vec); exp_sum_vec += y_vec; xsimd::store_aligned(&out[i], y_vec); } @@ -218,7 +205,7 @@ static inline void softmax(const T* in, T* out, int dim) noexcept // Remaining part that cannot be vectorize for(auto i = vec_size; i < dim; ++i) { - out[i] = std::exp(in[i]); + out[i] = MathsProvider::exp(in[i]); exp_sum += out[i]; } @@ -237,7 +224,7 @@ static inline void softmax(const T* in, T* out, int dim) noexcept } } -template +template static inline void tanh(const T* in, T* out, int dim) noexcept { using b_type = xsimd::simd_type; @@ -248,16 +235,16 @@ static inline void tanh(const T* in, T* out, int dim) noexcept for(int i = 0; i < vec_size; i += inc) { b_type x_vec = xsimd::load_aligned(&in[i]); - b_type y_vec = xsimd::tanh(x_vec); + b_type y_vec = MathsProvider::tanh(x_vec); xsimd::store_aligned(&out[i], y_vec); } // Remaining part that cannot be vectorize for(auto i = vec_size; i < dim; ++i) - out[i] = std::tanh(in[i]); + out[i] = MathsProvider::tanh(in[i]); } -template +template static inline void elu(const T* in, T* out, int dim, T alpha) noexcept { using b_type = xsimd::simd_type; @@ -268,58 +255,14 @@ static inline void elu(const T* in, T* out, int dim, T alpha) noexcept for(int i = 0; i < vec_size; i += inc) { b_type x_vec = xsimd::load_aligned(&in[i]); - b_type y_vec = xsimd::select(x_vec > (T)0, x_vec, alpha * (xsimd::exp(x_vec) - (T)1)); + b_type y_vec = xsimd::select(x_vec > (T)0, x_vec, alpha * (MathsProvider::exp(x_vec) - (T)1)); xsimd::store_aligned(&out[i], y_vec); } // Remaining part that cannot be vectorized for(auto i = vec_size; i < dim; ++i) - out[i] = in[i] > (T)0 ? in[i] : (alpha * (std::exp(in[i]) - (T)1)); -} - -template -static inline xsimd::simd_type fast_tanh(const xsimd::simd_type& x) noexcept -{ - using b_type = xsimd::simd_type; - - static const b_type clamp_hi((T)5.7); - static const b_type clamp_lo((T)-5.7); - auto xc = xsimd::clip(x, clamp_lo, clamp_hi); // clamp to range [-clamp, clamp] - - static const b_type v2027025((T)2027025); - static const b_type v270270((T)270270); - static const b_type v6930((T)6930); - static const b_type v36((T)36); - static const b_type v945945((T)945945); - static const b_type v51975((T)51975); - static const b_type v630((T)630); - - auto x2 = xc * xc; - auto numerator = xc * (v2027025 + x2 * (v270270 + x2 * (v6930 + v36 * x2))); - auto denominator = v2027025 + x2 * (v945945 + x2 * (v51975 + x2 * (v630 + x2))); - return numerator / denominator; -} - -template -static inline void fast_tanh(const T* in, T* out, int dim) noexcept -{ - using b_type = xsimd::simd_type; - constexpr auto inc = (int)b_type::size; - - // size for which the vectorization is possible - auto vec_size = dim - dim % inc; - for(int i = 0; i < vec_size; i += inc) - { - b_type x_vec = xsimd::load_aligned(&in[i]); - b_type y_vec = fast_tanh(x_vec); - xsimd::store_aligned(&out[i], y_vec); - } - - // Remaining part that cannot be vectorize - for(auto i = vec_size; i < dim; ++i) - out[i] = tanh_approx(in[i]); + out[i] = in[i] > (T)0 ? in[i] : (alpha * (MathsProvider::exp(in[i]) - (T)1)); } - } // namespace RTNeural #else // STL backend @@ -329,36 +272,11 @@ static inline void fast_tanh(const T* in, T* out, int dim) noexcept namespace RTNeural { - template static inline T vMult(const T* arg1, const T* arg2, int dim) noexcept { return std::inner_product(arg1, arg1 + dim, arg2, (T)0); } - -template -static inline T sigmoid(T value) noexcept -{ - return (T)1 / ((T)1 + std::exp(-value)); -} - -template -static inline void softmax(const T* input, T* out, int size) noexcept -{ - T exp_sum = 0; - for(int i = 0; i < size; ++i) - { - out[i] = std::exp(input[i]); - exp_sum += out[i]; - } - - const auto exp_sum_recip = (T)1 / exp_sum; - for(int i = 0; i < size; ++i) - { - out[i] *= exp_sum_recip; - } -} - } // namespace RTNeural #endif diff --git a/RTNeural/gru/gru.h b/RTNeural/gru/gru.h index 1d1bcb96..406cf4ca 100644 --- a/RTNeural/gru/gru.h +++ b/RTNeural/gru/gru.h @@ -12,6 +12,7 @@ #else #include "../Layer.h" #include "../common.h" +#include "../maths/maths_stl.h" #include namespace RTNeural @@ -25,7 +26,7 @@ namespace RTNeural * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class GRULayer final : public Layer { public: @@ -47,9 +48,9 @@ class GRULayer final : public Layer { for(int i = 0; i < Layer::out_size; ++i) { - zVec[i] = sigmoid(vMult(zWeights.W[i], input, Layer::in_size) + vMult(zWeights.U[i], ht1, Layer::out_size) + zWeights.b[0][i] + zWeights.b[1][i]); - rVec[i] = sigmoid(vMult(rWeights.W[i], input, Layer::in_size) + vMult(rWeights.U[i], ht1, Layer::out_size) + rWeights.b[0][i] + rWeights.b[1][i]); - cVec[i] = std::tanh(vMult(cWeights.W[i], input, Layer::in_size) + rVec[i] * (vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[1][i]) + cWeights.b[0][i]); + zVec[i] = MathsProvider::sigmoid(vMult(zWeights.W[i], input, Layer::in_size) + vMult(zWeights.U[i], ht1, Layer::out_size) + zWeights.b[0][i] + zWeights.b[1][i]); + rVec[i] = MathsProvider::sigmoid(vMult(rWeights.W[i], input, Layer::in_size) + vMult(rWeights.U[i], ht1, Layer::out_size) + rWeights.b[0][i] + rWeights.b[1][i]); + cVec[i] = MathsProvider::tanh(vMult(cWeights.W[i], input, Layer::in_size) + rVec[i] * (vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[1][i]) + cWeights.b[0][i]); h[i] = ((T)1 - zVec[i]) * cVec[i] + zVec[i] * ht1[i]; } @@ -142,7 +143,9 @@ class GRULayer final : public Layer * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class GRULayerT { public: @@ -179,19 +182,19 @@ class GRULayerT recurrent_mat_mul(outs, Uz, zt); kernel_mat_mul(ins, Wz, kernel_outs); for(int i = 0; i < out_size; ++i) - zt[i] = sigmoid(zt[i] + bz[i] + kernel_outs[i]); + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + kernel_outs[i]); // compute rt recurrent_mat_mul(outs, Ur, rt); kernel_mat_mul(ins, Wr, kernel_outs); for(int i = 0; i < out_size; ++i) - rt[i] = sigmoid(rt[i] + br[i] + kernel_outs[i]); + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + kernel_outs[i]); // compute h_hat recurrent_mat_mul(outs, Uh, ct); kernel_mat_mul(ins, Wh, kernel_outs); for(int i = 0; i < out_size; ++i) - ht[i] = std::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + kernel_outs[i]); + ht[i] = MathsProvider::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + kernel_outs[i]); computeOutput(); } @@ -204,17 +207,17 @@ class GRULayerT // compute zt recurrent_mat_mul(outs, Uz, zt); for(int i = 0; i < out_size; ++i) - zt[i] = sigmoid(zt[i] + bz[i] + (Wz_1[i] * ins[0])); + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + (Wz_1[i] * ins[0])); // compute rt recurrent_mat_mul(outs, Ur, rt); for(int i = 0; i < out_size; ++i) - rt[i] = sigmoid(rt[i] + br[i] + (Wr_1[i] * ins[0])); + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + (Wr_1[i] * ins[0])); // compute h_hat recurrent_mat_mul(outs, Uh, ct); for(int i = 0; i < out_size; ++i) - ht[i] = std::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + (Wh_1[i] * ins[0])); + ht[i] = MathsProvider::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + (Wh_1[i] * ins[0])); computeOutput(); } diff --git a/RTNeural/gru/gru.tpp b/RTNeural/gru/gru.tpp index 98cbcebd..e4d9b540 100644 --- a/RTNeural/gru/gru.tpp +++ b/RTNeural/gru/gru.tpp @@ -4,8 +4,8 @@ namespace RTNeural { #if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD -template -GRULayer::GRULayer(int in_size, int out_size) +template +GRULayer::GRULayer(int in_size, int out_size) : Layer(in_size, out_size) , zWeights(in_size, out_size) , rWeights(in_size, out_size) @@ -17,28 +17,28 @@ GRULayer::GRULayer(int in_size, int out_size) cVec = new T[out_size]; } -template -GRULayer::GRULayer(std::initializer_list sizes) - : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) +template +GRULayer::GRULayer(std::initializer_list sizes) + : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) { } -template -GRULayer::GRULayer(const GRULayer& other) - : GRULayer(other.in_size, other.out_size) +template +GRULayer::GRULayer(const GRULayer& other) + : GRULayer(other.in_size, other.out_size) { } -template -GRULayer& GRULayer::operator=(const GRULayer& other) +template +GRULayer& GRULayer::operator=(const GRULayer& other) { if(&other != this) - *this = GRULayer(other); + *this = GRULayer(other); return *this; } -template -GRULayer::~GRULayer() +template +GRULayer::~GRULayer() { delete[] ht1; delete[] zVec; @@ -46,8 +46,8 @@ GRULayer::~GRULayer() delete[] cVec; } -template -GRULayer::WeightSet::WeightSet(int in_size, int out_size) +template +GRULayer::WeightSet::WeightSet(int in_size, int out_size) : out_size(out_size) { W = new T*[out_size]; @@ -66,8 +66,8 @@ GRULayer::WeightSet::WeightSet(int in_size, int out_size) } } -template -GRULayer::WeightSet::~WeightSet() +template +GRULayer::WeightSet::~WeightSet() { for(int i = 0; i < kNumBiasLayers; ++i) { @@ -85,8 +85,8 @@ GRULayer::WeightSet::~WeightSet() delete[] U; } -template -void GRULayer::setWVals(const std::vector>& wVals) +template +void GRULayer::setWVals(const std::vector>& wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -99,8 +99,8 @@ void GRULayer::setWVals(const std::vector>& wVals) } } -template -void GRULayer::setWVals(T** wVals) +template +void GRULayer::setWVals(T** wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -113,8 +113,8 @@ void GRULayer::setWVals(T** wVals) } } -template -void GRULayer::setUVals(const std::vector>& uVals) +template +void GRULayer::setUVals(const std::vector>& uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -127,8 +127,8 @@ void GRULayer::setUVals(const std::vector>& uVals) } } -template -void GRULayer::setUVals(T** uVals) +template +void GRULayer::setUVals(T** uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -141,8 +141,8 @@ void GRULayer::setUVals(T** uVals) } } -template -void GRULayer::setBVals(const std::vector>& bVals) +template +void GRULayer::setBVals(const std::vector>& bVals) { for(int i = 0; i < 2; ++i) { @@ -155,8 +155,8 @@ void GRULayer::setBVals(const std::vector>& bVals) } } -template -void GRULayer::setBVals(T** bVals) +template +void GRULayer::setBVals(T** bVals) { for(int i = 0; i < 2; ++i) { @@ -169,8 +169,8 @@ void GRULayer::setBVals(T** bVals) } } -template -T GRULayer::getWVal(int i, int k) const noexcept +template +T GRULayer::getWVal(int i, int k) const noexcept { T** set = zWeights.W; if(k > 2 * Layer::out_size) @@ -187,8 +187,8 @@ T GRULayer::getWVal(int i, int k) const noexcept return set[i][k]; } -template -T GRULayer::getUVal(int i, int k) const noexcept +template +T GRULayer::getUVal(int i, int k) const noexcept { T** set = zWeights.U; if(k > 2 * Layer::out_size) @@ -205,8 +205,8 @@ T GRULayer::getUVal(int i, int k) const noexcept return set[i][k]; } -template -T GRULayer::getBVal(int i, int k) const noexcept +template +T GRULayer::getBVal(int i, int k) const noexcept { T** set = zWeights.b; if(k > 2 * Layer::out_size) @@ -224,8 +224,8 @@ T GRULayer::getBVal(int i, int k) const noexcept } //==================================================== -template -GRULayerT::GRULayerT() +template +GRULayerT::GRULayerT() { for(int i = 0; i < out_size; ++i) { @@ -269,10 +269,10 @@ GRULayerT::GRULayerT() reset(); } -template +template template std::enable_if_t -GRULayerT::prepare(int delaySamples) +GRULayerT::prepare(int delaySamples) { delayWriteIdx = delaySamples - 1; outs_delayed.resize(delayWriteIdx + 1, {}); @@ -280,10 +280,10 @@ GRULayerT::prepare(int delaySamples) reset(); } -template +template template std::enable_if_t -GRULayerT::prepare(T delaySamples) +GRULayerT::prepare(T delaySamples) { const auto delayOffFactor = delaySamples - std::floor(delaySamples); delayMult = (T)1 - delayOffFactor; @@ -295,8 +295,8 @@ GRULayerT::prepare(T delaySamples) reset(); } -template -void GRULayerT::reset() +template +void GRULayerT::reset() { if(sampleRateCorr != SampleRateCorrectionMode::None) { @@ -310,8 +310,8 @@ void GRULayerT::reset() } // kernel weights -template -void GRULayerT::setWVals(const std::vector>& wVals) +template +void GRULayerT::setWVals(const std::vector>& wVals) { for(int i = 0; i < in_size; ++i) { @@ -332,8 +332,8 @@ void GRULayerT::setWVals(const std::vect } // recurrent weights -template -void GRULayerT::setUVals(const std::vector>& uVals) +template +void GRULayerT::setUVals(const std::vector>& uVals) { for(int i = 0; i < out_size; ++i) { @@ -347,8 +347,8 @@ void GRULayerT::setUVals(const std::vect } // biases -template -void GRULayerT::setBVals(const std::vector>& bVals) +template +void GRULayerT::setBVals(const std::vector>& bVals) { for(int k = 0; k < out_size; ++k) { diff --git a/RTNeural/gru/gru_xsimd.h b/RTNeural/gru/gru_xsimd.h index d0527f0e..556628ea 100644 --- a/RTNeural/gru/gru_xsimd.h +++ b/RTNeural/gru/gru_xsimd.h @@ -3,6 +3,7 @@ #include "../Layer.h" #include "../common.h" +#include "../maths/maths_xsimd.h" #include namespace RTNeural { @@ -15,7 +16,7 @@ namespace RTNeural * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class GRULayer : public Layer { public: @@ -45,17 +46,17 @@ class GRULayer : public Layer vAdd(zVec.data(), zWeights.b[0].data(), zVec.data(), Layer::out_size); vAdd(zVec.data(), zWeights.b[1].data(), zVec.data(), Layer::out_size); - sigmoid(zVec.data(), zVec.data(), Layer::out_size); + sigmoid(zVec.data(), zVec.data(), Layer::out_size); vAdd(rVec.data(), rWeights.b[0].data(), rVec.data(), Layer::out_size); vAdd(rVec.data(), rWeights.b[1].data(), rVec.data(), Layer::out_size); - sigmoid(rVec.data(), rVec.data(), Layer::out_size); + sigmoid(rVec.data(), rVec.data(), Layer::out_size); vAdd(cTmp.data(), cWeights.b[1].data(), cTmp.data(), Layer::out_size); vProd(cTmp.data(), rVec.data(), cTmp.data(), Layer::out_size); vAdd(cTmp.data(), cVec.data(), cVec.data(), Layer::out_size); vAdd(cVec.data(), cWeights.b[0].data(), cVec.data(), Layer::out_size); - tanh(cVec.data(), cVec.data(), Layer::out_size); + tanh(cVec.data(), cVec.data(), Layer::out_size); vSub(ones.data(), zVec.data(), h, Layer::out_size); vProd(h, cVec.data(), h, Layer::out_size); @@ -156,7 +157,9 @@ class GRULayer : public Layer * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class GRULayerT { using v_type = xsimd::simd_type; @@ -198,19 +201,19 @@ class GRULayerT recurrent_mat_mul(outs, Uz, zt); kernel_mat_mul(ins, Wz, kernel_outs); for(int i = 0; i < v_out_size; ++i) - zt[i] = sigmoid(zt[i] + bz[i] + kernel_outs[i]); + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + kernel_outs[i]); // compute rt recurrent_mat_mul(outs, Ur, rt); kernel_mat_mul(ins, Wr, kernel_outs); for(int i = 0; i < v_out_size; ++i) - rt[i] = sigmoid(rt[i] + br[i] + kernel_outs[i]); + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + kernel_outs[i]); // compute h_hat recurrent_mat_mul(outs, Uh, ct); kernel_mat_mul(ins, Wh, kernel_outs); for(int i = 0; i < v_out_size; ++i) - ht[i] = xsimd::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], bh0[i] + kernel_outs[i])); + ht[i] = MathsProvider::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], bh0[i] + kernel_outs[i])); computeOutput(); } @@ -223,17 +226,17 @@ class GRULayerT // compute zt recurrent_mat_mul(outs, Uz, zt); for(int i = 0; i < v_out_size; ++i) - zt[i] = sigmoid(xsimd::fma(Wz_1[i], ins[0], zt[i] + bz[i])); + zt[i] = MathsProvider::sigmoid(xsimd::fma(Wz_1[i], ins[0], zt[i] + bz[i])); // compute rt recurrent_mat_mul(outs, Ur, rt); for(int i = 0; i < v_out_size; ++i) - rt[i] = sigmoid(xsimd::fma(Wr_1[i], ins[0], rt[i] + br[i])); + rt[i] = MathsProvider::sigmoid(xsimd::fma(Wr_1[i], ins[0], rt[i] + br[i])); // compute h_hat recurrent_mat_mul(outs, Uh, ct); for(int i = 0; i < v_out_size; ++i) - ht[i] = xsimd::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], xsimd::fma(Wh_1[i], ins[0], bh0[i]))); + ht[i] = MathsProvider::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], xsimd::fma(Wh_1[i], ins[0], bh0[i]))); computeOutput(); } @@ -342,11 +345,6 @@ class GRULayerT } } - static inline v_type sigmoid(v_type x) noexcept - { - return (T)1.0 / ((T)1.0 + xsimd::exp(-x)); - } - // kernel weights v_type Wz[in_size][v_out_size]; v_type Wr[in_size][v_out_size]; diff --git a/RTNeural/gru/gru_xsimd.tpp b/RTNeural/gru/gru_xsimd.tpp index 141ea7e5..7833b0ed 100644 --- a/RTNeural/gru/gru_xsimd.tpp +++ b/RTNeural/gru/gru_xsimd.tpp @@ -3,8 +3,8 @@ namespace RTNeural { -template -GRULayer::GRULayer(int in_size, int out_size) +template +GRULayer::GRULayer(int in_size, int out_size) : Layer(in_size, out_size) , zWeights(in_size, out_size) , rWeights(in_size, out_size) @@ -22,29 +22,29 @@ GRULayer::GRULayer(int in_size, int out_size) ones.resize(out_size, (T)1); } -template -GRULayer::GRULayer(std::initializer_list sizes) - : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) +template +GRULayer::GRULayer(std::initializer_list sizes) + : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) { } -template -GRULayer::GRULayer(const GRULayer& other) - : GRULayer(other.in_size, other.out_size) +template +GRULayer::GRULayer(const GRULayer& other) + : GRULayer(other.in_size, other.out_size) { } -template -GRULayer& GRULayer::operator=(const GRULayer& other) +template +GRULayer& GRULayer::operator=(const GRULayer& other) { - return *this = GRULayer(other); + return *this = GRULayer(other); } -template -GRULayer::~GRULayer() = default; +template +GRULayer::~GRULayer() = default; -template -GRULayer::WeightSet::WeightSet(int in_size, int out_size) +template +GRULayer::WeightSet::WeightSet(int in_size, int out_size) : out_size(out_size) { W = vec2_type(out_size, vec_type(in_size, (T)0)); @@ -54,11 +54,11 @@ GRULayer::WeightSet::WeightSet(int in_size, int out_size) b[1].resize(out_size, (T)0); } -template -GRULayer::WeightSet::~WeightSet() = default; +template +GRULayer::WeightSet::~WeightSet() = default; -template -void GRULayer::setWVals(const std::vector>& wVals) +template +void GRULayer::setWVals(const std::vector>& wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -71,8 +71,8 @@ void GRULayer::setWVals(const std::vector>& wVals) } } -template -void GRULayer::setWVals(T** wVals) +template +void GRULayer::setWVals(T** wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -85,8 +85,8 @@ void GRULayer::setWVals(T** wVals) } } -template -void GRULayer::setUVals(const std::vector>& uVals) +template +void GRULayer::setUVals(const std::vector>& uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -99,8 +99,8 @@ void GRULayer::setUVals(const std::vector>& uVals) } } -template -void GRULayer::setUVals(T** uVals) +template +void GRULayer::setUVals(T** uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -113,8 +113,8 @@ void GRULayer::setUVals(T** uVals) } } -template -void GRULayer::setBVals(const std::vector>& bVals) +template +void GRULayer::setBVals(const std::vector>& bVals) { for(int i = 0; i < 2; ++i) { @@ -127,8 +127,8 @@ void GRULayer::setBVals(const std::vector>& bVals) } } -template -void GRULayer::setBVals(T** bVals) +template +void GRULayer::setBVals(T** bVals) { for(int i = 0; i < 2; ++i) { @@ -141,8 +141,8 @@ void GRULayer::setBVals(T** bVals) } } -template -T GRULayer::getWVal(int i, int k) const noexcept +template +T GRULayer::getWVal(int i, int k) const noexcept { T** set = zWeights.W; if(k > 2 * Layer::out_size) @@ -159,8 +159,8 @@ T GRULayer::getWVal(int i, int k) const noexcept return set[i][k]; } -template -T GRULayer::getUVal(int i, int k) const noexcept +template +T GRULayer::getUVal(int i, int k) const noexcept { T** set = zWeights.U; if(k > 2 * Layer::out_size) @@ -177,8 +177,8 @@ T GRULayer::getUVal(int i, int k) const noexcept return set[i][k]; } -template -T GRULayer::getBVal(int i, int k) const noexcept +template +T GRULayer::getBVal(int i, int k) const noexcept { T** set = zWeights.b; if(k > 2 * Layer::out_size) @@ -196,8 +196,8 @@ T GRULayer::getBVal(int i, int k) const noexcept } //==================================================== -template -GRULayerT::GRULayerT() +template +GRULayerT::GRULayerT() { for(int i = 0; i < v_out_size; ++i) { @@ -244,10 +244,10 @@ GRULayerT::GRULayerT() reset(); } -template +template template std::enable_if_t -GRULayerT::prepare(int delaySamples) +GRULayerT::prepare(int delaySamples) { delayWriteIdx = delaySamples - 1; outs_delayed.resize(delayWriteIdx + 1, {}); @@ -255,10 +255,10 @@ GRULayerT::prepare(int delaySamples) reset(); } -template +template template std::enable_if_t -GRULayerT::prepare(T delaySamples) +GRULayerT::prepare(T delaySamples) { const auto delayOffFactor = delaySamples - std::floor(delaySamples); delayMult = (T)1 - delayOffFactor; @@ -270,8 +270,8 @@ GRULayerT::prepare(T delaySamples) reset(); } -template -void GRULayerT::reset() +template +void GRULayerT::reset() { if(sampleRateCorr != SampleRateCorrectionMode::None) { @@ -285,8 +285,8 @@ void GRULayerT::reset() } // kernel weights -template -void GRULayerT::setWVals(const std::vector>& wVals) +template +void GRULayerT::setWVals(const std::vector>& wVals) { for(int i = 0; i < out_size; ++i) { @@ -307,8 +307,8 @@ void GRULayerT::setWVals(const std::vect } // recurrent weights -template -void GRULayerT::setUVals(const std::vector>& uVals) +template +void GRULayerT::setUVals(const std::vector>& uVals) { for(int i = 0; i < out_size; ++i) { @@ -322,8 +322,8 @@ void GRULayerT::setUVals(const std::vect } // biases -template -void GRULayerT::setBVals(const std::vector>& bVals) +template +void GRULayerT::setBVals(const std::vector>& bVals) { for(int k = 0; k < out_size; ++k) { diff --git a/RTNeural/lstm/lstm.h b/RTNeural/lstm/lstm.h index 3dea391e..a9d3f374 100644 --- a/RTNeural/lstm/lstm.h +++ b/RTNeural/lstm/lstm.h @@ -10,6 +10,7 @@ #else #include "../Layer.h" #include "../common.h" +#include "../maths/maths_stl.h" #include namespace RTNeural @@ -23,7 +24,7 @@ namespace RTNeural * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class LSTMLayer final : public Layer { public: @@ -45,12 +46,12 @@ class LSTMLayer final : public Layer { for(int i = 0; i < Layer::out_size; ++i) { - fVec[i] = sigmoid(vMult(fWeights.W[i], input, Layer::in_size) + vMult(fWeights.U[i], ht1, Layer::out_size) + fWeights.b[i]); - iVec[i] = sigmoid(vMult(iWeights.W[i], input, Layer::in_size) + vMult(iWeights.U[i], ht1, Layer::out_size) + iWeights.b[i]); - oVec[i] = sigmoid(vMult(oWeights.W[i], input, Layer::in_size) + vMult(oWeights.U[i], ht1, Layer::out_size) + oWeights.b[i]); - ctVec[i] = std::tanh(vMult(cWeights.W[i], input, Layer::in_size) + vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[i]); + fVec[i] = MathsProvider::sigmoid(vMult(fWeights.W[i], input, Layer::in_size) + vMult(fWeights.U[i], ht1, Layer::out_size) + fWeights.b[i]); + iVec[i] = MathsProvider::sigmoid(vMult(iWeights.W[i], input, Layer::in_size) + vMult(iWeights.U[i], ht1, Layer::out_size) + iWeights.b[i]); + oVec[i] = MathsProvider::sigmoid(vMult(oWeights.W[i], input, Layer::in_size) + vMult(oWeights.U[i], ht1, Layer::out_size) + oWeights.b[i]); + ctVec[i] = MathsProvider::tanh(vMult(cWeights.W[i], input, Layer::in_size) + vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[i]); cVec[i] = fVec[i] * ct1[i] + iVec[i] * ctVec[i]; - h[i] = oVec[i] * std::tanh(cVec[i]); + h[i] = oVec[i] * MathsProvider::tanh(cVec[i]); } std::copy(cVec, cVec + Layer::out_size, ct1); @@ -115,7 +116,9 @@ class LSTMLayer final : public Layer * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class LSTMLayerT { public: @@ -152,19 +155,19 @@ class LSTMLayerT recurrent_mat_mul(outs, Uf, ft); kernel_mat_mul(ins, Wf, kernel_outs); for(int i = 0; i < out_size; ++i) - ft[i] = sigmoid(ft[i] + bf[i] + kernel_outs[i]); + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + kernel_outs[i]); // compute it recurrent_mat_mul(outs, Ui, it); kernel_mat_mul(ins, Wi, kernel_outs); for(int i = 0; i < out_size; ++i) - it[i] = sigmoid(it[i] + bi[i] + kernel_outs[i]); + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + kernel_outs[i]); // compute ot recurrent_mat_mul(outs, Uo, ot); kernel_mat_mul(ins, Wo, kernel_outs); for(int i = 0; i < out_size; ++i) - ot[i] = sigmoid(ot[i] + bo[i] + kernel_outs[i]); + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + kernel_outs[i]); computeOutputs(ins); } @@ -177,17 +180,17 @@ class LSTMLayerT // compute ft recurrent_mat_mul(outs, Uf, ft); for(int i = 0; i < out_size; ++i) - ft[i] = sigmoid(ft[i] + bf[i] + (Wf_1[i] * ins[0])); + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + (Wf_1[i] * ins[0])); // compute it recurrent_mat_mul(outs, Ui, it); for(int i = 0; i < out_size; ++i) - it[i] = sigmoid(it[i] + bi[i] + (Wi_1[i] * ins[0])); + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + (Wi_1[i] * ins[0])); // compute ot recurrent_mat_mul(outs, Uo, ot); for(int i = 0; i < out_size; ++i) - ot[i] = sigmoid(ot[i] + bo[i] + (Wo_1[i] * ins[0])); + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + (Wo_1[i] * ins[0])); computeOutputs(ins); } @@ -241,11 +244,11 @@ class LSTMLayerT recurrent_mat_mul(outs, Uc, ht); kernel_mat_mul(ins, Wc, kernel_outs); for(int i = 0; i < out_size; ++i) - ctVec[i] = it[i] * std::tanh(ht[i] + bc[i] + kernel_outs[i]) + ft[i] * ct[i]; + ctVec[i] = it[i] * MathsProvider::tanh(ht[i] + bc[i] + kernel_outs[i]) + ft[i] * ct[i]; // compute output for(int i = 0; i < out_size; ++i) - outsVec[i] = ot[i] * std::tanh(ctVec[i]); + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); } template @@ -255,11 +258,11 @@ class LSTMLayerT // compute ct recurrent_mat_mul(outs, Uc, ht); for(int i = 0; i < out_size; ++i) - ctVec[i] = it[i] * std::tanh(ht[i] + bc[i] + (Wc_1[i] * ins[0])) + ft[i] * ct[i]; + ctVec[i] = it[i] * MathsProvider::tanh(ht[i] + bc[i] + (Wc_1[i] * ins[0])) + ft[i] * ct[i]; // compute output for(int i = 0; i < out_size; ++i) - outsVec[i] = ot[i] * std::tanh(ctVec[i]); + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); } template diff --git a/RTNeural/lstm/lstm.tpp b/RTNeural/lstm/lstm.tpp index 0c1d2473..d41d7004 100644 --- a/RTNeural/lstm/lstm.tpp +++ b/RTNeural/lstm/lstm.tpp @@ -5,8 +5,8 @@ namespace RTNeural #if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD -template -LSTMLayer::LSTMLayer(int in_size, int out_size) +template +LSTMLayer::LSTMLayer(int in_size, int out_size) : Layer(in_size, out_size) , fWeights(in_size, out_size) , iWeights(in_size, out_size) @@ -23,28 +23,28 @@ LSTMLayer::LSTMLayer(int in_size, int out_size) cVec = new T[out_size]; } -template -LSTMLayer::LSTMLayer(std::initializer_list sizes) - : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) +template +LSTMLayer::LSTMLayer(std::initializer_list sizes) + : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) { } -template -LSTMLayer::LSTMLayer(const LSTMLayer& other) - : LSTMLayer(other.in_size, other.out_size) +template +LSTMLayer::LSTMLayer(const LSTMLayer& other) + : LSTMLayer(other.in_size, other.out_size) { } -template -LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) +template +LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) { if(&other != this) - *this = LSTMLayer(other); + *this = LSTMLayer(other); return *this; } -template -LSTMLayer::~LSTMLayer() +template +LSTMLayer::~LSTMLayer() { delete[] ht1; delete[] ct1; @@ -56,15 +56,15 @@ LSTMLayer::~LSTMLayer() delete[] cVec; } -template -void LSTMLayer::reset() +template +void LSTMLayer::reset() { std::fill(ht1, ht1 + Layer::out_size, (T)0); std::fill(ct1, ct1 + Layer::out_size, (T)0); } -template -LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) +template +LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) : out_size(out_size) { W = new T*[out_size]; @@ -78,8 +78,8 @@ LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) } } -template -LSTMLayer::WeightSet::~WeightSet() +template +LSTMLayer::WeightSet::~WeightSet() { delete[] b; @@ -93,8 +93,8 @@ LSTMLayer::WeightSet::~WeightSet() delete[] U; } -template -void LSTMLayer::setWVals(const std::vector>& wVals) +template +void LSTMLayer::setWVals(const std::vector>& wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -108,8 +108,8 @@ void LSTMLayer::setWVals(const std::vector>& wVals) } } -template -void LSTMLayer::setUVals(const std::vector>& uVals) +template +void LSTMLayer::setUVals(const std::vector>& uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -123,8 +123,8 @@ void LSTMLayer::setUVals(const std::vector>& uVals) } } -template -void LSTMLayer::setBVals(const std::vector& bVals) +template +void LSTMLayer::setBVals(const std::vector& bVals) { for(int k = 0; k < Layer::out_size; ++k) { @@ -136,8 +136,8 @@ void LSTMLayer::setBVals(const std::vector& bVals) } //==================================================== -template -LSTMLayerT::LSTMLayerT() +template +LSTMLayerT::LSTMLayerT() { for(int i = 0; i < out_size; ++i) { @@ -184,10 +184,10 @@ LSTMLayerT::LSTMLayerT() reset(); } -template +template template std::enable_if_t -LSTMLayerT::prepare(int delaySamples) +LSTMLayerT::prepare(int delaySamples) { delayWriteIdx = delaySamples - 1; ct_delayed.resize(delayWriteIdx + 1, {}); @@ -196,10 +196,10 @@ LSTMLayerT::prepare(int delaySamples) reset(); } -template +template template std::enable_if_t -LSTMLayerT::prepare(T delaySamples) +LSTMLayerT::prepare(T delaySamples) { const auto delayOffFactor = delaySamples - std::floor(delaySamples); delayMult = (T)1 - delayOffFactor; @@ -212,8 +212,8 @@ LSTMLayerT::prepare(T delaySamples) reset(); } -template -void LSTMLayerT::reset() +template +void LSTMLayerT::reset() { if(sampleRateCorr != SampleRateCorrectionMode::None) { @@ -232,8 +232,8 @@ void LSTMLayerT::reset() } } -template -void LSTMLayerT::setWVals(const std::vector>& wVals) +template +void LSTMLayerT::setWVals(const std::vector>& wVals) { for(int i = 0; i < in_size; ++i) { @@ -255,8 +255,8 @@ void LSTMLayerT::setWVals(const std::vec } } -template -void LSTMLayerT::setUVals(const std::vector>& uVals) +template +void LSTMLayerT::setUVals(const std::vector>& uVals) { for(int i = 0; i < out_size; ++i) { @@ -270,8 +270,8 @@ void LSTMLayerT::setUVals(const std::vec } } -template -void LSTMLayerT::setBVals(const std::vector& bVals) +template +void LSTMLayerT::setBVals(const std::vector& bVals) { for(int k = 0; k < out_size; ++k) { diff --git a/RTNeural/lstm/lstm_xsimd.h b/RTNeural/lstm/lstm_xsimd.h index ab5a5882..203e3c47 100644 --- a/RTNeural/lstm/lstm_xsimd.h +++ b/RTNeural/lstm/lstm_xsimd.h @@ -3,6 +3,7 @@ #include "../Layer.h" #include "../common.h" +#include "../maths/maths_xsimd.h" #include namespace RTNeural @@ -16,7 +17,7 @@ namespace RTNeural * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class LSTMLayer : public Layer { public: @@ -45,22 +46,22 @@ class LSTMLayer : public Layer } vAdd(fVec.data(), fWeights.b.data(), fVec.data(), Layer::out_size); - sigmoid(fVec.data(), fVec.data(), Layer::out_size); + sigmoid(fVec.data(), fVec.data(), Layer::out_size); vAdd(iVec.data(), iWeights.b.data(), iVec.data(), Layer::out_size); - sigmoid(iVec.data(), iVec.data(), Layer::out_size); + sigmoid(iVec.data(), iVec.data(), Layer::out_size); vAdd(oVec.data(), oWeights.b.data(), oVec.data(), Layer::out_size); - sigmoid(oVec.data(), oVec.data(), Layer::out_size); + sigmoid(oVec.data(), oVec.data(), Layer::out_size); vAdd(ctVec.data(), cWeights.b.data(), ctVec.data(), Layer::out_size); - tanh(ctVec.data(), ctVec.data(), Layer::out_size); + tanh(ctVec.data(), ctVec.data(), Layer::out_size); vProd(fVec.data(), ct1.data(), cVec.data(), Layer::out_size); vProd(iVec.data(), ctVec.data(), prod_out.data(), Layer::out_size); vAdd(cVec.data(), prod_out.data(), cVec.data(), Layer::out_size); - tanh(cVec.data(), h, Layer::out_size); + tanh(cVec.data(), h, Layer::out_size); vProd(h, oVec.data(), h, Layer::out_size); vCopy(cVec.data(), ct1.data(), Layer::out_size); @@ -130,7 +131,9 @@ class LSTMLayer : public Layer * please make sure to call `reset()` before your first call to * the `forward()` method. */ -template +template class LSTMLayerT { using v_type = xsimd::simd_type; @@ -172,19 +175,19 @@ class LSTMLayerT recurrent_mat_mul(outs, Uf, ft); kernel_mat_mul(ins, Wf, kernel_outs); for(int i = 0; i < v_out_size; ++i) - ft[i] = sigmoid(ft[i] + bf[i] + kernel_outs[i]); + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + kernel_outs[i]); // compute it recurrent_mat_mul(outs, Ui, it); kernel_mat_mul(ins, Wi, kernel_outs); for(int i = 0; i < v_out_size; ++i) - it[i] = sigmoid(it[i] + bi[i] + kernel_outs[i]); + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + kernel_outs[i]); // compute ot recurrent_mat_mul(outs, Uo, ot); kernel_mat_mul(ins, Wo, kernel_outs); for(int i = 0; i < v_out_size; ++i) - ot[i] = sigmoid(ot[i] + bo[i] + kernel_outs[i]); + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + kernel_outs[i]); computeOutputs(ins); } @@ -197,17 +200,17 @@ class LSTMLayerT // compute ft recurrent_mat_mul(outs, Uf, ft); for(int i = 0; i < v_out_size; ++i) - ft[i] = sigmoid(xsimd::fma(Wf_1[i], ins[0], ft[i] + bf[i])); + ft[i] = MathsProvider::sigmoid(xsimd::fma(Wf_1[i], ins[0], ft[i] + bf[i])); // compute it recurrent_mat_mul(outs, Ui, it); for(int i = 0; i < v_out_size; ++i) - it[i] = sigmoid(xsimd::fma(Wi_1[i], ins[0], it[i] + bi[i])); + it[i] = MathsProvider::sigmoid(xsimd::fma(Wi_1[i], ins[0], it[i] + bi[i])); // compute ot recurrent_mat_mul(outs, Uo, ot); for(int i = 0; i < v_out_size; ++i) - ot[i] = sigmoid(xsimd::fma(Wo_1[i], ins[0], ot[i] + bo[i])); + ot[i] = MathsProvider::sigmoid(xsimd::fma(Wo_1[i], ins[0], ot[i] + bo[i])); computeOutputs(ins); } @@ -261,11 +264,11 @@ class LSTMLayerT recurrent_mat_mul(outs, Uc, ht); kernel_mat_mul(ins, Wc, kernel_outs); for(int i = 0; i < v_out_size; ++i) - ctVec[i] = xsimd::fma(it[i], xsimd::tanh(ht[i] + bc[i] + kernel_outs[i]), ft[i] * ct[i]); + ctVec[i] = xsimd::fma(it[i], MathsProvider::tanh(ht[i] + bc[i] + kernel_outs[i]), ft[i] * ct[i]); // compute output for(int i = 0; i < v_out_size; ++i) - outsVec[i] = ot[i] * xsimd::tanh(ctVec[i]); + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); } template @@ -275,11 +278,11 @@ class LSTMLayerT // compute ct recurrent_mat_mul(outs, Uc, ht); for(int i = 0; i < v_out_size; ++i) - ctVec[i] = xsimd::fma(it[i], xsimd::tanh(xsimd::fma(Wc_1[i], ins[0], ht[i] + bc[i])), ft[i] * ct[i]); + ctVec[i] = xsimd::fma(it[i], MathsProvider::tanh(xsimd::fma(Wc_1[i], ins[0], ht[i] + bc[i])), ft[i] * ct[i]); // compute output for(int i = 0; i < v_out_size; ++i) - outsVec[i] = ot[i] * xsimd::tanh(ctVec[i]); + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); } template diff --git a/RTNeural/lstm/lstm_xsimd.tpp b/RTNeural/lstm/lstm_xsimd.tpp index 21884f00..1b5ca605 100644 --- a/RTNeural/lstm/lstm_xsimd.tpp +++ b/RTNeural/lstm/lstm_xsimd.tpp @@ -3,8 +3,8 @@ namespace RTNeural { -template -LSTMLayer::LSTMLayer(int in_size, int out_size) +template +LSTMLayer::LSTMLayer(int in_size, int out_size) : Layer(in_size, out_size) , fWeights(in_size, out_size) , iWeights(in_size, out_size) @@ -24,36 +24,36 @@ LSTMLayer::LSTMLayer(int in_size, int out_size) prod_out.resize(out_size, (T)0); } -template -LSTMLayer::LSTMLayer(std::initializer_list sizes) - : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) +template +LSTMLayer::LSTMLayer(std::initializer_list sizes) + : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) { } -template -LSTMLayer::LSTMLayer(const LSTMLayer& other) - : LSTMLayer(other.in_size, other.out_size) +template +LSTMLayer::LSTMLayer(const LSTMLayer& other) + : LSTMLayer(other.in_size, other.out_size) { } -template -LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) +template +LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) { - return *this = LSTMLayer(other); + return *this = LSTMLayer(other); } -template -LSTMLayer::~LSTMLayer() = default; +template +LSTMLayer::~LSTMLayer() = default; -template -void LSTMLayer::reset() +template +void LSTMLayer::reset() { std::fill(ht1.begin(), ht1.end(), (T)0); std::fill(ct1.begin(), ct1.end(), (T)0); } -template -LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) +template +LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) : out_size(out_size) { W = vec2_type(out_size, vec_type(in_size, (T)0)); @@ -61,11 +61,11 @@ LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) b.resize(out_size, (T)0); } -template -LSTMLayer::WeightSet::~WeightSet() = default; +template +LSTMLayer::WeightSet::~WeightSet() = default; -template -void LSTMLayer::setWVals(const std::vector>& wVals) +template +void LSTMLayer::setWVals(const std::vector>& wVals) { for(int i = 0; i < Layer::in_size; ++i) { @@ -79,8 +79,8 @@ void LSTMLayer::setWVals(const std::vector>& wVals) } } -template -void LSTMLayer::setUVals(const std::vector>& uVals) +template +void LSTMLayer::setUVals(const std::vector>& uVals) { for(int i = 0; i < Layer::out_size; ++i) { @@ -94,8 +94,8 @@ void LSTMLayer::setUVals(const std::vector>& uVals) } } -template -void LSTMLayer::setBVals(const std::vector& bVals) +template +void LSTMLayer::setBVals(const std::vector& bVals) { for(int k = 0; k < Layer::out_size; ++k) { @@ -107,8 +107,8 @@ void LSTMLayer::setBVals(const std::vector& bVals) } //==================================================== -template -LSTMLayerT::LSTMLayerT() +template +LSTMLayerT::LSTMLayerT() { for(int i = 0; i < v_out_size; ++i) { @@ -158,10 +158,10 @@ LSTMLayerT::LSTMLayerT() reset(); } -template +template template std::enable_if_t -LSTMLayerT::prepare(int delaySamples) +LSTMLayerT::prepare(int delaySamples) { delayWriteIdx = delaySamples - 1; ct_delayed.resize(delayWriteIdx + 1, {}); @@ -170,10 +170,10 @@ LSTMLayerT::prepare(int delaySamples) reset(); } -template +template template std::enable_if_t -LSTMLayerT::prepare(T delaySamples) +LSTMLayerT::prepare(T delaySamples) { const auto delayOffFactor = delaySamples - std::floor(delaySamples); delayMult = (T)1 - delayOffFactor; @@ -186,8 +186,8 @@ LSTMLayerT::prepare(T delaySamples) reset(); } -template -void LSTMLayerT::reset() +template +void LSTMLayerT::reset() { if constexpr(sampleRateCorr != SampleRateCorrectionMode::None) { @@ -206,8 +206,8 @@ void LSTMLayerT::reset() } } -template -void LSTMLayerT::setWVals(const std::vector>& wVals) +template +void LSTMLayerT::setWVals(const std::vector>& wVals) { for(int i = 0; i < out_size; ++i) { @@ -229,8 +229,8 @@ void LSTMLayerT::setWVals(const std::vec } } -template -void LSTMLayerT::setUVals(const std::vector>& uVals) +template +void LSTMLayerT::setUVals(const std::vector>& uVals) { for(int i = 0; i < out_size; ++i) { @@ -244,8 +244,8 @@ void LSTMLayerT::setUVals(const std::vec } } -template -void LSTMLayerT::setBVals(const std::vector& bVals) +template +void LSTMLayerT::setBVals(const std::vector& bVals) { for(int k = 0; k < out_size; ++k) { diff --git a/RTNeural/maths/maths_stl.h b/RTNeural/maths/maths_stl.h new file mode 100644 index 00000000..3210e7cc --- /dev/null +++ b/RTNeural/maths/maths_stl.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace RTNeural +{ +struct DefaultMathsProvider +{ + template + static T tanh(T x) + { + return std::tanh(x); + } + + template + static T sigmoid(T x) + { + return (T)1 / ((T)1 + std::exp(-x)); + } + + template + static T exp(T x) + { + return std::exp(x); + } +}; +} diff --git a/RTNeural/maths/maths_xsimd.h b/RTNeural/maths/maths_xsimd.h new file mode 100644 index 00000000..80c20dbf --- /dev/null +++ b/RTNeural/maths/maths_xsimd.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace RTNeural +{ +struct DefaultMathsProvider +{ + template + static T tanh(T x) + { + using std::tanh; + using xsimd::tanh; + return tanh(x); + } + + template + static T sigmoid(T x) + { + using std::exp; + using xsimd::exp; + return (T)1 / ((T)1 + exp(-x)); + } + + template + static T exp(T x) + { + using std::exp; + using xsimd::exp; + return exp(x); + } +}; +} diff --git a/bench/layer_creator.hpp b/bench/layer_creator.hpp index 557f1a19..3959bc15 100644 --- a/bench/layer_creator.hpp +++ b/bench/layer_creator.hpp @@ -164,11 +164,6 @@ create_layer(const std::string &layer_type, size_t in_size, size_t out_size) { return std::move(layer); } - if (layer_type == "fast_tanh") { - auto layer = std::make_unique>(in_size); - return std::move(layer); - } - if (layer_type == "relu") { auto layer = std::make_unique>(in_size); return std::move(layer); diff --git a/examples/custom_layer_model/GatedActivation_STL.h b/examples/custom_layer_model/GatedActivation_STL.h index 67aa54a9..66b65229 100644 --- a/examples/custom_layer_model/GatedActivation_STL.h +++ b/examples/custom_layer_model/GatedActivation_STL.h @@ -27,8 +27,13 @@ class GatedActivation /** Performs forward propagation for gated activation. */ inline void forward(const T (&ins)[in_size]) noexcept { + const auto sigmoid = [] (T x) + { + return (T) 1 / ((T) 1 + std::exp (-x)); + }; + for(int i = 0; i < out_size; ++i) - outs[i] = std::tanh(ins[i]) * RTNeural::sigmoid(ins[i + out_size]); + outs[i] = std::tanh(ins[i]) * sigmoid(ins[i + out_size]); } T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4462ea44..a519386f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,7 +4,6 @@ add_executable(rtneural_tests tests.cpp) target_link_libraries(rtneural_tests LINK_PUBLIC RTNeural) target_compile_definitions(rtneural_tests PRIVATE RTNEURAL_ROOT_DIR="${CMAKE_SOURCE_DIR}/") -add_test(NAME "RTNeural_Approx_Test" COMMAND $ approx) add_test(NAME "RTNeural_Bad_Model_Test" COMMAND $ bad_model) add_test(NAME "RTNeural_Model_Test" COMMAND $ model) add_test(NAME "RTNeural_Sample_Rate_RNN_Test" COMMAND $ sample_rate_rnn) diff --git a/tests/approx_tests.hpp b/tests/approx_tests.hpp deleted file mode 100644 index 584ed5db..00000000 --- a/tests/approx_tests.hpp +++ /dev/null @@ -1,119 +0,0 @@ -#pragma once - -#include -#include - -template -int fastTanhTest(T limit) -{ - using namespace RTNeural; - constexpr int layerSize = 8; - constexpr int nIter = 100; - constexpr T range = (T) 10; - - auto testTanh = [=] (auto forwardFunc, const T (&test_outs)[layerSize]) - { - std::default_random_engine generator; - std::uniform_real_distribution distribution(-range, range); - - constexpr int layerSize = 8; // MSVC can't capture this in the lambda - T test_ins alignas(RTNEURAL_DEFAULT_ALIGNMENT) [layerSize]; - T actual_outs[layerSize]; - - auto maxError = (T) 0; - auto maxErrorInput = (T) 0; - for(int i = 0; i < nIter; ++i) - { - // generate random input and reference - for(int n = 0; n < layerSize; ++n) - { - test_ins[n] = distribution(generator); - actual_outs[n] = std::tanh(test_ins[n]); - } - - // compute layer output - forwardFunc(test_ins); - // layer.forward(test_ins, test_outs); - - // check for errors - for(int n = 0; n < layerSize; ++n) - { - auto error = std::abs(actual_outs[n] - test_outs[n]); - if(error > maxError) - { - maxError = error; - maxErrorInput = test_ins[n]; - } - } - } - - std::cout << " Maximum error: " << maxError << ", at input value: " << maxErrorInput << std::endl; - if(maxError > limit) - { - std::cout << " FAIL: Error is too high!" << std::endl; - return 1; - } - - return 0; - }; - - int result = 0; - auto dtype = std::is_same::value ? "float" : "double"; - - T test_outs alignas(RTNEURAL_DEFAULT_ALIGNMENT) [layerSize]; - FastTanh fastTanh { layerSize }; - std::cout << "Testing FastTanh for data type " << dtype << std::endl; - result |= testTanh([&fastTanh, &test_outs] (const T (&test_ins)[layerSize]) - { - fastTanh.forward(test_ins, test_outs); - }, test_outs); - - FastTanhT fastTanhT; - std::cout << "Testing FastTanhT for data type " << dtype << std::endl; -#if RTNEURAL_USE_XSIMD - result |= testTanh([&fastTanhT, &test_outs] (const T (&test_ins)[layerSize]) - { - constexpr int layerSize = 8; // MSVC can't capture this in the lambda - using b_type = xsimd::simd_type; - constexpr auto b_size = (int)b_type::size; - constexpr auto v_size = layerSize / b_size; - - b_type test_ins_v[v_size]; - for(int i = 0; i < v_size; ++i) - test_ins_v[i] = xsimd::load_aligned(test_ins + i * b_size); - - fastTanhT.forward(test_ins_v); - - for(int i = 0; i < v_size; ++i) - xsimd::store_aligned(test_outs + i * b_size, fastTanhT.outs[i]); - }, test_outs); -#elif RTNEURAL_USE_EIGEN - result |= testTanh([&fastTanhT, &test_outs] (const T (&test_ins)[layerSize]) - { - constexpr int layerSize = 8; // MSVC can't capture this in the lambda - using MatType = Eigen::Matrix; - Eigen::Map test_ins_v (test_ins); - - fastTanhT.forward(test_ins_v); - - for(int i = 0; i < layerSize; ++i) - test_outs[i] = fastTanhT.outs(i); - }, test_outs); -#else - result |= testTanh([&fastTanhT] (const T (&test_ins)[layerSize]) - { - fastTanhT.forward(test_ins); - }, fastTanhT.outs); -#endif - - return result; -} - -int approximationTests() -{ - int result = 0; - result |= fastTanhTest(5.1e-5f); - result |= fastTanhTest(5.1e-5); - - return result; -} diff --git a/tests/tests.cpp b/tests/tests.cpp index 37a72688..a24f02ad 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -1,4 +1,3 @@ -#include "approx_tests.hpp" #include "bad_model_test.hpp" #include "conv2d_model.h" #include "load_csv.hpp" @@ -22,7 +21,6 @@ void help() std::cout << " all" << std::endl; std::cout << " util" << std::endl; std::cout << " model" << std::endl; - std::cout << " approx" << std::endl; std::cout << " sample_rate_rnn" << std::endl; std::cout << " bad_model" << std::endl; std::cout << " torch" << std::endl; @@ -108,7 +106,6 @@ int main(int argc, char* argv[]) int result = 0; result |= model_test::model_test(); - result |= approximationTests(); result |= sampleRateRNNTest(); result |= conv2d_test(); result |= torchGRUTest(); @@ -140,11 +137,6 @@ int main(int argc, char* argv[]) return model_test::model_test(); } - if(arg == "approx") - { - return approximationTests(); - } - if(arg == "sample_rate_rnn") { return sampleRateRNNTest();