One more try for the Dense layer

Apparently, the += operator for NoAlias is about as fast as the just the = operator.

So I figured I'd give it a shot. On M2 it's yet a little faster.
I'll try it on Windows in a minute.

RTNeural/dense/dense_eigen.h

@@ -19,12 +19,11 @@ class Dense : public Layer<T>
     Dense(int in_size, int out_size)
         : Layer<T>(in_size, out_size)
     {
-        weights = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>::Zero(out_size, in_size + 1);
+        weights = Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>::Zero(out_size, in_size);
 
-        inVec = Eigen::Matrix<T, Eigen::Dynamic, 1>::Zero(in_size + 1);
+        inVec = Eigen::Matrix<T, Eigen::Dynamic, 1>::Zero(in_size);
         outVec = Eigen::Matrix<T, Eigen::Dynamic, 1>::Zero(out_size);
-
-        inVec(in_size, 0) = (T)1;
+        bias = Eigen::Matrix<T, Eigen::Dynamic, 1>::Zero(out_size);
     }
 
     Dense(std::initializer_list<int> sizes)
@@ -50,17 +49,15 @@ class Dense : public Layer<T>
     /** Performs forward propagation for this layer. */
     inline void forward(const T* input, T* out) noexcept override
     {
-        for (int i = 0; i < Layer<T>::in_size; ++i)
-            inVec(i, 0) = input[i];
+        inVec = Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, 1>, RTNeuralEigenAlignment>(input, Layer<T>::in_size);
 
-        /**
-         * out = | w b | * | input |
-         *                 | 1     |
-         */
-        outVec.noalias() = weights * inVec;
+        outVec.noalias() += weights * inVec;
 
-        for (int i = 0; i < Layer<T>::out_size; ++i)
-            out[i] = outVec(i, 0);
+        for(int i = 0; i < Layer<T>::out_size; ++i)
+        {
+            out[i] = outVec(i);
+            outVec(i) = bias(i);
+        }
     }
 
     /**
@@ -96,17 +93,21 @@ class Dense : public Layer<T>
     void setBias(const T* b)
     {
         for(int i = 0; i < Layer<T>::out_size; ++i)
-            weights(i, Layer<T>::in_size) = b[i];
+        {
+            bias(i) = b[i];
+            outVec(i) = b[i];
+        }
     }
 
     /** Returns the weights value at the given indices. */
     T getWeight(int i, int k) const noexcept { return weights(i, k); }
 
     /** Returns the bias value at the given index. */
-    T getBias(int i) const noexcept { return weights(i, Layer<T>::in_size); }
+    T getBias(int i) const noexcept { return bias(i); }
 
 private:
     Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic> weights;
+    Eigen::Matrix<T, Eigen::Dynamic, 1> bias;
 
     Eigen::Matrix<T, Eigen::Dynamic, 1> inVec;
     Eigen::Matrix<T, Eigen::Dynamic, 1> outVec;
@@ -121,8 +122,7 @@ template <typename T, int in_sizet, int out_sizet>
 class DenseT
 {
     using out_vec_type = Eigen::Matrix<T, out_sizet, 1>;
-    using in_vec_type = Eigen::Matrix<T, in_sizet + 1, 1>;
-    using mat_type = Eigen::Matrix<T, out_sizet, in_sizet + 1>;
+    using mat_type = Eigen::Matrix<T, out_sizet, in_sizet>;
 
 public:
     static constexpr auto in_size = in_sizet;
@@ -132,8 +132,7 @@ class DenseT
         : outs(outs_internal)
     {
         weights = mat_type::Zero();
-        ins_internal = in_vec_type::Zero();
-        ins_internal(in_size, 0) = (T)1;
+        bias = out_vec_type::Zero();
         outs = out_vec_type::Zero();
     }
 
@@ -149,14 +148,8 @@ class DenseT
     /** Performs forward propagation for this layer. */
     inline void forward(const Eigen::Matrix<T, in_size, 1>& ins) noexcept
     {
-        for (int i = 0; i < in_size; ++i)
-            ins_internal(i, 0) = ins(i, 0);
-
-        /**
-         * out = | w b | * | input |
-         *                 | 1     |
-         */
-        outs.noalias() = weights * ins_internal;
+        outs = bias;
+        outs.noalias() += weights * ins;
     }
 
     /**
@@ -192,16 +185,16 @@ class DenseT
     void setBias(const T* b)
     {
         for(int i = 0; i < out_size; ++i)
-            weights(i, in_size) = b[i];
+            bias(i) = b[i];
     }
 
     Eigen::Map<out_vec_type, RTNeuralEigenAlignment> outs;
 
 private:
     T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size];
-    in_vec_type ins_internal;
 
     mat_type weights;
+    out_vec_type bias;
 };
 
 } // namespace RTNeural