Add support for Torch's Conv1d strides and ConvTranspose1d

RTNeural/conv1d/conv1d.h

@@ -50,6 +50,19 @@ class Conv1D final : public Layer<T>
     /** Returns the name of this layer. */
     std::string getName() const noexcept override { return "conv1d"; }
 
+
+    /** Performs a stride step for this layer. */
+    RTNEURAL_REALTIME inline void skip(const T* input)
+    {
+        // insert input into a circular buffer
+        std::copy(input, input + Layer<T>::in_size, state[state_ptr]);
+
+        // set state pointers to particular columns of the buffer
+        setStatePointers();
+
+        state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards
+    }
+
     /** Performs forward propagation for this layer. */
     RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override
     {

RTNeural/conv1d/conv1d.tpp

@@ -41,7 +41,8 @@ Conv1D<T>::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int
 
 template <typename T>
 Conv1D<T>::Conv1D(std::initializer_list<int> sizes)
-    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3))
+    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2),
+                *(sizes.begin() + 3), *(sizes.begin() + 4))
 {
 }
 

RTNeural/conv1d/conv1d_eigen.h

@@ -42,6 +42,19 @@ class Conv1D : public Layer<T>
     /** Returns the name of this layer. */
     std::string getName() const noexcept override { return "conv1d"; }
 
+    /** Performs a stride step for this layer. */
+    RTNEURAL_REALTIME inline void skip(const T* input)
+    {
+        // insert input into a circular buffer
+        state.col(state_ptr) = Eigen::Map<const Eigen::Vector<T, Eigen::Dynamic>,
+            RTNeuralEigenAlignment>(input, Layer<T>::in_size);
+
+        // set state pointers to the particular columns of the buffer
+        setStatePointers();
+
+        state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards
+    }
+
     /** Performs forward propagation for this layer. */
     RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override
     {

RTNeural/conv1d/conv1d_eigen.tpp

@@ -25,7 +25,8 @@ Conv1D<T>::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int
 
 template <typename T>
 Conv1D<T>::Conv1D(std::initializer_list<int> sizes)
-    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3))
+    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2),
+                *(sizes.begin() + 3), *(sizes.begin() + 4))
 {
 }
 

RTNeural/conv1d/conv1d_xsimd.h

@@ -45,6 +45,19 @@ class Conv1D : public Layer<T>
     /** Returns the name of this layer. */
     std::string getName() const noexcept override { return "conv1d"; }
 
+    /** Performs a stride step for this layer. */
+    RTNEURAL_REALTIME inline void skip(const T* input)
+    {
+        // insert input into a circular buffer
+        vCopy(input, state[state_ptr].data(), Layer<T>::in_size);
+
+        // set state pointers to particular columns of the buffer
+        setStatePointers();
+
+        state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards
+    }
+
+
     /** Performs forward propagation for this layer. */
     RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override
     {

RTNeural/conv1d/conv1d_xsimd.tpp

@@ -23,7 +23,8 @@ Conv1D<T>::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int
 
 template <typename T>
 Conv1D<T>::Conv1D(std::initializer_list<int> sizes)
-    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3))
+    : Conv1D<T>(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2),
+                *(sizes.begin() + 3), *(sizes.begin() + 4))
 {
 }
 

RTNeural/torch_helpers.h

@@ -21,6 +21,28 @@ namespace torch_helpers
             }
         }
 
+        template <typename T>
+        std::vector<std::vector<std::vector<T>>>
+            reverseChannels(std::vector<std::vector<std::vector<T>>>& conv_weights)
+        {
+            std::vector<std::vector<std::vector<T>>> aux {conv_weights[0].size()};
+
+            // Reverse channels in auxiliary variable
+            for(size_t j = 0; j < conv_weights[0].size();j++)
+            {
+                aux[j].resize(conv_weights.size());
+                for(size_t i = 0; i < conv_weights.size();i++)
+                {
+                    aux[j][i].resize(conv_weights[i][j].size());
+                    std::copy(conv_weights[i][j].begin(),
+                              conv_weights[i][j].end(),
+                              aux[j][i].begin());
+                }
+            }
+
+            return aux;
+        }
+
         /** Transposes the rows and columns of a matrix stored as a 2D vector. */
         template <typename T>
         std::vector<std::vector<T>> transpose(const std::vector<std::vector<T>>& x)
@@ -66,6 +88,26 @@ namespace torch_helpers
         }
     }
 
+    /** Loads a ConvTranspose1D layer from a JSON object containing a PyTorch state_dict. */
+    template <typename T, typename Conv1DType>
+    void loadConvTranspose1D(const nlohmann::json& modelJson, const std::string& layerPrefix, Conv1DType& conv, bool hasBias = true)
+    {
+        std::vector<std::vector<std::vector<T>>> conv_weights = modelJson.at(layerPrefix + "weight");
+        conv_weights = detail::reverseChannels<T>(conv_weights);
+        conv.setWeights(conv_weights);
+
+        if(hasBias)
+        {
+            std::vector<T> conv_bias = modelJson.at(layerPrefix + "bias");
+            conv.setBias(conv_bias);
+        }
+        else
+        {
+            std::vector<T> conv_bias((size_t)conv.out_size, (T)0);
+            conv.setBias(conv_bias);
+        }
+    }
+
     /** Loads a Conv1D layer from a JSON object containing a PyTorch state_dict. */
     template <typename T, typename Conv1DType>
     void loadConv1D(const nlohmann::json& modelJson, const std::string& layerPrefix, Conv1DType& conv, bool hasBias = true)

models/conv1d_torch_stride_3.json

@@ -0,0 +1 @@
+{"weight": [[[0.42042818602691356, 0.1858797375490857, -0.03632900008519796, 0.37632816922892476, 0.12971351638863882]], [[0.2604105913683754, -0.2874633759199114, -0.133173389274643, 0.0727535304168383, -0.18940757615126635]], [[-0.042155383103857524, -0.28907929190745385, -0.12945334678288611, 0.10903536914457612, -0.016235283900972675]], [[-0.052949643348350395, -0.08292988099514709, -0.26346373142713686, 0.14761463194395663, 0.2547989625732606]], [[-0.25905610025815295, 0.15808027546219416, -0.34906339493213423, 0.021244487771611753, -0.24505498027654232]], [[0.05209419030753171, 0.07828828346651195, 0.16273716855750242, 0.21947111004375486, -0.24359220394331174]], [[0.27717606937734685, 0.11680352718221176, -0.27200591158308884, -0.31061109556281746, -0.016541087272268717]], [[0.373428551091498, 0.06443427563294302, -0.01714668517146567, 0.26480286942932707, -0.11809103926987097]], [[-0.20376508378726751, -0.4101489918413958, 0.22243354926660963, -0.1415378682437916, 0.03418948702732588]], [[-0.030083683922472015, 0.3251121740625388, -0.4104889739324586, 0.44476320803199676, 0.33958189380583503]], [[0.23523181888045974, -0.21832991723821316, -0.2776229753490371, 0.43626998347391177, 0.2500835212321163]], [[-0.12353886811696624, 0.05889253838767561, 0.21573556208629685, -0.44322181585645837, 0.0747782571361536]]], "bias": [0.06235345040330914, -0.4040258841158204, -0.3014178940814476, 0.4460338126973123, 0.09003928036486314, -0.31381897448216467, -0.09468682430373992, 0.32503807662859413, -0.34778085576777357, 0.2768484289218343, 0.25847621557696404, 0.26623018434095796]}

models/convtranspose1d_torch.json

@@ -0,0 +1 @@
+{"weight": [[[0.10855409085056199, 0.047993941861667624, -0.009380107487622635, 0.09716751547549014, 0.03349188591674675], [0.06723772556971618, -0.07422272450549192, -0.03438522125457931, 0.018784880778813948, -0.04890482587195784], [-0.010884473114282256, -0.07463995221940009, -0.033424710413395425, 0.02815281125630141, -0.004191932277961108], [-0.013671539125052229, -0.021412403199811178, -0.06802604294313097, 0.0381139340783654, 0.06578880924514434], [-0.06688799746888434, 0.04081615161526393, -0.09012778102285188, 0.005485303155877294, -0.06327292383441568], [0.013450662099682001, 0.020213947871264418, 0.04201855624214375, 0.05666719694487453, -0.06289523660922713], [0.07156655337771307, 0.03015854103699943, -0.07023162437541802, -0.08019944001746881, -0.004270890368910679], [0.09641883729171581, 0.016636858430088428, -0.004427255074116551, 0.0683718068885147, -0.030490975228569084], [-0.052611918403109734, -0.10590001432436583, 0.05743209546315907, -0.03654492043772696, 0.008827687591480518], [-0.007767573788286797, 0.08394360238623329, -0.10598779732281216, 0.11483736651424718, 0.08767966795890499], [0.06073659446813999, -0.056372542229496204, -0.07168194400342487, 0.11264442536297635, 0.06457128752619302], [-0.03189759858842163, 0.015205988026089898, 0.05570268260966159, -0.11443938076585823, 0.019307662969782605], [0.016099591666039095, -0.10431903470781999, -0.07782576560175716, 0.11516543522813923, 0.02324804222384101], [-0.08102771079288758, -0.02444803290891015, 0.08392447051105568, -0.08979663083455669, 0.07148195697586344], [0.06673827188801996, 0.0687403380140174, -0.021543481396869554, 0.05164308246001144, -0.11274158796428328]], [[-0.0562126679973018, -0.0719024036217785, -0.07792740247111404, -0.0011250515091510427, -0.07982232536823447], [-0.06246021454636065, -0.02064590169439688, -0.029079472170318282, -0.07323094688430261, -0.023147852681695377], [0.004082565380675057, 0.02586825350852455, -0.02536722160905032, 0.008873214840986063, -0.06960227712659682], [-0.016346003819581262, 0.08231882293777774, 0.09746173628626957, -0.10667086681021669, 0.04703659771396858], [-0.07484478600039904, -0.07425022659925512, -0.07383995912035284, 0.039070028346086375, 0.02686927710150179], [-0.07660912293459077, 0.11000142106065051, 0.04095815104938448, 0.014350663221161783, -0.01237133066242932], [0.10859360256074567, 0.000430891338553438, 0.1089793825493743, 0.08572035050998288, -0.06236046714967029], [0.04913838408713618, -0.012713561687525618, 0.007750426556594664, -0.08062641900055158, 0.03647349404784646], [-0.10536562294975574, -0.00014885584716753386, -0.012274473107960523, 0.0679002764348658, -0.04510267195756275], [-0.04139170244640031, 0.04903708217953713, -0.10177661402588217, 0.09051066180810897, -0.10273825643040921], [0.09083068173403981, -0.044415531042604844, 0.028919238980964643, -0.10729322053069305, -0.0972384078801588], [0.0451706536870133, -0.08815096267892288, -0.011340891920962187, 0.08826702681545738, -0.04892719401232902], [-0.09066274453799494, -0.06509649022378745, 0.058952188975965616, 0.0017334472494428543, 0.01038166511038234], [-0.08604057825411007, 0.06159477253361352, 0.03495399980194193, 0.03672818683426747, -0.09577191194798532], [0.014739391344169753, -0.05434858035441077, -0.09228561710060917, 0.061471478173757546, 0.032022677833838956]], [[-0.09927459465555441, 0.09749646733558244, 0.06593886409054588, 0.03833472924965019, 0.01924292905317647], [0.05100310579223451, -0.025581022265003292, 0.00423184858472761, 0.09500729573739035, 0.03556395888815174], [-0.038306919321364236, 0.0820782725563634, 0.06232596966981248, -0.03440919359090279, 0.08499907288166582], [0.04278086458141912, 0.11326210541538893, -0.061184017300455064, -0.11069634361923451, -0.0005589359305423758], [-0.09631051812922309, -0.04983789489085132, 0.021450675442646047, 0.09991264304254584, -0.0016408593726441878], [0.022574535001373422, -0.05075183109759833, 0.048610041885703684, 0.03249099477825619, -0.06786947023762957], [-0.04765847591603234, 0.06645924467856147, -0.07756286513855637, -0.06824617059083031, -0.05305254990245484], [-0.022569518594373728, 0.10992038981295678, -0.084146543063305, -0.04851813804704676, 0.08391234966565185], [0.013104952127194383, 0.06004637740336731, -0.040812845825535474, -0.09822991529066046, 0.02310130650446772], [-0.009323823380637059, 0.0566174856116385, 0.09747444842739837, 0.0793798318227001, 0.04278309530946585], [0.013384402287686267, -0.0897296440346613, 0.03135311860986521, 0.07582496488862851, 0.009708015650240334], [0.05247981271542114, -0.023924247057928894, -0.07594981772342256, 0.09565993493414736, -0.027065527702565928], [-0.06601154627811162, -0.06812333481731364, 0.11181227022072536, -0.049829878289145854, -0.047745060705701475], [-0.07863982670806921, -0.02422262760015334, -0.03832461876688724, 0.08624611502337846, 0.11252360816354194], [-0.07971437266908624, -0.07192703409627144, -0.06602939070204975, -0.023115034447640986, -0.0733438080997561]], [[-0.04218899963077655, 0.10992354137484232, -0.03239525734163032, -0.06582304586138898, 0.06759247935661115], [-0.09872923128750716, 0.08381394605005017, -0.05194834587509435, 0.013218431548173468, 0.09208073796771228], [0.05908210528309012, -0.04616114695927545, -0.02252453029626307, 0.019305412424775167, 0.0541184176060827], [-0.07476182655877242, 0.047399160631189055, 0.02134342848557992, -0.005660631609676184, -0.08576543441292203], [-0.08538466317605534, 0.11150678778629648, -0.032998673636587414, 0.03422942469837305, 0.009069234503183615], [-0.04737004502343968, -0.11439923558543262, -0.026300074293753445, -0.054709006581384965, 0.07344510795233199], [0.0806319818947568, 0.09010864776866594, -0.11385435607540288, -0.10491296758694524, -0.04199806875488263], [-0.06527844906897845, -0.02263246542544431, 0.03415260338590616, 0.040655508273482285, 0.11294793820477102], [0.11389058912418147, 0.11160268794531726, 0.1080864725458988, 0.11094006989910744, 0.015083093031248426], [0.09469408505754952, -0.09636281264489142, 0.011441247370326549, -0.04718496130973976, 0.005757213495047375], [-0.06813951858626041, 0.10196931953288206, 0.06163644103128289, -0.07912646968724378, 0.009644656920921466], [-0.09109614786776389, 0.031499973174086746, -0.02044615847601397, -0.09027837634022719, -0.08447039613395144], [-0.024830287965421374, 0.06618393542186325, 0.046739545560440146, 0.036377864267849294, 0.03226554142129033], [0.08343110583198983, -0.029615406944206127, 0.045523210866959166, 0.07125141123149102, -0.07360617748613277], [-0.022801730200142242, -0.02644084624496648, 0.06194302655280075, 0.009033469326570703, 0.06682779028239942]]], "bias": [0.029019965981479323, -0.057978684740668814, -0.05051116694898852, -0.0933802236567082, -0.10856219926003169, 0.02667874382306082, -0.03818072613995073, -0.018738495076416964, -0.1114826873618283, -0.02230227138926437, -0.0809736808851606, 0.10316613107836597, 0.08342337976931113, 0.0016194013237633975, -0.03374016058525993]}

python/conv1d_torch.py

@@ -15,9 +15,11 @@ def default(self, obj):
 torch.manual_seed(0)
 
 x = np.random.uniform(-1, 1, 1000)
+
 conv = torch.nn.Conv1d(1, 12, 5, dilation=1, padding='valid', bias=True, dtype=torch.float64)
 y = conv(torch.from_numpy(x).reshape(1, 1, -1)).detach().numpy()[0]
 
+print(torch.from_numpy(x).reshape(1, 1, -1).shape)
 print(np.shape(y))
 
 plt.plot(x)

python/conv1d_torch_stride.py

@@ -0,0 +1,45 @@
+import torch
+from torch.utils.data import Dataset
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+from json import JSONEncoder
+
+class EncodeTensor(JSONEncoder,Dataset):
+    def default(self, obj):
+        if isinstance(obj, torch.Tensor):
+            return obj.cpu().detach().numpy().tolist()
+        return super(json.NpEncoder, self).default(obj)
+
+np.random.seed(1001)
+torch.manual_seed(0)
+out_channels = 12
+x = np.random.uniform(-1, 1, 1000)
+# Fails if it doesn't have bias
+conv = torch.nn.Conv1d(1, out_channels, 5, dilation=1, stride=3, padding='valid', bias=True, dtype=torch.float64)
+y = conv(torch.from_numpy(x).reshape(1, 1, -1)).detach().numpy()[0]
+
+print(torch.from_numpy(x).reshape(1, 1, -1).shape)
+print(np.shape(y))
+
+plt.plot(x)
+plt.plot(y[0, :])
+# plt.show()
+
+np.savetxt('test_data/conv1d_torch_x_python_stride_3.csv', x, delimiter=',')
+np.savetxt('test_data/conv1d_torch_y_python_stride_3.csv', y, delimiter=',')
+
+with open('models/conv1d_torch_stride_3.json', 'w') as json_file:
+    json.dump(conv.state_dict(), json_file,cls=EncodeTensor)
+
+# print(x[:5])
+# print(conv.state_dict())
+#
+# ch_idx = 0
+# kernel_test = conv.state_dict()["weight"][ch_idx, 0, :].detach().numpy()
+# print(kernel_test)
+# y_test = np.correlate(x, kernel_test, mode='full')
+# print(y_test[:10])
+# print(y[ch_idx,:10])
+#
+# print(np.sum(kernel_test * x[:5]))

python/convtranspose1d_torch.py

@@ -0,0 +1,44 @@
+import torch
+from torch.utils.data import Dataset
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+from json import JSONEncoder
+
+class EncodeTensor(JSONEncoder,Dataset):
+    def default(self, obj):
+        if isinstance(obj, torch.Tensor):
+            return obj.cpu().detach().numpy().tolist()
+        return super(json.NpEncoder, self).default(obj)
+
+np.random.seed(1001)
+torch.manual_seed(0)
+
+in_channels = 4
+out_channels = 15
+kernel_size = 5
+padding = 3
+dilation = 1
+stride = 3
+output_padding = 0
+x = torch.tensor(np.random.uniform(-1, 1, [in_channels,100])).unsqueeze(0)
+
+in_length = x.shape[-1]
+conv = torch.nn.ConvTranspose1d(in_channels, out_channels, kernel_size, 
+                                dilation=dilation, padding=padding,
+                                output_padding=output_padding, stride=stride,
+                                bias=True, dtype=torch.float64)
+
+y = conv(x).detach().numpy()[0]
+
+# print('x',x.shape)
+# print('y',y.shape)
+# print('Before sliding the kernel, data is zero padded:',dilation*(kernel_size -1) - padding,'units in both sides.')
+# print('Expected Length:',(in_length-1)*stride-2*padding+dilation*(kernel_size -1)+output_padding+1)
+# plt.show()
+
+np.savetxt('test_data/convtranspose1d_torch_x_python.csv', x.squeeze(0), delimiter=',')
+np.savetxt('test_data/convtranspose1d_torch_y_python.csv', y, delimiter=',')
+
+with open('models/convtranspose1d_torch.json', 'w') as json_file:
+    json.dump(conv.state_dict(), json_file,cls=EncodeTensor)

python/convtranspose1d_torch_cc.py

@@ -0,0 +1,45 @@
+import torch
+from torch.utils.data import Dataset
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+from json import JSONEncoder
+import cached_conv as cc
+
+class EncodeTensor(JSONEncoder,Dataset):
+    def default(self, obj):
+        if isinstance(obj, torch.Tensor):
+            return obj.cpu().detach().numpy().tolist()
+        return super(json.NpEncoder, self).default(obj)
+
+cc.use_cached_conv(True)
+np.random.seed(1001)
+torch.manual_seed(0)
+
+in_channels = 4
+out_channels = 15
+kernel_size = 5
+padding = 3
+dilation = 1
+stride = 3
+output_padding = 0
+x = torch.tensor(np.random.uniform(-1, 1, [in_channels,100])).unsqueeze(0)
+
+in_length = x.shape[-1]
+conv = cc.ConvTranspose1d(in_channels, out_channels, kernel_size, 
+                                dilation=dilation, padding=padding,
+                                output_padding=output_padding, stride=stride,
+                                bias=True, dtype=torch.float64)
+
+y = conv(x).detach().numpy()[0]
+
+# print('x',x.shape)
+# print('y',y.shape)
+# print('Before sliding the kernel, data is zero padded:',dilation*(kernel_size -1) - padding,'units in both sides.')
+# print('Expected Length:',(in_length-1)*stride-2*padding+dilation*(kernel_size -1)+output_padding+1)
+# plt.show()
+np.savetxt('test_data/convtranspose1d_torch_x_python_cc.csv', x.squeeze(0), delimiter=',')
+np.savetxt('test_data/convtranspose1d_torch_y_python_cc.csv', y, delimiter=',')
+
+with open('models/convtranspose1d_torch.json', 'w') as json_file:
+    json.dump(conv.state_dict(), json_file,cls=EncodeTensor)