Merge pull request #138 from aigamedev/ELU

Exponential Linear Units

sknn/backend/lasagne/mlp.py

@@ -28,6 +28,10 @@
 from ...nn import Layer, Convolution, ansi
 
 
+def explin(x):
+    return x * (x>=0) + (x<0) * (T.exp(x) - 1)
+
+
 class MultiLayerPerceptronBackend(BaseBackend):
     """
     Abstract base class for wrapping the multi-layer perceptron functionality
@@ -98,7 +102,8 @@ def _get_activation(self, l):
                           'Sigmoid': nl.sigmoid,
                           'Tanh': nl.tanh,
                           'Softmax': nl.softmax,
-                          'Linear': nl.linear}
+                          'Linear': nl.linear,
+                          'ExpLin': explin}
 
         assert l.type in nonlinearities,\
             "Layer type `%s` is not supported for `%s`." % (l.type, l.name)

sknn/nn.py

@@ -37,7 +37,7 @@ class Layer(object):
 
     type: str
         Select which activation function this layer should use, as a string.  Specifically,
-        options are ``Rectifier``, ``Sigmoid``, ``Tanh``, and ``Maxout`` for non-linear layers
+        options are ``Rectifier``, ``Sigmoid``, ``Tanh``, and ``ExpLin`` for non-linear layers
         and ``Linear``, ``Softmax`` or ``Gaussian`` for linear layers.
 
     name: str, optional
@@ -89,7 +89,7 @@ def __init__(
         assert warning is None,\
             "Specify layer parameters as keyword arguments, not positional arguments."
 
-        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear', 'Softmax', 'Gaussian']:
+        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear', 'Softmax', 'Gaussian', 'ExpLin']:
             raise NotImplementedError("Layer type `%s` is not implemented." % type)
 
         self.name = name
@@ -130,7 +130,7 @@ class Convolution(Layer):
     type: str
         Select which activation function this convolution layer should use, as a string.
         For hidden layers, you can use the following convolution types ``Rectifier``,
-        ``Sigmoid``, ``Tanh`` or ``Linear``.
+        ``ExpLin``, ``Sigmoid``, ``Tanh`` or ``Linear``.
 
     name: str, optional
         You optionally can specify a name for this layer, and its parameters
@@ -225,7 +225,7 @@ def __init__(
         assert warning is None,\
             "Specify layer parameters as keyword arguments, not positional arguments."
 
-        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear']:
+        if type not in ['Rectifier', 'Sigmoid', 'Tanh', 'Linear', 'ExpLin']:
             raise NotImplementedError("Convolution type `%s` is not implemented." % (type,))
         if border_mode not in ['valid', 'full', 'same']:
             raise NotImplementedError("Convolution border_mode `%s` is not implemented." % (border_mode,))
@@ -261,7 +261,7 @@ class NeuralNetwork(object):
         contains its type, optional name, and any paramaters required.
 
             * For hidden layers, you can use the following layer types:
-              ``Rectifier``, ``Sigmoid``, ``Tanh``, or ``Convolution``.
+              ``Rectifier``, ``ExpLin``, ``Sigmoid``, ``Tanh``, or ``Convolution``.
             * For output layers, you can use the following layer types:
               ``Linear`` or ``Softmax``.
 

sknn/tests/test_conv.py

@@ -132,7 +132,7 @@ def test_UpscalingFactorFour(self):
     def test_DownscaleUpscale(self):
         self._run(MLPR(
                     layers=[
-                        C("Rectifier", channels=6, kernel_shape=(3,3), pool_shape=(2,2), border_mode='same'),
+                        C("ExpLin", channels=6, kernel_shape=(3,3), pool_shape=(2,2), border_mode='same'),
                         C("Rectifier", channels=3, kernel_shape=(3,3), scale_factor=(2,2), border_mode='same')],
                     n_iter=1),
                   scale=1)
@@ -151,7 +151,7 @@ def test_SmallSquareKernel(self):
 
     def test_SquareKernelFull(self):
         nn = MLPR(layers=[
-                    C("Rectifier", channels=4, kernel_shape=(3,3), border_mode='full'),
+                    C("ExpLin", channels=4, kernel_shape=(3,3), border_mode='full'),
                     L("Linear", units=5)])
 
         a_in = numpy.zeros((8,32,32,1))
@@ -178,7 +178,7 @@ def test_VerticalKernel(self):
 
     def test_SquareKernelPool(self):
         nn = MLPR(layers=[
-                    C("Rectifier", channels=4, kernel_shape=(3,3), pool_shape=(2,2), border_mode='valid'),
+                    C("ExpLin", channels=4, kernel_shape=(3,3), pool_shape=(2,2), border_mode='valid'),
                     L("Linear", units=5)])
 
         a_in = numpy.zeros((8,32,32,1))
@@ -201,7 +201,7 @@ def test_InvalidBorderMode(self):
     def test_MultiLayerPooling(self):
         nn = MLPR(layers=[
                     C("Rectifier", channels=4, kernel_shape=(3,3), pool_shape=(2,2)),
-                    C("Rectifier", channels=4, kernel_shape=(3,3), pool_shape=(2,2)),
+                    C("ExpLin", channels=4, kernel_shape=(3,3), pool_shape=(2,2)),
                     L("Linear")])
 
         a_in, a_out = numpy.zeros((8,32,32,1)), numpy.zeros((8,16))
@@ -235,6 +235,9 @@ def _run(self, activation):
     def test_RectifierConv(self):
         self._run("Rectifier")
 
+    def test_RectifierConv(self):
+        self._run("ExpLin")
+
     def test_SigmoidConv(self):
         self._run("Sigmoid")
 

sknn/tests/test_data.py

@@ -135,10 +135,9 @@ def test_MultipleOutputRandom(self):
 
 class TestMaskedDataClassification(unittest.TestCase):
 
-    def check(self, a_in, a_out, a_mask, act='Softmax'):
-        nn = MLPC(layers=[L(act)], learning_rule='rmsprop', n_iter=100)
+    def check(self, a_in, a_out, a_mask, act='Softmax', n_iter=100):
+        nn = MLPC(layers=[L(act)], learning_rule='rmsprop', n_iter=n_iter)
         nn.fit(a_in, a_out, a_mask)
-        print(nn.classes_)
         return nn.predict_proba(a_in)
 
     def test_TwoLabelsOne(self):
@@ -169,7 +168,7 @@ def test_FourLabels(self):
         a_mask = numpy.zeros((16,), dtype=numpy.int32)
         a_mask[chosen] = 1.0
 
-        a_test = self.check(a_in, a_out, a_mask, act="Sigmoid").mean(axis=0)
+        a_test = self.check(a_in, a_out, a_mask, act="Sigmoid", n_iter=250).mean(axis=0)
         for i in range(a_out.shape[1]):
             compare = assert_greater if a_out[chosen][i]==0 else assert_less
             compare(a_test[i*2], a_test[i*2+1])

sknn/tests/test_deep.py

@@ -23,6 +23,7 @@ def setUp(self):
             layers=[
                 L("Rectifier", units=16),
                 L("Sigmoid", units=12),
+                L("ExpLin", units=8),
                 L("Tanh", units=4),
                 L("Linear")],
             n_iter=1)
@@ -45,7 +46,7 @@ def setUp(self):
     def run_EqualityTest(self, copier, asserter):
         # Only PyLearn2 supports Maxout.
         extra =  ["Maxout"] if sknn.backend.name == 'pylearn2' else []
-        for activation in ["Rectifier", "Sigmoid", "Tanh"] + extra:
+        for activation in ["Rectifier", "Sigmoid", "Tanh", "ExpLin"] + extra:
             nn1 = MLPR(layers=[L(activation, units=16, pieces=2), L("Linear", units=1)], random_state=1234)
             nn1._initialize(self.a_in, self.a_out)
 
@@ -105,4 +106,3 @@ def test_UnusedParameterWarning(self):
 
         assert_in('Parameter `pieces` is unused', self.buf.getvalue())
         self.buf = io.StringIO() # clear
-