Add exponential linear unit (#252)

Author: Shashi456

Sources/TensorFlow/Operators/Math.swift

@@ -996,6 +996,24 @@ func _vjpLogSoftmax<T: TensorFlowFloatingPoint>(
     return (value, { v in v - v.sum(alongAxes: -1) * exp(value) })
 }
 
+/// Returns a tensor by applying an exponential linear unit.
+/// Specifically, computes `exp(features) - 1` if < 0, `features` otherwise.
+/// See [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
+/// ](http://arxiv.org/abs/1511.07289)
+@inlinable
+@differentiable(vjp: _vjpElu)
+public func elu<T: TensorFlowFloatingPoint>(_ x: Tensor<T>) -> Tensor<T> {
+    Raw.elu(features: x)
+}
+
+@inlinable
+func _vjpElu<T: TensorFlowFloatingPoint>(
+    _ x: Tensor<T>
+) -> (Tensor<T>, (Tensor<T>) -> Tensor<T>) {
+    let y = elu(x)
+    return (y, { v in Raw.eluGrad(gradients: v, outputs: y) })
+}
+
 /// Computes `relu` of the specified tensor element-wise.
 /// Specifically, computes `max(0, x)`.
 @inlinable

Tests/TensorFlowTests/OperatorTests/MathTests.swift

@@ -223,17 +223,24 @@ final class MathOperatorTests: XCTestCase {
     }
 
     func testSoftplus() {
-      let x = Tensor<Float>([1.0, 2.0, 3.0])
-      let y = softplus(x)
-      let expected = Tensor<Float>([1.3132616,  2.126928, 3.0485873])
-      XCTAssertEqual(y, expected)
+        let x = Tensor<Float>([1.0, 2.0, 3.0])
+        let y = softplus(x)
+        let expected = Tensor<Float>([1.3132616,  2.126928, 3.0485873])
+        XCTAssertEqual(y, expected)
     }
 
     func testSoftsign() {
-      let x = Tensor<Float>([1.0, 4.0, 3.0])
-      let y = softsign(x)
-      let expected = Tensor<Float>([0.5 , 0.8 , 0.75])
-      XCTAssertEqual(y, expected)
+        let x = Tensor<Float>([1.0, 4.0, 3.0])
+        let y = softsign(x)
+        let expected = Tensor<Float>([0.5 , 0.8 , 0.75])
+        XCTAssertEqual(y, expected)
+    }
+
+    func testElu() {
+        let x = Tensor<Float>([-1.0, 2.0, 3.0])
+        let y = elu(x)
+        let expected = Tensor<Float>([-0.63212055, 2, 3])
+        XCTAssertEqual(y, expected)
     }
 
     func testXORInference() {
@@ -303,6 +310,7 @@ final class MathOperatorTests: XCTestCase {
         ("testSign", testSign),
         ("testReduction", testReduction),
         ("testCosineSimilarity", testCosineSimilarity),
+        ("testElu",testElu),
         ("testArgmax", testArgmax),
         ("testSoftplus", testSoftplus),
         ("testSoftsign", testSoftsign),