Add leaky ReLU activation function (#260)

Author: Shashi456

Sources/TensorFlow/Operators/Math.swift

@@ -997,7 +997,7 @@ func _vjpLogSoftmax<T: TensorFlowFloatingPoint>(
 }
 
 /// Returns a tensor by applying an exponential linear unit.
-/// Specifically, computes `exp(features) - 1` if < 0, `features` otherwise.
+/// Specifically, computes `exp(x) - 1` if < 0, `x` otherwise.
 /// See [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
 /// ](http://arxiv.org/abs/1511.07289)
 @inlinable
@@ -1014,6 +1014,28 @@ func _vjpElu<T: TensorFlowFloatingPoint>(
     return (y, { v in Raw.eluGrad(gradients: v, outputs: y) })
 }
 
+/// Returns a tensor by applying the leaky ReLU activation function
+/// to the specified tensor element-wise.
+/// Specifically, computes `max(x, x * alpha)`.
+@inlinable
+@differentiable(wrt: x, vjp: _vjpLeakyRelu)
+public func leakyRelu<T: TensorFlowFloatingPoint>(
+    _ x: Tensor<T>,
+    alpha: Double = 0.2
+) -> Tensor<T> {
+    Raw.leakyRelu(features: x, alpha: alpha)
+}
+
+@inlinable
+func _vjpLeakyRelu<T: TensorFlowFloatingPoint>(
+    _ x: Tensor<T>,
+    alpha: Double
+) -> (Tensor<T>, (Tensor<T>) -> Tensor<T>) {
+    return (leakyRelu(x, alpha: alpha), { v in
+        Raw.leakyReluGrad(gradients: v, features: x, alpha: alpha)
+    })
+}
+
 /// Computes `relu` of the specified tensor element-wise.
 /// Specifically, computes `max(0, x)`.
 @inlinable

Tests/TensorFlowTests/OperatorTests/MathTests.swift

@@ -243,6 +243,13 @@ final class MathOperatorTests: XCTestCase {
         XCTAssertEqual(y, expected)
     }
 
+    func testLeakyRelu() {
+        let x = Tensor<Float>([[-1.0, 2.0, 3.0]])
+        let y = leakyRelu(x, alpha: 0.4)
+        let expected = Tensor<Float>([-0.4, 2, 3])
+        XCTAssertEqual(y, expected)
+    }
+
     func testXORInference() {
         func xor(_ x: Float, _ y: Float) -> Float {
             let x = Tensor<Float>([x, y]).reshaped(to: [1, 2])
@@ -314,6 +321,7 @@ final class MathOperatorTests: XCTestCase {
         ("testArgmax", testArgmax),
         ("testSoftplus", testSoftplus),
         ("testSoftsign", testSoftsign),
+        ("testLeakyRelu", testLeakyRelu),
         ("testCeilAndFloor", testCeilAndFloor),
         ("testSimpleMath", testSimpleMath),
         ("testStandardDeviation", testStandardDeviation),