Add softplus and softsign (#225)

Author: Shashi456

Sources/TensorFlow/Operators/Math.swift

@@ -896,6 +896,36 @@ internal func _vjpSigmoid<T: TensorFlowFloatingPoint>(
     (sigmoid(x), { v in Raw.sigmoidGrad(x, dy: v) })
 }
 
+/// Returns the softplus of the specified tensor element-wise.
+/// Specifically, computes `log(exp(features) + 1)`.
+@inlinable
+@differentiable(vjp: _vjpSoftplus)
+public func softplus<T: TensorFlowFloatingPoint>(_ features: Tensor<T>) -> Tensor<T> {
+    Raw.softplus(features: features)
+}
+
+@inlinable
+internal func _vjpSoftplus<T: TensorFlowFloatingPoint>(
+    _ features: Tensor<T>
+) -> (Tensor<T>, (Tensor<T>) -> Tensor<T>) {
+    (softplus(features), { v in Raw.softplusGrad(gradients: v, features: features)})
+}
+
+/// Returns the softsign of the specified tensor element-wise.
+/// Specifically, computes `features/ (abs(features) + 1)`.
+@inlinable
+@differentiable(vjp: _vjpSoftsign)
+public func softsign<T: TensorFlowFloatingPoint>(_ features: Tensor<T>) -> Tensor<T> {
+    Raw.softsign(features: features)
+}
+
+@inlinable
+internal func _vjpSoftsign<T: TensorFlowFloatingPoint>(
+    _ features: Tensor<T>
+) -> (Tensor<T>, (Tensor<T>) -> Tensor<T>) {
+    (softsign(features), { v in Raw.softsignGrad(gradients: v, features: features)})
+}
+
 /// Computes the softmax of the specified tensor along the last axis.
 /// Specifically, computes `exp(x) / exp(x).sum(alongAxes: -1)`.
 @inlinable

Tests/TensorFlowTests/OperatorTests/MathTests.swift

@@ -118,7 +118,7 @@ final class MathOperatorTests: XCTestCase {
             x.variance(squeezingAxes: 0),
             Tensor(shape: [5], scalars: [0, 0, 0, 0, 0]))
         XCTAssertEqual(
-            x.variance(alongAxes: 0), 
+            x.variance(alongAxes: 0),
             Tensor(shape: [5], scalars: [0, 0, 0, 0, 0]))
         XCTAssertEqual(
             x.variance(squeezingAxes: 1),
@@ -214,6 +214,20 @@ final class MathOperatorTests: XCTestCase {
         XCTAssertEqual(result.scalars, [12.5, 6.5])
     }
 
+    func testSoftplus() {
+      let x = Tensor<Float>([1.0, 2.0, 3.0])
+      let y = softplus(x)
+      let expected = Tensor<Float>([1.3132616,  2.126928, 3.0485873])
+      XCTAssertEqual(y, expected)
+    }
+
+    func testSoftsign() {
+      let x = Tensor<Float>([1.0, 4.0, 3.0])
+      let y = softsign(x)
+      let expected = Tensor<Float>([0.5 , 0.8 , 0.75])
+      XCTAssertEqual(y, expected)
+    }
+
     func testXORInference() {
         func xor(_ x: Float, _ y: Float) -> Float {
             let x = Tensor<Float>([x, y]).reshaped(to: [1, 2])
@@ -281,6 +295,8 @@ final class MathOperatorTests: XCTestCase {
         ("testSign", testSign),
         ("testReduction", testReduction),
         ("testArgmax", testArgmax),
+        ("testSoftplus", testSoftplus),
+        ("testSoftsign", testSoftsign),
         ("testCeilAndFloor", testCeilAndFloor),
         ("testSimpleMath", testSimpleMath),
         ("testStandardDeviation", testStandardDeviation),