add huber loss support

Author: jrabary

Sources/TensorFlow/Loss.swift

@@ -290,3 +290,31 @@ public func sigmoidCrossEntropy<Scalar: TensorFlowFloatingPoint>(
     let negAbsLogits = max(logits, -logits) // Custom `abs` to compute gradients at `0`.
     return reduction(maxLogitsWithZero - logits * labels + log1p(exp(-negAbsLogits)))
 }
+
+/// Returns the Huber loss between predictions and expectations.
+///
+/// For each value x in error = exprected - predicted
+///     loss = 0.5 * x^2                 if |x| <= ∂
+///     loss = 0.5 * ∂^2 + ∂ * (|x| - ∂) if |x| > ∂
+/// See: https://en.wikipedia.org/wiki/Huber_loss
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
+///   - delta: A floating point scalar, the point where the Huber loss function changes from a
+///     quadratic to linear. 
+///   - reduction: Reduction to apply on the computed element-wise loss values.
+@differentiable(wrt: predicted)
+public func huberLoss<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>,
+    expected: Tensor<Scalar>,
+    delta: Scalar,
+    reduction: @differentiable (Tensor<Scalar>) -> Tensor<Scalar> = { $0.sum() }
+) -> Tensor<Scalar> {
+
+    let error = expected - predicted
+    let absError = abs(error)
+    let quadratic = min(absError, delta)
+    let linear = absError - quadratic
+    return reduction((0.5 * quadratic * quadratic) + (delta * linear))
+}

Tests/TensorFlowTests/LossTests.swift

@@ -217,6 +217,23 @@ final class LossTests: XCTestCase {
         assertEqual(computedGradient, expectedGradient, accuracy: 1e-6)
     }
 
+    func testHuberLoss() {
+        let predictions = Tensor<Float>([[0.9, 0.2, 0.2], 
+                                         [0.8, 0.4, 0.6]])
+        let targets = Tensor<Float>([[1, 0, 1], 
+                                     [1, 0, 0]])
+
+        // test huber(x, x) = 0
+        let loss1 = huberLoss(predicted: predictions, expected: predictions, delta: Float(1))
+        XCTAssertTrue(loss1.isAlmostEqual(to: Tensor(0), tolerance: 0.001), "Huber Loss all correct")
+
+        // test huber(p, t) = 0.62500006 computed from tf.keras.losses.Huber
+        let loss2 = huberLoss(predicted: predictions, expected: targets, delta: Float(1))
+        XCTAssertTrue(
+            loss2.isAlmostEqual(to: Tensor(0.62500006), tolerance: 0.001), 
+            "Huber Loss unweighted")
+    }
+
     static var allTests = [
         ("testL1Loss", testL1Loss),
         ("testL2Loss", testL2Loss),
@@ -237,5 +254,6 @@ final class LossTests: XCTestCase {
          testSoftmaxCrossEntropyWithProbabilitiesGrad),
         ("testSigmoidCrossEntropyLoss", testSigmoidCrossEntropyLoss),
         ("testSigmoidCrossEntropyGradient", testSigmoidCrossEntropyGradient),
+        ("testHuberLoss", testHuberLoss)
     ]
 }