Adding kullback Leibler Divergence (#226)

Author: Shashi456

Sources/TensorFlow/Loss.swift

@@ -103,6 +103,18 @@ public func poissonLoss<Scalar: TensorFlowFloatingPoint>(
     return (predicted - expected * log(predicted)).mean()
 }
 
+/// Returns the Kullback-Leibler divergence between predictions and expectations.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: predicted)
+public func kullbackLeiblerDivergence<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    return (expected * log(expected / predicted)).sum()
+}
+
 /// Computes the softmax cross entropy (categorical cross entropy) between logits and labels.
 ///
 /// - Parameters:

Tests/TensorFlowTests/LossTests.swift

@@ -104,6 +104,14 @@ final class LossTests: XCTestCase {
         assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
     }
 
+    func testKullbackLeiblerDivergence() {
+        let predicted = Tensor<Float>([0.2, 0.3, 0.4])
+        let expected = Tensor<Float>([1.0, 4.0, 3.0])
+        let loss = kullbackLeiblerDivergence(predicted: predicted, expected: expected)
+        let expectedLoss: Float = 18.015217
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
+
     func testSoftmaxCrossEntropyWithProbabilitiesLoss() {
         let logits = Tensor<Float>(shape: [2, 4], scalars: [1, 2, 3, 4, 5, 6, 7, 8])
         let labels = Tensor<Float>(
@@ -199,6 +207,7 @@ final class LossTests: XCTestCase {
         ("testMeanSquaredLogarithmicError", testMeanSquaredLogarithmicError),
         ("testMeanAbsoluteError", testMeanAbsoluteError),
         ("testHingeLoss", testHingeLoss),
+        ("testKullbackLeiblerDivergence", testKullbackLeiblerDivergence),
         ("testCategoricalHingeLoss", testCategoricalHingeLoss),
         ("testSquaredHingeLoss", testSquaredHingeLoss),
         ("testPoissonLoss", testPoissonLoss),