Merge branch 'master' into lc

Author: rxwei

Sources/TensorFlow/Layers/Recurrent.swift

@@ -88,7 +88,7 @@ public struct SimpleRNNCell<Scalar: TensorFlowFloatingPoint>: RNNCell, VectorPro
     // TODO(TF-507): Revert to `typealias State = Tensor<Scalar>` after
     // SR-10697 is fixed.
     public struct State: Equatable, Differentiable, VectorProtocol, KeyPathIterable {
-        public let value: Tensor<Scalar>
+        public var value: Tensor<Scalar>
         public init(_ value: Tensor<Scalar>) {
             self.value = value
         }

Sources/TensorFlow/Loss.swift

@@ -12,7 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// Computes the mean squared error between predictions and labels.
+/// Returns the L1 loss between predictions and labels.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - labels: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: predicted)
+public func l1Loss<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    return abs(expected - predicted).sum()
+}
+
+/// Returns the L2 loss between predictions and labels.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - labels: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: predicted)
+public func l2Loss<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    return (expected - predicted).squared().sum()
+}
+
+/// Returns the mean squared error between predictions and labels.
 ///
 /// - Parameters:
 ///   - predicted: Predicted outputs from a neural network.
@@ -41,7 +65,7 @@ public func meanSquaredLogarithmicError<Scalar: TensorFlowFloatingPoint>(
     return (logPredicted - logExpected).squared().mean()
 }
 
-/// Computes the mean absolute error between predictions and expectations.
+/// Returns the mean absolute error between predictions and expectations.
 ///
 /// - Parameters:
 ///   - predicted: Predicted outputs from a neural network.
@@ -53,6 +77,19 @@ public func meanAbsoluteError<Scalar: TensorFlowFloatingPoint>(
     return abs(expected - predicted).mean()
 }
 
+/// Returns the mean absolute percentage error between predictions and expectations.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: predicted)
+public func meanAbsolutePercentageError<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    let diff = abs((expected - predicted) / abs(expected))
+    return 100 * diff.mean()
+}
+
 /// Returns the hinge loss between predictions and expectations.
 ///
 /// - Parameters:
@@ -65,6 +102,24 @@ public func hingeLoss<Scalar: TensorFlowFloatingPoint>(
     return max(Tensor(1) - expected * predicted, Tensor(0)).mean()
 }
 
+/// Returns the cosine similarity between predictions and expectations.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: (predicted, expected))
+public func cosineSimilarity<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    return -(expected * predicted).sum() /
+        (sqrt(expected.squared().sum()) * sqrt(predicted.squared().sum()))
+}
+
+/// Returns the squared hinge loss between predictions and expectations.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
 @differentiable(wrt: predicted)
 public func squaredHingeLoss<Scalar: TensorFlowFloatingPoint>(
     predicted: Tensor<Scalar>, expected: Tensor<Scalar>
@@ -119,7 +174,19 @@ public func poissonLoss<Scalar: TensorFlowFloatingPoint>(
     return (predicted - expected * log(predicted)).mean()
 }
 
-/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels.
+/// Returns the Kullback-Leibler divergence between predictions and expectations.
+///
+/// - Parameters:
+///   - predicted: Predicted outputs from a neural network.
+///   - expected: Expected values, i.e. targets, that correspond to the correct output.
+@differentiable(wrt: predicted)
+public func kullbackLeiblerDivergence<Scalar: TensorFlowFloatingPoint>(
+    predicted: Tensor<Scalar>, expected: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    return (expected * log(expected / predicted)).sum()
+}
+
+/// Returns the softmax cross entropy (categorical cross entropy) between logits and labels.
 ///
 /// - Parameters:
 ///   - logits: One-hot encoded outputs from a neural network.
@@ -140,7 +207,7 @@ func _vjpSoftmaxCrossEntropy<Scalar: TensorFlowFloatingPoint>(
     return (loss.mean(), { v in (v / batchSize) * grad })
 }
 
-/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels.
+/// Returns the softmax cross entropy (categorical cross entropy) between logits and labels.
 ///
 /// - Parameters:
 ///   - logits: Unscaled log probabilities from a neural network.
@@ -162,7 +229,7 @@ func _vjpSoftmaxCrossEntropy<Scalar: TensorFlowFloatingPoint>(
     return (loss.mean(), { v in v / batchSize * grad })
 }
 
-/// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels.
+/// Returns the sigmoid cross entropy (binary cross entropy) between logits and labels.
 ///
 /// The reduction is reduced over all elements. If reduced over batch size is intended, please
 /// consider to scale the loss.

Sources/TensorFlow/Operators/Math.swift

@@ -876,7 +876,7 @@ public func pow<T: TensorFlowFloatingPoint>(_ x: Tensor<T>, _ n: Int) -> Tensor<
 @inlinable
 // @differentiable
 public func root<T: TensorFlowFloatingPoint>(_ x: Tensor<T>, _ n: Int) -> Tensor<T> {
-    pow(x, Tensor(T(1) / T(n)))
+    sign(x) * pow(abs(x), Tensor(T(1) / T(n)))
 }
 
 /// Computes the element-wise maximum of two tensors.

Sources/TensorFlow/Optimizer.swift

@@ -291,7 +291,7 @@ public class RiemannSGD<Model: Layer, Scalar: FloatingPoint>: Optimizer
 
     public func update(_ model: inout Model.AllDifferentiableVariables,
                        along direction: Model.TangentVector) {
-        model = model.moved(along: learningRate * (.zero - direction))
+        model.move(along: learningRate * (.zero - direction))
     }
 }
 

Tests/TensorFlowTests/Helpers.swift

@@ -21,7 +21,9 @@ internal func assertEqual<T: TensorFlowFloatingPoint>(
 ) {
     for (x, y) in zip(x, y) {
         if x.isNaN || y.isNaN {
-            XCTAssertTrue(x.isNaN && y.isNaN, message, file: file, line: line)
+            XCTAssertTrue(x.isNaN && y.isNaN,
+                          "\(x) is not equal to \(y) - \(message)",
+                          file: file, line: line)
             continue
         }
         XCTAssertEqual(x, y, accuracy: accuracy, message, file: file, line: line)

Tests/TensorFlowTests/LossTests.swift

@@ -16,6 +16,22 @@ import XCTest
 @testable import TensorFlow
 
 final class LossTests: XCTestCase {
+    func testL1Loss() {
+        let predicted = Tensor<Float>([1, 2, 3, 4])
+        let expected = Tensor<Float>([0.1, 0.2, 0.3, 0.4])
+        let loss = l1Loss(predicted: predicted, expected: expected)
+        let expectedLoss: Float = 9.0
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
+
+    func testL2Loss() {
+        let predicted = Tensor<Float>([1, 2, 3, 4])
+        let expected = Tensor<Float>([0.5, 1.5, 2.5, 3.5])
+        let loss = l2Loss(predicted: predicted, expected: expected)
+        let expectedLoss: Float = 1.0
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
+
     func testMeanSquaredErrorLoss() {
         let predicted = Tensor<Float>(shape: [2, 4], scalars: [1, 2, 3, 4, 5, 6, 7, 8])
         let expected = Tensor<Float>(
@@ -49,6 +65,15 @@ final class LossTests: XCTestCase {
         assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
     }
 
+    func testMeanAbsolutePercentageError() {
+        let predicted = Tensor<Float>([1, 2, 3, 4, 5, 6, 7, 8])
+        let expected = Tensor<Float>([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8])
+
+        let loss = meanAbsolutePercentageError(predicted: predicted, expected: expected)
+        let expectedLoss: Float = 900.0
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
+
     func testMeanSquaredErrorGrad() {
         let predicted = Tensor<Float>(shape: [2, 4], scalars: [1, 2, 3, 4, 5, 6, 7, 8])
         let expected = Tensor<Float>(
@@ -78,6 +103,14 @@ final class LossTests: XCTestCase {
         let expectedLoss: Float = 0.225
         assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
     }
+    
+    func testCosineSimilarityLoss() {
+        let predicted = Tensor<Float>([1, 2, 3, 4, 5, 6, 7, 8])
+        let expected = Tensor<Float>([0.5, 1, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0])
+        let loss = cosineSimilarity(predicted: predicted, expected: expected)
+        let expectedLoss: Float = -1.0
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
 
     func testSquaredHingeLoss() {
         let predicted = Tensor<Float>([1, 2, 3, 4, 5, 6, 7, 8])
@@ -112,6 +145,14 @@ final class LossTests: XCTestCase {
         assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
     }
 
+    func testKullbackLeiblerDivergence() {
+        let predicted = Tensor<Float>([0.2, 0.3, 0.4])
+        let expected = Tensor<Float>([1.0, 4.0, 3.0])
+        let loss = kullbackLeiblerDivergence(predicted: predicted, expected: expected)
+        let expectedLoss: Float = 18.015217
+        assertElementsEqual(expected: Tensor(expectedLoss), actual: loss)
+    }
+
     func testSoftmaxCrossEntropyWithProbabilitiesLoss() {
         let logits = Tensor<Float>(shape: [2, 4], scalars: [1, 2, 3, 4, 5, 6, 7, 8])
         let labels = Tensor<Float>(
@@ -202,12 +243,17 @@ final class LossTests: XCTestCase {
     }
 
     static var allTests = [
+        ("testL1Loss", testL1Loss),
+        ("testL2Loss", testL2Loss),
         ("testMeanSquaredErrorLoss", testMeanSquaredErrorLoss),
         ("testMeanSquaredErrorGrad", testMeanSquaredErrorGrad),
         ("testMeanSquaredLogarithmicError", testMeanSquaredLogarithmicError),
         ("testMeanAbsoluteError", testMeanAbsoluteError),
+        ("testMeanAbsolutePercentageError", testMeanAbsolutePercentageError),
         ("testHingeLoss", testHingeLoss),
+        ("testKullbackLeiblerDivergence", testKullbackLeiblerDivergence),
         ("testCategoricalHingeLoss", testCategoricalHingeLoss),
+        ("testCosineSimilarityLoss", testCosineSimilarityLoss),
         ("testSquaredHingeLoss", testSquaredHingeLoss),
         ("testPoissonLoss",testPoissonLoss),
         ("testLogcoshLoss", testLogcoshLoss),