Change BatchNorm momentum and epsilon to scalars.

Change `BatchNorm.momentum` and `BatchNorm.epsilon` to `Scalar` instead of
`Tensor<Scalar>`. Semantically, these properties are scalars.

It is not clear why `BatchNorm` originally defined these properties as `Tensor<Scalar>`
since the beginning:
6ca3813

Author: dan-zheng

Sources/TensorFlow/Layers/Normalization.swift

@@ -25,13 +25,13 @@ public struct BatchNorm<Scalar: TensorFlowFloatingPoint>: Layer {
     /// The feature dimension.
     @noDerivative public let axis: Int
     /// The momentum for the running mean and running variance.
-    @noDerivative public let momentum: Tensor<Scalar>
+    @noDerivative public let momentum: Scalar
     /// The offset value, also known as beta.
     public var offset: Tensor<Scalar>
     /// The scale value, also known as gamma.
     public var scale: Tensor<Scalar>
     /// The variance epsilon value.
-    @noDerivative public let epsilon: Tensor<Scalar>
+    @noDerivative public let epsilon: Scalar
     /// The running mean.
     @noDerivative public let runningMean: Parameter<Scalar>
     /// The running variance.
@@ -49,10 +49,10 @@ public struct BatchNorm<Scalar: TensorFlowFloatingPoint>: Layer {
     ///   - runningVariance: The running variance.
     public init(
         axis: Int,
-        momentum: Tensor<Scalar>,
+        momentum: Scalar,
         offset: Tensor<Scalar>,
         scale: Tensor<Scalar>,
-        epsilon: Tensor<Scalar>,
+        epsilon: Scalar,
         runningMean: Tensor<Scalar>,
         runningVariance: Tensor<Scalar>
     ) {
@@ -105,8 +105,8 @@ public struct BatchNorm<Scalar: TensorFlowFloatingPoint>: Layer {
     public init(
         featureCount: Int,
         axis: Int = -1,
-        momentum: Tensor<Scalar> = Tensor(0.99),
-        epsilon: Tensor<Scalar> = Tensor(0.001)
+        momentum: Scalar = 0.99,
+        epsilon: Scalar = 0.001
     ) {
         self.init(
             axis: axis,

Tests/TensorFlowTests/LayerTests.swift

@@ -1222,15 +1222,12 @@ final class LayerTests: XCTestCase {
         Context.local.learningPhase = .inference
         // This tests for a specific failure that had impacted the MiniGo model.
         let miniGoTensor = Tensor<Float>(randomUniform: [2, 19, 19, 256])
-        let miniGoBatchNorm = BatchNorm(
-            featureCount: 256,
-            momentum: Tensor<Float>(0.95),
-            epsilon: Tensor<Float>(1e-5))
+        let miniGoBatchNorm = BatchNorm<Float>(featureCount: 256, momentum: 0.95, epsilon: 1e-5)
         let miniGoResult = miniGoBatchNorm(miniGoTensor)
         XCTAssertEqual(miniGoTensor.shape, miniGoResult.shape)
 
         let x = Tensor<Float>(rangeFrom: 0, to: 20, stride: 1).reshaped(to: [4,5])
-        let epsilon = Tensor<Float>(0.001)
+        let epsilon: Float = 0.001
         let bnLayer = BatchNorm<Float>(featureCount: 5, axis: 1, epsilon: epsilon)
         // Test inference before any training.
         assertEqual(bnLayer.inferring(from: x), x / TensorFlow.sqrt(1 + epsilon), accuracy: 1e-5)