Skip to content
This repository was archived by the owner on Jul 1, 2023. It is now read-only.

Commit 00f0398

Browse files
rickwierengadan-zheng
rickwierenga
authored and
dan-zheng
committed
Move Dense and Dropout layers to separate files (#541)
1 parent ba2cf5c commit 00f0398

File tree

3 files changed

+169
-140
lines changed

3 files changed

+169
-140
lines changed

‎Sources/TensorFlow/Layers/Core.swift

Lines changed: 0 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -12,70 +12,6 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
fileprivate extension Tensor where Scalar: TensorFlowFloatingPoint {
16-
/// Computes dropout given a probability.
17-
// TODO: Remove the underscore once `droppingOut(probability:)` has been removed.
18-
@differentiable(wrt: self where Scalar: Differentiable)
19-
func _droppingOut(probability: Double) -> Tensor {
20-
let noise = Tensor(randomUniform: shape)
21-
let keepMask = noise .>= Scalar(probability)
22-
let keepProbability = Scalar(1.0 - probability)
23-
return self * Tensor(keepMask) / Tensor(keepProbability)
24-
}
25-
}
26-
27-
public extension Tensor where Scalar: TensorFlowFloatingPoint {
28-
/// Computes dropout given a probability.
29-
@available(*, deprecated, message: """
30-
This API will be removed after Swift for TensorFlow 0.6.
31-
For dropout, use the `Dropout` layer.
32-
""")
33-
@differentiable(wrt: self where Scalar: Differentiable)
34-
func droppingOut(probability: Double) -> Tensor {
35-
_droppingOut(probability: probability)
36-
}
37-
}
38-
39-
/// A dropout layer.
40-
///
41-
/// Dropout consists in randomly setting a fraction of input units to `0` at each update during
42-
/// training time, which helps prevent overfitting.
43-
@frozen
44-
public struct Dropout<Scalar: TensorFlowFloatingPoint>: ParameterlessLayer {
45-
@noDerivative public let probability: Double
46-
47-
/// Creates a dropout layer.
48-
///
49-
/// - Parameter probability: The drop probability.
50-
public init(probability: Double) {
51-
self.probability = probability
52-
}
53-
54-
@differentiable
55-
private func applyingTraining(to input: Tensor<Scalar>) -> Tensor<Scalar> {
56-
return input._droppingOut(probability: probability)
57-
}
58-
59-
@differentiable
60-
private func applyingInference(to input: Tensor<Scalar>) -> Tensor<Scalar> {
61-
return input
62-
}
63-
64-
/// Returns the output obtained from applying the layer to the given input.
65-
///
66-
/// - Parameter input: The input to the layer.
67-
/// - Returns: The output.
68-
@differentiable
69-
public func callAsFunction(_ input: Tensor<Scalar>) -> Tensor<Scalar> {
70-
switch Context.local.learningPhase {
71-
case .training:
72-
return applyingTraining(to: input)
73-
case .inference:
74-
return applyingInference(to: input)
75-
}
76-
}
77-
}
78-
7915
/// A flatten layer.
8016
///
8117
/// A flatten layer flattens the input when applied without affecting the batch size.
@@ -130,82 +66,6 @@ public struct Reshape<Scalar: TensorFlowFloatingPoint>: ParameterlessLayer {
13066
}
13167
}
13268

133-
/// A densely-connected neural network layer.
134-
///
135-
/// `Dense` implements the operation `activation(matmul(input, weight) + bias)`, where `weight` is
136-
/// a weight matrix, `bias` is a bias vector, and `activation` is an element-wise activation
137-
/// function.
138-
///
139-
/// This layer also supports 3-D weight tensors with 2-D bias matrices. In this case the first
140-
/// dimension of both is treated as the batch size that is aligned with the first dimension of
141-
/// `input` and the batch variant of the `matmul(_:_:)` operation is used, thus using a different
142-
/// weight and bias for each element in input batch.
143-
@frozen
144-
public struct Dense<Scalar: TensorFlowFloatingPoint>: Layer {
145-
/// The weight matrix.
146-
public var weight: Tensor<Scalar>
147-
/// The bias vector.
148-
public var bias: Tensor<Scalar>
149-
/// The element-wise activation function.
150-
@noDerivative public let activation: Activation
151-
/// Indicates whether this is a batched dense layer.
152-
@noDerivative internal let batched: Bool
153-
154-
/// The element-wise activation function type.
155-
public typealias Activation = @differentiable (Tensor<Scalar>) -> Tensor<Scalar>
156-
157-
public init(
158-
weight: Tensor<Scalar>,
159-
bias: Tensor<Scalar>,
160-
activation: @escaping Activation
161-
) {
162-
precondition(weight.rank <= 3, "The rank of the 'weight' tensor must be less than 4.")
163-
precondition(bias.rank <= 2, "The rank of the 'bias' tensor must be less than 3.")
164-
self.weight = weight
165-
self.bias = bias
166-
self.activation = activation
167-
self.batched = weight.rank == 3
168-
}
169-
170-
/// Returns the output obtained from applying the layer to the given input.
171-
///
172-
/// - Parameter input: The input to the layer.
173-
/// - Returns: The output.
174-
@differentiable
175-
public func callAsFunction(_ input: Tensor<Scalar>) -> Tensor<Scalar> {
176-
if batched {
177-
let hidden = matmul(input.expandingShape(at: 1), weight)
178-
return activation(hidden.squeezingShape(at: 1) + bias)
179-
}
180-
return activation(matmul(input, weight) + bias)
181-
}
182-
}
183-
184-
public extension Dense {
185-
/// Creates a `Dense` layer with the specified input size, output size, and element-wise
186-
/// activation function. The weight matrix is created with shape `[inputSize, outputSize]` and
187-
/// the bias vector is created with shape `[outputSize]`.
188-
///
189-
/// - Parameters:
190-
/// - inputSize: The dimensionality of the input space.
191-
/// - outputSize: The dimensionality of the output space.
192-
/// - activation: The activation function to use. The default value is `identity(_:)`.
193-
/// - weightInitializer: Initializer to use for `weight`.
194-
/// - biasInitializer: Initializer to use for `bias`.
195-
init(
196-
inputSize: Int,
197-
outputSize: Int,
198-
activation: @escaping Activation = identity,
199-
weightInitializer: ParameterInitializer<Scalar> = glorotUniform(),
200-
biasInitializer: ParameterInitializer<Scalar> = zeros()
201-
) {
202-
self.init(
203-
weight: weightInitializer([inputSize, outputSize]),
204-
bias: biasInitializer([outputSize]),
205-
activation: activation)
206-
}
207-
}
208-
20969
/// A layer that encloses a custom differentiable function.
21070
public struct Function<Input: Differentiable, Output: Differentiable>: ParameterlessLayer {
21171
public typealias Body = @differentiable (Input) -> Output

‎Sources/TensorFlow/Layers/Dense.swift

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
/// A densely-connected neural network layer.
16+
///
17+
/// `Dense` implements the operation `activation(matmul(input, weight) + bias)`, where `weight` is
18+
/// a weight matrix, `bias` is a bias vector, and `activation` is an element-wise activation
19+
/// function.
20+
///
21+
/// This layer also supports 3-D weight tensors with 2-D bias matrices. In this case the first
22+
/// dimension of both is treated as the batch size that is aligned with the first dimension of
23+
/// `input` and the batch variant of the `matmul(_:_:)` operation is used, thus using a different
24+
/// weight and bias for each element in input batch.
25+
@frozen
26+
public struct Dense<Scalar: TensorFlowFloatingPoint>: Layer {
27+
/// The weight matrix.
28+
public var weight: Tensor<Scalar>
29+
/// The bias vector.
30+
public var bias: Tensor<Scalar>
31+
/// The element-wise activation function.
32+
@noDerivative public let activation: Activation
33+
/// Indicates whether this is a batched dense layer.
34+
@noDerivative internal let batched: Bool
35+
36+
/// The element-wise activation function type.
37+
public typealias Activation = @differentiable (Tensor<Scalar>) -> Tensor<Scalar>
38+
39+
public init(
40+
weight: Tensor<Scalar>,
41+
bias: Tensor<Scalar>,
42+
activation: @escaping Activation
43+
) {
44+
precondition(weight.rank <= 3, "The rank of the 'weight' tensor must be less than 4.")
45+
precondition(bias.rank <= 2, "The rank of the 'bias' tensor must be less than 3.")
46+
self.weight = weight
47+
self.bias = bias
48+
self.activation = activation
49+
self.batched = weight.rank == 3
50+
}
51+
52+
/// Returns the output obtained from applying the layer to the given input.
53+
///
54+
/// - Parameter input: The input to the layer.
55+
/// - Returns: The output.
56+
@differentiable
57+
public func callAsFunction(_ input: Tensor<Scalar>) -> Tensor<Scalar> {
58+
if batched {
59+
let hidden = matmul(input.expandingShape(at: 1), weight)
60+
return activation(hidden.squeezingShape(at: 1) + bias)
61+
}
62+
return activation(matmul(input, weight) + bias)
63+
}
64+
}
65+
66+
public extension Dense {
67+
/// Creates a `Dense` layer with the specified input size, output size, and element-wise
68+
/// activation function. The weight matrix is created with shape `[inputSize, outputSize]` and
69+
/// the bias vector is created with shape `[outputSize]`.
70+
///
71+
/// - Parameters:
72+
/// - inputSize: The dimensionality of the input space.
73+
/// - outputSize: The dimensionality of the output space.
74+
/// - activation: The activation function to use. The default value is `identity(_:)`.
75+
/// - weightInitializer: Initializer to use for `weight`.
76+
/// - biasInitializer: Initializer to use for `bias`.
77+
init(
78+
inputSize: Int,
79+
outputSize: Int,
80+
activation: @escaping Activation = identity,
81+
weightInitializer: ParameterInitializer<Scalar> = glorotUniform(),
82+
biasInitializer: ParameterInitializer<Scalar> = zeros()
83+
) {
84+
self.init(
85+
weight: weightInitializer([inputSize, outputSize]),
86+
bias: biasInitializer([outputSize]),
87+
activation: activation)
88+
}
89+
}

‎Sources/TensorFlow/Layers/Dropout.swift

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
fileprivate extension Tensor where Scalar: TensorFlowFloatingPoint {
16+
/// Computes dropout given a probability.
17+
// TODO: Remove the underscore once `droppingOut(probability:)` has been removed.
18+
@differentiable(wrt: self where Scalar: Differentiable)
19+
func _droppingOut(probability: Double) -> Tensor {
20+
let noise = Tensor(randomUniform: shape)
21+
let keepMask = noise .>= Scalar(probability)
22+
let keepProbability = Scalar(1.0 - probability)
23+
return self * Tensor(keepMask) / Tensor(keepProbability)
24+
}
25+
}
26+
27+
public extension Tensor where Scalar: TensorFlowFloatingPoint {
28+
/// Computes dropout given a probability.
29+
@available(*, deprecated, message: """
30+
This API will be removed after Swift for TensorFlow 0.6.
31+
For dropout, use the `Dropout` layer.
32+
""")
33+
@differentiable(wrt: self where Scalar: Differentiable)
34+
func droppingOut(probability: Double) -> Tensor {
35+
_droppingOut(probability: probability)
36+
}
37+
}
38+
39+
/// A dropout layer.
40+
///
41+
/// Dropout consists in randomly setting a fraction of input units to `0` at each update during
42+
/// training time, which helps prevent overfitting.
43+
@frozen
44+
public struct Dropout<Scalar: TensorFlowFloatingPoint>: ParameterlessLayer {
45+
@noDerivative public let probability: Double
46+
47+
/// Creates a dropout layer.
48+
///
49+
/// - Parameter probability: The drop probability.
50+
/// - Precondition: probability must be a value between 0 and 1 (inclusive).
51+
public init(probability: Double) {
52+
precondition(0...1 ~= probability,
53+
"Probability must be a value between 0 and 1 (inclusive) but is \(probability)")
54+
self.probability = probability
55+
}
56+
57+
@differentiable
58+
private func applyingTraining(to input: Tensor<Scalar>) -> Tensor<Scalar> {
59+
return input._droppingOut(probability: probability)
60+
}
61+
62+
@differentiable
63+
private func applyingInference(to input: Tensor<Scalar>) -> Tensor<Scalar> {
64+
return input
65+
}
66+
67+
/// Returns the output obtained from applying the layer to the given input.
68+
///
69+
/// - Parameter input: The input to the layer.
70+
/// - Returns: The output.
71+
@differentiable
72+
public func callAsFunction(_ input: Tensor<Scalar>) -> Tensor<Scalar> {
73+
switch Context.local.learningPhase {
74+
case .training:
75+
return applyingTraining(to: input)
76+
case .inference:
77+
return applyingInference(to: input)
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)