Skip to content
This repository was archived by the owner on Jul 1, 2023. It is now read-only.

TF-425: Make learning rates dynamically settable. #81

Merged
merged 1 commit into from
Apr 9, 2019
Merged

TF-425: Make learning rates dynamically settable. #81

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions Sources/DeepLearning/Optimizer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public protocol Optimizer {
/// The scalar parameter type.
associatedtype Scalar: FloatingPoint
/// The learning rate.
var learningRate: Scalar { get }
var learningRate: Scalar { get set }
/// Updates the specified differentiable variables along the specified
/// direction.
mutating func update(_ variables: inout Model.AllDifferentiableVariables,
Expand All @@ -42,17 +42,17 @@ public protocol Optimizer {
public class Adam<Model: Layer, Scalar: TensorFlowFloatingPoint>: Optimizer
where Model.AllDifferentiableVariables == Model.CotangentVector {
/// The learning rate.
public let learningRate: Scalar
public var learningRate: Scalar
/// A coefficient used to calculate the first and second moments of
/// gradients.
public var beta1: Scalar
/// A coefficient used to calculate the first and second moments of
/// gradients.
public var beta2: Scalar
/// A small scalar added to the denominator to improve numerical stability.
public let epsilon: Scalar
public var epsilon: Scalar
/// The weight decay.
public let decay: Scalar
public var decay: Scalar

public init(
learningRate: Scalar = 1e-3,
Expand Down Expand Up @@ -122,13 +122,13 @@ public class Adam<Model: Layer, Scalar: TensorFlowFloatingPoint>: Optimizer
public class RMSProp<Model: Layer, Scalar: TensorFlowFloatingPoint>: Optimizer
where Model.AllDifferentiableVariables == Model.CotangentVector {
/// The learning rate.
public let learningRate: Scalar
public var learningRate: Scalar
// TODO: Document `rho`. Keras doesn't document `rho`.
public let rho: Scalar
public var rho: Scalar
/// A small scalar added to the denominator to improve numerical stability.
public let epsilon: Scalar
public var epsilon: Scalar
/// The weight decay.
public let decay: Scalar
public var decay: Scalar

public init(
learningRate: Scalar = 0.001,
Expand Down Expand Up @@ -180,14 +180,14 @@ public class RMSProp<Model: Layer, Scalar: TensorFlowFloatingPoint>: Optimizer
public class SGD<Model: Layer, Scalar: TensorFlowFloatingPoint>: Optimizer
where Model.AllDifferentiableVariables == Model.CotangentVector {
/// The learning rate.
public let learningRate: Scalar
public var learningRate: Scalar
/// The momentum factor. It accelerates stochastic gradient descent in the relevant direction
/// and dampens oscillations.
public let momentum: Scalar
public var momentum: Scalar
/// The weight decay.
public let decay: Scalar
public var decay: Scalar
/// Use Nesterov momentum if true.
public let nesterov: Bool
public var nesterov: Bool

public init(
learningRate: Scalar = 0.01,
Expand Down