Skip to content
This repository was archived by the owner on Apr 23, 2025. It is now read-only.

Commit ab7fcf8

Browse files
shabalindshabalind
authored
Add support for machine-readable JSON output (#230)
* Add support for machine-readable JSON output * Address pull request feedback * Make helper methods private * Make average and standardDeviation be an extension on array
1 parent f4855d6 commit ab7fcf8

10 files changed

+364
-166
lines changed

‎Benchmarks/Benchmark.swift

Lines changed: 6 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -18,72 +18,18 @@ protocol Benchmark {
1818
func run()
1919
}
2020

21-
protocol BenchmarkModel {
22-
func inferenceDefaults() -> BenchmarkSettings
23-
func inferenceBenchmark(_ settings: BenchmarkSettings) -> Benchmark
24-
func trainingDefaults() -> BenchmarkSettings
25-
func trainingBenchmark(_ settings: BenchmarkSettings) -> Benchmark
26-
}
27-
28-
enum BenchmarkVariety {
29-
case inferenceThroughput
30-
case trainingTime
31-
}
32-
33-
struct BenchmarkSettings {
34-
let batches: Int
35-
let batchSize: Int
36-
let iterations: Int
37-
let epochs: Int
38-
39-
func withDefaults(_ defaults: BenchmarkSettings) -> BenchmarkSettings {
40-
let newBatches = batches == -1 ? defaults.batches : batches
41-
let newBatchSize = batchSize == -1 ? defaults.batchSize : batchSize
42-
let newIterations = iterations == -1 ? defaults.iterations : iterations
43-
let newEpochs = epochs == -1 ? defaults.epochs : epochs
44-
return BenchmarkSettings(
45-
batches: newBatches, batchSize: newBatchSize,
46-
iterations: newIterations, epochs: newEpochs)
47-
}
48-
}
49-
50-
struct BenchmarkResults {
51-
let name: String
52-
let timings: [Double]
53-
let settings: BenchmarkSettings
54-
let variety: BenchmarkVariety
55-
}
56-
57-
extension BenchmarkResults {
58-
var interpretedTimings: [Double] {
59-
switch self.variety {
60-
case .inferenceThroughput:
61-
let batches = settings.batches
62-
let batchSize = settings.batchSize
63-
return timings.map { Double(batches * batchSize) / ($0 / 1000.0) }
64-
case .trainingTime:
65-
return timings
66-
}
67-
}
68-
}
69-
7021
/// Performs the specified benchmark over a certain number of iterations and provides the result to a callback function.
71-
func benchmark(
72-
name: String,
73-
settings: BenchmarkSettings,
74-
variety: BenchmarkVariety,
75-
benchmark: Benchmark,
76-
callback: (BenchmarkResults) -> Void
77-
) {
22+
func measure(
23+
configuration: BenchmarkConfiguration,
24+
benchmark: Benchmark
25+
) -> BenchmarkResults {
7826
var timings: [Double] = []
79-
for _ in 0..<settings.iterations {
27+
for _ in 0..<configuration.settings.iterations {
8028
let timing = time(benchmark.run)
8129
timings.append(timing)
8230
}
8331

84-
let results = BenchmarkResults(
85-
name: name, timings: timings, settings: settings, variety: variety)
86-
callback(results)
32+
return BenchmarkResults(configuration: configuration, timings: timings)
8733
}
8834

8935
/// Returns the time elapsed while running `body` in milliseconds.
@@ -95,39 +41,3 @@ func time(_ body: () -> Void) -> Double {
9541
let elapsed = end - start
9642
return elapsed
9743
}
98-
99-
/// Provides the average and standard deviation of an array of values.
100-
func statistics(for values: [Double]) -> (average: Double, standardDeviation: Double) {
101-
guard values.count > 0 else { return (average: 0.0, standardDeviation: 0.0) }
102-
guard values.count > 1 else { return (average: values.first!, standardDeviation: 0.0) }
103-
104-
let average = (values.reduce(0.0) { $0 + $1 }) / Double(values.count)
105-
106-
let standardDeviation = sqrt(
107-
values.reduce(0.0) { $0 + ($1 - average) * ($1 - average) }
108-
/ Double(values.count - 1))
109-
110-
return (average: average, standardDeviation: standardDeviation)
111-
}
112-
113-
// This is a simple callback function example that only logs the result to the console.
114-
func logResults(_ result: BenchmarkResults) {
115-
let (average, standardDeviation) = statistics(for: result.interpretedTimings)
116-
117-
switch result.variety {
118-
case .inferenceThroughput:
119-
print(
120-
"""
121-
Benchmark: \(result.name):
122-
\tAfter \(result.settings.iterations) iterations:
123-
\tSamples per second: \(String(format: "%.2f", average)), standard deviation: \(String(format: "%.2f", standardDeviation))
124-
""")
125-
case .trainingTime:
126-
print(
127-
"""
128-
Benchmark: \(result.name):
129-
\tAfter \(result.settings.iterations) iterations:
130-
\tAverage: \(String(format: "%.2f", average)) ms, standard deviation: \(String(format: "%.2f", standardDeviation)) ms
131-
""")
132-
}
133-
}

‎Benchmarks/BenchmarkConfiguration.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
/// Auxiliary struct used to show benchmark default settings coupled with benchmark name.
16+
struct BenchmarkConfiguration: Codable {
17+
let name: String
18+
let variety: BenchmarkVariety
19+
let settings: BenchmarkSettings
20+
}

‎Benchmarks/BenchmarkModel.swift

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
/// Protocol that contains benchmark factory methods for a given model.
16+
protocol BenchmarkModel {
17+
/// Settings to use in inference benchmark if no custom flags are given.
18+
var defaultInferenceSettings: BenchmarkSettings { get }
19+
20+
/// Create an instance of inference benchmark with given settings.
21+
func makeInferenceBenchmark(settings: BenchmarkSettings) -> Benchmark
22+
23+
/// Settings to use in training benchmark if no custom flags are given.
24+
var defaultTrainingSettings: BenchmarkSettings { get }
25+
26+
/// Create an instance of training benchmark with given settings.
27+
func makeTrainingBenchmark(settings: BenchmarkSettings) -> Benchmark
28+
}

‎Benchmarks/BenchmarkResults.swift

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
struct BenchmarkResults: Codable {
16+
let configuration: BenchmarkConfiguration
17+
let timings: [Double]
18+
}
19+
20+
extension BenchmarkResults {
21+
var interpretedTimings: [Double] {
22+
switch configuration.variety {
23+
case .inferenceThroughput:
24+
let batches = configuration.settings.batches
25+
let batchSize = configuration.settings.batchSize
26+
return timings.map { Double(batches * batchSize) / ($0 / 1000.0) }
27+
case .trainingTime:
28+
return timings
29+
}
30+
}
31+
}

‎Benchmarks/BenchmarkSettings.swift

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
struct BenchmarkSettings: Codable {
16+
let batches: Int
17+
let batchSize: Int
18+
let iterations: Int
19+
let epochs: Int
20+
21+
func withDefaults(_ defaults: BenchmarkSettings) -> BenchmarkSettings {
22+
let newBatches = batches == -1 ? defaults.batches : batches
23+
let newBatchSize = batchSize == -1 ? defaults.batchSize : batchSize
24+
let newIterations = iterations == -1 ? defaults.iterations : iterations
25+
let newEpochs = epochs == -1 ? defaults.epochs : epochs
26+
return BenchmarkSettings(
27+
batches: newBatches, batchSize: newBatchSize,
28+
iterations: newIterations, epochs: newEpochs)
29+
}
30+
}

‎Benchmarks/BenchmarkVariety.swift

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
enum BenchmarkVariety: String, Codable {
16+
case inferenceThroughput = "inference"
17+
case trainingTime = "training"
18+
}

‎Benchmarks/Models.swift

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,17 @@
1-
import Datasets
2-
import ImageClassificationModels
3-
4-
struct LeNetMnist: BenchmarkModel {
5-
func inferenceDefaults() -> BenchmarkSettings {
6-
return BenchmarkSettings(batches: 1000, batchSize: 1, iterations: 10, epochs: -1)
7-
}
8-
9-
func inferenceBenchmark(_ settings: BenchmarkSettings) -> Benchmark {
10-
return ImageClassificationInference<LeNet, MNIST>(settings: settings)
11-
}
12-
13-
func trainingDefaults() -> BenchmarkSettings {
14-
return BenchmarkSettings(batches: -1, batchSize: 128, iterations: 10, epochs: 1)
15-
}
16-
17-
func trainingBenchmark(_ settings: BenchmarkSettings) -> Benchmark {
18-
return ImageClassificationTraining<LeNet, MNIST>(settings: settings)
19-
}
20-
}
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
2114

2215
let benchmarkModels = [
23-
"lenet-mnist": LeNetMnist(),
16+
"lenet-mnist": LeNetMNIST(),
2417
]

‎Benchmarks/Models/LeNetMnist.swift

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import Datasets
16+
import ImageClassificationModels
17+
18+
struct LeNetMNIST: BenchmarkModel {
19+
20+
var defaultInferenceSettings: BenchmarkSettings {
21+
return BenchmarkSettings(batches: 1000, batchSize: 1, iterations: 10, epochs: -1)
22+
}
23+
24+
func makeInferenceBenchmark(settings: BenchmarkSettings) -> Benchmark {
25+
return ImageClassificationInference<LeNet, MNIST>(settings: settings)
26+
}
27+
28+
var defaultTrainingSettings: BenchmarkSettings {
29+
return BenchmarkSettings(batches: -1, batchSize: 128, iterations: 10, epochs: 1)
30+
}
31+
32+
func makeTrainingBenchmark(settings: BenchmarkSettings) -> Benchmark {
33+
return ImageClassificationTraining<LeNet, MNIST>(settings: settings)
34+
}
35+
}

0 commit comments

Comments
 (0)