Skip to content

Commit 79708e6

Browse files
alexcrichtonalexcrichton
committed
rollup merge of #21462: ahmedcharles/remove-ratchet
Conflicts: src/libtest/lib.rs
2 parents 87c3ee8 + eb00913 commit 79708e6

File tree

1 file changed

+3
-246
lines changed

1 file changed

+3
-246
lines changed

‎src/libtest/lib.rs

Lines changed: 3 additions & 246 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ extern crate "serialize" as rustc_serialize;
4444
extern crate term;
4545

4646
pub use self::TestFn::*;
47-
pub use self::MetricChange::*;
4847
pub use self::ColorConfig::*;
4948
pub use self::TestResult::*;
5049
pub use self::TestName::*;
@@ -62,7 +61,6 @@ use term::color::{Color, RED, YELLOW, GREEN, CYAN};
6261
use std::any::Any;
6362
use std::cmp;
6463
use std::collections::BTreeMap;
65-
use std::f64;
6664
use std::fmt;
6765
use std::io::fs::PathExtensions;
6866
use std::io::stdio::StdWriter;
@@ -81,8 +79,7 @@ use std::time::Duration;
8179
pub mod test {
8280
pub use {Bencher, TestName, TestResult, TestDesc,
8381
TestDescAndFn, TestOpts, TrFailed, TrIgnored, TrOk,
84-
Metric, MetricMap, MetricAdded, MetricRemoved,
85-
MetricChange, Improvement, Regression, LikelyNoise,
82+
Metric, MetricMap,
8683
StaticTestFn, StaticTestName, DynTestName, DynTestFn,
8784
run_test, test_main, test_main_static, filter_tests,
8885
parse_opts, StaticBenchFn, ShouldFail};
@@ -241,18 +238,6 @@ impl Clone for MetricMap {
241238
}
242239
}
243240

244-
/// Analysis of a single change in metric
245-
#[derive(Copy, PartialEq, Show)]
246-
pub enum MetricChange {
247-
LikelyNoise,
248-
MetricAdded,
249-
MetricRemoved,
250-
Improvement(f64),
251-
Regression(f64)
252-
}
253-
254-
pub type MetricDiff = BTreeMap<String,MetricChange>;
255-
256241
// The default console test runner. It accepts the command line
257242
// arguments and a vector of test_descs.
258243
pub fn test_main(args: &[String], tests: Vec<TestDescAndFn> ) {
@@ -516,22 +501,6 @@ impl<T: Writer> ConsoleTestState<T> {
516501
self.write_pretty("bench", term::color::CYAN)
517502
}
518503

519-
pub fn write_added(&mut self) -> io::IoResult<()> {
520-
self.write_pretty("added", term::color::GREEN)
521-
}
522-
523-
pub fn write_improved(&mut self) -> io::IoResult<()> {
524-
self.write_pretty("improved", term::color::GREEN)
525-
}
526-
527-
pub fn write_removed(&mut self) -> io::IoResult<()> {
528-
self.write_pretty("removed", term::color::YELLOW)
529-
}
530-
531-
pub fn write_regressed(&mut self) -> io::IoResult<()> {
532-
self.write_pretty("regressed", term::color::RED)
533-
}
534-
535504
pub fn write_pretty(&mut self,
536505
word: &str,
537506
color: term::color::Color) -> io::IoResult<()> {
@@ -655,55 +624,6 @@ impl<T: Writer> ConsoleTestState<T> {
655624
Ok(())
656625
}
657626

658-
pub fn write_metric_diff(&mut self, diff: &MetricDiff) -> io::IoResult<()> {
659-
let mut noise = 0u;
660-
let mut improved = 0u;
661-
let mut regressed = 0u;
662-
let mut added = 0u;
663-
let mut removed = 0u;
664-
665-
for (k, v) in diff.iter() {
666-
match *v {
667-
LikelyNoise => noise += 1,
668-
MetricAdded => {
669-
added += 1;
670-
try!(self.write_added());
671-
try!(self.write_plain(format!(": {}\n", *k).as_slice()));
672-
}
673-
MetricRemoved => {
674-
removed += 1;
675-
try!(self.write_removed());
676-
try!(self.write_plain(format!(": {}\n", *k).as_slice()));
677-
}
678-
Improvement(pct) => {
679-
improved += 1;
680-
try!(self.write_plain(format!(": {} ", *k).as_slice()));
681-
try!(self.write_improved());
682-
try!(self.write_plain(format!(" by {:.2}%\n",
683-
pct as f64).as_slice()));
684-
}
685-
Regression(pct) => {
686-
regressed += 1;
687-
try!(self.write_plain(format!(": {} ", *k).as_slice()));
688-
try!(self.write_regressed());
689-
try!(self.write_plain(format!(" by {:.2}%\n",
690-
pct as f64).as_slice()));
691-
}
692-
}
693-
}
694-
try!(self.write_plain(format!("result of ratchet: {} metrics added, \
695-
{} removed, {} improved, {} regressed, \
696-
{} noise\n",
697-
added, removed, improved, regressed,
698-
noise).as_slice()));
699-
if regressed == 0 {
700-
try!(self.write_plain("updated ratchet file\n"));
701-
} else {
702-
try!(self.write_plain("left ratchet file untouched\n"));
703-
}
704-
Ok(())
705-
}
706-
707627
pub fn write_run_finish(&mut self,
708628
ratchet_metrics: &Option<Path>,
709629
ratchet_pct: Option<f64>) -> io::IoResult<bool> {
@@ -721,9 +641,7 @@ impl<T: Writer> ConsoleTestState<T> {
721641
forced to: {}%\n",
722642
pct).as_slice()))
723643
}
724-
let (diff, ok) = self.metrics.ratchet(pth, ratchet_pct);
725-
try!(self.write_metric_diff(&diff));
726-
ok
644+
true
727645
}
728646
};
729647

@@ -1116,63 +1034,6 @@ impl MetricMap {
11161034
write!(&mut file, "{}", json::as_json(map))
11171035
}
11181036

1119-
/// Compare against another MetricMap. Optionally compare all
1120-
/// measurements in the maps using the provided `noise_pct` as a
1121-
/// percentage of each value to consider noise. If `None`, each
1122-
/// measurement's noise threshold is independently chosen as the
1123-
/// maximum of that measurement's recorded noise quantity in either
1124-
/// map.
1125-
pub fn compare_to_old(&self, old: &MetricMap,
1126-
noise_pct: Option<f64>) -> MetricDiff {
1127-
let mut diff : MetricDiff = BTreeMap::new();
1128-
let MetricMap(ref selfmap) = *self;
1129-
let MetricMap(ref old) = *old;
1130-
for (k, vold) in old.iter() {
1131-
let r = match selfmap.get(k) {
1132-
None => MetricRemoved,
1133-
Some(v) => {
1134-
let delta = v.value - vold.value;
1135-
let noise = match noise_pct {
1136-
None => vold.noise.abs().max(v.noise.abs()),
1137-
Some(pct) => vold.value * pct / 100.0
1138-
};
1139-
if delta.abs() <= noise {
1140-
LikelyNoise
1141-
} else {
1142-
let pct = delta.abs() / vold.value.max(f64::EPSILON) * 100.0;
1143-
if vold.noise < 0.0 {
1144-
// When 'noise' is negative, it means we want
1145-
// to see deltas that go up over time, and can
1146-
// only tolerate slight negative movement.
1147-
if delta < 0.0 {
1148-
Regression(pct)
1149-
} else {
1150-
Improvement(pct)
1151-
}
1152-
} else {
1153-
// When 'noise' is positive, it means we want
1154-
// to see deltas that go down over time, and
1155-
// can only tolerate slight positive movements.
1156-
if delta < 0.0 {
1157-
Improvement(pct)
1158-
} else {
1159-
Regression(pct)
1160-
}
1161-
}
1162-
}
1163-
}
1164-
};
1165-
diff.insert((*k).clone(), r);
1166-
}
1167-
let MetricMap(ref map) = *self;
1168-
for (k, _) in map.iter() {
1169-
if !diff.contains_key(k) {
1170-
diff.insert((*k).clone(), MetricAdded);
1171-
}
1172-
}
1173-
diff
1174-
}
1175-
11761037
/// Insert a named `value` (+/- `noise`) metric into the map. The value
11771038
/// must be non-negative. The `noise` indicates the uncertainty of the
11781039
/// metric, which doubles as the "noise range" of acceptable
@@ -1194,33 +1055,6 @@ impl MetricMap {
11941055
let MetricMap(ref mut map) = *self;
11951056
map.insert(name.to_string(), m);
11961057
}
1197-
1198-
/// Attempt to "ratchet" an external metric file. This involves loading
1199-
/// metrics from a metric file (if it exists), comparing against
1200-
/// the metrics in `self` using `compare_to_old`, and rewriting the
1201-
/// file to contain the metrics in `self` if none of the
1202-
/// `MetricChange`s are `Regression`. Returns the diff as well
1203-
/// as a boolean indicating whether the ratchet succeeded.
1204-
pub fn ratchet(&self, p: &Path, pct: Option<f64>) -> (MetricDiff, bool) {
1205-
let old = if p.exists() {
1206-
MetricMap::load(p)
1207-
} else {
1208-
MetricMap::new()
1209-
};
1210-
1211-
let diff : MetricDiff = self.compare_to_old(&old, pct);
1212-
let ok = diff.iter().all(|(_, v)| {
1213-
match *v {
1214-
Regression(_) => false,
1215-
_ => true
1216-
}
1217-
});
1218-
1219-
if ok {
1220-
self.save(p).unwrap();
1221-
}
1222-
return (diff, ok)
1223-
}
12241058
}
12251059

12261060

@@ -1362,8 +1196,7 @@ pub mod bench {
13621196
mod tests {
13631197
use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts,
13641198
TestDesc, TestDescAndFn, TestOpts, run_test,
1365-
Metric, MetricMap, MetricAdded, MetricRemoved,
1366-
Improvement, Regression, LikelyNoise,
1199+
Metric, MetricMap,
13671200
StaticTestName, DynTestName, DynTestFn, ShouldFail};
13681201
use std::io::TempDir;
13691202
use std::thunk::Thunk;
@@ -1629,81 +1462,5 @@ mod tests {
16291462

16301463
m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
16311464
m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
1632-
1633-
let diff1 = m2.compare_to_old(&m1, None);
1634-
1635-
assert_eq!(*(diff1.get(&"in-both-noise".to_string()).unwrap()), LikelyNoise);
1636-
assert_eq!(*(diff1.get(&"in-first-noise".to_string()).unwrap()), MetricRemoved);
1637-
assert_eq!(*(diff1.get(&"in-second-noise".to_string()).unwrap()), MetricAdded);
1638-
assert_eq!(*(diff1.get(&"in-both-want-downwards-but-regressed".to_string()).unwrap()),
1639-
Regression(100.0));
1640-
assert_eq!(*(diff1.get(&"in-both-want-downwards-and-improved".to_string()).unwrap()),
1641-
Improvement(50.0));
1642-
assert_eq!(*(diff1.get(&"in-both-want-upwards-but-regressed".to_string()).unwrap()),
1643-
Regression(50.0));
1644-
assert_eq!(*(diff1.get(&"in-both-want-upwards-and-improved".to_string()).unwrap()),
1645-
Improvement(100.0));
1646-
assert_eq!(diff1.len(), 7);
1647-
1648-
let diff2 = m2.compare_to_old(&m1, Some(200.0));
1649-
1650-
assert_eq!(*(diff2.get(&"in-both-noise".to_string()).unwrap()), LikelyNoise);
1651-
assert_eq!(*(diff2.get(&"in-first-noise".to_string()).unwrap()), MetricRemoved);
1652-
assert_eq!(*(diff2.get(&"in-second-noise".to_string()).unwrap()), MetricAdded);
1653-
assert_eq!(*(diff2.get(&"in-both-want-downwards-but-regressed".to_string()).unwrap()),
1654-
LikelyNoise);
1655-
assert_eq!(*(diff2.get(&"in-both-want-downwards-and-improved".to_string()).unwrap()),
1656-
LikelyNoise);
1657-
assert_eq!(*(diff2.get(&"in-both-want-upwards-but-regressed".to_string()).unwrap()),
1658-
LikelyNoise);
1659-
assert_eq!(*(diff2.get(&"in-both-want-upwards-and-improved".to_string()).unwrap()),
1660-
LikelyNoise);
1661-
assert_eq!(diff2.len(), 7);
1662-
}
1663-
1664-
#[test]
1665-
pub fn ratchet_test() {
1666-
1667-
let dpth = TempDir::new("test-ratchet").ok().expect("missing test for ratchet");
1668-
let pth = dpth.path().join("ratchet.json");
1669-
1670-
let mut m1 = MetricMap::new();
1671-
m1.insert_metric("runtime", 1000.0, 2.0);
1672-
m1.insert_metric("throughput", 50.0, 2.0);
1673-
1674-
let mut m2 = MetricMap::new();
1675-
m2.insert_metric("runtime", 1100.0, 2.0);
1676-
m2.insert_metric("throughput", 50.0, 2.0);
1677-
1678-
m1.save(&pth).unwrap();
1679-
1680-
// Ask for a ratchet that should fail to advance.
1681-
let (diff1, ok1) = m2.ratchet(&pth, None);
1682-
assert_eq!(ok1, false);
1683-
assert_eq!(diff1.len(), 2);
1684-
assert_eq!(*(diff1.get(&"runtime".to_string()).unwrap()), Regression(10.0));
1685-
assert_eq!(*(diff1.get(&"throughput".to_string()).unwrap()), LikelyNoise);
1686-
1687-
// Check that it was not rewritten.
1688-
let m3 = MetricMap::load(&pth);
1689-
let MetricMap(m3) = m3;
1690-
assert_eq!(m3.len(), 2);
1691-
assert_eq!(*(m3.get(&"runtime".to_string()).unwrap()), Metric::new(1000.0, 2.0));
1692-
assert_eq!(*(m3.get(&"throughput".to_string()).unwrap()), Metric::new(50.0, 2.0));
1693-
1694-
// Ask for a ratchet with an explicit noise-percentage override,
1695-
// that should advance.
1696-
let (diff2, ok2) = m2.ratchet(&pth, Some(10.0));
1697-
assert_eq!(ok2, true);
1698-
assert_eq!(diff2.len(), 2);
1699-
assert_eq!(*(diff2.get(&"runtime".to_string()).unwrap()), LikelyNoise);
1700-
assert_eq!(*(diff2.get(&"throughput".to_string()).unwrap()), LikelyNoise);
1701-
1702-
// Check that it was rewritten.
1703-
let m4 = MetricMap::load(&pth);
1704-
let MetricMap(m4) = m4;
1705-
assert_eq!(m4.len(), 2);
1706-
assert_eq!(*(m4.get(&"runtime".to_string()).unwrap()), Metric::new(1100.0, 2.0));
1707-
assert_eq!(*(m4.get(&"throughput".to_string()).unwrap()), Metric::new(50.0, 2.0));
17081465
}
17091466
}

0 commit comments

Comments
 (0)