Skip to content

Commit 369d296

Browse files
robertroeserNiteshKant
robertroeser
authored and
NiteshKant
committed
added sliding window histogram (#200)
Added sliding window histogram Problem Current histogram doesn't resets on every get on every dimension. This creates choppy metrics. Modifications Modified the timer to have n histrograms in a queue and rotate of over them to smooth out the histogram over a sliding window. Switched the dimensions of the histrogram to a label called value. Switched the reseting of the histrogram to a closure that is called by the timer instead so it will rotate the histrogram once per window. Result Correct metrics over a rolling window.
1 parent b39d30b commit 369d296

File tree

6 files changed

+198
-45
lines changed

6 files changed

+198
-45
lines changed

‎reactivesocket-stats-servo/src/main/java/io/reactivesocket/loadbalancer/servo/internal/HdrHistogramGauge.java

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,37 +21,27 @@
2121
import com.netflix.servo.monitor.NumberGauge;
2222
import org.HdrHistogram.Histogram;
2323

24-
import java.util.concurrent.TimeUnit;
25-
2624
/**
2725
* Gauge that wraps a {@link Histogram} and when it's polled returns a particular percentage
2826
*/
2927
public class HdrHistogramGauge extends NumberGauge {
30-
private static final long TIMEOUT = TimeUnit.MINUTES.toMillis(1);
31-
private final Histogram histogram;
28+
private final SlidingWindowHistogram histogram;
3229
private final double percentile;
33-
private volatile long lastCleared = System.currentTimeMillis();
30+
private final Runnable slide;
3431

35-
public HdrHistogramGauge(MonitorConfig monitorConfig, Histogram histogram, double percentile) {
32+
public HdrHistogramGauge(MonitorConfig monitorConfig, SlidingWindowHistogram histogram, double percentile, Runnable slide) {
3633
super(monitorConfig);
3734
this.histogram = histogram;
3835
this.percentile = percentile;
36+
this.slide = slide;
3937

4038
DefaultMonitorRegistry.getInstance().register(this);
4139
}
4240

4341
@Override
4442
public Long getValue() {
45-
long value = histogram.getValueAtPercentile(percentile);
46-
if (System.currentTimeMillis() - lastCleared > TIMEOUT) {
47-
synchronized (histogram) {
48-
if (System.currentTimeMillis() - lastCleared > TIMEOUT) {
49-
histogram.reset();
50-
lastCleared = System.currentTimeMillis();
51-
}
52-
}
53-
}
54-
43+
long value = histogram.aggregateHistogram().getValueAtPercentile(percentile);
44+
slide.run();
5545
return value;
5646
}
5747
}

‎reactivesocket-stats-servo/src/main/java/io/reactivesocket/loadbalancer/servo/internal/HdrHistogramMaxGauge.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@
2525
* Gauge that wraps a {@link Histogram} and when its polled returns it's max
2626
*/
2727
public class HdrHistogramMaxGauge extends NumberGauge {
28-
private final Histogram histogram;
28+
private final SlidingWindowHistogram histogram;
2929

30-
public HdrHistogramMaxGauge(MonitorConfig monitorConfig, Histogram histogram) {
30+
public HdrHistogramMaxGauge(MonitorConfig monitorConfig, SlidingWindowHistogram histogram) {
3131
super(monitorConfig);
3232
this.histogram = histogram;
3333

@@ -36,6 +36,6 @@ public HdrHistogramMaxGauge(MonitorConfig monitorConfig, Histogram histogram) {
3636

3737
@Override
3838
public Long getValue() {
39-
return histogram.getMaxValue();
39+
return histogram.aggregateHistogram().getMaxValue();
4040
}
4141
}

‎reactivesocket-stats-servo/src/main/java/io/reactivesocket/loadbalancer/servo/internal/HdrHistogramMinGauge.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@
2525
* Gauge that wraps a {@link Histogram} and when its polled returns it's min
2626
*/
2727
public class HdrHistogramMinGauge extends NumberGauge {
28-
private final Histogram histogram;
28+
private final SlidingWindowHistogram histogram;
2929

30-
public HdrHistogramMinGauge(MonitorConfig monitorConfig, Histogram histogram) {
30+
public HdrHistogramMinGauge(MonitorConfig monitorConfig, SlidingWindowHistogram histogram) {
3131
super(monitorConfig);
3232
this.histogram = histogram;
3333

@@ -36,6 +36,6 @@ public HdrHistogramMinGauge(MonitorConfig monitorConfig, Histogram histogram) {
3636

3737
@Override
3838
public Long getValue() {
39-
return histogram.getMinValue();
39+
return histogram.aggregateHistogram().getMinValue();
4040
}
4141
}

‎reactivesocket-stats-servo/src/main/java/io/reactivesocket/loadbalancer/servo/internal/HdrHistogramServoTimer.java

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
import com.netflix.servo.monitor.MonitorConfig;
1919
import com.netflix.servo.tag.Tag;
20-
import org.HdrHistogram.ConcurrentHistogram;
21-
import org.HdrHistogram.Histogram;
2220

2321
import java.util.Arrays;
2422
import java.util.List;
@@ -29,7 +27,11 @@
2927
* The buckets are min, max, 50%, 90%, 99%, 99.9%, and 99.99%
3028
*/
3129
public class HdrHistogramServoTimer {
32-
private final Histogram histogram = new ConcurrentHistogram(TimeUnit.MINUTES.toNanos(1), 2);
30+
private final SlidingWindowHistogram histogram = new SlidingWindowHistogram();
31+
32+
private static final long TIMEOUT = TimeUnit.MINUTES.toMillis(1);
33+
34+
private volatile long lastCleared = System.currentTimeMillis();
3335

3436
private HdrHistogramMinGauge min;
3537

@@ -46,31 +48,26 @@ public class HdrHistogramServoTimer {
4648
private HdrHistogramGauge p99_99;
4749

4850
private HdrHistogramServoTimer(String label) {
49-
histogram.setAutoResize(true);
5051

51-
min = new HdrHistogramMinGauge(MonitorConfig.builder(label + "_min").build(), histogram);
52-
max = new HdrHistogramMaxGauge(MonitorConfig.builder(label + "_max").build(), histogram);
52+
min = new HdrHistogramMinGauge(MonitorConfig.builder(label).withTag("value", "min").build(), histogram);
53+
max = new HdrHistogramMaxGauge(MonitorConfig.builder(label).withTag("value", "max").build(), histogram);
5354

54-
p50 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p50").build(), histogram, 50);
55-
p90 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p90").build(), histogram, 90);
56-
p99 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99").build(), histogram, 99);
57-
p99_9 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99_9").build(), histogram, 99.9);
58-
p99_99 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99_99").build(), histogram, 99.99);
55+
p50 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p50").build(), histogram, 50, this::slide);
56+
p90 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p90").build(), histogram, 90, this::slide);
57+
p99 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p99").build(), histogram, 99, this::slide);
58+
p99_9 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p99_9").build(), histogram, 99.9, this::slide);
59+
p99_99 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p99_99").build(), histogram, 99.99, this::slide);
5960
}
6061

61-
6262
private HdrHistogramServoTimer(String label, List<Tag> tags) {
63-
histogram.setAutoResize(true);
64-
65-
66-
min = new HdrHistogramMinGauge(MonitorConfig.builder(label + "_min").withTags(tags).build(), histogram);
67-
max = new HdrHistogramMaxGauge(MonitorConfig.builder(label + "_max").withTags(tags).build(), histogram);
68-
69-
p50 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p50").withTags(tags).build(), histogram, 50);
70-
p90 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p90").withTags(tags).build(), histogram, 90);
71-
p99 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99").withTags(tags).build(), histogram, 99);
72-
p99_9 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99_9").withTags(tags).build(), histogram, 99.9);
73-
p99_99 = new HdrHistogramGauge(MonitorConfig.builder(label + "_p99_99").withTags(tags).build(), histogram, 99.99);
63+
min = new HdrHistogramMinGauge(MonitorConfig.builder(label).withTag("value", "min").withTags(tags).build(), histogram);
64+
max = new HdrHistogramMaxGauge(MonitorConfig.builder(label).withTag("value", "min").withTags(tags).build(), histogram);
65+
66+
p50 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p50").withTags(tags).build(), histogram, 50, this::slide);
67+
p90 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p90").withTags(tags).build(), histogram, 90, this::slide);
68+
p99 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p90").withTags(tags).build(), histogram, 99, this::slide);
69+
p99_9 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p99_9").withTags(tags).build(), histogram, 99.9, this::slide);
70+
p99_99 = new HdrHistogramGauge(MonitorConfig.builder(label).withTag("value", "p99_99").withTags(tags).build(), histogram, 99.99, this::slide);
7471
}
7572

7673
public static HdrHistogramServoTimer newInstance(String label) {
@@ -87,6 +84,7 @@ public static HdrHistogramServoTimer newInstance(String label, List<Tag> tags) {
8784

8885
/**
8986
* Records a value for to the histogram and updates the Servo counter buckets
87+
*
9088
* @param value the value to update
9189
*/
9290
public void record(long value) {
@@ -120,4 +118,12 @@ public Long getP99_9() {
120118
public Long getP99_99() {
121119
return p99_99.getValue();
122120
}
121+
122+
private synchronized void slide() {
123+
if (System.currentTimeMillis() - lastCleared > TIMEOUT) {
124+
histogram.rotateHistogram();
125+
lastCleared = System.currentTimeMillis();
126+
}
127+
}
128+
123129
}

‎reactivesocket-stats-servo/src/main/java/io/reactivesocket/loadbalancer/servo/internal/SlidingWindowHistogram.java

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package io.reactivesocket.loadbalancer.servo.internal;
2+
3+
import org.HdrHistogram.ConcurrentHistogram;
4+
import org.HdrHistogram.Histogram;
5+
6+
import java.io.PrintStream;
7+
import java.util.ArrayDeque;
8+
import java.util.concurrent.TimeUnit;
9+
10+
/**
11+
* Wraps HdrHistogram to create a sliding window of n histogramQueue. Default window number is five.
12+
*/
13+
public class SlidingWindowHistogram {
14+
private volatile Histogram liveHistogram;
15+
16+
private final ArrayDeque<Histogram> histogramQueue;
17+
18+
private final Object LOCK = new Object();
19+
20+
public SlidingWindowHistogram() {
21+
this(5);
22+
}
23+
24+
public SlidingWindowHistogram(final int numOfWindows) {
25+
if (numOfWindows < 2) {
26+
throw new IllegalArgumentException("number of windows must be greater than 1");
27+
}
28+
this.histogramQueue = new ArrayDeque<>(numOfWindows - 1);
29+
this.liveHistogram = createHistogram();
30+
31+
for (int i = 0; i < numOfWindows - 1; i++) {
32+
histogramQueue.offer(createHistogram());
33+
}
34+
}
35+
36+
private static Histogram createHistogram() {
37+
ConcurrentHistogram histogram = new ConcurrentHistogram(TimeUnit.MINUTES.toNanos(1), 2);
38+
histogram.setAutoResize(true);
39+
return histogram;
40+
}
41+
42+
/**
43+
* Records a value to the in window liveHistogram
44+
*
45+
* @param value value to record
46+
*/
47+
public void recordValue(long value) {
48+
liveHistogram.recordValue(value);
49+
}
50+
51+
/**
52+
* Slides the Histogram window. Pops a Histogram off a queue, resets it, and places the old
53+
* on in the queue.
54+
*/
55+
public void rotateHistogram() {
56+
synchronized (LOCK) {
57+
Histogram onDeck = histogramQueue.poll();
58+
if (onDeck != null) {
59+
onDeck.reset();
60+
Histogram old = liveHistogram;
61+
liveHistogram = onDeck;
62+
histogramQueue.offer(old);
63+
}
64+
}
65+
}
66+
67+
/**
68+
* Aggregates the Histograms into a single Histogram and returns it.
69+
*
70+
* @return Aggregated liveHistogram
71+
*/
72+
public Histogram aggregateHistogram() {
73+
Histogram aggregate = createHistogram();
74+
75+
synchronized (LOCK) {
76+
aggregate.add(liveHistogram);
77+
histogramQueue
78+
.forEach(aggregate::add);
79+
}
80+
81+
return aggregate;
82+
}
83+
84+
/**
85+
* Prints HdrHistogram to System.out
86+
*/
87+
public void print() {
88+
print(System.out);
89+
}
90+
91+
public void print(PrintStream printStream) {
92+
aggregateHistogram().outputPercentileDistribution(printStream, 1000.0);
93+
}
94+
95+
}

‎reactivesocket-stats-servo/src/test/java/io/reactivesocket/loadbalancer/servo/internal/SlidingWindowHistogramTest.java

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package io.reactivesocket.loadbalancer.servo.internal;
2+
3+
import org.HdrHistogram.Histogram;
4+
import org.junit.Assert;
5+
import org.junit.Test;
6+
7+
public class SlidingWindowHistogramTest {
8+
@Test
9+
public void test() {
10+
SlidingWindowHistogram slidingWindowHistogram = new SlidingWindowHistogram(2);
11+
12+
for (int i = 0; i < 100_000; i++) {
13+
slidingWindowHistogram.recordValue(i);
14+
}
15+
16+
slidingWindowHistogram.print();
17+
slidingWindowHistogram.rotateHistogram();
18+
Histogram histogram =
19+
slidingWindowHistogram.aggregateHistogram();
20+
21+
long totalCount = histogram.getTotalCount();
22+
Assert.assertTrue(totalCount == 100_000);
23+
24+
long p90 = histogram.getValueAtPercentile(90);
25+
Assert.assertTrue(p90 < 100_000);
26+
27+
for (int i = 0; i < 100_000; i++) {
28+
slidingWindowHistogram.recordValue(i * 10_000);
29+
}
30+
31+
slidingWindowHistogram.print();
32+
slidingWindowHistogram.rotateHistogram();
33+
34+
histogram =
35+
slidingWindowHistogram.aggregateHistogram();
36+
37+
p90 = histogram.getValueAtPercentile(90);
38+
Assert.assertTrue(p90 >= 100_000);
39+
40+
for (int i = 0; i < 100_000; i++) {
41+
slidingWindowHistogram.recordValue(i);
42+
}
43+
44+
slidingWindowHistogram.print();
45+
slidingWindowHistogram.rotateHistogram();
46+
47+
for (int i = 0; i < 100_000; i++) {
48+
slidingWindowHistogram.recordValue(i);
49+
}
50+
51+
slidingWindowHistogram.print();
52+
53+
histogram =
54+
slidingWindowHistogram.aggregateHistogram();
55+
56+
totalCount = histogram.getTotalCount();
57+
Assert.assertTrue(totalCount == 200_000);
58+
59+
p90 = histogram.getValueAtPercentile(90);
60+
Assert.assertTrue(p90 < 100_000);
61+
}
62+
}

0 commit comments

Comments
 (0)