Skip to content

Commit 37f6024

Browse files
committed
Improve performance of node reading
This replaces the runtime check on record size and unrolls the node calculation. Before: ``` goos: linux goarch: amd64 pkg: github.com/oschwald/maxminddb-golang BenchmarkLookup-8 1000000 15928 ns/op BenchmarkLookupNetwork-8 1000000 16638 ns/op BenchmarkCountryCode-8 10000000 1244 ns/op PASS ok github.com/oschwald/maxminddb-golang 46.654s ``` After: ``` goos: linux goarch: amd64 pkg: github.com/oschwald/maxminddb-golang BenchmarkLookup-8 1000000 14918 ns/op BenchmarkLookupNetwork-8 1000000 15681 ns/op BenchmarkCountryCode-8 20000000 1137 ns/op PASS ok github.com/oschwald/maxminddb-golang 54.967s ```
1 parent 27395cf commit 37f6024

File tree

3 files changed

+75
-54
lines changed

3 files changed

+75
-54
lines changed

node.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package maxminddb
2+
3+
type nodeReader interface {
4+
readLeft(uint) uint
5+
readRight(uint) uint
6+
}
7+
8+
type nodeReader24 struct {
9+
buffer []byte
10+
}
11+
12+
func (n nodeReader24) readLeft(nodeNumber uint) uint {
13+
return (uint(n.buffer[nodeNumber]) << 16) | (uint(n.buffer[nodeNumber+1]) << 8) | uint(n.buffer[nodeNumber+2])
14+
}
15+
16+
func (n nodeReader24) readRight(nodeNumber uint) uint {
17+
return (uint(n.buffer[nodeNumber+3]) << 16) | (uint(n.buffer[nodeNumber+4]) << 8) | uint(n.buffer[nodeNumber+5])
18+
}
19+
20+
type nodeReader28 struct {
21+
buffer []byte
22+
}
23+
24+
func (n nodeReader28) readLeft(nodeNumber uint) uint {
25+
return ((uint(n.buffer[nodeNumber+3]) & 0xF0) << 20) | (uint(n.buffer[nodeNumber]) << 16) | (uint(n.buffer[nodeNumber+1]) << 8) | uint(n.buffer[nodeNumber+2])
26+
}
27+
28+
func (n nodeReader28) readRight(nodeNumber uint) uint {
29+
return ((uint(n.buffer[nodeNumber+3]) & 0x0F) << 24) | (uint(n.buffer[nodeNumber+4]) << 16) | (uint(n.buffer[nodeNumber+5]) << 8) | uint(n.buffer[nodeNumber+6])
30+
}
31+
32+
type nodeReader32 struct {
33+
buffer []byte
34+
}
35+
36+
func (n nodeReader32) readLeft(nodeNumber uint) uint {
37+
return (uint(n.buffer[nodeNumber]) << 24) | (uint(n.buffer[nodeNumber+1]) << 16) | (uint(n.buffer[nodeNumber+2]) << 8) | uint(n.buffer[nodeNumber+3])
38+
}
39+
40+
func (n nodeReader32) readRight(nodeNumber uint) uint {
41+
return (uint(n.buffer[nodeNumber+4]) << 24) | (uint(n.buffer[nodeNumber+5]) << 16) | (uint(n.buffer[nodeNumber+6]) << 8) | uint(n.buffer[nodeNumber+7])
42+
}

reader.go

Lines changed: 30 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ var metadataStartMarker = []byte("\xAB\xCD\xEFMaxMind.com")
2323
type Reader struct {
2424
hasMappedFile bool
2525
buffer []byte
26+
nodeReader nodeReader
2627
decoder decoder
2728
Metadata Metadata
2829
ipv4Start uint
@@ -75,37 +76,46 @@ func FromBytes(buffer []byte) (*Reader, error) {
7576
buffer[searchTreeSize+dataSectionSeparatorSize : metadataStart-len(metadataStartMarker)],
7677
}
7778

79+
nodeBuffer := buffer[:searchTreeSize]
80+
var nodeReader nodeReader
81+
switch metadata.RecordSize {
82+
case 24:
83+
nodeReader = nodeReader24{buffer: nodeBuffer}
84+
case 28:
85+
nodeReader = nodeReader28{buffer: nodeBuffer}
86+
case 32:
87+
nodeReader = nodeReader32{buffer: nodeBuffer}
88+
default:
89+
return nil, newInvalidDatabaseError("unknown record size: %d", metadata.RecordSize)
90+
}
91+
7892
reader := &Reader{
79-
buffer: buffer,
80-
decoder: d,
81-
Metadata: metadata,
82-
ipv4Start: 0,
93+
buffer: buffer,
94+
nodeReader: nodeReader,
95+
decoder: d,
96+
Metadata: metadata,
97+
ipv4Start: 0,
8398
}
8499

85-
err = reader.setIPv4Start()
100+
reader.setIPv4Start()
86101

87102
return reader, err
88103
}
89104

90-
func (r *Reader) setIPv4Start() error {
105+
func (r *Reader) setIPv4Start() {
91106
if r.Metadata.IPVersion != 6 {
92-
return nil
107+
return
93108
}
94109

95110
nodeCount := r.Metadata.NodeCount
96111

97112
node := uint(0)
98-
var err error
99113
i := 0
100114
for ; i < 96 && node < nodeCount; i++ {
101-
node, err = r.readNode(node, 0)
102-
if err != nil {
103-
return err
104-
}
115+
node = r.nodeReader.readLeft(r.nodeOffset(node))
105116
}
106117
r.ipv4Start = node
107118
r.ipv4StartBitDepth = i
108-
return err
109119
}
110120

111121
// Lookup retrieves the database record for ip and stores it in the value
@@ -241,10 +251,11 @@ func (r *Reader) lookupPointer(ip net.IP) (uint, int, net.IP, error) {
241251
for ; i < bitCount && node < nodeCount; i++ {
242252
bit := uint(1) & (uint(ip[i>>3]) >> (7 - (i % 8)))
243253

244-
var err error
245-
node, err = r.readNode(node, bit)
246-
if err != nil {
247-
return 0, int(i), ip, err
254+
offset := r.nodeOffset(node)
255+
if bit == 0 {
256+
node = r.nodeReader.readLeft(offset)
257+
} else {
258+
node = r.nodeReader.readRight(offset)
248259
}
249260
}
250261
if node == nodeCount {
@@ -257,33 +268,8 @@ func (r *Reader) lookupPointer(ip net.IP) (uint, int, net.IP, error) {
257268
return 0, int(i), ip, newInvalidDatabaseError("invalid node in search tree")
258269
}
259270

260-
func (r *Reader) readNode(nodeNumber uint, index uint) (uint, error) {
261-
RecordSize := r.Metadata.RecordSize
262-
263-
baseOffset := nodeNumber * RecordSize / 4
264-
265-
var nodeBytes []byte
266-
var prefix uint
267-
switch RecordSize {
268-
case 24:
269-
offset := baseOffset + index*3
270-
nodeBytes = r.buffer[offset : offset+3]
271-
case 28:
272-
prefix = uint(r.buffer[baseOffset+3])
273-
if index != 0 {
274-
prefix &= 0x0F
275-
} else {
276-
prefix = (0xF0 & prefix) >> 4
277-
}
278-
offset := baseOffset + index*4
279-
nodeBytes = r.buffer[offset : offset+3]
280-
case 32:
281-
offset := baseOffset + index*4
282-
nodeBytes = r.buffer[offset : offset+4]
283-
default:
284-
return 0, newInvalidDatabaseError("unknown record size: %d", RecordSize)
285-
}
286-
return uintFromBytes(prefix, nodeBytes), nil
271+
func (r *Reader) nodeOffset(node uint) uint {
272+
return node * r.Metadata.RecordSize / 4
287273
}
288274

289275
func (r *Reader) retrieveData(pointer uint, result interface{}) error {

traverse.go

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,8 @@ func (n *Networks) Next() bool {
6060
}
6161
ipRight[node.bit>>3] |= 1 << (7 - (node.bit % 8))
6262

63-
rightPointer, err := n.reader.readNode(node.pointer, 1)
64-
if err != nil {
65-
n.err = err
66-
return false
67-
}
63+
offset := n.reader.nodeOffset(node.pointer)
64+
rightPointer := n.reader.nodeReader.readRight(offset)
6865

6966
node.bit++
7067
n.nodes = append(n.nodes, netNode{
@@ -73,11 +70,7 @@ func (n *Networks) Next() bool {
7370
bit: node.bit,
7471
})
7572

76-
node.pointer, err = n.reader.readNode(node.pointer, 0)
77-
if err != nil {
78-
n.err = err
79-
return false
80-
}
73+
node.pointer = n.reader.nodeReader.readLeft(offset)
8174
}
8275
}
8376

0 commit comments

Comments
 (0)