Skip to content

Commit 1ef1341

Browse files
schmidt-sebastianschmidt-sebastian
authored
Add double to bit encoding for OrderedCode (#5816)
1 parent 3b481f5 commit 1ef1341

File tree

2 files changed

+311
-15
lines changed

2 files changed

+311
-15
lines changed

‎packages/firestore/src/index/ordered_code_writer.ts

Lines changed: 153 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,51 +14,189 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
import { fail } from '../util/assert';
17+
import { debugAssert, fail } from '../util/assert';
1818
import { ByteString } from '../util/byte_string';
1919

20+
const LONG_SIZE = 64;
21+
const BYTE_SIZE = 8;
22+
23+
/**
24+
* The default size of the buffer. This is arbitrary, but likely larger than
25+
* most index values so that less copies of the underlying buffer will be made.
26+
* For large values, a single copy will made to double the buffer length.
27+
*/
28+
const DEFAULT_BUFFER_SIZE = 1024;
29+
30+
/** Converts a JavaScript number to a byte array (using big endian encoding). */
31+
function doubleToLongBits(value: number): Uint8Array {
32+
const dv = new DataView(new ArrayBuffer(8));
33+
dv.setFloat64(0, value, /* littleEndian= */ false);
34+
return new Uint8Array(dv.buffer);
35+
}
36+
37+
/**
38+
* Counts the number of zeros in a byte.
39+
*
40+
* Visible for testing.
41+
*/
42+
export function numberOfLeadingZerosInByte(x: number): number {
43+
debugAssert(x < 256, 'Provided value is not a byte: ' + x);
44+
if (x === 0) {
45+
return 8;
46+
}
47+
48+
let zeros = 0;
49+
if (x >> 4 === 0) {
50+
// Test if the first four bits are zero.
51+
zeros += 4;
52+
x = x << 4;
53+
}
54+
if (x >> 6 === 0) {
55+
// Test if the first two (or next two) bits are zero.
56+
zeros += 2;
57+
x = x << 2;
58+
}
59+
if (x >> 7 === 0) {
60+
// Test if the remaining bit is zero.
61+
zeros += 1;
62+
}
63+
return zeros;
64+
}
65+
66+
/** Counts the number of leading zeros in the given byte array. */
67+
function numberOfLeadingZeros(bytes: Uint8Array): number {
68+
debugAssert(
69+
bytes.length === 8,
70+
'Can only count leading zeros in 64-bit numbers'
71+
);
72+
let leadingZeros = 0;
73+
for (let i = 0; i < 8; ++i) {
74+
const zeros = numberOfLeadingZerosInByte(bytes[i] & 0xff);
75+
leadingZeros += zeros;
76+
if (zeros !== 8) {
77+
break;
78+
}
79+
}
80+
return leadingZeros;
81+
}
82+
83+
/**
84+
* Returns the number of bytes required to store "value". Leading zero bytes
85+
* are skipped.
86+
*/
87+
function unsignedNumLength(value: Uint8Array): number {
88+
// This is just the number of bytes for the unsigned representation of the number.
89+
const numBits = LONG_SIZE - numberOfLeadingZeros(value);
90+
return Math.ceil(numBits / BYTE_SIZE);
91+
}
92+
93+
/**
94+
* OrderedCodeWriter is a minimal-allocation implementation of the writing
95+
* behavior defined by the backend.
96+
*
97+
* The code is ported from its Java counterpart.
98+
*/
2099
export class OrderedCodeWriter {
21-
writeBytesAscending(value: ByteString): void {
22-
fail('Not implemented');
100+
buffer = new Uint8Array(DEFAULT_BUFFER_SIZE);
101+
position = 0;
102+
103+
writeNumberAscending(val: number): void {
104+
// Values are encoded with a single byte length prefix, followed by the
105+
// actual value in big-endian format with leading 0 bytes dropped.
106+
const value = this.toOrderedBits(val);
107+
const len = unsignedNumLength(value);
108+
this.ensureAvailable(1 + len);
109+
this.buffer[this.position++] = len & 0xff; // Write the length
110+
for (let i = value.length - len; i < value.length; ++i) {
111+
this.buffer[this.position++] = value[i] & 0xff;
112+
}
23113
}
24114

25-
writeBytesDescending(value: ByteString): void {
26-
fail('Not implemented');
115+
writeNumberDescending(val: number): void {
116+
// Values are encoded with a single byte length prefix, followed by the
117+
// inverted value in big-endian format with leading 0 bytes dropped.
118+
const value = this.toOrderedBits(val);
119+
const len = unsignedNumLength(value);
120+
this.ensureAvailable(1 + len);
121+
this.buffer[this.position++] = ~(len & 0xff); // Write the length
122+
for (let i = value.length - len; i < value.length; ++i) {
123+
this.buffer[this.position++] = ~(value[i] & 0xff);
124+
}
27125
}
28126

29-
writeUtf8Ascending(sequence: string): void {
30-
fail('Not implemented');
127+
/**
128+
* Encodes `val` into an encoding so that the order matches the IEEE 754
129+
* floating-point comparison results with the following exceptions:
130+
* -0.0 < 0.0
131+
* all non-NaN < NaN
132+
* NaN = NaN
133+
*/
134+
private toOrderedBits(val: number): Uint8Array {
135+
const value = doubleToLongBits(val);
136+
// Check if the first bit is set. We use a bit mask since value[0] is
137+
// encoded as a number from 0 to 255.
138+
const isNegative = (value[0] & 0x80) !== 0;
139+
140+
// Revert the two complement to get natural ordering
141+
value[0] ^= isNegative ? 0xff : 0x80;
142+
for (let i = 1; i < value.length; ++i) {
143+
value[i] ^= isNegative ? 0xff : 0x00;
144+
}
145+
return value;
31146
}
32147

33-
writeUtf8Descending(sequence: string): void {
34-
fail('Not implemented');
148+
/** Resets the buffer such that it is the same as when it was newly constructed. */
149+
reset(): void {
150+
this.position = 0;
35151
}
36152

37-
writeNumberAscending(val: number): void {
153+
/** Makes a copy of the encoded bytes in this buffer. */
154+
encodedBytes(): Uint8Array {
155+
return this.buffer.slice(0, this.position);
156+
}
157+
158+
writeBytesAscending(value: ByteString): void {
38159
fail('Not implemented');
39160
}
40161

41-
writeNumberDescending(val: number): void {
162+
writeBytesDescending(value: ByteString): void {
42163
fail('Not implemented');
43164
}
44165

45-
writeInfinityAscending(): void {
166+
writeUtf8Ascending(sequence: string): void {
46167
fail('Not implemented');
47168
}
48169

49-
writeInfinityDescending(): void {
170+
writeUtf8Descending(sequence: string): void {
50171
fail('Not implemented');
51172
}
52173

53-
reset(): void {
174+
writeInfinityAscending(): void {
54175
fail('Not implemented');
55176
}
56177

57-
encodedBytes(): Uint8Array {
178+
writeInfinityDescending(): void {
58179
fail('Not implemented');
59180
}
60181

61182
seed(encodedBytes: Uint8Array): void {
62183
fail('Not implemented');
63184
}
185+
186+
private ensureAvailable(bytes: number): void {
187+
const minCapacity = bytes + this.position;
188+
if (minCapacity <= this.buffer.length) {
189+
return;
190+
}
191+
// Try doubling.
192+
let newLength = this.buffer.length * 2;
193+
// Still not big enough? Just allocate the right size.
194+
if (newLength < minCapacity) {
195+
newLength = minCapacity;
196+
}
197+
// Create the new buffer.
198+
const newBuffer = new Uint8Array(newLength);
199+
newBuffer.set(this.buffer); // copy old data
200+
this.buffer = newBuffer;
201+
}
64202
}

‎packages/firestore/test/unit/index/ordered_code_writer.test.ts

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0x00 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0x00
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
import { expect } from 'chai';
18+
19+
import {
20+
numberOfLeadingZerosInByte,
21+
OrderedCodeWriter
22+
} from '../../../src/index/ordered_code_writer';
23+
24+
class ValueTestCase<T> {
25+
constructor(
26+
readonly val: T,
27+
readonly ascString: string,
28+
readonly descString: string
29+
) {}
30+
}
31+
32+
const NUMBER_TEST_CASES: Array<ValueTestCase<number>> = [
33+
new ValueTestCase(
34+
Number.NEGATIVE_INFINITY,
35+
// Note: This values are taken from the Android reference implementation
36+
'070fffffffffffff',
37+
'f8f0000000000000'
38+
),
39+
new ValueTestCase(
40+
Number.MIN_SAFE_INTEGER,
41+
'083cc0000000000000',
42+
'f7c33fffffffffffff'
43+
),
44+
new ValueTestCase(-2, '083fffffffffffffff', 'f7c000000000000000'),
45+
new ValueTestCase(-1, '08400fffffffffffff', 'f7bff0000000000000'),
46+
new ValueTestCase(-0.1, '084046666666666665', 'f7bfb999999999999a'),
47+
new ValueTestCase(-0.0, '087fffffffffffffff', 'f78000000000000000'),
48+
new ValueTestCase(0, '088000000000000000', 'f77fffffffffffffff'),
49+
new ValueTestCase(
50+
Number.MIN_VALUE,
51+
'088000000000000001',
52+
'f77ffffffffffffffe'
53+
),
54+
new ValueTestCase(0.1, '08bfb999999999999a', 'f74046666666666665'),
55+
new ValueTestCase(1, '08bff0000000000000', 'f7400fffffffffffff'),
56+
new ValueTestCase(2, '08c000000000000000', 'f73fffffffffffffff'),
57+
new ValueTestCase(4, '08c010000000000000', 'f73fefffffffffffff'),
58+
new ValueTestCase(8, '08c020000000000000', 'f73fdfffffffffffff'),
59+
new ValueTestCase(16, '08c030000000000000', 'f73fcfffffffffffff'),
60+
new ValueTestCase(32, '08c040000000000000', 'f73fbfffffffffffff'),
61+
new ValueTestCase(64, '08c050000000000000', 'f73fafffffffffffff'),
62+
new ValueTestCase(128, '08c060000000000000', 'f73f9fffffffffffff'),
63+
new ValueTestCase(255, '08c06fe00000000000', 'f73f901fffffffffff'),
64+
new ValueTestCase(256, '08c070000000000000', 'f73f8fffffffffffff'),
65+
new ValueTestCase(257, '08c070100000000000', 'f73f8fefffffffffff'),
66+
new ValueTestCase(
67+
Number.MAX_SAFE_INTEGER,
68+
'08c33fffffffffffff',
69+
'f73cc0000000000000'
70+
),
71+
new ValueTestCase(
72+
Number.POSITIVE_INFINITY,
73+
'08fff0000000000000',
74+
'f7000fffffffffffff'
75+
),
76+
new ValueTestCase(Number.NaN, '08fff8000000000000', 'f70007ffffffffffff')
77+
];
78+
79+
describe('Ordered Code Writer', () => {
80+
it('computes number of leading zeros', () => {
81+
for (let i = 0; i < 0xff; ++i) {
82+
let zeros = 0;
83+
for (let bit = 7; bit >= 0; --bit) {
84+
if ((i & (1 << bit)) === 0) {
85+
++zeros;
86+
} else {
87+
break;
88+
}
89+
}
90+
expect(numberOfLeadingZerosInByte(i)).to.equal(zeros, `for number ${i}`);
91+
}
92+
});
93+
94+
it('converts numbers to bits', () => {
95+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
96+
const bytes = getBytes(NUMBER_TEST_CASES[i].val);
97+
expect(bytes.asc).to.deep.equal(
98+
fromHex(NUMBER_TEST_CASES[i].ascString),
99+
'Ascending for ' + NUMBER_TEST_CASES[i].val
100+
);
101+
expect(bytes.desc).to.deep.equal(
102+
fromHex(NUMBER_TEST_CASES[i].descString),
103+
'Descending for ' + NUMBER_TEST_CASES[i].val
104+
);
105+
}
106+
});
107+
108+
it('orders numbers correctly', () => {
109+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
110+
for (let j = i; j < NUMBER_TEST_CASES.length; ++j) {
111+
const left = NUMBER_TEST_CASES[i].val;
112+
const leftBytes = getBytes(left);
113+
const right = NUMBER_TEST_CASES[j].val;
114+
const rightBytes = getBytes(right);
115+
expect(compare(leftBytes.asc, rightBytes.asc)).to.equal(
116+
i === j ? 0 : -1,
117+
`Ascending order: ${left} vs ${right}`
118+
);
119+
expect(compare(leftBytes.desc, rightBytes.desc)).to.equal(
120+
i === j ? 0 : 1,
121+
`Descending order: ${left} vs ${right}`
122+
);
123+
}
124+
}
125+
});
126+
});
127+
128+
function fromHex(hexString: string): Uint8Array {
129+
const bytes = new Uint8Array(hexString.length / 2);
130+
for (let c = 0; c < hexString.length; c += 2) {
131+
bytes[c / 2] = parseInt(hexString.substr(c, 2), 16);
132+
}
133+
return bytes;
134+
}
135+
136+
function compare(left: Uint8Array, right: Uint8Array): number {
137+
for (let i = 0; i < Math.min(left.length, right.length); ++i) {
138+
if (left[i] < right[i]) {
139+
return -1;
140+
}
141+
if (left[i] > right[i]) {
142+
return 1;
143+
}
144+
}
145+
return left.length - right.length;
146+
}
147+
148+
function getBytes(val: unknown): { asc: Uint8Array; desc: Uint8Array } {
149+
const ascWriter = new OrderedCodeWriter();
150+
const descWriter = new OrderedCodeWriter();
151+
if (typeof val === 'number') {
152+
ascWriter.writeNumberAscending(val);
153+
descWriter.writeNumberDescending(val);
154+
} else {
155+
throw new Error('Encoding not yet supported for ' + val);
156+
}
157+
return { asc: ascWriter.encodedBytes(), desc: descWriter.encodedBytes() };
158+
}

0 commit comments

Comments
 (0)