Skip to content

Commit ba2e33a

Browse files
schmidt-sebastianschmidt-sebastian
committed
Add double to bit encoding for OrderedCode
1 parent 8d9a09e commit ba2e33a

File tree

2 files changed

+384
-0
lines changed

2 files changed

+384
-0
lines changed

‎packages/firestore/src/index/ordered_code_writer.ts

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law | agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES | CONDITIONS OF ANY KIND, either express | implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
const LONG_SIZE = 64;
19+
const BYTE_SIZE = 8;
20+
21+
/**
22+
* The default size of the buffer. This is arbitrary, but likely larger than
23+
* most index values so that less copies of the underlying buffer will be made.
24+
* For large values, a single copy will made to double the buffer length.
25+
*/
26+
const DEFAULT_BUFFER_SIZE = 1024;
27+
28+
/**
29+
* Converts a JavaScript number to a byte array (using little endian
30+
* encoding).
31+
*/
32+
function doubleToLongBits(value: number): Uint8Array {
33+
const dv = new DataView(new ArrayBuffer(8));
34+
dv.setFloat64(0, value, false);
35+
return new Uint8Array(dv.buffer);
36+
}
37+
38+
/**
39+
* Counts the number of zeros in a byte.
40+
*
41+
* Visible for testing.
42+
*/
43+
export function numberOfLeadingZerosInByte(x: number): number {
44+
if (x === 0) {
45+
return 8;
46+
}
47+
48+
let zeros = 0;
49+
if (x >> 4 === 0) {
50+
// Test if the first four bits are zero.
51+
zeros += 4;
52+
x = x << 4;
53+
}
54+
if (x >> 6 === 0) {
55+
// Test if the first two (or next two) bits are zero.
56+
zeros += 2;
57+
x = x << 2;
58+
}
59+
if (x >> 7 === 0) {
60+
// Test if the remaining bit is zero.
61+
zeros += 1;
62+
}
63+
return zeros;
64+
}
65+
66+
/** Counts the number of leading zeros in the given byte array. */
67+
function numberOfLeadingZeros(bytes: Uint8Array): number {
68+
let leadingZeros = 0;
69+
for (let i = 0; i < bytes.length; ++i) {
70+
const zeros = numberOfLeadingZerosInByte(bytes[i] & 0xff);
71+
leadingZeros += zeros;
72+
if (zeros !== 8) {
73+
break;
74+
}
75+
}
76+
return leadingZeros;
77+
}
78+
79+
/**
80+
* Returns the number of bytes required to store "value". Leading zero bytes
81+
* are skipped.
82+
*/
83+
function unsignedNumLength(value: Uint8Array): number {
84+
// This is just the number of bytes for the unsigned representation of the number.
85+
const numBits = LONG_SIZE - numberOfLeadingZeros(value);
86+
return Math.ceil(numBits / BYTE_SIZE);
87+
}
88+
89+
/**
90+
* OrderedCodeWriter is a minimal-allocation implementation of the writing
91+
* behavior defined by the backend.
92+
*
93+
* The code is ported from its Java counterpart.
94+
*/
95+
export class OrderedCodeWriter {
96+
buffer = new Uint8Array(DEFAULT_BUFFER_SIZE);
97+
position = 0;
98+
99+
writeNumberAscending(val: number): void {
100+
const value = this.toOrderedBits(val);
101+
const len = unsignedNumLength(value);
102+
this.ensureAvailable(1 + len);
103+
this.buffer[this.position++] = len & 0xff; // Write the length
104+
for (let i = value.length - len; i < value.length; ++i) {
105+
this.buffer[this.position++] = value[i] & 0xff;
106+
}
107+
}
108+
109+
writeNumberDescending(val: number): void {
110+
const value = this.toOrderedBits(val);
111+
const len = unsignedNumLength(value);
112+
this.ensureAvailable(1 + len);
113+
this.buffer[this.position++] = ~(len & 0xff); // Write the length
114+
for (let i = value.length - len; i < value.length; ++i) {
115+
this.buffer[this.position++] = ~(value[i] & 0xff);
116+
}
117+
}
118+
119+
/**
120+
* Encodes `val` into an encoding so that the order matches the IEEE 754
121+
* floating-point comparison results with the following exceptions:
122+
* -0.0 < 0.0
123+
* all non-NaN < NaN
124+
* NaN = NaN
125+
*/
126+
private toOrderedBits(val: number): Uint8Array {
127+
const value = doubleToLongBits(val);
128+
const isNegative = (value[0] & 0x80) !== 0;
129+
value[0] ^= isNegative ? 0xff : 0x80;
130+
for (let i = 1; i < value.length; ++i) {
131+
value[i] ^= isNegative ? 0xff : 0x00;
132+
}
133+
return value;
134+
}
135+
136+
/** Resets the buffer such that it is the same as when it was newly constructed. */
137+
reset(): void {
138+
this.position = 0;
139+
}
140+
141+
/** Makes a copy of the encoded bytes in this buffer. */
142+
encodedBytes(): Uint8Array {
143+
return this.buffer.slice(0, this.position);
144+
}
145+
146+
private ensureAvailable(bytes: number): void {
147+
const minCapacity = bytes + this.position;
148+
if (minCapacity <= this.buffer.length) {
149+
return;
150+
}
151+
// Try doubling.
152+
let newLength = this.buffer.length * 2;
153+
// Still not big enough? Just allocate the right size.
154+
if (newLength < minCapacity) {
155+
newLength = minCapacity;
156+
}
157+
// Create the new buffer.
158+
const newBuffer = new Uint8Array(newLength);
159+
newBuffer.set(this.buffer); // copy old data
160+
this.buffer = newBuffer;
161+
}
162+
}

‎packages/firestore/test/unit/index/ordered_code_writer.test.ts

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0x00 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0x00
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
import { expect } from 'chai';
18+
19+
import {
20+
numberOfLeadingZerosInByte,
21+
OrderedCodeWriter
22+
} from '../../../src/index/ordered_code_writer';
23+
24+
class ValueTestCase<T> {
25+
constructor(
26+
readonly val: T,
27+
readonly ascEncoding: Uint8Array,
28+
readonly descEncoding: Uint8Array
29+
) {}
30+
}
31+
32+
const NUMBER_TEST_CASES: Array<ValueTestCase<number>> = [
33+
new ValueTestCase(
34+
Number.NEGATIVE_INFINITY,
35+
// Note: This values are taken from the Android reference implementation
36+
new Uint8Array([0x07, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
37+
new Uint8Array([0xf8, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
38+
),
39+
new ValueTestCase(
40+
Number.MIN_SAFE_INTEGER,
41+
new Uint8Array([0x08, 0x3c, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
42+
new Uint8Array([0xf7, 0xc3, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
43+
),
44+
new ValueTestCase(
45+
-2,
46+
new Uint8Array([0x08, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
47+
new Uint8Array([0xf7, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
48+
),
49+
new ValueTestCase(
50+
-1,
51+
new Uint8Array([0x08, 0x40, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
52+
new Uint8Array([0xf7, 0xbf, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
53+
),
54+
new ValueTestCase(
55+
-0.1,
56+
new Uint8Array([0x08, 0x40, 0x46, 0x66, 0x66, 0x66, 0x66, 0x66, 0x65]),
57+
new Uint8Array([0xf7, 0xbf, 0xb9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a])
58+
),
59+
new ValueTestCase(
60+
-0.0,
61+
new Uint8Array([0x08, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
62+
new Uint8Array([0xf7, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
63+
),
64+
new ValueTestCase(
65+
0,
66+
new Uint8Array([0x08, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
67+
new Uint8Array([0xf7, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
68+
),
69+
new ValueTestCase(
70+
Number.MIN_VALUE,
71+
new Uint8Array([0x08, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]),
72+
new Uint8Array([0xf7, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe])
73+
),
74+
new ValueTestCase(
75+
0.1,
76+
new Uint8Array([0x08, 0xbf, 0xb9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a]),
77+
new Uint8Array([0xf7, 0x40, 0x46, 0x66, 0x66, 0x66, 0x66, 0x66, 0x65])
78+
),
79+
new ValueTestCase(
80+
1,
81+
new Uint8Array([0x08, 0xbf, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
82+
new Uint8Array([0xf7, 0x40, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
83+
),
84+
new ValueTestCase(
85+
2,
86+
new Uint8Array([0x08, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
87+
new Uint8Array([0xf7, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
88+
),
89+
new ValueTestCase(
90+
4,
91+
new Uint8Array([0x08, 0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
92+
new Uint8Array([0xf7, 0x3f, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
93+
),
94+
new ValueTestCase(
95+
8,
96+
new Uint8Array([0x08, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
97+
new Uint8Array([0xf7, 0x3f, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
98+
),
99+
new ValueTestCase(
100+
16,
101+
new Uint8Array([0x08, 0xc0, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
102+
new Uint8Array([0xf7, 0x3f, 0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
103+
),
104+
new ValueTestCase(
105+
32,
106+
new Uint8Array([0x08, 0xc0, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
107+
new Uint8Array([0xf7, 0x3f, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
108+
),
109+
new ValueTestCase(
110+
64,
111+
new Uint8Array([0x08, 0xc0, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
112+
new Uint8Array([0xf7, 0x3f, 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
113+
),
114+
new ValueTestCase(
115+
128,
116+
new Uint8Array([0x08, 0xc0, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
117+
new Uint8Array([0xf7, 0x3f, 0x9f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
118+
),
119+
new ValueTestCase(
120+
255,
121+
new Uint8Array([0x08, 0xc0, 0x6f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00]),
122+
new Uint8Array([0xf7, 0x3f, 0x90, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff])
123+
),
124+
new ValueTestCase(
125+
256,
126+
new Uint8Array([0x08, 0xc0, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
127+
new Uint8Array([0xf7, 0x3f, 0x8f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
128+
),
129+
new ValueTestCase(
130+
257,
131+
new Uint8Array([0x08, 0xc0, 0x70, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00]),
132+
new Uint8Array([0xf7, 0x3f, 0x8f, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff])
133+
),
134+
new ValueTestCase(
135+
Number.MAX_SAFE_INTEGER,
136+
new Uint8Array([0x08, 0xc3, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
137+
new Uint8Array([0xf7, 0x3c, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
138+
),
139+
new ValueTestCase(
140+
Number.POSITIVE_INFINITY,
141+
new Uint8Array([0x08, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
142+
new Uint8Array([0xf7, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
143+
),
144+
new ValueTestCase(
145+
Number.NaN,
146+
new Uint8Array([0x08, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
147+
new Uint8Array([0xf7, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
148+
)
149+
];
150+
151+
describe('Ordered Code Writer', () => {
152+
it('computes number of leading zeros', () => {
153+
for (let i = 0; i < 0xff; ++i) {
154+
let zeros = 0;
155+
for (let bit = 7; bit >= 0; --bit) {
156+
if ((i & (1 << bit)) === 0) {
157+
++zeros;
158+
} else {
159+
break;
160+
}
161+
}
162+
expect(numberOfLeadingZerosInByte(i)).to.equal(zeros, `for number ${i}`);
163+
}
164+
});
165+
166+
it('converts numbers to bits', () => {
167+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
168+
const bytes = getBytes(NUMBER_TEST_CASES[i].val);
169+
expect(bytes.asc).to.deep.equal(
170+
NUMBER_TEST_CASES[i].ascEncoding,
171+
'Ascending for ' + NUMBER_TEST_CASES[i].val
172+
);
173+
expect(bytes.desc).to.deep.equal(
174+
NUMBER_TEST_CASES[i].descEncoding,
175+
'Descending for ' + NUMBER_TEST_CASES[i].val
176+
);
177+
}
178+
});
179+
180+
it('orders numbers correctly', () => {
181+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
182+
for (let j = i; j < NUMBER_TEST_CASES.length; ++j) {
183+
const left = NUMBER_TEST_CASES[i].val;
184+
const leftBytes = getBytes(left);
185+
const right = NUMBER_TEST_CASES[j].val;
186+
const rightBytes = getBytes(right);
187+
expect(compare(leftBytes.asc, rightBytes.asc)).to.equal(
188+
i === j ? 0 : -1,
189+
`Ascending order: ${left} vs ${right}`
190+
);
191+
expect(compare(leftBytes.desc, rightBytes.desc)).to.equal(
192+
i === j ? 0 : 1,
193+
`Descending order: ${left} vs ${right}`
194+
);
195+
}
196+
}
197+
});
198+
});
199+
200+
function compare(left: Uint8Array, right: Uint8Array): number {
201+
for (let i = 0; i < Math.min(left.length, right.length); ++i) {
202+
if (left[i] < right[i]) {
203+
return -1;
204+
}
205+
if (left[i] > right[i]) {
206+
return 1;
207+
}
208+
}
209+
return left.length - right.length;
210+
}
211+
212+
function getBytes(val: unknown): { asc: Uint8Array; desc: Uint8Array } {
213+
const ascWriter = new OrderedCodeWriter();
214+
const descWriter = new OrderedCodeWriter();
215+
if (typeof val === 'number') {
216+
ascWriter.writeNumberAscending(val);
217+
descWriter.writeNumberDescending(val);
218+
} else {
219+
throw new Error('Encoding not yet supported for ' + val);
220+
}
221+
return { asc: ascWriter.encodedBytes(), desc: descWriter.encodedBytes() };
222+
}

0 commit comments

Comments
 (0)