Skip to content

Commit 8a4d444

Browse files
schmidt-sebastianschmidt-sebastian
committed
Add double to bit encoding for OrderedCode
1 parent 8d9a09e commit 8a4d444

File tree

2 files changed

+375
-0
lines changed

2 files changed

+375
-0
lines changed

‎packages/firestore/src/index/ordered_code_writer.ts

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law | agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES | CONDITIONS OF ANY KIND, either express | implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
const LONG_SIZE = 64;
19+
const BYTE_SIZE = 8;
20+
21+
/**
22+
* The default size of the buffer. This is arbitrary, but likely larger than
23+
* most index values so that less copies of the underlying buffer will be made.
24+
* For large values, a single copy will made to double the buffer length.
25+
*/
26+
const DEFAULT_BUFFER_SIZE = 1024;
27+
28+
/**
29+
* Converts a JavaScript number to a byte array (using little endian
30+
* encoding).
31+
*/
32+
function doubleToLongBits(value: number): Uint8Array {
33+
const dv = new DataView(new ArrayBuffer(8));
34+
dv.setFloat64(0, value, false);
35+
return new Uint8Array(dv.buffer);
36+
}
37+
38+
/** Counts the number of zeros in a byte. */
39+
export function numberOfLeadingZerosInByte(x: number): number {
40+
if (x === 0) {
41+
return 8;
42+
}
43+
44+
let zeros = 0;
45+
if (x >> 4 === 0) {
46+
// Test if the first four bits are zero.
47+
zeros += 4;
48+
x = x << 4;
49+
}
50+
if (x >> 6 === 0) {
51+
// Test if the first two (or next two) bits are zero.
52+
zeros += 2;
53+
x = x << 2;
54+
}
55+
if (x >> 7 === 0) {
56+
// Test if the remaining bit is zero.
57+
zeros += 1;
58+
}
59+
return zeros;
60+
}
61+
62+
/** Counts the number of leading zeros in the given byte array. */
63+
function numberOfLeadingZeros(bytes: Uint8Array): number {
64+
let leadingZeros = 0;
65+
for (let i = 0; i < bytes.length; ++i) {
66+
const zeros = numberOfLeadingZerosInByte(bytes[i] & 0xff);
67+
leadingZeros += zeros;
68+
if (zeros !== 8) {
69+
break;
70+
}
71+
}
72+
return leadingZeros;
73+
}
74+
75+
/**
76+
* Returns of bytes required to store "value". Leading zero bytes are skipped.
77+
*/
78+
function unsignedNumLength(value: Uint8Array): number {
79+
// This is just the number of bytes for the unsigned representation of the number.
80+
const numBits = LONG_SIZE - numberOfLeadingZeros(value);
81+
return Math.ceil(numBits / BYTE_SIZE);
82+
}
83+
84+
/**
85+
* OrderedCodeWriter is a minimal-allocation implementation of the writing
86+
* behavior defined by the backend.
87+
*
88+
* The code is ported from its Java counterpart.
89+
*/
90+
export class OrderedCodeWriter {
91+
buffer = new Uint8Array(DEFAULT_BUFFER_SIZE);
92+
position = 0;
93+
94+
writeNumberAscending(val: number): void {
95+
const value = this.toOrderedBits(val);
96+
const len = unsignedNumLength(value);
97+
this.ensureAvailable(1 + len);
98+
this.buffer[this.position++] = len & 0xff; // Write the length
99+
for (let i = value.length - len; i < value.length; ++i) {
100+
this.buffer[this.position++] = value[i] & 0xff;
101+
}
102+
}
103+
104+
writeNumberDescending(val: number): void {
105+
const value = this.toOrderedBits(val);
106+
const len = unsignedNumLength(value);
107+
this.ensureAvailable(1 + ~len);
108+
this.buffer[this.position++] = ~(len & 0xff); // Write the length
109+
for (let i = value.length - len; i < value.length; ++i) {
110+
this.buffer[this.position++] = ~(value[i] & 0xff);
111+
}
112+
}
113+
114+
/**
115+
* Encodes `val` into an encoding that has the following properties:
116+
* The order matches the IEEE 754 floating-point comparison results with the
117+
* following exceptions:
118+
* -0.0 < 0.0
119+
* all non-NaN < NaN
120+
* NaN = NaN
121+
*/
122+
private toOrderedBits(val: number): Uint8Array {
123+
const value = doubleToLongBits(val);
124+
const isNegative = (value[0] & 0x80) !== 0;
125+
value[0] ^= isNegative ? 0xff : 0x80;
126+
for (let i = 1; i < value.length; ++i) {
127+
value[i] ^= isNegative ? 0xff : 0x00;
128+
}
129+
return value;
130+
}
131+
132+
/** Resets the buffer such that it is the same as when it was newly constructed. */
133+
reset(): void {
134+
this.position = 0;
135+
}
136+
137+
/** Makes a copy of the encoded bytes in this buffer. */
138+
encodedBytes(): Uint8Array {
139+
return this.buffer.slice(0, this.position);
140+
}
141+
142+
private ensureAvailable(bytes: number): void {
143+
const minCapacity = bytes + this.position;
144+
if (minCapacity <= this.buffer.length) {
145+
return;
146+
}
147+
// Try doubling.
148+
let newLength = this.buffer.length * 2;
149+
// Still not big enough? Just allocate the right size.
150+
if (newLength < minCapacity) {
151+
newLength = minCapacity;
152+
}
153+
// Create the new buffer.
154+
const newBuffer = new Uint8Array(newLength);
155+
newBuffer.set(this.buffer); // copy old data
156+
this.buffer = newBuffer;
157+
}
158+
}

‎packages/firestore/test/unit/index/ordered_code_writer.test.ts

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/**
2+
* @license
3+
* Copyright 2017 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0x00 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0x00
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
import { expect } from 'chai';
18+
19+
import {
20+
numberOfLeadingZerosInByte,
21+
OrderedCodeWriter
22+
} from '../../../src/index/ordered_code_writer';
23+
24+
class ValueTestCase<T> {
25+
constructor(
26+
readonly val: T,
27+
readonly ascEncoding: Uint8Array,
28+
readonly descEncoding: Uint8Array
29+
) {}
30+
}
31+
32+
const NUMBER_TEST_CASES: Array<ValueTestCase<number>> = [
33+
new ValueTestCase(
34+
Number.NEGATIVE_INFINITY,
35+
// Note: This values are taken from the Android reference implementation
36+
new Uint8Array([0x07, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
37+
new Uint8Array([0xf8, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
38+
),
39+
new ValueTestCase(
40+
Number.MIN_SAFE_INTEGER,
41+
new Uint8Array([0x08, 0x3c, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
42+
new Uint8Array([0xf7, 0xc3, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
43+
),
44+
new ValueTestCase(
45+
-2,
46+
new Uint8Array([0x08, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
47+
new Uint8Array([0xf7, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
48+
),
49+
new ValueTestCase(
50+
-1,
51+
new Uint8Array([0x08, 0x40, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
52+
new Uint8Array([0xf7, 0xbf, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
53+
),
54+
new ValueTestCase(
55+
-0.1,
56+
new Uint8Array([0x08, 0x40, 0x46, 0x66, 0x66, 0x66, 0x66, 0x66, 0x65]),
57+
new Uint8Array([0xf7, 0xbf, 0xb9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a])
58+
),
59+
new ValueTestCase(
60+
0,
61+
new Uint8Array([0x08, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
62+
new Uint8Array([0xf7, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
63+
),
64+
new ValueTestCase(
65+
Number.MIN_VALUE,
66+
new Uint8Array([0x08, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]),
67+
new Uint8Array([0xf7, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe])
68+
),
69+
new ValueTestCase(
70+
0.1,
71+
new Uint8Array([0x08, 0xbf, 0xb9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a]),
72+
new Uint8Array([0xf7, 0x40, 0x46, 0x66, 0x66, 0x66, 0x66, 0x66, 0x65])
73+
),
74+
new ValueTestCase(
75+
1,
76+
new Uint8Array([0x08, 0xbf, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
77+
new Uint8Array([0xf7, 0x40, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
78+
),
79+
new ValueTestCase(
80+
2,
81+
new Uint8Array([0x08, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
82+
new Uint8Array([0xf7, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
83+
),
84+
new ValueTestCase(
85+
4,
86+
new Uint8Array([0x08, 0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
87+
new Uint8Array([0xf7, 0x3f, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
88+
),
89+
new ValueTestCase(
90+
0x08,
91+
new Uint8Array([0x08, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
92+
new Uint8Array([0xf7, 0x3f, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
93+
),
94+
new ValueTestCase(
95+
0x10,
96+
new Uint8Array([0x08, 0xc0, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
97+
new Uint8Array([0xf7, 0x3f, 0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
98+
),
99+
new ValueTestCase(
100+
0x20,
101+
new Uint8Array([0x08, 0xc0, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
102+
new Uint8Array([0xf7, 0x3f, 0xbf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
103+
),
104+
new ValueTestCase(
105+
0x40,
106+
new Uint8Array([0x08, 0xc0, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
107+
new Uint8Array([0xf7, 0x3f, 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
108+
),
109+
new ValueTestCase(
110+
0x80,
111+
new Uint8Array([0x08, 0xc0, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
112+
new Uint8Array([0xf7, 0x3f, 0x9f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
113+
),
114+
new ValueTestCase(
115+
0xff,
116+
new Uint8Array([0x08, 0xc0, 0x6f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00]),
117+
new Uint8Array([0xf7, 0x3f, 0x90, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff])
118+
),
119+
new ValueTestCase(
120+
256,
121+
new Uint8Array([0x08, 0xc0, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
122+
new Uint8Array([0xf7, 0x3f, 0x8f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
123+
),
124+
new ValueTestCase(
125+
257,
126+
new Uint8Array([0x08, 0xc0, 0x70, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00]),
127+
new Uint8Array([0xf7, 0x3f, 0x8f, 0xef, 0xff, 0xff, 0xff, 0xff, 0xff])
128+
),
129+
new ValueTestCase(
130+
Number.MAX_SAFE_INTEGER,
131+
new Uint8Array([0x08, 0xc3, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]),
132+
new Uint8Array([0xf7, 0x3c, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
133+
),
134+
new ValueTestCase(
135+
Number.POSITIVE_INFINITY,
136+
new Uint8Array([0x08, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
137+
new Uint8Array([0xf7, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
138+
),
139+
new ValueTestCase(
140+
Number.NaN,
141+
new Uint8Array([0x08, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]),
142+
new Uint8Array([0xf7, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])
143+
)
144+
];
145+
146+
describe('Ordered Code Writer', () => {
147+
it('computes number of leading zeros', () => {
148+
for (let i = 0; i < 0xff; ++i) {
149+
let zeros = 0;
150+
for (let bit = 7; bit >= 0; --bit) {
151+
if ((i & (1 << bit)) === 0) {
152+
++zeros;
153+
} else {
154+
break;
155+
}
156+
}
157+
expect(numberOfLeadingZerosInByte(i)).to.equal(zeros, `for number ${i}`);
158+
}
159+
});
160+
161+
it('converts numbers to bits', () => {
162+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
163+
const bytes = getBytes(NUMBER_TEST_CASES[i].val);
164+
expect(bytes.asc).to.deep.equal(
165+
NUMBER_TEST_CASES[i].ascEncoding,
166+
'Ascending for ' + NUMBER_TEST_CASES[i].val
167+
);
168+
expect(bytes.desc).to.deep.equal(
169+
NUMBER_TEST_CASES[i].descEncoding,
170+
'Descending for ' + NUMBER_TEST_CASES[i].val
171+
);
172+
}
173+
});
174+
175+
it('orders numbers correctly', () => {
176+
for (let i = 0; i < NUMBER_TEST_CASES.length; ++i) {
177+
for (let j = i; j < NUMBER_TEST_CASES.length; ++j) {
178+
const left = NUMBER_TEST_CASES[i].val;
179+
const leftBytes = getBytes(left);
180+
const right = NUMBER_TEST_CASES[j].val;
181+
const rightBytes = getBytes(right);
182+
expect(compare(leftBytes.asc, rightBytes.asc)).to.equal(
183+
i === j ? 0 : -1,
184+
`Ascending order: ${left} vs ${right}`
185+
);
186+
expect(compare(leftBytes.desc, rightBytes.desc)).to.equal(
187+
i === j ? 0 : 1,
188+
`Descending order: ${left} vs ${right}`
189+
);
190+
}
191+
}
192+
});
193+
});
194+
195+
function compare(left: Uint8Array, right: Uint8Array): number {
196+
for (let i = 0; i < Math.min(left.length, right.length); ++i) {
197+
if (left[i] < right[i]) {
198+
return -1;
199+
}
200+
if (left[i] > right[i]) {
201+
return 1;
202+
}
203+
}
204+
return left.length - right.length;
205+
}
206+
207+
function getBytes(val: unknown): { asc: Uint8Array; desc: Uint8Array } {
208+
const ascWriter = new OrderedCodeWriter();
209+
const descWriter = new OrderedCodeWriter();
210+
if (typeof val === 'number') {
211+
ascWriter.writeNumberAscending(val);
212+
descWriter.writeNumberDescending(val);
213+
} else {
214+
throw new Error('Encoding not yet supported for ' + val);
215+
}
216+
return { asc: ascWriter.encodedBytes(), desc: descWriter.encodedBytes() };
217+
}

0 commit comments

Comments
 (0)