Skip to content

Small improvements to the dictionary compression #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 13, 2020
Merged

Small improvements to the dictionary compression #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 54 additions & 81 deletions py/makeqstrdata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import re
import sys

from math import log
import collections
import gettext
import os.path
Expand Down Expand Up @@ -111,9 +112,10 @@ def __init__(self, words):

def iter_words(self, text):
s = []
words = self.words
for m in self.pat.finditer(text):
t = m.group(0)
if t in self.words:
if t in words:
if s:
yield (False, "".join(s))
s = []
Expand All @@ -124,33 +126,35 @@ def iter_words(self, text):
yield (False, "".join(s))

def iter(self, text):
s = []
for m in self.pat.finditer(text):
yield m.group(0)

def iter_substrings(s, minlen, maxlen):
maxlen = min(len(s), maxlen)
len_s = len(s)
maxlen = min(len_s, maxlen)
for n in range(minlen, maxlen + 1):
for begin in range(0, len(s) - n + 1):
for begin in range(0, len_s - n + 1):
yield s[begin : begin + n]

def compute_huffman_coding(translations, compression_filename):
texts = [t[1] for t in translations]
all_strings_concat = "".join(texts)
words = []

start_unused = 0x80
end_unused = 0xff
max_ord = 0
begin_unused = 128
end_unused = 256
for text in texts:
for c in text:
ord_c = ord(c)
max_ord = max(max_ord, ord_c)
if 128 <= ord_c < 256:
max_ord = max(ord_c, max_ord)
if 0x80 <= ord_c < 0xff:
end_unused = min(ord_c, end_unused)
max_words = end_unused - begin_unused
char_size = 1 if max_ord < 256 else 2
max_words = end_unused - 0x80

values_type = "uint16_t" if max_ord > 255 else "uint8_t"
max_words_len = 160 if max_ord > 255 else 255

sum_word_len = 0
sum_len = 0
while True:
extractor = TextSplitter(words)
counter = collections.Counter()
Expand All @@ -162,30 +166,30 @@ def compute_huffman_coding(translations, compression_filename):

scores = sorted(
(
# I don't know why this works good. This could be better.
(s, (len(s) - 1) ** ((max(occ - 2, 1) + 0.5) ** 0.8), occ)
(s, (len(s) - 1) ** log(max(occ - 2, 1)), occ)
for (s, occ) in counter.items()
),
key=lambda x: x[1],
reverse=True,
)

w = None
word = None
for (s, score, occ) in scores:
if score < 0:
break
if len(s) > 1:
w = s
if occ < 5:
continue
if score < 5:
break
word = s
break

if not w:
if not word:
break
if len(w) + sum_word_len > 256:
if sum_len + len(word) - 2 > max_words_len:
break
if len(words) == max_words:
break
words.append(w)
sum_word_len += len(w)
words.append(word)
sum_len += len(word) - 2

extractor = TextSplitter(words)
counter = collections.Counter()
Expand All @@ -194,26 +198,26 @@ def compute_huffman_coding(translations, compression_filename):
counter[atom] += 1
cb = huffman.codebook(counter.items())

word_start = begin_unused
word_start = start_unused
word_end = word_start + len(words) - 1
print("// # words", len(words))
print("// words", words)

values = []
length_count = {}
renumbered = 0
last_l = None
last_length = None
canonical = {}
for atom, code in sorted(cb.items(), key=lambda x: (len(x[1]), x[0])):
values.append(atom)
l = len(code)
if l not in length_count:
length_count[l] = 0
length_count[l] += 1
if last_l:
renumbered <<= (l - last_l)
canonical[atom] = '{0:0{width}b}'.format(renumbered, width=l)
#print(f"atom={repr(atom)} code={code}", file=sys.stderr)
length = len(code)
if length not in length_count:
length_count[length] = 0
length_count[length] += 1
if last_length:
renumbered <<= (length - last_length)
canonical[atom] = '{0:0{width}b}'.format(renumbered, width=length)
# print(f"atom={repr(atom)} code={code}", file=sys.stderr)
if len(atom) > 1:
o = words.index(atom) + 0x80
s = "".join(C_ESCAPES.get(ch1, ch1) for ch1 in atom)
Expand All @@ -222,34 +226,37 @@ def compute_huffman_coding(translations, compression_filename):
o = ord(atom)
print("//", o, s, counter[atom], canonical[atom], renumbered)
renumbered += 1
last_l = l
last_length = length
lengths = bytearray()
print("// length count", length_count)

for i in range(1, max(length_count) + 2):
lengths.append(length_count.get(i, 0))
print("// values", values, "lengths", len(lengths), lengths)
maxord = max(ord(u) for u in values if len(u) == 1)
values_type = "uint16_t" if maxord > 255 else "uint8_t"
ch_size = 1 if maxord > 255 else 2

print("//", values, lengths)
values = [(atom if len(atom) == 1 else chr(0x80 + words.index(atom))) for atom in values]
print("//", values, lengths)
max_translation_encoded_length = max(len(translation.encode("utf-8")) for original,translation in translations)
max_translation_encoded_length = max(
len(translation.encode("utf-8")) for (original, translation) in translations)

wends = list(len(w) - 2 for w in words)
for i in range(1, len(wends)):
wends[i] += wends[i - 1]

with open(compression_filename, "w") as f:
f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths))))
f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values)))
f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length()))
f.write("const {} words[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(c)) for w in words for c in w)))
f.write("const uint8_t wlen[] = {{ {} }};\n".format(", ".join(str(len(w)) for w in words)))
f.write("const uint8_t wends[] = {{ {} }};\n".format(", ".join(str(p) for p in wends)))
f.write("#define word_start {}\n".format(word_start))
f.write("#define word_end {}\n".format(word_end))

extractor = TextSplitter(words)
return values, lengths, words, extractor
return (values, lengths, words, canonical, extractor)

def decompress(encoding_table, encoded, encoded_length_bits):
values, lengths, words, extractor = encoding_table
(values, lengths, words, _, _) = encoding_table
dec = []
this_byte = 0
this_bit = 7
Expand Down Expand Up @@ -306,66 +313,32 @@ def decompress(encoding_table, encoded, encoded_length_bits):
def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded):
if not isinstance(decompressed, str):
raise TypeError()
values, lengths, words, extractor = encoding_table
(_, _, _, canonical, extractor) = encoding_table

enc = bytearray(len(decompressed) * 3)
#print(decompressed)
#print(lengths)
current_bit = 7
current_byte = 0

code = len_translation_encoded
bits = encoded_length_bits+1
bits = encoded_length_bits + 1
for i in range(bits - 1, 0, -1):
if len_translation_encoded & (1 << (i - 1)):
enc[current_byte] |= 1 << current_bit
if current_bit == 0:
current_bit = 7
#print("packed {0:0{width}b}".format(enc[current_byte], width=8))
current_byte += 1
else:
current_bit -= 1

#print("values = ", values, file=sys.stderr)
for atom in extractor.iter(decompressed):
#print("", file=sys.stderr)
if len(atom) > 1:
c = chr(0x80 + words.index(atom))
else:
c = atom
assert c in values

start = 0
end = lengths[0]
bits = 1
compressed = None
code = 0
while compressed is None:
s = start
e = end
#print("{0:0{width}b}".format(code, width=bits))
# Linear search!
for i in range(s, e):
if values[i] == c:
compressed = code + (i - start)
#print("found {0:0{width}b}".format(compressed, width=bits), file=sys.stderr)
break
code += end - start
code <<= 1
start = end
end += lengths[bits]
bits += 1
#print("next bit", bits)

for i in range(bits - 1, 0, -1):
if compressed & (1 << (i - 1)):
for b in canonical[atom]:
if b == "1":
enc[current_byte] |= 1 << current_bit
if current_bit == 0:
current_bit = 7
#print("packed {0:0{width}b}".format(enc[current_byte], width=8))
current_byte += 1
else:
current_bit -= 1

if current_bit != 7:
current_byte += 1
return enc[:current_byte]
Expand Down
12 changes: 6 additions & 6 deletions supervisor/shared/translate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,17 @@ STATIC int put_utf8(char *buf, int u) {
*buf = u;
return 1;
} else if(word_start <= u && u <= word_end) {
int n = (u - 0x80);
size_t off = 0;
for(int i=0; i<n; i++) {
off += wlen[i];
uint n = (u - word_start);
size_t pos = 0;
if (n > 0) {
pos = wends[n - 1] + (n * 2);
}
int ret = 0;
// note that at present, entries in the words table are
// guaranteed not to represent words themselves, so this adds
// at most 1 level of recursive call
for(int i=0; i<wlen[n]; i++) {
int len = put_utf8(buf, words[off+i]);
for(; pos < wends[n] + (n + 1) * 2; pos++) {
int len = put_utf8(buf, words[pos]);
buf += len;
ret += len;
}
Expand Down