Skip to content

Commit a2f68b2

Browse files
graydongraydon
committed
Intern metadata while writing, shrink stage1 from 12mb to 5.7mb.
1 parent 6daf440 commit a2f68b2

File tree

4 files changed

+224
-94
lines changed

4 files changed

+224
-94
lines changed

‎src/comp/front/creader.rs

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -43,34 +43,29 @@ tag resolve_result {
4343

4444
// Compact string representation for ty.t values. API ty_str & parse_from_str.
4545
// (The second has to be authed pure.) Extra parameters are for converting
46-
// to/from def_ids in the string rep. Whatever format you choose should not
46+
// to/from def_ids in the data buffer. Whatever format you choose should not
4747
// contain pipe characters.
4848

4949
// Callback to translate defs to strs or back.
5050
type str_def = fn(str) -> ast.def_id;
5151

52-
type pstate = rec(str rep, mutable uint pos, uint len, ty.ctxt tcx);
52+
type pstate = rec(vec[u8] data, int crate,
53+
mutable uint pos, uint len, ty.ctxt tcx);
5354

5455
fn peek(@pstate st) -> u8 {
55-
if (st.pos < st.len) {ret st.rep.(st.pos) as u8;}
56-
else {ret ' ' as u8;}
56+
ret st.data.(st.pos);
5757
}
5858
fn next(@pstate st) -> u8 {
59-
if (st.pos >= st.len) {fail;}
60-
auto ch = st.rep.(st.pos);
59+
auto ch = st.data.(st.pos);
6160
st.pos = st.pos + 1u;
62-
ret ch as u8;
61+
ret ch;
6362
}
6463

65-
fn parse_ty_str(str rep, str_def sd, ty.ctxt tcx) -> ty.t {
66-
auto len = _str.byte_len(rep);
67-
auto st = @rec(rep=rep, mutable pos=0u, len=len, tcx=tcx);
64+
fn parse_ty_data(vec[u8] data, int crate_num, uint pos, uint len,
65+
str_def sd, ty.ctxt tcx) -> ty.t {
66+
auto st = @rec(data=data, crate=crate_num,
67+
mutable pos=pos, len=len, tcx=tcx);
6868
auto result = parse_ty(st, sd);
69-
if (st.pos != len) {
70-
log_err "parse_ty_str: incomplete parse, stopped at byte "
71-
+ _uint.to_str(st.pos, 10u) + " of "
72-
+ _uint.to_str(len, 10u) + " in str '" + rep + "'";
73-
}
7469
ret result;
7570
}
7671

@@ -178,6 +173,26 @@ fn parse_ty(@pstate st, str_def sd) -> ty.t {
178173
case ('X') { ret ty.mk_var(st.tcx, parse_int(st)); }
179174
case ('E') { ret ty.mk_native(st.tcx); }
180175
case ('Y') { ret ty.mk_type(st.tcx); }
176+
case ('#') {
177+
auto pos = parse_hex(st);
178+
check (next(st) as char == ':');
179+
auto len = parse_hex(st);
180+
check (next(st) as char == '#');
181+
alt (st.tcx.rcache.find(tup(st.crate,pos,len))) {
182+
case (some[ty.t](?tt)) { ret tt; }
183+
case (none[ty.t]) {
184+
auto ps = @rec(pos=pos, len=len with *st);
185+
auto tt = parse_ty(ps, sd);
186+
st.tcx.rcache.insert(tup(st.crate,pos,len), tt);
187+
ret tt;
188+
}
189+
}
190+
}
191+
case (?c) {
192+
log_err "unexpected char in type string: ";
193+
log_err c;
194+
fail;
195+
}
181196
}
182197
}
183198

@@ -212,6 +227,23 @@ fn parse_int(@pstate st) -> int {
212227
ret n;
213228
}
214229

230+
fn parse_hex(@pstate st) -> uint {
231+
auto n = 0u;
232+
while (true) {
233+
auto cur = peek(st) as char;
234+
if ((cur < '0' || cur > '9') &&
235+
(cur < 'a' || cur > 'f')) {break;}
236+
st.pos = st.pos + 1u;
237+
n *= 16u;
238+
if ('0' <= cur && cur <= '9') {
239+
n += (cur as uint) - ('0' as uint);
240+
} else {
241+
n += (10u + (cur as uint) - ('a' as uint));
242+
}
243+
}
244+
ret n;
245+
}
246+
215247
fn parse_ty_fn(@pstate st, str_def sd) -> tup(vec[ty.arg], ty.t) {
216248
check(next(st) as char == '[');
217249
let vec[ty.arg] inputs = vec();
@@ -343,7 +375,8 @@ fn item_type(&ebml.doc item, int this_cnum, ty.ctxt tcx) -> ty.t {
343375

344376
auto tp = ebml.get_doc(item, metadata.tag_items_data_item_type);
345377
auto s = _str.unsafe_from_bytes(ebml.doc_data(tp));
346-
ret parse_ty_str(s, bind parse_external_def_id(this_cnum, _), tcx);
378+
ret parse_ty_data(item.data, this_cnum, tp.start, tp.end - tp.start,
379+
bind parse_external_def_id(this_cnum, _), tcx);
347380
}
348381

349382
fn item_ty_param_count(&ebml.doc item, int this_cnum) -> uint {

‎src/comp/middle/metadata.rs

Lines changed: 138 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
import std._str;
22
import std._uint;
33
import std._vec;
4+
import std.map.hashmap;
45
import std.ebml;
56
import std.io;
67
import std.option;
8+
import std.option.some;
9+
import std.option.none;
710

811
import front.ast;
912
import middle.fold;
@@ -48,119 +51,185 @@ const uint tag_index_table = 0x15u;
4851
// Extra parameters are for converting to/from def_ids in the string rep.
4952
// Whatever format you choose should not contain pipe characters.
5053

54+
type ty_abbrev = rec(uint pos, uint len, str s);
55+
5156
mod Encode {
5257

5358
type ctxt = rec(
54-
fn(ast.def_id) -> str ds, // Callback to translate defs to strs.
55-
ty.ctxt tcx // The type context.
59+
fn(ast.def_id) -> str ds, // Def -> str Callback.
60+
ty.ctxt tcx, // The type context.
61+
bool use_abbrevs,
62+
hashmap[ty.t, ty_abbrev] abbrevs // Type abbrevs.
5663
);
5764

5865
fn ty_str(@ctxt cx, ty.t t) -> str {
59-
ret sty_str(cx, ty.struct(cx.tcx, t));
66+
check (! cx.use_abbrevs);
67+
auto sw = io.string_writer();
68+
enc_ty(sw.get_writer(), cx, t);
69+
ret sw.get_str();
70+
}
71+
72+
fn enc_ty(io.writer w, @ctxt cx, ty.t t) {
73+
if (cx.use_abbrevs) {
74+
alt (cx.abbrevs.find(t)) {
75+
case (some[ty_abbrev](?a)) {
76+
w.write_str(a.s);
77+
ret;
78+
}
79+
case (none[ty_abbrev]) {
80+
auto pos = w.get_buf_writer().tell();
81+
auto ss = enc_sty(w, cx, ty.struct(cx.tcx, t));
82+
auto end = w.get_buf_writer().tell();
83+
auto len = end-pos;
84+
fn estimate_sz(uint u) -> uint {
85+
auto n = u;
86+
auto len = 0u;
87+
while (n != 0u) {
88+
len += 1u;
89+
n = n >> 4u;
90+
}
91+
ret len;
92+
}
93+
auto abbrev_len =
94+
3u + estimate_sz(pos) + estimate_sz(len);
95+
96+
if (abbrev_len < len) {
97+
// I.e. it's actually an abbreviation.
98+
auto s = ("#"
99+
+ _uint.to_str(pos, 16u) + ":"
100+
+ _uint.to_str(len, 16u) + "#");
101+
auto a = rec(pos=pos, len=len, s=s);
102+
cx.abbrevs.insert(t, a);
103+
}
104+
ret;
105+
}
106+
}
107+
}
108+
enc_sty(w, cx, ty.struct(cx.tcx, t));
60109
}
61110

62-
fn mt_str(@ctxt cx, &ty.mt mt) -> str {
63-
auto mut_str;
111+
fn enc_mt(io.writer w, @ctxt cx, &ty.mt mt) {
64112
alt (mt.mut) {
65-
case (ast.imm) { mut_str = ""; }
66-
case (ast.mut) { mut_str = "m"; }
67-
case (ast.maybe_mut) { mut_str = "?"; }
113+
case (ast.imm) { }
114+
case (ast.mut) { w.write_char('m'); }
115+
case (ast.maybe_mut) { w.write_char('?'); }
68116
}
69-
ret mut_str + ty_str(cx, mt.ty);
117+
enc_ty(w, cx, mt.ty);
70118
}
71119

72-
fn sty_str(@ctxt cx, ty.sty st) -> str {
120+
fn enc_sty(io.writer w, @ctxt cx, ty.sty st) {
73121
alt (st) {
74-
case (ty.ty_nil) {ret "n";}
75-
case (ty.ty_bool) {ret "b";}
76-
case (ty.ty_int) {ret "i";}
77-
case (ty.ty_uint) {ret "u";}
78-
case (ty.ty_float) {ret "l";}
122+
case (ty.ty_nil) { w.write_char('n'); }
123+
case (ty.ty_bool) { w.write_char('b'); }
124+
case (ty.ty_int) { w.write_char('i'); }
125+
case (ty.ty_uint) { w.write_char('u'); }
126+
case (ty.ty_float) { w.write_char('l'); }
79127
case (ty.ty_machine(?mach)) {
80128
alt (mach) {
81-
case (common.ty_u8) {ret "Mb";}
82-
case (common.ty_u16) {ret "Mw";}
83-
case (common.ty_u32) {ret "Ml";}
84-
case (common.ty_u64) {ret "Md";}
85-
case (common.ty_i8) {ret "MB";}
86-
case (common.ty_i16) {ret "MW";}
87-
case (common.ty_i32) {ret "ML";}
88-
case (common.ty_i64) {ret "MD";}
89-
case (common.ty_f32) {ret "Mf";}
90-
case (common.ty_f64) {ret "MF";}
129+
case (common.ty_u8) { w.write_str("Mb"); }
130+
case (common.ty_u16) { w.write_str("Mw"); }
131+
case (common.ty_u32) { w.write_str("Ml"); }
132+
case (common.ty_u64) { w.write_str("Md"); }
133+
case (common.ty_i8) { w.write_str("MB"); }
134+
case (common.ty_i16) { w.write_str("MW"); }
135+
case (common.ty_i32) { w.write_str("ML"); }
136+
case (common.ty_i64) { w.write_str("MD"); }
137+
case (common.ty_f32) { w.write_str("Mf"); }
138+
case (common.ty_f64) { w.write_str("MF"); }
91139
}
92140
}
93-
case (ty.ty_char) {ret "c";}
94-
case (ty.ty_str) {ret "s";}
141+
case (ty.ty_char) {w.write_char('c');}
142+
case (ty.ty_str) {w.write_char('s');}
95143
case (ty.ty_tag(?def,?tys)) { // TODO restore def_id
96-
auto acc = "t[" + cx.ds(def) + "|";
97-
for (ty.t t in tys) {acc += ty_str(cx, t);}
98-
ret acc + "]";
144+
w.write_str("t[");
145+
w.write_str(cx.ds(def));
146+
w.write_char('|');
147+
for (ty.t t in tys) {
148+
enc_ty(w, cx, t);
149+
}
150+
w.write_char(']');
99151
}
100-
case (ty.ty_box(?mt)) {ret "@" + mt_str(cx, mt);}
101-
case (ty.ty_vec(?mt)) {ret "V" + mt_str(cx, mt);}
102-
case (ty.ty_port(?t)) {ret "P" + ty_str(cx, t);}
103-
case (ty.ty_chan(?t)) {ret "C" + ty_str(cx, t);}
152+
case (ty.ty_box(?mt)) {w.write_char('@'); enc_mt(w, cx, mt); }
153+
case (ty.ty_vec(?mt)) {w.write_char('V'); enc_mt(w, cx, mt); }
154+
case (ty.ty_port(?t)) {w.write_char('P'); enc_ty(w, cx, t); }
155+
case (ty.ty_chan(?t)) {w.write_char('C'); enc_ty(w, cx, t); }
104156
case (ty.ty_tup(?mts)) {
105-
auto acc = "T[";
106-
for (ty.mt mt in mts) {acc += mt_str(cx, mt);}
107-
ret acc + "]";
157+
w.write_str("T[");
158+
for (ty.mt mt in mts) {
159+
enc_mt(w, cx, mt);
160+
}
161+
w.write_char(']');
108162
}
109163
case (ty.ty_rec(?fields)) {
110-
auto acc = "R[";
164+
w.write_str("R[");
111165
for (ty.field field in fields) {
112-
acc += field.ident + "=";
113-
acc += mt_str(cx, field.mt);
166+
w.write_str(field.ident);
167+
w.write_char('=');
168+
enc_mt(w, cx, field.mt);
114169
}
115-
ret acc + "]";
170+
w.write_char(']');
116171
}
117172
case (ty.ty_fn(?proto,?args,?out)) {
118-
ret proto_str(proto) + ty_fn_str(cx, args, out);
173+
enc_proto(w, proto);
174+
enc_ty_fn(w, cx, args, out);
119175
}
120176
case (ty.ty_native_fn(?abi,?args,?out)) {
121-
auto abistr;
177+
w.write_char('N');
122178
alt (abi) {
123-
case (ast.native_abi_rust) {abistr = "r";}
124-
case (ast.native_abi_cdecl) {abistr = "c";}
125-
case (ast.native_abi_llvm) {abistr = "l";}
179+
case (ast.native_abi_rust) { w.write_char('r'); }
180+
case (ast.native_abi_cdecl) { w.write_char('c'); }
181+
case (ast.native_abi_llvm) { w.write_char('l'); }
126182
}
127-
ret "N" + abistr + ty_fn_str(cx, args, out);
183+
enc_ty_fn(w, cx, args, out);
128184
}
129185
case (ty.ty_obj(?methods)) {
130-
auto acc = "O[";
186+
w.write_str("O[");
131187
for (ty.method m in methods) {
132-
acc += proto_str(m.proto);
133-
acc += m.ident;
134-
acc += ty_fn_str(cx, m.inputs, m.output);
188+
enc_proto(w, m.proto);
189+
w.write_str(m.ident);
190+
enc_ty_fn(w, cx, m.inputs, m.output);
135191
}
136-
ret acc + "]";
192+
w.write_char(']');
193+
}
194+
case (ty.ty_var(?id)) {
195+
w.write_char('X');
196+
w.write_str(common.istr(id));
137197
}
138-
case (ty.ty_var(?id)) {ret "X" + common.istr(id);}
139-
case (ty.ty_native) {ret "E";}
140-
case (ty.ty_param(?id)) {ret "p" + common.uistr(id);}
141-
case (ty.ty_type) {ret "Y";}
198+
case (ty.ty_native) {w.write_char('E');}
199+
case (ty.ty_param(?id)) {
200+
w.write_char('p');
201+
w.write_str(common.uistr(id));
202+
}
203+
case (ty.ty_type) {w.write_char('Y');}
142204

143205
// These two don't appear in crate metadata, but are here because
144206
// `hash_ty()` uses this function.
145-
case (ty.ty_bound_param(?id)) {ret "o" + common.uistr(id);}
146-
case (ty.ty_local(?def)) {ret "L" + cx.ds(def);}
207+
case (ty.ty_bound_param(?id)) {
208+
w.write_char('o');
209+
w.write_str(common.uistr(id));
210+
}
211+
case (ty.ty_local(?def)) {
212+
w.write_char('L');
213+
w.write_str(cx.ds(def));
214+
}
147215
}
148216
}
149217

150-
fn proto_str(ast.proto proto) -> str {
218+
fn enc_proto(io.writer w, ast.proto proto) {
151219
alt (proto) {
152-
case (ast.proto_iter) {ret "W";}
153-
case (ast.proto_fn) {ret "F";}
220+
case (ast.proto_iter) { w.write_char('W'); }
221+
case (ast.proto_fn) { w.write_char('F'); }
154222
}
155223
}
156224

157-
fn ty_fn_str(@ctxt cx, vec[ty.arg] args, ty.t out) -> str {
158-
auto acc = "[";
225+
fn enc_ty_fn(io.writer w, @ctxt cx, vec[ty.arg] args, ty.t out) {
226+
w.write_char('[');
159227
for (ty.arg arg in args) {
160-
if (arg.mode == ast.alias) {acc += "&";}
161-
acc += ty_str(cx, arg.ty);
228+
if (arg.mode == ast.alias) { w.write_char('&'); }
229+
enc_ty(w, cx, arg.ty);
162230
}
163-
ret acc + "]" + ty_str(cx, out);
231+
w.write_char(']');
232+
enc_ty(w, cx, out);
164233
}
165234

166235
}
@@ -336,9 +405,9 @@ fn encode_type(@trans.crate_ctxt cx, &ebml.writer ebml_w, ty.t typ) {
336405
ebml.start_tag(ebml_w, tag_items_data_item_type);
337406

338407
auto f = def_to_str;
339-
auto ty_str_ctxt = @rec(ds=f, tcx=cx.tcx);
340-
ebml_w.writer.write(_str.bytes(Encode.ty_str(ty_str_ctxt, typ)));
341-
408+
auto ty_str_ctxt = @rec(ds=f, tcx=cx.tcx,
409+
use_abbrevs=true, abbrevs=cx.type_abbrevs);
410+
Encode.enc_ty(io.new_writer_(ebml_w.writer), ty_str_ctxt, typ);
342411
ebml.end_tag(ebml_w);
343412
}
344413

@@ -565,7 +634,6 @@ fn encode_index[T](&ebml.writer ebml_w, vec[vec[tup(T, uint)]] buckets,
565634
ebml.end_tag(ebml_w);
566635
}
567636

568-
569637
fn write_str(io.writer writer, &str s) {
570638
writer.write_str(s);
571639
}

0 commit comments

Comments
 (0)