Skip to content

Commit b00f307

Browse files
graydongraydon
committed
Remove boxes from token.t.
1 parent 70c7590 commit b00f307

File tree

3 files changed

+79
-42
lines changed

3 files changed

+79
-42
lines changed

‎src/comp/front/lexer.rs

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ state type reader = state obj {
1818
fn bump();
1919
fn mark();
2020
fn get_mark_chpos() -> uint;
21+
fn add_str(str) -> token.str_num;
22+
fn get_str(token.str_num) -> str;
2123
fn get_chpos() -> uint;
2224
fn get_keywords() -> hashmap[str,token.token];
2325
fn get_reserved() -> hashmap[str,()];
@@ -32,6 +34,7 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
3234
mutable char ch,
3335
mutable uint mark_chpos,
3436
mutable uint chpos,
37+
mutable vec[str] strs,
3538
hashmap[str,token.token] keywords,
3639
hashmap[str,()] reserved,
3740
codemap.filemap fm) {
@@ -79,6 +82,15 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
7982
ret keywords;
8083
}
8184

85+
fn add_str(str s) -> token.str_num {
86+
strs += vec(s);
87+
ret Vec.len[str](strs) - 1u;
88+
}
89+
90+
fn get_str(token.str_num i) -> str {
91+
ret strs.(i);
92+
}
93+
8294
fn get_reserved() -> hashmap[str,()] {
8395
ret reserved;
8496
}
@@ -88,9 +100,10 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
88100
}
89101
}
90102
auto file = Str.unsafe_from_bytes(rdr.read_whole_stream());
103+
let vec[str] strs = vec();
91104
auto rd = reader(file, Str.byte_len(file), 0u, -1 as char,
92105
filemap.start_pos, filemap.start_pos,
93-
keyword_table(),
106+
strs, keyword_table(),
94107
reserved_word_table(),
95108
filemap);
96109
rd.init();
@@ -500,25 +513,25 @@ fn scan_number(char c, reader rdr) -> token.token {
500513
if (c == '3' && n == '2') {
501514
rdr.bump(); rdr.bump();
502515
ret token.LIT_MACH_FLOAT(util.common.ty_f32,
503-
float_str);
516+
rdr.add_str(float_str));
504517
}
505518
else if (c == '6' && n == '4') {
506519
rdr.bump(); rdr.bump();
507520
ret token.LIT_MACH_FLOAT(util.common.ty_f64,
508-
float_str);
521+
rdr.add_str(float_str));
509522
/* FIXME: if this is out of range for either a 32-bit or
510523
64-bit float, it won't be noticed till the back-end */
511524
}
512525
}
513526
else {
514-
ret token.LIT_FLOAT(float_str);
527+
ret token.LIT_FLOAT(rdr.add_str(float_str));
515528
}
516529
}
517530

518531
auto maybe_exponent = scan_exponent(rdr);
519532
alt(maybe_exponent) {
520533
case(some[str](?s)) {
521-
ret token.LIT_FLOAT(dec_str + s);
534+
ret token.LIT_FLOAT(rdr.add_str(dec_str + s));
522535
}
523536
case(none[str]) {
524537
ret token.LIT_INT(accum_int);
@@ -594,7 +607,7 @@ fn next_token(reader rdr) -> token.token {
594607
fail;
595608
}
596609

597-
ret token.IDENT(accum_str);
610+
ret token.IDENT(rdr.add_str(accum_str));
598611
}
599612

600613
if (is_dec_digit(c)) {
@@ -786,7 +799,7 @@ fn next_token(reader rdr) -> token.token {
786799
rdr.bump();
787800
}
788801
rdr.bump();
789-
ret token.LIT_STR(accum_str);
802+
ret token.LIT_STR(rdr.add_str(accum_str));
790803
}
791804

792805
case ('-') {

‎src/comp/front/parser.rs

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ state type parser =
3838
fn next_def_id() -> ast.def_id;
3939
fn set_def(ast.def_num);
4040
fn get_prec_table() -> vec[op_spec];
41+
fn get_str(token.str_num) -> str;
42+
fn get_reader() -> lexer.reader;
4143
fn get_filemap() -> codemap.filemap;
4244
fn get_chpos() -> uint;
4345
};
@@ -111,6 +113,14 @@ fn new_parser(session.session sess,
111113
ret precs;
112114
}
113115

116+
fn get_str(token.str_num i) -> str {
117+
ret rdr.get_str(i);
118+
}
119+
120+
fn get_reader() -> lexer.reader {
121+
ret rdr;
122+
}
123+
114124
fn get_filemap() -> codemap.filemap {
115125
ret rdr.get_filemap();
116126
}
@@ -135,7 +145,7 @@ fn new_parser(session.session sess,
135145

136146
fn unexpected(parser p, token.token t) {
137147
let str s = "unexpected token: ";
138-
s += token.to_str(t);
148+
s += token.to_str(p.get_reader(), t);
139149
p.err(s);
140150
}
141151

@@ -144,9 +154,9 @@ fn expect(parser p, token.token t) {
144154
p.bump();
145155
} else {
146156
let str s = "expecting ";
147-
s += token.to_str(t);
157+
s += token.to_str(p.get_reader(), t);
148158
s += ", found ";
149-
s += token.to_str(p.peek());
159+
s += token.to_str(p.get_reader(), p.peek());
150160
p.err(s);
151161
}
152162
}
@@ -157,7 +167,7 @@ fn spanned[T](uint lo, uint hi, &T node) -> ast.spanned[T] {
157167

158168
fn parse_ident(parser p) -> ast.ident {
159169
alt (p.peek()) {
160-
case (token.IDENT(?i)) { p.bump(); ret i; }
170+
case (token.IDENT(?i)) { p.bump(); ret p.get_str(i); }
161171
case (_) {
162172
p.err("expecting ident");
163173
fail;
@@ -173,10 +183,10 @@ fn parse_ident(parser p) -> ast.ident {
173183
*/
174184
fn parse_str_lit_or_env_ident(parser p) -> ast.ident {
175185
alt (p.peek()) {
176-
case (token.LIT_STR(?s)) { p.bump(); ret s; }
186+
case (token.LIT_STR(?s)) { p.bump(); ret p.get_str(s); }
177187
case (token.IDENT(?i)) {
178188
auto v = eval.lookup(p.get_session(), p.get_env(),
179-
p.get_span(), i);
189+
p.get_span(), p.get_str(i));
180190
if (!eval.val_is_str(v)) {
181191
p.err("expecting string-valued variable");
182192
}
@@ -549,15 +559,15 @@ fn parse_lit(parser p) -> ast.lit {
549559
}
550560
case (token.LIT_FLOAT(?s)) {
551561
p.bump();
552-
lit = ast.lit_float(s);
562+
lit = ast.lit_float(p.get_str(s));
553563
}
554564
case (token.LIT_MACH_INT(?tm, ?i)) {
555565
p.bump();
556566
lit = ast.lit_mach_int(tm, i);
557567
}
558568
case (token.LIT_MACH_FLOAT(?tm, ?s)) {
559569
p.bump();
560-
lit = ast.lit_mach_float(tm, s);
570+
lit = ast.lit_mach_float(tm, p.get_str(s));
561571
}
562572
case (token.LIT_CHAR(?c)) {
563573
p.bump();
@@ -569,7 +579,7 @@ fn parse_lit(parser p) -> ast.lit {
569579
}
570580
case (token.LIT_STR(?s)) {
571581
p.bump();
572-
lit = ast.lit_str(s);
582+
lit = ast.lit_str(p.get_str(s));
573583
}
574584
case (?t) {
575585
unexpected(p, t);
@@ -617,7 +627,7 @@ fn parse_path(parser p, greed g) -> ast.path {
617627
alt (p.peek()) {
618628
case (token.IDENT(?i)) {
619629
hi = p.get_hi_pos();
620-
ids += vec(i);
630+
ids += vec(p.get_str(i));
621631
p.bump();
622632
if (p.peek() == token.DOT) {
623633
if (g == GREEDY) {
@@ -1025,7 +1035,7 @@ fn parse_dot_or_call_expr(parser p) -> @ast.expr {
10251035
case (token.IDENT(?i)) {
10261036
hi = p.get_hi_pos();
10271037
p.bump();
1028-
e = extend_expr_by_ident(p, lo, hi, e, i);
1038+
e = extend_expr_by_ident(p, lo, hi, e, p.get_str(i));
10291039
}
10301040

10311041
case (token.LPAREN) {
@@ -1373,7 +1383,7 @@ fn parse_alt_expr(parser p) -> @ast.expr {
13731383
case (token.RBRACE) { /* empty */ }
13741384
case (?tok) {
13751385
p.err("expected 'case' or '}' when parsing 'alt' statement " +
1376-
"but found " + token.to_str(tok));
1386+
"but found " + token.to_str(p.get_reader(), tok));
13771387
}
13781388
}
13791389
}
@@ -1483,16 +1493,17 @@ fn parse_pat(parser p) -> @ast.pat {
14831493
case (token.IDENT(?id)) {
14841494
hi = p.get_hi_pos();
14851495
p.bump();
1486-
pat = ast.pat_bind(id, p.next_def_id(), ast.ann_none);
1496+
pat = ast.pat_bind(p.get_str(id), p.next_def_id(),
1497+
ast.ann_none);
14871498
}
14881499
case (?tok) {
14891500
p.err("expected identifier after '?' in pattern but " +
1490-
"found " + token.to_str(tok));
1501+
"found " + token.to_str(p.get_reader(), tok));
14911502
fail;
14921503
}
14931504
}
14941505
}
1495-
case (token.IDENT(?id)) {
1506+
case (token.IDENT(_)) {
14961507
auto tag_path = parse_path(p, GREEDY);
14971508
hi = tag_path.span.hi;
14981509

@@ -1723,7 +1734,7 @@ fn parse_block(parser p) -> ast.block {
17231734
if (stmt_ends_with_semi(stmt)) {
17241735
p.err("expected ';' or '}' after " +
17251736
"expression but found " +
1726-
token.to_str(t));
1737+
token.to_str(p.get_reader(), t));
17271738
fail;
17281739
}
17291740
stmts += vec(stmt);
@@ -2102,13 +2113,14 @@ fn parse_item_tag(parser p) -> @ast.item {
21022113
expect(p, token.SEMI);
21032114

21042115
auto id = p.next_def_id();
2105-
auto vr = rec(name=name, args=args, id=id, ann=ast.ann_none);
2116+
auto vr = rec(name=p.get_str(name), args=args,
2117+
id=id, ann=ast.ann_none);
21062118
variants += vec(spanned[ast.variant_](vlo, vhi, vr));
21072119
}
21082120
case (token.RBRACE) { /* empty */ }
21092121
case (_) {
21102122
p.err("expected name of variant or '}' but found " +
2111-
token.to_str(tok));
2123+
token.to_str(p.get_reader(), tok));
21122124
}
21132125
}
21142126
}
@@ -2210,7 +2222,8 @@ fn parse_item(parser p) -> @ast.item {
22102222
ret parse_item_obj(p, lyr);
22112223
}
22122224
case (?t) {
2213-
p.err("expected item but found " + token.to_str(t));
2225+
p.err("expected item but found " +
2226+
token.to_str(p.get_reader(), t));
22142227
}
22152228
}
22162229
fail;
@@ -2224,7 +2237,8 @@ fn parse_meta_item(parser p) -> @ast.meta_item {
22242237
case (token.LIT_STR(?s)) {
22252238
auto hi = p.get_hi_pos();
22262239
p.bump();
2227-
ret @spanned(lo, hi, rec(name = ident, value = s));
2240+
ret @spanned(lo, hi, rec(name = ident,
2241+
value = p.get_str(s)));
22282242
}
22292243
case (_) {
22302244
p.err("Metadata items must be string literals");
@@ -2294,9 +2308,9 @@ fn parse_rest_import_name(parser p, ast.ident first,
22942308
fn parse_full_import_name(parser p, ast.ident def_ident)
22952309
-> @ast.view_item {
22962310
alt (p.peek()) {
2297-
case (token.IDENT(?ident)) {
2311+
case (token.IDENT(?i)) {
22982312
p.bump();
2299-
ret parse_rest_import_name(p, ident, some(def_ident));
2313+
ret parse_rest_import_name(p, p.get_str(i), some(def_ident));
23002314
}
23012315
case (_) {
23022316
p.err("expecting an identifier");
@@ -2308,15 +2322,16 @@ fn parse_full_import_name(parser p, ast.ident def_ident)
23082322
fn parse_import(parser p) -> @ast.view_item {
23092323
expect(p, token.IMPORT);
23102324
alt (p.peek()) {
2311-
case (token.IDENT(?ident)) {
2325+
case (token.IDENT(?i)) {
23122326
p.bump();
23132327
alt (p.peek()) {
23142328
case (token.EQ) {
23152329
p.bump();
2316-
ret parse_full_import_name(p, ident);
2330+
ret parse_full_import_name(p, p.get_str(i));
23172331
}
23182332
case (_) {
2319-
ret parse_rest_import_name(p, ident, none[ast.ident]);
2333+
ret parse_rest_import_name(p, p.get_str(i),
2334+
none[ast.ident]);
23202335
}
23212336
}
23222337
}

‎src/comp/front/token.rs

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import std.Int;
55
import std.UInt;
66
import std.Str;
77

8+
type str_num = uint;
9+
810
tag binop {
911
PLUS;
1012
MINUS;
@@ -127,14 +129,14 @@ tag token {
127129
LIT_INT(int);
128130
LIT_UINT(uint);
129131
LIT_MACH_INT(ty_mach, int);
130-
LIT_FLOAT(str);
131-
LIT_MACH_FLOAT(ty_mach, str);
132-
LIT_STR(str);
132+
LIT_FLOAT(str_num);
133+
LIT_MACH_FLOAT(ty_mach, str_num);
134+
LIT_STR(str_num);
133135
LIT_CHAR(char);
134136
LIT_BOOL(bool);
135137

136138
/* Name components */
137-
IDENT(str);
139+
IDENT(str_num);
138140
IDX(int);
139141
UNDERSCORE;
140142

@@ -168,7 +170,7 @@ tag token {
168170
PORT;
169171
TASK;
170172

171-
BRACEQUOTE(str);
173+
BRACEQUOTE(str_num);
172174
EOF;
173175
}
174176

@@ -188,7 +190,7 @@ fn binop_to_str(binop o) -> str {
188190
}
189191
}
190192
191-
fn to_str(token t) -> str {
193+
fn to_str(lexer.reader r, token t) -> str {
192194
alt (t) {
193195
194196
case (EQ) { ret "="; }
@@ -301,10 +303,14 @@ fn to_str(token t) -> str {
301303
ret Int.to_str(i, 10u)
302304
+ "_" + ty_mach_to_str(tm);
303305
}
304-
case (LIT_FLOAT(?s)) { ret s; }
306+
case (LIT_MACH_FLOAT(?tm, ?s)) {
307+
ret r.get_str(s) + "_" + ty_mach_to_str(tm);
308+
}
309+
310+
case (LIT_FLOAT(?s)) { ret r.get_str(s); }
305311
case (LIT_STR(?s)) {
306312
// FIXME: escape.
307-
ret "\"" + s + "\"";
313+
ret "\"" + r.get_str(s) + "\"";
308314
}
309315
case (LIT_CHAR(?c)) {
310316
// FIXME: escape.
@@ -319,7 +325,11 @@ fn to_str(token t) -> str {
319325
}
320326

321327
/* Name components */
322-
case (IDENT(?s)) { auto si = "ident:"; si += s; ret si; }
328+
case (IDENT(?s)) {
329+
auto si = "ident:";
330+
si += r.get_str(s);
331+
ret si;
332+
}
323333
case (IDX(?i)) { ret "_" + Int.to_str(i, 10u); }
324334
case (UNDERSCORE) { ret "_"; }
325335

@@ -360,7 +370,6 @@ fn to_str(token t) -> str {
360370
}
361371

362372

363-
364373
// Local Variables:
365374
// fill-column: 78;
366375
// indent-tabs-mode: nil

0 commit comments

Comments
 (0)