Skip to content

Commit 16bdbf5

Browse files
graydongraydon
committed
---
yaml --- r: 2428 b: refs/heads/master c: b00f307 h: refs/heads/master v: v3
1 parent 0363268 commit 16bdbf5

File tree

4 files changed

+80
-43
lines changed

4 files changed

+80
-43
lines changed

‎[refs]

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
---
2-
refs/heads/master: 70c759030ccf27222a04f918e235bc9dbcf88c94
2+
refs/heads/master: b00f3074d4b4d68b3233a7f062ea13384fdccf2c

‎trunk/src/comp/front/lexer.rs

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ state type reader = state obj {
1818
fn bump();
1919
fn mark();
2020
fn get_mark_chpos() -> uint;
21+
fn add_str(str) -> token.str_num;
22+
fn get_str(token.str_num) -> str;
2123
fn get_chpos() -> uint;
2224
fn get_keywords() -> hashmap[str,token.token];
2325
fn get_reserved() -> hashmap[str,()];
@@ -32,6 +34,7 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
3234
mutable char ch,
3335
mutable uint mark_chpos,
3436
mutable uint chpos,
37+
mutable vec[str] strs,
3538
hashmap[str,token.token] keywords,
3639
hashmap[str,()] reserved,
3740
codemap.filemap fm) {
@@ -79,6 +82,15 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
7982
ret keywords;
8083
}
8184

85+
fn add_str(str s) -> token.str_num {
86+
strs += vec(s);
87+
ret Vec.len[str](strs) - 1u;
88+
}
89+
90+
fn get_str(token.str_num i) -> str {
91+
ret strs.(i);
92+
}
93+
8294
fn get_reserved() -> hashmap[str,()] {
8395
ret reserved;
8496
}
@@ -88,9 +100,10 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
88100
}
89101
}
90102
auto file = Str.unsafe_from_bytes(rdr.read_whole_stream());
103+
let vec[str] strs = vec();
91104
auto rd = reader(file, Str.byte_len(file), 0u, -1 as char,
92105
filemap.start_pos, filemap.start_pos,
93-
keyword_table(),
106+
strs, keyword_table(),
94107
reserved_word_table(),
95108
filemap);
96109
rd.init();
@@ -500,25 +513,25 @@ fn scan_number(char c, reader rdr) -> token.token {
500513
if (c == '3' && n == '2') {
501514
rdr.bump(); rdr.bump();
502515
ret token.LIT_MACH_FLOAT(util.common.ty_f32,
503-
float_str);
516+
rdr.add_str(float_str));
504517
}
505518
else if (c == '6' && n == '4') {
506519
rdr.bump(); rdr.bump();
507520
ret token.LIT_MACH_FLOAT(util.common.ty_f64,
508-
float_str);
521+
rdr.add_str(float_str));
509522
/* FIXME: if this is out of range for either a 32-bit or
510523
64-bit float, it won't be noticed till the back-end */
511524
}
512525
}
513526
else {
514-
ret token.LIT_FLOAT(float_str);
527+
ret token.LIT_FLOAT(rdr.add_str(float_str));
515528
}
516529
}
517530

518531
auto maybe_exponent = scan_exponent(rdr);
519532
alt(maybe_exponent) {
520533
case(some[str](?s)) {
521-
ret token.LIT_FLOAT(dec_str + s);
534+
ret token.LIT_FLOAT(rdr.add_str(dec_str + s));
522535
}
523536
case(none[str]) {
524537
ret token.LIT_INT(accum_int);
@@ -594,7 +607,7 @@ fn next_token(reader rdr) -> token.token {
594607
fail;
595608
}
596609

597-
ret token.IDENT(accum_str);
610+
ret token.IDENT(rdr.add_str(accum_str));
598611
}
599612

600613
if (is_dec_digit(c)) {
@@ -786,7 +799,7 @@ fn next_token(reader rdr) -> token.token {
786799
rdr.bump();
787800
}
788801
rdr.bump();
789-
ret token.LIT_STR(accum_str);
802+
ret token.LIT_STR(rdr.add_str(accum_str));
790803
}
791804

792805
case ('-') {

‎trunk/src/comp/front/parser.rs

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ state type parser =
3838
fn next_def_id() -> ast.def_id;
3939
fn set_def(ast.def_num);
4040
fn get_prec_table() -> vec[op_spec];
41+
fn get_str(token.str_num) -> str;
42+
fn get_reader() -> lexer.reader;
4143
fn get_filemap() -> codemap.filemap;
4244
fn get_chpos() -> uint;
4345
};
@@ -111,6 +113,14 @@ fn new_parser(session.session sess,
111113
ret precs;
112114
}
113115

116+
fn get_str(token.str_num i) -> str {
117+
ret rdr.get_str(i);
118+
}
119+
120+
fn get_reader() -> lexer.reader {
121+
ret rdr;
122+
}
123+
114124
fn get_filemap() -> codemap.filemap {
115125
ret rdr.get_filemap();
116126
}
@@ -135,7 +145,7 @@ fn new_parser(session.session sess,
135145

136146
fn unexpected(parser p, token.token t) {
137147
let str s = "unexpected token: ";
138-
s += token.to_str(t);
148+
s += token.to_str(p.get_reader(), t);
139149
p.err(s);
140150
}
141151

@@ -144,9 +154,9 @@ fn expect(parser p, token.token t) {
144154
p.bump();
145155
} else {
146156
let str s = "expecting ";
147-
s += token.to_str(t);
157+
s += token.to_str(p.get_reader(), t);
148158
s += ", found ";
149-
s += token.to_str(p.peek());
159+
s += token.to_str(p.get_reader(), p.peek());
150160
p.err(s);
151161
}
152162
}
@@ -157,7 +167,7 @@ fn spanned[T](uint lo, uint hi, &T node) -> ast.spanned[T] {
157167

158168
fn parse_ident(parser p) -> ast.ident {
159169
alt (p.peek()) {
160-
case (token.IDENT(?i)) { p.bump(); ret i; }
170+
case (token.IDENT(?i)) { p.bump(); ret p.get_str(i); }
161171
case (_) {
162172
p.err("expecting ident");
163173
fail;
@@ -173,10 +183,10 @@ fn parse_ident(parser p) -> ast.ident {
173183
*/
174184
fn parse_str_lit_or_env_ident(parser p) -> ast.ident {
175185
alt (p.peek()) {
176-
case (token.LIT_STR(?s)) { p.bump(); ret s; }
186+
case (token.LIT_STR(?s)) { p.bump(); ret p.get_str(s); }
177187
case (token.IDENT(?i)) {
178188
auto v = eval.lookup(p.get_session(), p.get_env(),
179-
p.get_span(), i);
189+
p.get_span(), p.get_str(i));
180190
if (!eval.val_is_str(v)) {
181191
p.err("expecting string-valued variable");
182192
}
@@ -549,15 +559,15 @@ fn parse_lit(parser p) -> ast.lit {
549559
}
550560
case (token.LIT_FLOAT(?s)) {
551561
p.bump();
552-
lit = ast.lit_float(s);
562+
lit = ast.lit_float(p.get_str(s));
553563
}
554564
case (token.LIT_MACH_INT(?tm, ?i)) {
555565
p.bump();
556566
lit = ast.lit_mach_int(tm, i);
557567
}
558568
case (token.LIT_MACH_FLOAT(?tm, ?s)) {
559569
p.bump();
560-
lit = ast.lit_mach_float(tm, s);
570+
lit = ast.lit_mach_float(tm, p.get_str(s));
561571
}
562572
case (token.LIT_CHAR(?c)) {
563573
p.bump();
@@ -569,7 +579,7 @@ fn parse_lit(parser p) -> ast.lit {
569579
}
570580
case (token.LIT_STR(?s)) {
571581
p.bump();
572-
lit = ast.lit_str(s);
582+
lit = ast.lit_str(p.get_str(s));
573583
}
574584
case (?t) {
575585
unexpected(p, t);
@@ -617,7 +627,7 @@ fn parse_path(parser p, greed g) -> ast.path {
617627
alt (p.peek()) {
618628
case (token.IDENT(?i)) {
619629
hi = p.get_hi_pos();
620-
ids += vec(i);
630+
ids += vec(p.get_str(i));
621631
p.bump();
622632
if (p.peek() == token.DOT) {
623633
if (g == GREEDY) {
@@ -1025,7 +1035,7 @@ fn parse_dot_or_call_expr(parser p) -> @ast.expr {
10251035
case (token.IDENT(?i)) {
10261036
hi = p.get_hi_pos();
10271037
p.bump();
1028-
e = extend_expr_by_ident(p, lo, hi, e, i);
1038+
e = extend_expr_by_ident(p, lo, hi, e, p.get_str(i));
10291039
}
10301040

10311041
case (token.LPAREN) {
@@ -1373,7 +1383,7 @@ fn parse_alt_expr(parser p) -> @ast.expr {
13731383
case (token.RBRACE) { /* empty */ }
13741384
case (?tok) {
13751385
p.err("expected 'case' or '}' when parsing 'alt' statement " +
1376-
"but found " + token.to_str(tok));
1386+
"but found " + token.to_str(p.get_reader(), tok));
13771387
}
13781388
}
13791389
}
@@ -1483,16 +1493,17 @@ fn parse_pat(parser p) -> @ast.pat {
14831493
case (token.IDENT(?id)) {
14841494
hi = p.get_hi_pos();
14851495
p.bump();
1486-
pat = ast.pat_bind(id, p.next_def_id(), ast.ann_none);
1496+
pat = ast.pat_bind(p.get_str(id), p.next_def_id(),
1497+
ast.ann_none);
14871498
}
14881499
case (?tok) {
14891500
p.err("expected identifier after '?' in pattern but " +
1490-
"found " + token.to_str(tok));
1501+
"found " + token.to_str(p.get_reader(), tok));
14911502
fail;
14921503
}
14931504
}
14941505
}
1495-
case (token.IDENT(?id)) {
1506+
case (token.IDENT(_)) {
14961507
auto tag_path = parse_path(p, GREEDY);
14971508
hi = tag_path.span.hi;
14981509

@@ -1723,7 +1734,7 @@ fn parse_block(parser p) -> ast.block {
17231734
if (stmt_ends_with_semi(stmt)) {
17241735
p.err("expected ';' or '}' after " +
17251736
"expression but found " +
1726-
token.to_str(t));
1737+
token.to_str(p.get_reader(), t));
17271738
fail;
17281739
}
17291740
stmts += vec(stmt);
@@ -2102,13 +2113,14 @@ fn parse_item_tag(parser p) -> @ast.item {
21022113
expect(p, token.SEMI);
21032114

21042115
auto id = p.next_def_id();
2105-
auto vr = rec(name=name, args=args, id=id, ann=ast.ann_none);
2116+
auto vr = rec(name=p.get_str(name), args=args,
2117+
id=id, ann=ast.ann_none);
21062118
variants += vec(spanned[ast.variant_](vlo, vhi, vr));
21072119
}
21082120
case (token.RBRACE) { /* empty */ }
21092121
case (_) {
21102122
p.err("expected name of variant or '}' but found " +
2111-
token.to_str(tok));
2123+
token.to_str(p.get_reader(), tok));
21122124
}
21132125
}
21142126
}
@@ -2210,7 +2222,8 @@ fn parse_item(parser p) -> @ast.item {
22102222
ret parse_item_obj(p, lyr);
22112223
}
22122224
case (?t) {
2213-
p.err("expected item but found " + token.to_str(t));
2225+
p.err("expected item but found " +
2226+
token.to_str(p.get_reader(), t));
22142227
}
22152228
}
22162229
fail;
@@ -2224,7 +2237,8 @@ fn parse_meta_item(parser p) -> @ast.meta_item {
22242237
case (token.LIT_STR(?s)) {
22252238
auto hi = p.get_hi_pos();
22262239
p.bump();
2227-
ret @spanned(lo, hi, rec(name = ident, value = s));
2240+
ret @spanned(lo, hi, rec(name = ident,
2241+
value = p.get_str(s)));
22282242
}
22292243
case (_) {
22302244
p.err("Metadata items must be string literals");
@@ -2294,9 +2308,9 @@ fn parse_rest_import_name(parser p, ast.ident first,
22942308
fn parse_full_import_name(parser p, ast.ident def_ident)
22952309
-> @ast.view_item {
22962310
alt (p.peek()) {
2297-
case (token.IDENT(?ident)) {
2311+
case (token.IDENT(?i)) {
22982312
p.bump();
2299-
ret parse_rest_import_name(p, ident, some(def_ident));
2313+
ret parse_rest_import_name(p, p.get_str(i), some(def_ident));
23002314
}
23012315
case (_) {
23022316
p.err("expecting an identifier");
@@ -2308,15 +2322,16 @@ fn parse_full_import_name(parser p, ast.ident def_ident)
23082322
fn parse_import(parser p) -> @ast.view_item {
23092323
expect(p, token.IMPORT);
23102324
alt (p.peek()) {
2311-
case (token.IDENT(?ident)) {
2325+
case (token.IDENT(?i)) {
23122326
p.bump();
23132327
alt (p.peek()) {
23142328
case (token.EQ) {
23152329
p.bump();
2316-
ret parse_full_import_name(p, ident);
2330+
ret parse_full_import_name(p, p.get_str(i));
23172331
}
23182332
case (_) {
2319-
ret parse_rest_import_name(p, ident, none[ast.ident]);
2333+
ret parse_rest_import_name(p, p.get_str(i),
2334+
none[ast.ident]);
23202335
}
23212336
}
23222337
}

0 commit comments

Comments
 (0)