Skip to content

Commit 3ea3d9e

Browse files
pcwaltonpcwalton
committed
rustc: Make the parser use the interner
1 parent b91ef94 commit 3ea3d9e

File tree

3 files changed

+31
-29
lines changed

3 files changed

+31
-29
lines changed

‎src/comp/front/lexer.rs

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import std::option::none;
1010
import driver::session::session;
1111
import util::common;
1212
import util::common::new_str_hash;
13+
import util::interner;
1314

1415
state type reader = state obj {
1516
fn is_eof() -> bool;
@@ -19,15 +20,15 @@ state type reader = state obj {
1920
fn bump();
2021
fn mark();
2122
fn get_mark_chpos() -> uint;
22-
fn add_str(str) -> token::str_num;
23-
fn get_str(token::str_num) -> str;
23+
fn get_interner() -> @interner::interner[str];
2424
fn get_chpos() -> uint;
2525
fn get_filemap() -> codemap::filemap;
2626
fn err(str m);
2727
};
2828

2929
fn new_reader(session sess, io::reader rdr,
30-
codemap::filemap filemap) -> reader {
30+
codemap::filemap filemap,
31+
@interner::interner[str] itr) -> reader {
3132

3233
state obj reader(session sess,
3334
str file,
@@ -37,7 +38,8 @@ fn new_reader(session sess, io::reader rdr,
3738
mutable uint mark_chpos,
3839
mutable uint chpos,
3940
mutable vec[str] strs,
40-
codemap::filemap fm) {
41+
codemap::filemap fm,
42+
@interner::interner[str] itr) {
4143

4244
fn is_eof() -> bool {
4345
ret ch == -1 as char;
@@ -78,14 +80,7 @@ fn new_reader(session sess, io::reader rdr,
7880
}
7981
}
8082

81-
fn add_str(str s) -> token::str_num {
82-
strs += vec(s);
83-
ret _vec::len[str](strs) - 1u;
84-
}
85-
86-
fn get_str(token::str_num i) -> str {
87-
ret strs.(i);
88-
}
83+
fn get_interner() -> @interner::interner[str] { ret itr; }
8984

9085
fn get_filemap() -> codemap::filemap {
9186
ret fm;
@@ -99,7 +94,7 @@ fn new_reader(session sess, io::reader rdr,
9994
let vec[str] strs = vec();
10095
auto rd = reader(sess, file, _str::byte_len(file), 0u, -1 as char,
10196
filemap.start_pos, filemap.start_pos,
102-
strs, filemap);
97+
strs, filemap, itr);
10398
rd.init();
10499
ret rd;
105100
}
@@ -387,28 +382,30 @@ fn scan_number(char c, reader rdr) -> token::token {
387382
if (c == '3' && n == '2') {
388383
rdr.bump(); rdr.bump();
389384
ret token::LIT_MACH_FLOAT(util::common::ty_f32,
390-
rdr.add_str(float_str));
385+
interner::intern[str](*rdr.get_interner(), float_str));
391386
}
392387
else if (c == '6' && n == '4') {
393388
rdr.bump(); rdr.bump();
394389
ret token::LIT_MACH_FLOAT(util::common::ty_f64,
395-
rdr.add_str(float_str));
390+
interner::intern[str](*rdr.get_interner(), float_str));
396391
/* FIXME: if this is out of range for either a 32-bit or
397392
64-bit float, it won't be noticed till the back-end */
398393
}
399394
}
400395
else {
401-
ret token::LIT_FLOAT(rdr.add_str(float_str));
396+
ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(),
397+
float_str));
402398
}
403399
}
404400

405401
auto maybe_exponent = scan_exponent(rdr);
406402
alt(maybe_exponent) {
407403
case(some[str](?s)) {
408-
ret token::LIT_FLOAT(rdr.add_str(dec_str + s));
404+
ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(),
405+
dec_str + s));
409406
}
410407
case(none[str]) {
411-
ret token::LIT_INT(accum_int);
408+
ret token::LIT_INT(accum_int);
412409
}
413410
}
414411
}
@@ -471,7 +468,8 @@ fn next_token(reader rdr) -> token::token {
471468
}
472469

473470
auto is_mod_name = c == ':' && rdr.next() == ':';
474-
ret token::IDENT(rdr.add_str(accum_str), is_mod_name);
471+
ret token::IDENT(interner::intern[str](*rdr.get_interner(),
472+
accum_str), is_mod_name);
475473
}
476474

477475
if (is_dec_digit(c)) {
@@ -671,7 +669,8 @@ fn next_token(reader rdr) -> token::token {
671669
rdr.bump();
672670
}
673671
rdr.bump();
674-
ret token::LIT_STR(rdr.add_str(accum_str));
672+
ret token::LIT_STR(interner::intern[str](*rdr.get_interner(),
673+
accum_str));
675674
}
676675

677676
case ('-') {
@@ -801,7 +800,8 @@ fn read_block_comment(reader rdr) -> cmnt {
801800

802801
fn gather_comments(session sess, str path) -> vec[cmnt] {
803802
auto srdr = io::file_reader(path);
804-
auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u));
803+
auto itr = @interner::mk_interner[str](_str::hash, _str::eq);
804+
auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u), itr);
805805
let vec[cmnt] comments = vec();
806806
while (!rdr.is_eof()) {
807807
while (true) {

‎src/comp/front/parser.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import util::common;
1111
import util::common::filename;
1212
import util::common::span;
1313
import util::common::new_str_hash;
14+
import util::interner;
1415

1516
tag restriction {
1617
UNRESTRICTED;
@@ -123,7 +124,7 @@ fn new_parser(session::session sess,
123124
}
124125

125126
fn get_str(token::str_num i) -> str {
126-
ret rdr.get_str(i);
127+
ret interner::get[str](*rdr.get_interner(), i);
127128
}
128129

129130
fn get_reader() -> lexer::reader {
@@ -156,7 +157,8 @@ fn new_parser(session::session sess,
156157
auto srdr = io::file_reader(path);
157158
auto filemap = codemap::new_filemap(path, pos);
158159
_vec::push[codemap::filemap](sess.get_codemap().files, filemap);
159-
auto rdr = lexer::new_reader(sess, srdr, filemap);
160+
auto itr = @interner::mk_interner[str](_str::hash, _str::eq);
161+
auto rdr = lexer::new_reader(sess, srdr, filemap, itr);
160162
// Make sure npos points at first actual token:
161163
lexer::consume_any_whitespace(rdr);
162164
auto npos = rdr.get_chpos();

‎src/comp/front/token.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import util::common::ty_mach;
22
import util::common::ty_mach_to_str;
33
import util::common::new_str_hash;
4+
import util::interner;
45
import std::_int;
56
import std::_uint;
67
import std::_str;
@@ -139,13 +140,14 @@ fn to_str(lexer::reader r, token t) -> str {
139140
+ "_" + ty_mach_to_str(tm);
140141
}
141142
case (LIT_MACH_FLOAT(?tm, ?s)) {
142-
ret r.get_str(s) + "_" + ty_mach_to_str(tm);
143+
ret interner::get[str](*r.get_interner(), s) + "_" +
144+
ty_mach_to_str(tm);
143145
}
144146

145-
case (LIT_FLOAT(?s)) { ret r.get_str(s); }
147+
case (LIT_FLOAT(?s)) { ret interner::get[str](*r.get_interner(), s); }
146148
case (LIT_STR(?s)) {
147149
// FIXME: escape.
148-
ret "\"" + r.get_str(s) + "\"";
150+
ret "\"" + interner::get[str](*r.get_interner(), s) + "\"";
149151
}
150152
case (LIT_CHAR(?c)) {
151153
// FIXME: escape.
@@ -160,9 +162,7 @@ fn to_str(lexer::reader r, token t) -> str {
160162
}
161163

162164
/* Name components */
163-
case (IDENT(?s, _)) {
164-
ret r.get_str(s);
165-
}
165+
case (IDENT(?s, _)) { ret interner::get[str](*r.get_interner(), s); }
166166
case (IDX(?i)) { ret "_" + _int::to_str(i, 10u); }
167167
case (UNDERSCORE) { ret "_"; }
168168

0 commit comments

Comments
 (0)