Skip to content

Commit 46f48d3

Browse files
petrochenkovpetrochenkov
committed
rustc_ast: Stop using "string typing" for doc comment tokens
Explicitly store their kind and style retrieved during lexing in the token
1 parent c15bae5 commit 46f48d3

File tree

20 files changed

+172
-159
lines changed

20 files changed

+172
-159
lines changed

‎src/librustc_ast/ast.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ pub use GenericArgs::*;
2323
pub use UnsafeSource::*;
2424

2525
use crate::ptr::P;
26-
use crate::token::{self, DelimToken};
26+
use crate::token::{self, CommentKind, DelimToken};
2727
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
2828

2929
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -2365,7 +2365,7 @@ pub enum AttrKind {
23652365
/// A doc comment (e.g. `/// ...`, `//! ...`, `/** ... */`, `/*! ... */`).
23662366
/// Doc attributes (e.g. `#[doc="..."]`) are represented with the `Normal`
23672367
/// variant (which is much less compact and thus more expensive).
2368-
DocComment(Symbol),
2368+
DocComment(CommentKind, Symbol),
23692369
}
23702370

23712371
/// `TraitRef`s appear in impls.

‎src/librustc_ast/attr/mod.rs

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem};
77
use crate::ast::{Path, PathSegment};
88
use crate::mut_visit::visit_clobber;
99
use crate::ptr::P;
10-
use crate::token::{self, Token};
10+
use crate::token::{self, CommentKind, Token};
1111
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
1212

1313
use rustc_data_structures::sync::Lock;
@@ -169,7 +169,7 @@ impl Attribute {
169169
pub fn has_name(&self, name: Symbol) -> bool {
170170
match self.kind {
171171
AttrKind::Normal(ref item) => item.path == name,
172-
AttrKind::DocComment(_) => false,
172+
AttrKind::DocComment(..) => false,
173173
}
174174
}
175175

@@ -198,7 +198,7 @@ impl Attribute {
198198
None
199199
}
200200
}
201-
AttrKind::DocComment(_) => None,
201+
AttrKind::DocComment(..) => None,
202202
}
203203
}
204204
pub fn name_or_empty(&self) -> Symbol {
@@ -218,7 +218,7 @@ impl Attribute {
218218
Some(MetaItem { kind: MetaItemKind::List(list), .. }) => Some(list),
219219
_ => None,
220220
},
221-
AttrKind::DocComment(_) => None,
221+
AttrKind::DocComment(..) => None,
222222
}
223223
}
224224

@@ -314,13 +314,13 @@ impl Attribute {
314314
pub fn is_doc_comment(&self) -> bool {
315315
match self.kind {
316316
AttrKind::Normal(_) => false,
317-
AttrKind::DocComment(_) => true,
317+
AttrKind::DocComment(..) => true,
318318
}
319319
}
320320

321321
pub fn doc_str(&self) -> Option<Symbol> {
322322
match self.kind {
323-
AttrKind::DocComment(symbol) => Some(symbol),
323+
AttrKind::DocComment(.., data) => Some(data),
324324
AttrKind::Normal(ref item) if item.path == sym::doc => {
325325
item.meta(self.span).and_then(|meta| meta.value_str())
326326
}
@@ -331,14 +331,14 @@ impl Attribute {
331331
pub fn get_normal_item(&self) -> &AttrItem {
332332
match self.kind {
333333
AttrKind::Normal(ref item) => item,
334-
AttrKind::DocComment(_) => panic!("unexpected doc comment"),
334+
AttrKind::DocComment(..) => panic!("unexpected doc comment"),
335335
}
336336
}
337337

338338
pub fn unwrap_normal_item(self) -> AttrItem {
339339
match self.kind {
340340
AttrKind::Normal(item) => item,
341-
AttrKind::DocComment(_) => panic!("unexpected doc comment"),
341+
AttrKind::DocComment(..) => panic!("unexpected doc comment"),
342342
}
343343
}
344344

@@ -405,8 +405,13 @@ pub fn mk_attr_outer(item: MetaItem) -> Attribute {
405405
mk_attr(AttrStyle::Outer, item.path, item.kind.mac_args(item.span), item.span)
406406
}
407407

408-
pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute {
409-
Attribute { kind: AttrKind::DocComment(comment), id: mk_attr_id(), style, span }
408+
pub fn mk_doc_comment(
409+
comment_kind: CommentKind,
410+
style: AttrStyle,
411+
data: Symbol,
412+
span: Span,
413+
) -> Attribute {
414+
Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span }
410415
}
411416

412417
pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {

‎src/librustc_ast/mut_visit.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -582,7 +582,7 @@ pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
582582
vis.visit_path(path);
583583
visit_mac_args(args, vis);
584584
}
585-
AttrKind::DocComment(_) => {}
585+
AttrKind::DocComment(..) => {}
586586
}
587587
vis.visit_span(span);
588588
}

‎src/librustc_ast/token.rs

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,12 @@ use rustc_span::{self, Span, DUMMY_SP};
1717
use std::borrow::Cow;
1818
use std::{fmt, mem};
1919

20+
#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
21+
pub enum CommentKind {
22+
Line,
23+
Block,
24+
}
25+
2026
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
2127
#[derive(HashStable_Generic)]
2228
pub enum BinOpToken {
@@ -238,9 +244,10 @@ pub enum TokenKind {
238244

239245
Interpolated(Lrc<Nonterminal>),
240246

241-
// Can be expanded into several tokens.
242-
/// A doc comment.
243-
DocComment(Symbol),
247+
/// A doc comment token.
248+
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
249+
/// similarly to symbols in string literal tokens.
250+
DocComment(CommentKind, ast::AttrStyle, Symbol),
244251

245252
// Junk. These carry no data because we don't really care about the data
246253
// they *would* carry, and don't really want to allocate a new ident for

‎src/librustc_ast/util/comments.rs

Lines changed: 51 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
pub use CommentStyle::*;
22

3-
use crate::ast;
3+
use crate::ast::AttrStyle;
4+
use crate::token::CommentKind;
45
use rustc_span::source_map::SourceMap;
56
use rustc_span::{BytePos, CharPos, FileName, Pos, Symbol};
67

7-
use log::debug;
8-
98
#[cfg(test)]
109
mod tests;
1110

@@ -28,43 +27,46 @@ pub struct Comment {
2827
pub pos: BytePos,
2928
}
3029

31-
pub fn is_line_doc_comment(s: &str) -> bool {
32-
let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/')
33-
|| s.starts_with("//!");
34-
debug!("is {:?} a doc comment? {}", s, res);
35-
res
36-
}
37-
38-
pub fn is_block_doc_comment(s: &str) -> bool {
39-
// Prevent `/**/` from being parsed as a doc comment
40-
let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*')
41-
|| s.starts_with("/*!"))
42-
&& s.len() >= 5;
43-
debug!("is {:?} a doc comment? {}", s, res);
44-
res
45-
}
46-
47-
// FIXME(#64197): Try to privatize this again.
48-
pub fn is_doc_comment(s: &str) -> bool {
49-
(s.starts_with("///") && is_line_doc_comment(s))
50-
|| s.starts_with("//!")
51-
|| (s.starts_with("/**") && is_block_doc_comment(s))
52-
|| s.starts_with("/*!")
30+
/// For a full line comment string returns its doc comment style if it's a doc comment
31+
/// and returns `None` if it's a regular comment.
32+
pub fn line_doc_comment_style(line_comment: &str) -> Option<AttrStyle> {
33+
let line_comment = line_comment.as_bytes();
34+
assert!(line_comment.starts_with(b"//"));
35+
match line_comment.get(2) {
36+
// `//!` is an inner line doc comment.
37+
Some(b'!') => Some(AttrStyle::Inner),
38+
Some(b'/') => match line_comment.get(3) {
39+
// `////` (more than 3 slashes) is not considered a doc comment.
40+
Some(b'/') => None,
41+
// Otherwise `///` is an outer line doc comment.
42+
_ => Some(AttrStyle::Outer),
43+
},
44+
_ => None,
45+
}
5346
}
5447

55-
pub fn doc_comment_style(comment: Symbol) -> ast::AttrStyle {
56-
let comment = &comment.as_str();
57-
assert!(is_doc_comment(comment));
58-
if comment.starts_with("//!") || comment.starts_with("/*!") {
59-
ast::AttrStyle::Inner
60-
} else {
61-
ast::AttrStyle::Outer
48+
/// For a full block comment string returns its doc comment style if it's a doc comment
49+
/// and returns `None` if it's a regular comment.
50+
pub fn block_doc_comment_style(block_comment: &str, terminated: bool) -> Option<AttrStyle> {
51+
let block_comment = block_comment.as_bytes();
52+
assert!(block_comment.starts_with(b"/*"));
53+
assert!(!terminated || block_comment.ends_with(b"*/"));
54+
match block_comment.get(2) {
55+
// `/*!` is an inner block doc comment.
56+
Some(b'!') => Some(AttrStyle::Inner),
57+
Some(b'*') => match block_comment.get(3) {
58+
// `/***` (more than 2 stars) is not considered a doc comment.
59+
Some(b'*') => None,
60+
// `/**/` is not considered a doc comment.
61+
Some(b'/') if block_comment.len() == 4 => None,
62+
// Otherwise `/**` is an outer block doc comment.
63+
_ => Some(AttrStyle::Outer),
64+
},
65+
_ => None,
6266
}
6367
}
6468

65-
pub fn strip_doc_comment_decoration(comment: Symbol) -> String {
66-
let comment = &comment.as_str();
67-
69+
pub fn strip_doc_comment_decoration(data: Symbol, comment_kind: CommentKind) -> String {
6870
/// remove whitespace-only lines from the start/end of lines
6971
fn vertical_trim(lines: Vec<String>) -> Vec<String> {
7072
let mut i = 0;
@@ -126,26 +128,19 @@ pub fn strip_doc_comment_decoration(comment: Symbol) -> String {
126128
}
127129
}
128130

129-
// one-line comments lose their prefix
130-
const ONELINERS: &[&str] = &["///!", "///", "//!", "//"];
131-
132-
for prefix in ONELINERS {
133-
if comment.starts_with(*prefix) {
134-
return (&comment[prefix.len()..]).to_string();
131+
match comment_kind {
132+
CommentKind::Line => {
133+
let data = data.as_str();
134+
let prefix_len = if data.starts_with('!') { 1 } else { 0 };
135+
data[prefix_len..].to_string()
136+
}
137+
CommentKind::Block => {
138+
let lines = data.as_str().lines().map(|s| s.to_string()).collect::<Vec<String>>();
139+
let lines = vertical_trim(lines);
140+
let lines = horizontal_trim(lines);
141+
lines.join("\n")
135142
}
136143
}
137-
138-
if comment.starts_with("/*") {
139-
let lines =
140-
comment[3..comment.len() - 2].lines().map(|s| s.to_string()).collect::<Vec<String>>();
141-
142-
let lines = vertical_trim(lines);
143-
let lines = horizontal_trim(lines);
144-
145-
return lines.join("\n");
146-
}
147-
148-
panic!("not a doc-comment: {}", comment);
149144
}
150145

151146
/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char.
@@ -226,8 +221,8 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec<Comme
226221
}
227222
}
228223
}
229-
rustc_lexer::TokenKind::BlockComment { terminated: _ } => {
230-
if !is_block_doc_comment(token_text) {
224+
rustc_lexer::TokenKind::BlockComment { terminated } => {
225+
if block_doc_comment_style(token_text, terminated).is_none() {
231226
let code_to_the_right = match text[pos + token.len..].chars().next() {
232227
Some('\r' | '\n') => false,
233228
_ => true,
@@ -249,7 +244,7 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec<Comme
249244
}
250245
}
251246
rustc_lexer::TokenKind::LineComment => {
252-
if !is_doc_comment(token_text) {
247+
if line_doc_comment_style(token_text).is_none() {
253248
comments.push(Comment {
254249
style: if code_to_the_left { Trailing } else { Isolated },
255250
lines: vec![token_text.to_string()],

‎src/librustc_ast/util/comments/tests.rs

Lines changed: 17 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,50 @@
11
use super::*;
22
use crate::with_default_session_globals;
33

4+
#[test]
5+
fn line_doc_comments() {
6+
assert!(line_doc_comment_style("///").is_some());
7+
assert!(line_doc_comment_style("/// blah").is_some());
8+
assert!(line_doc_comment_style("////").is_none());
9+
}
10+
411
#[test]
512
fn test_block_doc_comment_1() {
613
with_default_session_globals(|| {
7-
let comment = "/**\n * Test \n ** Test\n * Test\n*/";
8-
let stripped = strip_doc_comment_decoration(Symbol::intern(comment));
14+
let comment = "\n * Test \n ** Test\n * Test\n";
15+
let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block);
916
assert_eq!(stripped, " Test \n* Test\n Test");
1017
})
1118
}
1219

1320
#[test]
1421
fn test_block_doc_comment_2() {
1522
with_default_session_globals(|| {
16-
let comment = "/**\n * Test\n * Test\n*/";
17-
let stripped = strip_doc_comment_decoration(Symbol::intern(comment));
23+
let comment = "\n * Test\n * Test\n";
24+
let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block);
1825
assert_eq!(stripped, " Test\n Test");
1926
})
2027
}
2128

2229
#[test]
2330
fn test_block_doc_comment_3() {
2431
with_default_session_globals(|| {
25-
let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
26-
let stripped = strip_doc_comment_decoration(Symbol::intern(comment));
32+
let comment = "\n let a: *i32;\n *a = 5;\n";
33+
let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block);
2734
assert_eq!(stripped, " let a: *i32;\n *a = 5;");
2835
})
2936
}
3037

31-
#[test]
32-
fn test_block_doc_comment_4() {
33-
with_default_session_globals(|| {
34-
let comment = "/*******************\n test\n *********************/";
35-
let stripped = strip_doc_comment_decoration(Symbol::intern(comment));
36-
assert_eq!(stripped, " test");
37-
})
38-
}
39-
4038
#[test]
4139
fn test_line_doc_comment() {
4240
with_default_session_globals(|| {
43-
let stripped = strip_doc_comment_decoration(Symbol::intern("/// test"));
41+
let stripped = strip_doc_comment_decoration(Symbol::intern(" test"), CommentKind::Line);
4442
assert_eq!(stripped, " test");
45-
let stripped = strip_doc_comment_decoration(Symbol::intern("///! test"));
43+
let stripped = strip_doc_comment_decoration(Symbol::intern("! test"), CommentKind::Line);
4644
assert_eq!(stripped, " test");
47-
let stripped = strip_doc_comment_decoration(Symbol::intern("// test"));
48-
assert_eq!(stripped, " test");
49-
let stripped = strip_doc_comment_decoration(Symbol::intern("// test"));
50-
assert_eq!(stripped, " test");
51-
let stripped = strip_doc_comment_decoration(Symbol::intern("///test"));
52-
assert_eq!(stripped, "test");
53-
let stripped = strip_doc_comment_decoration(Symbol::intern("///!test"));
45+
let stripped = strip_doc_comment_decoration(Symbol::intern("test"), CommentKind::Line);
5446
assert_eq!(stripped, "test");
55-
let stripped = strip_doc_comment_decoration(Symbol::intern("//test"));
47+
let stripped = strip_doc_comment_decoration(Symbol::intern("!test"), CommentKind::Line);
5648
assert_eq!(stripped, "test");
5749
})
5850
}

‎src/librustc_ast/visit.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -880,7 +880,7 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
880880
pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) {
881881
match attr.kind {
882882
AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args),
883-
AttrKind::DocComment(_) => {}
883+
AttrKind::DocComment(..) => {}
884884
}
885885
}
886886

‎src/librustc_ast_lowering/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -981,7 +981,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
981981
path: item.path.clone(),
982982
args: self.lower_mac_args(&item.args),
983983
}),
984-
AttrKind::DocComment(comment) => AttrKind::DocComment(comment),
984+
AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
985985
};
986986

987987
Attribute { kind, id: attr.id, style: attr.style, span: attr.span }

0 commit comments

Comments
 (0)