Skip to content

Commit 58c35ab

Browse files
MichaReiseryouknowone
MichaReiser
authored and
youknowone
committed
Replace row/column based Location with byte-offsets.
1 parent 7b8844b commit 58c35ab

File tree

131 files changed

+12125
-23203
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

131 files changed

+12125
-23203
lines changed

‎Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1212
resolver = "2"
1313
members = [
1414
"ast", "core", "literal", "parser",
15+
"ruff_text_size",
1516
]
1617

1718
[workspace.dependencies]

‎ast/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@ unparse = ["rustpython-literal"]
1616
[dependencies]
1717
rustpython-compiler-core = { path = "../core", version = "0.2.0" }
1818
rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
19+
ruff_text_size = { path = "../ruff_text_size" }
1920

2021
num-bigint = { workspace = true }

‎ast/asdl_rs.py

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ def visitModule(self, mod, depth):
352352
depth,
353353
)
354354
self.emit(
355-
"Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, end_location: node.end_location, node: f(folder, node.node)? })",
355+
"Ok(Located { custom: folder.map_user(node.custom)?, range: node.range, node: f(folder, node.node)? })",
356356
depth + 1,
357357
)
358358
self.emit("}", depth)
@@ -718,7 +718,7 @@ def write_ast_def(mod, typeinfo, f):
718718
#![allow(clippy::derive_partial_eq_without_eq)]
719719
720720
pub use crate::constant::*;
721-
pub use crate::Location;
721+
pub use ruff_text_size::{TextSize, TextRange};
722722
723723
type Ident = String;
724724
\n
@@ -730,26 +730,54 @@ def write_ast_def(mod, typeinfo, f):
730730
textwrap.dedent(
731731
"""
732732
pub struct Located<T, U = ()> {
733-
pub location: Location,
734-
pub end_location: Option<Location>,
733+
pub range: TextRange,
735734
pub custom: U,
736735
pub node: T,
737736
}
738737
739738
impl<T> Located<T> {
740-
pub fn new(location: Location, end_location: Location, node: T) -> Self {
741-
Self { location, end_location: Some(end_location), custom: (), node }
739+
pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
740+
Self { range: TextRange::new(start, end), custom: (), node }
742741
}
743742
744-
pub const fn start(&self) -> Location {
745-
self.location
743+
/// Creates a new node that spans the position specified by `range`.
744+
pub fn with_range(node: T, range: TextRange) -> Self {
745+
Self {
746+
range,
747+
custom: (),
748+
node,
749+
}
750+
}
751+
752+
/// Returns the absolute start position of the node from the beginning of the document.
753+
#[inline]
754+
pub const fn start(&self) -> TextSize {
755+
self.range.start()
756+
}
757+
758+
/// Returns the node
759+
#[inline]
760+
pub fn node(&self) -> &T {
761+
&self.node
762+
}
763+
764+
/// Consumes self and returns the node.
765+
#[inline]
766+
pub fn into_node(self) -> T {
767+
self.node
768+
}
769+
770+
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
771+
#[inline]
772+
pub const fn range(&self) -> TextRange {
773+
self.range
774+
}
775+
776+
/// Returns the absolute position at which the node ends in the source document.
777+
#[inline]
778+
pub const fn end(&self) -> TextSize {
779+
self.range.end()
746780
}
747-
748-
/// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if
749-
/// [`end_location`](Located::end_location) is `None`.
750-
pub fn end(&self) -> Location {
751-
self.end_location.unwrap_or(self.location)
752-
}
753781
}
754782
755783
impl<T, U> std::ops::Deref for Located<T, U> {

‎ast/src/ast_gen.rs

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,36 +3,63 @@
33
#![allow(clippy::derive_partial_eq_without_eq)]
44

55
pub use crate::constant::*;
6-
pub use crate::Location;
6+
pub use ruff_text_size::{TextRange, TextSize};
77

88
type Ident = String;
99

1010
#[derive(Clone, Debug, PartialEq)]
1111
pub struct Located<T, U = ()> {
12-
pub location: Location,
13-
pub end_location: Option<Location>,
12+
pub range: TextRange,
1413
pub custom: U,
1514
pub node: T,
1615
}
1716

1817
impl<T> Located<T> {
19-
pub fn new(location: Location, end_location: Location, node: T) -> Self {
18+
pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
2019
Self {
21-
location,
22-
end_location: Some(end_location),
20+
range: TextRange::new(start, end),
2321
custom: (),
2422
node,
2523
}
2624
}
2725

28-
pub const fn start(&self) -> Location {
29-
self.location
26+
/// Creates a new node that spans the position specified by `range`.
27+
pub fn with_range(node: T, range: TextRange) -> Self {
28+
Self {
29+
range,
30+
custom: (),
31+
node,
32+
}
33+
}
34+
35+
/// Returns the absolute start position of the node from the beginning of the document.
36+
#[inline]
37+
pub const fn start(&self) -> TextSize {
38+
self.range.start()
39+
}
40+
41+
/// Returns the node
42+
#[inline]
43+
pub fn node(&self) -> &T {
44+
&self.node
45+
}
46+
47+
/// Consumes self and returns the node.
48+
#[inline]
49+
pub fn into_node(self) -> T {
50+
self.node
51+
}
52+
53+
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
54+
#[inline]
55+
pub const fn range(&self) -> TextRange {
56+
self.range
3057
}
3158

32-
/// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if
33-
/// [`end_location`](Located::end_location) is `None`.
34-
pub fn end(&self) -> Location {
35-
self.end_location.unwrap_or(self.location)
59+
/// Returns the absolute position at which the node ends in the source document.
60+
#[inline]
61+
pub const fn end(&self) -> TextSize {
62+
self.range.end()
3663
}
3764
}
3865

@@ -1142,8 +1169,7 @@ pub mod fold {
11421169
) -> Result<Located<MT, F::TargetU>, F::Error> {
11431170
Ok(Located {
11441171
custom: folder.map_user(node.custom)?,
1145-
location: node.location,
1146-
end_location: node.end_location,
1172+
range: node.range,
11471173
node: f(folder, node.node)?,
11481174
})
11491175
}

‎ast/src/constant.rs

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
126126
Ok(crate::Expr {
127127
node: expr,
128128
custom: node.custom,
129-
location: node.location,
130-
end_location: node.end_location,
129+
range: node.range,
131130
})
132131
}
133132
_ => crate::fold::fold_expr(self, node),
@@ -144,19 +143,17 @@ mod tests {
144143
fn test_constant_opt() {
145144
use crate::{fold::Fold, *};
146145

147-
let start = Default::default();
148-
let end = None;
146+
let range = TextRange::default();
147+
#[allow(clippy::let_unit_value)]
149148
let custom = ();
150149
let ast = Located {
151-
location: start,
152-
end_location: end,
150+
range,
153151
custom,
154152
node: ExprTuple {
155153
ctx: ExprContext::Load,
156154
elts: vec![
157155
Located {
158-
location: start,
159-
end_location: end,
156+
range,
160157
custom,
161158
node: ExprConstant {
162159
value: BigInt::from(1).into(),
@@ -165,8 +162,7 @@ mod tests {
165162
.into(),
166163
},
167164
Located {
168-
location: start,
169-
end_location: end,
165+
range,
170166
custom,
171167
node: ExprConstant {
172168
value: BigInt::from(2).into(),
@@ -175,15 +171,13 @@ mod tests {
175171
.into(),
176172
},
177173
Located {
178-
location: start,
179-
end_location: end,
174+
range,
180175
custom,
181176
node: ExprTuple {
182177
ctx: ExprContext::Load,
183178
elts: vec![
184179
Located {
185-
location: start,
186-
end_location: end,
180+
range,
187181
custom,
188182
node: ExprConstant {
189183
value: BigInt::from(3).into(),
@@ -192,8 +186,7 @@ mod tests {
192186
.into(),
193187
},
194188
Located {
195-
location: start,
196-
end_location: end,
189+
range,
197190
custom,
198191
node: ExprConstant {
199192
value: BigInt::from(4).into(),
@@ -202,8 +195,7 @@ mod tests {
202195
.into(),
203196
},
204197
Located {
205-
location: start,
206-
end_location: end,
198+
range,
207199
custom,
208200
node: ExprConstant {
209201
value: BigInt::from(5).into(),
@@ -225,8 +217,7 @@ mod tests {
225217
assert_eq!(
226218
new_ast,
227219
Located {
228-
location: start,
229-
end_location: end,
220+
range,
230221
custom,
231222
node: ExprConstant {
232223
value: Constant::Tuple(vec![

‎ast/src/lib.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,5 @@ mod impls;
77
mod unparse;
88

99
pub use ast_gen::*;
10-
pub use rustpython_compiler_core::Location;
1110

1211
pub type Suite<U = ()> = Vec<Stmt<U>>;

‎core/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,11 @@ license = "MIT"
99

1010
[dependencies]
1111
bitflags = { workspace = true }
12-
bstr = { workspace = true }
1312
itertools = { workspace = true }
1413
num-bigint = { workspace = true }
1514
num-complex = { workspace = true }
1615
serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
16+
ruff_text_size = { path = "../ruff_text_size" }
1717

1818
lz4_flex = "0.9.2"
19+

‎core/src/error.rs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
use crate::Location;
1+
use ruff_text_size::TextSize;
22
use std::error::Error as StdError;
33
use std::fmt::Display;
44

55
#[derive(Debug, PartialEq, Eq)]
66
pub struct BaseError<T> {
77
pub error: T,
8-
pub location: Location,
8+
pub location: TextSize,
99
pub source_path: String,
1010
}
1111

@@ -31,7 +31,12 @@ where
3131
T: std::fmt::Display,
3232
{
3333
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
34-
self.location.fmt_with(f, &self.error)
34+
write!(
35+
f,
36+
"{} at byte offset {}",
37+
&self.error,
38+
u32::from(self.location)
39+
)
3540
}
3641
}
3742

‎parser/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ tiny-keccak = { version = "2", features = ["sha3"] }
2121
[dependencies]
2222
rustpython-ast = { path = "../ast", version = "0.2.0" }
2323
rustpython-compiler-core = { path = "../core", version = "0.2.0" }
24+
ruff_text_size = { path = "../ruff_text_size" }
2425

2526
ahash = { workspace = true }
2627
itertools = { workspace = true }

‎parser/src/function.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use crate::{
44
ast,
55
lexer::{LexicalError, LexicalErrorType},
66
};
7+
use ruff_text_size::TextSize;
78
use rustc_hash::FxHashSet;
89

910
pub(crate) struct ArgumentList {
@@ -83,10 +84,7 @@ pub(crate) fn parse_params(
8384
Ok((pos_only, names, defaults))
8485
}
8586

86-
type FunctionArgument = (
87-
Option<(ast::Location, ast::Location, Option<String>)>,
88-
ast::Expr,
89-
);
87+
type FunctionArgument = (Option<(TextSize, TextSize, Option<String>)>, ast::Expr);
9088

9189
// Parse arguments as supplied during a function/lambda *call*.
9290
pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {

0 commit comments

Comments
 (0)