Skip to content

Commit b526da9

Browse files
BoshenBoshen
committed
refactor(parser): make Token fields private (#10936)
The goal of this PR is to reduce the diff for #10933
1 parent 0961296 commit b526da9

27 files changed

Lines changed: 385 additions & 136 deletions

‎crates/oxc_parser/src/cursor.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ pub struct ParserCheckpoint<'a> {
2222
impl<'a> ParserImpl<'a> {
2323
#[inline]
2424
pub(crate) fn start_span(&self) -> u32 {
25-
self.token.start
25+
self.token.start()
2626
}
2727

2828
#[inline]
@@ -39,7 +39,7 @@ impl<'a> ParserImpl<'a> {
3939
/// Get current Kind
4040
#[inline]
4141
pub(crate) fn cur_kind(&self) -> Kind {
42-
self.token.kind
42+
self.token.kind()
4343
}
4444

4545
/// Get current source text
@@ -58,7 +58,7 @@ impl<'a> ParserImpl<'a> {
5858

5959
/// Get current template string
6060
pub(crate) fn cur_template_string(&self) -> Option<&'a str> {
61-
self.lexer.get_template_string(self.token.start)
61+
self.lexer.get_template_string(self.token.start())
6262
}
6363

6464
/// Peek next token, returns EOF for final peek
@@ -70,13 +70,13 @@ impl<'a> ParserImpl<'a> {
7070
/// Peek next kind, returns EOF for final peek
7171
#[inline]
7272
pub(crate) fn peek_kind(&mut self) -> Kind {
73-
self.peek_token().kind
73+
self.peek_token().kind()
7474
}
7575

7676
/// Peek at kind
7777
#[inline]
7878
pub(crate) fn peek_at(&mut self, kind: Kind) -> bool {
79-
self.peek_token().kind == kind
79+
self.peek_token().kind() == kind
8080
}
8181

8282
/// Peek nth token
@@ -91,13 +91,13 @@ impl<'a> ParserImpl<'a> {
9191
/// Peek at nth kind
9292
#[inline]
9393
pub(crate) fn nth_at(&mut self, n: u8, kind: Kind) -> bool {
94-
self.nth(n).kind == kind
94+
self.nth(n).kind() == kind
9595
}
9696

9797
/// Peek nth kind
9898
#[inline]
9999
pub(crate) fn nth_kind(&mut self, n: u8) -> Kind {
100-
self.nth(n).kind
100+
self.nth(n).kind()
101101
}
102102

103103
/// Checks if the current index has token `Kind`
@@ -121,15 +121,15 @@ impl<'a> ParserImpl<'a> {
121121
/// Checks if the current token is escaped if it is a keyword
122122
fn advance(&mut self, kind: Kind) {
123123
self.test_escaped_keyword(kind);
124-
self.prev_token_end = self.token.end;
124+
self.prev_token_end = self.token.end();
125125
self.token = self.lexer.next_token();
126126
}
127127

128128
/// Move to the next `JSXChild`
129129
/// Checks if the current token is escaped if it is a keyword
130130
fn advance_for_jsx_child(&mut self, kind: Kind) {
131131
self.test_escaped_keyword(kind);
132-
self.prev_token_end = self.token.end;
132+
self.prev_token_end = self.token.end();
133133
self.token = self.lexer.next_jsx_child();
134134
}
135135

@@ -182,7 +182,7 @@ impl<'a> ParserImpl<'a> {
182182
if kind == Kind::Semicolon {
183183
return true;
184184
}
185-
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line
185+
kind == Kind::RCurly || kind.is_eof() || self.cur_token().is_on_new_line()
186186
}
187187

188188
/// # Errors
@@ -242,7 +242,7 @@ impl<'a> ParserImpl<'a> {
242242
let kind = self.cur_kind();
243243
if kind == Kind::RAngle {
244244
self.token = self.lexer.next_right_angle();
245-
self.token.kind
245+
self.token.kind()
246246
} else {
247247
kind
248248
}
@@ -252,7 +252,7 @@ impl<'a> ParserImpl<'a> {
252252
let kind = self.cur_kind();
253253
if matches!(kind, Kind::ShiftLeft | Kind::ShiftLeftEq | Kind::LtEq) {
254254
self.token = self.lexer.re_lex_as_typescript_l_angle(kind);
255-
self.token.kind
255+
self.token.kind()
256256
} else {
257257
kind
258258
}
@@ -262,7 +262,7 @@ impl<'a> ParserImpl<'a> {
262262
let kind = self.cur_kind();
263263
if matches!(kind, Kind::ShiftRight | Kind::ShiftRight3) {
264264
self.token = self.lexer.re_lex_as_typescript_r_angle(kind);
265-
self.token.kind
265+
self.token.kind()
266266
} else {
267267
kind
268268
}

‎crates/oxc_parser/src/js/arrow.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ impl<'a> ParserImpl<'a> {
6565

6666
if self.at(Kind::Async) {
6767
let second_token = self.peek_token();
68-
let second = second_token.kind;
69-
if second_token.is_on_new_line {
68+
let second = second_token.kind();
69+
if second_token.is_on_new_line() {
7070
return Tristate::False;
7171
}
7272
if second != Kind::LParen && second != Kind::LAngle {
@@ -191,10 +191,10 @@ impl<'a> ParserImpl<'a> {
191191
fn is_un_parenthesized_async_arrow_function_worker(&mut self) -> Tristate {
192192
if self.at(Kind::Async) {
193193
let first_token = self.peek_token();
194-
let first = first_token.kind;
194+
let first = first_token.kind();
195195
// If the "async" is followed by "=>" token then it is not a beginning of an async arrow-function
196196
// but instead a simple arrow-function which will be parsed inside "parseAssignmentExpressionOrHigher"
197-
if first_token.is_on_new_line || first == Kind::Arrow {
197+
if first_token.is_on_new_line() || first == Kind::Arrow {
198198
return Tristate::False;
199199
}
200200
// Check for un-parenthesized AsyncArrowFunction
@@ -240,7 +240,7 @@ impl<'a> ParserImpl<'a> {
240240

241241
self.ctx = self.ctx.and_await(has_await);
242242

243-
if self.cur_token().is_on_new_line {
243+
if self.cur_token().is_on_new_line() {
244244
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
245245
}
246246

@@ -281,7 +281,7 @@ impl<'a> ParserImpl<'a> {
281281

282282
self.ctx = self.ctx.and_await(has_await);
283283

284-
if self.cur_token().is_on_new_line {
284+
if self.cur_token().is_on_new_line() {
285285
self.error(diagnostics::lineterminator_before_arrow(self.cur_token().span()));
286286
}
287287

@@ -345,7 +345,7 @@ impl<'a> ParserImpl<'a> {
345345
&mut self,
346346
allow_return_type_in_arrow_function: bool,
347347
) -> Option<Expression<'a>> {
348-
let pos = self.cur_token().start;
348+
let pos = self.cur_token().start();
349349
if self.state.not_parenthesized_arrow.contains(&pos) {
350350
return None;
351351
}

‎crates/oxc_parser/src/js/class.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ impl<'a> ParserImpl<'a> {
227227

228228
// async ...
229229
if key_name.is_none() && self.at(Kind::Async) && !self.peek_at(Kind::Question) {
230-
if !self.peek_token().is_on_new_line
230+
if !self.peek_token().is_on_new_line()
231231
&& (self.peek_kind().is_class_element_name_start() || self.peek_at(Kind::Star))
232232
{
233233
self.bump(Kind::Async);

‎crates/oxc_parser/src/js/declaration.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ impl<'a> ParserImpl<'a> {
9595
let mut definite = false;
9696
if binding_kind.is_binding_identifier()
9797
&& self.at(Kind::Bang)
98-
&& !self.cur_token().is_on_new_line
98+
&& !self.cur_token().is_on_new_line()
9999
{
100100
self.eat(Kind::Bang);
101101
definite = true;

‎crates/oxc_parser/src/js/expression.rs‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -290,9 +290,9 @@ impl<'a> ParserImpl<'a> {
290290
let span = self.start_span();
291291
let token = self.cur_token();
292292
let src = self.cur_src();
293-
let value = match token.kind {
293+
let value = match token.kind() {
294294
Kind::Decimal | Kind::Binary | Kind::Octal | Kind::Hex => {
295-
parse_int(src, token.kind, token.has_separator())
295+
parse_int(src, token.kind(), token.has_separator())
296296
}
297297
Kind::Float | Kind::PositiveExponential | Kind::NegativeExponential => {
298298
parse_float(src, token.has_separator())
@@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> {
303303
self.set_fatal_error(diagnostics::invalid_number(err, token.span()));
304304
0.0 // Dummy value
305305
});
306-
let base = match token.kind {
306+
let base = match token.kind() {
307307
Kind::Decimal => NumberBase::Decimal,
308308
Kind::Float => NumberBase::Float,
309309
Kind::Binary => NumberBase::Binary,
@@ -334,7 +334,7 @@ impl<'a> ParserImpl<'a> {
334334
let token = self.cur_token();
335335
let raw = self.cur_src();
336336
let src = raw.strip_suffix('n').unwrap();
337-
let _value = parse_big_int(src, token.kind, token.has_separator())
337+
let _value = parse_big_int(src, token.kind(), token.has_separator())
338338
.map_err(|err| diagnostics::invalid_number(err, token.span()));
339339
self.bump_any();
340340
self.ast.big_int_literal(self.end_span(span), raw, base)
@@ -346,10 +346,10 @@ impl<'a> ParserImpl<'a> {
346346
if !self.lexer.errors.is_empty() {
347347
return self.unexpected();
348348
}
349-
let pattern_start = self.cur_token().start + 1; // +1 to exclude left `/`
349+
let pattern_start = self.cur_token().start() + 1; // +1 to exclude left `/`
350350
let pattern_text = &self.source_text[pattern_start as usize..pattern_end as usize];
351351
let flags_start = pattern_end + 1; // +1 to include right `/`
352-
let flags_text = &self.source_text[flags_start as usize..self.cur_token().end as usize];
352+
let flags_text = &self.source_text[flags_start as usize..self.cur_token().end() as usize];
353353
let raw = self.cur_src();
354354
self.bump_any();
355355

@@ -406,7 +406,7 @@ impl<'a> ParserImpl<'a> {
406406
}
407407
let value = self.cur_string();
408408
let span = self.start_span();
409-
let lone_surrogates = self.cur_token().lone_surrogates;
409+
let lone_surrogates = self.cur_token().lone_surrogates();
410410
self.bump_any();
411411
let span = self.end_span(span);
412412
// SAFETY:
@@ -547,13 +547,13 @@ impl<'a> ParserImpl<'a> {
547547
// Also replace `\r` with `\n` in `raw`.
548548
// If contains `\r`, then `escaped` must be `true` (because `\r` needs unescaping),
549549
// so we can skip searching for `\r` in common case where contains no escapes.
550-
let (cooked, lone_surrogates) = if self.cur_token().escaped {
550+
let (cooked, lone_surrogates) = if self.cur_token().escaped() {
551551
// `cooked = None` when template literal has invalid escape sequence
552552
let cooked = self.cur_template_string().map(Atom::from);
553553
if cooked.is_some() && raw.contains('\r') {
554554
raw = self.ast.atom(&raw.cow_replace("\r\n", "\n").cow_replace('\r', "\n"));
555555
}
556-
(cooked, self.cur_token().lone_surrogates)
556+
(cooked, self.cur_token().lone_surrogates())
557557
} else {
558558
(Some(raw), false)
559559
};
@@ -774,7 +774,7 @@ impl<'a> ParserImpl<'a> {
774774
}
775775

776776
if !question_dot {
777-
if self.at(Kind::Bang) && !self.cur_token().is_on_new_line && self.is_ts {
777+
if self.at(Kind::Bang) && !self.cur_token().is_on_new_line() && self.is_ts {
778778
self.bump_any();
779779
lhs = self.ast.expression_ts_non_null(self.end_span(lhs_span), lhs);
780780
continue;
@@ -1021,7 +1021,7 @@ impl<'a> ParserImpl<'a> {
10211021
let span = self.start_span();
10221022
let lhs = self.parse_lhs_expression_or_higher();
10231023
// ++ -- postfix update expressions
1024-
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line {
1024+
if self.cur_kind().is_update_operator() && !self.cur_token().is_on_new_line() {
10251025
let operator = map_update_operator(self.cur_kind());
10261026
self.bump_any();
10271027
let lhs = SimpleAssignmentTarget::cover(lhs, self);
@@ -1131,7 +1131,7 @@ impl<'a> ParserImpl<'a> {
11311131
}
11321132

11331133
if self.is_ts && matches!(kind, Kind::As | Kind::Satisfies) {
1134-
if self.cur_token().is_on_new_line {
1134+
if self.cur_token().is_on_new_line() {
11351135
break;
11361136
}
11371137
self.bump_any();
@@ -1396,7 +1396,7 @@ impl<'a> ParserImpl<'a> {
13961396
if self.at(Kind::Await) {
13971397
let peek_token = self.peek_token();
13981398
// Allow arrow expression `await => {}`
1399-
if peek_token.kind == Kind::Arrow {
1399+
if peek_token.kind() == Kind::Arrow {
14001400
return false;
14011401
}
14021402
if self.ctx.has_await() {
@@ -1405,13 +1405,13 @@ impl<'a> ParserImpl<'a> {
14051405
// The following expressions are ambiguous
14061406
// await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await ``, await of []
14071407
if matches!(
1408-
peek_token.kind,
1408+
peek_token.kind(),
14091409
Kind::Of | Kind::LParen | Kind::LBrack | Kind::Slash | Kind::RegExp
14101410
) {
14111411
return false;
14121412
}
14131413

1414-
return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
1414+
return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield();
14151415
}
14161416
false
14171417
}
@@ -1420,13 +1420,13 @@ impl<'a> ParserImpl<'a> {
14201420
if self.at(Kind::Yield) {
14211421
let peek_token = self.peek_token();
14221422
// Allow arrow expression `yield => {}`
1423-
if peek_token.kind == Kind::Arrow {
1423+
if peek_token.kind() == Kind::Arrow {
14241424
return false;
14251425
}
14261426
if self.ctx.has_yield() {
14271427
return true;
14281428
}
1429-
return !peek_token.is_on_new_line && peek_token.kind.is_after_await_or_yield();
1429+
return !peek_token.is_on_new_line() && peek_token.kind().is_after_await_or_yield();
14301430
}
14311431
false
14321432
}

‎crates/oxc_parser/src/js/function.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ impl<'a> ParserImpl<'a> {
2424
self.at(Kind::Function)
2525
|| self.at(Kind::Async)
2626
&& self.peek_at(Kind::Function)
27-
&& !self.peek_token().is_on_new_line
27+
&& !self.peek_token().is_on_new_line()
2828
}
2929

3030
pub(crate) fn parse_function_body(&mut self) -> Box<'a, FunctionBody<'a>> {
@@ -303,7 +303,7 @@ impl<'a> ParserImpl<'a> {
303303
let mut delegate = false;
304304
let mut argument = None;
305305

306-
if !self.cur_token().is_on_new_line {
306+
if !self.cur_token().is_on_new_line() {
307307
delegate = self.eat(Kind::Star);
308308
let not_assignment_expr = matches!(
309309
self.cur_kind(),

‎crates/oxc_parser/src/js/module.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ impl<'a> ParserImpl<'a> {
172172
/// [Import Attributes](https://tc39.es/proposal-import-attributes)
173173
fn parse_import_attributes(&mut self) -> Option<WithClause<'a>> {
174174
let attributes_keyword = match self.cur_kind() {
175-
Kind::Assert if !self.cur_token().is_on_new_line => self.parse_identifier_name(),
175+
Kind::Assert if !self.cur_token().is_on_new_line() => self.parse_identifier_name(),
176176
Kind::With => self.parse_identifier_name(),
177177
_ => {
178178
return None;
@@ -403,7 +403,7 @@ impl<'a> ParserImpl<'a> {
403403
self.parse_class_declaration(decl_span, &modifiers),
404404
)
405405
}
406-
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line && self.is_ts => {
406+
_ if self.at(Kind::Interface) && !self.peek_token().is_on_new_line() && self.is_ts => {
407407
let decl = self.parse_ts_interface_declaration(decl_span, &Modifiers::empty());
408408
match decl {
409409
Declaration::TSInterfaceDeclaration(decl) => {

‎crates/oxc_parser/src/js/object.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ impl<'a> ParserImpl<'a> {
4646
// AsyncGeneratorMethod
4747
Kind::Async
4848
if (class_element_name || peek_kind == Kind::Star)
49-
&& !self.peek_token().is_on_new_line =>
49+
&& !self.peek_token().is_on_new_line() =>
5050
{
5151
self.parse_property_definition_method()
5252
}

0 commit comments

Comments
 (0)