From 3157eb9e12e1c346b6217004cd0f85fdaa022e32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Tue, 15 Sep 2026 12:23:00 +0200
Subject: [PATCH 1/2] parser: Collapse parser and tokenizer.

Now they are 1:1 so the distinction is mostly internal. This allows
state tracking to be a bit simpler.
---
 src/parser.rs                 | 215 +++-------
 src/rules_and_declarations.rs |   4 +-
 src/size_of_tests.rs          |   1 -
 src/tokenizer.rs              | 725 +++++++++++++++++-----------------
 4 files changed, 417 insertions(+), 528 deletions(-)

diff --git a/src/parser.rs b/src/parser.rs
index 1ee17817..5d0454f9 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -3,11 +3,10 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use crate::cow_rc_str::CowRcStr;
-use crate::tokenizer::{SourceLocation, SourcePosition, Token, Tokenizer};
+use crate::tokenizer::{SeenStatus, SourceLocation, SourcePosition, Token};
 use smallvec::SmallVec;
 use std::fmt;
 use std::ops::BitOr;
-use std::ops::Range;
 
 /// A capture of the internal state of a `Parser` (including the position within the input),
 /// obtained from the `Parser::position` method.
@@ -223,14 +222,16 @@ impl<E: fmt::Display + fmt::Debug> std::error::Error for ParseError<E> {}
 /// A CSS parser that borrows its `&str` input, yields `Token`s, and keeps track of nested blocks
 /// and functions.
 pub struct Parser<'i> {
-    tokenizer: Tokenizer<'i>,
+    pub(crate) input: &'i str,
+    pub(crate) state: ParserState,
     cached_token: CachedToken<'i>,
     current_block_depth: u8,
     nested_block_limit: u8,
-    /// If `Some(_)`, .parse_nested_block() can be called.
-    at_start_of: Option<BlockType>,
     /// For parsers from `parse_until` or `parse_nested_block`
     stop_before: Delimiters,
+    pub(crate) arbitrary_substitution_functions: SeenStatus<'i>,
+    pub(crate) source_map_url: Option<&'i str>,
+    pub(crate) source_url: Option<&'i str>,
 }
 
 struct CachedToken<'i> {
@@ -365,8 +366,8 @@ impl<'i> Parser<'i> {
     #[inline]
     pub fn new(input: &'i str) -> Self {
         Self {
-            tokenizer: Tokenizer::new(input),
-            at_start_of: None,
+            input,
+            state: ParserState::default(),
             stop_before: Delimiter::None,
             nested_block_limit: Self::REASONABLE_NESTED_BLOCK_LIMIT,
             current_block_depth: 0,
@@ -375,6 +376,9 @@ impl<'i> Parser<'i> {
                 start_position: SourcePosition(usize::MAX), // No token would match this cache.
                 end_state: ParserState::default(),
             },
+            arbitrary_substitution_functions: SeenStatus::DontCare,
+            source_map_url: None,
+            source_url: None,
         }
     }
 
@@ -385,11 +389,6 @@ impl<'i> Parser<'i> {
         self.nested_block_limit = limit;
     }
 
-    /// Return the current line that is being parsed.
-    pub fn current_line(&self) -> &'i str {
-        self.tokenizer.current_source_line()
-    }
-
     /// Check whether the input is exhausted. That is, if `.next()` would return a token.
     ///
     /// This ignores whitespace and comments.
@@ -417,38 +416,6 @@ impl<'i> Parser<'i> {
         result
     }
 
-    /// Return the current position within the input.
-    ///
-    /// This can be used with the `Parser::slice` and `slice_from` methods.
-    #[inline]
-    pub fn position(&self) -> SourcePosition {
-        self.tokenizer.position()
-    }
-
-    /// The current line number and column number.
-    #[inline]
-    pub fn current_source_location(&self) -> SourceLocation {
-        self.tokenizer.current_source_location()
-    }
-
-    /// The source map URL, if known.
-    ///
-    /// The source map URL is extracted from a specially formatted
-    /// comment.  The last such comment is used, so this value may
-    /// change as parsing proceeds.
-    pub fn current_source_map_url(&self) -> Option<&str> {
-        self.tokenizer.current_source_map_url()
-    }
-
-    /// The source URL, if known.
-    ///
-    /// The source URL is extracted from a specially formatted
-    /// comment.  The last such comment is used, so this value may
-    /// change as parsing proceeds.
-    pub fn current_source_url(&self) -> Option<&str> {
-        self.tokenizer.current_source_url()
-    }
-
     /// Create a new unexpected token or EOF ParseError at the current location
     #[inline]
     pub fn new_error_for_next_token<E>(&mut self) -> ParseError<E> {
@@ -463,34 +430,14 @@ impl<'i> Parser<'i> {
     /// This state can later be restored with the `Parser::reset` method.
     #[inline]
     pub fn state(&self) -> ParserState {
-        ParserState {
-            at_start_of: self.at_start_of,
-            ..self.tokenizer.state()
-        }
-    }
-
-    /// Advance the input until the next token that’s not whitespace or a comment.
-    #[inline]
-    pub fn skip_whitespace(&mut self) {
-        if let Some(block_type) = self.at_start_of.take() {
-            consume_until_end_of_block(block_type, &mut self.tokenizer);
-        }
-
-        self.tokenizer.skip_whitespace()
+        self.state.clone()
     }
 
+    /// Like `next_byte`, but returns `None` if the next byte is one of the delimiters this
+    /// parser was told to stop before.
     #[inline]
-    pub(crate) fn skip_cdc_and_cdo(&mut self) {
-        if let Some(block_type) = self.at_start_of.take() {
-            consume_until_end_of_block(block_type, &mut self.tokenizer);
-        }
-
-        self.tokenizer.skip_cdc_and_cdo()
-    }
-
-    #[inline]
-    pub(crate) fn next_byte(&self) -> Option<u8> {
-        let byte = self.tokenizer.next_byte()?;
+    pub(crate) fn next_byte_before_delimiter(&self) -> Option<u8> {
+        let byte = self.next_byte()?;
         if self.stop_before.contains(Delimiters::from_byte(byte)) {
             return None;
         }
@@ -503,26 +450,7 @@ impl<'i> Parser<'i> {
     /// Should only be used with `SourcePosition` values from the same `Parser` instance.
     #[inline]
     pub fn reset(&mut self, state: &ParserState) {
-        self.tokenizer.reset(state);
-        self.at_start_of = state.at_start_of;
-    }
-
-    /// Start looking for arbitrary substitution functions like `var()` / `env()` functions.
-    /// (See the `.seen_arbitrary_substitution_functions()` method.)
-    #[inline]
-    pub fn look_for_arbitrary_substitution_functions(
-        &mut self,
-        fns: ArbitrarySubstitutionFunctions<'i>,
-    ) {
-        self.tokenizer
-            .look_for_arbitrary_substitution_functions(fns)
-    }
-
-    /// Return whether a relevant function has been seen by the tokenizer since
-    /// `look_for_arbitrary_substitution_functions` was called, and stop looking.
-    #[inline]
-    pub fn seen_arbitrary_substitution_functions(&mut self) -> bool {
-        self.tokenizer.seen_arbitrary_substitution_functions()
+        self.state = state.clone();
     }
 
     /// The old name of `try_parse`, which requires raw identifiers in the Rust 2018 edition.
@@ -551,18 +479,6 @@ impl<'i> Parser<'i> {
         result
     }
 
-    /// Return a slice of the CSS input
-    #[inline]
-    pub fn slice(&self, range: Range<SourcePosition>) -> &'i str {
-        self.tokenizer.slice(range)
-    }
-
-    /// Return a slice of the CSS input, from the given position to the current one.
-    #[inline]
-    pub fn slice_from(&self, start_position: SourcePosition) -> &'i str {
-        self.tokenizer.slice_from(start_position)
-    }
-
     /// Return the next token in the input that is neither whitespace or a comment,
     /// and advance the position accordingly.
     ///
@@ -597,41 +513,31 @@ impl<'i> Parser<'i> {
     pub fn next_including_whitespace_and_comments(
         &mut self,
     ) -> Result<&Token<'i>, BasicParseError> {
-        if let Some(block_type) = self.at_start_of.take() {
-            consume_until_end_of_block(block_type, &mut self.tokenizer);
+        if let Some(block_type) = self.state.at_start_of.take() {
+            self.consume_until_end_of_block(block_type);
         }
 
-        let Some(byte) = self.tokenizer.next_byte() else {
-            return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput));
-        };
-
-        if self.stop_before.contains(Delimiters::from_byte(byte)) {
+        if self.next_byte_before_delimiter().is_none() {
             return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput));
         }
 
-        let token_start_position = self.tokenizer.position();
+        let token_start_position = self.position();
         let using_cached_token = self.cached_token.start_position == token_start_position;
-        let token = if using_cached_token {
-            let cached_token = &self.cached_token;
-            self.tokenizer.reset(&cached_token.end_state);
-            if let Token::Function(ref name) = cached_token.token {
-                self.tokenizer.see_function(name)
-            }
-            &cached_token.token
+        if using_cached_token {
+            self.state = self.cached_token.end_state.clone();
         } else {
-            let new_token = self.tokenizer.next_unchecked();
+            let new_token = self.next_unchecked();
+            if let Some(block_type) = BlockType::opening(&new_token) {
+                self.state.at_start_of = Some(block_type);
+            }
             self.cached_token = CachedToken {
                 token: new_token,
                 start_position: token_start_position,
-                end_state: self.tokenizer.state(),
+                end_state: self.state.clone(),
             };
-            &self.cached_token.token
-        };
-
-        if let Some(block_type) = BlockType::opening(token) {
-            self.at_start_of = Some(block_type);
         }
-        Ok(token)
+
+        Ok(&self.cached_token.token)
     }
 
     /// Have the given closure parse something, then check the the input is exhausted.
@@ -1001,17 +907,17 @@ where
     if error_behavior == ParseUntilErrorBehavior::Stop && result.is_err() {
         return result;
     }
-    if let Some(block_type) = parser.at_start_of.take() {
-        consume_until_end_of_block(block_type, &mut parser.tokenizer);
+    if let Some(block_type) = parser.state.at_start_of.take() {
+        parser.consume_until_end_of_block(block_type);
     }
     // FIXME: have a special-purpose tokenizer method for this that does less work.
-    while let Some(next_byte) = parser.tokenizer.next_byte() {
+    while let Some(next_byte) = parser.next_byte() {
         if delimiters.contains(Delimiters::from_byte(next_byte)) {
             break;
         }
-        let token = parser.tokenizer.next_unchecked();
+        let token = parser.next_unchecked();
         if let Some(block_type) = BlockType::opening(&token) {
-            consume_until_end_of_block(block_type, &mut parser.tokenizer);
+            parser.consume_until_end_of_block(block_type);
         }
     }
     result
@@ -1030,14 +936,14 @@ where
     if error_behavior == ParseUntilErrorBehavior::Stop && result.is_err() {
         return result;
     }
-    if let Some(next_byte) = parser.tokenizer.next_byte() {
+    if let Some(next_byte) = parser.next_byte() {
         let delimiter = Delimiters::from_byte(next_byte);
         if !parser.stop_before.contains(delimiter) {
             debug_assert!(delimiters.contains(delimiter));
             // We know this byte is ASCII.
-            parser.tokenizer.advance(1);
+            parser.advance(1);
             if next_byte == b'{' {
-                consume_until_end_of_block(BlockType::CurlyBracket, &mut parser.tokenizer);
+                parser.consume_until_end_of_block(BlockType::CurlyBracket);
             }
         }
     }
@@ -1051,7 +957,7 @@ pub fn parse_nested_block<'i, F, T, E>(
 where
     F: FnOnce(&mut Parser<'i>) -> Result<T, ParseError<E>>,
 {
-    let block_type = parser.at_start_of.take().expect(
+    let block_type = parser.state.at_start_of.take().expect(
         "\
          A nested parser can only be created when a Function, \
          ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \
@@ -1073,34 +979,39 @@ where
         BlockType::Parenthesis => ClosingDelimiter::CloseParenthesis,
     };
     let result = parser.parse_entirely(parse);
-    if let Some(nested_block_type) = parser.at_start_of.take() {
-        consume_until_end_of_block(nested_block_type, &mut parser.tokenizer);
+    if let Some(nested_block_type) = parser.state.at_start_of.take() {
+        parser.consume_until_end_of_block(nested_block_type);
     }
-    consume_until_end_of_block(block_type, &mut parser.tokenizer);
+    parser.consume_until_end_of_block(block_type);
     parser.stop_before = old_stop_before;
     parser.current_block_depth = parser.current_block_depth.wrapping_sub(1);
     result
 }
 
-#[inline(never)]
-#[cold]
-fn consume_until_end_of_block(block_type: BlockType, tokenizer: &mut Tokenizer) {
-    let mut stack = SmallVec::<[BlockType; 16]>::new();
-    stack.push(block_type);
-
-    // FIXME: have a special-purpose tokenizer method for this that does less work.
-    while let Ok(ref token) = tokenizer.next() {
-        if let Some(b) = BlockType::closing(token) {
-            if *stack.last().unwrap() == b {
-                stack.pop();
-                if stack.is_empty() {
-                    return;
+impl Parser<'_> {
+    /// Consume tokens until the end of a block of the given type that we're at the start of,
+    /// ignoring any `stop_before` delimiters.
+    #[inline(never)]
+    #[cold]
+    pub(crate) fn consume_until_end_of_block(&mut self, block_type: BlockType) {
+        let mut stack = SmallVec::<[BlockType; 16]>::new();
+        stack.push(block_type);
+
+        // FIXME: have a special-purpose tokenizer method for this that does less work.
+        while !self.is_eof() {
+            let token = self.next_unchecked();
+            if let Some(b) = BlockType::closing(&token) {
+                if *stack.last().unwrap() == b {
+                    stack.pop();
+                    if stack.is_empty() {
+                        return;
+                    }
                 }
             }
-        }
 
-        if let Some(block_type) = BlockType::opening(token) {
-            stack.push(block_type);
+            if let Some(block_type) = BlockType::opening(&token) {
+                stack.push(block_type);
+            }
         }
     }
 }
diff --git a/src/rules_and_declarations.rs b/src/rules_and_declarations.rs
index f33eadb8..0a72b850 100644
--- a/src/rules_and_declarations.rs
+++ b/src/rules_and_declarations.rs
@@ -381,7 +381,7 @@ where
         loop {
             self.input.skip_cdc_and_cdo();
             let start = self.input.state();
-            let at_keyword = match self.input.next_byte()? {
+            let at_keyword = match self.input.next_byte_before_delimiter()? {
                 b'@' => match self.input.next_including_whitespace_and_comments() {
                     Ok(Token::AtKeyword(name)) => Some(name.clone()),
                     _ => {
@@ -458,7 +458,7 @@ where
     input.parse_entirely(|input| {
         input.skip_whitespace();
         let start = input.state();
-        let at_keyword = if input.next_byte() == Some(b'@') {
+        let at_keyword = if input.next_byte_before_delimiter() == Some(b'@') {
             match *input.next_including_whitespace_and_comments()? {
                 Token::AtKeyword(ref name) => Some(name.clone()),
                 _ => {
diff --git a/src/size_of_tests.rs b/src/size_of_tests.rs
index e407578b..9d58dee1 100644
--- a/src/size_of_tests.rs
+++ b/src/size_of_tests.rs
@@ -42,7 +42,6 @@ size_of_test!(token, Token, 32);
 size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32);
 size_of_test!(cow_rc_str, CowRcStr, 16);
 
-size_of_test!(tokenizer, crate::tokenizer::Tokenizer, 96);
 size_of_test!(parser, crate::parser::Parser, 168);
 size_of_test!(source_position, crate::SourcePosition, 8);
 size_of_test!(parser_state, crate::ParserState, 24);
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index a1bca2d3..87fa4e12 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -6,7 +6,7 @@
 
 use self::Token::*;
 use crate::cow_rc_str::CowRcStr;
-use crate::parser::{ArbitrarySubstitutionFunctions, ParserState};
+use crate::parser::{ArbitrarySubstitutionFunctions, Parser};
 use std::char;
 use std::ops::Range;
 
@@ -182,7 +182,7 @@ pub enum Token<'a> {
 impl Token<'_> {
     /// Return whether this token represents a parse error.
     ///
-    /// `BadUrl` and `BadString` are tokenizer-level parse errors.
+    /// `BadUrl` and `BadString` are parser-level parse errors.
     ///
     /// `CloseParenthesis`, `CloseSquareBracket`, and `CloseCurlyBracket` are *unmatched*
     /// and therefore parse errors when returned by one of the `Parser::next*` methods.
@@ -194,42 +194,29 @@ impl Token<'_> {
     }
 }
 
-#[derive(Clone)]
-pub struct Tokenizer<'a> {
-    input: &'a str,
-    /// Counted in bytes, not code points. From 0.
-    position: usize,
-    /// The position at the start of the current line; but adjusted to
-    /// ensure that computing the column will give the result in units
-    /// of UTF-16 characters.
-    current_line_start_position: usize,
-    current_line_number: u32,
-    arbitrary_substitution_functions: SeenStatus<'a>,
-    source_map_url: Option<&'a str>,
-    source_url: Option<&'a str>,
-}
-
+/// Tracks whether the parser has seen any of the arbitrary substitution functions the caller
+/// asked about via `Parser::look_for_arbitrary_substitution_functions`.
 #[derive(Copy, Clone, PartialEq, Eq)]
-enum SeenStatus<'a> {
+pub(crate) enum SeenStatus<'a> {
     DontCare,
     LookingForThem(ArbitrarySubstitutionFunctions<'a>),
     SeenAtLeastOne,
 }
 
-impl<'a> Tokenizer<'a> {
+impl SeenStatus<'_> {
     #[inline]
-    pub fn new(input: &'a str) -> Self {
-        Tokenizer {
-            input,
-            position: 0,
-            current_line_start_position: 0,
-            current_line_number: 0,
-            arbitrary_substitution_functions: SeenStatus::DontCare,
-            source_map_url: None,
-            source_url: None,
+    pub(crate) fn see_function(&mut self, name: &str) {
+        if let SeenStatus::LookingForThem(fns) = *self {
+            if fns.iter().any(|a| name.eq_ignore_ascii_case(a)) {
+                *self = SeenStatus::SeenAtLeastOne;
+            }
         }
     }
+}
 
+impl<'a> Parser<'a> {
+    /// Start looking for arbitrary substitution functions like `var()` / `env()` functions.
+    /// (See the `.seen_arbitrary_substitution_functions()` method.)
     #[inline]
     pub fn look_for_arbitrary_substitution_functions(
         &mut self,
@@ -238,6 +225,8 @@ impl<'a> Tokenizer<'a> {
         self.arbitrary_substitution_functions = SeenStatus::LookingForThem(fns);
     }
 
+    /// Return whether a relevant function has been seen by the parser since
+    /// `look_for_arbitrary_substitution_functions` was called, and stop looking.
     #[inline]
     pub fn seen_arbitrary_substitution_functions(&mut self) -> bool {
         let seen = self.arbitrary_substitution_functions == SeenStatus::SeenAtLeastOne;
@@ -245,92 +234,74 @@ impl<'a> Tokenizer<'a> {
         seen
     }
 
+    /// Tokenize the next token, without any of the block / delimiter handling that
+    /// `Parser::next` and friends do. Assumes non-EOF.
     #[inline]
-    pub fn see_function(&mut self, name: &str) {
-        if let SeenStatus::LookingForThem(fns) = self.arbitrary_substitution_functions {
-            if fns.iter().any(|a| name.eq_ignore_ascii_case(a)) {
-                self.arbitrary_substitution_functions = SeenStatus::SeenAtLeastOne;
-            }
-        }
-    }
-
-    #[inline]
-    pub fn next(&mut self) -> Result<Token<'a>, ()> {
-        if self.is_eof() {
-            return Err(());
-        }
-        Ok(self.next_unchecked())
-    }
-
-    #[inline]
-    pub fn next_unchecked(&mut self) -> Token<'a> {
+    pub(crate) fn next_unchecked(&mut self) -> Token<'a> {
         next_token_unchecked(self)
     }
 
+    /// Return the current position within the input.
+    ///
+    /// This can be used with the `Parser::slice` and `slice_from` methods.
     #[inline]
     pub fn position(&self) -> SourcePosition {
-        debug_assert!(self.input.is_char_boundary(self.position));
-        SourcePosition(self.position)
+        debug_assert!(self.input.is_char_boundary(self.state.position));
+        SourcePosition(self.state.position)
     }
 
+    /// The current line number and column number.
     #[inline]
     pub fn current_source_location(&self) -> SourceLocation {
-        SourceLocation {
-            line: self.current_line_number,
-            column: (self.position - self.current_line_start_position + 1) as u32,
-        }
+        self.state.source_location()
     }
 
+    /// The source map URL, if known.
+    ///
+    /// The source map URL is extracted from a specially formatted
+    /// comment.  The last such comment is used, so this value may
+    /// change as parsing proceeds.
     #[inline]
     pub fn current_source_map_url(&self) -> Option<&'a str> {
         self.source_map_url
     }
 
+    /// The source URL, if known.
+    ///
+    /// The source URL is extracted from a specially formatted
+    /// comment.  The last such comment is used, so this value may
+    /// change as parsing proceeds.
     #[inline]
     pub fn current_source_url(&self) -> Option<&'a str> {
         self.source_url
     }
 
+    /// Return a slice of the CSS input, from the given position to the current one.
     #[inline]
-    pub fn state(&self) -> ParserState {
-        ParserState {
-            position: self.position,
-            current_line_start_position: self.current_line_start_position,
-            current_line_number: self.current_line_number,
-            at_start_of: None,
-        }
-    }
-
-    #[inline]
-    pub fn reset(&mut self, state: &ParserState) {
-        self.position = state.position;
-        self.current_line_start_position = state.current_line_start_position;
-        self.current_line_number = state.current_line_number;
-    }
-
-    #[inline]
-    pub(crate) fn slice_from(&self, start_pos: SourcePosition) -> &'a str {
+    pub fn slice_from(&self, start_pos: SourcePosition) -> &'a str {
         self.slice(start_pos..self.position())
     }
 
+    /// Return a slice of the CSS input
     #[inline]
-    pub(crate) fn slice(&self, range: Range<SourcePosition>) -> &'a str {
+    pub fn slice(&self, range: Range<SourcePosition>) -> &'a str {
         debug_assert!(self.input.is_char_boundary(range.start.0));
         debug_assert!(self.input.is_char_boundary(range.end.0));
         unsafe { self.input.get_unchecked(range.start.0..range.end.0) }
     }
 
     #[inline]
-    pub(crate) fn byte_slice(&self, range: Range<usize>) -> &'a [u8] {
+    fn byte_slice(&self, range: Range<usize>) -> &'a [u8] {
         &self.input.as_bytes()[range]
     }
 
     #[inline]
-    pub(crate) fn byte_slice_from(&self, start: usize) -> &'a [u8] {
-        self.byte_slice(start..self.position)
+    fn byte_slice_from(&self, start: usize) -> &'a [u8] {
+        self.byte_slice(start..self.state.position)
     }
 
-    pub fn current_source_line(&self) -> &'a str {
+    /// Return the current line that is being parsed.
+    pub fn current_line(&self) -> &'a str {
         let current = self.position();
         let start = self
             .slice(SourcePosition(0)..current)
@@ -344,32 +315,32 @@ impl<'a> Tokenizer<'a> {
     }
 
     #[inline]
-    pub fn next_byte(&self) -> Option<u8> {
+    pub(crate) fn next_byte(&self) -> Option<u8> {
         if self.is_eof() {
             None
         } else {
-            Some(self.input.as_bytes()[self.position])
+            Some(self.input.as_bytes()[self.state.position])
         }
     }
 
-    // If false, `tokenizer.next_char()` will not panic.
+    // If false, `parser.next_char()` will not panic.
     #[inline]
-    fn is_eof(&self) -> bool {
+    pub(crate) fn is_eof(&self) -> bool {
         !self.has_at_least(0)
     }
 
     // If true, the input has at least `n` bytes left *after* the current one.
-    // That is, `tokenizer.char_at(n)` will not panic.
+    // That is, `parser.char_at(n)` will not panic.
     #[inline]
     fn has_at_least(&self, n: usize) -> bool {
-        self.position + n < self.input.len()
+        self.state.position + n < self.input.len()
     }
 
     // Advance over N bytes in the input.  This function can advance
     // over ASCII bytes (excluding newlines), or UTF-8 sequence
     // leaders (excluding leaders for 4-byte sequences).
     #[inline]
-    pub fn advance(&mut self, n: usize) {
+    pub(crate) fn advance(&mut self, n: usize) {
         if cfg!(debug_assertions) {
             // Each byte must either be an ASCII byte or a sequence
             // leader, but not a 4-byte leader; also newlines are
@@ -380,13 +351,13 @@ impl<'a> Tokenizer<'a> {
                 debug_assert!(b != b'\r' && b != b'\n' && b != b'\x0C');
             }
         }
-        self.position += n
+        self.state.position += n
     }
 
     /// Equivalent to calling advance() for runs of bytes for which `matches` returns true.
     /// Returns the byte slice advanced over.
     fn advance_while(&mut self, mut matches: impl FnMut(u8) -> bool) -> &[u8] {
-        let start = self.position;
+        let start = self.state.position;
         let mut position = start;
 
         let bytes = &self.input.as_bytes()[start..];
@@ -397,8 +368,8 @@ impl<'a> Tokenizer<'a> {
             position += 1;
         }
 
-        // Equivalent to self.position = position, but with advance()'s debug_assert!s
-        self.advance(position - self.position);
+        // Equivalent to self.state.position = position, but with advance()'s debug_assert!s
+        self.advance(position - self.state.position);
 
         self.byte_slice_from(start)
     }
@@ -411,7 +382,7 @@ impl<'a> Tokenizer<'a> {
 
     #[inline]
     fn byte_at(&self, offset: usize) -> u8 {
-        self.input.as_bytes()[self.position + offset]
+        self.input.as_bytes()[self.state.position + offset]
     }
 
     // Advance over a single byte; the byte must be a UTF-8 sequence
@@ -421,8 +392,9 @@ impl<'a> Tokenizer<'a> {
         debug_assert!(self.next_byte_unchecked() & 0xF0 == 0xF0);
         // This takes two UTF-16 characters to represent, so we
         // actually have an undercount.
-        self.current_line_start_position = self.current_line_start_position.wrapping_sub(1);
-        self.position += 1;
+        self.state.current_line_start_position =
+            self.state.current_line_start_position.wrapping_sub(1);
+        self.state.position += 1;
     }
 
     // Advance over a single byte; the byte must be a UTF-8
@@ -433,30 +405,33 @@ impl<'a> Tokenizer<'a> {
         // Continuation bytes contribute to column overcount.  Note
         // that due to the special case for the 4-byte sequence intro,
         // we must use wrapping add here.
-        self.current_line_start_position = self.current_line_start_position.wrapping_add(1);
-        self.position += 1;
+        self.state.current_line_start_position =
+            self.state.current_line_start_position.wrapping_add(1);
+        self.state.position += 1;
     }
 
     // Advance over any kind of byte, excluding newlines.
     #[inline(never)]
     fn consume_known_byte(&mut self, byte: u8) {
         debug_assert!(byte != b'\r' && byte != b'\n' && byte != b'\x0C');
-        self.position += 1;
+        self.state.position += 1;
         // Continuation bytes contribute to column overcount.
         if byte & 0xF0 == 0xF0 {
             // This takes two UTF-16 characters to represent, so we
             // actually have an undercount.
-            self.current_line_start_position = self.current_line_start_position.wrapping_sub(1);
+            self.state.current_line_start_position =
+                self.state.current_line_start_position.wrapping_sub(1);
         } else if byte & 0xC0 == 0x80 {
             // Note that due to the special case for the 4-byte
             // sequence intro, we must use wrapping add here.
-            self.current_line_start_position = self.current_line_start_position.wrapping_add(1);
+            self.state.current_line_start_position =
+                self.state.current_line_start_position.wrapping_add(1);
         }
     }
 
     #[inline]
     fn next_char(&self) -> char {
-        unsafe { self.input.get_unchecked(self.position().0..) }
+        unsafe { self.input.get_unchecked(self.state.position().0..) }
             .chars()
             .next()
             .unwrap()
@@ -468,17 +443,17 @@ impl<'a> Tokenizer<'a> {
     fn consume_newline(&mut self) {
         let byte = self.next_byte_unchecked();
         debug_assert!(byte == b'\r' || byte == b'\n' || byte == b'\x0C');
-        self.position += 1;
+        self.state.position += 1;
         if byte == b'\r' && self.next_byte() == Some(b'\n') {
-            self.position += 1;
+            self.state.position += 1;
         }
-        self.current_line_start_position = self.position;
-        self.current_line_number += 1;
+        self.state.current_line_start_position = self.state.position;
+        self.state.current_line_number += 1;
     }
 
     #[inline]
     fn has_newline_at(&self, offset: usize) -> bool {
-        self.position + offset < self.input.len()
+        self.state.position + offset < self.input.len()
             && matches!(self.byte_at(offset), b'\n' | b'\r' | b'\x0C')
     }
 
@@ -486,10 +461,11 @@ impl<'a> Tokenizer<'a> {
     fn consume_char(&mut self) -> char {
         let c = self.next_char();
         let len_utf8 = c.len_utf8();
-        self.position += len_utf8;
+        self.state.position += len_utf8;
         // Note that due to the special case for the 4-byte sequence
         // intro, we must use wrapping add here.
-        self.current_line_start_position = self
+        self.state.current_line_start_position = self
+            .state
             .current_line_start_position
             .wrapping_add(len_utf8 - c.len_utf16());
         c
@@ -497,10 +473,14 @@ impl<'a> Tokenizer<'a> {
 
     #[inline]
     fn starts_with(&self, needle: &[u8]) -> bool {
-        self.input.as_bytes()[self.position..].starts_with(needle)
+        self.input.as_bytes()[self.state.position..].starts_with(needle)
     }
 
+    /// Advance the input until the next token that’s not whitespace or a comment.
     pub fn skip_whitespace(&mut self) {
+        if let Some(block_type) = self.state.at_start_of.take() {
+            self.consume_until_end_of_block(block_type);
+        }
         while !self.is_eof() {
             match_byte! { self.next_byte_unchecked(),
                 b' ' | b'\t' => {
@@ -521,7 +501,10 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
-    pub fn skip_cdc_and_cdo(&mut self) {
+    pub(crate) fn skip_cdc_and_cdo(&mut self) {
+        if let Some(block_type) = self.state.at_start_of.take() {
+            self.consume_until_end_of_block(block_type);
+        }
         while !self.is_eof() {
             match_byte! { self.next_byte_unchecked(),
                 b' ' | b'\t' => {
@@ -588,132 +571,132 @@ pub struct SourceLocation {
 #[cfg(feature = "malloc_size_of")]
 malloc_size_of::malloc_size_of_is_0!(SourceLocation);
 
-fn next_token_unchecked<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
-    debug_assert!(!tokenizer.is_eof());
-    let b = tokenizer.next_byte_unchecked();
+fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
+    debug_assert!(!parser.is_eof());
+    let b = parser.next_byte_unchecked();
     let token = match_byte! { b,
         b' ' | b'\t' => {
-            consume_whitespace(tokenizer, false)
+            consume_whitespace(parser, false)
         },
-        b'\n' | b'\x0C' | b'\r' => consume_whitespace(tokenizer, true),
-        b'"' => consume_string(tokenizer, false),
+        b'\n' | b'\x0C' | b'\r' => consume_whitespace(parser, true),
+        b'"' => consume_string(parser, false),
         b'#' => {
-            tokenizer.advance(1);
-            if is_ident_start(tokenizer) { IDHash(consume_name(tokenizer)) }
-            else if !tokenizer.is_eof() &&
-                matches!(tokenizer.next_byte_unchecked(), b'0'..=b'9' | b'-') {
+            parser.advance(1);
+            if is_ident_start(parser) { IDHash(consume_name(parser)) }
+            else if !parser.is_eof() &&
+                matches!(parser.next_byte_unchecked(), b'0'..=b'9' | b'-') {
                 // Any other valid case here already resulted in IDHash.
-                Hash(consume_name(tokenizer))
+                Hash(consume_name(parser))
             }
             else { Delim('#') }
         },
         b'$' => {
-            if tokenizer.starts_with(b"$=") { tokenizer.advance(2); SuffixMatch }
-            else { tokenizer.advance(1); Delim('$') }
+            if parser.starts_with(b"$=") { parser.advance(2); SuffixMatch }
+            else { parser.advance(1); Delim('$') }
         },
-        b'\'' => consume_string(tokenizer, true),
-        b'(' => { tokenizer.advance(1); ParenthesisBlock },
-        b')' => { tokenizer.advance(1); CloseParenthesis },
+        b'\'' => consume_string(parser, true),
+        b'(' => { parser.advance(1); ParenthesisBlock },
+        b')' => { parser.advance(1); CloseParenthesis },
         b'*' => {
-            if tokenizer.starts_with(b"*=") { tokenizer.advance(2); SubstringMatch }
-            else { tokenizer.advance(1); Delim('*') }
+            if parser.starts_with(b"*=") { parser.advance(2); SubstringMatch }
+            else { parser.advance(1); Delim('*') }
         },
         b'+' => {
             if (
-                tokenizer.has_at_least(1)
-                && tokenizer.byte_at(1).is_ascii_digit()
+                parser.has_at_least(1)
+                && parser.byte_at(1).is_ascii_digit()
             ) || (
-                tokenizer.has_at_least(2)
-                && tokenizer.byte_at(1) == b'.'
-                && tokenizer.byte_at(2).is_ascii_digit()
+                parser.has_at_least(2)
+                && parser.byte_at(1) == b'.'
+                && parser.byte_at(2).is_ascii_digit()
             ) {
-                consume_numeric(tokenizer)
+                consume_numeric(parser)
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim('+')
             }
         },
-        b',' => { tokenizer.advance(1); Comma },
+        b',' => { parser.advance(1); Comma },
         b'-' => {
             if (
-                tokenizer.has_at_least(1)
-                && tokenizer.byte_at(1).is_ascii_digit()
+                parser.has_at_least(1)
+                && parser.byte_at(1).is_ascii_digit()
             ) || (
-                tokenizer.has_at_least(2)
-                && tokenizer.byte_at(1) == b'.'
-                && tokenizer.byte_at(2).is_ascii_digit()
+                parser.has_at_least(2)
+                && parser.byte_at(1) == b'.'
+                && parser.byte_at(2).is_ascii_digit()
             ) {
-                consume_numeric(tokenizer)
-            } else if tokenizer.starts_with(b"-->") {
-                tokenizer.advance(3);
+                consume_numeric(parser)
+            } else if parser.starts_with(b"-->") {
+                parser.advance(3);
                 CDC
-            } else if is_ident_start(tokenizer) {
-                consume_ident_like(tokenizer)
+            } else if is_ident_start(parser) {
+                consume_ident_like(parser)
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim('-')
             }
         },
         b'.' => {
-            if tokenizer.has_at_least(1)
-                && tokenizer.byte_at(1).is_ascii_digit() {
-                consume_numeric(tokenizer)
+            if parser.has_at_least(1)
+                && parser.byte_at(1).is_ascii_digit() {
+                consume_numeric(parser)
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim('.')
             }
         }
         b'/' => {
-            if tokenizer.starts_with(b"/*") {
-                Comment(consume_comment(tokenizer))
+            if parser.starts_with(b"/*") {
+                Comment(consume_comment(parser))
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim('/')
             }
         }
-        b'0'..=b'9' => consume_numeric(tokenizer),
-        b':' => { tokenizer.advance(1); Colon },
-        b';' => { tokenizer.advance(1); Semicolon },
+        b'0'..=b'9' => consume_numeric(parser),
+        b':' => { parser.advance(1); Colon },
+        b';' => { parser.advance(1); Semicolon },
         b'<' => {
-            if tokenizer.starts_with(b"<!--") {
-                tokenizer.advance(4);
+            if parser.starts_with(b"<!--") {
+                parser.advance(4);
                 CDO
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim('<')
             }
         },
         b'@' => {
-            tokenizer.advance(1);
-            if is_ident_start(tokenizer) { AtKeyword(consume_name(tokenizer)) }
+            parser.advance(1);
+            if is_ident_start(parser) { AtKeyword(consume_name(parser)) }
             else { Delim('@') }
         },
-        b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(tokenizer),
-        b'[' => { tokenizer.advance(1); SquareBracketBlock },
+        b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser),
+        b'[' => { parser.advance(1); SquareBracketBlock },
         b'\\' => {
-            if !tokenizer.has_newline_at(1) { consume_ident_like(tokenizer) }
-            else { tokenizer.advance(1); Delim('\\') }
+            if !parser.has_newline_at(1) { consume_ident_like(parser) }
+            else { parser.advance(1); Delim('\\') }
         },
-        b']' => { tokenizer.advance(1); CloseSquareBracket },
+        b']' => { parser.advance(1); CloseSquareBracket },
         b'^' => {
-            if tokenizer.starts_with(b"^=") { tokenizer.advance(2); PrefixMatch }
-            else { tokenizer.advance(1); Delim('^') }
+            if parser.starts_with(b"^=") { parser.advance(2); PrefixMatch }
+            else { parser.advance(1); Delim('^') }
         },
-        b'{' => { tokenizer.advance(1); CurlyBracketBlock },
+        b'{' => { parser.advance(1); CurlyBracketBlock },
         b'|' => {
-            if tokenizer.starts_with(b"|=") { tokenizer.advance(2); DashMatch }
-            else { tokenizer.advance(1); Delim('|') }
+            if parser.starts_with(b"|=") { parser.advance(2); DashMatch }
+            else { parser.advance(1); Delim('|') }
         },
-        b'}' => { tokenizer.advance(1); CloseCurlyBracket },
+        b'}' => { parser.advance(1); CloseCurlyBracket },
         b'~' => {
-            if tokenizer.starts_with(b"~=") { tokenizer.advance(2); IncludeMatch }
-            else { tokenizer.advance(1); Delim('~') }
+            if parser.starts_with(b"~=") { parser.advance(2); IncludeMatch }
+            else { parser.advance(1); Delim('~') }
         },
         _ => {
             if !b.is_ascii() {
-                consume_ident_like(tokenizer)
+                consume_ident_like(parser)
             } else {
-                tokenizer.advance(1);
+                parser.advance(1);
                 Delim(b as char)
             }
         },
@@ -721,40 +704,40 @@ fn next_token_unchecked<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     token
 }
 
-fn consume_whitespace<'a>(tokenizer: &mut Tokenizer<'a>, newline: bool) -> Token<'a> {
-    let start_position = tokenizer.position();
+fn consume_whitespace<'a>(parser: &mut Parser<'a>, newline: bool) -> Token<'a> {
+    let start_position = parser.position();
     if newline {
-        tokenizer.consume_newline();
+        parser.consume_newline();
     } else {
-        tokenizer.advance(1);
+        parser.advance(1);
     }
-    while !tokenizer.is_eof() {
-        let b = tokenizer.next_byte_unchecked();
+    while !parser.is_eof() {
+        let b = parser.next_byte_unchecked();
         match_byte! { b,
             b' ' | b'\t' => {
-                tokenizer.advance(1);
+                parser.advance(1);
             }
             b'\n' | b'\x0C' | b'\r' => {
-                tokenizer.consume_newline();
+                parser.consume_newline();
             }
             _ => {
                 break
             }
         }
     }
-    WhiteSpace(tokenizer.slice_from(start_position))
+    WhiteSpace(parser.slice_from(start_position))
 }
 
 // Check for sourceMappingURL or sourceURL comments and update the
-// tokenizer appropriately.
-fn check_for_source_map<'a>(tokenizer: &mut Tokenizer<'a>, contents: &'a str) {
+// parser appropriately.
+fn check_for_source_map<'a>(parser: &mut Parser<'a>, contents: &'a str) {
     let directive = "# sourceMappingURL=";
     let directive_old = "@ sourceMappingURL=";
 
     // If there is a source map directive, extract the URL.
     if contents.starts_with(directive) || contents.starts_with(directive_old) {
         let contents = &contents[directive.len()..];
-        tokenizer.source_map_url = contents.split([' ', '\t', '\x0C', '\r', '\n']).next();
+        parser.source_map_url = contents.split([' ', '\t', '\x0C', '\r', '\n']).next();
     }
 
     let directive = "# sourceURL=";
@@ -763,43 +746,43 @@ fn check_for_source_map<'a>(tokenizer: &mut Tokenizer<'a>, contents: &'a str) {
     // If there is a source map directive, extract the URL.
     if contents.starts_with(directive) || contents.starts_with(directive_old) {
         let contents = &contents[directive.len()..];
-        tokenizer.source_url = contents.split([' ', '\t', '\x0C', '\r', '\n']).next()
+        parser.source_url = contents.split([' ', '\t', '\x0C', '\r', '\n']).next()
     }
 }
 
-fn consume_comment<'a>(tokenizer: &mut Tokenizer<'a>) -> &'a str {
-    tokenizer.advance(2); // consume "/*"
-    let start_position = tokenizer.position();
-    while !tokenizer.is_eof() {
-        match_byte! { tokenizer.next_byte_unchecked(),
+fn consume_comment<'a>(parser: &mut Parser<'a>) -> &'a str {
+    parser.advance(2); // consume "/*"
+    let start_position = parser.position();
+    while !parser.is_eof() {
+        match_byte! { parser.next_byte_unchecked(),
             b'*' => {
-                let end_position = tokenizer.position();
-                tokenizer.advance(1);
-                if tokenizer.next_byte() == Some(b'/') {
-                    tokenizer.advance(1);
-                    let contents = tokenizer.slice(start_position..end_position);
-                    check_for_source_map(tokenizer, contents);
+                let end_position = parser.position();
+                parser.advance(1);
+                if parser.next_byte() == Some(b'/') {
+                    parser.advance(1);
+                    let contents = parser.slice(start_position..end_position);
+                    check_for_source_map(parser, contents);
                     return contents
                 }
             }
             b'\n' | b'\x0C' | b'\r' => {
-                tokenizer.consume_newline();
+                parser.consume_newline();
             }
-            b'\x80'..=b'\xBF' => { tokenizer.consume_continuation_byte(); }
-            b'\xF0'..=b'\xFF' => { tokenizer.consume_4byte_intro(); }
+            b'\x80'..=b'\xBF' => { parser.consume_continuation_byte(); }
+            b'\xF0'..=b'\xFF' => { parser.consume_4byte_intro(); }
             _ => {
                 // ASCII or other leading byte.
-                tokenizer.advance(1);
+                parser.advance(1);
             }
         }
     }
-    let contents = tokenizer.slice_from(start_position);
-    check_for_source_map(tokenizer, contents);
+    let contents = parser.slice_from(start_position);
+    check_for_source_map(parser, contents);
     contents
 }
 
-fn consume_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool) -> Token<'a> {
-    match consume_quoted_string(tokenizer, single_quote) {
+fn consume_string<'a>(parser: &mut Parser<'a>, single_quote: bool) -> Token<'a> {
+    match consume_quoted_string(parser, single_quote) {
         Ok(value) => QuotedString(value),
         Err(value) => BadString(value),
     }
@@ -807,57 +790,57 @@ fn consume_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool) -> Toke
 
 /// Return `Err(())` on syntax error (ie. unescaped newline)
 fn consume_quoted_string<'a>(
-    tokenizer: &mut Tokenizer<'a>,
+    parser: &mut Parser<'a>,
     single_quote: bool,
 ) -> Result<CowRcStr<'a>, CowRcStr<'a>> {
-    tokenizer.advance(1); // Skip the initial quote
+    parser.advance(1); // Skip the initial quote
     // start_pos is at code point boundary, after " or '
-    let start_pos = tokenizer.position();
+    let start_pos = parser.position();
     let mut string_bytes;
     loop {
-        if tokenizer.is_eof() {
-            return Ok(tokenizer.slice_from(start_pos).into());
+        if parser.is_eof() {
+            return Ok(parser.slice_from(start_pos).into());
         }
-        match_byte! { tokenizer.next_byte_unchecked(),
+        match_byte! { parser.next_byte_unchecked(),
             b'"' => {
                 if !single_quote {
-                    let value = tokenizer.slice_from(start_pos);
-                    tokenizer.advance(1);
+                    let value = parser.slice_from(start_pos);
+                    parser.advance(1);
                     return Ok(value.into())
                 }
-                tokenizer.advance(1);
+                parser.advance(1);
             }
             b'\'' => {
                 if single_quote {
-                    let value = tokenizer.slice_from(start_pos);
-                    tokenizer.advance(1);
+                    let value = parser.slice_from(start_pos);
+                    parser.advance(1);
                     return Ok(value.into())
                 }
-                tokenizer.advance(1);
+                parser.advance(1);
             }
             b'\\' | b'\0' => {
-                // * The tokenizer’s input is UTF-8 since it’s `&str`.
+                // * The parser’s input is UTF-8 since it’s `&str`.
                 // * start_pos is at a code point boundary
                 // * so is the current position (which is before '\\' or '\0'
                 //
                 // So `string_bytes` is well-formed UTF-8.
-                string_bytes = tokenizer.slice_from(start_pos).as_bytes().to_owned();
+                string_bytes = parser.slice_from(start_pos).as_bytes().to_owned();
                 break
             }
             b'\n' | b'\r' | b'\x0C' => {
-                return Err(tokenizer.slice_from(start_pos).into())
+                return Err(parser.slice_from(start_pos).into())
             },
-            b'\x80'..=b'\xBF' => { tokenizer.consume_continuation_byte(); }
-            b'\xF0'..=b'\xFF' => { tokenizer.consume_4byte_intro(); }
+            b'\x80'..=b'\xBF' => { parser.consume_continuation_byte(); }
+            b'\xF0'..=b'\xFF' => { parser.consume_4byte_intro(); }
             _ => {
                 // ASCII or other leading byte.
-                tokenizer.advance(1);
+                parser.advance(1);
             }
         }
     }
 
-    while !tokenizer.is_eof() {
-        let b = tokenizer.next_byte_unchecked();
+    while !parser.is_eof() {
+        let b = parser.next_byte_unchecked();
         match_byte! { b,
             b'\n' | b'\r' | b'\x0C' => {
                 return Err(
@@ -868,42 +851,42 @@ fn consume_quoted_string<'a>(
                 );
             }
             b'"' => {
-                tokenizer.advance(1);
+                parser.advance(1);
                 if !single_quote {
                     break;
                 }
             }
             b'\'' => {
-                tokenizer.advance(1);
+                parser.advance(1);
                 if single_quote {
                     break;
                 }
             }
             b'\\' => {
-                tokenizer.advance(1);
-                if !tokenizer.is_eof() {
-                    match tokenizer.next_byte_unchecked() {
+                parser.advance(1);
+                if !parser.is_eof() {
+                    match parser.next_byte_unchecked() {
                         // Escaped newline
                         b'\n' | b'\x0C' | b'\r' => {
-                            tokenizer.consume_newline();
+                            parser.consume_newline();
                         }
                         // This pushes one well-formed code point
-                        _ => consume_escape_and_write(tokenizer, &mut string_bytes)
+                        _ => consume_escape_and_write(parser, &mut string_bytes)
                     }
                 }
                 // else: escaped EOF, do nothing.
                 continue;
             }
             b'\0' => {
-                tokenizer.advance(1);
+                parser.advance(1);
                 string_bytes.extend("\u{FFFD}".as_bytes());
                 continue;
             }
-            b'\x80'..=b'\xBF' => { tokenizer.consume_continuation_byte(); }
-            b'\xF0'..=b'\xFF' => { tokenizer.consume_4byte_intro(); }
+            b'\x80'..=b'\xBF' => { parser.consume_continuation_byte(); }
+            b'\xF0'..=b'\xFF' => { parser.consume_4byte_intro(); }
             _ => {
                 // ASCII or other leading byte.
-                tokenizer.advance(1);
+                parser.advance(1);
             },
         }
 
@@ -919,32 +902,32 @@ fn consume_quoted_string<'a>(
 }
 
 #[inline]
-fn is_ident_start(tokenizer: &Tokenizer) -> bool {
-    !tokenizer.is_eof()
-        && match_byte! { tokenizer.next_byte_unchecked(),
+fn is_ident_start(parser: &Parser) -> bool {
+    !parser.is_eof()
+        && match_byte! { parser.next_byte_unchecked(),
             b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => true,
             b'-' => {
-                tokenizer.has_at_least(1) && match_byte! { tokenizer.byte_at(1),
+                parser.has_at_least(1) && match_byte! { parser.byte_at(1),
                     b'a'..=b'z' | b'A'..=b'Z' | b'-' | b'_' | b'\0' => {
                         true
                     }
-                    b'\\' => !tokenizer.has_newline_at(1),
+                    b'\\' => !parser.has_newline_at(1),
                     b => !b.is_ascii(),
                 }
             },
-            b'\\' => !tokenizer.has_newline_at(1),
+            b'\\' => !parser.has_newline_at(1),
             b => !b.is_ascii(),
         }
 }
 
-fn consume_ident_like<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
-    let value = consume_name(tokenizer);
-    if !tokenizer.is_eof() && tokenizer.next_byte_unchecked() == b'(' {
-        tokenizer.advance(1);
+fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> {
+    let value = consume_name(parser);
+    if !parser.is_eof() && parser.next_byte_unchecked() == b'(' {
+        parser.advance(1);
         if value.eq_ignore_ascii_case("url") {
-            consume_unquoted_url(tokenizer).unwrap_or(Function(value))
+            consume_unquoted_url(parser).unwrap_or(Function(value))
         } else {
-            tokenizer.see_function(&value);
+            parser.arbitrary_substitution_functions.see_function(&value);
             Function(value)
         }
     } else {
@@ -952,7 +935,7 @@ fn consume_ident_like<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     }
 }
 
-fn consume_name<'a>(tokenizer: &mut Tokenizer<'a>) -> CowRcStr<'a> {
+fn consume_name<'a>(parser: &mut Parser<'a>) -> CowRcStr<'a> {
     // These are the overwhelmingly common bytes, that we can just skip over in a tight loop.
     static IS_SIMPLE_NAME_BYTE: [bool; 256] = {
         let mut table = [false; 256];
@@ -965,54 +948,54 @@ fn consume_name<'a>(tokenizer: &mut Tokenizer<'a>) -> CowRcStr<'a> {
     };
 
     // start_pos is the end of the previous token, therefore at a code point boundary
-    let start_pos = tokenizer.position();
+    let start_pos = parser.position();
     let mut value_bytes;
     loop {
-        tokenizer.advance_while(|b| IS_SIMPLE_NAME_BYTE[b as usize]);
-        if tokenizer.is_eof() {
-            return tokenizer.slice_from(start_pos).into();
+        parser.advance_while(|b| IS_SIMPLE_NAME_BYTE[b as usize]);
+        if parser.is_eof() {
+            return parser.slice_from(start_pos).into();
         }
-        match_byte! { tokenizer.next_byte_unchecked(),
+        match_byte! { parser.next_byte_unchecked(),
             b'\\' | b'\0' => {
-                // * The tokenizer’s input is UTF-8 since it’s `&str`.
+                // * The parser’s input is UTF-8 since it’s `&str`.
                 // * start_pos is at a code point boundary
                 // * so is the current position (which is before '\\' or '\0'
                 //
                 // So `value_bytes` is well-formed UTF-8.
-                value_bytes = tokenizer.slice_from(start_pos).as_bytes().to_owned();
+                value_bytes = parser.slice_from(start_pos).as_bytes().to_owned();
                 break
             }
-            b'\x80'..=b'\xBF' => { tokenizer.consume_continuation_byte(); }
-            b'\xF0'..=b'\xFF' => { tokenizer.consume_4byte_intro(); }
+            b'\x80'..=b'\xBF' => { parser.consume_continuation_byte(); }
+            b'\xF0'..=b'\xFF' => { parser.consume_4byte_intro(); }
             _b => {
-                return tokenizer.slice_from(start_pos).into();
+                return parser.slice_from(start_pos).into();
             }
         }
     }
 
     loop {
-        value_bytes.extend(tokenizer.advance_while(|b| IS_SIMPLE_NAME_BYTE[b as usize]));
-        if tokenizer.is_eof() {
+        value_bytes.extend(parser.advance_while(|b| IS_SIMPLE_NAME_BYTE[b as usize]));
+        if parser.is_eof() {
             break;
         }
-        let b = tokenizer.next_byte_unchecked();
+        let b = parser.next_byte_unchecked();
         match_byte! { b,
             b'\\' => {
-                if tokenizer.has_newline_at(1) { break }
-                tokenizer.advance(1);
+                if parser.has_newline_at(1) { break }
+                parser.advance(1);
                 // This pushes one well-formed code point
-                consume_escape_and_write(tokenizer, &mut value_bytes)
+                consume_escape_and_write(parser, &mut value_bytes)
             }
             b'\0' => {
-                tokenizer.advance(1);
+                parser.advance(1);
                 value_bytes.extend("\u{FFFD}".as_bytes());
             },
             b'\x80'..=b'\xBF' => {
-                tokenizer.consume_continuation_byte();
+                parser.consume_continuation_byte();
                 value_bytes.push(b)
             }
             b'\xF0'..=b'\xFF' => {
-                tokenizer.consume_4byte_intro();
+                parser.consume_4byte_intro();
                 value_bytes.push(b)
             }
             _ => {
@@ -1044,27 +1027,27 @@ fn byte_to_decimal_digit(b: u8) -> Option<u32> {
     }
 }
 
-fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
+fn consume_numeric<'a>(parser: &mut Parser<'a>) -> Token<'a> {
     // Parse [+-]?\d*(\.\d+)?([eE][+-]?\d+)?
     // But this is always called so that there is at least one digit in \d*(\.\d+)?
 
     // Do all the math in f64 so that large numbers overflow to +/-inf
     // and i32::{MIN, MAX} are within range.
 
-    let (has_sign, sign) = match tokenizer.next_byte_unchecked() {
+    let (has_sign, sign) = match parser.next_byte_unchecked() {
         b'-' => (true, -1.),
         b'+' => (true, 1.),
         _ => (false, 1.),
     };
     if has_sign {
-        tokenizer.advance(1);
+        parser.advance(1);
     }
 
     let mut integral_part: f64 = 0.;
-    while let Some(digit) = byte_to_decimal_digit(tokenizer.next_byte_unchecked()) {
+    while let Some(digit) = byte_to_decimal_digit(parser.next_byte_unchecked()) {
         integral_part = integral_part * 10. + digit as f64;
-        tokenizer.advance(1);
-        if tokenizer.is_eof() {
+        parser.advance(1);
+        if parser.is_eof() {
             break;
         }
     }
@@ -1072,18 +1055,18 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     let mut is_integer = true;
 
     let mut fractional_part: f64 = 0.;
-    if tokenizer.has_at_least(1)
-        && tokenizer.next_byte_unchecked() == b'.'
-        && tokenizer.byte_at(1).is_ascii_digit()
+    if parser.has_at_least(1)
+        && parser.next_byte_unchecked() == b'.'
+        && parser.byte_at(1).is_ascii_digit()
     {
         is_integer = false;
-        tokenizer.advance(1); // Consume '.'
+        parser.advance(1); // Consume '.'
         let mut factor = 0.1;
-        while let Some(digit) = byte_to_decimal_digit(tokenizer.next_byte_unchecked()) {
+        while let Some(digit) = byte_to_decimal_digit(parser.next_byte_unchecked()) {
             fractional_part += digit as f64 * factor;
             factor *= 0.1;
-            tokenizer.advance(1);
-            if tokenizer.is_eof() {
+            parser.advance(1);
+            if parser.is_eof() {
                 break;
             }
         }
@@ -1091,28 +1074,28 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
 
     let mut value = sign * (integral_part + fractional_part);
 
-    if tokenizer.has_at_least(1)
-        && matches!(tokenizer.next_byte_unchecked(), b'e' | b'E')
-        && (tokenizer.byte_at(1).is_ascii_digit()
-            || (tokenizer.has_at_least(2)
-                && matches!(tokenizer.byte_at(1), b'+' | b'-')
-                && tokenizer.byte_at(2).is_ascii_digit()))
+    if parser.has_at_least(1)
+        && matches!(parser.next_byte_unchecked(), b'e' | b'E')
+        && (parser.byte_at(1).is_ascii_digit()
+            || (parser.has_at_least(2)
+                && matches!(parser.byte_at(1), b'+' | b'-')
+                && parser.byte_at(2).is_ascii_digit()))
     {
         is_integer = false;
-        tokenizer.advance(1);
-        let (has_sign, sign) = match tokenizer.next_byte_unchecked() {
+        parser.advance(1);
+        let (has_sign, sign) = match parser.next_byte_unchecked() {
             b'-' => (true, -1.),
             b'+' => (true, 1.),
             _ => (false, 1.),
         };
         if has_sign {
-            tokenizer.advance(1);
+            parser.advance(1);
         }
         let mut exponent: f64 = 0.;
-        while let Some(digit) = byte_to_decimal_digit(tokenizer.next_byte_unchecked()) {
+        while let Some(digit) = byte_to_decimal_digit(parser.next_byte_unchecked()) {
             exponent = exponent * 10. + digit as f64;
-            tokenizer.advance(1);
-            if tokenizer.is_eof() {
+            parser.advance(1);
+            if parser.is_eof() {
                 break;
             }
         }
@@ -1131,8 +1114,8 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         None
     };
 
-    if !tokenizer.is_eof() && tokenizer.next_byte_unchecked() == b'%' {
-        tokenizer.advance(1);
+    if !parser.is_eof() && parser.next_byte_unchecked() == b'%' {
+        parser.advance(1);
         return Percentage {
             unit_value: (value / 100.) as f32,
             int_value,
@@ -1140,8 +1123,8 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         };
     }
     let value = value as f32;
-    if is_ident_start(tokenizer) {
-        let unit = consume_name(tokenizer);
+    if is_ident_start(parser) {
+        let unit = consume_name(parser);
         Dimension {
             value,
             int_value,
@@ -1168,10 +1151,10 @@ unsafe fn from_utf8_release_unchecked(string_bytes: Vec<u8>) -> String {
     }
 }
 
-fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
+fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result<Token<'a>, ()> {
     // This is only called after "url(", so the current position is a code point boundary.
-    let start_position = tokenizer.position;
-    let from_start = &tokenizer.input[tokenizer.position..];
+    let start_position = parser.state.position;
+    let from_start = &parser.input[parser.state.position..];
     let mut newlines = 0;
     let mut last_newline = 0;
     let mut found_printable_char = false;
@@ -1180,7 +1163,7 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
         let (offset, b) = match iter.next() {
             Some(item) => item,
             None => {
-                tokenizer.position = tokenizer.input.len();
+                parser.state.position = parser.input.len();
                 break;
             }
         };
@@ -1200,13 +1183,13 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
             b')' => {
                 // Don't use advance, because we may be skipping
                 // newlines here, and we want to avoid the assert.
-                tokenizer.position += offset + 1;
+                parser.state.position += offset + 1;
                 break
             }
             _ => {
                 // Don't use advance, because we may be skipping
                 // newlines here, and we want to avoid the assert.
-                tokenizer.position += offset;
+                parser.state.position += offset;
                 found_printable_char = true;
                 break
             }
@@ -1214,107 +1197,107 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
     }
 
     if newlines > 0 {
-        tokenizer.current_line_number += newlines;
+        parser.state.current_line_number += newlines;
         // No need for wrapping_add here, because there's no possible
         // way to wrap.
-        tokenizer.current_line_start_position = start_position + last_newline + 1;
+        parser.state.current_line_start_position = start_position + last_newline + 1;
     }
 
     if found_printable_char {
         // This function only consumed ASCII (whitespace) bytes,
         // so the current position is a code point boundary.
-        return Ok(consume_unquoted_url_internal(tokenizer));
+        return Ok(consume_unquoted_url_internal(parser));
     } else {
         return Ok(UnquotedUrl("".into()));
     }
 
-    fn consume_unquoted_url_internal<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
+    fn consume_unquoted_url_internal<'a>(parser: &mut Parser<'a>) -> Token<'a> {
         // This function is only called with start_pos at a code point boundary.
-        let start_pos = tokenizer.position();
+        let start_pos = parser.position();
         let mut string_bytes: Vec<u8>;
         loop {
-            if tokenizer.is_eof() {
-                return UnquotedUrl(tokenizer.slice_from(start_pos).into());
+            if parser.is_eof() {
+                return UnquotedUrl(parser.slice_from(start_pos).into());
             }
-            match_byte! { tokenizer.next_byte_unchecked(),
+            match_byte! { parser.next_byte_unchecked(),
                 b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {
-                    let value = tokenizer.slice_from(start_pos);
-                    return consume_url_end(tokenizer, start_pos, value.into())
+                    let value = parser.slice_from(start_pos);
+                    return consume_url_end(parser, start_pos, value.into())
                 }
                 b')' => {
-                    let value = tokenizer.slice_from(start_pos);
-                    tokenizer.advance(1);
+                    let value = parser.slice_from(start_pos);
+                    parser.advance(1);
                     return UnquotedUrl(value.into())
                 }
                 b'\x01'..=b'\x08' | b'\x0B' | b'\x0E'..=b'\x1F' | b'\x7F'  // non-printable
                     | b'"' | b'\'' | b'(' => {
-                    tokenizer.advance(1);
-                    return consume_bad_url(tokenizer, start_pos)
+                    parser.advance(1);
+                    return consume_bad_url(parser, start_pos)
                 },
                 b'\\' | b'\0' => {
-                    // * The tokenizer’s input is UTF-8 since it’s `&str`.
+                    // * The parser’s input is UTF-8 since it’s `&str`.
                     // * start_pos is at a code point boundary
                     // * so is the current position (which is before '\\' or '\0'
                     //
                     // So `string_bytes` is well-formed UTF-8.
-                    string_bytes = tokenizer.slice_from(start_pos).as_bytes().to_owned();
+                    string_bytes = parser.slice_from(start_pos).as_bytes().to_owned();
                     break
                 }
-                b'\x80'..=b'\xBF' => { tokenizer.consume_continuation_byte(); }
-                b'\xF0'..=b'\xFF' => { tokenizer.consume_4byte_intro(); }
+                b'\x80'..=b'\xBF' => { parser.consume_continuation_byte(); }
+                b'\xF0'..=b'\xFF' => { parser.consume_4byte_intro(); }
                 _ => {
                     // ASCII or other leading byte.
-                    tokenizer.advance(1);
+                    parser.advance(1);
                 }
             }
         }
-        while !tokenizer.is_eof() {
-            let b = tokenizer.next_byte_unchecked();
+        while !parser.is_eof() {
+            let b = parser.next_byte_unchecked();
             match_byte! { b,
                 b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {
                     // string_bytes is well-formed UTF-8, see other comments.
                     let string = unsafe { from_utf8_release_unchecked(string_bytes) }.into();
-                    return consume_url_end(tokenizer, start_pos, string)
+                    return consume_url_end(parser, start_pos, string)
                 }
                 b')' => {
-                    tokenizer.advance(1);
+                    parser.advance(1);
                     break;
                 }
                 b'\x01'..=b'\x08' | b'\x0B' | b'\x0E'..=b'\x1F' | b'\x7F'  // non-printable
                     | b'"' | b'\'' | b'(' => {
-                    tokenizer.advance(1);
-                    return consume_bad_url(tokenizer, start_pos);
+                    parser.advance(1);
+                    return consume_bad_url(parser, start_pos);
                 }
                 b'\\' => {
-                    tokenizer.advance(1);
-                    if tokenizer.has_newline_at(0) {
-                        return consume_bad_url(tokenizer, start_pos)
+                    parser.advance(1);
+                    if parser.has_newline_at(0) {
+                        return consume_bad_url(parser, start_pos)
                     }
 
                     // This pushes one well-formed code point to string_bytes
-                    consume_escape_and_write(tokenizer, &mut string_bytes)
+                    consume_escape_and_write(parser, &mut string_bytes)
                 },
                 b'\0' => {
-                    tokenizer.advance(1);
+                    parser.advance(1);
                     string_bytes.extend("\u{FFFD}".as_bytes());
                 }
                 b'\x80'..=b'\xBF' => {
                     // We’ll end up copying the whole code point
                     // before this loop does something else.
-                    tokenizer.consume_continuation_byte();
+                    parser.consume_continuation_byte();
                     string_bytes.push(b);
                 }
                 b'\xF0'..=b'\xFF' => {
                     // We’ll end up copying the whole code point
                     // before this loop does something else.
-                    tokenizer.consume_4byte_intro();
+                    parser.consume_4byte_intro();
                     string_bytes.push(b);
                 }
                 // If this byte is part of a multi-byte code point,
                 // we’ll end up copying the whole code point before this loop does something else.
                 b => {
                     // ASCII or other leading byte.
-                    tokenizer.advance(1);
+                    parser.advance(1);
                     string_bytes.push(b)
                 }
             }
@@ -1326,66 +1309,66 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
     }
 
     fn consume_url_end<'a>(
-        tokenizer: &mut Tokenizer<'a>,
+        parser: &mut Parser<'a>,
         start_pos: SourcePosition,
         string: CowRcStr<'a>,
     ) -> Token<'a> {
-        while !tokenizer.is_eof() {
-            match_byte! { tokenizer.next_byte_unchecked(),
+        while !parser.is_eof() {
+            match_byte! { parser.next_byte_unchecked(),
                 b')' => {
-                    tokenizer.advance(1);
+                    parser.advance(1);
                     break
                 }
-                b' ' | b'\t' => { tokenizer.advance(1); }
+                b' ' | b'\t' => { parser.advance(1); }
                 b'\n' | b'\x0C' | b'\r' => {
-                    tokenizer.consume_newline();
+                    parser.consume_newline();
                 }
                 b => {
-                    tokenizer.consume_known_byte(b);
-                    return consume_bad_url(tokenizer, start_pos);
+                    parser.consume_known_byte(b);
+                    return consume_bad_url(parser, start_pos);
                 }
             }
         }
         UnquotedUrl(string)
     }
 
-    fn consume_bad_url<'a>(tokenizer: &mut Tokenizer<'a>, start_pos: SourcePosition) -> Token<'a> {
+    fn consume_bad_url<'a>(parser: &mut Parser<'a>, start_pos: SourcePosition) -> Token<'a> {
         // Consume up to the closing )
-        while !tokenizer.is_eof() {
-            match_byte! { tokenizer.next_byte_unchecked(),
+        while !parser.is_eof() {
+            match_byte! { parser.next_byte_unchecked(),
                 b')' => {
-                    let contents = tokenizer.slice_from(start_pos).into();
-                    tokenizer.advance(1);
+                    let contents = parser.slice_from(start_pos).into();
+                    parser.advance(1);
                     return BadUrl(contents)
                 }
                 b'\\' => {
-                    tokenizer.advance(1);
-                    if matches!(tokenizer.next_byte(), Some(b')') | Some(b'\\')) {
-                        tokenizer.advance(1); // Skip an escaped ')' or '\'
+                    parser.advance(1);
+                    if matches!(parser.next_byte(), Some(b')') | Some(b'\\')) {
+                        parser.advance(1); // Skip an escaped ')' or '\'
                     }
                 }
                 b'\n' | b'\x0C' | b'\r' => {
-                    tokenizer.consume_newline();
+                    parser.consume_newline();
                 }
                 b => {
-                    tokenizer.consume_known_byte(b);
+                    parser.consume_known_byte(b);
                 }
             }
         }
-        BadUrl(tokenizer.slice_from(start_pos).into())
+        BadUrl(parser.slice_from(start_pos).into())
     }
 }
 
 // (value, number of digits up to 6)
-fn consume_hex_digits(tokenizer: &mut Tokenizer<'_>) -> (u32, u32) {
+fn consume_hex_digits(parser: &mut Parser<'_>) -> (u32, u32) {
     let mut value = 0;
     let mut digits = 0;
-    while digits < 6 && !tokenizer.is_eof() {
-        match byte_to_hex_digit(tokenizer.next_byte_unchecked()) {
+    while digits < 6 && !parser.is_eof() {
+        match byte_to_hex_digit(parser.next_byte_unchecked()) {
             Some(digit) => {
                 value = value * 16 + digit;
                 digits += 1;
-                tokenizer.advance(1);
+                parser.advance(1);
             }
             None => break,
         }
@@ -1395,31 +1378,27 @@ fn consume_hex_digits(tokenizer: &mut Tokenizer<'_>) -> (u32, u32) {
 
 // Same constraints as consume_escape except it writes into `bytes` the result
 // instead of returning it.
-fn consume_escape_and_write(tokenizer: &mut Tokenizer, bytes: &mut Vec<u8>) {
-    bytes.extend(
-        consume_escape(tokenizer)
-            .encode_utf8(&mut [0; 4])
-            .as_bytes(),
-    )
+fn consume_escape_and_write(parser: &mut Parser, bytes: &mut Vec<u8>) {
+    bytes.extend(consume_escape(parser).encode_utf8(&mut [0; 4]).as_bytes())
 }
 
 // Assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed
 // and that the next input character has already been verified
 // to not be a newline.
-fn consume_escape(tokenizer: &mut Tokenizer) -> char {
-    if tokenizer.is_eof() {
+fn consume_escape(parser: &mut Parser) -> char {
+    if parser.is_eof() {
         return '\u{FFFD}';
     } // Escaped EOF
-    match_byte! { tokenizer.next_byte_unchecked(),
+    match_byte! { parser.next_byte_unchecked(),
         b'0'..=b'9' | b'A'..=b'F' | b'a'..=b'f' => {
-            let (c, _) = consume_hex_digits(tokenizer);
-            if !tokenizer.is_eof() {
-                match_byte! { tokenizer.next_byte_unchecked(),
+            let (c, _) = consume_hex_digits(parser);
+            if !parser.is_eof() {
+                match_byte! { parser.next_byte_unchecked(),
                     b' ' | b'\t' => {
-                        tokenizer.advance(1)
+                        parser.advance(1)
                     }
                     b'\n' | b'\x0C' | b'\r' => {
-                        tokenizer.consume_newline();
+                        parser.consume_newline();
                     }
                     _ => {}
                 }
@@ -1433,9 +1412,9 @@ fn consume_escape(tokenizer: &mut Tokenizer) -> char {
             }
         },
         b'\0' => {
-            tokenizer.advance(1);
+            parser.advance(1);
             '\u{FFFD}'
         }
-        _ => tokenizer.consume_char(),
+        _ => parser.consume_char(),
     }
 }

From d520608f61f0b662198c6a2152e75fa2f3512d03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Tue, 15 Sep 2026 12:45:26 +0200
Subject: [PATCH 2/2] parser: Move at_start_of handling to the tokenization
 loop.

---
 src/parser.rs    | 38 ++++++++------------------------
 src/tokenizer.rs | 56 ++++++++++++++++++++++++++++++++----------------
 2 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/src/parser.rs b/src/parser.rs
index 5d0454f9..2ca1b4dc 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -248,15 +248,6 @@ pub(crate) enum BlockType {
 }
 
 impl BlockType {
-    fn opening(token: &Token) -> Option<BlockType> {
-        match *token {
-            Token::Function(_) | Token::ParenthesisBlock => Some(BlockType::Parenthesis),
-            Token::SquareBracketBlock => Some(BlockType::SquareBracket),
-            Token::CurlyBracketBlock => Some(BlockType::CurlyBracket),
-            _ => None,
-        }
-    }
-
     fn closing(token: &Token) -> Option<BlockType> {
         match *token {
             Token::CloseParenthesis => Some(BlockType::Parenthesis),
@@ -513,9 +504,7 @@ impl<'i> Parser<'i> {
     pub fn next_including_whitespace_and_comments(
         &mut self,
     ) -> Result<&Token<'i>, BasicParseError> {
-        if let Some(block_type) = self.state.at_start_of.take() {
-            self.consume_until_end_of_block(block_type);
-        }
+        self.skip_block_at_start();
 
         if self.next_byte_before_delimiter().is_none() {
             return Err(BasicParseError::new(BasicParseErrorKind::EndOfInput));
@@ -527,9 +516,6 @@ impl<'i> Parser<'i> {
             self.state = self.cached_token.end_state.clone();
         } else {
             let new_token = self.next_unchecked();
-            if let Some(block_type) = BlockType::opening(&new_token) {
-                self.state.at_start_of = Some(block_type);
-            }
             self.cached_token = CachedToken {
                 token: new_token,
                 start_position: token_start_position,
@@ -907,18 +893,14 @@ where
     if error_behavior == ParseUntilErrorBehavior::Stop && result.is_err() {
         return result;
     }
-    if let Some(block_type) = parser.state.at_start_of.take() {
-        parser.consume_until_end_of_block(block_type);
-    }
+    parser.skip_block_at_start();
     // FIXME: have a special-purpose tokenizer method for this that does less work.
     while let Some(next_byte) = parser.next_byte() {
         if delimiters.contains(Delimiters::from_byte(next_byte)) {
             break;
         }
-        let token = parser.next_unchecked();
-        if let Some(block_type) = BlockType::opening(&token) {
-            parser.consume_until_end_of_block(block_type);
-        }
+        parser.next_unchecked();
+        parser.skip_block_at_start();
     }
     result
 }
@@ -979,9 +961,7 @@ where
         BlockType::Parenthesis => ClosingDelimiter::CloseParenthesis,
     };
     let result = parser.parse_entirely(parse);
-    if let Some(nested_block_type) = parser.state.at_start_of.take() {
-        parser.consume_until_end_of_block(nested_block_type);
-    }
+    parser.skip_block_at_start();
     parser.consume_until_end_of_block(block_type);
     parser.stop_before = old_stop_before;
     parser.current_block_depth = parser.current_block_depth.wrapping_sub(1);
@@ -1000,6 +980,10 @@ impl Parser<'_> {
         // FIXME: have a special-purpose tokenizer method for this that does less work.
         while !self.is_eof() {
             let token = self.next_unchecked();
+            if let Some(nested_block_type) = self.state.at_start_of.take() {
+                stack.push(nested_block_type);
+                continue;
+            }
             if let Some(b) = BlockType::closing(&token) {
                 if *stack.last().unwrap() == b {
                     stack.pop();
@@ -1008,10 +992,6 @@ impl Parser<'_> {
                     }
                 }
             }
-
-            if let Some(block_type) = BlockType::opening(&token) {
-                stack.push(block_type);
-            }
         }
     }
 }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 87fa4e12..7b3ef09f 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -6,7 +6,7 @@
 
 use self::Token::*;
 use crate::cow_rc_str::CowRcStr;
-use crate::parser::{ArbitrarySubstitutionFunctions, Parser};
+use crate::parser::{ArbitrarySubstitutionFunctions, BlockType, Parser};
 use std::char;
 use std::ops::Range;
 
@@ -234,13 +234,22 @@ impl<'a> Parser<'a> {
         seen
     }
 
-    /// Tokenize the next token, without any of the block / delimiter handling that
-    /// `Parser::next` and friends do. Assumes non-EOF.
+    /// Tokenize the next token, without the delimiter handling that `Parser::next` and friends do.
+    /// Assumes non-EOF.
     #[inline]
     pub(crate) fn next_unchecked(&mut self) -> Token<'a> {
+        debug_assert!(self.state.at_start_of.is_none());
         next_token_unchecked(self)
     }
 
+    /// If the last token returned opened a block, skip until after the end of that block.
+    #[inline]
+    pub(crate) fn skip_block_at_start(&mut self) {
+        if let Some(block_type) = self.state.at_start_of.take() {
+            self.consume_until_end_of_block(block_type);
+        }
+    }
+
     /// Return the current position within the input.
     ///
     /// This can be used with the `Parser::slice` and `slice_from` methods.
@@ -478,9 +487,7 @@ impl<'a> Parser<'a> {
 
     /// Advance the input until the next token that’s not whitespace or a comment.
     pub fn skip_whitespace(&mut self) {
-        if let Some(block_type) = self.state.at_start_of.take() {
-            self.consume_until_end_of_block(block_type);
-        }
+        self.skip_block_at_start();
         while !self.is_eof() {
             match_byte! { self.next_byte_unchecked(),
                 b' ' | b'\t' => {
@@ -502,9 +509,7 @@ impl<'a> Parser<'a> {
     }
 
     pub(crate) fn skip_cdc_and_cdo(&mut self) {
-        if let Some(block_type) = self.state.at_start_of.take() {
-            self.consume_until_end_of_block(block_type);
-        }
+        self.skip_block_at_start();
         while !self.is_eof() {
             match_byte! { self.next_byte_unchecked(),
                 b' ' | b'\t' => {
@@ -574,7 +579,7 @@ malloc_size_of::malloc_size_of_is_0!(SourceLocation);
 fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
     debug_assert!(!parser.is_eof());
     let b = parser.next_byte_unchecked();
-    let token = match_byte! { b,
+    match_byte! { b,
         b' ' | b'\t' => {
             consume_whitespace(parser, false)
         },
@@ -595,7 +600,11 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
             else { parser.advance(1); Delim('$') }
         },
         b'\'' => consume_string(parser, true),
-        b'(' => { parser.advance(1); ParenthesisBlock },
+        b'(' => {
+            parser.advance(1);
+            parser.state.at_start_of = Some(BlockType::Parenthesis);
+            ParenthesisBlock
+        },
         b')' => { parser.advance(1); CloseParenthesis },
         b'*' => {
             if parser.starts_with(b"*=") { parser.advance(2); SubstringMatch }
@@ -672,7 +681,11 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
             else { Delim('@') }
         },
         b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser),
-        b'[' => { parser.advance(1); SquareBracketBlock },
+        b'[' => {
+            parser.advance(1);
+            parser.state.at_start_of = Some(BlockType::SquareBracket);
+            SquareBracketBlock
+        },
         b'\\' => {
             if !parser.has_newline_at(1) { consume_ident_like(parser) }
             else { parser.advance(1); Delim('\\') }
@@ -682,7 +695,11 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
             if parser.starts_with(b"^=") { parser.advance(2); PrefixMatch }
             else { parser.advance(1); Delim('^') }
         },
-        b'{' => { parser.advance(1); CurlyBracketBlock },
+        b'{' => {
+            parser.advance(1);
+            parser.state.at_start_of = Some(BlockType::CurlyBracket);
+            CurlyBracketBlock
+        },
         b'|' => {
             if parser.starts_with(b"|=") { parser.advance(2); DashMatch }
             else { parser.advance(1); Delim('|') }
@@ -700,8 +717,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
                 Delim(b as char)
             }
         },
-    };
-    token
+    }
 }
 
 fn consume_whitespace<'a>(parser: &mut Parser<'a>, newline: bool) -> Token<'a> {
@@ -925,11 +941,13 @@ fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> {
     if !parser.is_eof() && parser.next_byte_unchecked() == b'(' {
         parser.advance(1);
         if value.eq_ignore_ascii_case("url") {
-            consume_unquoted_url(parser).unwrap_or(Function(value))
-        } else {
-            parser.arbitrary_substitution_functions.see_function(&value);
-            Function(value)
+            if let Ok(url) = consume_unquoted_url(parser) {
+                return url;
+            }
         }
+        parser.state.at_start_of = Some(BlockType::Parenthesis);
+        parser.arbitrary_substitution_functions.see_function(&value);
+        Function(value)
     } else {
         Ident(value)
     }