From e57c799d42938e6a71b809f4a478c47a16788925 Mon Sep 17 00:00:00 2001 From: Romain Menke Date: Thu, 17 Sep 2026 22:34:24 +0200 Subject: [PATCH 1/2] improve performance --- package-lock.json | 8 +- packages/css-tokenizer/CHANGELOG.md | 5 + packages/css-tokenizer/dist/index.mjs | 2 +- packages/css-tokenizer/package.json | 2 +- .../four-code-points-would-start-cdo.ts | 7 - .../src/checks/matches-url-ident.ts | 19 - .../three-code-points-would-start-cdc.ts | 7 - ...-code-points-would-start-ident-sequence.ts | 37 - .../three-code-points-would-start-number.ts | 29 - ...e-code-points-would-start-unicode-range.ts | 27 - .../two-code-points-are-valid-escape.ts | 13 - .../checks/two-code-points-start-comment.ts | 10 - .../css-tokenizer/src/code-points/ranges.ts | 42 +- packages/css-tokenizer/src/consume/bad-url.ts | 29 - packages/css-tokenizer/src/consume/comment.ts | 58 - .../src/consume/escaped-code-point.ts | 60 - .../css-tokenizer/src/consume/hash-token.ts | 49 - .../src/consume/ident-like-token.ts | 74 - .../src/consume/ident-sequence.ts | 34 - packages/css-tokenizer/src/consume/number.ts | 63 - .../src/consume/numeric-token.ts | 68 - .../css-tokenizer/src/consume/string-token.ts | 95 -- .../src/consume/unicode-range-token.ts | 102 -- .../css-tokenizer/src/consume/url-token.ts | 181 --- .../src/consume/whitespace-token.ts | 18 - .../src/interfaces/code-point-reader.ts | 15 - .../css-tokenizer/src/interfaces/context.ts | 5 - packages/css-tokenizer/src/reader.ts | 43 - packages/css-tokenizer/src/tokenizer.ts | 1357 +++++++++++++++-- packages/css-tokenizer/test/test.mjs | 4 + packages/css-tokenizer/test/token/basic.mjs | 28 + packages/css-tokenizer/test/token/cdo.mjs | 33 + .../css-tokenizer/test/token/is-token.mjs | 164 +- packages/css-tokenizer/test/token/numeric.mjs | 128 ++ packages/css-tokenizer/test/token/string.mjs | 42 + .../test/token/unicode-range.mjs | 14 + packages/css-tokenizer/test/token/url.mjs | 104 ++ .../css-tokenizer/test/util/clone-tokens.mjs | 30 + .../test/util/mirror-variant.mjs | 35 + .../test/wpt/cdc-vs-ident-tokens.mjs | 15 + 40 files changed, 1843 insertions(+), 1213 deletions(-) delete mode 100644 packages/css-tokenizer/src/checks/four-code-points-would-start-cdo.ts delete mode 100644 packages/css-tokenizer/src/checks/matches-url-ident.ts delete mode 100644 packages/css-tokenizer/src/checks/three-code-points-would-start-cdc.ts delete mode 100644 packages/css-tokenizer/src/checks/three-code-points-would-start-ident-sequence.ts delete mode 100644 packages/css-tokenizer/src/checks/three-code-points-would-start-number.ts delete mode 100644 packages/css-tokenizer/src/checks/three-code-points-would-start-unicode-range.ts delete mode 100644 packages/css-tokenizer/src/checks/two-code-points-are-valid-escape.ts delete mode 100644 packages/css-tokenizer/src/checks/two-code-points-start-comment.ts delete mode 100644 packages/css-tokenizer/src/consume/bad-url.ts delete mode 100644 packages/css-tokenizer/src/consume/comment.ts delete mode 100644 packages/css-tokenizer/src/consume/escaped-code-point.ts delete mode 100644 packages/css-tokenizer/src/consume/hash-token.ts delete mode 100644 packages/css-tokenizer/src/consume/ident-like-token.ts delete mode 100644 packages/css-tokenizer/src/consume/ident-sequence.ts delete mode 100644 packages/css-tokenizer/src/consume/number.ts delete mode 100644 packages/css-tokenizer/src/consume/numeric-token.ts delete mode 100644 packages/css-tokenizer/src/consume/string-token.ts delete mode 100644 packages/css-tokenizer/src/consume/unicode-range-token.ts delete mode 100644 packages/css-tokenizer/src/consume/url-token.ts delete mode 100644 packages/css-tokenizer/src/consume/whitespace-token.ts delete mode 100644 packages/css-tokenizer/src/interfaces/code-point-reader.ts delete mode 100644 packages/css-tokenizer/src/interfaces/context.ts delete mode 100644 packages/css-tokenizer/src/reader.ts create mode 100644 packages/css-tokenizer/test/util/clone-tokens.mjs create mode 100644 packages/css-tokenizer/test/util/mirror-variant.mjs diff --git a/package-lock.json b/package-lock.json index 9f3b890e07..a05a271380 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2694,9 +2694,9 @@ "license": "MIT" }, "node_modules/@rmenke/css-tokenizer-tests": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@rmenke/css-tokenizer-tests/-/css-tokenizer-tests-1.2.0.tgz", - "integrity": "sha512-XfdeXzW5QGc3inl69eid2FTLGY/514xs+VXQWlEzdUVm1QdU6MicU5S2hcEbHoC9WMzIMALTzxiZb49w+xJk0Q==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@rmenke/css-tokenizer-tests/-/css-tokenizer-tests-1.4.0.tgz", + "integrity": "sha512-vQstlz0BOhWz5fxKXm0lnmgDuXLmi0ZozUDwBreXV5vY2K/k2SY02iaGygj9jr+qaEQse1YalJ+2pPNdReef7Q==", "dev": true, "license": "MIT" }, @@ -8299,7 +8299,7 @@ ], "license": "MIT", "devDependencies": { - "@rmenke/css-tokenizer-tests": "^1.2.0", + "@rmenke/css-tokenizer-tests": "^1.4.0", "postcss": "^8.5.10", "postcss-parser-tests": "^8.10.0" }, diff --git a/packages/css-tokenizer/CHANGELOG.md b/packages/css-tokenizer/CHANGELOG.md index c6d7842a98..7d42e1cc19 100644 --- a/packages/css-tokenizer/CHANGELOG.md +++ b/packages/css-tokenizer/CHANGELOG.md @@ -1,5 +1,10 @@ # Changes to CSS Tokenizer +### Unreleased (patch) + +- Fix decoding of astral code points in `string-token` and `url-token` values. +- Improve performance + ### 4.0.0 _January 14, 2026_ diff --git a/packages/css-tokenizer/dist/index.mjs b/packages/css-tokenizer/dist/index.mjs index e0237c0b7e..00d8e9e953 100644 --- a/packages/css-tokenizer/dist/index.mjs +++ b/packages/css-tokenizer/dist/index.mjs @@ -1 +1 @@ -class ParseError extends Error{sourceStart;sourceEnd;parserState;constructor(e,n,t,o){super(e),this.name="ParseError",this.sourceStart=n,this.sourceEnd=t,this.parserState=o}}class ParseErrorWithToken extends ParseError{token;constructor(e,n,t,o,r){super(e,n,t,o),this.token=r}}const e={UnexpectedNewLineInString:"Unexpected newline while consuming a string token.",UnexpectedEOFInString:"Unexpected EOF while consuming a string token.",UnexpectedEOFInComment:"Unexpected EOF while consuming a comment.",UnexpectedEOFInURL:"Unexpected EOF while consuming a url token.",UnexpectedEOFInEscapedCodePoint:"Unexpected EOF while consuming an escaped code point.",UnexpectedCharacterInURL:"Unexpected character while consuming a url token.",InvalidEscapeSequenceInURL:"Invalid escape sequence while consuming a url token.",InvalidEscapeSequenceAfterBackslash:'Invalid escape sequence after "\\"'},n="undefined"!=typeof globalThis&&"structuredClone"in globalThis;function cloneTokens(e){return n?structuredClone(e):JSON.parse(JSON.stringify(e))}function stringify(...e){let n="";for(let t=0;t=48&&e<=57}function isUppercaseLetterCodePoint(e){return e>=65&&e<=90}function isLowercaseLetterCodePoint(e){return e>=97&&e<=122}function isHexDigitCodePoint(e){return e>=48&&e<=57||e>=97&&e<=102||e>=65&&e<=70}function isLetterCodePoint(e){return isLowercaseLetterCodePoint(e)||isUppercaseLetterCodePoint(e)}function isIdentStartCodePoint(e){return isLetterCodePoint(e)||isNonASCII_IdentCodePoint(e)||95===e}function isIdentCodePoint(e){return isIdentStartCodePoint(e)||isDigitCodePoint(e)||e===o}function isNonASCII_IdentCodePoint(e){return 183===e||8204===e||8205===e||8255===e||8256===e||8204===e||(192<=e&&e<=214||216<=e&&e<=246||248<=e&&e<=893||895<=e&&e<=8191||8304<=e&&e<=8591||11264<=e&&e<=12271||12289<=e&&e<=55295||63744<=e&&e<=64975||65008<=e&&e<=65533||(0===e||(!!isSurrogate(e)||e>=65536)))}function isNonPrintableCodePoint(e){return 11===e||127===e||0<=e&&e<=8||14<=e&&e<=31}function isNewLine(e){return e===r||e===t||12===e}function isWhitespace(e){return 32===e||e===r||9===e||e===t||12===e}function isSurrogate(e){return e>=55296&&e<=57343}function checkIfTwoCodePointsAreAValidEscape(e){return 92===e.source.codePointAt(e.cursor)&&!isNewLine(e.source.codePointAt(e.cursor+1)??-1)}function checkIfThreeCodePointsWouldStartAnIdentSequence(e,n){return n.source.codePointAt(n.cursor)===o?n.source.codePointAt(n.cursor+1)===o||(!!isIdentStartCodePoint(n.source.codePointAt(n.cursor+1)??-1)||92===n.source.codePointAt(n.cursor+1)&&!isNewLine(n.source.codePointAt(n.cursor+2)??-1)):!!isIdentStartCodePoint(n.source.codePointAt(n.cursor)??-1)||checkIfTwoCodePointsAreAValidEscape(n)}function checkIfThreeCodePointsWouldStartANumber(e){return e.source.codePointAt(e.cursor)===i||e.source.codePointAt(e.cursor)===o?!!isDigitCodePoint(e.source.codePointAt(e.cursor+1)??-1)||46===e.source.codePointAt(e.cursor+1)&&isDigitCodePoint(e.source.codePointAt(e.cursor+2)??-1):46===e.source.codePointAt(e.cursor)?isDigitCodePoint(e.source.codePointAt(e.cursor+1)??-1):isDigitCodePoint(e.source.codePointAt(e.cursor)??-1)}function checkIfTwoCodePointsStartAComment(e){return 47===e.source.codePointAt(e.cursor)&&42===e.source.codePointAt(e.cursor+1)}function checkIfThreeCodePointsWouldStartCDC(e){return e.source.codePointAt(e.cursor)===o&&e.source.codePointAt(e.cursor+1)===o&&62===e.source.codePointAt(e.cursor+2)}var c,a,u;function mirrorVariantType(e){switch(e){case c.OpenParen:return c.CloseParen;case c.CloseParen:return c.OpenParen;case c.OpenCurly:return c.CloseCurly;case c.CloseCurly:return c.OpenCurly;case c.OpenSquare:return c.CloseSquare;case c.CloseSquare:return c.OpenSquare;default:return null}}function mirrorVariant(e){switch(e[0]){case c.OpenParen:return[c.CloseParen,")",-1,-1,void 0];case c.CloseParen:return[c.OpenParen,"(",-1,-1,void 0];case c.OpenCurly:return[c.CloseCurly,"}",-1,-1,void 0];case c.CloseCurly:return[c.OpenCurly,"{",-1,-1,void 0];case c.OpenSquare:return[c.CloseSquare,"]",-1,-1,void 0];case c.CloseSquare:return[c.OpenSquare,"[",-1,-1,void 0];default:return null}}function consumeComment(n,t){for(t.advanceCodePoint(2);;){const o=t.readCodePoint();if(void 0===o){const o=[c.Comment,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInComment,t.representationStart,t.representationEnd,["4.3.2. Consume comments","Unexpected EOF"],o)),o}if(42===o&&(void 0!==t.source.codePointAt(t.cursor)&&47===t.source.codePointAt(t.cursor))){t.advanceCodePoint();break}}return[c.Comment,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0]}function consumeEscapedCodePoint(n,o){const i=o.readCodePoint();if(void 0===i)return n.onParseError(new ParseError(e.UnexpectedEOFInEscapedCodePoint,o.representationStart,o.representationEnd,["4.3.7. Consume an escaped code point","Unexpected EOF"])),s;if(isHexDigitCodePoint(i)){const e=[i];let n;for(;void 0!==(n=o.source.codePointAt(o.cursor))&&isHexDigitCodePoint(n)&&e.length<6;)e.push(n),o.advanceCodePoint();isWhitespace(o.source.codePointAt(o.cursor)??-1)&&(o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r&&o.advanceCodePoint(),o.advanceCodePoint());const c=parseInt(String.fromCodePoint(...e),16);return 0===c||isSurrogate(c)||c>1114111?s:c}return 0===i||isSurrogate(i)?s:i}function consumeIdentSequence(e,n){const t=[];for(;;){const o=n.source.codePointAt(n.cursor)??-1;if(0===o||isSurrogate(o))t.push(s),n.advanceCodePoint(+(o>65535)+1);else if(isIdentCodePoint(o))t.push(o),n.advanceCodePoint(+(o>65535)+1);else{if(!checkIfTwoCodePointsAreAValidEscape(n))return t;n.advanceCodePoint(),t.push(consumeEscapedCodePoint(e,n))}}}function consumeHashToken(e,n){n.advanceCodePoint();const t=n.source.codePointAt(n.cursor);if(void 0!==t&&(isIdentCodePoint(t)||checkIfTwoCodePointsAreAValidEscape(n))){let t=u.Unrestricted;checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)&&(t=u.ID);const o=consumeIdentSequence(e,n);return[c.Hash,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...o),type:t}]}return[c.Delim,"#",n.representationStart,n.representationEnd,{value:"#"}]}function consumeNumber(e,n){let t=a.Integer;for(n.source.codePointAt(n.cursor)!==i&&n.source.codePointAt(n.cursor)!==o||n.advanceCodePoint();isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint();if(46===n.source.codePointAt(n.cursor)&&isDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))for(n.advanceCodePoint(2),t=a.Number;isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint();if(101===n.source.codePointAt(n.cursor)||69===n.source.codePointAt(n.cursor)){if(isDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))n.advanceCodePoint(2);else{if(n.source.codePointAt(n.cursor+1)!==o&&n.source.codePointAt(n.cursor+1)!==i||!isDigitCodePoint(n.source.codePointAt(n.cursor+2)??-1))return t;n.advanceCodePoint(3)}for(t=a.Number;isDigitCodePoint(n.source.codePointAt(n.cursor)??-1);)n.advanceCodePoint()}return t}function consumeNumericToken(e,n){let t;{const e=n.source.codePointAt(n.cursor);e===o?t="-":e===i&&(t="+")}const r=consumeNumber(0,n),s=parseFloat(n.source.slice(n.representationStart,n.representationEnd+1));if(checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)){const o=consumeIdentSequence(e,n);return[c.Dimension,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t,type:r,unit:String.fromCodePoint(...o)}]}return 37===n.source.codePointAt(n.cursor)?(n.advanceCodePoint(),[c.Percentage,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t}]):[c.Number,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:s,signCharacter:t,type:r}]}function consumeWhiteSpace(e){for(;isWhitespace(e.source.codePointAt(e.cursor)??-1);)e.advanceCodePoint();return[c.Whitespace,e.source.slice(e.representationStart,e.representationEnd+1),e.representationStart,e.representationEnd,void 0]}!function(e){e.Comment="comment",e.AtKeyword="at-keyword-token",e.BadString="bad-string-token",e.BadURL="bad-url-token",e.CDC="CDC-token",e.CDO="CDO-token",e.Colon="colon-token",e.Comma="comma-token",e.Delim="delim-token",e.Dimension="dimension-token",e.EOF="EOF-token",e.Function="function-token",e.Hash="hash-token",e.Ident="ident-token",e.Number="number-token",e.Percentage="percentage-token",e.Semicolon="semicolon-token",e.String="string-token",e.URL="url-token",e.Whitespace="whitespace-token",e.OpenParen="(-token",e.CloseParen=")-token",e.OpenSquare="[-token",e.CloseSquare="]-token",e.OpenCurly="{-token",e.CloseCurly="}-token",e.UnicodeRange="unicode-range-token"}(c||(c={})),function(e){e.Integer="integer",e.Number="number"}(a||(a={})),function(e){e.Unrestricted="unrestricted",e.ID="id"}(u||(u={}));class Reader{cursor=0;source="";representationStart=0;representationEnd=-1;constructor(e){this.source=e}advanceCodePoint(e=1){this.cursor=this.cursor+e,this.representationEnd=this.cursor-1}readCodePoint(){const e=this.source.codePointAt(this.cursor);if(void 0!==e)return this.cursor=this.cursor+1,this.representationEnd=this.cursor-1,e}unreadCodePoint(e=1){this.cursor=this.cursor-e,this.representationEnd=this.cursor-1}resetRepresentation(){this.representationStart=this.cursor,this.representationEnd=-1}}function consumeStringToken(n,o){let i="";const a=o.readCodePoint();for(;;){const u=o.readCodePoint();if(void 0===u){const t=[c.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:i}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInString,o.representationStart,o.representationEnd,["4.3.5. Consume a string token","Unexpected EOF"],t)),t}if(isNewLine(u)){o.unreadCodePoint();const i=[c.BadString,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedNewLineInString,o.representationStart,o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r?o.representationEnd+2:o.representationEnd+1,["4.3.5. Consume a string token","Unexpected newline"],i)),i}if(u===a)return[c.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:i}];if(92!==u)0===u||isSurrogate(u)?i+=String.fromCodePoint(s):i+=String.fromCodePoint(u);else{if(void 0===o.source.codePointAt(o.cursor))continue;if(isNewLine(o.source.codePointAt(o.cursor)??-1)){o.source.codePointAt(o.cursor)===t&&o.source.codePointAt(o.cursor+1)===r&&o.advanceCodePoint(),o.advanceCodePoint();continue}i+=String.fromCodePoint(consumeEscapedCodePoint(n,o))}}}function checkIfCodePointsMatchURLIdent(e){return!(3!==e.length||117!==e[0]&&85!==e[0]||114!==e[1]&&82!==e[1]||108!==e[2]&&76!==e[2])}function consumeBadURL(e,n){for(;;){const t=n.source.codePointAt(n.cursor);if(void 0===t)return;if(41===t)return void n.advanceCodePoint();checkIfTwoCodePointsAreAValidEscape(n)?(n.advanceCodePoint(),consumeEscapedCodePoint(e,n)):n.advanceCodePoint()}}function consumeUrlToken(n,t){for(;isWhitespace(t.source.codePointAt(t.cursor)??-1);)t.advanceCodePoint();let o="";for(;;){if(void 0===t.source.codePointAt(t.cursor)){const r=[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected EOF"],r)),r}if(41===t.source.codePointAt(t.cursor))return t.advanceCodePoint(),[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];if(isWhitespace(t.source.codePointAt(t.cursor)??-1)){for(t.advanceCodePoint();isWhitespace(t.source.codePointAt(t.cursor)??-1);)t.advanceCodePoint();if(void 0===t.source.codePointAt(t.cursor)){const r=[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Consume as much whitespace as possible","Unexpected EOF"],r)),r}return 41===t.source.codePointAt(t.cursor)?(t.advanceCodePoint(),[c.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:o}]):(consumeBadURL(n,t),[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0])}const r=t.source.codePointAt(t.cursor);if(34===r||39===r||40===r||isNonPrintableCodePoint(r??-1)){consumeBadURL(n,t);const o=[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedCharacterInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected U+0022 QUOTATION MARK (\"), U+0027 APOSTROPHE ('), U+0028 LEFT PARENTHESIS (() or non-printable code point"],o)),o}if(92===r){if(checkIfTwoCodePointsAreAValidEscape(t)){t.advanceCodePoint(),o+=String.fromCodePoint(consumeEscapedCodePoint(n,t));continue}consumeBadURL(n,t);const r=[c.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],r)),r}0===t.source.codePointAt(t.cursor)||isSurrogate(t.source.codePointAt(t.cursor)??-1)?(o+=String.fromCodePoint(s),t.advanceCodePoint()):(o+=t.source[t.cursor],t.advanceCodePoint())}}function consumeIdentLikeToken(e,n){const t=consumeIdentSequence(e,n);if(40!==n.source.codePointAt(n.cursor))return[c.Ident,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];if(checkIfCodePointsMatchURLIdent(t)){n.advanceCodePoint();let o=0;for(;;){const e=isWhitespace(n.source.codePointAt(n.cursor)??-1),r=isWhitespace(n.source.codePointAt(n.cursor+1)??-1);if(e&&r){o+=1,n.advanceCodePoint(1);continue}const i=e?n.source.codePointAt(n.cursor+1):n.source.codePointAt(n.cursor);if(34===i||39===i)return o>0&&n.unreadCodePoint(o),[c.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];break}return consumeUrlToken(e,n)}return n.advanceCodePoint(),[c.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}]}function checkIfThreeCodePointsWouldStartAUnicodeRange(e){return!(117!==e.source.codePointAt(e.cursor)&&85!==e.source.codePointAt(e.cursor)||e.source.codePointAt(e.cursor+1)!==i||63!==e.source.codePointAt(e.cursor+2)&&!isHexDigitCodePoint(e.source.codePointAt(e.cursor+2)??-1))}function consumeUnicodeRangeToken(e,n){n.advanceCodePoint(2);const t=[],r=[];let i;for(;void 0!==(i=n.source.codePointAt(n.cursor))&&t.length<6&&isHexDigitCodePoint(i);)t.push(i),n.advanceCodePoint();for(;void 0!==(i=n.source.codePointAt(n.cursor))&&t.length<6&&63===i;)0===r.length&&r.push(...t),t.push(48),r.push(70),n.advanceCodePoint();if(!r.length&&n.source.codePointAt(n.cursor)===o&&isHexDigitCodePoint(n.source.codePointAt(n.cursor+1)??-1))for(n.advanceCodePoint();void 0!==(i=n.source.codePointAt(n.cursor))&&r.length<6&&isHexDigitCodePoint(i);)r.push(i),n.advanceCodePoint();if(!r.length){const e=parseInt(String.fromCodePoint(...t),16);return[c.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:e,endOfRange:e}]}const s=parseInt(String.fromCodePoint(...t),16),a=parseInt(String.fromCodePoint(...r),16);return[c.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:s,endOfRange:a}]}function tokenize(e,n){const t=tokenizer(e,n),o=[];for(;!t.endOfFile();)o.push(t.nextToken());return o.push(t.nextToken()),o}function tokenizer(n,s){const a=n.css.valueOf(),u=n.unicodeRangesAllowed??!1,d=new Reader(a),p={onParseError:s?.onParseError??noop};return{nextToken:function nextToken(){d.resetRepresentation();const n=d.source.codePointAt(d.cursor);if(void 0===n)return[c.EOF,"",-1,-1,void 0];if(47===n&&checkIfTwoCodePointsStartAComment(d))return consumeComment(p,d);if(u&&(117===n||85===n)&&checkIfThreeCodePointsWouldStartAUnicodeRange(d))return consumeUnicodeRangeToken(0,d);if(isIdentStartCodePoint(n))return consumeIdentLikeToken(p,d);if(isDigitCodePoint(n))return consumeNumericToken(p,d);switch(n){case 44:return d.advanceCodePoint(),[c.Comma,",",d.representationStart,d.representationEnd,void 0];case 58:return d.advanceCodePoint(),[c.Colon,":",d.representationStart,d.representationEnd,void 0];case 59:return d.advanceCodePoint(),[c.Semicolon,";",d.representationStart,d.representationEnd,void 0];case 40:return d.advanceCodePoint(),[c.OpenParen,"(",d.representationStart,d.representationEnd,void 0];case 41:return d.advanceCodePoint(),[c.CloseParen,")",d.representationStart,d.representationEnd,void 0];case 91:return d.advanceCodePoint(),[c.OpenSquare,"[",d.representationStart,d.representationEnd,void 0];case 93:return d.advanceCodePoint(),[c.CloseSquare,"]",d.representationStart,d.representationEnd,void 0];case 123:return d.advanceCodePoint(),[c.OpenCurly,"{",d.representationStart,d.representationEnd,void 0];case 125:return d.advanceCodePoint(),[c.CloseCurly,"}",d.representationStart,d.representationEnd,void 0];case 39:case 34:return consumeStringToken(p,d);case 35:return consumeHashToken(p,d);case i:case 46:return checkIfThreeCodePointsWouldStartANumber(d)?consumeNumericToken(p,d):(d.advanceCodePoint(),[c.Delim,d.source[d.representationStart],d.representationStart,d.representationEnd,{value:d.source[d.representationStart]}]);case r:case t:case 12:case 9:case 32:return consumeWhiteSpace(d);case o:return checkIfThreeCodePointsWouldStartANumber(d)?consumeNumericToken(p,d):checkIfThreeCodePointsWouldStartCDC(d)?(d.advanceCodePoint(3),[c.CDC,"--\x3e",d.representationStart,d.representationEnd,void 0]):checkIfThreeCodePointsWouldStartAnIdentSequence(0,d)?consumeIdentLikeToken(p,d):(d.advanceCodePoint(),[c.Delim,"-",d.representationStart,d.representationEnd,{value:"-"}]);case 60:return checkIfFourCodePointsWouldStartCDO(d)?(d.advanceCodePoint(4),[c.CDO,"\x3c!--",d.representationStart,d.representationEnd,void 0]):(d.advanceCodePoint(),[c.Delim,"<",d.representationStart,d.representationEnd,{value:"<"}]);case 64:if(d.advanceCodePoint(),checkIfThreeCodePointsWouldStartAnIdentSequence(0,d)){const e=consumeIdentSequence(p,d);return[c.AtKeyword,d.source.slice(d.representationStart,d.representationEnd+1),d.representationStart,d.representationEnd,{value:String.fromCodePoint(...e)}]}return[c.Delim,"@",d.representationStart,d.representationEnd,{value:"@"}];case 92:{if(checkIfTwoCodePointsAreAValidEscape(d))return consumeIdentLikeToken(p,d);d.advanceCodePoint();const n=[c.Delim,"\\",d.representationStart,d.representationEnd,{value:"\\"}];return p.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceAfterBackslash,d.representationStart,d.representationEnd,["4.3.1. Consume a token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],n)),n}}return d.advanceCodePoint(),[c.Delim,d.source[d.representationStart],d.representationStart,d.representationEnd,{value:d.source[d.representationStart]}]},endOfFile:function endOfFile(){return void 0===d.source.codePointAt(d.cursor)}}}function noop(){}function mutateIdent(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=String.fromCodePoint(...serializeIdent(t));e[1]=o,e[4].value=n}function mutateUnit(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=serializeIdent(t);101===o[0]&&insertEscapedCodePoint(o,0,o[0]);const r=String.fromCodePoint(...o),i="+"===e[4].signCharacter?e[4].signCharacter:"",s=e[4].value.toString();e[1]=`${i}${s}${r}`,e[4].unit=n}function serializeIdent(e){let n;if(0===e[0])e.splice(0,1,s),n=1;else if(e[0]===o&&e[1]===o)n=2;else if(e[0]===o&&e[1])n=2,isIdentStartCodePoint(e[1])||(n+=insertEscapedCodePoint(e,1,e[1]));else{if(e[0]===o&&!e[1])return[92,e[0]];isIdentStartCodePoint(e[0])?n=1:(n=1,n+=insertEscapedCodePoint(e,0,e[0]))}for(let t=n;t=48&&e<=57||e>=97&&e<=102||e>=65&&e<=70}function isIdentStartCodePoint(e){return e>=65&&e<=90||e>=97&&e<=122||95===e||isNonASCII_IdentCodePoint(e)}function isIdentCodePoint(e){return e>=65&&e<=90||e>=97&&e<=122||e>=48&&e<=57||e===o||95===e||isNonASCII_IdentCodePoint(e)}function isNonASCII_IdentCodePoint(e){return e<183?0===e:183===e||8204===e||8205===e||8255===e||8256===e||8204===e||(192<=e&&e<=214||216<=e&&e<=246||248<=e&&e<=893||895<=e&&e<=8191||8304<=e&&e<=8591||11264<=e&&e<=12271||12289<=e&&e<=55295||63744<=e&&e<=64975||65008<=e&&e<=65533||(0===e||(e>=55296&&e<=57343||e>=65536)))}function isNonPrintableCodePoint(e){return 11===e||127===e||0<=e&&e<=8||14<=e&&e<=31}function isNewLine(e){return e===r||e===t||12===e}function isWhitespace(e){return 32===e||e===r||9===e||e===t||12===e}function isSurrogate(e){return e>=55296&&e<=57343}function tokenize(e,n){const t=tokenizer(e,n),o=[];for(;!t.endOfFile();)o.push(t.nextToken());return o.push(t.nextToken()),o}function tokenizer(n,C){const l=n.css.valueOf(),f=n.unicodeRangesAllowed??!1;let m=0,h=0,k=-1,P=!1;const p=C?.onParseError??noop;function consumeIdentSequence(){const e=m;for(;;){const e=l.charCodeAt(m);if(e>=65&&e<=90||e>=97&&e<=122||e>=48&&e<=57||e===o||95===e){m+=1;continue}const n=l.codePointAt(m)??-1;if(0===n||n>=55296&&n<=57343||n===s&&!isNewLine(l.codePointAt(m+1)??-1))break;if(!isNonASCII_IdentCodePoint(n))break;m+=+(n>65535)+1}const n=l.codePointAt(m)??-1;if(0!==n&&!(n>=55296&&n<=57343)&&(n!==s||isNewLine(l.codePointAt(m+1)??-1)))return P=!0,k=m-1,l.slice(e,m);P=!1,m=e,k=-1;let t="";for(;;){const e=l.codePointAt(m)??-1;if(0===e||isSurrogate(e))t+=String.fromCharCode(c),m+=+(e>65535)+1,k=m-1;else if(e>=65&&e<=90||e>=97&&e<=122||e>=48&&e<=57||e===o||95===e||isNonASCII_IdentCodePoint(e))t+=e>65535?String.fromCodePoint(e):String.fromCharCode(e),m+=+(e>65535)+1,k=m-1;else{if(l.codePointAt(m)!==s||isNewLine(l.codePointAt(m+1)??-1))return t;m+=1,k=m-1,t+=String.fromCodePoint(consumeEscapedCodePoint())}}}function consumeEscapedCodePoint(){const n=l.codePointAt(m);if(void 0===n)return p(new ParseError(e.UnexpectedEOFInEscapedCodePoint,h,k,["4.3.7. Consume an escaped code point","Unexpected EOF"])),c;if(m+=+(n>65535)+1,k=m-1,isHexDigitCodePoint(n)){const e=[n];let o;for(;void 0!==(o=l.codePointAt(m))&&isHexDigitCodePoint(o)&&e.length<6;)e.push(o),m+=1,k=m-1;isWhitespace(l.codePointAt(m)??-1)&&(l.codePointAt(m)===t&&l.codePointAt(m+1)===r&&(m+=1),m+=1,k=m-1);const i=parseInt(String.fromCodePoint(...e),16);return 0===i||isSurrogate(i)||i>1114111?c:i}return 0===n||isSurrogate(n)?c:n}function consumeBadURL(){for(;;){const e=l.codePointAt(m);if(void 0===e)return;if(41===e)return m+=1,void(k=m-1);e!==s||isNewLine(l.codePointAt(m+1)??-1)?(m+=1,k=m-1):(m+=1,k=m-1,consumeEscapedCodePoint())}}function consumeComment(){for(m+=2,k=m-1;;){const n=l.codePointAt(m);if(void 0===n){const n=[u.Comment,l.slice(h,k+1),h,k,void 0];return p(new ParseErrorWithToken(e.UnexpectedEOFInComment,h,k,["4.3.2. Consume comments","Unexpected EOF"],n)),n}if(m+=1,k=m-1,42===n&&void 0!==l.codePointAt(m)&&47===l.codePointAt(m)){m+=1,k=m-1;break}}return[u.Comment,l.slice(h,k+1),h,k,void 0]}function consumeStringToken(){let n="";const o=l.codePointAt(m);for(m+=1,k=m-1;;){const i=l.charCodeAt(m);if(m>=l.length){const t=[u.String,l.slice(h,k+1),h,k,{value:n}];return p(new ParseErrorWithToken(e.UnexpectedEOFInString,h,k,["4.3.5. Consume a string token","Unexpected EOF"],t)),t}if(m+=1,k=m-1,isNewLine(i)){m-=1,k=m-1;const n=[u.BadString,l.slice(h,k+1),h,k,void 0];return p(new ParseErrorWithToken(e.UnexpectedNewLineInString,h,l.codePointAt(m)===t&&l.codePointAt(m+1)===r?k+2:k+1,["4.3.5. Consume a string token","Unexpected newline"],n)),n}if(i===o)return[u.String,l.slice(h,k+1),h,k,{value:n}];if(i!==s)if(0!==i)if(i<128)n+=String.fromCharCode(i);else{if(i>=55296&&i<=57343){const e=l.codePointAt(m-1)??-1;e>=65536?(n+=String.fromCodePoint(e),m+=1,k=m-1):n+=String.fromCharCode(c);continue}n+=String.fromCharCode(i)}else n+=String.fromCharCode(c);else{if(void 0===l.codePointAt(m))continue;if(isNewLine(l.codePointAt(m)??-1)){l.codePointAt(m)===t&&l.codePointAt(m+1)===r&&(m+=1),m+=1,k=m-1;continue}n+=String.fromCodePoint(consumeEscapedCodePoint())}}}function consumeUnicodeRangeToken(){m+=2,k=m-1;const e=[],n=[];let t;for(;void 0!==(t=l.codePointAt(m))&&e.length<6&&isHexDigitCodePoint(t);)e.push(t),m+=1,k=m-1;for(;void 0!==(t=l.codePointAt(m))&&e.length<6&&63===t;)0===n.length&&n.push(...e),e.push(48),n.push(70),m+=1,k=m-1;if(!n.length&&l.codePointAt(m)===o&&isHexDigitCodePoint(l.codePointAt(m+1)??-1))for(m+=1,k=m-1;void 0!==(t=l.codePointAt(m))&&n.length<6&&isHexDigitCodePoint(t);)n.push(t),m+=1,k=m-1;if(!n.length){const n=parseInt(String.fromCodePoint(...e),16);return[u.UnicodeRange,l.slice(h,k+1),h,k,{startOfRange:n,endOfRange:n}]}const r=parseInt(String.fromCodePoint(...e),16),i=parseInt(String.fromCodePoint(...n),16);return[u.UnicodeRange,l.slice(h,k+1),h,k,{startOfRange:r,endOfRange:i}]}function consumeUrlToken(){for(;isWhitespace(l.codePointAt(m)??-1);)m+=1,k=m-1;let n="";for(;;){if(void 0===l.codePointAt(m)){const t=[u.URL,l.slice(h,k+1),h,k,{value:n}];return p(new ParseErrorWithToken(e.UnexpectedEOFInURL,h,k,["4.3.6. Consume a url token","Unexpected EOF"],t)),t}if(41===l.codePointAt(m))return m+=1,k=m-1,[u.URL,l.slice(h,k+1),h,k,{value:n}];if(isWhitespace(l.codePointAt(m)??-1)){for(m+=1,k=m-1;isWhitespace(l.codePointAt(m)??-1);)m+=1,k=m-1;if(void 0===l.codePointAt(m)){const t=[u.URL,l.slice(h,k+1),h,k,{value:n}];return p(new ParseErrorWithToken(e.UnexpectedEOFInURL,h,k,["4.3.6. Consume a url token","Consume as much whitespace as possible","Unexpected EOF"],t)),t}return 41===l.codePointAt(m)?(m+=1,k=m-1,[u.URL,l.slice(h,k+1),h,k,{value:n}]):(consumeBadURL(),[u.BadURL,l.slice(h,k+1),h,k,void 0])}const t=l.codePointAt(m);if(34===t||39===t||40===t||isNonPrintableCodePoint(t??-1)){consumeBadURL();const n=[u.BadURL,l.slice(h,k+1),h,k,void 0];return p(new ParseErrorWithToken(e.UnexpectedCharacterInURL,h,k,["4.3.6. Consume a url token","Unexpected U+0022 QUOTATION MARK (\"), U+0027 APOSTROPHE ('), U+0028 LEFT PARENTHESIS (() or non-printable code point"],n)),n}if(t===s){if(t===s&&!isNewLine(l.codePointAt(m+1)??-1)){m+=1,k=m-1,n+=String.fromCodePoint(consumeEscapedCodePoint());continue}consumeBadURL();const o=[u.BadURL,l.slice(h,k+1),h,k,void 0];return p(new ParseErrorWithToken(e.InvalidEscapeSequenceInURL,h,k,["4.3.6. Consume a url token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],o)),o}0===t||isSurrogate(t??-1)?(n+=String.fromCharCode(c),m+=1,k=m-1):(n+=String.fromCodePoint(t??c),m+=+((t??-1)>65535)+1,k=m-1)}}function consumeNumericToken(){let e;{const n=l.charCodeAt(m);n===o?e="-":n===i&&(e="+")}let n=a.Integer;const t=l.charCodeAt(m);t!==i&&t!==o||(m+=1);let r=l.charCodeAt(m);for(;r>=48&&r<=57;)m+=1,r=l.charCodeAt(m);if(46===l.charCodeAt(m)&&l.charCodeAt(m+1)>=48&&l.charCodeAt(m+1)<=57)for(m+=2,n=a.Number,r=l.charCodeAt(m);r>=48&&r<=57;)m+=1,r=l.charCodeAt(m);const c=l.charCodeAt(m);if(101===c||69===c){const e=l.charCodeAt(m+1);if(e>=48&&e<=57)for(m+=2,n=a.Number,r=l.charCodeAt(m);r>=48&&r<=57;)m+=1,r=l.charCodeAt(m);else if((e===o||e===i)&&l.charCodeAt(m+2)>=48&&l.charCodeAt(m+2)<=57)for(m+=3,n=a.Number,r=l.charCodeAt(m);r>=48&&r<=57;)m+=1,r=l.charCodeAt(m)}k=m-1;const s=parseFloat(l.slice(h,k+1));if(checkIfThreeCodePointsWouldStartAnIdentSequence()){const t=consumeIdentSequence();return[u.Dimension,l.slice(h,k+1),h,k,{value:s,signCharacter:e,type:n,unit:t}]}return 37===l.charCodeAt(m)?(m+=1,k=m-1,[u.Percentage,l.slice(h,k+1),h,k,{value:s,signCharacter:e}]):[u.Number,l.slice(h,k+1),h,k,{value:s,signCharacter:e,type:n}]}function consumeIdentLikeToken(){const e=consumeIdentSequence();if(40!==l.charCodeAt(m))return[u.Ident,P?e:l.slice(h,k+1),h,k,{value:e}];if(!(3!==e.length||117!==e.charCodeAt(0)&&85!==e.charCodeAt(0)||114!==e.charCodeAt(1)&&82!==e.charCodeAt(1)||108!==e.charCodeAt(2)&&76!==e.charCodeAt(2))){m+=1,k=m-1;let n=0;for(;isWhitespace(l.charCodeAt(m));)n+=1,m+=1,k=m-1;const t=l.charCodeAt(m);return 34===t||39===t?(n>0&&(m-=n,k=m-1),[u.Function,l.slice(h,k+1),h,k,{value:e}]):consumeUrlToken()}return m+=1,k=m-1,[u.Function,l.slice(h,k+1),h,k,{value:e}]}function consumeHashToken(){m+=1,k=m-1;const e=l.charCodeAt(m);if(e>=65&&e<=90||e>=97&&e<=122||e>=48&&e<=57||e===o||95===e||isNonASCII_IdentCodePoint(e)||e===s&&!isNewLine(l.codePointAt(m+1)??-1)){let e=d.Unrestricted;checkIfThreeCodePointsWouldStartAnIdentSequence()&&(e=d.ID);const n=consumeIdentSequence();return[u.Hash,l.slice(h,k+1),h,k,{value:n,type:e}]}return[u.Delim,"#",h,k,{value:"#"}]}function consumeAtKeywordToken(){if(m+=1,k=m-1,checkIfThreeCodePointsWouldStartAnIdentSequence()){const e=consumeIdentSequence();return[u.AtKeyword,l.slice(h,k+1),h,k,{value:e}]}return[u.Delim,"@",h,k,{value:"@"}]}function consumeInvalidEscapeToken(){m+=1,k=m-1;const n=[u.Delim,"\\",h,k,{value:"\\"}];return p(new ParseErrorWithToken(e.InvalidEscapeSequenceAfterBackslash,h,k,["4.3.1. Consume a token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],n)),n}function checkIfThreeCodePointsWouldStartAnIdentSequence(e){const n=void 0===e?l.charCodeAt(m):e;return n===o?l.charCodeAt(m+1)===o||(!!isIdentStartCodePoint(l.charCodeAt(m+1))||l.charCodeAt(m+1)===s&&!isNewLine(l.charCodeAt(m+2))):!!isIdentStartCodePoint(n)||l.charCodeAt(m)===s&&!isNewLine(l.charCodeAt(m+1))}function checkIfThreeCodePointsWouldStartANumber(e){return e===i||e===o?l.charCodeAt(m+1)>=48&&l.charCodeAt(m+1)<=57||46===l.charCodeAt(m+1)&&(l.charCodeAt(m+2)>=48&&l.charCodeAt(m+2)<=57):46===e?l.charCodeAt(m+1)>=48&&l.charCodeAt(m+1)<=57:e>=48&&e<=57}return{nextToken:function nextToken(){if(h=m,k=-1,m>=l.length)return[u.EOF,"",-1,-1,void 0];const e=l.charCodeAt(m);if(47===e&&42===l.charCodeAt(m+1))return consumeComment();if(f&&(117===e||85===e)&&l.codePointAt(m+1)===i&&(63===l.codePointAt(m+2)||isHexDigitCodePoint(l.codePointAt(m+2)??-1)))return consumeUnicodeRangeToken();if(isIdentStartCodePoint(e))return consumeIdentLikeToken();if((n=e)>=48&&n<=57)return consumeNumericToken();switch(e){case 44:return m+=1,k=m-1,[u.Comma,",",h,k,void 0];case 58:return m+=1,k=m-1,[u.Colon,":",h,k,void 0];case 59:return m+=1,k=m-1,[u.Semicolon,";",h,k,void 0];case 40:return m+=1,k=m-1,[u.OpenParen,"(",h,k,void 0];case 41:return m+=1,k=m-1,[u.CloseParen,")",h,k,void 0];case 91:return m+=1,k=m-1,[u.OpenSquare,"[",h,k,void 0];case 93:return m+=1,k=m-1,[u.CloseSquare,"]",h,k,void 0];case 123:return m+=1,k=m-1,[u.OpenCurly,"{",h,k,void 0];case 125:return m+=1,k=m-1,[u.CloseCurly,"}",h,k,void 0];case 39:case 34:return consumeStringToken();case 35:return consumeHashToken();case i:case 46:return checkIfThreeCodePointsWouldStartANumber(e)?consumeNumericToken():(m+=1,k=m-1,[u.Delim,l[h],h,k,{value:l[h]}]);case r:case t:case 12:case 9:case 32:{let e=l.charCodeAt(m);for(;32===e||e===r||9===e||e===t||12===e;)m+=1,e=l.charCodeAt(m);k=m-1}return[u.Whitespace,l.slice(h,k+1),h,k,void 0];case o:return checkIfThreeCodePointsWouldStartANumber(e)?consumeNumericToken():l.charCodeAt(m+1)===o&&62===l.charCodeAt(m+2)?(m+=3,k=m-1,[u.CDC,"--\x3e",h,k,void 0]):checkIfThreeCodePointsWouldStartAnIdentSequence(e)?consumeIdentLikeToken():(m+=1,k=m-1,[u.Delim,"-",h,k,{value:"-"}]);case 60:return 33===l.charCodeAt(m+1)&&l.charCodeAt(m+2)===o&&l.charCodeAt(m+3)===o?(m+=4,k=m-1,[u.CDO,"\x3c!--",h,k,void 0]):(m+=1,k=m-1,[u.Delim,"<",h,k,{value:"<"}]);case 64:return consumeAtKeywordToken();case s:return e!==s||isNewLine(l.charCodeAt(m+1))?consumeInvalidEscapeToken():consumeIdentLikeToken();default:return m+=1,k=m-1,[u.Delim,l[h],h,k,{value:l[h]}]}var n},endOfFile:function endOfFile(){return m>=l.length}}}function noop(){}function mutateIdent(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=String.fromCodePoint(...serializeIdent(t));e[1]=o,e[4].value=n}function mutateUnit(e,n){const t=[];for(const e of n)t.push(e.codePointAt(0));const o=serializeIdent(t);101===o[0]&&insertEscapedCodePoint(o,0,o[0]);const r=String.fromCodePoint(...o),i="+"===e[4].signCharacter?e[4].signCharacter:"",c=e[4].value.toString();e[1]=`${i}${c}${r}`,e[4].unit=n}function serializeIdent(e){let n;if(0===e[0])e.splice(0,1,c),n=1;else if(e[0]===o&&e[1]===o)n=2;else if(e[0]===o&&e[1])n=2,isIdentStartCodePoint(e[1])||(n+=insertEscapedCodePoint(e,1,e[1]));else{if(e[0]===o&&!e[1])return[92,e[0]];isIdentStartCodePoint(e[0])?n=1:(n=1,n+=insertEscapedCodePoint(e,0,e[0]))}for(let t=n;t): boolean { - return ( - codePoints.length === 3 && - ( - codePoints[0] === LATIN_SMALL_LETTER_U || - codePoints[0] === LATIN_CAPITAL_LETTER_U - ) && - ( - codePoints[1] === LATIN_SMALL_LETTER_R || - codePoints[1] === LATIN_CAPITAL_LETTER_R - ) && - ( - codePoints[2] === LATIN_SMALL_LETTER_L || - codePoints[2] === LATIN_CAPITAL_LETTER_L - ) - ); -} diff --git a/packages/css-tokenizer/src/checks/three-code-points-would-start-cdc.ts b/packages/css-tokenizer/src/checks/three-code-points-would-start-cdc.ts deleted file mode 100644 index 87c728752b..0000000000 --- a/packages/css-tokenizer/src/checks/three-code-points-would-start-cdc.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { GREATER_THAN_SIGN, HYPHEN_MINUS } from '../code-points/code-points'; -import type { CodePointReader } from '../interfaces/code-point-reader'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token -export function checkIfThreeCodePointsWouldStartCDC(reader: CodePointReader): boolean { - return reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 2) === GREATER_THAN_SIGN; -} diff --git a/packages/css-tokenizer/src/checks/three-code-points-would-start-ident-sequence.ts b/packages/css-tokenizer/src/checks/three-code-points-would-start-ident-sequence.ts deleted file mode 100644 index 91855018d9..0000000000 --- a/packages/css-tokenizer/src/checks/three-code-points-would-start-ident-sequence.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { HYPHEN_MINUS, REVERSE_SOLIDUS } from '../code-points/code-points'; -import { isIdentStartCodePoint, isNewLine } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { checkIfTwoCodePointsAreAValidEscape } from './two-code-points-are-valid-escape'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#would-start-an-identifier -export function checkIfThreeCodePointsWouldStartAnIdentSequence(ctx: Context, reader: CodePointReader): boolean { - // // U+002D HYPHEN-MINUS - if (reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { - // If the second code point is a U+002D HYPHEN-MINUS return true - if (reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS) { - return true; - } - - // If the second code point is an ident-start code point return true - if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1)) { - return true; - } - - // If the second and third code points are a valid escape return true - if (reader.source.codePointAt(reader.cursor + 1) === REVERSE_SOLIDUS && !isNewLine(reader.source.codePointAt(reader.cursor + 2) ?? -1)) { - return true; - } - - return false; - } - - // ident-start code point - // Return true. - if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor) ?? -1)) { - return true; - } - - // U+005C REVERSE SOLIDUS (\) - return checkIfTwoCodePointsAreAValidEscape(reader); -} diff --git a/packages/css-tokenizer/src/checks/three-code-points-would-start-number.ts b/packages/css-tokenizer/src/checks/three-code-points-would-start-number.ts deleted file mode 100644 index a9c14d318b..0000000000 --- a/packages/css-tokenizer/src/checks/three-code-points-would-start-number.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { FULL_STOP, HYPHEN_MINUS, PLUS_SIGN } from '../code-points/code-points'; -import { isDigitCodePoint } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-number -export function checkIfThreeCodePointsWouldStartANumber(reader: CodePointReader): boolean { - if (reader.source.codePointAt(reader.cursor) === PLUS_SIGN || reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { // U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-) - // If the second code point is a digit, return true. - if (isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1)) { - return true; - } - - // Otherwise, if the second code point is a U+002E FULL STOP (.) - if (reader.source.codePointAt(reader.cursor + 1) === FULL_STOP) { - // and the third code point is a digit, return true. - return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 2) ?? -1); - } - - // Otherwise, return false. - return false; - - } else if (reader.source.codePointAt(reader.cursor) === FULL_STOP) { // U+002E FULL STOP (.) - // If the second code point is a digit, return true. - // Otherwise, return false. - return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1); - } - - return isDigitCodePoint(reader.source.codePointAt(reader.cursor) ?? -1); // digit -} diff --git a/packages/css-tokenizer/src/checks/three-code-points-would-start-unicode-range.ts b/packages/css-tokenizer/src/checks/three-code-points-would-start-unicode-range.ts deleted file mode 100644 index a45bb78e82..0000000000 --- a/packages/css-tokenizer/src/checks/three-code-points-would-start-unicode-range.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { LATIN_CAPITAL_LETTER_U, LATIN_SMALL_LETTER_U, PLUS_SIGN, QUESTION_MARK } from '../code-points/code-points'; -import { isHexDigitCodePoint } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; - -// https://drafts.csswg.org/css-syntax/#starts-a-unicode-range -export function checkIfThreeCodePointsWouldStartAUnicodeRange(reader: CodePointReader): boolean { - if ( - // The first code point is either U+0055 LATIN CAPITAL LETTER U (U) or U+0075 LATIN SMALL LETTER U (u) - ( - reader.source.codePointAt(reader.cursor) === LATIN_SMALL_LETTER_U || - reader.source.codePointAt(reader.cursor) === LATIN_CAPITAL_LETTER_U - ) && - // The second code point is U+002B PLUS SIGN (+). - reader.source.codePointAt(reader.cursor + 1) === PLUS_SIGN && - // The third code point is either U+003F QUESTION MARK (?) or a hex digit - ( - reader.source.codePointAt(reader.cursor + 2) === QUESTION_MARK || - isHexDigitCodePoint(reader.source.codePointAt(reader.cursor + 2) ?? -1) - ) - ) { - // then return true. - return true; - } - - // Otherwise return false. - return false; -} diff --git a/packages/css-tokenizer/src/checks/two-code-points-are-valid-escape.ts b/packages/css-tokenizer/src/checks/two-code-points-are-valid-escape.ts deleted file mode 100644 index ab7f48901c..0000000000 --- a/packages/css-tokenizer/src/checks/two-code-points-are-valid-escape.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { REVERSE_SOLIDUS } from '../code-points/code-points'; -import { isNewLine } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-valid-escape -export function checkIfTwoCodePointsAreAValidEscape(reader: CodePointReader): boolean { - return ( - // If the first code point is not U+005C REVERSE SOLIDUS (\), return false. - reader.source.codePointAt(reader.cursor) === REVERSE_SOLIDUS && - // Otherwise, if the second code point is a newline, return false. - !isNewLine(reader.source.codePointAt(reader.cursor + 1) ?? -1) - ); -} diff --git a/packages/css-tokenizer/src/checks/two-code-points-start-comment.ts b/packages/css-tokenizer/src/checks/two-code-points-start-comment.ts deleted file mode 100644 index 796b2ae6ae..0000000000 --- a/packages/css-tokenizer/src/checks/two-code-points-start-comment.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ASTERISK, SOLIDUS } from '../code-points/code-points'; -import type { CodePointReader } from '../interfaces/code-point-reader'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comments -export function checkIfTwoCodePointsStartAComment(reader: CodePointReader): boolean { - return ( - reader.source.codePointAt(reader.cursor) === SOLIDUS && - reader.source.codePointAt(reader.cursor + 1) === ASTERISK - ); -} diff --git a/packages/css-tokenizer/src/code-points/ranges.ts b/packages/css-tokenizer/src/code-points/ranges.ts index ae99324cef..2b277562ee 100644 --- a/packages/css-tokenizer/src/code-points/ranges.ts +++ b/packages/css-tokenizer/src/code-points/ranges.ts @@ -1,4 +1,4 @@ -import { BACKSPACE, DELETE, INFORMATION_SEPARATOR_ONE, LINE_TABULATION, LOW_LINE, HYPHEN_MINUS, NULL, SHIFT_OUT, LINE_FEED, CARRIAGE_RETURN, FORM_FEED, SPACE, CHARACTER_TABULATION } from './code-points'; +import { BACKSPACE, CARRIAGE_RETURN, CHARACTER_TABULATION, DELETE, FORM_FEED, HYPHEN_MINUS, INFORMATION_SEPARATOR_ONE, LINE_FEED, LINE_TABULATION, LOW_LINE, NULL, SHIFT_OUT, SPACE } from './code-points'; // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokenizer-definitions // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#digit @@ -6,16 +6,6 @@ export function isDigitCodePoint(search: number): boolean { return search >= 0x0030 && search <= 0x0039; } -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#uppercase-letter -function isUppercaseLetterCodePoint(search: number): boolean { - return search >= 0x0041 && search <= 0x005a; -} - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#lowercase-letter -function isLowercaseLetterCodePoint(search: number): boolean { - return search >= 0x0061 && search <= 0x007a; -} - // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#hex-digit export function isHexDigitCodePoint(search: number): boolean { return ( @@ -25,23 +15,35 @@ export function isHexDigitCodePoint(search: number): boolean { ); } -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#letter -function isLetterCodePoint(search: number): boolean { - return isLowercaseLetterCodePoint(search) || isUppercaseLetterCodePoint(search); -} - // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-start-code-point export function isIdentStartCodePoint(search: number): boolean { - return isLetterCodePoint(search) || isNonASCII_IdentCodePoint(search) || search === LOW_LINE; + return ( + (search >= 0x0041 && search <= 0x005a) || // A .. Z + (search >= 0x0061 && search <= 0x007a) || // a .. z + search === LOW_LINE || + isNonASCII_IdentCodePoint(search) + ); } // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-code-point export function isIdentCodePoint(search: number): boolean { - return isIdentStartCodePoint(search) || isDigitCodePoint(search) || search === HYPHEN_MINUS; + return ( + (search >= 0x0041 && search <= 0x005a) || // A .. Z + (search >= 0x0061 && search <= 0x007a) || // a .. z + (search >= 0x0030 && search <= 0x0039) || // 0 .. 9 + search === HYPHEN_MINUS || + search === LOW_LINE || + isNonASCII_IdentCodePoint(search) + ); } // https://drafts.csswg.org/css-syntax/#non-ascii-ident-code-point -function isNonASCII_IdentCodePoint(search: number): boolean { +export function isNonASCII_IdentCodePoint(search: number): boolean { + // The only code points below U+00B7 that are non-ASCII ident code points is U+0000 NULL. + if (search < 0x00B7) { + return search === 0x000; + } + if ( search === 0x00B7 || search === 0x200C || @@ -70,7 +72,7 @@ function isNonASCII_IdentCodePoint(search: number): boolean { // Input preprocessing if (search === 0x000) { return true; - } else if (isSurrogate(search)) { + } else if (search >= 0xd800 && search <= 0xdfff) { // surrogate return true; } diff --git a/packages/css-tokenizer/src/consume/bad-url.ts b/packages/css-tokenizer/src/consume/bad-url.ts deleted file mode 100644 index 20981e5cb5..0000000000 --- a/packages/css-tokenizer/src/consume/bad-url.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { checkIfTwoCodePointsAreAValidEscape } from '../checks/two-code-points-are-valid-escape'; -import { RIGHT_PARENTHESIS } from '../code-points/code-points'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { consumeEscapedCodePoint } from './escaped-code-point'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-remnants-of-bad-url -export function consumeBadURL(ctx: Context, reader: CodePointReader): void { - while (true) { - const codePoint = reader.source.codePointAt(reader.cursor); - if (typeof codePoint === "undefined") { - return; - } - - if (codePoint === RIGHT_PARENTHESIS) { - reader.advanceCodePoint(); - return; - } - - if (checkIfTwoCodePointsAreAValidEscape(reader)) { - reader.advanceCodePoint(); - consumeEscapedCodePoint(ctx, reader); - continue; - } - - reader.advanceCodePoint(); - continue; - } -} diff --git a/packages/css-tokenizer/src/consume/comment.ts b/packages/css-tokenizer/src/consume/comment.ts deleted file mode 100644 index 02c4d6b257..0000000000 --- a/packages/css-tokenizer/src/consume/comment.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { ASTERISK, SOLIDUS } from '../code-points/code-points'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { ParseErrorMessage, ParseErrorWithToken } from '../interfaces/error'; -import type { CSSToken, TokenComment} from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comment -export function consumeComment(ctx: Context, reader: CodePointReader): TokenComment { - reader.advanceCodePoint(2); - - while (true) { - const codePoint = reader.readCodePoint(); - if (typeof codePoint === "undefined") { - const token: CSSToken = [ - TokenType.Comment, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedEOFInComment, - reader.representationStart, - reader.representationEnd, - [ - '4.3.2. Consume comments', - 'Unexpected EOF', - ], - token - )); - - return token; - } - - if (codePoint !== ASTERISK) { - continue; - } - - if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { - continue; - } - - if (reader.source.codePointAt(reader.cursor) === SOLIDUS) { - reader.advanceCodePoint(); - break; - } - } - - return [ - TokenType.Comment, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; -} diff --git a/packages/css-tokenizer/src/consume/escaped-code-point.ts b/packages/css-tokenizer/src/consume/escaped-code-point.ts deleted file mode 100644 index 964007da10..0000000000 --- a/packages/css-tokenizer/src/consume/escaped-code-point.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { CARRIAGE_RETURN, LINE_FEED, MAXIMUM_ALLOWED_CODEPOINT, REPLACEMENT_CHARACTER } from '../code-points/code-points'; -import { isHexDigitCodePoint, isSurrogate, isWhitespace } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { ParseError, ParseErrorMessage } from '../interfaces/error'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-escaped-code-point -export function consumeEscapedCodePoint(ctx: Context, reader: CodePointReader): number { - const codePoint = reader.readCodePoint(); - if (typeof codePoint === "undefined") { - ctx.onParseError(new ParseError( - ParseErrorMessage.UnexpectedEOFInEscapedCodePoint, - reader.representationStart, - reader.representationEnd, - [ - '4.3.7. Consume an escaped code point', - 'Unexpected EOF', - ], - )); - - return REPLACEMENT_CHARACTER; - } - - if (isHexDigitCodePoint(codePoint)) { - const hexSequence: Array = [codePoint]; - - let nextCodePoint: number | undefined; - while ((typeof (nextCodePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && isHexDigitCodePoint(nextCodePoint) && hexSequence.length < 6) { - hexSequence.push(nextCodePoint); - reader.advanceCodePoint(); - } - - if (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) { - if ( - reader.source.codePointAt(reader.cursor) === CARRIAGE_RETURN && - reader.source.codePointAt(reader.cursor + 1) === LINE_FEED - ) { - reader.advanceCodePoint(); - } - - reader.advanceCodePoint(); - } - - const codePointLiteral = parseInt(String.fromCodePoint(...hexSequence), 16); - if (codePointLiteral === 0 || isSurrogate(codePointLiteral)) { - return REPLACEMENT_CHARACTER; - } - if (codePointLiteral > MAXIMUM_ALLOWED_CODEPOINT) { - return REPLACEMENT_CHARACTER; - } - - return codePointLiteral; - } - - if (codePoint === 0 || isSurrogate(codePoint)) { - return REPLACEMENT_CHARACTER; - } - - return codePoint; -} diff --git a/packages/css-tokenizer/src/consume/hash-token.ts b/packages/css-tokenizer/src/consume/hash-token.ts deleted file mode 100644 index 3f04ea9eae..0000000000 --- a/packages/css-tokenizer/src/consume/hash-token.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { checkIfThreeCodePointsWouldStartAnIdentSequence } from '../checks/three-code-points-would-start-ident-sequence'; -import { checkIfTwoCodePointsAreAValidEscape } from '../checks/two-code-points-are-valid-escape'; -import { isIdentCodePoint } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import type { TokenDelim, TokenHash} from '../interfaces/token'; -import { HashType, TokenType } from '../interfaces/token'; -import { consumeIdentSequence } from './ident-sequence'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token -export function consumeHashToken(ctx: Context, reader: CodePointReader): TokenDelim|TokenHash { - reader.advanceCodePoint(); - - const codePoint = reader.source.codePointAt(reader.cursor); - if ( - (typeof codePoint !== "undefined") && ( - isIdentCodePoint(codePoint) || - checkIfTwoCodePointsAreAValidEscape(reader) - ) - ) { - let hashType = HashType.Unrestricted; - - if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { - hashType = HashType.ID; - } - - const identSequence = consumeIdentSequence(ctx, reader); - return [ - TokenType.Hash, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: String.fromCodePoint(...identSequence), - type: hashType, - }, - ]; - } - - return [ - TokenType.Delim, - '#', - reader.representationStart, - reader.representationEnd, - { - value: '#', - }, - ]; -} diff --git a/packages/css-tokenizer/src/consume/ident-like-token.ts b/packages/css-tokenizer/src/consume/ident-like-token.ts deleted file mode 100644 index 0370c08c84..0000000000 --- a/packages/css-tokenizer/src/consume/ident-like-token.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { checkIfCodePointsMatchURLIdent } from '../checks/matches-url-ident'; -import { APOSTROPHE, LEFT_PARENTHESIS, QUOTATION_MARK } from '../code-points/code-points'; -import { isWhitespace } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import type { TokenBadURL, TokenFunction, TokenIdent, TokenURL } from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; -import { consumeIdentSequence } from './ident-sequence'; -import { consumeUrlToken } from './url-token'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-ident-like-token -export function consumeIdentLikeToken(ctx: Context, reader: CodePointReader): TokenIdent | TokenFunction | TokenURL | TokenBadURL { - const codePoints = consumeIdentSequence(ctx, reader); - - if (reader.source.codePointAt(reader.cursor) !== LEFT_PARENTHESIS) { - return [ - TokenType.Ident, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: String.fromCodePoint(...codePoints), - }, - ]; - } - - if (checkIfCodePointsMatchURLIdent(codePoints)) { - reader.advanceCodePoint(); - - let read = 0; - while (true) { - const firstIsWhitespace = isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1); - const secondIsWhitespace = isWhitespace(reader.source.codePointAt(reader.cursor + 1) ?? -1); - if (firstIsWhitespace && secondIsWhitespace) { - read = read + 1; - reader.advanceCodePoint(1); - continue; - } - - const firstNonWhitespace = firstIsWhitespace ? reader.source.codePointAt(reader.cursor + 1) : reader.source.codePointAt(reader.cursor); - if (firstNonWhitespace === QUOTATION_MARK || firstNonWhitespace === APOSTROPHE) { - if (read > 0) { - // https://github.com/w3c/csswg-drafts/issues/8280#issuecomment-1370566921 - reader.unreadCodePoint(read); - } - - return [ - TokenType.Function, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: String.fromCodePoint(...codePoints), - }, - ]; - } - - break; - } - - return consumeUrlToken(ctx, reader); - } - - reader.advanceCodePoint(); - return [ - TokenType.Function, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: String.fromCodePoint(...codePoints), - }, - ]; -} diff --git a/packages/css-tokenizer/src/consume/ident-sequence.ts b/packages/css-tokenizer/src/consume/ident-sequence.ts deleted file mode 100644 index 53d5ea9574..0000000000 --- a/packages/css-tokenizer/src/consume/ident-sequence.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { checkIfTwoCodePointsAreAValidEscape } from '../checks/two-code-points-are-valid-escape'; -import { NULL, REPLACEMENT_CHARACTER } from '../code-points/code-points'; -import { isIdentCodePoint, isSurrogate } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { consumeEscapedCodePoint } from './escaped-code-point'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-name -export function consumeIdentSequence(ctx: Context, reader: CodePointReader): Array { - const result: Array = []; - - while (true) { - const codePoint = reader.source.codePointAt(reader.cursor) ?? -1; - if (codePoint === NULL || isSurrogate(codePoint)) { - result.push(REPLACEMENT_CHARACTER); - reader.advanceCodePoint(1 + +(codePoint > 0xffff)); - continue; - } - - if (isIdentCodePoint(codePoint)) { - result.push(codePoint); - reader.advanceCodePoint(1 + +(codePoint > 0xffff)); - continue; - } - - if (checkIfTwoCodePointsAreAValidEscape(reader)) { - reader.advanceCodePoint(); - result.push(consumeEscapedCodePoint(ctx, reader)); - continue; - } - - return result; - } -} diff --git a/packages/css-tokenizer/src/consume/number.ts b/packages/css-tokenizer/src/consume/number.ts deleted file mode 100644 index 95e57a5c7b..0000000000 --- a/packages/css-tokenizer/src/consume/number.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { FULL_STOP, HYPHEN_MINUS, LATIN_CAPITAL_LETTER_E, LATIN_SMALL_LETTER_E, PLUS_SIGN } from '../code-points/code-points'; -import { isDigitCodePoint } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { NumberType } from '../interfaces/token'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-number -export function consumeNumber(ctx: Context, reader: CodePointReader): NumberType { - // 1. Initially set type to "integer". - let type = NumberType.Integer; - - // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), consume it and append it to repr. - if (reader.source.codePointAt(reader.cursor) === PLUS_SIGN || reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { - reader.advanceCodePoint(); - } - - // 3. While the next input code point is a digit, consume it and append it to repr. - while (isDigitCodePoint(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - - // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then: - if (reader.source.codePointAt(reader.cursor) === FULL_STOP && isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1)) { - // 4.1. Consume them. - reader.advanceCodePoint(2); - - // 4.3. Set type to "number". - type = NumberType.Number; - - // 4.4. While the next input code point is a digit, consume it and append it to repr. - while (isDigitCodePoint(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - } - - // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) or U+0065 LATIN SMALL LETTER E (e), - // optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+), - // followed by a digit, then: - if (reader.source.codePointAt(reader.cursor) === LATIN_SMALL_LETTER_E || reader.source.codePointAt(reader.cursor) === LATIN_CAPITAL_LETTER_E) { - if (isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1)) { - // 5.1. Consume them. - reader.advanceCodePoint(2); - } else if ( - (reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS || reader.source.codePointAt(reader.cursor + 1) === PLUS_SIGN) && - isDigitCodePoint(reader.source.codePointAt(reader.cursor + 2) ?? -1) - ) { - // 5.1. Consume them. - reader.advanceCodePoint(3); - } else { - return type; - } - - // 5.3. Set type to "number". - type = NumberType.Number; - - // 5.4. While the next input code point is a digit, consume it and append it to repr. - while (isDigitCodePoint(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - } - - return type; -} diff --git a/packages/css-tokenizer/src/consume/numeric-token.ts b/packages/css-tokenizer/src/consume/numeric-token.ts deleted file mode 100644 index fb27bb1cce..0000000000 --- a/packages/css-tokenizer/src/consume/numeric-token.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { checkIfThreeCodePointsWouldStartAnIdentSequence } from '../checks/three-code-points-would-start-ident-sequence'; -import { HYPHEN_MINUS, PERCENTAGE_SIGN, PLUS_SIGN } from '../code-points/code-points'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import type { TokenDimension, TokenNumber, TokenPercentage} from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; -import { consumeIdentSequence } from './ident-sequence'; -import { consumeNumber } from './number'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-numeric-token -export function consumeNumericToken(ctx: Context, reader: CodePointReader): TokenPercentage|TokenNumber|TokenDimension { - let signCharacter: undefined | '+' | '-' = undefined; - - { - const peeked = reader.source.codePointAt(reader.cursor); - if (peeked === HYPHEN_MINUS) { - signCharacter = '-'; - } else if (peeked === PLUS_SIGN) { - signCharacter = '+'; - } - } - - const numberType = consumeNumber(ctx, reader); - const numberValue = parseFloat(reader.source.slice(reader.representationStart, reader.representationEnd + 1)); - - if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { - const unit = consumeIdentSequence(ctx, reader); - return [ - TokenType.Dimension, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: numberValue, - signCharacter: signCharacter, - type: numberType, - unit: String.fromCodePoint(...unit), - }, - ]; - } - - if (reader.source.codePointAt(reader.cursor) === PERCENTAGE_SIGN) { - reader.advanceCodePoint(); - - return [ - TokenType.Percentage, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: numberValue, - signCharacter: signCharacter, - }, - ]; - } - - return [ - TokenType.Number, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: numberValue, - signCharacter: signCharacter, - type: numberType, - }, - ]; -} diff --git a/packages/css-tokenizer/src/consume/string-token.ts b/packages/css-tokenizer/src/consume/string-token.ts deleted file mode 100644 index da4a3e9e76..0000000000 --- a/packages/css-tokenizer/src/consume/string-token.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { CARRIAGE_RETURN, LINE_FEED, NULL, REPLACEMENT_CHARACTER, REVERSE_SOLIDUS } from '../code-points/code-points'; -import { isNewLine, isSurrogate } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { ParseErrorWithToken, ParseErrorMessage } from '../interfaces/error'; -import type { CSSToken, TokenBadString, TokenString} from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; -import { consumeEscapedCodePoint } from './escaped-code-point'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-string-token -export function consumeStringToken(ctx: Context, reader: CodePointReader): TokenBadString|TokenString { - let result = ''; - - const first = reader.readCodePoint(); - - while (true) { - const next = reader.readCodePoint(); - - if (typeof next === "undefined") { - const token: CSSToken = [TokenType.String, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, { value: result }]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedEOFInString, - reader.representationStart, - reader.representationEnd, - [ - '4.3.5. Consume a string token', - 'Unexpected EOF', - ], - token - )); - - return token; - } - - if (isNewLine(next)) { - reader.unreadCodePoint(); - - const token: CSSToken = [TokenType.BadString, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, undefined]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedNewLineInString, - reader.representationStart, - ( - ( - reader.source.codePointAt(reader.cursor) === CARRIAGE_RETURN && - reader.source.codePointAt(reader.cursor + 1) === LINE_FEED - ) ? - // CR LF - reader.representationEnd + 2 : - // LF - reader.representationEnd + 1 - ), - [ - '4.3.5. Consume a string token', - 'Unexpected newline', - ], - token - )); - - return token; - } - - if (next === first) { - return [TokenType.String, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, { value: result }]; - } - - if (next === REVERSE_SOLIDUS) { - if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { - continue; - } - if (isNewLine(reader.source.codePointAt(reader.cursor) ?? -1)) { - if ( - reader.source.codePointAt(reader.cursor) === CARRIAGE_RETURN && - reader.source.codePointAt(reader.cursor + 1) === LINE_FEED - ) { - reader.advanceCodePoint(); - } - - reader.advanceCodePoint(); - continue; - } - - result = result + String.fromCodePoint(consumeEscapedCodePoint(ctx, reader)); - continue; - } - - if (next === NULL || isSurrogate(next)) { - result = result + String.fromCodePoint(REPLACEMENT_CHARACTER); - continue; - } - - result = result + String.fromCodePoint(next); - } -} diff --git a/packages/css-tokenizer/src/consume/unicode-range-token.ts b/packages/css-tokenizer/src/consume/unicode-range-token.ts deleted file mode 100644 index 2d03100c02..0000000000 --- a/packages/css-tokenizer/src/consume/unicode-range-token.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { DIGIT_ZERO, HYPHEN_MINUS, LATIN_CAPITAL_LETTER_F, QUESTION_MARK } from '../code-points/code-points'; -import { isHexDigitCodePoint } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import type { TokenUnicodeRange } from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; - -// https://drafts.csswg.org/css-syntax/#starts-a-unicode-range -export function consumeUnicodeRangeToken(ctx: Context, reader: CodePointReader): TokenUnicodeRange { - // 1. Consume the next two input code points and discard them. - reader.advanceCodePoint(2); - - const firstSegment: Array = []; - const secondSegment: Array = []; - - // 2. Consume as many hex digits as possible, - // but no more than 6. - let codePoint: number | undefined; - while ( - (typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && - firstSegment.length < 6 && - isHexDigitCodePoint(codePoint) - ) { - firstSegment.push(codePoint); - reader.advanceCodePoint(); - } - - // 2. If less than 6 hex digits were consumed, - // consume as many U+003F QUESTION MARK (?) code points as possible, - // but no more than enough to make the total of hex digits and U+003F QUESTION MARK (?) code points equal to 6. - while ( - (typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && - firstSegment.length < 6 && - codePoint === QUESTION_MARK - ) { - if (secondSegment.length === 0) { - secondSegment.push(...firstSegment); - } - - // 3. If first segment contains any question mark code points, then: - // 3.1 Replace the question marks in first segment with U+0030 DIGIT ZERO (0) code points. - firstSegment.push(DIGIT_ZERO); - // 3.2.Replace the question marks in first segment with U+0046 LATIN CAPITAL LETTER F (F) code points. - secondSegment.push(LATIN_CAPITAL_LETTER_F); - reader.advanceCodePoint(); - } - - if (!secondSegment.length) { - // 5. If the next 2 input code points are U+002D HYPHEN-MINUS (-) followed by a hex digit - if ( - reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS && - isHexDigitCodePoint(reader.source.codePointAt(reader.cursor + 1) ?? -1) - ) { - // 5.1. Consume the next input code point. - reader.advanceCodePoint(); - - // 5.2 Consume as many hex digits as possible, - // but no more than 6. - while ( - (typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && - secondSegment.length < 6 && - isHexDigitCodePoint(codePoint) - ) { - secondSegment.push(codePoint); - reader.advanceCodePoint(); - } - } - } - - if (!secondSegment.length) { - // Interpret the consumed code points as a hexadecimal number. - const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); - - // Return a new both starting and ending at start of range. - return [ - TokenType.UnicodeRange, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - startOfRange: startOfRange, - endOfRange: startOfRange, - }, - ]; - } - - // Interpret the consumed code points as a hexadecimal number. - const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); - const endOfRange = parseInt(String.fromCodePoint(...secondSegment), 16); - - // Return a new starting at start of range and ending at end of range. - return [ - TokenType.UnicodeRange, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - startOfRange: startOfRange, - endOfRange: endOfRange, - }, - ]; -} diff --git a/packages/css-tokenizer/src/consume/url-token.ts b/packages/css-tokenizer/src/consume/url-token.ts deleted file mode 100644 index 1b3f9599de..0000000000 --- a/packages/css-tokenizer/src/consume/url-token.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { checkIfTwoCodePointsAreAValidEscape } from '../checks/two-code-points-are-valid-escape'; -import { APOSTROPHE, LEFT_PARENTHESIS, NULL, QUOTATION_MARK, REPLACEMENT_CHARACTER, REVERSE_SOLIDUS, RIGHT_PARENTHESIS } from '../code-points/code-points'; -import { isNonPrintableCodePoint, isSurrogate, isWhitespace } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { Context } from '../interfaces/context'; -import { ParseErrorWithToken, ParseErrorMessage } from '../interfaces/error'; -import type { CSSToken, TokenBadURL, TokenURL } from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; -import { consumeBadURL } from './bad-url'; -import { consumeEscapedCodePoint } from './escaped-code-point'; - -// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-url-token -export function consumeUrlToken(ctx: Context, reader: CodePointReader): TokenURL|TokenBadURL { - while (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - - let string = ''; - - while (true) { - if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { - const token: CSSToken = [ - TokenType.URL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: string, - }, - ]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedEOFInURL, - reader.representationStart, - reader.representationEnd, - [ - '4.3.6. Consume a url token', - 'Unexpected EOF', - ], - token, - )); - - return token; - } - - if (reader.source.codePointAt(reader.cursor) === RIGHT_PARENTHESIS) { - reader.advanceCodePoint(); - return [ - TokenType.URL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: string, - }, - ]; - } - - if (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - while (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - - if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { - const token: CSSToken = [ - TokenType.URL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: string, - }, - ]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedEOFInURL, - reader.representationStart, - reader.representationEnd, - [ - '4.3.6. Consume a url token', - 'Consume as much whitespace as possible', - 'Unexpected EOF', - ], - token - )); - - return token; - } - - if (reader.source.codePointAt(reader.cursor) === RIGHT_PARENTHESIS) { - reader.advanceCodePoint(); - return [ - TokenType.URL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - { - value: string, - }, - ]; - } - - consumeBadURL(ctx, reader); - return [ - TokenType.BadURL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; - } - - const codePoint = reader.source.codePointAt(reader.cursor); - if (codePoint === QUOTATION_MARK || codePoint === APOSTROPHE || codePoint === LEFT_PARENTHESIS || isNonPrintableCodePoint(codePoint ?? -1)) { - consumeBadURL(ctx, reader); - - const token: CSSToken = [ - TokenType.BadURL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.UnexpectedCharacterInURL, - reader.representationStart, - reader.representationEnd, - [ - '4.3.6. Consume a url token', - 'Unexpected U+0022 QUOTATION MARK ("), U+0027 APOSTROPHE (\'), U+0028 LEFT PARENTHESIS (() or non-printable code point', - ], - token - )); - - return token; - } - - if (codePoint === REVERSE_SOLIDUS) { - if (checkIfTwoCodePointsAreAValidEscape(reader)) { - reader.advanceCodePoint(); - string = string + String.fromCodePoint(consumeEscapedCodePoint(ctx, reader)); - continue; - } - - consumeBadURL(ctx, reader); - - const token: CSSToken = [ - TokenType.BadURL, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; - - ctx.onParseError(new ParseErrorWithToken( - ParseErrorMessage.InvalidEscapeSequenceInURL, - reader.representationStart, - reader.representationEnd, - [ - '4.3.6. Consume a url token', - 'U+005C REVERSE SOLIDUS (\\)', - 'The input stream does not start with a valid escape sequence', - ], - token - )); - - return token; - } - - if (reader.source.codePointAt(reader.cursor) === NULL || isSurrogate(reader.source.codePointAt(reader.cursor) ?? -1)) { - string = string + String.fromCodePoint(REPLACEMENT_CHARACTER); - reader.advanceCodePoint(); - continue; - } - - string = string + reader.source[reader.cursor]; - reader.advanceCodePoint(); - } -} diff --git a/packages/css-tokenizer/src/consume/whitespace-token.ts b/packages/css-tokenizer/src/consume/whitespace-token.ts deleted file mode 100644 index 21a74c27f2..0000000000 --- a/packages/css-tokenizer/src/consume/whitespace-token.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { isWhitespace } from '../code-points/ranges'; -import type { CodePointReader } from '../interfaces/code-point-reader'; -import type { TokenWhitespace } from '../interfaces/token'; -import { TokenType } from '../interfaces/token'; - -export function consumeWhiteSpace(reader: CodePointReader): TokenWhitespace { - while (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) { - reader.advanceCodePoint(); - } - - return [ - TokenType.Whitespace, - reader.source.slice(reader.representationStart, reader.representationEnd + 1), - reader.representationStart, - reader.representationEnd, - undefined, - ]; -} diff --git a/packages/css-tokenizer/src/interfaces/code-point-reader.ts b/packages/css-tokenizer/src/interfaces/code-point-reader.ts deleted file mode 100644 index 953696eb27..0000000000 --- a/packages/css-tokenizer/src/interfaces/code-point-reader.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** - * @internal - */ -export type CodePointReader = { - representationStart: number; - representationEnd: number; - - cursor: number; - source: string; - - advanceCodePoint(n?: number): void - readCodePoint(): number | undefined - unreadCodePoint(n?: number): void - resetRepresentation(): void -}; diff --git a/packages/css-tokenizer/src/interfaces/context.ts b/packages/css-tokenizer/src/interfaces/context.ts deleted file mode 100644 index 5d2eac72a2..0000000000 --- a/packages/css-tokenizer/src/interfaces/context.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { ParseError } from './error'; - -export type Context = { - onParseError: (error: ParseError) => void -}; diff --git a/packages/css-tokenizer/src/reader.ts b/packages/css-tokenizer/src/reader.ts deleted file mode 100644 index d95b0cbcc2..0000000000 --- a/packages/css-tokenizer/src/reader.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { CodePointReader } from './interfaces/code-point-reader'; - -/** - * @internal - */ -export class Reader implements CodePointReader { - cursor = 0; - source = ''; - - representationStart = 0; - representationEnd = -1; - - constructor(source: string) { - this.source = source; - } - - advanceCodePoint(n = 1): void { - this.cursor = this.cursor + n; - this.representationEnd = this.cursor - 1; - } - - readCodePoint(): number | undefined { - const codePoint = this.source.codePointAt(this.cursor); - if (typeof codePoint === "undefined") { - return undefined; - } - - this.cursor = this.cursor + 1; - this.representationEnd = this.cursor - 1; - - return codePoint; - } - - unreadCodePoint(n = 1): void { - this.cursor = this.cursor - n; - this.representationEnd = this.cursor - 1; - } - - resetRepresentation(): void { - this.representationStart = this.cursor; - this.representationEnd = -1; - } -} diff --git a/packages/css-tokenizer/src/tokenizer.ts b/packages/css-tokenizer/src/tokenizer.ts index 83176c7878..5bec873a4c 100644 --- a/packages/css-tokenizer/src/tokenizer.ts +++ b/packages/css-tokenizer/src/tokenizer.ts @@ -1,25 +1,52 @@ -import { checkIfFourCodePointsWouldStartCDO } from './checks/four-code-points-would-start-cdo'; -import { checkIfThreeCodePointsWouldStartAnIdentSequence } from './checks/three-code-points-would-start-ident-sequence'; -import { checkIfThreeCodePointsWouldStartANumber } from './checks/three-code-points-would-start-number'; -import { checkIfTwoCodePointsStartAComment } from './checks/two-code-points-start-comment'; -import { checkIfThreeCodePointsWouldStartCDC } from './checks/three-code-points-would-start-cdc'; -import { APOSTROPHE, CARRIAGE_RETURN, CHARACTER_TABULATION, COLON, COMMA, COMMERCIAL_AT, FORM_FEED, FULL_STOP, HYPHEN_MINUS, LATIN_CAPITAL_LETTER_U, LATIN_SMALL_LETTER_U, LEFT_CURLY_BRACKET, LEFT_PARENTHESIS, LEFT_SQUARE_BRACKET, LESS_THAN_SIGN, LINE_FEED, NUMBER_SIGN, PLUS_SIGN, QUOTATION_MARK, REVERSE_SOLIDUS, RIGHT_CURLY_BRACKET, RIGHT_PARENTHESIS, RIGHT_SQUARE_BRACKET, SEMICOLON, SOLIDUS, SPACE } from './code-points/code-points'; -import { isDigitCodePoint, isIdentStartCodePoint } from './code-points/ranges'; -import { consumeComment } from './consume/comment'; -import { consumeHashToken } from './consume/hash-token'; -import { consumeIdentSequence } from './consume/ident-sequence'; -import { consumeNumericToken } from './consume/numeric-token'; -import { consumeWhiteSpace } from './consume/whitespace-token'; -import type { CSSToken} from './interfaces/token'; -import { TokenType } from './interfaces/token'; -import { Reader } from './reader'; -import { consumeStringToken } from './consume/string-token'; -import { consumeIdentLikeToken } from './consume/ident-like-token'; -import { checkIfTwoCodePointsAreAValidEscape } from './checks/two-code-points-are-valid-escape'; -import type { ParseError} from './interfaces/error'; -import { ParseErrorMessage, ParseErrorWithToken } from './interfaces/error'; -import { checkIfThreeCodePointsWouldStartAUnicodeRange } from './checks/three-code-points-would-start-unicode-range'; -import { consumeUnicodeRangeToken } from './consume/unicode-range-token'; +import { + APOSTROPHE, + ASTERISK, + CARRIAGE_RETURN, + CHARACTER_TABULATION, + COLON, + COMMA, + COMMERCIAL_AT, + DIGIT_ZERO, + EXCLAMATION_MARK, + FORM_FEED, + FULL_STOP, + GREATER_THAN_SIGN, + HYPHEN_MINUS, + LATIN_CAPITAL_LETTER_E, + LATIN_CAPITAL_LETTER_F, + LATIN_CAPITAL_LETTER_L, + LATIN_CAPITAL_LETTER_R, + LATIN_CAPITAL_LETTER_U, + LATIN_SMALL_LETTER_E, + LATIN_SMALL_LETTER_L, + LATIN_SMALL_LETTER_R, + LATIN_SMALL_LETTER_U, + LEFT_CURLY_BRACKET, + LEFT_PARENTHESIS, + LEFT_SQUARE_BRACKET, + LESS_THAN_SIGN, + LINE_FEED, + LOW_LINE, + MAXIMUM_ALLOWED_CODEPOINT, + NULL, + NUMBER_SIGN, + PERCENTAGE_SIGN, + PLUS_SIGN, + QUESTION_MARK, + QUOTATION_MARK, + REPLACEMENT_CHARACTER, + REVERSE_SOLIDUS, + RIGHT_CURLY_BRACKET, + RIGHT_PARENTHESIS, + RIGHT_SQUARE_BRACKET, + SEMICOLON, + SOLIDUS, + SPACE, +} from './code-points/code-points'; +import type { CSSToken } from './interfaces/token'; +import { HashType, NumberType, TokenType } from './interfaces/token'; +import { ParseError, ParseErrorMessage, ParseErrorWithToken } from './interfaces/error'; +import { isNewLine, isSurrogate, isNonASCII_IdentCodePoint, isHexDigitCodePoint, isWhitespace, isNonPrintableCodePoint, isIdentStartCodePoint, isDigitCodePoint } from './code-points/ranges'; /** * Tokenize a CSS string into a list of tokens. @@ -60,175 +87,637 @@ export function tokenizer( onParseError?: (error: ParseError) => void }, ): { nextToken: () => CSSToken, endOfFile: () => boolean } { - const css = input.css.valueOf(); + const source = input.css.valueOf(); const unicodeRangesAllowed = input.unicodeRangesAllowed ?? false; - const reader = new Reader(css); + /** The position of the next code point that will be read. */ + let cursor = 0; + /** The index of the first code point of the current token. */ + let representationStart = 0; + /** The index of the last code point of the current token. */ + let representationEnd = -1; - const ctx = { - onParseError: options?.onParseError ?? noop, - }; + /** + * Set by {@link consumeIdentSequence}: true when the returned ident value is the raw + * source substring (no NULL, surrogate or escape code points were decoded). + */ + let identIsPlainRepr = false; - function endOfFile(): boolean { - return typeof reader.source.codePointAt(reader.cursor) === "undefined"; - } + const onParseError = options?.onParseError ?? noop; - function nextToken(): CSSToken { - reader.resetRepresentation(); + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-name + // + // Fast path: when the ident sequence contains no NULL, surrogate or escape code points, + // the decoded value is the raw source substring and is returned as a single slice. + function consumeIdentSequence(): string { + const start = cursor; - const peeked = reader.source.codePointAt(reader.cursor); - if (typeof peeked === "undefined") { - return [TokenType.EOF, '', -1, -1, undefined]; - } + // Scan the ident sequence without building a string. + // NULL, surrogates and escapes must be decoded and fall through to the decode pass. + while (true) { + // ASCII fast path: letters, digits, underscore and hyphen are always ident code points. + const asciiCodeUnit = source.charCodeAt(cursor); + if ( + (asciiCodeUnit >= 0x0041 && asciiCodeUnit <= 0x005a) || // A .. Z + (asciiCodeUnit >= 0x0061 && asciiCodeUnit <= 0x007a) || // a .. z + (asciiCodeUnit >= 0x0030 && asciiCodeUnit <= 0x0039) || // 0 .. 9 + asciiCodeUnit === HYPHEN_MINUS || + asciiCodeUnit === LOW_LINE + ) { + cursor += 1; + continue; + } + + const codePoint = source.codePointAt(cursor) ?? -1; + if ( + codePoint === NULL || + (codePoint >= 0xd800 && codePoint <= 0xdfff) || // surrogate + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-valid-escape + (codePoint === REVERSE_SOLIDUS && !isNewLine(source.codePointAt(cursor + 1) ?? -1)) + ) { + break; + } - if (peeked === SOLIDUS && checkIfTwoCodePointsStartAComment(reader)) { - return consumeComment(ctx, reader); + // A non-ASCII ident code point. + // https://drafts.csswg.org/css-syntax/#non-ascii-ident-code-point + if (isNonASCII_IdentCodePoint(codePoint)) { + cursor += 1 + +(codePoint > 0xffff); + continue; + } + + break; } + // If the scan stopped on a code point that does not need decoding, the ident is plain. + const stopCodePoint = source.codePointAt(cursor) ?? -1; if ( - unicodeRangesAllowed && ( - peeked === LATIN_SMALL_LETTER_U || - peeked === LATIN_CAPITAL_LETTER_U - ) && - checkIfThreeCodePointsWouldStartAUnicodeRange(reader) + stopCodePoint !== NULL && + !(stopCodePoint >= 0xd800 && stopCodePoint <= 0xdfff) && // surrogate + !(stopCodePoint === REVERSE_SOLIDUS && !isNewLine(source.codePointAt(cursor + 1) ?? -1)) // valid escape ) { - return consumeUnicodeRangeToken(ctx, reader); + identIsPlainRepr = true; + representationEnd = cursor - 1; + return source.slice(start, cursor); } - if (isIdentStartCodePoint(peeked)) { - return consumeIdentLikeToken(ctx, reader); + // Decode pass: NULL, surrogate and escape code points require decoding. + identIsPlainRepr = false; + cursor = start; + representationEnd = -1; + + let result = ''; + + while (true) { + const codePoint = source.codePointAt(cursor) ?? -1; + if (codePoint === NULL || isSurrogate(codePoint)) { + result += String.fromCharCode(REPLACEMENT_CHARACTER); + cursor += 1 + +(codePoint > 0xffff); + representationEnd = cursor - 1; + continue; + } + + // An ident code point is a letter, a digit, an underscore, a hyphen or a non-ASCII ident code point. + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-code-point + if ( + (codePoint >= 0x0041 && codePoint <= 0x005a) || // A .. Z + (codePoint >= 0x0061 && codePoint <= 0x007a) || // a .. z + (codePoint >= 0x0030 && codePoint <= 0x0039) || // 0 .. 9 + codePoint === HYPHEN_MINUS || + codePoint === LOW_LINE || + isNonASCII_IdentCodePoint(codePoint) + ) { + result += codePoint > 0xffff ? String.fromCodePoint(codePoint) : String.fromCharCode(codePoint); + cursor += 1 + +(codePoint > 0xffff); + representationEnd = cursor - 1; + continue; + } + + if (source.codePointAt(cursor) === REVERSE_SOLIDUS && !isNewLine(source.codePointAt(cursor + 1) ?? -1)) { + cursor += 1; + representationEnd = cursor - 1; + result += String.fromCodePoint(consumeEscapedCodePoint()); + continue; + } + + return result; + } + } + + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-escaped-code-point + function consumeEscapedCodePoint(): number { + const codePoint = source.codePointAt(cursor); + if (typeof codePoint === "undefined") { + onParseError(new ParseError( + ParseErrorMessage.UnexpectedEOFInEscapedCodePoint, + representationStart, + representationEnd, + [ + '4.3.7. Consume an escaped code point', + 'Unexpected EOF', + ], + )); + + return REPLACEMENT_CHARACTER; + } + + cursor += 1 + +(codePoint > 0xffff); + representationEnd = cursor - 1; + + if (isHexDigitCodePoint(codePoint)) { + const hexSequence: Array = [codePoint]; + + let nextCodePoint: number | undefined; + while ((typeof (nextCodePoint = source.codePointAt(cursor)) !== "undefined") && isHexDigitCodePoint(nextCodePoint) && hexSequence.length < 6) { + hexSequence.push(nextCodePoint); + cursor += 1; + representationEnd = cursor - 1; + } + + if (isWhitespace(source.codePointAt(cursor) ?? -1)) { + if ( + source.codePointAt(cursor) === CARRIAGE_RETURN && + source.codePointAt(cursor + 1) === LINE_FEED + ) { + cursor += 1; + } + + cursor += 1; + representationEnd = cursor - 1; + } + + const codePointLiteral = parseInt(String.fromCodePoint(...hexSequence), 16); + if (codePointLiteral === 0 || isSurrogate(codePointLiteral)) { + return REPLACEMENT_CHARACTER; + } + if (codePointLiteral > MAXIMUM_ALLOWED_CODEPOINT) { + return REPLACEMENT_CHARACTER; + } + + return codePointLiteral; } - if (isDigitCodePoint(peeked)) { - return consumeNumericToken(ctx, reader); + if (codePoint === 0 || isSurrogate(codePoint)) { + return REPLACEMENT_CHARACTER; + } + + return codePoint; + } + + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-remnants-of-bad-url + function consumeBadURL(): void { + while (true) { + const codePoint = source.codePointAt(cursor); + if (typeof codePoint === "undefined") { + return; + } + + if (codePoint === RIGHT_PARENTHESIS) { + cursor += 1; + representationEnd = cursor - 1; + return; + } + + if (codePoint === REVERSE_SOLIDUS && !isNewLine(source.codePointAt(cursor + 1) ?? -1)) { + cursor += 1; + representationEnd = cursor - 1; + consumeEscapedCodePoint(); + continue; + } + + cursor += 1; + representationEnd = cursor - 1; + continue; } + } + + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comment + function consumeComment(): CSSToken { + cursor += 2; + representationEnd = cursor - 1; + + while (true) { + const codePoint = source.codePointAt(cursor); + if (typeof codePoint === "undefined") { + const token: CSSToken = [ + TokenType.Comment, + source.slice(representationStart, representationEnd + 1), + representationStart, + representationEnd, + undefined, + ]; - // Simple, one character tokens: - switch (peeked) { - case COMMA: - reader.advanceCodePoint(); - return [TokenType.Comma, ',', reader.representationStart, reader.representationEnd, undefined]; + onParseError(new ParseErrorWithToken( + ParseErrorMessage.UnexpectedEOFInComment, + representationStart, + representationEnd, + [ + '4.3.2. Consume comments', + 'Unexpected EOF', + ], + token + )); - case COLON: - reader.advanceCodePoint(); - return [TokenType.Colon, ':', reader.representationStart, reader.representationEnd, undefined]; + return token; + } + + cursor += 1; + representationEnd = cursor - 1; - case SEMICOLON: - reader.advanceCodePoint(); - return [TokenType.Semicolon, ';', reader.representationStart, reader.representationEnd, undefined]; + if (codePoint !== ASTERISK) { + continue; + } - case LEFT_PARENTHESIS: - reader.advanceCodePoint(); - return [TokenType.OpenParen, '(', reader.representationStart, reader.representationEnd, undefined]; + if (typeof source.codePointAt(cursor) === "undefined") { + continue; + } - case RIGHT_PARENTHESIS: - reader.advanceCodePoint(); - return [TokenType.CloseParen, ')', reader.representationStart, reader.representationEnd, undefined]; + if (source.codePointAt(cursor) === SOLIDUS) { + cursor += 1; + representationEnd = cursor - 1; + break; + } + } - case LEFT_SQUARE_BRACKET: - reader.advanceCodePoint(); - return [TokenType.OpenSquare, '[', reader.representationStart, reader.representationEnd, undefined]; + return [ + TokenType.Comment, + source.slice(representationStart, representationEnd + 1), + representationStart, + representationEnd, + undefined, + ]; + } - case RIGHT_SQUARE_BRACKET: - reader.advanceCodePoint(); - return [TokenType.CloseSquare, ']', reader.representationStart, reader.representationEnd, undefined]; + // https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-string-token + function consumeStringToken(): CSSToken { + let stringValue = ''; + const first = source.codePointAt(cursor); + cursor += 1; + representationEnd = cursor - 1; - case LEFT_CURLY_BRACKET: - reader.advanceCodePoint(); - return [TokenType.OpenCurly, '{', reader.representationStart, reader.representationEnd, undefined]; + while (true) { + const next = source.charCodeAt(cursor); + if (cursor >= source.length) { + const token: CSSToken = [TokenType.String, source.slice(representationStart, representationEnd + 1), representationStart, representationEnd, { value: stringValue }]; - case RIGHT_CURLY_BRACKET: - reader.advanceCodePoint(); - return [TokenType.CloseCurly, '}', reader.representationStart, reader.representationEnd, undefined]; + onParseError(new ParseErrorWithToken( + ParseErrorMessage.UnexpectedEOFInString, + representationStart, + representationEnd, + [ + '4.3.5. Consume a string token', + 'Unexpected EOF', + ], + token + )); - case APOSTROPHE: - case QUOTATION_MARK: - return consumeStringToken(ctx, reader); + return token; + } - case NUMBER_SIGN: - return consumeHashToken(ctx, reader); + cursor += 1; + representationEnd = cursor - 1; - case PLUS_SIGN: - case FULL_STOP: - if (checkIfThreeCodePointsWouldStartANumber(reader)) { - return consumeNumericToken(ctx, reader); + if (isNewLine(next)) { + cursor -= 1; + representationEnd = cursor - 1; + + const token: CSSToken = [TokenType.BadString, source.slice(representationStart, representationEnd + 1), representationStart, representationEnd, undefined]; + + onParseError(new ParseErrorWithToken( + ParseErrorMessage.UnexpectedNewLineInString, + representationStart, + ( + ( + source.codePointAt(cursor) === CARRIAGE_RETURN && + source.codePointAt(cursor + 1) === LINE_FEED + ) ? + // CR LF + representationEnd + 2 : + // LF + representationEnd + 1 + ), + [ + '4.3.5. Consume a string token', + 'Unexpected newline', + ], + token + )); + + return token; + } + + if (next === first) { + return [TokenType.String, source.slice(representationStart, representationEnd + 1), representationStart, representationEnd, { value: stringValue }]; + } + + if (next === REVERSE_SOLIDUS) { + if (typeof source.codePointAt(cursor) === "undefined") { + continue; } + if (isNewLine(source.codePointAt(cursor) ?? -1)) { + if ( + source.codePointAt(cursor) === CARRIAGE_RETURN && + source.codePointAt(cursor + 1) === LINE_FEED + ) { + cursor += 1; + } - reader.advanceCodePoint(); - return [TokenType.Delim, reader.source[reader.representationStart], reader.representationStart, reader.representationEnd, { - value: reader.source[reader.representationStart], - }]; - - case LINE_FEED: - case CARRIAGE_RETURN: - case FORM_FEED: - case CHARACTER_TABULATION: - case SPACE: - return consumeWhiteSpace(reader); - - case HYPHEN_MINUS: - if (checkIfThreeCodePointsWouldStartANumber(reader)) { - return consumeNumericToken(ctx, reader); + cursor += 1; + representationEnd = cursor - 1; + continue; } - if (checkIfThreeCodePointsWouldStartCDC(reader)) { - reader.advanceCodePoint(3); + stringValue = stringValue + String.fromCodePoint(consumeEscapedCodePoint()); + continue; + } - return [TokenType.CDC, '-->', reader.representationStart, reader.representationEnd, undefined]; + if (next === NULL) { + stringValue = stringValue + String.fromCharCode(REPLACEMENT_CHARACTER); + continue; + } + + // ASCII (fast path) + if (next < 0x0080) { + stringValue = stringValue + String.fromCharCode(next); + continue; + } + + // Surrogates: an astral code point is read as a whole, a lone surrogate becomes the replacement character. + if (next >= 0xd800 && next <= 0xdfff) { + const codePoint = source.codePointAt(cursor - 1) ?? -1; + if (codePoint >= 0x10000) { + stringValue = stringValue + String.fromCodePoint(codePoint); + cursor += 1; + representationEnd = cursor - 1; + } else { + stringValue = stringValue + String.fromCharCode(REPLACEMENT_CHARACTER); } + continue; + } - if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { - return consumeIdentLikeToken(ctx, reader); + // Non-ASCII BMP code point. + stringValue = stringValue + String.fromCharCode(next); + } + } + + // https://drafts.csswg.org/css-syntax/#consume-a-unicode-range-token + function consumeUnicodeRangeToken(): CSSToken { + // 1. Consume the next two input code points and discard them. + cursor += 2; + representationEnd = cursor - 1; + + const firstSegment: Array = []; + const secondSegment: Array = []; + + // 2. Consume as many hex digits as possible, + // but no more than 6. + let codePoint: number | undefined; + while ( + (typeof (codePoint = source.codePointAt(cursor)) !== "undefined") && + firstSegment.length < 6 && + isHexDigitCodePoint(codePoint) + ) { + firstSegment.push(codePoint); + cursor += 1; + representationEnd = cursor - 1; + } + + // 2. If less than 6 hex digits were consumed, + // consume as many U+003F QUESTION MARK (?) code points as possible, + // but no more than enough to make the total of hex digits and U+003F QUESTION MARK (?) code points equal to 6. + while ( + (typeof (codePoint = source.codePointAt(cursor)) !== "undefined") && + firstSegment.length < 6 && + codePoint === QUESTION_MARK + ) { + if (secondSegment.length === 0) { + secondSegment.push(...firstSegment); + } + + // 3. If first segment contains any question mark code points, then: + // 3.1 Replace the question marks in first segment with U+0030 DIGIT ZERO (0) code points. + firstSegment.push(DIGIT_ZERO); + // 3.2. Replace the question marks in first segment with U+0046 LATIN CAPITAL LETTER F (F) code points. + secondSegment.push(LATIN_CAPITAL_LETTER_F); + cursor += 1; + representationEnd = cursor - 1; + } + + if (!secondSegment.length) { + // 5. If the next 2 input code points are U+002D HYPHEN-MINUS (-) followed by a hex digit + if ( + source.codePointAt(cursor) === HYPHEN_MINUS && + isHexDigitCodePoint(source.codePointAt(cursor + 1) ?? -1) + ) { + // 5.1. Consume the next input code point. + cursor += 1; + representationEnd = cursor - 1; + + // 5.2 Consume as many hex digits as possible, + // but no more than 6. + while ( + (typeof (codePoint = source.codePointAt(cursor)) !== "undefined") && + secondSegment.length < 6 && + isHexDigitCodePoint(codePoint) + ) { + secondSegment.push(codePoint); + cursor += 1; + representationEnd = cursor - 1; } + } + } - reader.advanceCodePoint(); - return [TokenType.Delim, '-', reader.representationStart, reader.representationEnd, { - value: '-', - }]; + if (!secondSegment.length) { + // Interpret the consumed code points as a hexadecimal number. + const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); + + // Return a new both starting and ending at start of range. + return [ + TokenType.UnicodeRange, + source.slice(representationStart, representationEnd + 1), + representationStart, + representationEnd, + { + startOfRange: startOfRange, + endOfRange: startOfRange, + }, + ]; + } - case LESS_THAN_SIGN: - if (checkIfFourCodePointsWouldStartCDO(reader)) { - reader.advanceCodePoint(4); + // Interpret the consumed code points as a hexadecimal number. + const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); + const endOfRange = parseInt(String.fromCodePoint(...secondSegment), 16); - return [TokenType.CDO, '', representationStart, representationEnd, undefined]; + } + + if (checkIfThreeCodePointsWouldStartAnIdentSequence(peeked)) { + return consumeIdentLikeToken(); + } + + cursor += 1; + representationEnd = cursor - 1; + return [TokenType.Delim, '-', representationStart, representationEnd, { + value: '-', + }]; + + case LESS_THAN_SIGN: + if (source.charCodeAt(cursor + 1) === EXCLAMATION_MARK && source.charCodeAt(cursor + 2) === HYPHEN_MINUS && source.charCodeAt(cursor + 3) === HYPHEN_MINUS) { // CDO + cursor += 4; + representationEnd = cursor - 1; + + return [TokenType.CDO, '', 0, 2, undefined], + [TokenType.CDO]: [TokenType.CDO, '