From 0238d793d2ed80b29a1e50e4be983bfb5615cbd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Mon, 7 Sep 2026 23:20:15 +0200 Subject: [PATCH 01/20] Stream CSS URL replacements across resumable file chunks --- .../DataLiberation/CSS/class-cssprocessor.php | 474 ++++++++++++++++-- components/DataLiberation/README.md | 67 +++ .../DataLiberation/Tests/CSSStreamTest.php | 51 ++ .../Tests/CSSURLStreamProcessTest.php | 74 +++ .../DataLiberation/Tests/CSSURLStreamTest.php | 168 +++++++ .../fixtures/css-stream/rewrite-file.php | 42 ++ .../URL/class-cssurlprocessor.php | 317 +++++++++++- 7 files changed, 1133 insertions(+), 60 deletions(-) create mode 100644 components/DataLiberation/Tests/CSSStreamTest.php create mode 100644 components/DataLiberation/Tests/CSSURLStreamProcessTest.php create mode 100644 components/DataLiberation/Tests/CSSURLStreamTest.php create mode 100644 components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php diff --git a/components/DataLiberation/CSS/class-cssprocessor.php b/components/DataLiberation/CSS/class-cssprocessor.php index e54f88864..c8b7afd08 100644 --- a/components/DataLiberation/CSS/class-cssprocessor.php +++ b/components/DataLiberation/CSS/class-cssprocessor.php @@ -308,6 +308,15 @@ class CSSProcessor { */ private $lexical_updates = array(); + /** @var array|null Durable lexical state for chunked input; null for whole strings. */ + private $stream; + + /** @var int|null Last byte at which a new CSS code point may begin in this chunk. */ + private $fragment_limit; + + /** @var bool Whether this fragment consumed the unquoted URL closing parenthesis. */ + private $url_closed = false; + /** * Constructor for the CSS processor. * @@ -341,6 +350,236 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { return new static( $css ); } + /** + * Opens a bounded-input tokenizer. Drain next_token_fragment() before appending. + * + * Fragments preserve source bytes. Strings, comments, names, and URLs may span + * fragments. url( is exposed as a function before its quoted or unquoted value; + * the whole-string next_token() API keeps its existing CSS Syntax token types. + * + * @param array|null $cursor { + * Optional state returned by get_reentrancy_cursor(). + * @type array $lexer Lexical continuation state. + * @type string $pending_b64 Unconsumed source bytes, base64 encoded. + * } + * @return static + */ + public static function create_for_streaming( ?array $cursor = null ) { + $processor = new static( '' ); + $processor->stream = array( + 'phase' => '', + 'quote' => '', + 'name' => '', + 'kind' => '', + 'number' => 'integer', + 'finished' => false, + ); + if ( null !== $cursor ) { + if ( ! isset( $cursor['lexer'], $cursor['pending_b64'] ) || ! is_array( $cursor['lexer'] ) || ! is_string( $cursor['pending_b64'] ) ) { + throw new \InvalidArgumentException( 'The CSS cursor must contain lexer state and a base64 input tail.' ); + } + $processor->stream = $cursor['lexer']; + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. + $processor->css = base64_decode( $cursor['pending_b64'], true ); + if ( false === $processor->css ) { + throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' ); + } + $processor->length = strlen( $processor->css ); + } + return $processor; + } + + /** + * Supplies at most 64 KiB of source bytes. The caller marks actual EOF explicitly. + * + * @param string $bytes Next source chunk. + * @param bool $is_last Whether this is the end of the stylesheet. + */ + public function append_bytes( string $bytes, bool $is_last = false ): void { + if ( null === $this->stream || $this->stream['finished'] ) { + throw new \LogicException( 'CSS input can only be appended to an unfinished streaming processor.' ); + } + if ( strlen( $bytes ) > 65536 || $this->length - $this->at > 13 ) { + throw new \InvalidArgumentException( 'CSS input must be drained before appending a chunk of at most 65536 bytes; received ' . strlen( $bytes ) . ' bytes with ' . ( $this->length - $this->at ) . ' unread bytes.' ); + } + $this->css = substr( $this->css, $this->at ) . $bytes; + $this->at = 0; + $this->length = strlen( $this->css ); + $this->stream['finished'] = $is_last; + } + + /** + * Returns one token fragment, or null when more input is needed or EOF is reached. + * + * Names are retained only up to 32 decoded bytes. Longer names cannot be url, + * import, image-set, or -webkit-image-set; their source bytes still pass through. + * + * @return array|null { + * @type string $text Original source bytes for this fragment. + * @type string $type CSS token type; a later fragment may finish a name as a function. + * @type string $name Complete short function or at-keyword name, otherwise empty. + * @type string $value Decoded string or URL value fragment, excluding its syntax. + * @type string $raw_value Original bytes of that value fragment. + * @type string $before Syntax preceding the value, such as an opening quote. + * @type string $after Syntax following the value, such as a closing parenthesis. + * @type bool $first For strings and URLs, whether this begins their value. + * @type bool $last For strings and URLs, whether this finishes their value. + * @type bool $closes_url Whether an unquoted URL consumed its closing parenthesis. + * } + */ + public function next_token_fragment(): ?array { + if ( null === $this->stream ) { + throw new \LogicException( 'Token fragments require create_for_streaming().' ); + } + // A CSS escape needs at most six hex digits and CRLF after its backslash. + // Keeping ten bytes also covers UTF-8, number lookahead, and comment delimiters. + $this->fragment_limit = $this->stream['finished'] ? $this->length : max( 0, $this->length - 10 ); + if ( ! $this->stream['finished'] && $this->fragment_limit > 0 ) { + $start = $this->fragment_limit; + for ( $back = 0; $back < 3 && $start > 0; ++$back ) { + if ( 0x80 !== ( ord( $this->css[ $start ] ) & 0xc0 ) ) { + break; + } + --$start; + } + $end = $start; + $invalid = 0; + if ( $start < $this->fragment_limit && 1 === _wp_scan_utf8( $this->css, $end, $invalid, null, 1 ) && $end > $this->fragment_limit ) { + $this->fragment_limit = $start; + } + } + if ( $this->at >= $this->fragment_limit ) { + return null; + } + $start = $this->at; + $phase = $this->stream['phase']; + $this->after_token(); + $this->token_starts_at = $start; + switch ( $phase ) { + case 'comment': + $this->consume_comment_fragment( false ); + break; + case 'string': + $this->consume_string( $this->stream['quote'] ); + break; + case 'url-start': + $whitespace = strspn( $this->css, "\t\n\f\r ", $this->at, $this->fragment_limit - $this->at ); + if ( $whitespace ) { + $this->at += $whitespace; + $this->token_type = self::TOKEN_WHITESPACE; + } elseif ( '"' === $this->css[ $this->at ] || "'" === $this->css[ $this->at ] ) { + $this->stream['phase'] = ''; + $this->consume_string(); + } else { + $this->consume_url( true ); + } + break; + case 'url': + case 'url-space': + $this->consume_url( true, 'url-space' === $phase ); + break; + case 'bad-url': + $this->consume_remnants_of_bad_url(); + break; + case 'name': + $this->consume_name_fragment(); + break; + case 'number': + $this->consume_numeric( true ); + break; + default: + $this->stream['name'] = ''; + if ( '/*' === substr( $this->css, $this->at, 2 ) ) { + $this->consume_comment_fragment( true ); + } else { + $this->scan_next_token(); + } + } + $type = $this->token_type; + $name = ''; + if ( in_array( $type, array( self::TOKEN_IDENT, self::TOKEN_FUNCTION, self::TOKEN_AT_KEYWORD, self::TOKEN_HASH, self::TOKEN_DIMENSION ), true ) ) { + if ( 'name' !== $phase ) { + $this->stream['kind'] = $type; + $value = $this->get_token_value(); + $this->stream['name'] = is_string( $value ) && strlen( $value ) <= 32 ? $value : null; + } + if ( in_array( $type, array( self::TOKEN_FUNCTION, self::TOKEN_AT_KEYWORD ), true ) && 'name' !== $this->stream['phase'] ) { + $name = $this->stream['name'] ?? ''; + } + } + $end = $this->at; + $text = substr( $this->css, $start, $end - $start ); + $value_start = $this->token_value_starts_at; + $value_length = $this->token_value_length; + $has_value = in_array( $type, array( self::TOKEN_STRING, self::TOKEN_URL, self::TOKEN_BAD_STRING, self::TOKEN_BAD_URL ), true ) && null !== $value_start; + return array( + 'text' => $text, + 'type' => $type, + 'name' => $name, + 'value' => $has_value ? $this->decode_range( $value_start, $value_length, ! in_array( $type, array( self::TOKEN_URL, self::TOKEN_BAD_URL ), true ) ) : '', + 'raw_value' => $has_value ? substr( $this->css, $value_start, $value_length ) : '', + 'before' => $has_value ? substr( $this->css, $start, $value_start - $start ) : $text, + 'after' => $has_value ? substr( $this->css, $value_start + $value_length, $end - $value_start - $value_length ) : '', + 'first' => ! in_array( $phase, array( 'string', 'url', 'url-space', 'bad-url' ), true ), + 'closes_url' => $this->url_closed, + 'last' => self::TOKEN_BAD_URL === $type || 'url-space' === $this->stream['phase'] || ! in_array( $this->stream['phase'], array( 'string', 'url', 'url-space', 'bad-url' ), true ), + ); + } + + /** + * Saves only the unfinished lexical state and unread input tail, without handles. + * + * @return array { + * @type array $lexer Lexical continuation fields used by create_for_streaming(). + * @type string $pending_b64 Unconsumed source bytes, base64 encoded. + * } + */ + public function get_reentrancy_cursor(): array { + if ( null === $this->stream ) { + throw new \LogicException( 'CSS cursors require create_for_streaming().' ); + } + return array( + 'lexer' => $this->stream, + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes. + 'pending_b64' => base64_encode( substr( $this->css, $this->at ) ), + ); + } + + /** Continues a comment without retaining the already returned comment bytes. */ + private function consume_comment_fragment( bool $first ): void { + $end = strpos( $this->css, '*/', $this->at + ( $first ? 2 : 0 ) ); + $this->token_type = self::TOKEN_COMMENT; + if ( false !== $end ) { + $this->at = $end + 2; + $this->stream['phase'] = ''; + } else { + $this->at = $this->fragment_limit; + $this->stream['phase'] = $this->stream['finished'] ? '' : 'comment'; + } + } + + /** Continues a name; only its short decoded spelling can identify a URL context. */ + private function consume_name_fragment(): void { + $start = $this->at; + $this->consume_ident_sequence(); + $name = $this->decode_range( $start, $this->at - $start ); + if ( null !== $this->stream['name'] ) { + $this->stream['name'] .= $name; + if ( strlen( $this->stream['name'] ) > 32 ) { + $this->stream['name'] = null; + } + } + $this->token_type = $this->stream['kind']; + $this->token_length = $this->at - $start; + if ( '' === $this->stream['phase'] && self::TOKEN_IDENT === $this->token_type && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { + ++$this->at; + $this->token_type = self::TOKEN_FUNCTION; + if ( 'url' === strtolower( $this->stream['name'] ?? '' ) ) { + $this->stream['phase'] = 'url-start'; + } + } + } + /** * Moves to the next token in the CSS stream. * @@ -351,6 +590,14 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { * @return bool Whether a token was found. */ public function next_token(): bool { + if ( null !== $this->stream ) { + throw new \LogicException( 'Use next_token_fragment() for streamed CSS input.' ); + } + return $this->scan_next_token(); + } + + /** Consumes a token using the same CSS rules for whole strings and input fragments. */ + private function scan_next_token(): bool { $this->after_token(); // Bale out once we reach the end. @@ -885,20 +1132,23 @@ public function get_token_value_length(): ?int { * @return bool Whether the value was successfully updated. */ public function set_token_value( string $new_value ): bool { + if ( null !== $this->stream ) { + throw new \LogicException( 'Streaming callers rewrite returned value fragments instead of queuing whole-token edits.' ); + } // Only URL and string tokens are currently supported. switch ( $this->token_type ) { case self::TOKEN_URL: $this->lexical_updates[] = array( 'start' => $this->token_value_starts_at, 'length' => $this->token_value_length, - 'text' => $this->escape_url_value( $new_value ), + 'text' => self::escape_url_value( $new_value ), ); return true; case self::TOKEN_STRING: $this->lexical_updates[] = array( 'start' => $this->token_starts_at, 'length' => $this->token_length, - 'text' => $this->escape_url_value( $new_value ), + 'text' => self::escape_url_value( $new_value ), ); return true; default: @@ -907,10 +1157,61 @@ public function set_token_value( string $new_value ): bool { } } + /** + * Finds the source byte length of a decoded CSS value prefix. + * + * The caller retains only a candidate URL base, not an entire URL. Decode with + * the same escape and UTF-8 rules used by token values before cutting its source. + * + * @param string $raw_value CSS value bytes without quotes or url(). + * @param int $decoded_bytes Number of decoded UTF-8 bytes to consume. + * @param bool $is_string Whether CSS string line continuations are allowed. + * @return int Source byte length of that prefix. + */ + public static function measure_value_prefix( string $raw_value, int $decoded_bytes, bool $is_string ): int { + $processor = new static( $raw_value ); + $at = 0; + $decoded = 0; + while ( $decoded < $decoded_bytes && $at < $processor->length ) { + $char = $raw_value[ $at ]; + if ( '\\' === $char && $is_string && $at + 1 < $processor->length && false !== strpos( "\r\n\f", $raw_value[ $at + 1 ] ) ) { + $at += "\r" === $raw_value[ $at + 1 ] && "\n" === substr( $raw_value, $at + 2, 1 ) ? 3 : 2; + } elseif ( '\\' === $char && $processor->is_valid_escape( $at ) ) { + ++$at; + $decoded += strlen( $processor->decode_escape_at( $at, $consumed ) ); + $at += $consumed; + } else { + $next = $at; + $invalid = 0; + if ( 1 === _wp_scan_utf8( $raw_value, $next, $invalid, null, 1 ) ) { + $decoded += "\x00" === $char ? 3 : $next - $at; + $at = $next; + } else { + $decoded += 3; + $at += $invalid; + } + } + } + return $at; + } + + /** + * Escapes a replacement prefix for either quoted or unquoted CSS URL syntax. + * + * Keep delimiters outside the replacement. In particular, do not turn an + * unfinished unquoted URL into a quoted string before its remaining bytes arrive. + * + * @param string $value Decoded replacement URL base. + * @return string CSS value bytes without surrounding quotes. + */ + public static function escape_value_prefix( string $value ): string { + return self::escape_url_value( $value, false ); + } + /** * Escapes a URL value for use in quoted url() syntax. * - * Always returns a quoted URL string since they're easier + * Whole-value replacements use quoted URL strings because they are easier * to escape. Quoted URLs are consumed using the string token * rules, and the only values we need to escape in strings, are: * @@ -918,13 +1219,20 @@ public function set_token_value( string $new_value ): bool { * * Newlines. That amounts to \n, \r, \f, \r\n when preprocessing is considered. * * U+005C REVERSE SOLIDUS (\) * + * Prefix replacements keep the surrounding syntax and also escape spaces, + * controls, apostrophes, and parentheses to work in unquoted URLs. + * + * @param string $unescaped Decoded URL value or prefix. + * @param bool $quote Whether to wrap a complete replacement in quotes. + * @return string Escaped CSS value bytes. * @see https://www.w3.org/TR/css-syntax-3/#consume-url-token */ - private function escape_url_value( string $unescaped ): string { + private static function escape_url_value( string $unescaped, bool $quote = true ): string { $escaped = ''; $at = 0; + $unsafe = $quote ? "\n\r\f\\\"" : "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f \x7f\\\"'()"; while ( $at < strlen( $unescaped ) ) { - $safe_len = strcspn( $unescaped, "\n\r\f\\\"", $at ); + $safe_len = strcspn( $unescaped, $unsafe, $at ); if ( $safe_len > 0 ) { $escaped .= substr( $unescaped, $at, $safe_len ); $at += $safe_len; @@ -945,7 +1253,7 @@ private function escape_url_value( string $unescaped ): string { * what the caller intended. */ $escaped .= '\\a '; - if ( strlen( $unescaped ) > $at + 1 && "\n" === $unescaped[ $at + 1 ] ) { + if ( strlen( $unescaped ) > $at && "\n" === $unescaped[ $at ] ) { ++$at; } break; @@ -963,11 +1271,12 @@ private function escape_url_value( string $unescaped ): string { $escaped .= '\\22 '; break; default: - _doing_it_wrong( __METHOD__, 'Unexpected character in URL value: ' . $unsafe_char, '1.0.0' ); + ++$at; + $escaped .= '\\' . dechex( ord( $unsafe_char ) ) . ' '; break; } } - return '"' . $escaped . '"'; + return $quote ? '"' . $escaped . '"' : $escaped; } /** @@ -991,6 +1300,9 @@ private function escape_url_value( string $unescaped ): string { * @return string The modified CSS. */ public function get_updated_css(): string { + if ( null !== $this->stream ) { + throw new \LogicException( 'Streaming callers collect returned token fragments instead of a whole stylesheet.' ); + } if ( empty( $this->lexical_updates ) ) { return $this->css; } @@ -1023,6 +1335,7 @@ function ( $a, $b ) { * Clears token state between tokens. */ private function after_token(): void { + $this->url_closed = false; $this->token_type = null; $this->token_type_flag = null; $this->token_starts_at = null; @@ -1043,27 +1356,33 @@ private function after_token(): void { * * @return bool */ - private function consume_string(): bool { + private function consume_string( ?string $ending_char = null ): bool { // Initially create a with its value set to the empty string. $this->token_starts_at = $this->at; - $ending_char = $this->css[ $this->at ]; - - // Skip past the opening quote. - ++$this->at; + if ( null === $ending_char ) { + $ending_char = $this->css[ $this->at ]; + // Skip past the opening quote, but not when continuing its value. + ++$this->at; + } + $limit = $this->fragment_limit ?? $this->length; + if ( null !== $this->stream ) { + $this->stream['phase'] = ''; + $this->stream['quote'] = $ending_char; + } $value_starts_at = $this->at; // Characters that need special handling: the ending quote, newlines, backslashes. $special_chars = "'" === $ending_char ? "'\n\f\r\\" : "\"\n\f\r\\"; - while ( $this->at < $this->length ) { + while ( $this->at < $limit ) { // Consume normal characters until we hit a special character. - $normal_len = strcspn( $this->css, $special_chars, $this->at ); + $normal_len = strcspn( $this->css, $special_chars, $this->at, $limit - $this->at ); if ( $normal_len > 0 ) { $this->at += $normal_len; } - if ( $this->at >= $this->length ) { - break; // EOF. + if ( $this->at >= $limit ) { + break; // Input boundary. } $char = $this->css[ $this->at ]; @@ -1133,7 +1452,11 @@ private function consume_string(): bool { } } - // EOF + if ( null !== $this->stream && ! $this->stream['finished'] ) { + $this->stream['phase'] = 'string'; + } + + // EOF, or a value fragment awaiting the next input chunk. // This is a parse error. Return the . $this->token_type = self::TOKEN_STRING; $this->token_length = $this->at - $this->token_starts_at; @@ -1153,26 +1476,31 @@ private function consume_string(): bool { * * @return bool */ - private function consume_numeric(): bool { + private function consume_numeric( bool $continuing = false ): bool { // Consume a number and let number be the result. // The type flag defaults to "integer". - $number_type = 'integer'; + $limit = $this->fragment_limit ?? $this->length; + $stage = $continuing ? $this->stream['number'] : 'integer'; + $number_type = 'integer' === $stage ? 'integer' : 'number'; + if ( null !== $this->stream ) { + $this->stream['phase'] = ''; + } // If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), // consume it and append it to repr. - if ( '+' === $this->css[ $this->at ] || '-' === $this->css[ $this->at ] ) { + if ( ! $continuing && ( '+' === $this->css[ $this->at ] || '-' === $this->css[ $this->at ] ) ) { ++$this->at; } // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at ); + $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); if ( $digits > 0 ) { $this->at += $digits; } // If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then. if ( - $this->at + 1 < $this->length && + 'integer' === $stage && $this->at < $limit && $this->at + 1 < $this->length && '.' === $this->css[ $this->at ] && $this->css[ $this->at + 1 ] >= '0' && $this->css[ $this->at + 1 ] <= '9' @@ -1181,8 +1509,9 @@ private function consume_numeric(): bool { ++$this->at; // Set type to "number". $number_type = 'number'; + $stage = 'fraction'; // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at ); + $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); if ( $digits > 0 ) { $this->at += $digits; } @@ -1191,7 +1520,7 @@ private function consume_numeric(): bool { // If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) // or U+0065 LATIN SMALL LETTER E (e), optionally followed by U+002D HYPHEN-MINUS (-) // or U+002B PLUS SIGN (+), followed by a digit, then. - if ( $this->at < $this->length ) { + if ( 'exponent' !== $stage && $this->at < $limit ) { $e = $this->css[ $this->at ]; if ( 'e' === $e || 'E' === $e ) { $save_pos = $this->at; @@ -1213,8 +1542,9 @@ private function consume_numeric(): bool { if ( $has_exp ) { // Set type to "number". $number_type = 'number'; + $stage = 'exponent'; // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at ); + $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); if ( $digits > 0 ) { $this->at += $digits; } @@ -1224,6 +1554,15 @@ private function consume_numeric(): bool { } } + if ( null !== $this->stream && $this->at >= $limit && ! $this->stream['finished'] ) { + $this->stream['phase'] = 'number'; + $this->stream['number'] = $stage; + $this->token_type = self::TOKEN_NUMBER; + $this->token_type_flag = $number_type; + $this->token_length = $this->at - $this->token_starts_at; + return true; + } + /** * This is the end of spec section 4.3.12. Consume a number. * We still have some work to do as specified in section 4.3.3. Consume a numeric token: @@ -1277,6 +1616,20 @@ private function consume_ident_like(): bool { $decoded = $this->consume_ident_sequence(); $string = $decoded ?? $this->decode_range( $ident_start, $this->at - $ident_start ); + if ( null !== $this->stream ) { + $this->token_value = $string; + $this->token_type = self::TOKEN_IDENT; + if ( '' === $this->stream['phase'] && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { + ++$this->at; + $this->token_type = self::TOKEN_FUNCTION; + if ( 0 === strcasecmp( $string, 'url' ) ) { + $this->stream['phase'] = 'url-start'; + } + } + $this->token_length = $this->at - $this->token_starts_at; + return true; + } + // If string's value is an ASCII case-insensitive match for "url", // and the next input code point is U+0028 LEFT PARENTHESIS ((). if ( 0 === strcasecmp( $string, 'url' ) && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { @@ -1340,18 +1693,31 @@ private function consume_ident_like(): bool { * * @return bool */ - private function consume_url(): bool { + private function consume_url( bool $continuing = false, bool $trailing_space = false ): bool { // Initially create a with its value set to the empty string. // Consume as much whitespace as possible. - $this->at += strspn( $this->css, "\t\n\f\r ", $this->at ); + $limit = $this->fragment_limit ?? $this->length; + if ( ! $continuing ) { + $this->at += strspn( $this->css, "\t\n\f\r ", $this->at, max( 0, $limit - $this->at ) ); + } + if ( null !== $this->stream ) { + $this->stream['phase'] = ''; + } - $value_starts_at = $this->at; + $value_starts_at = $this->at; + $this->token_value_starts_at = $value_starts_at; // Repeatedly consume the next input code point from the stream. - while ( $this->at < $this->length ) { + while ( $this->at < $limit ) { + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~:/?#[]@!$&*+,;=%', $this->at, $limit - $this->at ); + if ( $plain > 0 ) { + $this->at += $plain; + continue; + } // U+0029 RIGHT PARENTHESIS ()) // Return the . if ( ')' === $this->css[ $this->at ] ) { + $this->url_closed = true; ++$this->at; $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; @@ -1365,12 +1731,15 @@ private function consume_url(): bool { // U+0029 RIGHT PARENTHESIS ()) or EOF, consume it and return the // (if EOF was encountered, this is a parse error); otherwise, consume the // remnants of a bad url, create a , and return it. - $ws_len = strspn( $this->css, "\t\n\f\r ", $this->at ); - if ( $ws_len > 0 ) { + $ws_len = strspn( $this->css, "\t\n\f\r ", $this->at, $limit - $this->at ); + if ( $ws_len > 0 || $trailing_space ) { $value_ends_at = $this->at; $this->at += $ws_len; // Accept either ) or EOF after whitespace. - if ( $this->at >= $this->length ) { + if ( $this->at >= $limit ) { + if ( null !== $this->stream && ! $this->stream['finished'] ) { + $this->stream['phase'] = 'url-space'; + } // EOF is a parse error, but we return the anyway. $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; @@ -1380,6 +1749,7 @@ private function consume_url(): bool { } if ( ')' === $this->css[ $this->at ] ) { + $this->url_closed = true; // Skip the closing parenthesis and return the . ++$this->at; $this->token_type = self::TOKEN_URL; @@ -1447,8 +1817,12 @@ private function consume_url(): bool { } } - // EOF - // This is a parse error. Return the . + if ( null !== $this->stream && ! $this->stream['finished'] ) { + $this->stream['phase'] = 'url'; + } + + // Return the URL value fragment at an input boundary. Actual EOF before + // the closing parenthesis is a parse error, but still returns a URL token. $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; $this->token_value_starts_at = $value_starts_at; @@ -1467,10 +1841,18 @@ private function consume_url(): bool { * @return bool */ private function consume_remnants_of_bad_url(): bool { - while ( $this->at < $this->length ) { - $this->at += strcspn( $this->css, ')\\', $this->at ); + $limit = $this->fragment_limit ?? $this->length; + if ( null !== $this->stream ) { + $this->token_value_length = null === $this->token_value_starts_at ? null : $this->at - $this->token_value_starts_at; + $this->stream['phase'] = ''; + } + while ( $this->at < $limit ) { + $this->at += strcspn( $this->css, ')\\', $this->at, $limit - $this->at ); - if ( $this->at >= $this->length ) { + if ( $this->at >= $limit ) { + if ( null !== $this->stream && ! $this->stream['finished'] ) { + $this->stream['phase'] = 'bad-url'; + } break; } @@ -1482,6 +1864,7 @@ private function consume_remnants_of_bad_url(): bool { continue; } } elseif ( ')' === $this->css[ $this->at ] ) { + $this->url_closed = true; ++$this->at; break; } @@ -1504,7 +1887,13 @@ private function consume_remnants_of_bad_url(): bool { * @see https://www.w3.org/TR/css-syntax-3/#consume-name */ private function consume_ident_sequence() { - while ( $this->at < $this->length ) { + $limit = $this->fragment_limit ?? $this->length; + while ( $this->at < $limit ) { + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_', $this->at, $limit - $this->at ); + if ( $plain > 0 ) { + $this->at += $plain; + continue; + } $codepoint_bytes = $this->consume_ident_codepoint( $this->at ); if ( $codepoint_bytes > 0 ) { $this->at += $codepoint_bytes; @@ -1521,6 +1910,9 @@ private function consume_ident_sequence() { break; } + if ( null !== $this->stream ) { + $this->stream['phase'] = $this->at >= $limit && ! $this->stream['finished'] ? 'name' : ''; + } } /** @@ -1747,8 +2139,8 @@ private function decode_escape_at( int $offset, &$bytes_consumed ): string { // Hex digits (CSS spec allows at most 6). $hex_len = strspn( $this->css, '0123456789ABCDEFabcdef', $at, 6 ); if ( $hex_len > 0 ) { - $hex = substr( $this->css, $at, $hex_len ); - $at += $hex_len; + $hex = substr( $this->css, $at, $hex_len ); + $at += $hex_len; // If the next input code point is whitespace, consume it as well. if ( $at < $this->length ) { diff --git a/components/DataLiberation/README.md b/components/DataLiberation/README.md index 2c19f1232..b21b7a7e7 100644 --- a/components/DataLiberation/README.md +++ b/components/DataLiberation/README.md @@ -281,3 +281,70 @@ posts: 2 block markup exported frontmatter title exported ``` + +## Rewrite a CSS file in chunks and resume + +A download can stop between `https://old.exa` and `mple/photo.png`. The CSS +processor keeps that unfinished prefix and its CSS context in a cursor. +It uses the same tokenizer as inline CSS rewriting. Comments and displayed +text stay unchanged; `url()`, bare `@import` strings, and `image-set()` URL +strings are recognized. + + +```php + 'https://new.example' ); +$processor = CSSURLProcessor::create_for_streaming( $mapping ); +foreach ( $processor->rewrite_chunk( 'a{src:url(https://old.exa', false ) as $bytes ) { + echo $bytes; +} + +// Save this cursor only after writing every output chunk. A new process can +// restore it without reading the completed source bytes again. +$cursor = json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ); +$processor = CSSURLProcessor::create_for_streaming( $mapping, $cursor ); +foreach ( $processor->rewrite_chunk( 'mple/photo.png)}', true ) as $bytes ) { + echo $bytes; +} +echo "\n"; +``` + + +``` +a{src:url(https://new.example/photo.png)} +``` + +For files, read bounded source chunks and write each yielded output chunk +before saving a checkpoint. Output chunks are at most 64 KiB. A checkpoint +must contain the source byte offset, destination byte offset, and processor +cursor together. Flush destination writes before publishing that checkpoint. +On resume, seek the source to its saved offset and truncate the destination +to its saved offset before appending. Keep the source and URL mapping unchanged. +If a write fails or the output iterator is abandoned, reopen from the last +saved checkpoint rather than continuing the partially consumed iterator. +[The separate-process file test](Tests/fixtures/css-stream/rewrite-file.php) +shows this write, checkpoint, and replay order. + +Matching decodes CSS escapes, compares the HTTP(S) origin without case, and +compares the path with case. The longest matching source base wins, at a +`/`, `?`, `#`, or URL end boundary. Protocol-relative URLs keep their `//` +form. Relative paths, data URLs, and unrelated hosts remain unchanged. The +replacement preserves surrounding CSS syntax and the raw unmatched suffix; +escapes within the replaced prefix can change spelling. This is base-prefix +matching, not full URL canonicalization: dot segments and alternate encoded +host spellings are not resolved. + +The tokenizer retains at most 13 unread source bytes after a drained chunk. +An undecided URL prefix can retain up to 1 MiB of raw bytes between chunks; +source and escaped target bases are also capped at 1 MiB. Nested `image-set` +contexts are capped at 128. Exceeding a cap throws instead of buffering an +unbounded token. Completed comments, strings, and data URLs are not retained. +Call `rewrite_chunk('', true)` if EOF is learned after the final nonempty read. +EOF must mean the actual end of the stylesheet, not an interrupted response. diff --git a/components/DataLiberation/Tests/CSSStreamTest.php b/components/DataLiberation/Tests/CSSStreamTest.php new file mode 100644 index 000000000..8db7bd344 --- /dev/null +++ b/components/DataLiberation/Tests/CSSStreamTest.php @@ -0,0 +1,51 @@ +append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $steps = 0; + while ( null !== ( $fragment = $processor->next_token_fragment() ) ) { + $output .= $fragment['text']; + $this->assertLessThan( 64, ++$steps, 'The lexer must consume input or finish its current token.' ); + } + $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + } + $this->assertSame( $input, $output ); + } + + public static function corpus() { + $cases = json_decode( file_get_contents( __DIR__ . '/css-test-cases.json' ), true ); + foreach ( $cases as $name => $case ) { + yield $name => array( $case['css'] ); + } + yield 'invalid UTF-8' => array( "a{content:\"bad\xff\xc3x\xe2\x82\x80\"}" ); + yield 'UTF-8 at every position' => array( 'a{content:"aé東京😀\1f600 \0000e9 b"}' ); + } + + /** The decoded fragments must use the same UTF-8 and escape rules as a whole value. */ + public function test_decoded_string_fragments_match_the_whole_token() { + foreach ( array( '"é東京😀\1f600 \0000e9 b"', "\"before\\\r\nafter\\\nend\\\ftail\"", "\"\xc3x\xe2\x82\x80\xff\x00\"" ) as $input ) { + $whole = CSSProcessor::create( $input ); + $this->assertTrue( $whole->next_token() ); + $expected = $whole->get_token_value(); + $processor = CSSProcessor::create_for_streaming(); + $decoded = ''; + for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { + $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + while ( null !== ( $fragment = $processor->next_token_fragment() ) ) { + $decoded .= $fragment['value']; + } + $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + } + $this->assertSame( $expected, $decoded ); + } + } +} diff --git a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php new file mode 100644 index 000000000..be0664775 --- /dev/null +++ b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php @@ -0,0 +1,74 @@ +directory = sys_get_temp_dir() . '/css-stream-' . bin2hex( random_bytes( 8 ) ); + mkdir( $this->directory ); + } + + /** @after */ + public function remove_directory() { + foreach ( glob( $this->directory . '/*' ) as $path ) { + unlink( $path ); + } + rmdir( $this->directory ); + } + + /** @dataProvider interruptions */ + public function test_file_rewrite_resumes_after_process_death( $stop ) { + // The first boundary splits a CSS escape; the second remains inside a comment. + $prefix = '/*' . str_repeat( 'a', 32743 ) . '*/a{src:url("https://\\6f ld.example/a.png")}'; + $input = $prefix . '/*' . str_repeat( 'b', 32768 ) . '*/' + . '@import "https://old.example/theme.css";' + . 'a{src:url(https://old.example/' . str_repeat( 'c', 131072 ) . ')}'; + $this->assertSame( '\\6', substr( $input, 32766, 2 ) ); + $expected = strtr( $input, array( 'https://\\6f ld.example/' => 'https://old.example/moved/', 'https://old.example/' => 'https://old.example/moved/' ) ); + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); + if ( 'none' !== $stop ) { + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertGreaterThan( 0, $state['source_bytes'] ); + $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + } + $this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + } + + public function test_file_rewrite_reports_a_prefix_limit_and_keeps_the_last_checkpoint() { + $input = 'a{src:url("h' . str_repeat( "\\\n", 550000 ) . 'ttps://old.example/a")}'; + file_put_contents( $this->directory . '/source.css', $input ); + for ( $attempt = 0; $attempt < 2; ++$attempt ) { + $this->assertNotSame( 0, $this->run_worker( 'none' ) ); + $this->assertStringContainsString( 'exceeding 1048576', file_get_contents( $this->directory . '/worker.log' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); + $this->assertFalse( $state['css']['urls']['finished'] ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + } + } + + public static function interruptions() { + return array( array( 'none' ), array( 'before' ), array( 'after' ) ); + } + + /** Runs the same caller used for uninterrupted and resumed file rewrites. */ + private function run_worker( $stop ) { + $arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-stream/rewrite-file.php', $this->directory . '/source.css', $this->directory . '/target.css', $this->directory . '/state.json', $stop ); + $command = implode( ' ', array_map( 'escapeshellarg', $arguments ) ); + $process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes ); + $this->assertIsResource( $process ); + fclose( $pipes[0] ); + return proc_close( $process ); + } +} diff --git a/components/DataLiberation/Tests/CSSURLStreamTest.php b/components/DataLiberation/Tests/CSSURLStreamTest.php new file mode 100644 index 000000000..89736b2b4 --- /dev/null +++ b/components/DataLiberation/Tests/CSSURLStreamTest.php @@ -0,0 +1,168 @@ + 'http://new.example/local' ); + for ( $split = 0; $split <= strlen( $input ); ++$split ) { + $processor = CSSURLProcessor::create_for_streaming( $mapping ); + $output = $this->rewrite_chunk( $processor, substr( $input, 0, $split ), false ); + $cursor = json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ); + $processor = CSSURLProcessor::create_for_streaming( $mapping, $cursor ); + $output .= $this->rewrite_chunk( $processor, substr( $input, $split ), true ); + $this->assertSame( $expected, $output, 'Split at byte ' . $split ); + } + } + + public static function stylesheets() { + return array( + 'trailing URL spaces' => array( 'a{src:url(https://old.example )}', 'a{src:url(http://new.example/local )}' ), + 'leading URL spaces' => array( 'a{src:url( "https://old.example/a")}', 'a{src:url( "http://new.example/local/a")}' ), + 'bad URL keeps its syntax' => array( 'a{src:url(https://old.example/a(broken)}', 'a{src:url(http://new.example/local/a(broken)}' ), + 'bad string keeps its syntax' => array( "@import \"https://old.example/a\n", "@import \"http://new.example/local/a\n" ), + 'not a url function' => array( 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}', 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}' ), + 'nested image set' => array( 'a{src:image-set(image-set("https://old.example/a" 1x) 1x,"https://old.example/b" type("https://old.example/mime"))}', 'a{src:image-set(image-set("http://new.example/local/a" 1x) 1x,"http://new.example/local/b" type("https://old.example/mime"))}' ), + 'parenthesized resolution' => array( 'a{src:image-set("https://old.example/a" calc(1x * (2 + 3)), "https://old.example/b" 2x)}', 'a{src:image-set("http://new.example/local/a" calc(1x * (2 + 3)), "http://new.example/local/b" 2x)}' ), + 'long non-url names' => array( str_repeat( 'x', 70 ) . 'url("https://old.example/a")', str_repeat( 'x', 70 ) . 'url("https://old.example/a")' ), + 'long numeric dimension' => array( str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")', str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")' ), + 'quoted' => array( 'a{background:url("https://old.example/a.png")}', 'a{background:url("http://new.example/local/a.png")}' ), + 'unquoted' => array( 'a{background:url(https://old.example/a.png)}', 'a{background:url(http://new.example/local/a.png)}' ), + 'hex host and function' => array( 'a{background:\\75rl(https://\\6f ld.example/a.png)}', 'a{background:\\75rl(http://new.example/local/a.png)}' ), + 'slash escapes' => array( 'a{background:url(https:\\/\\/old.example\\/a.png)}', 'a{background:url(http://new.example/local\\/a.png)}' ), + 'protocol relative' => array( 'a{src:url(//old.example/font.woff2)}', 'a{src:url(//new.example/local/font.woff2)}' ), + 'import string' => array( '@import "https://old.example/a.css" screen;', '@import "http://new.example/local/a.css" screen;' ), + 'image set' => array( 'a{background:image-set("https://old.example/a.png" 1x, url(https://old.example/b.png) 2x)}', 'a{background:image-set("http://new.example/local/a.png" 1x, url(http://new.example/local/b.png) 2x)}' ), + 'comment and displayed text' => array( '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}', '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}' ), + 'unrelated and relative' => array( 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}', 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}' ), + 'EOF string' => array( 'a{src:url("https://old.example/a', 'a{src:url("http://new.example/local/a' ), + 'EOF URL' => array( 'a{src:url(https://old.example/a', 'a{src:url(http://new.example/local/a' ), + 'escaped line continuation' => array( "a{src:url(\"https://old.exa\\\r\nmple/a\")}", 'a{src:url("http://new.example/local/a")}' ), + ); + } + + /** Tiny chunks repeatedly cross the same token instead of just splitting it once. */ + public function test_one_byte_chunks_match_whole_chunk_output() { + $mapping = array( 'https://old.example' => 'http://new.example/local' ); + foreach ( self::stylesheets() as $case ) { + $processor = CSSURLProcessor::create_for_streaming( $mapping ); + $output = ''; + for ( $at = 0; $at < strlen( $case[0] ); ++$at ) { + $output .= $this->rewrite_chunk( $processor, $case[0][$at], false ); + $processor = CSSURLProcessor::create_for_streaming( $mapping, $processor->get_reentrancy_cursor() ); + } + $output .= $this->rewrite_chunk( $processor, '', true ); + $this->assertSame( $case[1], $output ); + } + } + + /** A single token, not just a stylesheet, can exceed the input chunk size. */ + public function test_large_tokens_keep_memory_and_cursor_bounded() { + $mapping = array( 'https://old.example' => 'https://old.example/moved' ); + foreach ( array( array( '/*', '*/' ), array( 'a{content:"', '"}' ), array( 'a{src:url(data:image/png;base64,', ')}' ), array( 'a{src:url(https://old.example/', ')}' ), array( '.long', '{}' ) ) as $token ) { + $processor = CSSURLProcessor::create_for_streaming( $mapping ); + $input_hash = hash_init( 'sha256' ); + $output_hash = hash_init( 'sha256' ); + $expected_prefix = str_replace( 'https://old.example/', 'https://old.example/moved/', $token[0] ); + hash_update( $input_hash, $expected_prefix ); + hash_update( $output_hash, $this->rewrite_chunk( $processor, $token[0], false ) ); + $start_memory = memory_get_usage(); + for ( $chunk = 0; $chunk < 128; ++$chunk ) { + $bytes = str_repeat( 'a', 32768 ); + hash_update( $input_hash, $bytes ); + hash_update( $output_hash, $this->rewrite_chunk( $processor, $bytes, false ) ); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertLessThan( 2048, strlen( json_encode( $cursor ) ) ); + $this->assertLessThan( 2 * 1024 * 1024, memory_get_usage() - $start_memory ); + $processor = CSSURLProcessor::create_for_streaming( $mapping, $cursor ); + } + hash_update( $input_hash, $token[1] ); + hash_update( $output_hash, $this->rewrite_chunk( $processor, $token[1], true ) ); + $this->assertSame( hash_final( $input_hash ), hash_final( $output_hash ) ); + } + } + /** Expanding a short source URL many times must not buffer a large output string. */ + public function test_expanded_output_is_yielded_in_bounded_chunks() { + $target = 'https://new.example/' . str_repeat( 'a', 4096 ); + $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => $target ) ); + $input = str_repeat( 'a{src:url(https://old.example/a)}', 1500 ); + $expected = hash_init( 'sha256' ); + for ( $index = 0; $index < 1500; ++$index ) { + hash_update( $expected, 'a{src:url(' . $target . '/a)}' ); + } + $actual = hash_init( 'sha256' ); + $bytes = 0; + $memory = memory_get_usage(); + foreach ( $processor->rewrite_chunk( $input, true ) as $chunk ) { + $this->assertLessThanOrEqual( 65536, strlen( $chunk ) ); + $this->assertLessThan( 2 * 1024 * 1024, memory_get_usage() - $memory ); + $bytes += strlen( $chunk ); + hash_update( $actual, $chunk ); + } + $this->assertGreaterThan( 5 * 1024 * 1024, $bytes ); + $this->assertSame( hash_final( $expected ), hash_final( $actual ) ); + } + + public function test_cannot_checkpoint_unconsumed_output() { + $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); + $output = $processor->rewrite_chunk( 'a{src:url(https://old.example/a)}', true ); + $output->rewind(); + $this->expectException( LogicException::class ); + $this->expectExceptionMessage( 'Consume all CSS output chunks before saving a cursor.' ); + $processor->get_reentrancy_cursor(); + } + + public function test_changed_mapping_cannot_resume_an_open_url() { + $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); + $this->rewrite_chunk( $processor, 'a{src:url("https://old.exa', false ); + $this->expectException( InvalidArgumentException::class ); + $this->expectExceptionMessage( 'different URL mappings' ); + CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://other.example' ), $processor->get_reentrancy_cursor() ); + } + + /** Zero-width line continuations cannot make an undecided prefix consume unlimited memory. */ + public function test_undecided_url_prefix_limit_reports_a_failure() { + $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); + $this->rewrite_chunk( $processor, 'a{src:url("h', false ); + $this->expectException( RuntimeException::class ); + $this->expectExceptionMessage( 'exceeding 1048576' ); + for ( $index = 0; $index < 34; ++$index ) { + $this->rewrite_chunk( $processor, str_repeat( "\\\n", 16384 ), false ); + } + } + + public function test_replacement_prefix_escapes_controls_and_delimiters() { + $input = "https://new.example/" . chr( 1 ) . "\rX\n \"'()"; + $escaped = CSSProcessor::escape_value_prefix( $input ); + foreach ( array( 'url(' . $escaped . ')', 'url("' . $escaped . '")', "url('" . $escaped . "')" ) as $css ) { + $processor = new CSSURLProcessor( $css ); + $this->assertTrue( $processor->next_url() ); + $this->assertSame( str_replace( "\r", "\n", $input ), $processor->get_raw_url() ); + } + } + + public function test_whole_string_finder_uses_the_same_url_contexts() { + $css = '@import "https://old.example/theme.css";a{src:image-set("https://old.example/a" 1x,url(https://old.example/b) 2x,"https://old.example/c" 3x);content:"https://old.example/text"}'; + $processor = new CSSURLProcessor( $css ); + $urls = array(); + while ( $processor->next_url() ) { + $urls[] = $processor->get_raw_url(); + } + $this->assertSame( array( 'https://old.example/theme.css', 'https://old.example/a', 'https://old.example/b', 'https://old.example/c' ), $urls ); + } + + /** Collects only the small input chunks supplied by these assertions. */ + private function rewrite_chunk( CSSURLProcessor $processor, string $input, bool $last ): string { + $output = ''; + foreach ( $processor->rewrite_chunk( $input, $last ) as $chunk ) { + $this->assertLessThanOrEqual( 65536, strlen( $chunk ) ); + $output .= $chunk; + } + return $output; + } + +} diff --git a/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php new file mode 100644 index 000000000..dede4ae72 --- /dev/null +++ b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php @@ -0,0 +1,42 @@ + 0, 'output_bytes' => 0, 'css' => null ); +$input = fopen( $input_path, 'rb' ); +$output = fopen( $output_path, 'c+b' ); +fseek( $input, $state['source_bytes'] ); +// Bytes after the last saved boundary were written by an interrupted process. +// Discard them before replaying the corresponding source chunk. +ftruncate( $output, $state['output_bytes'] ); +fseek( $output, $state['output_bytes'] ); +$processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://old.example/moved' ), $state['css'] ); +$chunks = 0; +while ( ! feof( $input ) ) { + $chunk = fread( $input, 32768 ); + foreach ( $processor->rewrite_chunk( $chunk, feof( $input ) ) as $rewritten ) { + $written = fwrite( $output, $rewritten ); + if ( strlen( $rewritten ) !== $written ) { + throw new RuntimeException( 'CSS output wrote ' . $written . ' of ' . strlen( $rewritten ) . ' bytes.' ); + } + } + fflush( $output ); + ++$chunks; + if ( 'before' === $stop && 2 === $chunks ) { + exit( 99 ); + } + $state = array( 'source_bytes' => ftell( $input ), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + file_put_contents( $state_path . '.tmp', json_encode( $state ) ); + rename( $state_path . '.tmp', $state_path ); + if ( 'after' === $stop && 2 === $chunks ) { + exit( 99 ); + } +} +fclose( $input ); +fclose( $output ); diff --git a/components/DataLiberation/URL/class-cssurlprocessor.php b/components/DataLiberation/URL/class-cssurlprocessor.php index 92f4876d5..a96b6800a 100644 --- a/components/DataLiberation/URL/class-cssurlprocessor.php +++ b/components/DataLiberation/URL/class-cssurlprocessor.php @@ -8,11 +8,21 @@ * Provides URL specific helpers on top of the CSSProcessor tokenizer. */ class CSSURLProcessor { + /** A URL base can contain arbitrarily many zero-width CSS line continuations. */ + private const MAX_BASE_BYTES = 1048576; + /** * @var CSSProcessor */ private $processor; + /** @var array URL syntax context shared by whole-string and streaming callers. */ + private $context = array( + 'depth' => 0, + 'images' => array(), + 'expect' => '', + ); + /** * @param string $css CSS source without wrapping braces. */ @@ -20,6 +30,248 @@ public function __construct( string $css ) { $this->processor = CSSProcessor::create( $css ); } + /** @var array|null State for chunked rewriting; null for the whole-string iterator. */ + private $stream; + + /** @var array Compiled source bases, ordered from longest path to shortest. */ + private $mappings = array(); + + /** @var bool Whether the caller still has output chunks to consume for this input. */ + private $input_open = false; + + /** @var string Binds resumed prefix decisions to the same compiled mapping. */ + private $mapping_hash; + + /** + * Opens URL rewriting for a CSS file without retaining completed input. + * + * @param array $url_mapping Source HTTP(S) bases mapped to target HTTP(S) bases. + * @param array|null $cursor { + * Optional state returned by get_reentrancy_cursor(). + * @type array $css CSS tokenizer cursor. + * @type array $urls Undecided URL prefix and EOF state. + * @type array $context Function depth and expected URL syntax. + * @type string $mapping_hash Hash of the compiled mapping. + * } + * @return static + */ + public static function create_for_streaming( array $url_mapping, ?array $cursor = null ) { + if ( null !== $cursor && ( ! isset( $cursor['css'], $cursor['urls'], $cursor['context'], $cursor['mapping_hash'] ) || ! is_array( $cursor['css'] ) || ! is_array( $cursor['urls'] ) || ! is_array( $cursor['context'] ) ) ) { + throw new \InvalidArgumentException( 'The CSS URL cursor must contain CSS lexer state and URL context. Restart downloads created by another CSS rewriter.' ); + } + $processor = new static( '' ); + $processor->processor = CSSProcessor::create_for_streaming( $cursor['css'] ?? null ); + $processor->stream = array( + 'pending' => null, + 'finished' => false, + ); + if ( null !== $cursor ) { + $processor->stream = $cursor['urls']; + $processor->context = $cursor['context']; + if ( null !== $processor->stream['pending'] ) { + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. + $processor->stream['pending']['raw'] = base64_decode( $processor->stream['pending']['raw'], true ); + if ( false === $processor->stream['pending']['raw'] ) { + throw new \InvalidArgumentException( 'The CSS URL cursor contains an invalid base64 URL prefix.' ); + } + } + } + foreach ( $url_mapping as $source => $target ) { + $source_url = WPURL::parse( $source ); + $target_url = WPURL::parse( $target ); + foreach ( array( $source_url, $target_url ) as $url ) { + if ( false === $url || ! in_array( $url->protocol, array( 'http:', 'https:' ), true ) || '' !== $url->username || '' !== $url->password || '' !== $url->search || '' !== $url->hash ) { + throw new \InvalidArgumentException( 'CSS URL bases must be HTTP(S) addresses without credentials, query, or fragment: ' . $source . ' => ' . $target ); + } + } + $path = rtrim( $source_url->pathname, '/' ); + foreach ( array( $source_url->protocol, '' ) as $scheme ) { + $origin = $scheme . '//' . $source_url->host; + $processor->mappings[] = array( + 'origin' => $origin, + 'prefix' => $origin . $path, + 'target' => CSSProcessor::escape_value_prefix( ( '' === $scheme ? '' : $target_url->protocol ) . '//' . $target_url->host . rtrim( $target_url->pathname, '/' ) ), + 'position' => count( $processor->mappings ), + ); + $entry = end( $processor->mappings ); + if ( strlen( $entry['prefix'] ) > self::MAX_BASE_BYTES || strlen( $entry['target'] ) > self::MAX_BASE_BYTES ) { + throw new \InvalidArgumentException( 'CSS URL bases must not exceed ' . self::MAX_BASE_BYTES . ' bytes; the source has ' . strlen( $entry['prefix'] ) . ' bytes and the escaped target has ' . strlen( $entry['target'] ) . ' bytes.' ); + } + } + } + usort( + $processor->mappings, + static function ( $first, $second ) { + return ( strlen( $second['prefix'] ) <=> strlen( $first['prefix'] ) ) ? ( strlen( $second['prefix'] ) <=> strlen( $first['prefix'] ) ) : ( $first['position'] <=> $second['position'] ); + } + ); + $processor->mapping_hash = hash( 'sha256', json_encode( $processor->mappings ) ); + if ( null !== $cursor && ( $cursor['mapping_hash'] ?? null ) !== $processor->mapping_hash ) { + throw new \InvalidArgumentException( 'Cannot resume CSS rewriting with different URL mappings. Start a new stylesheet rewrite.' ); + } + return $processor; + } + + /** + * Rewrites one supplied chunk and yields bytes ready to write. + * + * Only URL base bytes change. CSS delimiters and the unmatched URL suffix keep + * their source spelling. Comments and displayed strings are never URL contexts. + * The caller writes every yielded chunk before saving its source/output offsets + * and this processor cursor together at the input boundary. If writing fails or + * the generator is abandoned, reopen from the last saved cursor. A cursor covers + * all supplied source bytes, including the small undecided tail kept inside it. + * + * @param string $chunk Next source bytes. + * @param bool $is_last Whether this is the actual end of the stylesheet. + * @return \Generator Output chunks of at most 64 KiB; drain them before saving the cursor. + */ + public function rewrite_chunk( string $chunk, bool $is_last ): \Generator { + if ( null === $this->stream ) { + throw new \LogicException( 'Chunked CSS rewriting requires create_for_streaming().' ); + } + if ( $this->input_open ) { + throw new \LogicException( 'Consume all CSS output chunks before supplying another input chunk.' ); + } + if ( $this->stream['finished'] ) { + if ( '' === $chunk ) { + return; + } + throw new \LogicException( 'Cannot append CSS bytes after the end of the stylesheet.' ); + } + $this->input_open = true; + $output = ''; + $length = strlen( $chunk ); + $offset = 0; + do { + $bytes = substr( $chunk, $offset, 65536 ); + $offset += strlen( $bytes ); + $this->processor->append_bytes( $bytes, $is_last && $offset === $length ); + // phpcs:ignore Generic.CodeAnalysis.AssignmentInCondition.FoundInWhileCondition -- Read each fragment once and stop at the input boundary. + while ( null !== ( $fragment = $this->processor->next_token_fragment() ) ) { + $type = $fragment['type']; + $is_value = in_array( $type, array( CSSProcessor::TOKEN_STRING, CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_BAD_STRING, CSSProcessor::TOKEN_BAD_URL ), true ); + $is_url = false; + if ( ! $is_value || $fragment['first'] ) { + $is_url = $this->inspect_url_context( $type, $fragment['name'] ); + } + $piece = $fragment['text']; + if ( $is_value ) { + if ( $fragment['first'] ) { + $this->stream['pending'] = $is_url ? array( + 'raw' => '', + 'decoded' => '', + 'string' => ! in_array( $type, array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_BAD_URL ), true ), + ) : null; + } + $piece = $fragment['before']; + if ( null !== $this->stream['pending'] ) { + $this->stream['pending']['raw'] .= $fragment['raw_value']; + $this->stream['pending']['decoded'] .= $fragment['value']; + $piece .= $this->rewrite_pending_prefix( $fragment['last'] ); + } else { + $piece .= $fragment['raw_value']; + } + $piece .= $fragment['after']; + if ( $fragment['closes_url'] ) { + $this->inspect_url_context( CSSProcessor::TOKEN_RIGHT_PAREN, '' ); + } + } + $output .= $piece; + if ( strlen( $output ) >= 65536 ) { + // Write expanded replacements before reading more source tokens. + yield from $this->split_output_chunks( $output ); + $output = ''; + } + } + } while ( $offset < $length ); + if ( $is_last ) { + $output .= $this->rewrite_pending_prefix( true ); + $this->stream['finished'] = true; + } + yield from $this->split_output_chunks( $output ); + $this->input_open = false; + } + + /** + * Returns state to save beside the source and destination byte offsets. + * + * @return array { + * @type array $css CSS tokenizer cursor. + * @type array $urls Any undecided base prefix and EOF state; raw bytes use base64. + * @type array $context Function depth and expected URL syntax. + * @type string $mapping_hash Hash of the compiled mapping; resume requires the same mapping. + * } + */ + public function get_reentrancy_cursor(): array { + if ( null === $this->stream ) { + throw new \LogicException( 'CSS URL cursors require create_for_streaming().' ); + } + if ( $this->input_open ) { + throw new \LogicException( 'Consume all CSS output chunks before saving a cursor.' ); + } + $urls = $this->stream; + if ( null !== $urls['pending'] ) { + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes. + $urls['pending']['raw'] = base64_encode( $urls['pending']['raw'] ); + } + return array( + 'css' => $this->processor->get_reentrancy_cursor(), + 'urls' => $urls, + 'context' => $this->context, + 'mapping_hash' => $this->mapping_hash, + ); + } + + /** + * Splits expanded URL replacements without holding the whole rewritten input. + * + * @param string $bytes Current output buffer. + * @return \Generator Bounded output chunks. + */ + private function split_output_chunks( string $bytes ): \Generator { + $length = strlen( $bytes ); + for ( $offset = 0; $offset < $length; $offset += 65536 ) { + yield substr( $bytes, $offset, 65536 ); + } + } + + /** Replaces one decoded source base, after its following byte rules out a host/path prefix collision. */ + private function rewrite_pending_prefix( bool $last ): string { + if ( null === $this->stream['pending'] ) { + return ''; + } + $pending = $this->stream['pending']; + $decoded = $pending['decoded']; + foreach ( $this->mappings as $mapping ) { + $prefix = $mapping['prefix']; + $origin_bytes = strlen( $mapping['origin'] ); + $compare_bytes = min( strlen( $decoded ), strlen( $prefix ) ); + $origin_compare_bytes = min( $compare_bytes, $origin_bytes ); + if ( 0 !== strncasecmp( $decoded, $prefix, $origin_compare_bytes ) || + ( $compare_bytes > $origin_bytes && substr( $decoded, $origin_bytes, $compare_bytes - $origin_bytes ) !== substr( $prefix, $origin_bytes, $compare_bytes - $origin_bytes ) ) ) { + continue; + } + if ( strlen( $decoded ) < strlen( $prefix ) || ( strlen( $decoded ) === strlen( $prefix ) && ! $last ) ) { + if ( ! $last ) { + if ( strlen( $pending['raw'] ) > self::MAX_BASE_BYTES ) { + throw new \RuntimeException( 'An undecided CSS URL base has ' . strlen( $pending['raw'] ) . ' source bytes, exceeding ' . self::MAX_BASE_BYTES . '; shorten the escaped URL prefix before migrating this stylesheet.' ); + } + return ''; + } + continue; + } + if ( strlen( $decoded ) > strlen( $prefix ) && false === strpos( '/?#', $decoded[ strlen( $prefix ) ] ) ) { + continue; + } + $raw_prefix_bytes = CSSProcessor::measure_value_prefix( $pending['raw'], strlen( $prefix ), $pending['string'] ); + $this->stream['pending'] = null; + return $mapping['target'] . substr( $pending['raw'], $raw_prefix_bytes ); + } + $this->stream['pending'] = null; + return $pending['raw']; + } + /** * Moves the cursor to the next URL token, if available. * @@ -27,31 +279,58 @@ public function __construct( string $css ) { */ public function next_url(): bool { while ( $this->processor->next_token() ) { - $type = $this->processor->get_token_type(); - - // Direct URL token. - if ( CSSProcessor::TOKEN_URL === $type ) { + $type = $this->processor->get_token_type(); + $name = in_array( $type, array( CSSProcessor::TOKEN_FUNCTION, CSSProcessor::TOKEN_AT_KEYWORD ), true ) ? $this->processor->get_token_value() : ''; + $is_url = $this->inspect_url_context( $type, $name ); + if ( $is_url && in_array( $type, array( CSSProcessor::TOKEN_STRING, CSSProcessor::TOKEN_URL ), true ) ) { return true; } + } + return false; + } - // url() function with STRING token. - if ( CSSProcessor::TOKEN_FUNCTION === $type && - 0 === strcasecmp( $this->processor->get_token_value(), 'url' ) ) { - // Look ahead for STRING token, skipping whitespace. - while ( $this->processor->next_token() ) { - $inner_type = $this->processor->get_token_type(); - if ( CSSProcessor::TOKEN_WHITESPACE === $inner_type ) { - continue; // Skip whitespace. - } - if ( CSSProcessor::TOKEN_STRING === $inner_type ) { - return true; // Found the URL string. - } - // Hit something else (like RIGHT_PAREN or another token). - break; + /** + * Recognizes URL values without treating comments or displayed text as URLs. + * + * The url() function expects a STRING token after whitespace. Bare @import and + * image-set strings use the same lookahead; another token clears that expectation. + * Direct unquoted URL tokens need no preceding function token in whole-string mode. + * + * @param string $type CSS token type. + * @param string $name Decoded function or at-keyword name, otherwise empty. + * @return bool Whether this begins a URL value. + */ + private function inspect_url_context( string $type, string $name ): bool { + if ( in_array( $type, array( CSSProcessor::TOKEN_WHITESPACE, CSSProcessor::TOKEN_COMMENT ), true ) ) { + return false; + } + $expected = $this->context['expect']; + $this->context['expect'] = ''; + if ( CSSProcessor::TOKEN_FUNCTION === $type ) { + ++$this->context['depth']; + $name = strtolower( $name ); + $this->context['expect'] = 'url' === $name ? 'url' : ''; + if ( in_array( $name, array( 'image-set', '-webkit-image-set' ), true ) ) { + if ( count( $this->context['images'] ) >= 128 ) { + throw new \RuntimeException( 'CSS image-set nesting exceeds 128 open functions.' ); } + $this->context['images'][] = $this->context['depth']; + $this->context['expect'] = 'image'; } + } elseif ( CSSProcessor::TOKEN_AT_KEYWORD === $type ) { + $this->context['expect'] = 'import' === strtolower( $name ) ? 'import' : ''; + } elseif ( CSSProcessor::TOKEN_LEFT_PAREN === $type ) { + ++$this->context['depth']; + } elseif ( CSSProcessor::TOKEN_RIGHT_PAREN === $type ) { + if ( end( $this->context['images'] ) === $this->context['depth'] ) { + array_pop( $this->context['images'] ); + } + $this->context['depth'] = max( 0, $this->context['depth'] - 1 ); + } elseif ( CSSProcessor::TOKEN_COMMA === $type && end( $this->context['images'] ) === $this->context['depth'] ) { + $this->context['expect'] = 'image'; } - return false; + return in_array( $type, array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_BAD_URL ), true ) || + ( '' !== $expected && in_array( $type, array( CSSProcessor::TOKEN_STRING, CSSProcessor::TOKEN_BAD_STRING ), true ) ); } /** From 39c501521944dfbb730a826bb348a6d8d0875781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Mon, 7 Sep 2026 23:35:32 +0200 Subject: [PATCH 02/20] Launch CSS file test workers directly on Windows --- components/DataLiberation/Tests/CSSURLStreamProcessTest.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php index be0664775..483cf1aac 100644 --- a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php +++ b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php @@ -66,7 +66,9 @@ public static function interruptions() { private function run_worker( $stop ) { $arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-stream/rewrite-file.php', $this->directory . '/source.css', $this->directory . '/target.css', $this->directory . '/state.json', $stop ); $command = implode( ' ', array_map( 'escapeshellarg', $arguments ) ); - $process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes ); + // Windows cmd.exe strips quotes from this command. Launch PHP directly; + // the command stays a string for PHP 7.2, which cannot accept an argument array. + $process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes, null, null, array( 'bypass_shell' => true ) ); $this->assertIsResource( $process ); fclose( $pipes[0] ); return proc_close( $process ); From 2450d71041220985a1b158bc6501f195f35f00bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Tue, 8 Sep 2026 00:48:32 +0200 Subject: [PATCH 03/20] Describe CSS streaming checkpoint tests and input boundaries --- components/DataLiberation/CSS/class-cssprocessor.php | 4 ++-- components/DataLiberation/Tests/CSSStreamTest.php | 1 + components/DataLiberation/Tests/CSSURLStreamProcessTest.php | 2 ++ components/DataLiberation/Tests/CSSURLStreamTest.php | 5 +++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/components/DataLiberation/CSS/class-cssprocessor.php b/components/DataLiberation/CSS/class-cssprocessor.php index c8b7afd08..2b429f712 100644 --- a/components/DataLiberation/CSS/class-cssprocessor.php +++ b/components/DataLiberation/CSS/class-cssprocessor.php @@ -1456,8 +1456,8 @@ private function consume_string( ?string $ending_char = null ): bool { $this->stream['phase'] = 'string'; } - // EOF, or a value fragment awaiting the next input chunk. - // This is a parse error. Return the . + // EOF without a closing quote is a parse error; an input boundary is not. + // Return the in either case, retaining the phase when more bytes may arrive. $this->token_type = self::TOKEN_STRING; $this->token_length = $this->at - $this->token_starts_at; $this->token_value_starts_at = $value_starts_at; diff --git a/components/DataLiberation/Tests/CSSStreamTest.php b/components/DataLiberation/Tests/CSSStreamTest.php index 8db7bd344..d604ee717 100644 --- a/components/DataLiberation/Tests/CSSStreamTest.php +++ b/components/DataLiberation/Tests/CSSStreamTest.php @@ -21,6 +21,7 @@ public function test_source_bytes_survive_one_byte_input_and_resume( $input ) { $this->assertSame( $input, $output ); } + /** Supplies the CSS corpus plus byte sequences that cross UTF-8 and escape boundaries. */ public static function corpus() { $cases = json_decode( file_get_contents( __DIR__ . '/css-test-cases.json' ), true ); foreach ( $cases as $name => $case ) { diff --git a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php index 483cf1aac..16eab67df 100644 --- a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php +++ b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php @@ -45,6 +45,7 @@ public function test_file_rewrite_resumes_after_process_death( $stop ) { $this->assertSame( strlen( $expected ), $state['output_bytes'] ); } + /** Prefix-limit failure must preserve a resumable boundary on both the first run and resume. */ public function test_file_rewrite_reports_a_prefix_limit_and_keeps_the_last_checkpoint() { $input = 'a{src:url("h' . str_repeat( "\\\n", 550000 ) . 'ttps://old.example/a")}'; file_put_contents( $this->directory . '/source.css', $input ); @@ -58,6 +59,7 @@ public function test_file_rewrite_reports_a_prefix_limit_and_keeps_the_last_chec } } + /** Runs through completion or exits on either side of the second file checkpoint. */ public static function interruptions() { return array( array( 'none' ), array( 'before' ), array( 'after' ) ); } diff --git a/components/DataLiberation/Tests/CSSURLStreamTest.php b/components/DataLiberation/Tests/CSSURLStreamTest.php index 89736b2b4..ab1f55ac0 100644 --- a/components/DataLiberation/Tests/CSSURLStreamTest.php +++ b/components/DataLiberation/Tests/CSSURLStreamTest.php @@ -19,6 +19,7 @@ public function test_rewrites_and_resumes_at_every_byte( $input, $expected ) { } } + /** Pairs URL syntax and non-URL text with the exact bytes expected after rewriting. */ public static function stylesheets() { return array( 'trailing URL spaces' => array( 'a{src:url(https://old.example )}', 'a{src:url(http://new.example/local )}' ), @@ -107,6 +108,7 @@ public function test_expanded_output_is_yielded_in_bounded_chunks() { $this->assertSame( hash_final( $expected ), hash_final( $actual ) ); } + /** A saved cursor must not skip output still held by the generator. */ public function test_cannot_checkpoint_unconsumed_output() { $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); $output = $processor->rewrite_chunk( 'a{src:url(https://old.example/a)}', true ); @@ -116,6 +118,7 @@ public function test_cannot_checkpoint_unconsumed_output() { $processor->get_reentrancy_cursor(); } + /** An unfinished source base cannot be resumed with a different replacement. */ public function test_changed_mapping_cannot_resume_an_open_url() { $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); $this->rewrite_chunk( $processor, 'a{src:url("https://old.exa', false ); @@ -135,6 +138,7 @@ public function test_undecided_url_prefix_limit_reports_a_failure() { } } + /** Replacement bytes must stay inside the value in all three URL quoting forms. */ public function test_replacement_prefix_escapes_controls_and_delimiters() { $input = "https://new.example/" . chr( 1 ) . "\rX\n \"'()"; $escaped = CSSProcessor::escape_value_prefix( $input ); @@ -145,6 +149,7 @@ public function test_replacement_prefix_escapes_controls_and_delimiters() { } } + /** The existing iterator recognizes import and image-set URLs without matching displayed text. */ public function test_whole_string_finder_uses_the_same_url_contexts() { $css = '@import "https://old.example/theme.css";a{src:image-set("https://old.example/a" 1x,url(https://old.example/b) 2x,"https://old.example/c" 3x);content:"https://old.example/text"}'; $processor = new CSSURLProcessor( $css ); From ca22e2b4ec6514b5d8251118fd0591cf11efc8d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Zieli=C5=84ski?= Date: Tue, 8 Sep 2026 10:10:17 +0200 Subject: [PATCH 04/20] Buffer unfinished CSS tokens and retry with more input --- .../DataLiberation/CSS/class-cssprocessor.php | 432 +++++------------- components/DataLiberation/README.md | 24 +- .../DataLiberation/Tests/CSSStreamTest.php | 44 +- .../Tests/CSSURLStreamProcessTest.php | 10 +- .../DataLiberation/Tests/CSSURLStreamTest.php | 35 +- .../URL/class-cssurlprocessor.php | 159 ++----- 6 files changed, 233 insertions(+), 471 deletions(-) diff --git a/components/DataLiberation/CSS/class-cssprocessor.php b/components/DataLiberation/CSS/class-cssprocessor.php index 2b429f712..5fcb63cd7 100644 --- a/components/DataLiberation/CSS/class-cssprocessor.php +++ b/components/DataLiberation/CSS/class-cssprocessor.php @@ -308,14 +308,8 @@ class CSSProcessor { */ private $lexical_updates = array(); - /** @var array|null Durable lexical state for chunked input; null for whole strings. */ - private $stream; - - /** @var int|null Last byte at which a new CSS code point may begin in this chunk. */ - private $fragment_limit; - - /** @var bool Whether this fragment consumed the unquoted URL closing parenthesis. */ - private $url_closed = false; + /** @var bool Whether an unfinished token may receive more source bytes. */ + private $expecting_more_input = false; /** * Constructor for the CSS processor. @@ -351,235 +345,97 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { } /** - * Opens a bounded-input tokenizer. Drain next_token_fragment() before appending. - * - * Fragments preserve source bytes. Strings, comments, names, and URLs may span - * fragments. url( is exposed as a function before its quoted or unquoted value; - * the whole-string next_token() API keeps its existing CSS Syntax token types. + * Opens a processor that keeps unfinished tokens until more input arrives. * * @param array|null $cursor { - * Optional state returned by get_reentrancy_cursor(). - * @type array $lexer Lexical continuation state. - * @type string $pending_b64 Unconsumed source bytes, base64 encoded. + * Optional state from get_reentrancy_cursor(), after flushing processed CSS. + * @type string $pending_b64 Unprocessed source bytes, base64 encoded. + * @type bool $expecting_more_input Whether the source has more bytes. * } * @return static */ public static function create_for_streaming( ?array $cursor = null ) { - $processor = new static( '' ); - $processor->stream = array( - 'phase' => '', - 'quote' => '', - 'name' => '', - 'kind' => '', - 'number' => 'integer', - 'finished' => false, - ); - if ( null !== $cursor ) { - if ( ! isset( $cursor['lexer'], $cursor['pending_b64'] ) || ! is_array( $cursor['lexer'] ) || ! is_string( $cursor['pending_b64'] ) ) { - throw new \InvalidArgumentException( 'The CSS cursor must contain lexer state and a base64 input tail.' ); - } - $processor->stream = $cursor['lexer']; - // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. - $processor->css = base64_decode( $cursor['pending_b64'], true ); - if ( false === $processor->css ) { - throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' ); - } - $processor->length = strlen( $processor->css ); + if ( null !== $cursor && ( ! isset( $cursor['pending_b64'], $cursor['expecting_more_input'] ) || ! is_string( $cursor['pending_b64'] ) || ! is_bool( $cursor['expecting_more_input'] ) ) ) { + throw new \InvalidArgumentException( 'The CSS cursor must contain a base64 input tail and whether more input is expected.' ); } + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. + $pending = null === $cursor ? '' : base64_decode( $cursor['pending_b64'], true ); + if ( false === $pending ) { + throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' ); + } + $processor = new static( $pending ); + $processor->expecting_more_input = $cursor['expecting_more_input'] ?? true; return $processor; } /** - * Supplies at most 64 KiB of source bytes. The caller marks actual EOF explicitly. + * Appends source bytes after the caller has flushed completed tokens. + * + * The unfinished token stays in memory and is parsed again. Neither its size + * nor the saved cursor size is capped. Use small reads to limit other input. * - * @param string $bytes Next source chunk. - * @param bool $is_last Whether this is the end of the stylesheet. + * @param string $bytes Next source bytes. + * @param bool $is_last Whether this is the actual end of the stylesheet. */ public function append_bytes( string $bytes, bool $is_last = false ): void { - if ( null === $this->stream || $this->stream['finished'] ) { - throw new \LogicException( 'CSS input can only be appended to an unfinished streaming processor.' ); + if ( ! $this->expecting_more_input ) { + throw new \LogicException( 'CSS input cannot be appended after the end of the stylesheet.' ); } - if ( strlen( $bytes ) > 65536 || $this->length - $this->at > 13 ) { - throw new \InvalidArgumentException( 'CSS input must be drained before appending a chunk of at most 65536 bytes; received ' . strlen( $bytes ) . ' bytes with ' . ( $this->length - $this->at ) . ' unread bytes.' ); + if ( 0 !== $this->at ) { + throw new \LogicException( 'Flush processed CSS before appending more input.' ); } - $this->css = substr( $this->css, $this->at ) . $bytes; - $this->at = 0; - $this->length = strlen( $this->css ); - $this->stream['finished'] = $is_last; + $this->css .= $bytes; + $this->length = strlen( $this->css ); + $this->expecting_more_input = ! $is_last; + } + + /** Returns whether more source bytes may be appended. */ + public function is_expecting_more_input(): bool { + return $this->expecting_more_input; } /** - * Returns one token fragment, or null when more input is needed or EOF is reached. - * - * Names are retained only up to 32 decoded bytes. Longer names cannot be url, - * import, image-set, or -webkit-image-set; their source bytes still pass through. - * - * @return array|null { - * @type string $text Original source bytes for this fragment. - * @type string $type CSS token type; a later fragment may finish a name as a function. - * @type string $name Complete short function or at-keyword name, otherwise empty. - * @type string $value Decoded string or URL value fragment, excluding its syntax. - * @type string $raw_value Original bytes of that value fragment. - * @type string $before Syntax preceding the value, such as an opening quote. - * @type string $after Syntax following the value, such as a closing parenthesis. - * @type bool $first For strings and URLs, whether this begins their value. - * @type bool $last For strings and URLs, whether this finishes their value. - * @type bool $closes_url Whether an unquoted URL consumed its closing parenthesis. - * } + * Returns edited, completed input and keeps the unfinished token for another read. + * + * Flush before appending input or saving a cursor. Existing whole-token setters + * work in streaming mode; their edits are applied only to the returned prefix. + * + * @return string Processed CSS, including edits made with set_token_value(). */ - public function next_token_fragment(): ?array { - if ( null === $this->stream ) { - throw new \LogicException( 'Token fragments require create_for_streaming().' ); - } - // A CSS escape needs at most six hex digits and CRLF after its backslash. - // Keeping ten bytes also covers UTF-8, number lookahead, and comment delimiters. - $this->fragment_limit = $this->stream['finished'] ? $this->length : max( 0, $this->length - 10 ); - if ( ! $this->stream['finished'] && $this->fragment_limit > 0 ) { - $start = $this->fragment_limit; - for ( $back = 0; $back < 3 && $start > 0; ++$back ) { - if ( 0x80 !== ( ord( $this->css[ $start ] ) & 0xc0 ) ) { - break; - } - --$start; - } - $end = $start; - $invalid = 0; - if ( $start < $this->fragment_limit && 1 === _wp_scan_utf8( $this->css, $end, $invalid, null, 1 ) && $end > $this->fragment_limit ) { - $this->fragment_limit = $start; - } - } - if ( $this->at >= $this->fragment_limit ) { - return null; - } - $start = $this->at; - $phase = $this->stream['phase']; + public function flush_processed_css(): string { + if ( 0 === $this->at ) { + return ''; + } + $pending = substr( $this->css, $this->at ); + $updated = $this->get_updated_css(); + $output = substr( $updated, 0, strlen( $updated ) - strlen( $pending ) ); + $this->css = $pending; + $this->length = strlen( $pending ); + $this->at = 0; + $this->lexical_updates = array(); $this->after_token(); - $this->token_starts_at = $start; - switch ( $phase ) { - case 'comment': - $this->consume_comment_fragment( false ); - break; - case 'string': - $this->consume_string( $this->stream['quote'] ); - break; - case 'url-start': - $whitespace = strspn( $this->css, "\t\n\f\r ", $this->at, $this->fragment_limit - $this->at ); - if ( $whitespace ) { - $this->at += $whitespace; - $this->token_type = self::TOKEN_WHITESPACE; - } elseif ( '"' === $this->css[ $this->at ] || "'" === $this->css[ $this->at ] ) { - $this->stream['phase'] = ''; - $this->consume_string(); - } else { - $this->consume_url( true ); - } - break; - case 'url': - case 'url-space': - $this->consume_url( true, 'url-space' === $phase ); - break; - case 'bad-url': - $this->consume_remnants_of_bad_url(); - break; - case 'name': - $this->consume_name_fragment(); - break; - case 'number': - $this->consume_numeric( true ); - break; - default: - $this->stream['name'] = ''; - if ( '/*' === substr( $this->css, $this->at, 2 ) ) { - $this->consume_comment_fragment( true ); - } else { - $this->scan_next_token(); - } - } - $type = $this->token_type; - $name = ''; - if ( in_array( $type, array( self::TOKEN_IDENT, self::TOKEN_FUNCTION, self::TOKEN_AT_KEYWORD, self::TOKEN_HASH, self::TOKEN_DIMENSION ), true ) ) { - if ( 'name' !== $phase ) { - $this->stream['kind'] = $type; - $value = $this->get_token_value(); - $this->stream['name'] = is_string( $value ) && strlen( $value ) <= 32 ? $value : null; - } - if ( in_array( $type, array( self::TOKEN_FUNCTION, self::TOKEN_AT_KEYWORD ), true ) && 'name' !== $this->stream['phase'] ) { - $name = $this->stream['name'] ?? ''; - } - } - $end = $this->at; - $text = substr( $this->css, $start, $end - $start ); - $value_start = $this->token_value_starts_at; - $value_length = $this->token_value_length; - $has_value = in_array( $type, array( self::TOKEN_STRING, self::TOKEN_URL, self::TOKEN_BAD_STRING, self::TOKEN_BAD_URL ), true ) && null !== $value_start; - return array( - 'text' => $text, - 'type' => $type, - 'name' => $name, - 'value' => $has_value ? $this->decode_range( $value_start, $value_length, ! in_array( $type, array( self::TOKEN_URL, self::TOKEN_BAD_URL ), true ) ) : '', - 'raw_value' => $has_value ? substr( $this->css, $value_start, $value_length ) : '', - 'before' => $has_value ? substr( $this->css, $start, $value_start - $start ) : $text, - 'after' => $has_value ? substr( $this->css, $value_start + $value_length, $end - $value_start - $value_length ) : '', - 'first' => ! in_array( $phase, array( 'string', 'url', 'url-space', 'bad-url' ), true ), - 'closes_url' => $this->url_closed, - 'last' => self::TOKEN_BAD_URL === $type || 'url-space' === $this->stream['phase'] || ! in_array( $this->stream['phase'], array( 'string', 'url', 'url-space', 'bad-url' ), true ), - ); + return $output; } /** - * Saves only the unfinished lexical state and unread input tail, without handles. + * Returns unfinished input to save beside the caller's source and output offsets. * * @return array { - * @type array $lexer Lexical continuation fields used by create_for_streaming(). - * @type string $pending_b64 Unconsumed source bytes, base64 encoded. + * @type string $pending_b64 Unprocessed source bytes, base64 encoded. + * @type bool $expecting_more_input Whether the source has more bytes. * } */ public function get_reentrancy_cursor(): array { - if ( null === $this->stream ) { - throw new \LogicException( 'CSS cursors require create_for_streaming().' ); + if ( 0 !== $this->at ) { + throw new \LogicException( 'Flush processed CSS before saving a cursor.' ); } return array( - 'lexer' => $this->stream, // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes. - 'pending_b64' => base64_encode( substr( $this->css, $this->at ) ), + 'pending_b64' => base64_encode( $this->css ), + 'expecting_more_input' => $this->expecting_more_input, ); } - /** Continues a comment without retaining the already returned comment bytes. */ - private function consume_comment_fragment( bool $first ): void { - $end = strpos( $this->css, '*/', $this->at + ( $first ? 2 : 0 ) ); - $this->token_type = self::TOKEN_COMMENT; - if ( false !== $end ) { - $this->at = $end + 2; - $this->stream['phase'] = ''; - } else { - $this->at = $this->fragment_limit; - $this->stream['phase'] = $this->stream['finished'] ? '' : 'comment'; - } - } - - /** Continues a name; only its short decoded spelling can identify a URL context. */ - private function consume_name_fragment(): void { - $start = $this->at; - $this->consume_ident_sequence(); - $name = $this->decode_range( $start, $this->at - $start ); - if ( null !== $this->stream['name'] ) { - $this->stream['name'] .= $name; - if ( strlen( $this->stream['name'] ) > 32 ) { - $this->stream['name'] = null; - } - } - $this->token_type = $this->stream['kind']; - $this->token_length = $this->at - $start; - if ( '' === $this->stream['phase'] && self::TOKEN_IDENT === $this->token_type && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { - ++$this->at; - $this->token_type = self::TOKEN_FUNCTION; - if ( 'url' === strtolower( $this->stream['name'] ?? '' ) ) { - $this->stream['phase'] = 'url-start'; - } - } - } - /** * Moves to the next token in the CSS stream. * @@ -587,16 +443,26 @@ private function consume_name_fragment(): void { * * @see https://www.w3.org/TR/css-syntax-3/#consume-token * - * @return bool Whether a token was found. + * @return bool Whether a complete token was found; false also means more input is needed. */ public function next_token(): bool { - if ( null !== $this->stream ) { - throw new \LogicException( 'Use next_token_fragment() for streamed CSS input.' ); + $start = $this->at; + if ( ! $this->scan_next_token() ) { + return false; + } + // A CSS escape needs at most six hex digits and CRLF after its backslash. + // Ten bytes also cover UTF-8 and the lookahead used to classify a token. + // A name may become a function, and a number may acquire a unit. Retry the + // whole token when more bytes arrive instead of saving its internal state. + if ( $this->expecting_more_input && $this->at > $this->length - 10 ) { + $this->at = $start; + $this->after_token(); + return false; } - return $this->scan_next_token(); + return true; } - /** Consumes a token using the same CSS rules for whole strings and input fragments. */ + /** Reads a whole token with the same CSS rules for complete and growing input. */ private function scan_next_token(): bool { $this->after_token(); @@ -1132,9 +998,6 @@ public function get_token_value_length(): ?int { * @return bool Whether the value was successfully updated. */ public function set_token_value( string $new_value ): bool { - if ( null !== $this->stream ) { - throw new \LogicException( 'Streaming callers rewrite returned value fragments instead of queuing whole-token edits.' ); - } // Only URL and string tokens are currently supported. switch ( $this->token_type ) { case self::TOKEN_URL: @@ -1160,8 +1023,8 @@ public function set_token_value( string $new_value ): bool { /** * Finds the source byte length of a decoded CSS value prefix. * - * The caller retains only a candidate URL base, not an entire URL. Decode with - * the same escape and UTF-8 rules used by token values before cutting its source. + * Decode with the same escape and UTF-8 rules used by token values before + * cutting the source bytes of the matched URL base. * * @param string $raw_value CSS value bytes without quotes or url(). * @param int $decoded_bytes Number of decoded UTF-8 bytes to consume. @@ -1198,8 +1061,8 @@ public static function measure_value_prefix( string $raw_value, int $decoded_byt /** * Escapes a replacement prefix for either quoted or unquoted CSS URL syntax. * - * Keep delimiters outside the replacement. In particular, do not turn an - * unfinished unquoted URL into a quoted string before its remaining bytes arrive. + * Keep the existing quotes or url() delimiters outside the replacement so + * the unmatched suffix can keep its original source spelling. * * @param string $value Decoded replacement URL base. * @return string CSS value bytes without surrounding quotes. @@ -1300,9 +1163,6 @@ private static function escape_url_value( string $unescaped, bool $quote = true * @return string The modified CSS. */ public function get_updated_css(): string { - if ( null !== $this->stream ) { - throw new \LogicException( 'Streaming callers collect returned token fragments instead of a whole stylesheet.' ); - } if ( empty( $this->lexical_updates ) ) { return $this->css; } @@ -1335,7 +1195,6 @@ function ( $a, $b ) { * Clears token state between tokens. */ private function after_token(): void { - $this->url_closed = false; $this->token_type = null; $this->token_type_flag = null; $this->token_starts_at = null; @@ -1356,33 +1215,27 @@ private function after_token(): void { * * @return bool */ - private function consume_string( ?string $ending_char = null ): bool { + private function consume_string(): bool { // Initially create a with its value set to the empty string. $this->token_starts_at = $this->at; - if ( null === $ending_char ) { - $ending_char = $this->css[ $this->at ]; - // Skip past the opening quote, but not when continuing its value. - ++$this->at; - } - $limit = $this->fragment_limit ?? $this->length; - if ( null !== $this->stream ) { - $this->stream['phase'] = ''; - $this->stream['quote'] = $ending_char; - } + $ending_char = $this->css[ $this->at ]; + + // Skip past the opening quote. + ++$this->at; $value_starts_at = $this->at; // Characters that need special handling: the ending quote, newlines, backslashes. $special_chars = "'" === $ending_char ? "'\n\f\r\\" : "\"\n\f\r\\"; - while ( $this->at < $limit ) { + while ( $this->at < $this->length ) { // Consume normal characters until we hit a special character. - $normal_len = strcspn( $this->css, $special_chars, $this->at, $limit - $this->at ); + $normal_len = strcspn( $this->css, $special_chars, $this->at ); if ( $normal_len > 0 ) { $this->at += $normal_len; } - if ( $this->at >= $limit ) { - break; // Input boundary. + if ( $this->at >= $this->length ) { + break; // EOF. } $char = $this->css[ $this->at ]; @@ -1452,12 +1305,8 @@ private function consume_string( ?string $ending_char = null ): bool { } } - if ( null !== $this->stream && ! $this->stream['finished'] ) { - $this->stream['phase'] = 'string'; - } - - // EOF without a closing quote is a parse error; an input boundary is not. - // Return the in either case, retaining the phase when more bytes may arrive. + // EOF + // This is a parse error. Return the . $this->token_type = self::TOKEN_STRING; $this->token_length = $this->at - $this->token_starts_at; $this->token_value_starts_at = $value_starts_at; @@ -1476,31 +1325,26 @@ private function consume_string( ?string $ending_char = null ): bool { * * @return bool */ - private function consume_numeric( bool $continuing = false ): bool { + private function consume_numeric(): bool { // Consume a number and let number be the result. // The type flag defaults to "integer". - $limit = $this->fragment_limit ?? $this->length; - $stage = $continuing ? $this->stream['number'] : 'integer'; - $number_type = 'integer' === $stage ? 'integer' : 'number'; - if ( null !== $this->stream ) { - $this->stream['phase'] = ''; - } + $number_type = 'integer'; // If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), // consume it and append it to repr. - if ( ! $continuing && ( '+' === $this->css[ $this->at ] || '-' === $this->css[ $this->at ] ) ) { + if ( '+' === $this->css[ $this->at ] || '-' === $this->css[ $this->at ] ) { ++$this->at; } // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); + $digits = strspn( $this->css, '0123456789', $this->at ); if ( $digits > 0 ) { $this->at += $digits; } // If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then. if ( - 'integer' === $stage && $this->at < $limit && $this->at + 1 < $this->length && + $this->at + 1 < $this->length && '.' === $this->css[ $this->at ] && $this->css[ $this->at + 1 ] >= '0' && $this->css[ $this->at + 1 ] <= '9' @@ -1509,9 +1353,8 @@ private function consume_numeric( bool $continuing = false ): bool { ++$this->at; // Set type to "number". $number_type = 'number'; - $stage = 'fraction'; // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); + $digits = strspn( $this->css, '0123456789', $this->at ); if ( $digits > 0 ) { $this->at += $digits; } @@ -1520,7 +1363,7 @@ private function consume_numeric( bool $continuing = false ): bool { // If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) // or U+0065 LATIN SMALL LETTER E (e), optionally followed by U+002D HYPHEN-MINUS (-) // or U+002B PLUS SIGN (+), followed by a digit, then. - if ( 'exponent' !== $stage && $this->at < $limit ) { + if ( $this->at < $this->length ) { $e = $this->css[ $this->at ]; if ( 'e' === $e || 'E' === $e ) { $save_pos = $this->at; @@ -1542,9 +1385,8 @@ private function consume_numeric( bool $continuing = false ): bool { if ( $has_exp ) { // Set type to "number". $number_type = 'number'; - $stage = 'exponent'; // While the next input code point is a digit, consume it and append it to repr. - $digits = strspn( $this->css, '0123456789', $this->at, max( 0, $limit - $this->at ) ); + $digits = strspn( $this->css, '0123456789', $this->at ); if ( $digits > 0 ) { $this->at += $digits; } @@ -1554,15 +1396,6 @@ private function consume_numeric( bool $continuing = false ): bool { } } - if ( null !== $this->stream && $this->at >= $limit && ! $this->stream['finished'] ) { - $this->stream['phase'] = 'number'; - $this->stream['number'] = $stage; - $this->token_type = self::TOKEN_NUMBER; - $this->token_type_flag = $number_type; - $this->token_length = $this->at - $this->token_starts_at; - return true; - } - /** * This is the end of spec section 4.3.12. Consume a number. * We still have some work to do as specified in section 4.3.3. Consume a numeric token: @@ -1616,20 +1449,6 @@ private function consume_ident_like(): bool { $decoded = $this->consume_ident_sequence(); $string = $decoded ?? $this->decode_range( $ident_start, $this->at - $ident_start ); - if ( null !== $this->stream ) { - $this->token_value = $string; - $this->token_type = self::TOKEN_IDENT; - if ( '' === $this->stream['phase'] && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { - ++$this->at; - $this->token_type = self::TOKEN_FUNCTION; - if ( 0 === strcasecmp( $string, 'url' ) ) { - $this->stream['phase'] = 'url-start'; - } - } - $this->token_length = $this->at - $this->token_starts_at; - return true; - } - // If string's value is an ASCII case-insensitive match for "url", // and the next input code point is U+0028 LEFT PARENTHESIS ((). if ( 0 === strcasecmp( $string, 'url' ) && $this->at < $this->length && '(' === $this->css[ $this->at ] ) { @@ -1693,23 +1512,16 @@ private function consume_ident_like(): bool { * * @return bool */ - private function consume_url( bool $continuing = false, bool $trailing_space = false ): bool { + private function consume_url(): bool { // Initially create a with its value set to the empty string. // Consume as much whitespace as possible. - $limit = $this->fragment_limit ?? $this->length; - if ( ! $continuing ) { - $this->at += strspn( $this->css, "\t\n\f\r ", $this->at, max( 0, $limit - $this->at ) ); - } - if ( null !== $this->stream ) { - $this->stream['phase'] = ''; - } + $this->at += strspn( $this->css, "\t\n\f\r ", $this->at ); - $value_starts_at = $this->at; - $this->token_value_starts_at = $value_starts_at; + $value_starts_at = $this->at; // Repeatedly consume the next input code point from the stream. - while ( $this->at < $limit ) { - $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~:/?#[]@!$&*+,;=%', $this->at, $limit - $this->at ); + while ( $this->at < $this->length ) { + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~:/?#[]@!$&*+,;=%', $this->at ); if ( $plain > 0 ) { $this->at += $plain; continue; @@ -1717,7 +1529,6 @@ private function consume_url( bool $continuing = false, bool $trailing_space = f // U+0029 RIGHT PARENTHESIS ()) // Return the . if ( ')' === $this->css[ $this->at ] ) { - $this->url_closed = true; ++$this->at; $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; @@ -1731,15 +1542,12 @@ private function consume_url( bool $continuing = false, bool $trailing_space = f // U+0029 RIGHT PARENTHESIS ()) or EOF, consume it and return the // (if EOF was encountered, this is a parse error); otherwise, consume the // remnants of a bad url, create a , and return it. - $ws_len = strspn( $this->css, "\t\n\f\r ", $this->at, $limit - $this->at ); - if ( $ws_len > 0 || $trailing_space ) { + $ws_len = strspn( $this->css, "\t\n\f\r ", $this->at ); + if ( $ws_len > 0 ) { $value_ends_at = $this->at; $this->at += $ws_len; // Accept either ) or EOF after whitespace. - if ( $this->at >= $limit ) { - if ( null !== $this->stream && ! $this->stream['finished'] ) { - $this->stream['phase'] = 'url-space'; - } + if ( $this->at >= $this->length ) { // EOF is a parse error, but we return the anyway. $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; @@ -1749,7 +1557,6 @@ private function consume_url( bool $continuing = false, bool $trailing_space = f } if ( ')' === $this->css[ $this->at ] ) { - $this->url_closed = true; // Skip the closing parenthesis and return the . ++$this->at; $this->token_type = self::TOKEN_URL; @@ -1817,12 +1624,8 @@ private function consume_url( bool $continuing = false, bool $trailing_space = f } } - if ( null !== $this->stream && ! $this->stream['finished'] ) { - $this->stream['phase'] = 'url'; - } - - // Return the URL value fragment at an input boundary. Actual EOF before - // the closing parenthesis is a parse error, but still returns a URL token. + // EOF + // This is a parse error. Return the . $this->token_type = self::TOKEN_URL; $this->token_length = $this->at - $this->token_starts_at; $this->token_value_starts_at = $value_starts_at; @@ -1841,18 +1644,10 @@ private function consume_url( bool $continuing = false, bool $trailing_space = f * @return bool */ private function consume_remnants_of_bad_url(): bool { - $limit = $this->fragment_limit ?? $this->length; - if ( null !== $this->stream ) { - $this->token_value_length = null === $this->token_value_starts_at ? null : $this->at - $this->token_value_starts_at; - $this->stream['phase'] = ''; - } - while ( $this->at < $limit ) { - $this->at += strcspn( $this->css, ')\\', $this->at, $limit - $this->at ); + while ( $this->at < $this->length ) { + $this->at += strcspn( $this->css, ')\\', $this->at ); - if ( $this->at >= $limit ) { - if ( null !== $this->stream && ! $this->stream['finished'] ) { - $this->stream['phase'] = 'bad-url'; - } + if ( $this->at >= $this->length ) { break; } @@ -1864,7 +1659,6 @@ private function consume_remnants_of_bad_url(): bool { continue; } } elseif ( ')' === $this->css[ $this->at ] ) { - $this->url_closed = true; ++$this->at; break; } @@ -1887,9 +1681,8 @@ private function consume_remnants_of_bad_url(): bool { * @see https://www.w3.org/TR/css-syntax-3/#consume-name */ private function consume_ident_sequence() { - $limit = $this->fragment_limit ?? $this->length; - while ( $this->at < $limit ) { - $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_', $this->at, $limit - $this->at ); + while ( $this->at < $this->length ) { + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_', $this->at ); if ( $plain > 0 ) { $this->at += $plain; continue; @@ -1910,9 +1703,6 @@ private function consume_ident_sequence() { break; } - if ( null !== $this->stream ) { - $this->stream['phase'] = $this->at >= $limit && ! $this->stream['finished'] ? 'name' : ''; - } } /** diff --git a/components/DataLiberation/README.md b/components/DataLiberation/README.md index b21b7a7e7..9b7717e36 100644 --- a/components/DataLiberation/README.md +++ b/components/DataLiberation/README.md @@ -285,10 +285,11 @@ frontmatter title exported ## Rewrite a CSS file in chunks and resume A download can stop between `https://old.exa` and `mple/photo.png`. The CSS -processor keeps that unfinished prefix and its CSS context in a cursor. +processor keeps the entire unfinished token and its CSS context in a cursor. +It parses that token again when more input arrives, like the XML processor. It uses the same tokenizer as inline CSS rewriting. Comments and displayed text stay unchanged; `url()`, bare `@import` strings, and `image-set()` URL -strings are recognized. +strings are recognized. Malformed string and URL tokens stay unchanged. ``` -a{src:url(https://new.example/photo.png)} +a{src:url("https://new.example/photo.png")} ``` -For files, use a fixed input chunk size and save progress in this order: +The caller chooses which URLs to change. `set_raw_url()` uses the existing +whole-value setter: it quotes an unquoted URL and escapes the replacement for +CSS. For example, replacing `old.png` with `new.png` changes `url(old.png)` to +`url("new.png")`. It does not apply a separate prefix-only rewrite rule. -1. Read a source chunk and pass it to `rewrite_chunk()`. -2. Write every output piece from the `foreach` loop. Finish the loop, then - flush the output file. -3. Save the source byte offset, output byte offset, and parser cursor together. - This saved state is a checkpoint. The source offset counts all bytes read, - including the unfinished bytes held in the cursor. +### Write and release completed output -The 64 KiB output threshold is checked after each complete token. A large -token can exceed it and is returned without splitting. Any remaining output -is returned after the input chunk has been processed. +`flush_processed_css()` returns completed CSS with edits applied and removes +those source bytes from memory. An unfinished token remains for the next read. +Flushing clears the current token or URL, so edit it before flushing. + +Appending input does not require a flush. Without flushing, `get_updated_css()` +returns all supplied CSS with edits applied. After a flush, it returns only the +retained CSS. Choose when to flush based on how much output the caller wants to +keep in memory. For example, flushing after each edited URL avoids storing many +large replacements. Flush once more after scanning stops to get completed CSS +after the last URL. There is no automatic output-size threshold or byte slicing. + +There is no token-size cap. A large comment, string, identifier, or embedded +image can use a lot of memory even with small input chunks. Each read reparses +the unfinished token. Small chunks therefore do not bound the largest token's +memory use or parsing work. The cursor does not copy these bytes. +More than 128 open, nested `image-set()` functions causes an error. + +### Resume in a new process + +Suppose the source is `a{src:url(https://old.example/photo.png)}`. A read ends +inside the URL. After flushing `a{src:`, the saved source offset points at +`url(`, not at the end of that read. A new process reads the unfinished URL +again from `url(`. It does not repeat the flushed prefix. + +`get_reentrancy_cursor()` returns an opaque string. Save it with +`get_token_byte_offset_in_the_input_stream()`. Supply source bytes from that +offset to `create_for_streaming($css, $cursor)`. The cursor contains parsing +state, including the URL position after `@import` or inside `image-set()`, but +no source bytes or edits. Do not inspect or change its internal format. + +A cursor saved while a token or URL is current reads that token again on +resume, as XML does. If `input_finished()` was already called, pass all remaining +source bytes to the factory; appending after the source end is rejected. +For file rewrites, save a checkpoint after flushing completed output instead: + +1. Read a source chunk and append it. Mark the source end when it is reached. +2. Scan and edit using the ordinary token or URL methods. +3. Write the string from `flush_processed_css()` and flush the output file. +4. Save the processor's source byte offset, the output file offset, and the + parser cursor together. Do not use the input file handle's current offset: + it can be past bytes that the processor still needs to read again. On resume, seek the source to its saved offset. Remove output bytes after the saved output offset, then append there. Those extra bytes may have been written before the previous process stopped, but after its last checkpoint. Removing them prevents duplicate output when the corresponding source is read again. -Keep the source file and URL mapping unchanged between runs. The cursor stores -a hash of the prepared replacement rules; resume rejects different rules. - -If a write fails or the output loop stops early, discard the processor and -resume from the last checkpoint. Do not continue the unfinished output loop. -The [file-rewrite test caller](Tests/fixtures/css-stream/rewrite-file.php) -shows how to save and restore both file positions and the parser state. - -Before matching, CSS escapes in a URL are decoded. Scheme and host matching -ignores letter case; path matching uses letter case. The longest source base -wins. For example, `/blog` matches `/blog/photo.png`, `/blog?x=1`, and `/blog#top`, -but not `/blogger`. A match can also end at the URL end. - -URLs that start with `//` keep that form. Relative paths, data URLs, and unrelated -hosts stay unchanged. Only the matched base is replaced. The remaining URL bytes -and surrounding quotes, parentheses, and spaces keep their original spelling. -Escapes inside the replaced base can change spelling. URLs read from CSS are -not fully normalized: path parts such as `/a/../b` and alternate encoded host -spellings are not resolved before matching. - -The [token-streaming limits above](#stream-css-tokens-and-resume) still apply. -An unfinished token, such as a comment or URL, is kept and parsed again when -more input arrives. There is no size limit for that token. Small input and output -chunks therefore do not limit its memory use, cursor size, or parsing work. -More than 128 open, nested `image-set()` functions causes an error before the -file is complete. `rewrite_chunk()` releases completed source bytes itself; -the caller does not need to call `flush_processed_css()`. - -Set `$is_last` to `true` only at the actual file end. If that is known only after -the last nonempty read, call `rewrite_chunk('', true)`. A download that stops -early has not reached the file end and must not be marked as complete. +Keep the source file and the caller's edit rules unchanged between runs. +The processor does not check either of them. + +If a write fails, discard the processor and resume from the last checkpoint. +The [token file caller](Tests/fixtures/css-token-stream/rewrite-file.php) and +[URL file caller](Tests/fixtures/css-stream/rewrite-file.php) show how to save +and restore both file positions and the parser state. Their tests stop on both +sides of a checkpoint and start a fresh PHP process to finish the output. diff --git a/components/DataLiberation/Tests/CSSPreprocessingTest.php b/components/DataLiberation/Tests/CSSPreprocessingTest.php index bac8565ca..4c114fe3d 100644 --- a/components/DataLiberation/Tests/CSSPreprocessingTest.php +++ b/components/DataLiberation/Tests/CSSPreprocessingTest.php @@ -24,24 +24,27 @@ public function test_preprocessed_token_keeps_source_bytes( $input, $type, $valu } /** - * Restores JSON state at every byte split, including beside NUL and inside CRLF or an escape. + * Restores at every byte split by rereading the unfinished source, including NUL, CRLF, and escapes. * The expected token is specified by the test, not copied from a whole-string parse. * * @dataProvider preprocessing_cases */ public function test_preprocessing_survives_split_input_and_resume( $input, $type, $value ) { for ( $split = 0; $split <= strlen( $input ); ++$split ) { - $processor = CSSProcessor::create_for_streaming(); + $processor = CSSProcessor::create_for_streaming( substr( $input, 0, $split ) ); $output = ''; $tokens = array(); - foreach ( array( substr( $input, 0, $split ), substr( $input, $split ) ) as $index => $chunk ) { - $processor->append_bytes( $chunk, 1 === $index ); + for ( $index = 0; $index < 2; ++$index ) { + if ( 1 === $index ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { $tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() ); } $output .= $processor->flush_processed_css(); - $cursor = json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ); - $processor = CSSProcessor::create_for_streaming( $cursor ); + if ( 0 === $index ) { + $processor = CSSProcessor::create_for_streaming( substr( $input, $processor->get_token_byte_offset_in_the_input_stream() ), $processor->get_reentrancy_cursor() ); + } } $this->assertSame( array( array( $type, $value, $input ) ), $tokens, 'Split at byte ' . $split ); $this->assertSame( $input, $output, 'Split at byte ' . $split ); diff --git a/components/DataLiberation/Tests/CSSStreamProcessTest.php b/components/DataLiberation/Tests/CSSStreamProcessTest.php index a3bc59ef9..cabe37893 100644 --- a/components/DataLiberation/Tests/CSSStreamProcessTest.php +++ b/components/DataLiberation/Tests/CSSStreamProcessTest.php @@ -43,7 +43,8 @@ public function test_token_edits_resume_after_process_death( $stop ) { $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); $this->assertGreaterThan( 0, $state['source_bytes'] ); $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); - $this->assertNotSame( '', $state['css']['pending_b64'] ); + $this->assertIsString( $state['css'] ); + $this->assertLessThan( 512, strlen( $state['css'] ) ); $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); } $this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) ); @@ -55,7 +56,7 @@ public function test_token_edits_resume_after_process_death( $stop ) { /** * Places NUL at the end of the second 32 KiB read, inside an unfinished URL. - * Resume must retain that byte, rewrite the URL, and leave a later bad URL unchanged. + * Resume must reread that byte, rewrite the URL, and leave a later bad URL unchanged. * * @dataProvider interruptions */ @@ -74,10 +75,7 @@ public function test_nul_preprocessing_preserves_file_offsets_after_resume( $sto $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); if ( 'none' !== $stop ) { $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); - $this->assertSame( 'before' === $stop ? 32768 : 65536, $state['source_bytes'] ); - if ( 'after' === $stop ) { - $this->assertStringContainsString( "\x00", base64_decode( $state['css']['pending_b64'] ) ); - } + $this->assertSame( 'before' === $stop ? strlen( $first ) : strlen( $first . $comment . 'a{src:' ), $state['source_bytes'] ); $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); } $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); diff --git a/components/DataLiberation/Tests/CSSStreamTest.php b/components/DataLiberation/Tests/CSSStreamTest.php index ad420415a..d4d64b680 100644 --- a/components/DataLiberation/Tests/CSSStreamTest.php +++ b/components/DataLiberation/Tests/CSSStreamTest.php @@ -16,14 +16,18 @@ public function test_source_bytes_survive_one_byte_input_and_resume( $input ) { $whole_tokens[] = array( $whole->get_token_type(), $whole->get_token_value(), $whole->get_unnormalized_token() ); } for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { - $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $processor->append_bytes( substr( $input, $offset, 1 ) ); + if ( strlen( $input ) === $offset ) { + $processor->input_finished(); + } $steps = 0; while ( $processor->next_token() ) { $tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() ); $this->assertLessThan( 64, ++$steps, 'The lexer must consume input or finish its current token.' ); } $output .= $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ) ); + $resume_offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSProcessor::create_for_streaming( substr( $input, $resume_offset, min( $offset + 1, strlen( $input ) ) - $resume_offset ), $processor->get_reentrancy_cursor() ); } $this->assertSame( $input, $output ); $this->assertSame( $whole_tokens, $tokens ); @@ -48,12 +52,16 @@ public function test_buffered_strings_match_the_whole_token() { $processor = CSSProcessor::create_for_streaming(); $decoded = ''; for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { - $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $processor->append_bytes( substr( $input, $offset, 1 ) ); + if ( strlen( $input ) === $offset ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { $decoded .= $processor->get_token_value(); } $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + $resume_offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSProcessor::create_for_streaming( substr( $input, $resume_offset, min( $offset + 1, strlen( $input ) ) - $resume_offset ), $processor->get_reentrancy_cursor() ); } $this->assertSame( $expected, $decoded ); } @@ -63,7 +71,9 @@ public function test_buffered_strings_match_the_whole_token() { public function test_buffered_tokens_use_the_existing_value_setter() { $processor = CSSProcessor::create_for_streaming(); $output = ''; + $source = ''; foreach ( array( 'a{src:url("https://old.exa', 'mple/a");color:red} ' ) as $input ) { + $source .= $input; $processor->append_bytes( $input ); while ( $processor->next_token() ) { if ( 'https://old.example/a' === $processor->get_token_value() ) { @@ -71,9 +81,9 @@ public function test_buffered_tokens_use_the_existing_value_setter() { } } $output .= $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + $processor = CSSProcessor::create_for_streaming( substr( $source, $processor->get_token_byte_offset_in_the_input_stream() ), $processor->get_reentrancy_cursor() ); } - $processor->append_bytes( '', true ); + $processor->input_finished(); while ( $processor->next_token() ) {} $output .= $processor->flush_processed_css(); $this->assertSame( 'a{src:url("https://new.example/moved/a");color:red} ', $output ); diff --git a/components/DataLiberation/Tests/CSSStreamingApiTest.php b/components/DataLiberation/Tests/CSSStreamingApiTest.php new file mode 100644 index 000000000..58d0c2297 --- /dev/null +++ b/components/DataLiberation/Tests/CSSStreamingApiTest.php @@ -0,0 +1,146 @@ +assertTrue( $processor->is_expecting_more_input() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->is_finished() ); + $processor->append_bytes( 'mple/a)' ); + $this->assertFalse( $processor->is_paused_at_incomplete_input() ); + $processor->input_finished(); + $this->assertFalse( $processor->is_expecting_more_input() ); + $this->assertTrue( $processor->$scan() ); + $this->assertSame( 'https://old.example/a', $processor->$get_value() ); + $this->assertTrue( $processor->$set_value( 'https://new.example/a' ) ); + $this->assertSame( 'url("https://new.example/a")', $processor->get_updated_css() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_finished() ); + $this->assertFalse( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->$scan() ); + } + + /** Appending must not discard earlier edits or require the caller to flush them first. @dataProvider processors */ + public function test_append_keeps_unflushed_edits( $class, $scan, $get_value, $set_value ) { + $processor = $class::create_for_streaming( 'url(first.png) ' ); + $this->assertTrue( $processor->$scan() ); + $this->assertTrue( $processor->$set_value( 'changed.png' ) ); + $processor->append_bytes( 'url(second.png)' ); + $processor->input_finished(); + $values = array(); + while ( $processor->$scan() ) { + if ( null !== $processor->$get_value() ) { + $values[] = $processor->$get_value(); + } + } + $this->assertSame( array( 'second.png' ), $values ); + $this->assertSame( 'url("changed.png") url(second.png)', $processor->get_updated_css() ); + } + + /** Like XML, a cursor at a token replays that token from the original source. @dataProvider processors */ + public function test_resume_replays_the_current_token_without_saving_edits( $class, $scan, $get_value, $set_value ) { + $input = 'url(first.png) url(second.png)'; + $processor = $class::create_for_streaming( $input ); + $processor->input_finished(); + $this->assertTrue( $processor->$scan() ); + $this->assertTrue( $processor->$set_value( 'changed.png' ) ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertSame( 0, $offset ); + $this->assertIsString( $cursor ); + $resumed = $class::create_for_streaming( substr( $input, $offset ), $cursor ); + $this->assertTrue( $resumed->$scan() ); + $this->assertSame( 'first.png', $resumed->$get_value() ); + $this->assertSame( $input, $resumed->get_updated_css() ); + } + + /** Unfinished bytes are reread from the source, rather than copied into the cursor. @dataProvider processors */ + public function test_resume_after_flushing_rereads_the_unfinished_token( $class, $scan, $get_value, $set_value ) { + $prefix = 'url(first.png) '; + $pending = '/*' . str_repeat( 'a', 131072 ); + $processor = $class::create_for_streaming( $prefix . $pending ); + while ( $processor->$scan() ) {} + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertSame( $prefix, $processor->flush_processed_css() ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertSame( strlen( $prefix ), $offset ); + $this->assertIsString( $cursor ); + $this->assertLessThan( 512, strlen( $cursor ) ); + $input = $prefix . $pending . '*/url(second.png)'; + $resumed = $class::create_for_streaming( substr( $input, $offset ), $cursor ); + $resumed->input_finished(); + $values = array(); + while ( $resumed->$scan() ) { + if ( null !== $resumed->$get_value() ) { + $values[] = $resumed->$get_value(); + } + } + $this->assertSame( array( 'second.png' ), $values ); + $this->assertSame( substr( $input, $offset ), $resumed->flush_processed_css() ); + $this->assertSame( strlen( $input ), $resumed->get_token_byte_offset_in_the_input_stream() ); + $this->assertTrue( $resumed->is_finished() ); + } + + /** Marking EOF twice is harmless; supplying more input afterwards is an error. @dataProvider processors */ + public function test_cannot_append_after_input_finished( $class, $scan, $get_value, $set_value ) { + $processor = $class::create_for_streaming(); + $processor->input_finished(); + $processor->input_finished(); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_finished() ); + $this->expectException( LogicException::class ); + $this->expectExceptionMessage( 'after the end of the stylesheet' ); + $processor->append_bytes( 'url(late.png)' ); + } + + /** Malformed cursor input must not silently start a new parse. @dataProvider processors */ + public function test_rejects_corrupt_cursor( $class, $scan, $get_value, $set_value ) { + $this->expectException( InvalidArgumentException::class ); + $class::create_for_streaming( '', 'not a cursor' ); + } + + /** + * Restoring on a string must keep the URL meaning given by its preceding syntax. + * + * @dataProvider url_contexts + */ + public function test_url_resume_replays_the_current_string( $input ) { + $processor = CSSURLProcessor::create_for_streaming( $input ); + $processor->input_finished(); + $this->assertTrue( $processor->next_url() ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $resumed = CSSURLProcessor::create_for_streaming( substr( $input, $offset ), $processor->get_reentrancy_cursor() ); + $this->assertTrue( $resumed->next_url() ); + $this->assertSame( 'first.png', $resumed->get_raw_url() ); + $this->assertTrue( $resumed->next_url() ); + $this->assertSame( 'second.png', $resumed->get_raw_url() ); + $this->assertFalse( $resumed->next_url() ); + } + + /** These strings are URLs only because of syntax before the saved byte offset. */ + public static function url_contexts() { + return array( + 'import' => array( '@import "first.png"; @import "second.png";' ), + 'quoted url' => array( 'a{src:url("first.png"),url("second.png")}' ), + 'image set' => array( 'a{src:image-set("first.png" type("image/png"),"second.png" 2x)}' ), + ); + } + + /** The APIs differ only in whether the caller scans all tokens or URL values. */ + public static function processors() { + return array( + 'tokens' => array( CSSProcessor::class, 'next_token', 'get_token_value', 'set_token_value' ), + 'URLs' => array( CSSURLProcessor::class, 'next_url', 'get_raw_url', 'set_raw_url' ), + ); + } +} diff --git a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php index 7311c2212..7fa84414d 100644 --- a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php +++ b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php @@ -1,6 +1,7 @@ assertSame( '\\6', substr( $input, 32766, 2 ) ); - $expected = strtr( $input, array( 'https://\\6f ld.example/' => 'https://old.example/moved/', 'https://old.example/' => 'https://old.example/moved/' ) ); + $whole = new CSSURLProcessor( $input ); + while ( $whole->next_url() ) { + $whole->set_raw_url( str_replace( 'https://old.example/', 'https://old.example/moved/', $whole->get_raw_url() ) ); + } + $expected = $whole->get_updated_css(); file_put_contents( $this->directory . '/source.css', $input ); $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); if ( 'none' !== $stop ) { @@ -64,14 +69,14 @@ public function test_file_rewrite_resumes_between_import_keyword_and_url( $stop $comment = '/*' . str_repeat( ' ', 65536 - strlen( $prefix ) - 4 ) . '*/'; $input = $prefix . $comment . '"https://old.example/theme.css";' . 'a{content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/last)}'; - $expected = 'a{src:url(https://old.example/moved/first)}@import' . $comment . '"https://old.example/moved/theme.css";' - . 'a{content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/moved/last)}'; + $expected = 'a{src:url("https://old.example/moved/first")}@import' . $comment . '"https://old.example/moved/theme.css";' + . 'a{content:"https://old.example/text";src:url(https://old.example/bad(image),url("https://old.example/moved/last")}'; file_put_contents( $this->directory . '/source.css', $input ); $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); if ( 'none' !== $stop ) { $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); - $this->assertSame( 'before' === $stop ? 32768 : 65536, $state['source_bytes'] ); - $this->assertSame( 'import', $state['css']['context']['expect'] ); + $this->assertSame( strlen( $prefix ), $state['source_bytes'] ); + $this->assertLessThan( 512, strlen( $state['css'] ) ); $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); } $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); @@ -93,11 +98,30 @@ public function test_file_rewrite_reports_a_nesting_limit_and_keeps_the_last_che $this->assertStringContainsString( 'nesting exceeds 128', file_get_contents( $this->directory . '/worker.log' ) ); $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); - $this->assertTrue( $state['css']['css']['expecting_more_input'] ); + $this->assertTrue( CSSURLProcessor::create_for_streaming( '', $state['css'] )->is_expecting_more_input() ); $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); } } + /** A stop after marking EOF must not skip a URL that has not yet been written. */ + public function test_resume_after_process_death_between_eof_and_final_edit() { + $path = str_repeat( 'a', 65536 ); + $input = '@import "https://old.example/' . $path . '";'; + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 99, $this->run_worker( 'eof' ), file_get_contents( $this->directory . '/worker.log' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( '@import ' ), $state['source_bytes'] ); + $this->assertSame( '@import ', file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + $expected = '@import "https://old.example/moved/' . $path . '";'; + $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + $this->assertTrue( CSSURLProcessor::create_for_streaming( '', $state['css'] )->is_finished() ); + } + /** * Selects normal completion, or exit just before or after the second saved state. * The worker has written the second chunk's output before either exit point. diff --git a/components/DataLiberation/Tests/CSSURLStreamTest.php b/components/DataLiberation/Tests/CSSURLStreamTest.php index 02b4c8604..b84f49244 100644 --- a/components/DataLiberation/Tests/CSSURLStreamTest.php +++ b/components/DataLiberation/Tests/CSSURLStreamTest.php @@ -3,155 +3,157 @@ use PHPUnit\Framework\TestCase; use WordPress\DataLiberation\URL\CSSURLProcessor; -/** Checks that splitting CSS input and restoring saved state do not change URL rewrite results. */ +/** Splitting input and restoring a cursor must preserve the whole-string scan-and-edit behavior. */ class CSSURLStreamTest extends TestCase { /** - * Splits each example at every byte position and restores the parser between the two parts. + * Replays unfinished source at every split and compares with the existing whole-string API. * * @dataProvider stylesheets */ - public function test_rewrites_and_resumes_at_every_byte( $input, $expected ) { - $mapping = array( 'https://old.example' => 'http://new.example/local' ); + public function test_rewrites_and_resumes_at_every_byte( $input ) { + $whole = new CSSURLProcessor( $input ); + $this->replace_urls( $whole ); + $expected = $whole->get_updated_css(); for ( $split = 0; $split <= strlen( $input ); ++$split ) { - $processor = CSSURLProcessor::create_for_streaming( $mapping ); - $output = $this->rewrite_chunk( $processor, substr( $input, 0, $split ), false ); - $cursor = json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ); - $processor = CSSURLProcessor::create_for_streaming( $mapping, $cursor ); - $output .= $this->rewrite_chunk( $processor, substr( $input, $split ), true ); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, 0, $split ) ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset ), $processor->get_reentrancy_cursor() ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); $this->assertSame( $expected, $output, 'Split at byte ' . $split ); + $this->assertTrue( $processor->is_finished() ); } } - /** - * Supplies CSS and its expected output, including text that must stay unchanged. - * Escapes and malformed CSS use exact strings so unwanted byte changes fail the test. - */ + /** Covers escapes, malformed tokens, URL context, and tokens accepted only at actual EOF. */ public static function stylesheets() { return array( - 'trailing URL spaces' => array( 'a{src:url(https://old.example )}', 'a{src:url(http://new.example/local )}' ), - 'leading URL spaces' => array( 'a{src:url( "https://old.example/a")}', 'a{src:url( "http://new.example/local/a")}' ), - 'bad URL keeps its syntax' => array( 'a{src:url(https://old.example/a(broken)}', 'a{src:url(https://old.example/a(broken)}' ), - 'bad string keeps its syntax' => array( "@import \"https://old.example/a\n", "@import \"https://old.example/a\n" ), - 'bad string consumes import position' => array( "@import \"https://old.example/bad\n" . '"https://old.example/text";@import/**/"https://old.example/theme.css";', "@import \"https://old.example/bad\n" . '"https://old.example/text";@import/**/"http://new.example/local/theme.css";' ), - 'URL position does not carry into text or a bad URL' => array( 'a{src:url(https://old.example/a);content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/b)}', 'a{src:url(http://new.example/local/a);content:"https://old.example/text";src:url(https://old.example/bad(image),url(http://new.example/local/b)}' ), - 'not a url function' => array( 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}', 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}' ), - 'nested image set' => array( 'a{src:image-set(image-set("https://old.example/a" 1x) 1x,"https://old.example/b" type("https://old.example/mime"))}', 'a{src:image-set(image-set("http://new.example/local/a" 1x) 1x,"http://new.example/local/b" type("https://old.example/mime"))}' ), - 'parenthesized resolution' => array( 'a{src:image-set("https://old.example/a" calc(1x * (2 + 3)), "https://old.example/b" 2x)}', 'a{src:image-set("http://new.example/local/a" calc(1x * (2 + 3)), "http://new.example/local/b" 2x)}' ), - 'long non-url names' => array( str_repeat( 'x', 70 ) . 'url("https://old.example/a")', str_repeat( 'x', 70 ) . 'url("https://old.example/a")' ), - 'long numeric dimension' => array( str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")', str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")' ), - 'quoted' => array( 'a{background:url("https://old.example/a.png")}', 'a{background:url("http://new.example/local/a.png")}' ), - 'unquoted' => array( 'a{background:url(https://old.example/a.png)}', 'a{background:url(http://new.example/local/a.png)}' ), - 'hex host and function' => array( 'a{background:\\75rl(https://\\6f ld.example/a.png)}', 'a{background:\\75rl(http://new.example/local/a.png)}' ), - 'slash escapes' => array( 'a{background:url(https:\\/\\/old.example\\/a.png)}', 'a{background:url(http://new.example/local\\/a.png)}' ), - 'protocol relative' => array( 'a{src:url(//old.example/font.woff2)}', 'a{src:url(//new.example/local/font.woff2)}' ), - 'import string' => array( '@import "https://old.example/a.css" screen;', '@import "http://new.example/local/a.css" screen;' ), - 'image set' => array( 'a{background:image-set("https://old.example/a.png" 1x, url(https://old.example/b.png) 2x)}', 'a{background:image-set("http://new.example/local/a.png" 1x, url(http://new.example/local/b.png) 2x)}' ), - 'comment and displayed text' => array( '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}', '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}' ), - 'unrelated and relative' => array( 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}', 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}' ), - 'EOF string' => array( 'a{src:url("https://old.example/a', 'a{src:url("http://new.example/local/a' ), - 'EOF URL' => array( 'a{src:url(https://old.example/a', 'a{src:url(http://new.example/local/a' ), - 'escaped line continuation' => array( "a{src:url(\"https://old.exa\\\r\nmple/a\")}", 'a{src:url("http://new.example/local/a")}' ), + 'trailing URL spaces' => array( 'a{src:url(https://old.example )}' ), + 'leading URL spaces' => array( 'a{src:url( "https://old.example/a")}' ), + 'bad URL keeps its syntax' => array( 'a{src:url(https://old.example/a(broken)}' ), + 'bad string keeps its syntax' => array( "@import \"https://old.example/a\n" ), + 'bad string consumes import position' => array( "@import \"https://old.example/bad\n" . '"https://old.example/text";@import/**/"https://old.example/theme.css";' ), + 'URL position does not carry into text or a bad URL' => array( 'a{src:url(https://old.example/a);content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/b)}' ), + 'not a url function' => array( 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}' ), + 'nested image set' => array( 'a{src:image-set(image-set("https://old.example/a" 1x) 1x,"https://old.example/b" type("https://old.example/mime"))}' ), + 'parenthesized resolution' => array( 'a{src:image-set("https://old.example/a" calc(1x * (2 + 3)), "https://old.example/b" 2x)}' ), + 'long non-url names' => array( str_repeat( 'x', 70 ) . 'url("https://old.example/a")' ), + 'long numeric dimension' => array( str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")' ), + 'quoted' => array( 'a{background:url("https://old.example/a.png")}' ), + 'unquoted' => array( 'a{background:url(https://old.example/a.png)}' ), + 'hex host and function' => array( 'a{background:\\75rl(https://\\6f ld.example/a.png)}' ), + 'slash escapes' => array( 'a{background:url(https:\\/\\/old.example\\/a.png)}' ), + 'protocol relative' => array( 'a{src:url(//old.example/font.woff2)}' ), + 'import string' => array( '@import "https://old.example/a.css" screen;' ), + 'image set' => array( 'a{background:image-set("https://old.example/a.png" 1x, url(https://old.example/b.png) 2x)}' ), + 'comment and displayed text' => array( '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}' ), + 'unrelated and relative' => array( 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}' ), + 'EOF string' => array( 'a{src:url("https://old.example/a' ), + 'EOF URL' => array( 'a{src:url(https://old.example/a' ), + 'escaped line continuation' => array( "a{src:url(\"https://old.exa\\\r\nmple/a\")}" ), ); } - /** Sends one byte per call and restores the parser after each byte, including inside escapes. */ - public function test_one_byte_chunks_match_whole_chunk_output() { - $mapping = array( 'https://old.example' => 'http://new.example/local' ); + /** Restores after every byte, including bytes inside CSS escapes and image-set() strings. */ + public function test_one_byte_chunks_match_whole_string_output() { foreach ( self::stylesheets() as $case ) { - $processor = CSSURLProcessor::create_for_streaming( $mapping ); + $input = $case[0]; + $whole = new CSSURLProcessor( $input ); + $this->replace_urls( $whole ); + $processor = CSSURLProcessor::create_for_streaming(); $output = ''; - for ( $at = 0; $at < strlen( $case[0] ); ++$at ) { - $output .= $this->rewrite_chunk( $processor, $case[0][$at], false ); - $processor = CSSURLProcessor::create_for_streaming( $mapping, $processor->get_reentrancy_cursor() ); + for ( $at = 0; $at < strlen( $input ); ++$at ) { + $processor->append_bytes( $input[ $at ] ); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset, $at + 1 - $offset ), $processor->get_reentrancy_cursor() ); } - $output .= $this->rewrite_chunk( $processor, '', true ); - $this->assertSame( $case[1], $output ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( $whole->get_updated_css(), $output ); } } /** - * Sends a single comment, string, URL, or name across many 32 KiB chunks. - * The saved unfinished input must grow until that item ends, then become empty. + * Sends one comment, string, URL, or name across 128 reads of 32 KiB each. + * The input buffer must grow until that item ends, but the cursor must not copy it. */ public function test_large_tokens_are_retained_until_complete_then_released() { - $mapping = array( 'https://old.example' => 'https://old.example/moved' ); foreach ( array( array( '/*', '*/' ), array( 'a{content:"', '"}' ), array( 'a{src:url(data:image/png;base64,', ')}' ), array( 'a{src:url(https://old.example/', ')}' ), array( '.long', '{}' ) ) as $token ) { - $processor = CSSURLProcessor::create_for_streaming( $mapping ); - $input_hash = hash_init( 'sha256' ); + $input = $token[0] . str_repeat( 'a', 128 * 32768 ) . $token[1]; + $whole = new CSSURLProcessor( $input ); + $this->replace_urls( $whole ); + $expected = hash( 'sha256', $whole->get_updated_css() ); + $processor = CSSURLProcessor::create_for_streaming( $token[0] ); + $this->replace_urls( $processor ); $output_hash = hash_init( 'sha256' ); - $expected_prefix = str_replace( 'https://old.example/', 'https://old.example/moved/', $token[0] ); - hash_update( $input_hash, $expected_prefix ); - hash_update( $output_hash, $this->rewrite_chunk( $processor, $token[0], false ) ); + hash_update( $output_hash, $processor->flush_processed_css() ); $retained_bytes = 0; for ( $chunk = 0; $chunk < 128; ++$chunk ) { - $bytes = str_repeat( 'a', 32768 ); - hash_update( $input_hash, $bytes ); - hash_update( $output_hash, $this->rewrite_chunk( $processor, $bytes, false ) ); + $processor->append_bytes( str_repeat( 'a', 32768 ) ); + $this->replace_urls( $processor ); + hash_update( $output_hash, $processor->flush_processed_css() ); + $this->assertGreaterThan( $retained_bytes, strlen( $processor->get_updated_css() ) ); + $retained_bytes = strlen( $processor->get_updated_css() ); $cursor = $processor->get_reentrancy_cursor(); - $this->assertGreaterThan( $retained_bytes, strlen( $cursor['css']['pending_b64'] ) ); - $retained_bytes = strlen( $cursor['css']['pending_b64'] ); - $processor = CSSURLProcessor::create_for_streaming( $mapping, $cursor ); + $this->assertLessThan( 512, strlen( $cursor ) ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset, strlen( $token[0] ) + ( $chunk + 1 ) * 32768 - $offset ), $cursor ); } - hash_update( $input_hash, $token[1] ); - hash_update( $output_hash, $this->rewrite_chunk( $processor, $token[1], true ) ); - $this->assertSame( hash_final( $input_hash ), hash_final( $output_hash ) ); - $this->assertSame( '', $processor->get_reentrancy_cursor()['css']['pending_b64'] ); + $processor->append_bytes( $token[1] ); + $processor->input_finished(); + $this->replace_urls( $processor ); + hash_update( $output_hash, $processor->flush_processed_css() ); + $this->assertSame( $expected, hash_final( $output_hash ) ); + $this->assertSame( '', $processor->get_updated_css() ); + $this->assertSame( strlen( $input ), $processor->get_token_byte_offset_in_the_input_stream() ); } } + /** * Expands 1,500 short URLs into over 5 MiB of output. - * Output must reach the caller before all replacements accumulate in memory. + * The caller flushes each edit instead of retaining all replacements until EOF. */ - public function test_expanded_output_is_yielded_as_it_grows() { + public function test_caller_can_flush_each_edit_to_bound_expanded_output() { $target = 'https://new.example/' . str_repeat( 'a', 4096 ); - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => $target ) ); $input = str_repeat( 'a{src:url(https://old.example/a)}', 1500 ); + $processor = CSSURLProcessor::create_for_streaming( $input ); + $processor->input_finished(); $expected = hash_init( 'sha256' ); for ( $index = 0; $index < 1500; ++$index ) { - hash_update( $expected, 'a{src:url(' . $target . '/a)}' ); + hash_update( $expected, 'a{src:url("' . $target . '/a")}' ); } $actual = hash_init( 'sha256' ); $bytes = 0; - $chunks = 0; $memory = memory_get_usage(); - foreach ( $processor->rewrite_chunk( $input, true ) as $chunk ) { + while ( $processor->next_url() ) { + $processor->set_raw_url( $target . '/a' ); + $output = $processor->flush_processed_css(); $this->assertLessThan( 2 * 1024 * 1024, memory_get_usage() - $memory ); - $bytes += strlen( $chunk ); - ++$chunks; - hash_update( $actual, $chunk ); + $bytes += strlen( $output ); + hash_update( $actual, $output ); } - $this->assertGreaterThan( 1, $chunks ); + $output = $processor->flush_processed_css(); + hash_update( $actual, $output ); + $bytes += strlen( $output ); $this->assertGreaterThan( 5 * 1024 * 1024, $bytes ); $this->assertSame( hash_final( $expected ), hash_final( $actual ) ); } - /** A large URL is returned whole, without byte slices or an empty final piece. */ - public function test_large_url_is_yielded_without_splitting() { - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); + /** Flushing a large URL returns it whole, with no fixed-size output slices. */ + public function test_large_url_is_flushed_without_splitting() { $path = str_repeat( 'a', 131072 ); - $input = 'url(https://old.example/' . $path . ')'; - $chunks = iterator_to_array( $processor->rewrite_chunk( $input, true ), false ); - $this->assertCount( 1, $chunks ); - $this->assertSame( 'url(https://new.example/' . $path . ')', $chunks[0] ); - } - - /** Stops at the first output piece and checks that saving a cursor is rejected until iteration ends. */ - public function test_cannot_checkpoint_unconsumed_output() { - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); - $output = $processor->rewrite_chunk( 'a{src:url(https://old.example/a)}', true ); - $output->rewind(); - $this->expectException( LogicException::class ); - $this->expectExceptionMessage( 'Consume all CSS output chunks before saving a cursor.' ); - $processor->get_reentrancy_cursor(); - } - - /** Changes the target host after saving half a URL; resume must reject the changed rule. */ - public function test_changed_mapping_cannot_resume_an_open_url() { - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); - $this->rewrite_chunk( $processor, 'a{src:url("https://old.exa', false ); - $this->expectException( InvalidArgumentException::class ); - $this->expectExceptionMessage( 'different URL mappings' ); - CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://other.example' ), $processor->get_reentrancy_cursor() ); + $processor = CSSURLProcessor::create_for_streaming( 'url(https://old.example/' . $path . ')' ); + $processor->input_finished(); + $this->assertTrue( $processor->next_url() ); + $processor->set_raw_url( 'https://new.example/' . $path ); + $this->assertSame( 'url("https://new.example/' . $path . '")', $processor->flush_processed_css() ); + $this->assertSame( '', $processor->flush_processed_css() ); } /** @@ -159,36 +161,44 @@ public function test_changed_mapping_cannot_resume_an_open_url() { * Those pairs decode to no characters, but must be retained until the quoted URL ends. */ public function test_long_escaped_url_is_rewritten_after_its_closing_quote() { - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); - $output = $this->rewrite_chunk( $processor, 'a{src:url("h', false ); + $processor = CSSURLProcessor::create_for_streaming( 'a{src:url("h' ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); for ( $index = 0; $index < 34; ++$index ) { - $output .= $this->rewrite_chunk( $processor, str_repeat( "\\\n", 16384 ), false ); + $processor->append_bytes( str_repeat( "\\\n", 16384 ) ); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); } - $output .= $this->rewrite_chunk( $processor, 'ttps://old.example/a")}', true ); + $processor->append_bytes( 'ttps://old.example/a")}' ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); $this->assertSame( 'a{src:url("https://new.example/a")}', $output ); } - /** Sends a URL without its closing ')'; only the preceding CSS may be returned before resume. */ + /** An unfinished URL cannot be written yet; resume must read it again from its start. */ public function test_unfinished_url_is_not_written_before_its_end() { - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ) ); - $output = $this->rewrite_chunk( $processor, 'a{src:url(https://old.example/' . str_repeat( 'a', 65536 ), false ); + $input = 'a{src:url(https://old.example/' . str_repeat( 'a', 65536 ); + $processor = CSSURLProcessor::create_for_streaming( $input ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); $this->assertSame( 'a{src:', $output ); - $processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://new.example' ), json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ) ); - $output .= $this->rewrite_chunk( $processor, ')}', true ); - $this->assertSame( 'a{src:url(https://new.example/' . str_repeat( 'a', 65536 ) . ')}', $output ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset ) . ')}', $processor->get_reentrancy_cursor() ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( 'a{src:url("https://new.example/' . str_repeat( 'a', 65536 ) . '")}', $output ); } - /** - * Joins output pieces so tests can compare their exact bytes, and rejects empty pieces. - * The large-output test reads the generator directly so this helper does not affect its memory check. - */ - private function rewrite_chunk( CSSURLProcessor $processor, string $input, bool $last ): string { - $output = ''; - foreach ( $processor->rewrite_chunk( $input, $last ) as $chunk ) { - $this->assertNotSame( '', $chunk ); - $output .= $chunk; + /** Uses the same caller-selected replacement for whole-string and streamed input. */ + private function replace_urls( CSSURLProcessor $processor ): void { + while ( $processor->next_url() ) { + $url = $processor->get_raw_url(); + $replacement = str_replace( 'https://old.example/', 'https://new.example/', $url ); + if ( $url !== $replacement ) { + $processor->set_raw_url( $replacement ); + } } - return $output; } - } diff --git a/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php index c2d6a911a..0ecde448b 100644 --- a/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php +++ b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php @@ -4,7 +4,8 @@ * Test caller that rewrites a CSS file and saves progress after each source chunk. * * Arguments are the source path, output path, state path, and stop mode. - * 'before' and 'after' exit around the second state save; 'none' runs to the end. + * 'before' and 'after' exit around the second state save. 'eof' exits after + * marking source EOF but before editing the last tokens. 'none' runs to the end. * Run again with the same paths and 'none' to resume from the state file. */ @@ -16,8 +17,8 @@ $output_path = $argv[2]; $state_path = $argv[3]; $stop = $argv[4]; -// The two offsets say where to resume reading and writing. The parser state -// keeps any unfinished CSS bytes already counted in source_bytes. +// The two offsets say where to resume reading and writing. The source offset +// points before unfinished bytes, which the new process reads again. $state = file_exists( $state_path ) ? json_decode( file_get_contents( $state_path ), true ) : array( 'source_bytes' => 0, 'output_bytes' => 0, 'css' => null ); $input = fopen( $input_path, 'rb' ); $output = fopen( $output_path, 'c+b' ); @@ -29,16 +30,25 @@ fseek( $output, $state['output_bytes'] ); // The target still starts with the source base. Rewriting a URL twice would // add '/moved' twice, which makes repeated replacements visible in the output. -$processor = CSSURLProcessor::create_for_streaming( array( 'https://old.example' => 'https://old.example/moved' ), $state['css'] ); +$processor = CSSURLProcessor::create_for_streaming( '', $state['css'] ); $chunks = 0; -while ( ! feof( $input ) ) { +while ( ! $processor->is_finished() ) { $chunk = fread( $input, 32768 ); - foreach ( $processor->rewrite_chunk( $chunk, feof( $input ) ) as $rewritten ) { - $written = fwrite( $output, $rewritten ); - if ( strlen( $rewritten ) !== $written ) { - throw new RuntimeException( 'CSS output wrote ' . $written . ' of ' . strlen( $rewritten ) . ' bytes.' ); + $processor->append_bytes( $chunk ); + if ( feof( $input ) ) { + $processor->input_finished(); + if ( 'eof' === $stop ) { + exit( 99 ); } } + while ( $processor->next_url() ) { + $processor->set_raw_url( str_replace( 'https://old.example/', 'https://old.example/moved/', $processor->get_raw_url() ) ); + } + $rewritten = $processor->flush_processed_css(); + $written = fwrite( $output, $rewritten ); + if ( strlen( $rewritten ) !== $written ) { + throw new RuntimeException( 'CSS output wrote ' . $written . ' of ' . strlen( $rewritten ) . ' bytes.' ); + } // Save offsets only after all output for this source chunk has been written. // A stopped process must not leave saved state ahead of the output file. fflush( $output ); @@ -46,7 +56,9 @@ if ( 'before' === $stop && 2 === $chunks ) { exit( 99 ); } - $state = array( 'source_bytes' => ftell( $input ), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + // The cursor contains no CSS bytes. Resume must reread unfinished input + // from the processor's source offset, which can be earlier than ftell($input). + $state = array( 'source_bytes' => $processor->get_token_byte_offset_in_the_input_stream(), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); // Replace the complete state file in one rename. A stop during the temporary // write leaves the previous saved offsets and parser state together. file_put_contents( $state_path . '.tmp', json_encode( $state ) ); diff --git a/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php index 124abdb0e..30a44177b 100644 --- a/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php +++ b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php @@ -16,11 +16,14 @@ // Discard them before replaying the corresponding source chunk. ftruncate( $output, $state['output_bytes'] ); fseek( $output, $state['output_bytes'] ); -$processor = CSSProcessor::create_for_streaming( $state['css'] ); +$processor = CSSProcessor::create_for_streaming( '', $state['css'] ); $chunks = 0; -while ( ! feof( $input ) ) { +while ( ! $processor->is_finished() ) { $chunk = fread( $input, 32768 ); - $processor->append_bytes( $chunk, feof( $input ) ); + $processor->append_bytes( $chunk ); + if ( feof( $input ) ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { if ( in_array( $processor->get_token_type(), array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_STRING ), true ) ) { $processor->set_token_value( str_replace( 'https://old.example/', 'https://old.example/moved/', $processor->get_token_value() ) ); @@ -36,7 +39,9 @@ if ( 'before' === $stop && 2 === $chunks ) { exit( 99 ); } - $state = array( 'source_bytes' => ftell( $input ), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + // The source offset follows completed tokens, not fread(): unfinished bytes + // must be read again from the file when a new process restores this cursor. + $state = array( 'source_bytes' => $processor->get_token_byte_offset_in_the_input_stream(), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); file_put_contents( $state_path . '.tmp', json_encode( $state ) ); rename( $state_path . '.tmp', $state_path ); if ( 'after' === $stop && 2 === $chunks ) { diff --git a/components/DataLiberation/URL/class-cssurlprocessor.php b/components/DataLiberation/URL/class-cssurlprocessor.php index e4248f4f7..9e4ef45da 100644 --- a/components/DataLiberation/URL/class-cssurlprocessor.php +++ b/components/DataLiberation/URL/class-cssurlprocessor.php @@ -5,7 +5,7 @@ use WordPress\DataLiberation\CSS\CSSProcessor; /** - * Finds CSS URLs in a complete string or rewrites them as source chunks arrive. + * Finds and edits CSS URLs in a complete string or as source bytes arrive. * * CSSProcessor reads one CSS item, called a token, at a time. A token can be * a quoted string, a comment, or an unquoted url(...). The surrounding syntax @@ -36,9 +36,9 @@ class CSSURLProcessor { /** * Remembers where a quoted string can be a URL as tokens are read. * - * Both next_url() and rewrite_chunk() advance through next_token(), which - * updates this state. Streaming callers save it in the cursor so a new - * process can continue inside an image-set(). + * The next_url() method advances through next_token(), which updates this + * state. Streaming callers save it in the cursor so a new process can + * continue inside an image-set(). * * @var array { * @type int $depth Number of function or '(' tokens not yet closed by ')'. @@ -53,6 +53,16 @@ class CSSURLProcessor { 'expect' => '', ); + /** + * Context before the current URL was read, for replaying that URL on resume. + * + * Reading the string in @import "theme.css" clears the next-string expectation. + * A cursor saved on that string needs the earlier expectation to find it again. + * + * @var array|null Same keys as $context; null before the first URL. + */ + private $context_before_url; + /** * @param string $css CSS source without wrapping braces. */ @@ -61,228 +71,128 @@ public function __construct( string $css ) { } /** - * URL replacement rules prepared once by create_for_streaming(). + * Opens a URL iterator that accepts CSS through append_bytes(). * - * Each input rule produces two entries: one for https://old.example and one - * for //old.example, for example. Entries with longer source bases come first - * so a rule for /blog wins over a rule for the whole site. + * Use the same next_url(), get_raw_url(), and set_raw_url() calls as for a + * complete string. The caller chooses replacements; this processor does not + * accept URL mappings. Call input_finished() at the actual source EOF. * - * Each entry contains 'origin' (scheme and host, with any port), 'prefix' - * (origin and path), 'target' (replacement escaped for CSS), and 'position' - * (original entry order, used when two prefixes have the same length). + * To resume, supply source bytes starting at the saved token byte offset + * and the cursor. A cursor saved on a URL reads that URL again. * - * @var array|null Null when constructed for next_url(), without streaming rules. + * @param string $css Initial source bytes; may be empty. + * @param string|null $cursor Opaque state from get_reentrancy_cursor(), or null for a new stream. + * @return static */ - private $mappings; + public static function create_for_streaming( string $css = '', ?string $cursor = null ) { + $state = null; + if ( null !== $cursor ) { + $state = json_decode( $cursor, true ); + if ( ! is_array( $state ) || ! isset( $state['css'], $state['context'] ) || ! is_string( $state['css'] ) || ! is_array( $state['context'] ) ) { + throw new \InvalidArgumentException( 'The CSS URL cursor must contain CSS parser state and URL context.' ); + } + } + $processor = new static( '' ); + $processor->processor = CSSProcessor::create_for_streaming( $css, $state['css'] ?? null ); + if ( null !== $state ) { + $processor->context = $state['context']; + } + return $processor; + } /** - * Whether iteration over the current rewrite_chunk() result has started but not finished. + * Adds source bytes without discarding earlier bytes or edits. * - * While true, output may still be waiting in the generator. Reject new input - * and saved cursors until the caller finishes the foreach loop. Otherwise - * a saved cursor could advance past output that the caller has not received. + * Continue with next_url(). Use flush_processed_css() separately when the + * caller is ready to write and release completed output. * - * @var bool + * @param string $bytes Next source bytes. */ - private $input_open = false; + public function append_bytes( string $bytes ): void { + $this->processor->append_bytes( $bytes ); + } - /** - * Identifies the URL replacement rules used before a rewrite stopped. - * - * A file started with old.example -> new.example must not resume with - * old.example -> other.example. That could leave two target hosts in one - * output file. create_for_streaming() rejects a cursor if its saved hash - * differs from the hash of the rules supplied for the new run. - * - * @var string|null SHA-256 hash of $mappings, including match order; null before streaming. - */ - private $mapping_hash; + /** Marks actual source EOF; a stopped download must not call this method. */ + public function input_finished(): void { + $this->processor->input_finished(); + } + + /** Returns whether the caller may still supply source bytes. */ + public function is_expecting_more_input(): bool { + return $this->processor->is_expecting_more_input(); + } + + /** Returns whether next_url() stopped because parsing needs more source bytes. */ + public function is_paused_at_incomplete_input(): bool { + return $this->processor->is_paused_at_incomplete_input(); + } + + /** Returns whether input has ended and no token remains to be read or inspected. */ + public function is_finished(): bool { + return $this->processor->is_finished(); + } /** - * Starts or resumes URL rewriting for CSS supplied in chunks. + * Returns completed CSS with edits applied and releases those source bytes. * - * For example, array( 'https://old.example' => 'https://new.example/local' ) - * changes https://old.example/photo.png to https://new.example/local/photo.png. - * Both bases must be HTTP(S) URLs without credentials, a query, or a fragment. + * An unfinished token stays in memory until more input arrives or the caller + * marks EOF. This clears the current URL, but keeps the surrounding syntax + * so next_url() can continue inside an image-set(). No output-size limit is + * imposed; callers can flush after each URL to avoid accumulating many edits. * - * Pass null for a new file. To resume, pass the cursor returned by - * get_reentrancy_cursor() and the same mapping in the same order. The cursor - * contains unfinished CSS bytes; supply only source bytes after the saved - * source offset. Completed source bytes are not kept by this processor. + * For files, write and flush this output before saving the parser cursor, + * source byte offset, and output byte offset together. If a write fails, + * discard this processor. Resume from the last saved cursor and source + * offset, first removing output bytes after the saved output offset so the + * repeated input does not duplicate them. Keep source and edit rules unchanged. * - * @param array $url_mapping Source URL bases as keys, replacement bases as values. - * @param array|null $cursor State from get_reentrancy_cursor(), or null to start. { - * @type array $css CSSProcessor state, including unfinished source bytes. - * @type array $context Saved $context: open parentheses, image sets, and expected strings. - * @type string $mapping_hash Hash used to reject a resume with different replacement rules. - * } - * @return static Processor ready for rewrite_chunk(). + * @return string Completed output, possibly empty; unfinished tokens have no size cap. */ - public static function create_for_streaming( array $url_mapping, ?array $cursor = null ) { - if ( null !== $cursor && ( ! isset( $cursor['css'], $cursor['context'], $cursor['mapping_hash'] ) || ! is_array( $cursor['css'] ) || ! is_array( $cursor['context'] ) ) ) { - throw new \InvalidArgumentException( 'The CSS URL cursor must contain buffered CSS input, URL context, and its mapping hash.' ); - } - $processor = new static( '' ); - $processor->processor = CSSProcessor::create_for_streaming( $cursor['css'] ?? null ); - $processor->mappings = array(); - if ( null !== $cursor ) { - $processor->context = $cursor['context']; - } - foreach ( $url_mapping as $source => $target ) { - $source_url = WPURL::parse( $source ); - $target_url = WPURL::parse( $target ); - foreach ( array( $source_url, $target_url ) as $url ) { - if ( false === $url || ! in_array( $url->protocol, array( 'http:', 'https:' ), true ) || '' !== $url->username || '' !== $url->password || '' !== $url->search || '' !== $url->hash ) { - throw new \InvalidArgumentException( 'CSS URL bases must be HTTP(S) addresses without credentials, query, or fragment: ' . $source . ' => ' . $target ); - } - } - $path = rtrim( $source_url->pathname, '/' ); - foreach ( array( $source_url->protocol, '' ) as $scheme ) { - $origin = $scheme . '//' . $source_url->host; - $processor->mappings[] = array( - 'origin' => $origin, - 'prefix' => $origin . $path, - 'target' => CSSProcessor::escape_value_prefix( ( '' === $scheme ? '' : $target_url->protocol ) . '//' . $target_url->host . rtrim( $target_url->pathname, '/' ) ), - 'position' => count( $processor->mappings ), - ); - } - } - usort( - $processor->mappings, - static function ( $first, $second ) { - return ( strlen( $second['prefix'] ) <=> strlen( $first['prefix'] ) ) ? ( strlen( $second['prefix'] ) <=> strlen( $first['prefix'] ) ) : ( $first['position'] <=> $second['position'] ); - } - ); - $processor->mapping_hash = hash( 'sha256', json_encode( $processor->mappings ) ); - if ( null !== $cursor && ( $cursor['mapping_hash'] ?? null ) !== $processor->mapping_hash ) { - throw new \InvalidArgumentException( 'Cannot resume CSS rewriting with different URL mappings. Start a new stylesheet rewrite.' ); - } - return $processor; + public function flush_processed_css(): string { + $output = $this->processor->flush_processed_css(); + $this->current_token_is_url = false; + return $output; } /** - * Rewrites URLs in the next source chunk and returns output through a generator. - * - * A chunk can end inside `url(https://old.exa`. That URL stays in memory until - * later input completes it, or $is_last marks the actual end of the file. - * Only the matched URL base changes. Quotes, parentheses, and the remaining - * URL bytes keep their original spelling. Comments and displayed text stay - * unchanged. Malformed string and URL tokens also stay unchanged. - * - * Use foreach to write each output chunk. Finish that loop before supplying - * more input or calling get_reentrancy_cursor(). For files, flush the output, - * then save the cursor and both file offsets together. The source offset - * includes every supplied byte, even bytes still held in the cursor. - * - * If writing fails or the loop stops early, discard this processor. Resume - * from the last saved cursor and source offset. First remove output bytes - * after the saved output offset so the repeated input does not duplicate them. + * Returns opaque state for a new processor at the current source position. * - * The 64 KiB output threshold is checked after each complete token. A large - * token can exceed it and is returned without splitting. Unfinished tokens - * have no size limit. A large comment or URL can increase memory use and cursor size. + * Save get_token_byte_offset_in_the_input_stream() beside this string and + * supply source bytes from that offset when resuming. The cursor contains + * parsing context, but no CSS bytes, pending edits, or replacement rules. + * Do not depend on its internal format. * - * @param string $chunk Source bytes immediately after the previous chunk; may be empty. - * @param bool $is_last True only at the actual end of the file, not when a download stops early. - * @return \Generator Nonempty output pieces, in source order. + * @return string State accepted by create_for_streaming(). */ - public function rewrite_chunk( string $chunk, bool $is_last ): \Generator { - if ( null === $this->mappings ) { - throw new \LogicException( 'Chunked CSS rewriting requires create_for_streaming().' ); - } - if ( $this->input_open ) { - throw new \LogicException( 'Consume all CSS output chunks before supplying another input chunk.' ); - } - if ( ! $this->processor->is_expecting_more_input() ) { - if ( '' === $chunk ) { - return; - } - throw new \LogicException( 'Cannot append CSS bytes after the end of the stylesheet.' ); - } - $this->input_open = true; - $output = ''; - $this->processor->append_bytes( $chunk, $is_last ); - while ( $this->next_token() ) { - $piece = $this->processor->get_unnormalized_token(); - if ( $this->current_token_is_url ) { - $type = $this->processor->get_token_type(); - $decoded = $this->get_raw_url(); - foreach ( $this->mappings as $mapping ) { - $prefix = $mapping['prefix']; - $origin_bytes = strlen( $mapping['origin'] ); - // Scheme and host ignore letter case; paths do not. Thus - // OLD.EXAMPLE/blog can match old.example/blog, but /Blog cannot. - if ( strlen( $decoded ) < strlen( $prefix ) || 0 !== strncasecmp( $decoded, $mapping['origin'], $origin_bytes ) || - substr( $decoded, $origin_bytes, strlen( $prefix ) - $origin_bytes ) !== substr( $prefix, $origin_bytes ) ) { - continue; - } - // A base ending in /blog must not match /blogger. A slash, query, - // fragment, or URL end must follow the matched base. - if ( strlen( $decoded ) > strlen( $prefix ) && false === strpos( '/?#', $decoded[ strlen( $prefix ) ] ) ) { - continue; - } - // Match decoded text, but edit the original bytes. For example, - // an escaped letter such as \6f takes more source bytes than 'o'. - // Measure only the base so escapes after it keep their spelling. - $value_start = $this->processor->get_token_value_start() - $this->processor->get_token_start(); - $raw_value = substr( $piece, $value_start, $this->processor->get_token_value_length() ); - $raw_prefix_bytes = CSSProcessor::measure_value_prefix( $raw_value, strlen( $prefix ), CSSProcessor::TOKEN_STRING === $type ); - $piece = substr_replace( $piece, $mapping['target'], $value_start, $raw_prefix_bytes ); - break; - } - } - $output .= $piece; - if ( strlen( $output ) >= 65536 ) { - // Replacements can be much longer than the input URLs. Return - // this output now so later replacements do not keep adding to it. - yield $output; - $output = ''; - } - } - // Completed tokens have been copied to output. Release their source - // bytes, but keep any unfinished token for the next input chunk. - $this->processor->flush_processed_css(); - if ( '' !== $output ) { - yield $output; - } - $this->input_open = false; + public function get_reentrancy_cursor(): string { + return json_encode( + array( + 'css' => $this->processor->get_reentrancy_cursor(), + 'context' => $this->current_token_is_url ? $this->context_before_url : $this->context, + ) + ); } /** - * Returns the parser state needed to resume this rewrite in a new process. + * Returns the source offset from which a saved cursor must resume. * - * Call this after writing all output from rewrite_chunk(). The array can be - * encoded as JSON. Save it with the source and output byte offsets, then pass - * it to create_for_streaming() with the same URL mapping to resume. This method - * returns data only; the caller must save it and manage the two files. + * This is the current URL's token start, or the next unread position when + * next_url() has paused or finished. After flushing, it is the first source + * byte not yet returned, even when replacements changed the output length. * - * @return array State for create_for_streaming(). { - * @type array $css CSSProcessor state, including unfinished source bytes. - * @type array $context Saved $context, so a string after resume keeps its URL meaning. - * @type string $mapping_hash Hash used to check that replacement rules did not change. - * } + * @return int Byte offset in the original source. */ - public function get_reentrancy_cursor(): array { - if ( null === $this->mappings ) { - throw new \LogicException( 'CSS URL cursors require create_for_streaming().' ); - } - if ( $this->input_open ) { - throw new \LogicException( 'Consume all CSS output chunks before saving a cursor.' ); - } - return array( - 'css' => $this->processor->get_reentrancy_cursor(), - 'context' => $this->context, - 'mapping_hash' => $this->mapping_hash, - ); + public function get_token_byte_offset_in_the_input_stream(): int { + return $this->processor->get_token_byte_offset_in_the_input_stream(); } /** - * Finds the next URL in the complete CSS string passed to the constructor. + * Finds the next URL in the supplied CSS. * * Recognizes url(), @import strings, and image-set() image strings. Skips * comments, displayed text, and malformed string or URL tokens. * - * @return bool True when get_raw_url() can read the next URL; false when none remain. + * @return bool True when get_raw_url() can read the next URL; false at EOF or when more bytes are needed. */ public function next_url(): bool { while ( $this->next_token() ) { @@ -321,8 +231,11 @@ private function next_token(): bool { // that the string is a URL before clearing the expectation for the next token. $this->current_token_is_url = CSSProcessor::TOKEN_URL === $type || ( '' !== $this->context['expect'] && CSSProcessor::TOKEN_STRING === $type ); - $name = in_array( $type, array( CSSProcessor::TOKEN_FUNCTION, CSSProcessor::TOKEN_AT_KEYWORD ), true ) ? $this->processor->get_token_value() : ''; - $this->context['expect'] = ''; + if ( $this->current_token_is_url ) { + $this->context_before_url = $this->context; + } + $name = in_array( $type, array( CSSProcessor::TOKEN_FUNCTION, CSSProcessor::TOKEN_AT_KEYWORD ), true ) ? $this->processor->get_token_value() : ''; + $this->context['expect'] = ''; if ( CSSProcessor::TOKEN_FUNCTION === $type ) { ++$this->context['depth']; $name = strtolower( $name );