Stream CSS tokens and resume unfinished input

components/DataLiberation/CSS/class-cssprocessor.php

@@ -308,6 +308,9 @@ class CSSProcessor {
 	 */
 	private $lexical_updates = array();
 
+	/** @var bool Whether an unfinished token may receive more source bytes. */
+	private $expecting_more_input = false;
+
 	/**
 	 * Constructor for the CSS processor.
 	 *
@@ -341,16 +344,126 @@ public static function create( string $css, string $encoding = 'UTF-8' ) {
 		return new static( $css );
 	}
 
+	/**
+	 * Opens a processor that keeps unfinished tokens until more input arrives.
+	 *
+	 * @param array|null $cursor {
+	 *     Optional state from get_reentrancy_cursor(), after flushing processed CSS.
+	 *     @type string $pending_b64 Unprocessed source bytes, base64 encoded.
+	 *     @type bool   $expecting_more_input Whether the source has more bytes.
+	 * }
+	 * @return static
+	 */
+	public static function create_for_streaming( ?array $cursor = null ) {
+		if ( null !== $cursor && ( ! isset( $cursor['pending_b64'], $cursor['expecting_more_input'] ) || ! is_string( $cursor['pending_b64'] ) || ! is_bool( $cursor['expecting_more_input'] ) ) ) {
+			throw new \InvalidArgumentException( 'The CSS cursor must contain a base64 input tail and whether more input is expected.' );
+		}
+		// phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes.
+		$pending = null === $cursor ? '' : base64_decode( $cursor['pending_b64'], true );
+		if ( false === $pending ) {
+			throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' );
+		}
+		$processor                       = new static( $pending );
+		$processor->expecting_more_input = $cursor['expecting_more_input'] ?? true;
+		return $processor;
+	}
+
+	/**
+	 * Appends source bytes after the caller has flushed completed tokens.
+	 *
+	 * The unfinished token stays in memory and is parsed again. Neither its size
+	 * nor the saved cursor size is capped. Use small reads to limit other input.
+	 *
+	 * @param string $bytes Next source bytes.
+	 * @param bool   $is_last Whether this is the actual end of the stylesheet.
+	 */
+	public function append_bytes( string $bytes, bool $is_last = false ): void {
+		if ( ! $this->expecting_more_input ) {
+			throw new \LogicException( 'CSS input cannot be appended after the end of the stylesheet.' );
+		}
+		if ( 0 !== $this->at ) {
+			throw new \LogicException( 'Flush processed CSS before appending more input.' );
+		}
+		$this->css                 .= $bytes;
+		$this->length               = strlen( $this->css );
+		$this->expecting_more_input = ! $is_last;
+	}
+
+	/** Returns whether more source bytes may be appended. */
+	public function is_expecting_more_input(): bool {
+		return $this->expecting_more_input;
+	}
+
+	/**
+	 * Returns edited, completed input and keeps the unfinished token for another read.
+	 *
+	 * Flush before appending input or saving a cursor. Existing whole-token setters
+	 * work in streaming mode; their edits are applied only to the returned prefix.
+	 *
+	 * @return string Processed CSS, including edits made with set_token_value().
+	 */
+	public function flush_processed_css(): string {
+		if ( 0 === $this->at ) {
+			return '';
+		}
+		$pending               = substr( $this->css, $this->at );
+		$updated               = $this->get_updated_css();
+		$output                = substr( $updated, 0, strlen( $updated ) - strlen( $pending ) );
+		$this->css             = $pending;
+		$this->length          = strlen( $pending );
+		$this->at              = 0;
+		$this->lexical_updates = array();
+		$this->after_token();
+		return $output;
+	}
+
+	/**
+	 * Returns unfinished input to save beside the caller's source and output offsets.
+	 *
+	 * @return array {
+	 *     @type string $pending_b64 Unprocessed source bytes, base64 encoded.
+	 *     @type bool   $expecting_more_input Whether the source has more bytes.
+	 * }
+	 */
+	public function get_reentrancy_cursor(): array {
+		if ( 0 !== $this->at ) {
+			throw new \LogicException( 'Flush processed CSS before saving a cursor.' );
+		}
+		return array(
+			// phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes.
+			'pending_b64' => base64_encode( $this->css ),
+			'expecting_more_input' => $this->expecting_more_input,
+		);
+	}
+
 	/**
 	 * Moves to the next token in the CSS stream.
 	 *
 	 * Implements the main tokenization loop, consuming the next token from the input stream.
 	 *
 	 * @see https://www.w3.org/TR/css-syntax-3/#consume-token
 	 *
-	 * @return bool Whether a token was found.
+	 * @return bool Whether a complete token was found; false also means more input is needed.
 	 */
 	public function next_token(): bool {
+		$start = $this->at;
+		if ( ! $this->scan_next_token() ) {
+			return false;
+		}
+		// A CSS escape needs at most six hex digits and CRLF after its backslash.
+		// Ten bytes also cover UTF-8 and the lookahead used to classify a token.
+		// A name may become a function, and a number may acquire a unit. Retry the
+		// whole token when more bytes arrive instead of saving its internal state.
+		if ( $this->expecting_more_input && $this->at > $this->length - 10 ) {
+			$this->at = $start;
+			$this->after_token();
+			return false;
+		}
+		return true;
+	}
+
+	/** Reads a whole token with the same CSS rules for complete and growing input. */
+	private function scan_next_token(): bool {
 		$this->after_token();
 
 		// Bale out once we reach the end.
@@ -1473,6 +1586,19 @@ private function consume_url(): bool {
 
 		// Repeatedly consume the next input code point from the stream.
 		while ( $this->at < $this->length ) {
+			// Scan runs such as https://example.com/photo.png in native code rather
+			// than one PHP iteration per byte. These ASCII bytes need no special
+			// handling in an unquoted URL. Quotes, parentheses, whitespace, escapes,
+			// and non-ASCII bytes fall through to the rules below.
+			// Only the cursor advances; source bytes stay unchanged. This also makes
+			// reparsing a long unfinished URL cheaper when another chunk arrives.
+			// This set only selects the fast path; it does not validate the URL or
+			// reject other bytes, which still reach the CSS token rules below.
+			$plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~:/?#[]@!$&*+,;=%', $this->at );
+			if ( $plain > 0 ) {
+				$this->at += $plain;
+				continue;
+			}
 			// U+0029 RIGHT PARENTHESIS ())
 			// Return the <url-token>.
 			if ( ')' === $this->css[ $this->at ] ) {
@@ -1629,6 +1755,17 @@ private function consume_remnants_of_bad_url(): bool {
 	 */
 	private function consume_ident_sequence() {
 		while ( $this->at < $this->length ) {
+			// Scan ASCII name characters, as in margin-top or item_2, in native
+			// code rather than one PHP iteration per byte. Letters, digits, "-",
+			// and "_" can continue a name; deciding whether an identifier may
+			// start here is a separate check, not the purpose of this byte list.
+			// Non-ASCII bytes, NULL, and escapes use the rules below. Advancing
+			// only the cursor preserves the source spelling for later decoding.
+			$plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_', $this->at );
+			if ( $plain > 0 ) {
+				$this->at += $plain;
+				continue;
+			}
 			$codepoint_bytes = $this->consume_ident_codepoint( $this->at );
 			if ( $codepoint_bytes > 0 ) {
 				$this->at += $codepoint_bytes;

components/DataLiberation/README.md

@@ -308,3 +308,25 @@ not returned. More than 128 nested image sets throw an error.
 
 [The file-rewrite caller](Tests/fixtures/css-context/rewrite-file.php) uses the
 whole-string iterator and writes the output only after iteration completes.
+
+## Stream CSS tokens and resume
+
+`CSSProcessor::create_for_streaming()` accepts source chunks through
+`append_bytes($bytes, $is_last)`. Read complete tokens with the existing
+`next_token()`, getters, and `set_token_value()`. If a token is unfinished,
+the processor keeps it and tries again after the next read, like XML.
+Only mark actual source EOF, not the end of an interrupted response.
+
+`flush_processed_css()` returns completed input with edits applied and releases
+its source bytes. Call it before appending more input or saving a cursor.
+`get_reentrancy_cursor()` saves the unfinished input in a JSON-safe array;
+pass it to `create_for_streaming()` in the next process. Save the source offset,
+output offset, and cursor together after writing and flushing the output.
+Resume truncates output to that saved offset before replaying more source.
+[The separate-process file caller](Tests/fixtures/css-token-stream/rewrite-file.php)
+shows both sides of that checkpoint boundary.
+
+There is no token-size cap. A large comment, string, identifier, or embedded
+image increases memory use and the saved cursor size. Each read reparses the
+unfinished token. Small chunks therefore do not bound the largest token's
+memory use or parsing work.

components/DataLiberation/Tests/CSSStreamProcessTest.php

@@ -0,0 +1,72 @@
+<?php
+
+use PHPUnit\Framework\TestCase;
+use WordPress\DataLiberation\CSS\CSSProcessor;
+
+/** Streams token edits through real files and resumes in a fresh PHP process. */
+class CSSStreamProcessTest extends TestCase {
+	/** @var string */
+	private $directory;
+
+	/** @before */
+	public function create_directory() {
+		$this->directory = sys_get_temp_dir() . '/css-token-stream-' . bin2hex( random_bytes( 8 ) );
+		mkdir( $this->directory );
+	}
+
+	/** @after */
+	public function remove_directory() {
+		foreach ( glob( $this->directory . '/*' ) as $path ) {
+			unlink( $path );
+		}
+		rmdir( $this->directory );
+	}
+
+	/** @dataProvider interruptions */
+	public function test_token_edits_resume_after_process_death( $stop ) {
+		// The first boundary splits a CSS escape; the second remains inside a comment.
+		$prefix = '/*' . str_repeat( 'a', 32743 ) . '*/a{src:url("https://\\6f ld.example/a.png")}';
+		$input = $prefix . '/*' . str_repeat( 'b', 32768 ) . '*/'
+			. '@import "https://old.example/theme.css";'
+			. 'a{src:url(https://old.example/' . str_repeat( 'c', 131072 ) . ')}';
+		$this->assertSame( '\\6', substr( $input, 32766, 2 ) );
+		$whole = CSSProcessor::create( $input );
+		while ( $whole->next_token() ) {
+			if ( in_array( $whole->get_token_type(), array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_STRING ), true ) ) {
+				$whole->set_token_value( str_replace( 'https://old.example/', 'https://old.example/moved/', $whole->get_token_value() ) );
+			}
+		}
+		$expected = $whole->get_updated_css();
+		file_put_contents( $this->directory . '/source.css', $input );
+		$this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) );
+		if ( 'none' !== $stop ) {
+			$state = json_decode( file_get_contents( $this->directory . '/state.json' ), true );
+			$this->assertGreaterThan( 0, $state['source_bytes'] );
+			$this->assertLessThan( strlen( $input ), $state['source_bytes'] );
+			$this->assertNotSame( '', $state['css']['pending_b64'] );
+			$this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) );
+		}
+		$this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) );
+		$this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) );
+		$state = json_decode( file_get_contents( $this->directory . '/state.json' ), true );
+		$this->assertSame( strlen( $input ), $state['source_bytes'] );
+		$this->assertSame( strlen( $expected ), $state['output_bytes'] );
+	}
+
+	/** Runs through completion or exits on either side of the second file checkpoint. */
+	public static function interruptions() {
+		return array( array( 'none' ), array( 'before' ), array( 'after' ) );
+	}
+
+	/** Runs the same caller used for uninterrupted and resumed file rewrites. */
+	private function run_worker( $stop ) {
+		$arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-token-stream/rewrite-file.php', $this->directory . '/source.css', $this->directory . '/target.css', $this->directory . '/state.json', $stop );
+		$command = implode( ' ', array_map( 'escapeshellarg', $arguments ) );
+		// Windows cmd.exe strips quotes from this command. Launch PHP directly;
+		// the command stays a string for PHP 7.2, which cannot accept an argument array.
+		$process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes, null, null, array( 'bypass_shell' => true ) );
+		$this->assertIsResource( $process );
+		fclose( $pipes[0] );
+		return proc_close( $process );
+	}
+}

components/DataLiberation/Tests/CSSStreamTest.php

@@ -0,0 +1,82 @@
+<?php
+
+use PHPUnit\Framework\TestCase;
+use WordPress\DataLiberation\CSS\CSSProcessor;
+
+/** The CSS lexer must keep every source byte while forgetting completed input. */
+class CSSStreamTest extends TestCase {
+	/** @dataProvider corpus */
+	public function test_source_bytes_survive_one_byte_input_and_resume( $input ) {
+		$processor = CSSProcessor::create_for_streaming();
+		$output = '';
+		$tokens = array();
+		$whole_tokens = array();
+		$whole = CSSProcessor::create( $input );
+		while ( $whole->next_token() ) {
+			$whole_tokens[] = array( $whole->get_token_type(), $whole->get_token_value(), $whole->get_unnormalized_token() );
+		}
+		for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) {
+			$processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset );
+			$steps = 0;
+			while ( $processor->next_token() ) {
+				$tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() );
+				$this->assertLessThan( 64, ++$steps, 'The lexer must consume input or finish its current token.' );
+			}
+			$output .= $processor->flush_processed_css();
+			$processor = CSSProcessor::create_for_streaming( json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ) );
+		}
+		$this->assertSame( $input, $output );
+		$this->assertSame( $whole_tokens, $tokens );
+	}
+
+	/** Supplies the CSS corpus plus byte sequences that cross UTF-8 and escape boundaries. */
+	public static function corpus() {
+		$cases = json_decode( file_get_contents( __DIR__ . '/css-test-cases.json' ), true );
+		foreach ( $cases as $name => $case ) {
+			yield $name => array( $case['css'] );
+		}
+		yield 'invalid UTF-8' => array( "a{content:\"bad\xff\xc3x\xe2\x82\x80\"}" );
+		yield 'UTF-8 at every position' => array( 'a{content:"aé東京😀\1f600 \0000e9 b"}' );
+	}
+
+	/** A buffered string uses the same UTF-8 and escape rules as whole-string input. */
+	public function test_buffered_strings_match_the_whole_token() {
+		foreach ( array( '"é東京😀\1f600 \0000e9 b"', "\"before\\\r\nafter\\\nend\\\ftail\"", "\"\xc3x\xe2\x82\x80\xff\x00\"" ) as $input ) {
+			$whole = CSSProcessor::create( $input );
+			$this->assertTrue( $whole->next_token() );
+			$expected = $whole->get_token_value();
+			$processor = CSSProcessor::create_for_streaming();
+			$decoded = '';
+			for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) {
+				$processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset );
+				while ( $processor->next_token() ) {
+					$decoded .= $processor->get_token_value();
+				}
+				$processor->flush_processed_css();
+				$processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() );
+			}
+			$this->assertSame( $expected, $decoded );
+		}
+	}
+
+	/** The existing whole-token setter must survive flushing and a fresh processor. */
+	public function test_buffered_tokens_use_the_existing_value_setter() {
+		$processor = CSSProcessor::create_for_streaming();
+		$output = '';
+		foreach ( array( 'a{src:url("https://old.exa', 'mple/a");color:red}            ' ) as $input ) {
+			$processor->append_bytes( $input );
+			while ( $processor->next_token() ) {
+				if ( 'https://old.example/a' === $processor->get_token_value() ) {
+					$this->assertTrue( $processor->set_token_value( 'https://new.example/moved/a' ) );
+				}
+			}
+			$output .= $processor->flush_processed_css();
+			$processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() );
+		}
+		$processor->append_bytes( '', true );
+		while ( $processor->next_token() ) {}
+		$output .= $processor->flush_processed_css();
+		$this->assertSame( 'a{src:url("https://new.example/moved/a");color:red}            ', $output );
+	}
+
+}