diff --git a/components/DataLiberation/CSS/class-cssprocessor.php b/components/DataLiberation/CSS/class-cssprocessor.php index 3abcbab1..3a9e5571 100644 --- a/components/DataLiberation/CSS/class-cssprocessor.php +++ b/components/DataLiberation/CSS/class-cssprocessor.php @@ -308,6 +308,9 @@ class CSSProcessor { */ private $lexical_updates = array(); + /** @var bool Whether an unfinished token may receive more source bytes. */ + private $expecting_more_input = false; + /** * Constructor for the CSS processor. * @@ -341,6 +344,98 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { return new static( $css ); } + /** + * Opens a processor that keeps unfinished tokens until more input arrives. + * + * @param array|null $cursor { + * Optional state from get_reentrancy_cursor(), after flushing processed CSS. + * @type string $pending_b64 Unprocessed source bytes, base64 encoded. + * @type bool $expecting_more_input Whether the source has more bytes. + * } + * @return static + */ + public static function create_for_streaming( ?array $cursor = null ) { + if ( null !== $cursor && ( ! isset( $cursor['pending_b64'], $cursor['expecting_more_input'] ) || ! is_string( $cursor['pending_b64'] ) || ! is_bool( $cursor['expecting_more_input'] ) ) ) { + throw new \InvalidArgumentException( 'The CSS cursor must contain a base64 input tail and whether more input is expected.' ); + } + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. + $pending = null === $cursor ? '' : base64_decode( $cursor['pending_b64'], true ); + if ( false === $pending ) { + throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' ); + } + $processor = new static( $pending ); + $processor->expecting_more_input = $cursor['expecting_more_input'] ?? true; + return $processor; + } + + /** + * Appends source bytes after the caller has flushed completed tokens. + * + * The unfinished token stays in memory and is parsed again. Neither its size + * nor the saved cursor size is capped. Use small reads to limit other input. + * + * @param string $bytes Next source bytes. + * @param bool $is_last Whether this is the actual end of the stylesheet. + */ + public function append_bytes( string $bytes, bool $is_last = false ): void { + if ( ! $this->expecting_more_input ) { + throw new \LogicException( 'CSS input cannot be appended after the end of the stylesheet.' ); + } + if ( 0 !== $this->at ) { + throw new \LogicException( 'Flush processed CSS before appending more input.' ); + } + $this->css .= $bytes; + $this->length = strlen( $this->css ); + $this->expecting_more_input = ! $is_last; + } + + /** Returns whether more source bytes may be appended. */ + public function is_expecting_more_input(): bool { + return $this->expecting_more_input; + } + + /** + * Returns edited, completed input and keeps the unfinished token for another read. + * + * Flush before appending input or saving a cursor. Existing whole-token setters + * work in streaming mode; their edits are applied only to the returned prefix. + * + * @return string Processed CSS, including edits made with set_token_value(). + */ + public function flush_processed_css(): string { + if ( 0 === $this->at ) { + return ''; + } + $pending = substr( $this->css, $this->at ); + $updated = $this->get_updated_css(); + $output = substr( $updated, 0, strlen( $updated ) - strlen( $pending ) ); + $this->css = $pending; + $this->length = strlen( $pending ); + $this->at = 0; + $this->lexical_updates = array(); + $this->after_token(); + return $output; + } + + /** + * Returns unfinished input to save beside the caller's source and output offsets. + * + * @return array { + * @type string $pending_b64 Unprocessed source bytes, base64 encoded. + * @type bool $expecting_more_input Whether the source has more bytes. + * } + */ + public function get_reentrancy_cursor(): array { + if ( 0 !== $this->at ) { + throw new \LogicException( 'Flush processed CSS before saving a cursor.' ); + } + return array( + // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes. + 'pending_b64' => base64_encode( $this->css ), + 'expecting_more_input' => $this->expecting_more_input, + ); + } + /** * Moves to the next token in the CSS stream. * @@ -348,9 +443,27 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { * * @see https://www.w3.org/TR/css-syntax-3/#consume-token * - * @return bool Whether a token was found. + * @return bool Whether a complete token was found; false also means more input is needed. */ public function next_token(): bool { + $start = $this->at; + if ( ! $this->scan_next_token() ) { + return false; + } + // A CSS escape needs at most six hex digits and CRLF after its backslash. + // Ten bytes also cover UTF-8 and the lookahead used to classify a token. + // A name may become a function, and a number may acquire a unit. Retry the + // whole token when more bytes arrive instead of saving its internal state. + if ( $this->expecting_more_input && $this->at > $this->length - 10 ) { + $this->at = $start; + $this->after_token(); + return false; + } + return true; + } + + /** Reads a whole token with the same CSS rules for complete and growing input. */ + private function scan_next_token(): bool { $this->after_token(); // Bale out once we reach the end. @@ -1473,6 +1586,19 @@ private function consume_url(): bool { // Repeatedly consume the next input code point from the stream. while ( $this->at < $this->length ) { + // Scan runs such as https://example.com/photo.png in native code rather + // than one PHP iteration per byte. These ASCII bytes need no special + // handling in an unquoted URL. Quotes, parentheses, whitespace, escapes, + // and non-ASCII bytes fall through to the rules below. + // Only the cursor advances; source bytes stay unchanged. This also makes + // reparsing a long unfinished URL cheaper when another chunk arrives. + // This set only selects the fast path; it does not validate the URL or + // reject other bytes, which still reach the CSS token rules below. + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~:/?#[]@!$&*+,;=%', $this->at ); + if ( $plain > 0 ) { + $this->at += $plain; + continue; + } // U+0029 RIGHT PARENTHESIS ()) // Return the . if ( ')' === $this->css[ $this->at ] ) { @@ -1629,6 +1755,17 @@ private function consume_remnants_of_bad_url(): bool { */ private function consume_ident_sequence() { while ( $this->at < $this->length ) { + // Scan ASCII name characters, as in margin-top or item_2, in native + // code rather than one PHP iteration per byte. Letters, digits, "-", + // and "_" can continue a name; deciding whether an identifier may + // start here is a separate check, not the purpose of this byte list. + // Non-ASCII bytes, NULL, and escapes use the rules below. Advancing + // only the cursor preserves the source spelling for later decoding. + $plain = strspn( $this->css, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_', $this->at ); + if ( $plain > 0 ) { + $this->at += $plain; + continue; + } $codepoint_bytes = $this->consume_ident_codepoint( $this->at ); if ( $codepoint_bytes > 0 ) { $this->at += $codepoint_bytes; diff --git a/components/DataLiberation/README.md b/components/DataLiberation/README.md index 92ea2dfe..b28358bc 100644 --- a/components/DataLiberation/README.md +++ b/components/DataLiberation/README.md @@ -308,3 +308,25 @@ not returned. More than 128 nested image sets throw an error. [The file-rewrite caller](Tests/fixtures/css-context/rewrite-file.php) uses the whole-string iterator and writes the output only after iteration completes. + +## Stream CSS tokens and resume + +`CSSProcessor::create_for_streaming()` accepts source chunks through +`append_bytes($bytes, $is_last)`. Read complete tokens with the existing +`next_token()`, getters, and `set_token_value()`. If a token is unfinished, +the processor keeps it and tries again after the next read, like XML. +Only mark actual source EOF, not the end of an interrupted response. + +`flush_processed_css()` returns completed input with edits applied and releases +its source bytes. Call it before appending more input or saving a cursor. +`get_reentrancy_cursor()` saves the unfinished input in a JSON-safe array; +pass it to `create_for_streaming()` in the next process. Save the source offset, +output offset, and cursor together after writing and flushing the output. +Resume truncates output to that saved offset before replaying more source. +[The separate-process file caller](Tests/fixtures/css-token-stream/rewrite-file.php) +shows both sides of that checkpoint boundary. + +There is no token-size cap. A large comment, string, identifier, or embedded +image increases memory use and the saved cursor size. Each read reparses the +unfinished token. Small chunks therefore do not bound the largest token's +memory use or parsing work. diff --git a/components/DataLiberation/Tests/CSSStreamProcessTest.php b/components/DataLiberation/Tests/CSSStreamProcessTest.php new file mode 100644 index 00000000..ec4643d0 --- /dev/null +++ b/components/DataLiberation/Tests/CSSStreamProcessTest.php @@ -0,0 +1,72 @@ +directory = sys_get_temp_dir() . '/css-token-stream-' . bin2hex( random_bytes( 8 ) ); + mkdir( $this->directory ); + } + + /** @after */ + public function remove_directory() { + foreach ( glob( $this->directory . '/*' ) as $path ) { + unlink( $path ); + } + rmdir( $this->directory ); + } + + /** @dataProvider interruptions */ + public function test_token_edits_resume_after_process_death( $stop ) { + // The first boundary splits a CSS escape; the second remains inside a comment. + $prefix = '/*' . str_repeat( 'a', 32743 ) . '*/a{src:url("https://\\6f ld.example/a.png")}'; + $input = $prefix . '/*' . str_repeat( 'b', 32768 ) . '*/' + . '@import "https://old.example/theme.css";' + . 'a{src:url(https://old.example/' . str_repeat( 'c', 131072 ) . ')}'; + $this->assertSame( '\\6', substr( $input, 32766, 2 ) ); + $whole = CSSProcessor::create( $input ); + while ( $whole->next_token() ) { + if ( in_array( $whole->get_token_type(), array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_STRING ), true ) ) { + $whole->set_token_value( str_replace( 'https://old.example/', 'https://old.example/moved/', $whole->get_token_value() ) ); + } + } + $expected = $whole->get_updated_css(); + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); + if ( 'none' !== $stop ) { + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertGreaterThan( 0, $state['source_bytes'] ); + $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); + $this->assertNotSame( '', $state['css']['pending_b64'] ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + } + $this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + } + + /** Runs through completion or exits on either side of the second file checkpoint. */ + public static function interruptions() { + return array( array( 'none' ), array( 'before' ), array( 'after' ) ); + } + + /** Runs the same caller used for uninterrupted and resumed file rewrites. */ + private function run_worker( $stop ) { + $arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-token-stream/rewrite-file.php', $this->directory . '/source.css', $this->directory . '/target.css', $this->directory . '/state.json', $stop ); + $command = implode( ' ', array_map( 'escapeshellarg', $arguments ) ); + // Windows cmd.exe strips quotes from this command. Launch PHP directly; + // the command stays a string for PHP 7.2, which cannot accept an argument array. + $process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes, null, null, array( 'bypass_shell' => true ) ); + $this->assertIsResource( $process ); + fclose( $pipes[0] ); + return proc_close( $process ); + } +} diff --git a/components/DataLiberation/Tests/CSSStreamTest.php b/components/DataLiberation/Tests/CSSStreamTest.php new file mode 100644 index 00000000..ad420415 --- /dev/null +++ b/components/DataLiberation/Tests/CSSStreamTest.php @@ -0,0 +1,82 @@ +next_token() ) { + $whole_tokens[] = array( $whole->get_token_type(), $whole->get_token_value(), $whole->get_unnormalized_token() ); + } + for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { + $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $steps = 0; + while ( $processor->next_token() ) { + $tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() ); + $this->assertLessThan( 64, ++$steps, 'The lexer must consume input or finish its current token.' ); + } + $output .= $processor->flush_processed_css(); + $processor = CSSProcessor::create_for_streaming( json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ) ); + } + $this->assertSame( $input, $output ); + $this->assertSame( $whole_tokens, $tokens ); + } + + /** Supplies the CSS corpus plus byte sequences that cross UTF-8 and escape boundaries. */ + public static function corpus() { + $cases = json_decode( file_get_contents( __DIR__ . '/css-test-cases.json' ), true ); + foreach ( $cases as $name => $case ) { + yield $name => array( $case['css'] ); + } + yield 'invalid UTF-8' => array( "a{content:\"bad\xff\xc3x\xe2\x82\x80\"}" ); + yield 'UTF-8 at every position' => array( 'a{content:"aé東京😀\1f600 \0000e9 b"}' ); + } + + /** A buffered string uses the same UTF-8 and escape rules as whole-string input. */ + public function test_buffered_strings_match_the_whole_token() { + foreach ( array( '"é東京😀\1f600 \0000e9 b"', "\"before\\\r\nafter\\\nend\\\ftail\"", "\"\xc3x\xe2\x82\x80\xff\x00\"" ) as $input ) { + $whole = CSSProcessor::create( $input ); + $this->assertTrue( $whole->next_token() ); + $expected = $whole->get_token_value(); + $processor = CSSProcessor::create_for_streaming(); + $decoded = ''; + for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { + $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + while ( $processor->next_token() ) { + $decoded .= $processor->get_token_value(); + } + $processor->flush_processed_css(); + $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + } + $this->assertSame( $expected, $decoded ); + } + } + + /** The existing whole-token setter must survive flushing and a fresh processor. */ + public function test_buffered_tokens_use_the_existing_value_setter() { + $processor = CSSProcessor::create_for_streaming(); + $output = ''; + foreach ( array( 'a{src:url("https://old.exa', 'mple/a");color:red} ' ) as $input ) { + $processor->append_bytes( $input ); + while ( $processor->next_token() ) { + if ( 'https://old.example/a' === $processor->get_token_value() ) { + $this->assertTrue( $processor->set_token_value( 'https://new.example/moved/a' ) ); + } + } + $output .= $processor->flush_processed_css(); + $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + } + $processor->append_bytes( '', true ); + while ( $processor->next_token() ) {} + $output .= $processor->flush_processed_css(); + $this->assertSame( 'a{src:url("https://new.example/moved/a");color:red} ', $output ); + } + +} diff --git a/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php new file mode 100644 index 00000000..124abdb0 --- /dev/null +++ b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php @@ -0,0 +1,47 @@ + 0, 'output_bytes' => 0, 'css' => null ); +$input = fopen( $input_path, 'rb' ); +$output = fopen( $output_path, 'c+b' ); +fseek( $input, $state['source_bytes'] ); +// Bytes after the last saved boundary were written by an interrupted process. +// Discard them before replaying the corresponding source chunk. +ftruncate( $output, $state['output_bytes'] ); +fseek( $output, $state['output_bytes'] ); +$processor = CSSProcessor::create_for_streaming( $state['css'] ); +$chunks = 0; +while ( ! feof( $input ) ) { + $chunk = fread( $input, 32768 ); + $processor->append_bytes( $chunk, feof( $input ) ); + while ( $processor->next_token() ) { + if ( in_array( $processor->get_token_type(), array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_STRING ), true ) ) { + $processor->set_token_value( str_replace( 'https://old.example/', 'https://old.example/moved/', $processor->get_token_value() ) ); + } + } + $rewritten = $processor->flush_processed_css(); + $written = fwrite( $output, $rewritten ); + if ( strlen( $rewritten ) !== $written ) { + throw new RuntimeException( 'CSS output wrote ' . $written . ' of ' . strlen( $rewritten ) . ' bytes.' ); + } + fflush( $output ); + ++$chunks; + if ( 'before' === $stop && 2 === $chunks ) { + exit( 99 ); + } + $state = array( 'source_bytes' => ftell( $input ), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + file_put_contents( $state_path . '.tmp', json_encode( $state ) ); + rename( $state_path . '.tmp', $state_path ); + if ( 'after' === $stop && 2 === $chunks ) { + exit( 99 ); + } +} +fclose( $input ); +fclose( $output );