diff --git a/components/DataLiberation/CSS/class-cssprocessor.php b/components/DataLiberation/CSS/class-cssprocessor.php index 02a4ea39..21f75ede 100644 --- a/components/DataLiberation/CSS/class-cssprocessor.php +++ b/components/DataLiberation/CSS/class-cssprocessor.php @@ -314,6 +314,12 @@ class CSSProcessor { /** @var bool Whether an unfinished token may receive more source bytes. */ private $expecting_more_input = false; + /** @var bool Whether the last scan needs more bytes before it can return a token. */ + private $paused_at_incomplete_input = false; + + /** @var int Source bytes before the retained buffer, including bytes discarded by earlier processes. */ + private $input_bytes_forgotten = 0; + /** * Constructor for the CSS processor. * @@ -348,48 +354,57 @@ public static function create( string $css, string $encoding = 'UTF-8' ) { } /** - * Opens a processor that keeps unfinished tokens until more input arrives. + * Opens a processor that accepts more CSS through append_bytes(). * - * @param array|null $cursor { - * Optional state from get_reentrancy_cursor(), after flushing processed CSS. - * @type string $pending_b64 Unprocessed source bytes, base64 encoded. - * @type bool $expecting_more_input Whether the source has more bytes. - * } + * To resume, supply source bytes starting at the saved token byte offset, + * together with the cursor. The cursor contains no CSS bytes or pending edits. + * A cursor saved on a token reads that token again, as XMLProcessor does. + * + * @param string $css Initial source bytes; may be empty. + * @param string|null $cursor Opaque state from get_reentrancy_cursor(), or null for a new stream. * @return static */ - public static function create_for_streaming( ?array $cursor = null ) { - if ( null !== $cursor && ( ! isset( $cursor['pending_b64'], $cursor['expecting_more_input'] ) || ! is_string( $cursor['pending_b64'] ) || ! is_bool( $cursor['expecting_more_input'] ) ) ) { - throw new \InvalidArgumentException( 'The CSS cursor must contain a base64 input tail and whether more input is expected.' ); - } - // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode -- Cursor JSON must preserve arbitrary source bytes. - $pending = null === $cursor ? '' : base64_decode( $cursor['pending_b64'], true ); - if ( false === $pending ) { - throw new \InvalidArgumentException( 'The CSS cursor contains an invalid base64 input tail.' ); + public static function create_for_streaming( string $css = '', ?string $cursor = null ) { + $processor = new static( $css ); + $processor->expecting_more_input = true; + if ( null !== $cursor ) { + $state = json_decode( $cursor, true ); + if ( ! is_array( $state ) || ! isset( $state['input_offset'], $state['expecting_more_input'] ) || ! is_int( $state['input_offset'] ) || $state['input_offset'] < 0 || ! is_bool( $state['expecting_more_input'] ) ) { + throw new \InvalidArgumentException( 'The CSS cursor must contain a nonnegative source byte offset and whether more input is expected.' ); + } + $processor->input_bytes_forgotten = $state['input_offset']; + $processor->expecting_more_input = $state['expecting_more_input']; } - $processor = new static( $pending ); - $processor->expecting_more_input = $cursor['expecting_more_input'] ?? true; return $processor; } /** - * Appends source bytes after the caller has flushed completed tokens. + * Adds source bytes and allows a paused scan to continue. * - * The unfinished token stays in memory and is parsed again. Neither its size - * nor the saved cursor size is capped. Use small reads to limit other input. + * Earlier bytes and edits remain available through get_updated_css(). Call + * flush_processed_css() separately to return and release completed output. + * An unfinished token stays in memory and is parsed again; its size is not capped. * * @param string $bytes Next source bytes. - * @param bool $is_last Whether this is the actual end of the stylesheet. */ - public function append_bytes( string $bytes, bool $is_last = false ): void { + public function append_bytes( string $bytes ): void { if ( ! $this->expecting_more_input ) { throw new \LogicException( 'CSS input cannot be appended after the end of the stylesheet.' ); } - if ( 0 !== $this->at ) { - throw new \LogicException( 'Flush processed CSS before appending more input.' ); - } - $this->css .= $bytes; - $this->length = strlen( $this->css ); - $this->expecting_more_input = ! $is_last; + $this->css .= $bytes; + $this->length = strlen( $this->css ); + $this->paused_at_incomplete_input = false; + } + + /** + * Marks the actual end of the source, so the next scan uses CSS's EOF rules. + * + * A download stopping early is not EOF. Unlike XML, CSS can return an unclosed + * string or URL at EOF; input_finished() preserves those existing CSS rules. + */ + public function input_finished(): void { + $this->expecting_more_input = false; + $this->paused_at_incomplete_input = false; } /** Returns whether more source bytes may be appended. */ @@ -397,11 +412,22 @@ public function is_expecting_more_input(): bool { return $this->expecting_more_input; } + /** Returns whether the last scan stopped for more bytes, including token lookahead. */ + public function is_paused_at_incomplete_input(): bool { + return $this->paused_at_incomplete_input; + } + + /** Returns whether input has ended and no token remains to be read or inspected. */ + public function is_finished(): bool { + return ! $this->expecting_more_input && $this->at >= $this->length && null === $this->token_type; + } + /** * Returns edited, completed input and keeps the unfinished token for another read. * - * Flush before appending input or saving a cursor. Existing whole-token setters - * work in streaming mode; their edits are applied only to the returned prefix. + * Existing whole-token setters work in streaming mode; their edits are applied + * only to the returned prefix. Flushing also clears the current token. Write and + * flush this output before saving a file-rewrite cursor and its source offset. * * @return string Processed CSS, including edits made with set_token_value(). */ @@ -409,36 +435,48 @@ public function flush_processed_css(): string { if ( 0 === $this->at ) { return ''; } - $pending = substr( $this->css, $this->at ); - $updated = $this->get_updated_css(); - $output = substr( $updated, 0, strlen( $updated ) - strlen( $pending ) ); - $this->css = $pending; - $this->length = strlen( $pending ); - $this->at = 0; - $this->lexical_updates = array(); + $pending = substr( $this->css, $this->at ); + $updated = $this->get_updated_css(); + $output = substr( $updated, 0, strlen( $updated ) - strlen( $pending ) ); + $this->css = $pending; + $this->length = strlen( $pending ); + $this->input_bytes_forgotten += $this->at; + $this->at = 0; + $this->lexical_updates = array(); $this->after_token(); return $output; } /** - * Returns unfinished input to save beside the caller's source and output offsets. + * Returns opaque parser state for a new processor at the current source position. + * + * Save get_token_byte_offset_in_the_input_stream() separately. Resume reads + * the original source again from that position; the cursor contains neither + * unfinished bytes nor edits. Do not depend on the string's internal format. * - * @return array { - * @type string $pending_b64 Unprocessed source bytes, base64 encoded. - * @type bool $expecting_more_input Whether the source has more bytes. - * } + * @return string State accepted by create_for_streaming(). */ - public function get_reentrancy_cursor(): array { - if ( 0 !== $this->at ) { - throw new \LogicException( 'Flush processed CSS before saving a cursor.' ); - } - return array( - // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode -- Cursor JSON must preserve arbitrary source bytes. - 'pending_b64' => base64_encode( $this->css ), - 'expecting_more_input' => $this->expecting_more_input, + public function get_reentrancy_cursor(): string { + return json_encode( + array( + 'input_offset' => $this->get_token_byte_offset_in_the_input_stream(), + 'expecting_more_input' => $this->expecting_more_input, + ) ); } + /** + * Returns the original source offset from which a saved cursor must resume. + * + * This is the current token's start, or the next unread position when no token + * is exposed. After flushing, it is the first source byte not yet returned. + * + * @return int Byte offset in the source, unaffected by replacement lengths. + */ + public function get_token_byte_offset_in_the_input_stream(): int { + return $this->input_bytes_forgotten + ( $this->token_starts_at ?? $this->at ); + } + /** * Moves to the next token in the CSS stream. * @@ -449,8 +487,12 @@ public function get_reentrancy_cursor(): array { * @return bool Whether a complete token was found; false also means more input is needed. */ public function next_token(): bool { + if ( $this->paused_at_incomplete_input ) { + return false; + } $start = $this->at; if ( ! $this->scan_next_token() ) { + $this->paused_at_incomplete_input = $this->expecting_more_input; return false; } // A CSS escape needs at most six hex digits and CRLF after its backslash. @@ -460,6 +502,7 @@ public function next_token(): bool { if ( $this->expecting_more_input && $this->at > $this->length - 10 ) { $this->at = $start; $this->after_token(); + $this->paused_at_incomplete_input = true; return false; } return true; diff --git a/components/DataLiberation/README.md b/components/DataLiberation/README.md index b28358bc..1f224acf 100644 --- a/components/DataLiberation/README.md +++ b/components/DataLiberation/README.md @@ -309,24 +309,122 @@ not returned. More than 128 nested image sets throw an error. [The file-rewrite caller](Tests/fixtures/css-context/rewrite-file.php) uses the whole-string iterator and writes the output only after iteration completes. -## Stream CSS tokens and resume - -`CSSProcessor::create_for_streaming()` accepts source chunks through -`append_bytes($bytes, $is_last)`. Read complete tokens with the existing -`next_token()`, getters, and `set_token_value()`. If a token is unfinished, -the processor keeps it and tries again after the next read, like XML. -Only mark actual source EOF, not the end of an interrupted response. - -`flush_processed_css()` returns completed input with edits applied and releases -its source bytes. Call it before appending more input or saving a cursor. -`get_reentrancy_cursor()` saves the unfinished input in a JSON-safe array; -pass it to `create_for_streaming()` in the next process. Save the source offset, -output offset, and cursor together after writing and flushing the output. -Resume truncates output to that saved offset before replaying more source. -[The separate-process file caller](Tests/fixtures/css-token-stream/rewrite-file.php) -shows both sides of that checkpoint boundary. +## Stream CSS with the existing scan-and-edit API + +For example, one read can end at `url(https://old.exa` and the next can supply +`mple/photo.png)`. The processor waits for the rest of that URL before returning +it. The caller then reads and edits it with the same methods used for a complete +CSS string. + +Both `CSSProcessor` and `CSSURLProcessor` follow the XML streaming API: + +1. Call `create_for_streaming($css = '', $cursor = null)` to supply initial + bytes or start with an empty buffer. +2. Call `append_bytes($bytes)` when more source bytes arrive. +3. Use `next_token()` or `next_url()` and the existing getters and setters. + A false result can mean the processor needs more input. Check + `is_paused_at_incomplete_input()` to distinguish that from completion. +4. Call `input_finished()` only at the actual source end. Continue scanning + to read the last tokens. A stopped download is not the source end. +5. `is_finished()` becomes true when input has ended and no current or unread + token remains. `is_expecting_more_input()` says whether more bytes can still + be appended. + +The whole-string API is unchanged: `CSSProcessor::create($css)` and +`new CSSURLProcessor($css)` still take a complete stylesheet. Getters, setters, +and `get_updated_css()` work the same way with either input mode. There is no +separate `rewrite_chunk()` API or built-in URL mapping policy. + + +```php + $bytes ) { + $processor->append_bytes( $bytes ); + if ( $index === count( $parts ) - 1 ) { + $processor->input_finished(); + } + while ( $processor->next_url() ) { + if ( 'https://old.example/photo.png' === $processor->get_raw_url() ) { + $processor->set_raw_url( 'https://new.example/photo.png' ); + } + } + echo $processor->flush_processed_css(); +} +echo "\n"; +``` + + +``` +a{src:url("https://new.example/photo.png")} +``` + +The caller chooses which URLs to change. `set_raw_url()` uses the existing +whole-value setter: it quotes an unquoted URL and escapes the replacement for +CSS. For example, replacing `old.png` with `new.png` changes `url(old.png)` to +`url("new.png")`. It does not apply a separate prefix-only rewrite rule. + +### Write and release completed output + +`flush_processed_css()` returns completed CSS with edits applied and removes +those source bytes from memory. An unfinished token remains for the next read. +Flushing clears the current token or URL, so edit it before flushing. + +Appending input does not require a flush. Without flushing, `get_updated_css()` +returns all supplied CSS with edits applied. After a flush, it returns only the +retained CSS. Choose when to flush based on how much output the caller wants to +keep in memory. For example, flushing after each edited URL avoids storing many +large replacements. Flush once more after scanning stops to get completed CSS +after the last URL. There is no automatic output-size threshold or byte slicing. There is no token-size cap. A large comment, string, identifier, or embedded -image increases memory use and the saved cursor size. Each read reparses the -unfinished token. Small chunks therefore do not bound the largest token's -memory use or parsing work. +image can use a lot of memory even with small input chunks. Each read reparses +the unfinished token. Small chunks therefore do not bound the largest token's +memory use or parsing work. The cursor does not copy these bytes. +More than 128 open, nested `image-set()` functions causes an error. + +### Resume in a new process + +Suppose the source is `a{src:url(https://old.example/photo.png)}`. A read ends +inside the URL. After flushing `a{src:`, the saved source offset points at +`url(`, not at the end of that read. A new process reads the unfinished URL +again from `url(`. It does not repeat the flushed prefix. + +`get_reentrancy_cursor()` returns an opaque string. Save it with +`get_token_byte_offset_in_the_input_stream()`. Supply source bytes from that +offset to `create_for_streaming($css, $cursor)`. The cursor contains parsing +state, including the URL position after `@import` or inside `image-set()`, but +no source bytes or edits. Do not inspect or change its internal format. + +A cursor saved while a token or URL is current reads that token again on +resume, as XML does. If `input_finished()` was already called, pass all remaining +source bytes to the factory; appending after the source end is rejected. +For file rewrites, save a checkpoint after flushing completed output instead: + +1. Read a source chunk and append it. Mark the source end when it is reached. +2. Scan and edit using the ordinary token or URL methods. +3. Write the string from `flush_processed_css()` and flush the output file. +4. Save the processor's source byte offset, the output file offset, and the + parser cursor together. Do not use the input file handle's current offset: + it can be past bytes that the processor still needs to read again. + +On resume, seek the source to its saved offset. Remove output bytes after the +saved output offset, then append there. Those extra bytes may have been written +before the previous process stopped, but after its last checkpoint. Removing +them prevents duplicate output when the corresponding source is read again. +Keep the source file and the caller's edit rules unchanged between runs. +The processor does not check either of them. + +If a write fails, discard the processor and resume from the last checkpoint. +The [token file caller](Tests/fixtures/css-token-stream/rewrite-file.php) and +[URL file caller](Tests/fixtures/css-stream/rewrite-file.php) show how to save +and restore both file positions and the parser state. Their tests stop on both +sides of a checkpoint and start a fresh PHP process to finish the output. diff --git a/components/DataLiberation/Tests/CSSPreprocessingTest.php b/components/DataLiberation/Tests/CSSPreprocessingTest.php index bac8565c..4c114fe3 100644 --- a/components/DataLiberation/Tests/CSSPreprocessingTest.php +++ b/components/DataLiberation/Tests/CSSPreprocessingTest.php @@ -24,24 +24,27 @@ public function test_preprocessed_token_keeps_source_bytes( $input, $type, $valu } /** - * Restores JSON state at every byte split, including beside NUL and inside CRLF or an escape. + * Restores at every byte split by rereading the unfinished source, including NUL, CRLF, and escapes. * The expected token is specified by the test, not copied from a whole-string parse. * * @dataProvider preprocessing_cases */ public function test_preprocessing_survives_split_input_and_resume( $input, $type, $value ) { for ( $split = 0; $split <= strlen( $input ); ++$split ) { - $processor = CSSProcessor::create_for_streaming(); + $processor = CSSProcessor::create_for_streaming( substr( $input, 0, $split ) ); $output = ''; $tokens = array(); - foreach ( array( substr( $input, 0, $split ), substr( $input, $split ) ) as $index => $chunk ) { - $processor->append_bytes( $chunk, 1 === $index ); + for ( $index = 0; $index < 2; ++$index ) { + if ( 1 === $index ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { $tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() ); } $output .= $processor->flush_processed_css(); - $cursor = json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ); - $processor = CSSProcessor::create_for_streaming( $cursor ); + if ( 0 === $index ) { + $processor = CSSProcessor::create_for_streaming( substr( $input, $processor->get_token_byte_offset_in_the_input_stream() ), $processor->get_reentrancy_cursor() ); + } } $this->assertSame( array( array( $type, $value, $input ) ), $tokens, 'Split at byte ' . $split ); $this->assertSame( $input, $output, 'Split at byte ' . $split ); diff --git a/components/DataLiberation/Tests/CSSStreamProcessTest.php b/components/DataLiberation/Tests/CSSStreamProcessTest.php index a3bc59ef..cabe3789 100644 --- a/components/DataLiberation/Tests/CSSStreamProcessTest.php +++ b/components/DataLiberation/Tests/CSSStreamProcessTest.php @@ -43,7 +43,8 @@ public function test_token_edits_resume_after_process_death( $stop ) { $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); $this->assertGreaterThan( 0, $state['source_bytes'] ); $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); - $this->assertNotSame( '', $state['css']['pending_b64'] ); + $this->assertIsString( $state['css'] ); + $this->assertLessThan( 512, strlen( $state['css'] ) ); $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); } $this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) ); @@ -55,7 +56,7 @@ public function test_token_edits_resume_after_process_death( $stop ) { /** * Places NUL at the end of the second 32 KiB read, inside an unfinished URL. - * Resume must retain that byte, rewrite the URL, and leave a later bad URL unchanged. + * Resume must reread that byte, rewrite the URL, and leave a later bad URL unchanged. * * @dataProvider interruptions */ @@ -74,10 +75,7 @@ public function test_nul_preprocessing_preserves_file_offsets_after_resume( $sto $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); if ( 'none' !== $stop ) { $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); - $this->assertSame( 'before' === $stop ? 32768 : 65536, $state['source_bytes'] ); - if ( 'after' === $stop ) { - $this->assertStringContainsString( "\x00", base64_decode( $state['css']['pending_b64'] ) ); - } + $this->assertSame( 'before' === $stop ? strlen( $first ) : strlen( $first . $comment . 'a{src:' ), $state['source_bytes'] ); $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); } $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); diff --git a/components/DataLiberation/Tests/CSSStreamTest.php b/components/DataLiberation/Tests/CSSStreamTest.php index ad420415..d4d64b68 100644 --- a/components/DataLiberation/Tests/CSSStreamTest.php +++ b/components/DataLiberation/Tests/CSSStreamTest.php @@ -16,14 +16,18 @@ public function test_source_bytes_survive_one_byte_input_and_resume( $input ) { $whole_tokens[] = array( $whole->get_token_type(), $whole->get_token_value(), $whole->get_unnormalized_token() ); } for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { - $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $processor->append_bytes( substr( $input, $offset, 1 ) ); + if ( strlen( $input ) === $offset ) { + $processor->input_finished(); + } $steps = 0; while ( $processor->next_token() ) { $tokens[] = array( $processor->get_token_type(), $processor->get_token_value(), $processor->get_unnormalized_token() ); $this->assertLessThan( 64, ++$steps, 'The lexer must consume input or finish its current token.' ); } $output .= $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( json_decode( json_encode( $processor->get_reentrancy_cursor() ), true ) ); + $resume_offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSProcessor::create_for_streaming( substr( $input, $resume_offset, min( $offset + 1, strlen( $input ) ) - $resume_offset ), $processor->get_reentrancy_cursor() ); } $this->assertSame( $input, $output ); $this->assertSame( $whole_tokens, $tokens ); @@ -48,12 +52,16 @@ public function test_buffered_strings_match_the_whole_token() { $processor = CSSProcessor::create_for_streaming(); $decoded = ''; for ( $offset = 0; $offset <= strlen( $input ); ++$offset ) { - $processor->append_bytes( substr( $input, $offset, 1 ), strlen( $input ) === $offset ); + $processor->append_bytes( substr( $input, $offset, 1 ) ); + if ( strlen( $input ) === $offset ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { $decoded .= $processor->get_token_value(); } $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + $resume_offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSProcessor::create_for_streaming( substr( $input, $resume_offset, min( $offset + 1, strlen( $input ) ) - $resume_offset ), $processor->get_reentrancy_cursor() ); } $this->assertSame( $expected, $decoded ); } @@ -63,7 +71,9 @@ public function test_buffered_strings_match_the_whole_token() { public function test_buffered_tokens_use_the_existing_value_setter() { $processor = CSSProcessor::create_for_streaming(); $output = ''; + $source = ''; foreach ( array( 'a{src:url("https://old.exa', 'mple/a");color:red} ' ) as $input ) { + $source .= $input; $processor->append_bytes( $input ); while ( $processor->next_token() ) { if ( 'https://old.example/a' === $processor->get_token_value() ) { @@ -71,9 +81,9 @@ public function test_buffered_tokens_use_the_existing_value_setter() { } } $output .= $processor->flush_processed_css(); - $processor = CSSProcessor::create_for_streaming( $processor->get_reentrancy_cursor() ); + $processor = CSSProcessor::create_for_streaming( substr( $source, $processor->get_token_byte_offset_in_the_input_stream() ), $processor->get_reentrancy_cursor() ); } - $processor->append_bytes( '', true ); + $processor->input_finished(); while ( $processor->next_token() ) {} $output .= $processor->flush_processed_css(); $this->assertSame( 'a{src:url("https://new.example/moved/a");color:red} ', $output ); diff --git a/components/DataLiberation/Tests/CSSStreamingApiTest.php b/components/DataLiberation/Tests/CSSStreamingApiTest.php new file mode 100644 index 00000000..58d0c229 --- /dev/null +++ b/components/DataLiberation/Tests/CSSStreamingApiTest.php @@ -0,0 +1,146 @@ +assertTrue( $processor->is_expecting_more_input() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->is_finished() ); + $processor->append_bytes( 'mple/a)' ); + $this->assertFalse( $processor->is_paused_at_incomplete_input() ); + $processor->input_finished(); + $this->assertFalse( $processor->is_expecting_more_input() ); + $this->assertTrue( $processor->$scan() ); + $this->assertSame( 'https://old.example/a', $processor->$get_value() ); + $this->assertTrue( $processor->$set_value( 'https://new.example/a' ) ); + $this->assertSame( 'url("https://new.example/a")', $processor->get_updated_css() ); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_finished() ); + $this->assertFalse( $processor->is_paused_at_incomplete_input() ); + $this->assertFalse( $processor->$scan() ); + } + + /** Appending must not discard earlier edits or require the caller to flush them first. @dataProvider processors */ + public function test_append_keeps_unflushed_edits( $class, $scan, $get_value, $set_value ) { + $processor = $class::create_for_streaming( 'url(first.png) ' ); + $this->assertTrue( $processor->$scan() ); + $this->assertTrue( $processor->$set_value( 'changed.png' ) ); + $processor->append_bytes( 'url(second.png)' ); + $processor->input_finished(); + $values = array(); + while ( $processor->$scan() ) { + if ( null !== $processor->$get_value() ) { + $values[] = $processor->$get_value(); + } + } + $this->assertSame( array( 'second.png' ), $values ); + $this->assertSame( 'url("changed.png") url(second.png)', $processor->get_updated_css() ); + } + + /** Like XML, a cursor at a token replays that token from the original source. @dataProvider processors */ + public function test_resume_replays_the_current_token_without_saving_edits( $class, $scan, $get_value, $set_value ) { + $input = 'url(first.png) url(second.png)'; + $processor = $class::create_for_streaming( $input ); + $processor->input_finished(); + $this->assertTrue( $processor->$scan() ); + $this->assertTrue( $processor->$set_value( 'changed.png' ) ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertSame( 0, $offset ); + $this->assertIsString( $cursor ); + $resumed = $class::create_for_streaming( substr( $input, $offset ), $cursor ); + $this->assertTrue( $resumed->$scan() ); + $this->assertSame( 'first.png', $resumed->$get_value() ); + $this->assertSame( $input, $resumed->get_updated_css() ); + } + + /** Unfinished bytes are reread from the source, rather than copied into the cursor. @dataProvider processors */ + public function test_resume_after_flushing_rereads_the_unfinished_token( $class, $scan, $get_value, $set_value ) { + $prefix = 'url(first.png) '; + $pending = '/*' . str_repeat( 'a', 131072 ); + $processor = $class::create_for_streaming( $prefix . $pending ); + while ( $processor->$scan() ) {} + $this->assertTrue( $processor->is_paused_at_incomplete_input() ); + $this->assertSame( $prefix, $processor->flush_processed_css() ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertSame( strlen( $prefix ), $offset ); + $this->assertIsString( $cursor ); + $this->assertLessThan( 512, strlen( $cursor ) ); + $input = $prefix . $pending . '*/url(second.png)'; + $resumed = $class::create_for_streaming( substr( $input, $offset ), $cursor ); + $resumed->input_finished(); + $values = array(); + while ( $resumed->$scan() ) { + if ( null !== $resumed->$get_value() ) { + $values[] = $resumed->$get_value(); + } + } + $this->assertSame( array( 'second.png' ), $values ); + $this->assertSame( substr( $input, $offset ), $resumed->flush_processed_css() ); + $this->assertSame( strlen( $input ), $resumed->get_token_byte_offset_in_the_input_stream() ); + $this->assertTrue( $resumed->is_finished() ); + } + + /** Marking EOF twice is harmless; supplying more input afterwards is an error. @dataProvider processors */ + public function test_cannot_append_after_input_finished( $class, $scan, $get_value, $set_value ) { + $processor = $class::create_for_streaming(); + $processor->input_finished(); + $processor->input_finished(); + $this->assertFalse( $processor->$scan() ); + $this->assertTrue( $processor->is_finished() ); + $this->expectException( LogicException::class ); + $this->expectExceptionMessage( 'after the end of the stylesheet' ); + $processor->append_bytes( 'url(late.png)' ); + } + + /** Malformed cursor input must not silently start a new parse. @dataProvider processors */ + public function test_rejects_corrupt_cursor( $class, $scan, $get_value, $set_value ) { + $this->expectException( InvalidArgumentException::class ); + $class::create_for_streaming( '', 'not a cursor' ); + } + + /** + * Restoring on a string must keep the URL meaning given by its preceding syntax. + * + * @dataProvider url_contexts + */ + public function test_url_resume_replays_the_current_string( $input ) { + $processor = CSSURLProcessor::create_for_streaming( $input ); + $processor->input_finished(); + $this->assertTrue( $processor->next_url() ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $resumed = CSSURLProcessor::create_for_streaming( substr( $input, $offset ), $processor->get_reentrancy_cursor() ); + $this->assertTrue( $resumed->next_url() ); + $this->assertSame( 'first.png', $resumed->get_raw_url() ); + $this->assertTrue( $resumed->next_url() ); + $this->assertSame( 'second.png', $resumed->get_raw_url() ); + $this->assertFalse( $resumed->next_url() ); + } + + /** These strings are URLs only because of syntax before the saved byte offset. */ + public static function url_contexts() { + return array( + 'import' => array( '@import "first.png"; @import "second.png";' ), + 'quoted url' => array( 'a{src:url("first.png"),url("second.png")}' ), + 'image set' => array( 'a{src:image-set("first.png" type("image/png"),"second.png" 2x)}' ), + ); + } + + /** The APIs differ only in whether the caller scans all tokens or URL values. */ + public static function processors() { + return array( + 'tokens' => array( CSSProcessor::class, 'next_token', 'get_token_value', 'set_token_value' ), + 'URLs' => array( CSSURLProcessor::class, 'next_url', 'get_raw_url', 'set_raw_url' ), + ); + } +} diff --git a/components/DataLiberation/Tests/CSSURLContextProcessTest.php b/components/DataLiberation/Tests/CSSURLContextProcessTest.php index 8b446678..85a4335f 100644 --- a/components/DataLiberation/Tests/CSSURLContextProcessTest.php +++ b/components/DataLiberation/Tests/CSSURLContextProcessTest.php @@ -32,6 +32,20 @@ public function test_file_rewrite_finds_only_url_contexts() { $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); } + /** A malformed import consumes its URL position; a later valid URL must still be found. */ + public function test_file_rewrite_skips_malformed_urls_and_following_text() { + $input = "@import \"https://old.example/bad\n" + . '"https://old.example/text";a{src:url(https://old.example/bad(image),url(https://old.example/good)}' + . '@import/**/"https://old.example/theme.css";'; + $expected = "@import \"https://old.example/bad\n" + . '"https://old.example/text";a{src:url(https://old.example/bad(image),url("https://new.example/good")}' + . '@import/**/"https://new.example/theme.css";'; + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 0, $this->run_worker(), file_get_contents( $this->directory . '/worker.log' ) ); + $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + } + /** NUL-containing URLs can be rewritten; other controls still make a URL invalid. */ public function test_file_rewrite_applies_nul_preprocessing_before_url_validation() { $comment = "/* Keep this NUL: \x00 and these line endings: \r\n\f */"; @@ -68,6 +82,19 @@ public function test_whole_string_finder_recognizes_import_and_image_set_urls() $this->assertSame( array( 'https://old.example/theme.css', 'https://old.example/a', 'https://old.example/b', 'https://old.example/c' ), $urls ); } + /** Reading a URL twice must not consume another token or lose an empty URL. */ + public function test_url_reads_leave_the_iterator_on_the_current_url() { + $css = '@import/**/"theme.css";a{content:"text";src:url(""),url(a.png),image-set("b.png" type("image/png"),"c.png" 2x)}'; + $processor = new CSSURLProcessor( $css ); + foreach ( array( 'theme.css', '', 'a.png', 'b.png', 'c.png' ) as $url ) { + $this->assertTrue( $processor->next_url() ); + $this->assertSame( $url, $processor->get_raw_url() ); + $this->assertSame( $url, $processor->get_raw_url() ); + } + $this->assertFalse( $processor->next_url() ); + $this->assertFalse( $processor->next_url() ); + } + /** Runs a whole-file caller without streamed input or a saved cursor. */ private function run_worker() { $arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-context/rewrite-file.php', $this->directory ); diff --git a/components/DataLiberation/Tests/CSSURLStreamProcessTest.php b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php new file mode 100644 index 00000000..7fa84414 --- /dev/null +++ b/components/DataLiberation/Tests/CSSURLStreamProcessTest.php @@ -0,0 +1,148 @@ +directory = sys_get_temp_dir() . '/css-stream-' . bin2hex( random_bytes( 8 ) ); + mkdir( $this->directory ); + } + + /** @after */ + public function remove_directory() { + foreach ( glob( $this->directory . '/*' ) as $path ) { + unlink( $path ); + } + rmdir( $this->directory ); + } + + /** + * Checks an uninterrupted rewrite and two runs that exit before or after saving state. + * A new process must finish each stopped run with the exact expected file bytes. + * + * @dataProvider interruptions + */ + public function test_file_rewrite_resumes_after_process_death( $stop ) { + // The worker reads 32 KiB at a time. Its first read ends after '\6', + // inside the '\6f ' escape for 'o'. Its second read ends inside a comment. + // Both saved positions require the next process to reread unfinished CSS. + $prefix = '/*' . str_repeat( 'a', 32743 ) . '*/a{src:url("https://\\6f ld.example/a.png")}'; + $input = $prefix . '/*' . str_repeat( 'b', 32768 ) . '*/' + . '@import "https://old.example/theme.css";' + . 'a{src:url(https://old.example/' . str_repeat( 'c', 131072 ) . ')}'; + $this->assertSame( '\\6', substr( $input, 32766, 2 ) ); + $whole = new CSSURLProcessor( $input ); + while ( $whole->next_url() ) { + $whole->set_raw_url( str_replace( 'https://old.example/', 'https://old.example/moved/', $whole->get_raw_url() ) ); + } + $expected = $whole->get_updated_css(); + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); + if ( 'none' !== $stop ) { + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertGreaterThan( 0, $state['source_bytes'] ); + $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + } + $this->assertSame( hash( 'sha256', $expected ), hash_file( 'sha256', $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + } + + /** + * Restores the URL position after @import, even when a comment spans the saved offsets. + * + * @dataProvider interruptions + */ + public function test_file_rewrite_resumes_between_import_keyword_and_url( $stop ) { + // The comment ends at the second 32 KiB read. Both stop modes save a + // position after @import but before the string that supplies its URL. + $prefix = 'a{src:url(https://old.example/first)}@import'; + $comment = '/*' . str_repeat( ' ', 65536 - strlen( $prefix ) - 4 ) . '*/'; + $input = $prefix . $comment . '"https://old.example/theme.css";' + . 'a{content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/last)}'; + $expected = 'a{src:url("https://old.example/moved/first")}@import' . $comment . '"https://old.example/moved/theme.css";' + . 'a{content:"https://old.example/text";src:url(https://old.example/bad(image),url("https://old.example/moved/last")}'; + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 'none' === $stop ? 0 : 99, $this->run_worker( $stop ), file_get_contents( $this->directory . '/worker.log' ) ); + if ( 'none' !== $stop ) { + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $prefix ), $state['source_bytes'] ); + $this->assertLessThan( 512, strlen( $state['css'] ) ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + } + $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + } + + /** + * Saves part of the file, then reaches 129 nested image-set() calls and fails. + * A second process must report the same error and leave the saved state before the file end. + */ + public function test_file_rewrite_reports_a_nesting_limit_and_keeps_the_last_checkpoint() { + $input = '/*' . str_repeat( 'a', 65536 ) . '*/a{src:' . str_repeat( 'image-set(', 129 ) . '"https://old.example/a"' . str_repeat( ')', 129 ) . '}'; + file_put_contents( $this->directory . '/source.css', $input ); + for ( $attempt = 0; $attempt < 2; ++$attempt ) { + $this->assertNotSame( 0, $this->run_worker( 'none' ) ); + $this->assertStringContainsString( 'nesting exceeds 128', file_get_contents( $this->directory . '/worker.log' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertLessThan( strlen( $input ), $state['source_bytes'] ); + $this->assertTrue( CSSURLProcessor::create_for_streaming( '', $state['css'] )->is_expecting_more_input() ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + } + } + + /** A stop after marking EOF must not skip a URL that has not yet been written. */ + public function test_resume_after_process_death_between_eof_and_final_edit() { + $path = str_repeat( 'a', 65536 ); + $input = '@import "https://old.example/' . $path . '";'; + file_put_contents( $this->directory . '/source.css', $input ); + $this->assertSame( 99, $this->run_worker( 'eof' ), file_get_contents( $this->directory . '/worker.log' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( '@import ' ), $state['source_bytes'] ); + $this->assertSame( '@import ', file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( 0, $this->run_worker( 'none' ), file_get_contents( $this->directory . '/worker.log' ) ); + $expected = '@import "https://old.example/moved/' . $path . '";'; + $this->assertSame( $expected, file_get_contents( $this->directory . '/target.css' ) ); + $this->assertSame( $input, file_get_contents( $this->directory . '/source.css' ) ); + $state = json_decode( file_get_contents( $this->directory . '/state.json' ), true ); + $this->assertSame( strlen( $input ), $state['source_bytes'] ); + $this->assertSame( strlen( $expected ), $state['output_bytes'] ); + $this->assertTrue( CSSURLProcessor::create_for_streaming( '', $state['css'] )->is_finished() ); + } + + /** + * Selects normal completion, or exit just before or after the second saved state. + * The worker has written the second chunk's output before either exit point. + */ + public static function interruptions() { + return array( array( 'none' ), array( 'before' ), array( 'after' ) ); + } + + /** + * Starts the file-rewrite script and waits for its exit code. + * Code 0 means completion, 99 means a test stop, and other codes report failures. + * The script loads any saved state from the previous process before reading more source bytes. + */ + private function run_worker( $stop ) { + $arguments = array( PHP_BINARY, __DIR__ . '/fixtures/css-stream/rewrite-file.php', $this->directory . '/source.css', $this->directory . '/target.css', $this->directory . '/state.json', $stop ); + $command = implode( ' ', array_map( 'escapeshellarg', $arguments ) ); + // Bypass cmd.exe on Windows because it strips the quotes around these + // paths. Keep a command string: proc_open() in PHP 7.2 cannot take an array. + $process = proc_open( $command, array( 0 => array( 'pipe', 'r' ), 1 => array( 'file', $this->directory . '/worker.log', 'w' ), 2 => array( 'file', $this->directory . '/worker.log', 'a' ) ), $pipes, null, null, array( 'bypass_shell' => true ) ); + $this->assertIsResource( $process ); + fclose( $pipes[0] ); + return proc_close( $process ); + } +} diff --git a/components/DataLiberation/Tests/CSSURLStreamTest.php b/components/DataLiberation/Tests/CSSURLStreamTest.php new file mode 100644 index 00000000..b84f4924 --- /dev/null +++ b/components/DataLiberation/Tests/CSSURLStreamTest.php @@ -0,0 +1,204 @@ +replace_urls( $whole ); + $expected = $whole->get_updated_css(); + for ( $split = 0; $split <= strlen( $input ); ++$split ) { + $processor = CSSURLProcessor::create_for_streaming( substr( $input, 0, $split ) ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset ), $processor->get_reentrancy_cursor() ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( $expected, $output, 'Split at byte ' . $split ); + $this->assertTrue( $processor->is_finished() ); + } + } + + /** Covers escapes, malformed tokens, URL context, and tokens accepted only at actual EOF. */ + public static function stylesheets() { + return array( + 'trailing URL spaces' => array( 'a{src:url(https://old.example )}' ), + 'leading URL spaces' => array( 'a{src:url( "https://old.example/a")}' ), + 'bad URL keeps its syntax' => array( 'a{src:url(https://old.example/a(broken)}' ), + 'bad string keeps its syntax' => array( "@import \"https://old.example/a\n" ), + 'bad string consumes import position' => array( "@import \"https://old.example/bad\n" . '"https://old.example/text";@import/**/"https://old.example/theme.css";' ), + 'URL position does not carry into text or a bad URL' => array( 'a{src:url(https://old.example/a);content:"https://old.example/text";src:url(https://old.example/bad(image),url(https://old.example/b)}' ), + 'not a url function' => array( 'a{src:10url("https://old.example/a"),noturl("https://old.example/a")}' ), + 'nested image set' => array( 'a{src:image-set(image-set("https://old.example/a" 1x) 1x,"https://old.example/b" type("https://old.example/mime"))}' ), + 'parenthesized resolution' => array( 'a{src:image-set("https://old.example/a" calc(1x * (2 + 3)), "https://old.example/b" 2x)}' ), + 'long non-url names' => array( str_repeat( 'x', 70 ) . 'url("https://old.example/a")' ), + 'long numeric dimension' => array( str_repeat( '1', 70 ) . '.23e+45url("https://old.example/a")' ), + 'quoted' => array( 'a{background:url("https://old.example/a.png")}' ), + 'unquoted' => array( 'a{background:url(https://old.example/a.png)}' ), + 'hex host and function' => array( 'a{background:\\75rl(https://\\6f ld.example/a.png)}' ), + 'slash escapes' => array( 'a{background:url(https:\\/\\/old.example\\/a.png)}' ), + 'protocol relative' => array( 'a{src:url(//old.example/font.woff2)}' ), + 'import string' => array( '@import "https://old.example/a.css" screen;' ), + 'image set' => array( 'a{background:image-set("https://old.example/a.png" 1x, url(https://old.example/b.png) 2x)}' ), + 'comment and displayed text' => array( '/* url(https://old.example/a) */ a{content:"url(https://old.example/b)"}' ), + 'unrelated and relative' => array( 'a{src:url(../a),url(data:image/png;base64,AAAA),url(https://old.example:8080/a),url(https://old.example.org/a)}' ), + 'EOF string' => array( 'a{src:url("https://old.example/a' ), + 'EOF URL' => array( 'a{src:url(https://old.example/a' ), + 'escaped line continuation' => array( "a{src:url(\"https://old.exa\\\r\nmple/a\")}" ), + ); + } + + /** Restores after every byte, including bytes inside CSS escapes and image-set() strings. */ + public function test_one_byte_chunks_match_whole_string_output() { + foreach ( self::stylesheets() as $case ) { + $input = $case[0]; + $whole = new CSSURLProcessor( $input ); + $this->replace_urls( $whole ); + $processor = CSSURLProcessor::create_for_streaming(); + $output = ''; + for ( $at = 0; $at < strlen( $input ); ++$at ) { + $processor->append_bytes( $input[ $at ] ); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset, $at + 1 - $offset ), $processor->get_reentrancy_cursor() ); + } + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( $whole->get_updated_css(), $output ); + } + } + + /** + * Sends one comment, string, URL, or name across 128 reads of 32 KiB each. + * The input buffer must grow until that item ends, but the cursor must not copy it. + */ + public function test_large_tokens_are_retained_until_complete_then_released() { + foreach ( array( array( '/*', '*/' ), array( 'a{content:"', '"}' ), array( 'a{src:url(data:image/png;base64,', ')}' ), array( 'a{src:url(https://old.example/', ')}' ), array( '.long', '{}' ) ) as $token ) { + $input = $token[0] . str_repeat( 'a', 128 * 32768 ) . $token[1]; + $whole = new CSSURLProcessor( $input ); + $this->replace_urls( $whole ); + $expected = hash( 'sha256', $whole->get_updated_css() ); + $processor = CSSURLProcessor::create_for_streaming( $token[0] ); + $this->replace_urls( $processor ); + $output_hash = hash_init( 'sha256' ); + hash_update( $output_hash, $processor->flush_processed_css() ); + $retained_bytes = 0; + for ( $chunk = 0; $chunk < 128; ++$chunk ) { + $processor->append_bytes( str_repeat( 'a', 32768 ) ); + $this->replace_urls( $processor ); + hash_update( $output_hash, $processor->flush_processed_css() ); + $this->assertGreaterThan( $retained_bytes, strlen( $processor->get_updated_css() ) ); + $retained_bytes = strlen( $processor->get_updated_css() ); + $cursor = $processor->get_reentrancy_cursor(); + $this->assertLessThan( 512, strlen( $cursor ) ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset, strlen( $token[0] ) + ( $chunk + 1 ) * 32768 - $offset ), $cursor ); + } + $processor->append_bytes( $token[1] ); + $processor->input_finished(); + $this->replace_urls( $processor ); + hash_update( $output_hash, $processor->flush_processed_css() ); + $this->assertSame( $expected, hash_final( $output_hash ) ); + $this->assertSame( '', $processor->get_updated_css() ); + $this->assertSame( strlen( $input ), $processor->get_token_byte_offset_in_the_input_stream() ); + } + } + + /** + * Expands 1,500 short URLs into over 5 MiB of output. + * The caller flushes each edit instead of retaining all replacements until EOF. + */ + public function test_caller_can_flush_each_edit_to_bound_expanded_output() { + $target = 'https://new.example/' . str_repeat( 'a', 4096 ); + $input = str_repeat( 'a{src:url(https://old.example/a)}', 1500 ); + $processor = CSSURLProcessor::create_for_streaming( $input ); + $processor->input_finished(); + $expected = hash_init( 'sha256' ); + for ( $index = 0; $index < 1500; ++$index ) { + hash_update( $expected, 'a{src:url("' . $target . '/a")}' ); + } + $actual = hash_init( 'sha256' ); + $bytes = 0; + $memory = memory_get_usage(); + while ( $processor->next_url() ) { + $processor->set_raw_url( $target . '/a' ); + $output = $processor->flush_processed_css(); + $this->assertLessThan( 2 * 1024 * 1024, memory_get_usage() - $memory ); + $bytes += strlen( $output ); + hash_update( $actual, $output ); + } + $output = $processor->flush_processed_css(); + hash_update( $actual, $output ); + $bytes += strlen( $output ); + $this->assertGreaterThan( 5 * 1024 * 1024, $bytes ); + $this->assertSame( hash_final( $expected ), hash_final( $actual ) ); + } + + /** Flushing a large URL returns it whole, with no fixed-size output slices. */ + public function test_large_url_is_flushed_without_splitting() { + $path = str_repeat( 'a', 131072 ); + $processor = CSSURLProcessor::create_for_streaming( 'url(https://old.example/' . $path . ')' ); + $processor->input_finished(); + $this->assertTrue( $processor->next_url() ); + $processor->set_raw_url( 'https://new.example/' . $path ); + $this->assertSame( 'url("https://new.example/' . $path . '")', $processor->flush_processed_css() ); + $this->assertSame( '', $processor->flush_processed_css() ); + } + + /** + * Inserts over 1 MiB of backslash-newline pairs between the 'h' and 'ttps://' of a short URL. + * Those pairs decode to no characters, but must be retained until the quoted URL ends. + */ + public function test_long_escaped_url_is_rewritten_after_its_closing_quote() { + $processor = CSSURLProcessor::create_for_streaming( 'a{src:url("h' ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); + for ( $index = 0; $index < 34; ++$index ) { + $processor->append_bytes( str_repeat( "\\\n", 16384 ) ); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + } + $processor->append_bytes( 'ttps://old.example/a")}' ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( 'a{src:url("https://new.example/a")}', $output ); + } + + /** An unfinished URL cannot be written yet; resume must read it again from its start. */ + public function test_unfinished_url_is_not_written_before_its_end() { + $input = 'a{src:url(https://old.example/' . str_repeat( 'a', 65536 ); + $processor = CSSURLProcessor::create_for_streaming( $input ); + $this->replace_urls( $processor ); + $output = $processor->flush_processed_css(); + $this->assertSame( 'a{src:', $output ); + $offset = $processor->get_token_byte_offset_in_the_input_stream(); + $processor = CSSURLProcessor::create_for_streaming( substr( $input, $offset ) . ')}', $processor->get_reentrancy_cursor() ); + $processor->input_finished(); + $this->replace_urls( $processor ); + $output .= $processor->flush_processed_css(); + $this->assertSame( 'a{src:url("https://new.example/' . str_repeat( 'a', 65536 ) . '")}', $output ); + } + + /** Uses the same caller-selected replacement for whole-string and streamed input. */ + private function replace_urls( CSSURLProcessor $processor ): void { + while ( $processor->next_url() ) { + $url = $processor->get_raw_url(); + $replacement = str_replace( 'https://old.example/', 'https://new.example/', $url ); + if ( $url !== $replacement ) { + $processor->set_raw_url( $replacement ); + } + } + } +} diff --git a/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php new file mode 100644 index 00000000..0ecde448 --- /dev/null +++ b/components/DataLiberation/Tests/fixtures/css-stream/rewrite-file.php @@ -0,0 +1,71 @@ + 0, 'output_bytes' => 0, 'css' => null ); +$input = fopen( $input_path, 'rb' ); +$output = fopen( $output_path, 'c+b' ); +fseek( $input, $state['source_bytes'] ); +// A process can stop after writing output but before saving the new offsets. +// The next run reads that source again. Remove the extra output bytes first +// so that the repeated read does not append a second copy of the same CSS. +ftruncate( $output, $state['output_bytes'] ); +fseek( $output, $state['output_bytes'] ); +// The target still starts with the source base. Rewriting a URL twice would +// add '/moved' twice, which makes repeated replacements visible in the output. +$processor = CSSURLProcessor::create_for_streaming( '', $state['css'] ); +$chunks = 0; +while ( ! $processor->is_finished() ) { + $chunk = fread( $input, 32768 ); + $processor->append_bytes( $chunk ); + if ( feof( $input ) ) { + $processor->input_finished(); + if ( 'eof' === $stop ) { + exit( 99 ); + } + } + while ( $processor->next_url() ) { + $processor->set_raw_url( str_replace( 'https://old.example/', 'https://old.example/moved/', $processor->get_raw_url() ) ); + } + $rewritten = $processor->flush_processed_css(); + $written = fwrite( $output, $rewritten ); + if ( strlen( $rewritten ) !== $written ) { + throw new RuntimeException( 'CSS output wrote ' . $written . ' of ' . strlen( $rewritten ) . ' bytes.' ); + } + // Save offsets only after all output for this source chunk has been written. + // A stopped process must not leave saved state ahead of the output file. + fflush( $output ); + ++$chunks; + if ( 'before' === $stop && 2 === $chunks ) { + exit( 99 ); + } + // The cursor contains no CSS bytes. Resume must reread unfinished input + // from the processor's source offset, which can be earlier than ftell($input). + $state = array( 'source_bytes' => $processor->get_token_byte_offset_in_the_input_stream(), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + // Replace the complete state file in one rename. A stop during the temporary + // write leaves the previous saved offsets and parser state together. + file_put_contents( $state_path . '.tmp', json_encode( $state ) ); + rename( $state_path . '.tmp', $state_path ); + if ( 'after' === $stop && 2 === $chunks ) { + exit( 99 ); + } +} +fclose( $input ); +fclose( $output ); diff --git a/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php index 124abdb0..30a44177 100644 --- a/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php +++ b/components/DataLiberation/Tests/fixtures/css-token-stream/rewrite-file.php @@ -16,11 +16,14 @@ // Discard them before replaying the corresponding source chunk. ftruncate( $output, $state['output_bytes'] ); fseek( $output, $state['output_bytes'] ); -$processor = CSSProcessor::create_for_streaming( $state['css'] ); +$processor = CSSProcessor::create_for_streaming( '', $state['css'] ); $chunks = 0; -while ( ! feof( $input ) ) { +while ( ! $processor->is_finished() ) { $chunk = fread( $input, 32768 ); - $processor->append_bytes( $chunk, feof( $input ) ); + $processor->append_bytes( $chunk ); + if ( feof( $input ) ) { + $processor->input_finished(); + } while ( $processor->next_token() ) { if ( in_array( $processor->get_token_type(), array( CSSProcessor::TOKEN_URL, CSSProcessor::TOKEN_STRING ), true ) ) { $processor->set_token_value( str_replace( 'https://old.example/', 'https://old.example/moved/', $processor->get_token_value() ) ); @@ -36,7 +39,9 @@ if ( 'before' === $stop && 2 === $chunks ) { exit( 99 ); } - $state = array( 'source_bytes' => ftell( $input ), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); + // The source offset follows completed tokens, not fread(): unfinished bytes + // must be read again from the file when a new process restores this cursor. + $state = array( 'source_bytes' => $processor->get_token_byte_offset_in_the_input_stream(), 'output_bytes' => ftell( $output ), 'css' => $processor->get_reentrancy_cursor() ); file_put_contents( $state_path . '.tmp', json_encode( $state ) ); rename( $state_path . '.tmp', $state_path ); if ( 'after' === $stop && 2 === $chunks ) { diff --git a/components/DataLiberation/URL/class-cssurlprocessor.php b/components/DataLiberation/URL/class-cssurlprocessor.php index a5e99972..9e4ef45d 100644 --- a/components/DataLiberation/URL/class-cssurlprocessor.php +++ b/components/DataLiberation/URL/class-cssurlprocessor.php @@ -5,7 +5,12 @@ use WordPress\DataLiberation\CSS\CSSProcessor; /** - * Provides URL specific helpers on top of the CSSProcessor tokenizer. + * Finds and edits CSS URLs in a complete string or as source bytes arrive. + * + * CSSProcessor reads one CSS item, called a token, at a time. A token can be + * a quoted string, a comment, or an unquoted url(...). The surrounding syntax + * tells whether a string holds a URL: @import "theme.css" does, but + * content: "theme.css" does not. */ class CSSURLProcessor { /** @@ -13,13 +18,51 @@ class CSSURLProcessor { */ private $processor; - /** @var array URL syntax context used by the whole-string iterator. */ + /** + * Whether the current token holds a URL, rather than a comment or displayed text. + * + * The string in @import "theme.css" holds a URL; the same string after + * content: does not. Malformed string and URL tokens give false. + * This checks the CSS token and its position, not full URL validity. + * + * The next_token() method records this before updating $context for the + * following token. It is not saved in the cursor: resume reads a new token + * and classifies it using the saved context. + * + * @var bool True also for an empty URL. + */ + private $current_token_is_url = false; + + /** + * Remembers where a quoted string can be a URL as tokens are read. + * + * The next_url() method advances through next_token(), which updates this + * state. Streaming callers save it in the cursor so a new process can + * continue inside an image-set(). + * + * @var array { + * @type int $depth Number of function or '(' tokens not yet closed by ')'. + * @type int[] $images Depth of each open image-set(), outermost first. + * @type string $expect Why the next string can be a URL: 'url', 'import', + * or 'image'. Empty when no URL string is expected. + * } + */ private $context = array( 'depth' => 0, 'images' => array(), 'expect' => '', ); + /** + * Context before the current URL was read, for replaying that URL on resume. + * + * Reading the string in @import "theme.css" clears the next-string expectation. + * A cursor saved on that string needs the earlier expectation to find it again. + * + * @var array|null Same keys as $context; null before the first URL. + */ + private $context_before_url; + /** * @param string $css CSS source without wrapping braces. */ @@ -28,61 +71,203 @@ public function __construct( string $css ) { } /** - * Moves the cursor to the next URL token, if available. + * Opens a URL iterator that accepts CSS through append_bytes(). * - * Recognizes URL values without treating comments or displayed text as URLs. - * The url() function expects a STRING token after whitespace. Bare @import and - * image-set strings use the same lookahead; another token clears that expectation. - * Direct unquoted URL tokens already include their url() wrapper. + * Use the same next_url(), get_raw_url(), and set_raw_url() calls as for a + * complete string. The caller chooses replacements; this processor does not + * accept URL mappings. Call input_finished() at the actual source EOF. * - * @return bool + * To resume, supply source bytes starting at the saved token byte offset + * and the cursor. A cursor saved on a URL reads that URL again. + * + * @param string $css Initial source bytes; may be empty. + * @param string|null $cursor Opaque state from get_reentrancy_cursor(), or null for a new stream. + * @return static + */ + public static function create_for_streaming( string $css = '', ?string $cursor = null ) { + $state = null; + if ( null !== $cursor ) { + $state = json_decode( $cursor, true ); + if ( ! is_array( $state ) || ! isset( $state['css'], $state['context'] ) || ! is_string( $state['css'] ) || ! is_array( $state['context'] ) ) { + throw new \InvalidArgumentException( 'The CSS URL cursor must contain CSS parser state and URL context.' ); + } + } + $processor = new static( '' ); + $processor->processor = CSSProcessor::create_for_streaming( $css, $state['css'] ?? null ); + if ( null !== $state ) { + $processor->context = $state['context']; + } + return $processor; + } + + /** + * Adds source bytes without discarding earlier bytes or edits. + * + * Continue with next_url(). Use flush_processed_css() separately when the + * caller is ready to write and release completed output. + * + * @param string $bytes Next source bytes. + */ + public function append_bytes( string $bytes ): void { + $this->processor->append_bytes( $bytes ); + } + + /** Marks actual source EOF; a stopped download must not call this method. */ + public function input_finished(): void { + $this->processor->input_finished(); + } + + /** Returns whether the caller may still supply source bytes. */ + public function is_expecting_more_input(): bool { + return $this->processor->is_expecting_more_input(); + } + + /** Returns whether next_url() stopped because parsing needs more source bytes. */ + public function is_paused_at_incomplete_input(): bool { + return $this->processor->is_paused_at_incomplete_input(); + } + + /** Returns whether input has ended and no token remains to be read or inspected. */ + public function is_finished(): bool { + return $this->processor->is_finished(); + } + + /** + * Returns completed CSS with edits applied and releases those source bytes. + * + * An unfinished token stays in memory until more input arrives or the caller + * marks EOF. This clears the current URL, but keeps the surrounding syntax + * so next_url() can continue inside an image-set(). No output-size limit is + * imposed; callers can flush after each URL to avoid accumulating many edits. + * + * For files, write and flush this output before saving the parser cursor, + * source byte offset, and output byte offset together. If a write fails, + * discard this processor. Resume from the last saved cursor and source + * offset, first removing output bytes after the saved output offset so the + * repeated input does not duplicate them. Keep source and edit rules unchanged. + * + * @return string Completed output, possibly empty; unfinished tokens have no size cap. + */ + public function flush_processed_css(): string { + $output = $this->processor->flush_processed_css(); + $this->current_token_is_url = false; + return $output; + } + + /** + * Returns opaque state for a new processor at the current source position. + * + * Save get_token_byte_offset_in_the_input_stream() beside this string and + * supply source bytes from that offset when resuming. The cursor contains + * parsing context, but no CSS bytes, pending edits, or replacement rules. + * Do not depend on its internal format. + * + * @return string State accepted by create_for_streaming(). + */ + public function get_reentrancy_cursor(): string { + return json_encode( + array( + 'css' => $this->processor->get_reentrancy_cursor(), + 'context' => $this->current_token_is_url ? $this->context_before_url : $this->context, + ) + ); + } + + /** + * Returns the source offset from which a saved cursor must resume. + * + * This is the current URL's token start, or the next unread position when + * next_url() has paused or finished. After flushing, it is the first source + * byte not yet returned, even when replacements changed the output length. + * + * @return int Byte offset in the original source. + */ + public function get_token_byte_offset_in_the_input_stream(): int { + return $this->processor->get_token_byte_offset_in_the_input_stream(); + } + + /** + * Finds the next URL in the supplied CSS. + * + * Recognizes url(), @import strings, and image-set() image strings. Skips + * comments, displayed text, and malformed string or URL tokens. + * + * @return bool True when get_raw_url() can read the next URL; false at EOF or when more bytes are needed. */ public function next_url(): bool { - while ( $this->processor->next_token() ) { - $type = $this->processor->get_token_type(); - $name = in_array( $type, array( CSSProcessor::TOKEN_FUNCTION, CSSProcessor::TOKEN_AT_KEYWORD ), true ) ? $this->processor->get_token_value() : ''; - if ( in_array( $type, array( CSSProcessor::TOKEN_WHITESPACE, CSSProcessor::TOKEN_COMMENT ), true ) ) { - continue; + while ( $this->next_token() ) { + if ( $this->current_token_is_url ) { + return true; } - $expected = $this->context['expect']; - $this->context['expect'] = ''; - if ( CSSProcessor::TOKEN_FUNCTION === $type ) { - ++$this->context['depth']; - $name = strtolower( $name ); - $this->context['expect'] = 'url' === $name ? 'url' : ''; - if ( in_array( $name, array( 'image-set', '-webkit-image-set' ), true ) ) { - // In image-set("a.png" type("image/png"), "b.png" 2x), - // type() is nested one level deeper than image-set(). Save each - // open image-set's depth so only commas at that depth start images. - // The matching ')' removes that entry. Cap the stack at 128 open - // image-set() calls, even for malformed input. This does not cap - // images within a set or separate image-set() calls in the file. - if ( count( $this->context['images'] ) >= 128 ) { - throw new \RuntimeException( 'CSS image-set nesting exceeds 128 open functions.' ); - } - $this->context['images'][] = $this->context['depth']; - $this->context['expect'] = 'image'; - } - } elseif ( CSSProcessor::TOKEN_AT_KEYWORD === $type ) { - $this->context['expect'] = 'import' === strtolower( $name ) ? 'import' : ''; - } elseif ( CSSProcessor::TOKEN_LEFT_PAREN === $type ) { - ++$this->context['depth']; - } elseif ( CSSProcessor::TOKEN_RIGHT_PAREN === $type ) { - if ( end( $this->context['images'] ) === $this->context['depth'] ) { - array_pop( $this->context['images'] ); + } + return false; + } + + /** + * Moves to the next CSS token and records whether it holds a URL. + * + * In @import "theme.css", the @import token makes the next string a URL. + * url("photo.png") and image-set("photo.png" 1x) use the same rule. Spaces + * and comments do not clear that expectation; any other token consumes it. + * An unquoted url(photo.png) is already one URL token, including its wrapper. + * + * Both whole-string and streaming callers advance here so they cannot skip + * the state changes needed to interpret later strings. A successful read can + * produce a comment or a malformed token. $current_token_is_url tells + * whether the current token holds a URL that the caller can read or replace. + * + * @return bool True when there is a current token; false at the end of input or when more bytes are needed. + */ + private function next_token(): bool { + $this->current_token_is_url = false; + if ( ! $this->processor->next_token() ) { + return false; + } + $type = $this->processor->get_token_type(); + if ( in_array( $type, array( CSSProcessor::TOKEN_WHITESPACE, CSSProcessor::TOKEN_COMMENT ), true ) ) { + return true; + } + // The expectation belongs to this token. For @import "theme.css", save + // that the string is a URL before clearing the expectation for the next token. + $this->current_token_is_url = CSSProcessor::TOKEN_URL === $type || + ( '' !== $this->context['expect'] && CSSProcessor::TOKEN_STRING === $type ); + if ( $this->current_token_is_url ) { + $this->context_before_url = $this->context; + } + $name = in_array( $type, array( CSSProcessor::TOKEN_FUNCTION, CSSProcessor::TOKEN_AT_KEYWORD ), true ) ? $this->processor->get_token_value() : ''; + $this->context['expect'] = ''; + if ( CSSProcessor::TOKEN_FUNCTION === $type ) { + ++$this->context['depth']; + $name = strtolower( $name ); + $this->context['expect'] = 'url' === $name ? 'url' : ''; + if ( in_array( $name, array( 'image-set', '-webkit-image-set' ), true ) ) { + // In image-set("a.png" type("image/png"), "b.png" 2x), the comma + // starts another image. A comma inside type(...) must not do so: + // type() is one level deeper than the image-set() around it. + // Save the depth of each open image-set() to tell them apart; + // its closing ')' removes that depth from the list. + // Keep at most 128 open image-set() calls, even in malformed CSS, + // so repeated openings cannot grow the list without a limit. + // This limits nesting, not images per set or separate sets in a file. + if ( count( $this->context['images'] ) >= 128 ) { + throw new \RuntimeException( 'CSS image-set nesting exceeds 128 open functions.' ); } - $this->context['depth'] = max( 0, $this->context['depth'] - 1 ); - } elseif ( CSSProcessor::TOKEN_COMMA === $type && end( $this->context['images'] ) === $this->context['depth'] ) { - $this->context['expect'] = 'image'; + $this->context['images'][] = $this->context['depth']; + $this->context['expect'] = 'image'; } - if ( - CSSProcessor::TOKEN_URL === $type || - ( '' !== $expected && CSSProcessor::TOKEN_STRING === $type ) - ) { - return true; + } elseif ( CSSProcessor::TOKEN_AT_KEYWORD === $type ) { + $this->context['expect'] = 'import' === strtolower( $name ) ? 'import' : ''; + } elseif ( CSSProcessor::TOKEN_LEFT_PAREN === $type ) { + ++$this->context['depth']; + } elseif ( CSSProcessor::TOKEN_RIGHT_PAREN === $type ) { + if ( end( $this->context['images'] ) === $this->context['depth'] ) { + array_pop( $this->context['images'] ); } + $this->context['depth'] = max( 0, $this->context['depth'] - 1 ); + } elseif ( CSSProcessor::TOKEN_COMMA === $type && end( $this->context['images'] ) === $this->context['depth'] ) { + $this->context['expect'] = 'image'; } - return false; + return true; } /**