From 00f7932f997718e57a00a176f3c6181d746c7bb3 Mon Sep 17 00:00:00 2001 From: whowes Date: Sat, 12 Sep 2026 15:44:12 +0000 Subject: [PATCH] feat(gax): add rewindable stream buffer for chunk recovery Introduce RewindableStreamBuffer managing a single-chunk buffer over an InputStream, supporting forward compaction and topping up upon recovery realignment without mark()/reset(). Enforces boundaries by throwing FailedPreconditionException when a server offset is below the base offset or beyond the current buffer window. Use payloadLength in ChunkUploadRequest to avoid allocating temporary byte arrays for full-sized chunks while reusing a single backing array. --- .../ResumableUploadChunkCallable.java | 10 +- .../api/gax/resumable/ChunkUploadRequest.java | 31 +- .../rpc/ResumableUploadChunkCoordinator.java | 45 ++- .../api/gax/rpc/RewindableStreamBuffer.java | 157 ++++++++++ .../com/google/api/gax/rpc/UploadErrors.java | 70 +++++ .../gax/resumable/ChunkUploadRequestTest.java | 51 ++++ .../rpc/ResumableUploadCallableImplTest.java | 52 +++- .../gax/rpc/RewindableStreamBufferTest.java | 278 ++++++++++++++++++ 8 files changed, 661 insertions(+), 33 deletions(-) create mode 100644 sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/RewindableStreamBuffer.java create mode 100644 sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UploadErrors.java create mode 100644 sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RewindableStreamBufferTest.java diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ResumableUploadChunkCallable.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ResumableUploadChunkCallable.java index 70a9e27f20e7..04145943e13e 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ResumableUploadChunkCallable.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ResumableUploadChunkCallable.java @@ -45,6 +45,7 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -82,7 +83,12 @@ public Map> getQueryParamNames(ChunkUploadRequest request) @Override public byte[] getBinaryRequestBody(ChunkUploadRequest request) { - return request.getPayload(); + int length = request.getPayloadLength(); + byte[] payload = request.getPayload(); + if (length == payload.length) { + return payload; + } + return Arrays.copyOf(payload, length); } @Override @@ -111,7 +117,7 @@ private ResumableUploadChunkCallable( public ApiFuture> futureCall( ChunkUploadRequest request, @Nullable ApiCallContext inputContext) { Preconditions.checkNotNull(request); - boolean isPayloadEmpty = request.getPayload().length == 0; + boolean isPayloadEmpty = request.getPayloadLength() == 0; String command; if (request.isFinal()) { command = !isPayloadEmpty ? COMMAND_UPLOAD_FINALIZE : COMMAND_FINALIZE; diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadRequest.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadRequest.java index 73b08d578208..15baf9a73396 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadRequest.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadRequest.java @@ -32,6 +32,7 @@ import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; import org.jspecify.annotations.NullMarked; /** Request value object for uploading a chunk to an active resumable upload session. */ @@ -48,6 +49,9 @@ public abstract class ChunkUploadRequest { @SuppressWarnings("mutable") public abstract byte[] getPayload(); + /** The number of bytes within {@link #getPayload()} to upload. */ + public abstract int getPayloadLength(); + /** The byte offset of this chunk in the overall stream. */ public abstract long getOffset(); @@ -56,8 +60,12 @@ public abstract class ChunkUploadRequest { public abstract Builder toBuilder(); + private static final int UNSET_PAYLOAD_LENGTH = Integer.MIN_VALUE; + public static Builder newBuilder() { - return new AutoValue_ChunkUploadRequest.Builder().setFinal(false); + return new AutoValue_ChunkUploadRequest.Builder() + .setFinal(false) + .setPayloadLength(UNSET_PAYLOAD_LENGTH); } @AutoValue.Builder @@ -66,10 +74,29 @@ public abstract static class Builder { public abstract Builder setPayload(byte[] payload); + public abstract Builder setPayloadLength(int payloadLength); + public abstract Builder setOffset(long offset); public abstract Builder setFinal(boolean isFinal); - public abstract ChunkUploadRequest build(); + abstract byte[] getPayload(); + + abstract int getPayloadLength(); + + abstract ChunkUploadRequest autoBuild(); + + public ChunkUploadRequest build() { + if (getPayloadLength() == UNSET_PAYLOAD_LENGTH) { + setPayloadLength(getPayload().length); + } + ChunkUploadRequest request = autoBuild(); + Preconditions.checkArgument( + request.getPayloadLength() >= 0, "payloadLength must be non-negative"); + Preconditions.checkArgument( + request.getPayloadLength() <= request.getPayload().length, + "payloadLength exceeds payload array length"); + return request; + } } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java index 64b167d99538..f34f6827bd08 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java @@ -44,13 +44,11 @@ import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.retrying.ScheduledRetryingExecutor; -import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.MoreExecutors; import com.google.errorprone.annotations.concurrent.GuardedBy; import java.io.IOException; import java.io.InputStream; import java.time.Duration; -import java.util.Arrays; import java.util.concurrent.CancellationException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -77,8 +75,6 @@ final class ResumableUploadChunkCoordinator { .setMaxAttempts(5) .build(); - private static final byte[] EMPTY_PAYLOAD = new byte[0]; - private final Object lock = new Object(); private final AtomicBoolean dispatching = new AtomicBoolean(false); private final AtomicLong nextChunkOffset = new AtomicLong(-1L); @@ -88,13 +84,13 @@ final class ResumableUploadChunkCoordinator { private final RetryingCallable> retryingChunkCallable; private final InputStream payload; - private final byte[] buffer; private final int chunkSize; private final ApiCallContext callContext; private final ClientContext clientContext; private final RetrySettings chunkRetrySettings; private volatile @Nullable String uploadSessionUrl; + private volatile @Nullable RewindableStreamBuffer buffer; @GuardedBy("lock") private boolean done; @@ -123,7 +119,6 @@ final class ResumableUploadChunkCoordinator { this.callContext = checkNotNull(callContext, "callContext must not be null"); this.clientContext = checkNotNull(clientContext, "clientContext must not be null"); this.chunkRetrySettings = DEFAULT_CHUNK_RETRY_SETTINGS; - this.buffer = new byte[chunkSize]; RetryAlgorithm> retryAlgorithm = new RetryAlgorithm<>( @@ -152,6 +147,7 @@ public void onSuccess(ResumableUploadSession session) { } } uploadSessionUrl = session.getUploadUrl(); + buffer = new RewindableStreamBuffer(payload, chunkSize, uploadSessionUrl); scheduleNextChunk(0L); } @@ -258,43 +254,40 @@ private void transmitSingleChunk(long currentOffset) { } } - int bytesRead; - try { - bytesRead = ByteStreams.read(payload, buffer, 0, chunkSize); - } catch (IOException e) { - finish(null, e); + String url = uploadSessionUrl; + if (url == null) { + finish(null, new IllegalStateException("Upload session URL not available")); return; } - boolean isFinal = bytesRead < chunkSize; - byte[] chunkPayload; - if (bytesRead == chunkSize) { - chunkPayload = buffer; - } else if (bytesRead == 0) { - chunkPayload = EMPTY_PAYLOAD; - } else { - chunkPayload = Arrays.copyOf(buffer, bytesRead); + RewindableStreamBuffer streamBuffer = buffer; + if (streamBuffer == null) { + finish(null, new IllegalStateException("Upload buffer not initialized")); + return; } - String url = uploadSessionUrl; - if (url == null) { - finish(null, new IllegalStateException("Upload session URL not available")); + try { + streamBuffer.fill(currentOffset); + } catch (IOException e) { + finish(null, e); return; } ChunkUploadRequest chunkRequest = ChunkUploadRequest.newBuilder() .setUploadUrl(url) - .setPayload(chunkPayload) - .setOffset(currentOffset) - .setFinal(isFinal) + .setPayload(streamBuffer.getBuffer()) + .setPayloadLength(streamBuffer.getPayloadLength()) + .setOffset(streamBuffer.getBufferBaseOffset()) + .setFinal(streamBuffer.isFinal()) .build(); RetryingFuture> retryingFuture = retryingChunkCallable.futureCall(chunkRequest, callContext); setInFlightFuture(retryingFuture); - long chunkLength = chunkPayload.length; + long chunkLength = chunkRequest.getPayloadLength(); + boolean isFinal = chunkRequest.isFinal(); ApiFutures.addCallback( retryingFuture, new ApiFutureCallback>() { diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/RewindableStreamBuffer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/RewindableStreamBuffer.java new file mode 100644 index 000000000000..24a27dafabcf --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/RewindableStreamBuffer.java @@ -0,0 +1,157 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import org.jspecify.annotations.NullMarked; + +/** + * Manages a single-chunk buffer over an {@link InputStream} for resumable uploads. + * + *

The buffer holds at most one chunk of data in a reused backing array. It supports forward + * compaction and topping up upon recovery realignment, and enforces the boundary condition that + * requests to rewind before the buffer's base offset fail with an unrecoverable {@link + * FailedPreconditionException}. + */ +@NullMarked +final class RewindableStreamBuffer { + + private final InputStream inputStream; + private final int chunkSize; + private final String uploadUrl; + private final byte[] buffer; + + private long bufferBaseOffset; + private int payloadLength; + private boolean isFinal; + private boolean streamExhausted; + + RewindableStreamBuffer(InputStream inputStream, int chunkSize, String uploadUrl) { + this.inputStream = checkNotNull(inputStream, "inputStream must not be null"); + checkArgument(chunkSize > 0, "chunkSize must be > 0"); + this.chunkSize = chunkSize; + this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null"); + this.buffer = new byte[chunkSize]; + this.bufferBaseOffset = 0L; + this.payloadLength = 0; + this.isFinal = false; + this.streamExhausted = false; + } + + /** + * Advances the buffer from the stream starting at {@code targetOffset}, reading up to chunk size. + * + * @param targetOffset the absolute stream offset corresponding to the start of this chunk + * @throws IOException if reading from the stream fails + */ + void fill(long targetOffset) throws IOException { + this.bufferBaseOffset = targetOffset; + this.payloadLength = ByteStreams.read(inputStream, buffer, 0, chunkSize); + this.isFinal = (payloadLength < chunkSize); + if (this.isFinal) { + this.streamExhausted = true; + } + } + + /** + * Realigns the buffer window to {@code committedOffset}. + * + *

Compacts forward within the existing buffer to discard already-committed bytes, and then + * tops up the buffer to capacity from the underlying stream. + * + * @param committedOffset the server's committed byte offset + * @throws FailedPreconditionException if {@code committedOffset} is below the buffer's base + * offset or beyond the current buffer window + * @throws IOException if reading from the stream fails + */ + void realignTo(long committedOffset) throws IOException { + if (committedOffset < bufferBaseOffset) { + throw UploadErrors.protocolViolation( + String.format( + "Server committed offset %d is below buffer base offset %d for upload URL %s; cannot" + + " rewind stream before buffer base", + committedOffset, bufferBaseOffset, uploadUrl)); + } + + if (committedOffset > bufferBaseOffset + payloadLength) { + throw UploadErrors.protocolViolation( + String.format( + "Server committed offset %d is beyond current buffer window [%d, %d] for upload URL" + + " %s", + committedOffset, bufferBaseOffset, bufferBaseOffset + payloadLength, uploadUrl)); + } + + int committedWithinBuffer = (int) (committedOffset - bufferBaseOffset); + int remainingBytes = payloadLength - committedWithinBuffer; + + if (remainingBytes > 0 && committedWithinBuffer > 0) { + System.arraycopy(buffer, committedWithinBuffer, buffer, 0, remainingBytes); + } + + this.bufferBaseOffset = committedOffset; + this.payloadLength = remainingBytes; + + if (!streamExhausted && payloadLength < chunkSize) { + int space = chunkSize - payloadLength; + int additionalRead = ByteStreams.read(inputStream, buffer, payloadLength, space); + payloadLength += additionalRead; + if (additionalRead < space) { + streamExhausted = true; + } + } + + this.isFinal = streamExhausted; + } + + byte[] getBuffer() { + return buffer; + } + + int getPayloadLength() { + return payloadLength; + } + + long getBufferBaseOffset() { + return bufferBaseOffset; + } + + boolean isFinal() { + return isFinal; + } + + boolean isEmpty() { + return payloadLength == 0; + } +} diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UploadErrors.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UploadErrors.java new file mode 100644 index 000000000000..d86699f23371 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UploadErrors.java @@ -0,0 +1,70 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** Package-private error singletons and factory methods for resumable upload failure paths. */ +@NullMarked +final class UploadErrors { + + static final StatusCode TIMEOUT_STATUS_CODE = + new StatusCode() { + @Override + public StatusCode.Code getCode() { + return StatusCode.Code.DEADLINE_EXCEEDED; + } + + @Override + public @Nullable Object getTransportCode() { + return null; + } + }; + + static final StatusCode FAILED_PRECONDITION_STATUS_CODE = + new StatusCode() { + @Override + public StatusCode.Code getCode() { + return StatusCode.Code.FAILED_PRECONDITION; + } + + @Override + public @Nullable Object getTransportCode() { + return null; + } + }; + + private UploadErrors() {} + + static FailedPreconditionException protocolViolation(String message) { + return new FailedPreconditionException(message, null, FAILED_PRECONDITION_STATUS_CODE, false); + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/resumable/ChunkUploadRequestTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/resumable/ChunkUploadRequestTest.java index a3f7f4d9a15b..d5e9e2f9136e 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/resumable/ChunkUploadRequestTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/resumable/ChunkUploadRequestTest.java @@ -63,4 +63,55 @@ void builder_explicitIsFinalTrue_preservesValue() { assertThat(request.isFinal()).isTrue(); assertThat(request.getPayload()).isEmpty(); } + + @Test + void builder_defaultsPayloadLengthToArrayLength() { + byte[] payload = "test-payload".getBytes(StandardCharsets.UTF_8); + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.example.com/session/1") + .setPayload(payload) + .setOffset(0L) + .build(); + + assertThat(request.getPayloadLength()).isEqualTo(payload.length); + } + + @Test + void builder_customPayloadLength_preservesValue() { + byte[] payload = new byte[1024]; + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.example.com/session/1") + .setPayload(payload) + .setPayloadLength(500) + .setOffset(2048L) + .build(); + + assertThat(request.getPayloadLength()).isEqualTo(500); + } + + @Test + void builder_invalidBounds_throwsIllegalArgumentException() { + byte[] payload = new byte[100]; + org.junit.jupiter.api.Assertions.assertThrows( + IllegalArgumentException.class, + () -> + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.example.com/session/1") + .setPayload(payload) + .setOffset(0L) + .setPayloadLength(-1) + .build()); + + org.junit.jupiter.api.Assertions.assertThrows( + IllegalArgumentException.class, + () -> + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.example.com/session/1") + .setPayload(payload) + .setOffset(0L) + .setPayloadLength(150) + .build()); + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java index 469ab676fbb3..bd23b302f158 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java @@ -51,6 +51,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; @@ -365,9 +366,11 @@ void testChunkRetry_cat1FailureThenSuccess_retriesAndSucceeds() throws Exception verify(mockChunkCallable, times(2)).futureCall(captor.capture(), any()); List requests = captor.getAllValues(); assertThat(requests.get(0).getOffset()).isEqualTo(0); - assertThat(requests.get(0).getPayload()).isEqualTo("hello".getBytes(StandardCharsets.UTF_8)); + assertThat(Arrays.copyOf(requests.get(0).getPayload(), requests.get(0).getPayloadLength())) + .isEqualTo("hello".getBytes(StandardCharsets.UTF_8)); assertThat(requests.get(1).getOffset()).isEqualTo(0); - assertThat(requests.get(1).getPayload()).isEqualTo("hello".getBytes(StandardCharsets.UTF_8)); + assertThat(Arrays.copyOf(requests.get(1).getPayload(), requests.get(1).getPayloadLength())) + .isEqualTo("hello".getBytes(StandardCharsets.UTF_8)); } @Test @@ -517,6 +520,49 @@ void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() { verify(mockChunkCallable, times(1)).futureCall(any(), any()); } + @Test + void testBufferWindow_noArrayCopyForPartialChunk_backingArrayIdentityPreserved() + throws Exception { + stubStartSession("https://upload.url/partial-no-copy"); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "ok"))); + + // 5 bytes with default chunkSize = 8 -> partial chunk + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("12345"), null); + + assertThat(future.get()).isEqualTo("ok"); + ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); + verify(mockChunkCallable).futureCall(captor.capture(), any()); + ChunkUploadRequest chunk = captor.getValue(); + assertThat(chunk.getPayloadLength()).isEqualTo(5); + // Backing array capacity is 8 (chunkSize), not 5 (no copy performed) + assertThat(chunk.getPayload().length).isEqualTo(8); + } + + @Test + void testBufferWindow_reusesSingleArrayAcrossChunks() throws Exception { + stubStartSession("https://upload.url/reuse-array"); + // 20 bytes with chunkSize = 8 -> 3 chunks: [0..8), [8..16), [16..20) + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null))) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null))) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "done"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("01234567890123456789"), null); + + assertThat(future.get()).isEqualTo("done"); + ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); + verify(mockChunkCallable, times(3)).futureCall(captor.capture(), any()); + List chunks = captor.getAllValues(); + + // All chunks must reference the exact same backing byte[] instance + byte[] backingArray = chunks.get(0).getPayload(); + assertThat(chunks.get(1).getPayload()).isSameInstanceAs(backingArray); + assertThat(chunks.get(2).getPayload()).isSameInstanceAs(backingArray); + } + private static class HttpStatusStatusCode implements StatusCode { private final int httpStatus; private final StatusCode.Code code; @@ -580,7 +626,7 @@ private static InputStream streamOf(String content) { private static void assertChunk( ChunkUploadRequest chunk, long expectedOffset, int expectedSize, boolean expectedFinal) { assertThat(chunk.getOffset()).isEqualTo(expectedOffset); - assertThat(chunk.getPayload().length).isEqualTo(expectedSize); + assertThat(chunk.getPayloadLength()).isEqualTo(expectedSize); assertThat(chunk.isFinal()).isEqualTo(expectedFinal); } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RewindableStreamBufferTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RewindableStreamBufferTest.java new file mode 100644 index 000000000000..7b3ac1f7ce8a --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RewindableStreamBufferTest.java @@ -0,0 +1,278 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +class RewindableStreamBufferTest { + + private static final String UPLOAD_URL = "https://upload.example.com/session-1"; + + @Test + void testExactMultiplePayloads() throws IOException { + byte[] data = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); // 16 bytes, chunk size 8 + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + // Chunk 0: 8 bytes + buffer.fill(0L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(buffer.isFinal()).isFalse(); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("01234567"); + + // Chunk 1: 8 bytes + buffer.fill(8L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(8L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(buffer.isFinal()).isFalse(); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("89abcdef"); + + // Final 0-byte finalize chunk + buffer.fill(16L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(16L); + assertThat(buffer.getPayloadLength()).isEqualTo(0); + assertThat(buffer.isFinal()).isTrue(); + assertThat(buffer.isEmpty()).isTrue(); + } + + @Test + void testShortFinalChunk() throws IOException { + byte[] data = "short".getBytes(StandardCharsets.UTF_8); // 5 bytes, chunk size 8 + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + buffer.fill(0L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(5); + assertThat(buffer.isFinal()).isTrue(); + assertThat(buffer.isEmpty()).isFalse(); + assertThat(new String(buffer.getBuffer(), 0, 5, StandardCharsets.UTF_8)).isEqualTo("short"); + } + + @Test + void testZeroBytePayload() throws IOException { + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(new byte[0]), 8, UPLOAD_URL); + + buffer.fill(0L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(0); + assertThat(buffer.isFinal()).isTrue(); + assertThat(buffer.isEmpty()).isTrue(); + } + + @Test + void testRealignToMidBufferOffset_compactsAndTopsUp() throws IOException { + // 20 bytes: chunk size 8 + byte[] data = "0123456789ABCDEFGHIJ".getBytes(StandardCharsets.UTF_8); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + // Initial fill: "01234567" (bytes 0..7) + buffer.fill(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("01234567"); + + // Server committed 5 bytes (0..4), so next committed offset is 5. + // Remaining uncommitted bytes in buffer: "567" (3 bytes). + // realignTo(5) compacts "567" to buffer[0..3) and tops up 5 more bytes ("89ABC") from stream. + buffer.realignTo(5L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(5L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); // 3 remaining + 5 topped up = 8 + assertThat(buffer.isFinal()).isFalse(); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("56789ABC"); + } + + @Test + void testRealignToBufferBaseOffset_isNoOp() throws IOException { + byte[] data = "0123456789".getBytes(StandardCharsets.UTF_8); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + buffer.fill(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("01234567"); + + // Realigning to exactly the buffer base offset (0) is a no-op + buffer.realignTo(0L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("01234567"); + } + + @Test + void testRealignToBelowBaseOffset_throwsFailedPreconditionException_classifiedFatal() + throws IOException { + byte[] data = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + // Advanced to chunk 1 (base offset 8) + buffer.fill(8L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(8L); + + // Server requests offset 4, which is below buffer base offset 8 + FailedPreconditionException exception = + assertThrows(FailedPreconditionException.class, () -> buffer.realignTo(4L)); + + assertThat(exception.getMessage()).contains("4"); + assertThat(exception.getMessage()).contains("8"); + assertThat(exception.getMessage()).contains(UPLOAD_URL); + + // Must be classified as FATAL by ResumableUploadErrorClassifier + ResumableUploadErrorClassifier.Category category = + ResumableUploadErrorClassifier.classify(exception, ResumableUploadCommand.UPLOAD); + assertThat(category).isEqualTo(ResumableUploadErrorClassifier.Category.FATAL); + } + + @Test + void testRealignToBeyondBufferWindow_throwsFailedPreconditionException_classifiedFatal() + throws IOException { + byte[] data = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + // Initial fill at 0: window is [0, 8] + buffer.fill(0L); + assertThat(buffer.getBufferBaseOffset()).isEqualTo(0L); + assertThat(buffer.getPayloadLength()).isEqualTo(8); + + // Server reports committed offset 10, which is beyond current buffer window [0, 8] + FailedPreconditionException exception = + assertThrows(FailedPreconditionException.class, () -> buffer.realignTo(10L)); + + assertThat(exception.getMessage()).contains("10"); + assertThat(exception.getMessage()).contains("8"); + assertThat(exception.getMessage()).contains(UPLOAD_URL); + + // Must be classified as FATAL by ResumableUploadErrorClassifier + ResumableUploadErrorClassifier.Category category = + ResumableUploadErrorClassifier.classify(exception, ResumableUploadCommand.UPLOAD); + assertThat(category).isEqualTo(ResumableUploadErrorClassifier.Category.FATAL); + } + + @Test + void testBufferNeverCallsMarkOrResetOnStream() throws IOException { + class MarkCountingInputStream extends FilterInputStream { + int markCount = 0; + int resetCount = 0; + + MarkCountingInputStream(InputStream in) { + super(in); + } + + @Override + public synchronized void mark(int readlimit) { + markCount++; + super.mark(readlimit); + } + + @Override + public synchronized void reset() throws IOException { + resetCount++; + super.reset(); + } + } + + byte[] data = "0123456789ABCDEF".getBytes(StandardCharsets.UTF_8); + MarkCountingInputStream countingStream = + new MarkCountingInputStream(new ByteArrayInputStream(data)); + RewindableStreamBuffer buffer = new RewindableStreamBuffer(countingStream, 8, UPLOAD_URL); + + buffer.fill(0L); + buffer.realignTo(4L); + buffer.fill(12L); + + assertThat(countingStream.markCount).isEqualTo(0); + assertThat(countingStream.resetCount).isEqualTo(0); + } + + @Test + void testFillWithShortReads_greedilyFillsBufferToCapacity() throws IOException { + byte[] data = "01234567".getBytes(StandardCharsets.UTF_8); // 8 bytes + // Stream that yields at most 2 bytes per read + InputStream shortReadingStream = + new FilterInputStream(new ByteArrayInputStream(data)) { + @Override + public int read(byte[] b, int off, int len) throws IOException { + return super.read(b, off, Math.min(len, 2)); + } + }; + + RewindableStreamBuffer buffer = new RewindableStreamBuffer(shortReadingStream, 8, UPLOAD_URL); + buffer.fill(0L); + + // Must greedily fill all 8 bytes despite short reads, and not be marked final yet + assertThat(buffer.getPayloadLength()).isEqualTo(8); + assertThat(buffer.isFinal()).isFalse(); + assertThat(new String(buffer.getBuffer(), 0, 8, StandardCharsets.UTF_8)).isEqualTo("01234567"); + } + + @Test + void testNoArrayCopyForPartialChunk_backingArrayIdentityPreserved() throws IOException { + byte[] data = "small".getBytes(StandardCharsets.UTF_8); // 5 bytes + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 10, UPLOAD_URL); + + byte[] backingArray = buffer.getBuffer(); + buffer.fill(0L); + + // Backing array reference identity must be preserved (no copy on partial read) + assertThat(buffer.getBuffer()).isSameInstanceAs(backingArray); + assertThat(buffer.getPayloadLength()).isEqualTo(5); + } + + @Test + void testReusesSingleArrayAcrossChunks() throws IOException { + byte[] data = "0123456789abcdefghij".getBytes(StandardCharsets.UTF_8); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(new ByteArrayInputStream(data), 8, UPLOAD_URL); + + byte[] initialArray = buffer.getBuffer(); + + buffer.fill(0L); + assertThat(buffer.getBuffer()).isSameInstanceAs(initialArray); + + buffer.fill(8L); + assertThat(buffer.getBuffer()).isSameInstanceAs(initialArray); + + buffer.fill(16L); + assertThat(buffer.getBuffer()).isSameInstanceAs(initialArray); + } +}