diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadResponse.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadResponse.java index 76ba857d5d35..765d18ce0b8d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadResponse.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/resumable/ChunkUploadResponse.java @@ -69,7 +69,7 @@ public static Builder newBuilder() { public static ChunkUploadResponse create( boolean isComplete, @Nullable ResponseT response) { - return create(isComplete, response, null); + return create(isComplete, response, isComplete ? "final" : "active"); } public static ChunkUploadResponse create( diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ChunkAttemptCallable.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ChunkAttemptCallable.java new file mode 100644 index 000000000000..e836f2b1aacd --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ChunkAttemptCallable.java @@ -0,0 +1,301 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; +import com.google.api.gax.resumable.ChunkUploadRequest; +import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.common.util.concurrent.MoreExecutors; +import java.time.Duration; +import java.util.concurrent.Callable; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * A {@link Callable} representing an attempt to transmit a single chunk in a resumable upload + * session. Used with {@link com.google.api.gax.retrying.ScheduledRetryingExecutor}. + * + *

Execution follows the standard attempt template with pre-attempt recovery handling. When the + * previous attempt failed with a Category 2 (recoverable) error or missing status header, {@code + * prepareAttempt} queries session status, realigns the buffer window, tops up from the stream, and + * dispatches the chunk upload request. The callable never blocks on {@code .get()}; results and + * cancellations propagate asynchronously. + * + * @param the type of the final response message once the upload completes + */ +@NullMarked +class ChunkAttemptCallable implements Callable> { + + private final UnaryCallable> + uploadChunkCallable; + private final UnaryCallable> + queryStatusCallable; + private final RewindableStreamBuffer buffer; + private final String uploadUrl; + private final ApiCallContext originalCallContext; + + private volatile ChunkUploadRequest currentRequest; + private volatile ResumableUploadCommand currentCommand; + + private volatile @Nullable RetryingFuture> retryingFuture; + private volatile @Nullable ApiFuture inFlightFuture; + private volatile @Nullable Throwable lastFailure; + private volatile @Nullable ChunkUploadResponse lastResponse; + + ChunkAttemptCallable( + UnaryCallable> uploadChunkCallable, + UnaryCallable> queryStatusCallable, + RewindableStreamBuffer buffer, + String uploadUrl, + ChunkUploadRequest request, + ApiCallContext callContext, + ResumableUploadCommand command) { + this.uploadChunkCallable = + checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null"); + this.queryStatusCallable = + checkNotNull(queryStatusCallable, "queryStatusCallable must not be null"); + this.buffer = checkNotNull(buffer, "buffer must not be null"); + this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null"); + this.currentRequest = checkNotNull(request, "request must not be null"); + this.originalCallContext = checkNotNull(callContext, "callContext must not be null"); + this.currentCommand = checkNotNull(command, "command must not be null"); + } + + void setRetryingFuture(RetryingFuture> retryingFuture) { + this.retryingFuture = checkNotNull(retryingFuture, "retryingFuture must not be null"); + } + + private boolean needsRecovery() { + if (lastFailure != null) { + ResumableUploadErrorClassifier.Category category = + ResumableUploadErrorClassifier.classify(lastFailure, currentCommand); + return category == ResumableUploadErrorClassifier.Category.RECOVERABLE; + } + if (lastResponse != null && lastResponse.getUploadStatus() == null) { + ResumableUploadErrorClassifier.Category category = + ResumableUploadErrorClassifier.classifyMissingStatusHeader(currentCommand); + return category == ResumableUploadErrorClassifier.Category.RECOVERABLE; + } + return false; + } + + private void failAttempt( + SettableApiFuture> attemptFuture, Throwable t) { + lastFailure = t; + lastResponse = null; + attemptFuture.setException(t); + } + + /** + * Pre-attempt recovery step invoked before transmitting an attempt when the previous attempt + * encountered a Category 2 (recoverable) error or missing status header. + */ + private void prepareAttempt( + SettableApiFuture> attemptFuture, + ApiCallContext attemptContext, + RetryingFuture> currentRetryingFuture) { + QueryStatusRequest queryRequest = QueryStatusRequest.create(uploadUrl); + ApiFuture> queryFuture = + queryStatusCallable.futureCall(queryRequest, attemptContext); + if (queryFuture == null) { + failAttempt( + attemptFuture, new IllegalStateException("queryStatusCallable returned a null future")); + return; + } + this.inFlightFuture = queryFuture; + + ApiFutures.addCallback( + queryFuture, + new ApiFutureCallback>() { + @Override + public void onSuccess(QueryStatusResponse queryResponse) { + handleQuerySuccess(queryResponse, attemptFuture, attemptContext, currentRetryingFuture); + } + + @Override + public void onFailure(Throwable t) { + failAttempt(attemptFuture, t); + } + }, + MoreExecutors.directExecutor()); + } + + private void handleQuerySuccess( + QueryStatusResponse queryResponse, + SettableApiFuture> attemptFuture, + ApiCallContext attemptContext, + RetryingFuture> currentRetryingFuture) { + if (currentRetryingFuture.isDone()) { + return; + } + + if (queryResponse.getUploadStatus() == null) { + failAttempt( + attemptFuture, + UploadErrors.protocolViolation( + "Query status response missing X-Goog-Upload-Status header for upload URL: " + + uploadUrl)); + return; + } + + // Server already finalized the upload. + if (queryResponse.isComplete()) { + ChunkUploadResponse response = + ChunkUploadResponse.create( + true, queryResponse.getResponse(), queryResponse.getUploadStatus()); + lastFailure = null; + lastResponse = response; + attemptFuture.set(response); + return; + } + + // Incomplete query response with null committed offset violates the protocol invariant. + Long committedOffset = queryResponse.getCommittedOffset(); + if (committedOffset == null) { + failAttempt( + attemptFuture, + UploadErrors.protocolViolation( + "Incomplete query status response did not include a committed offset for upload URL: " + + uploadUrl)); + return; + } + + // Normal path: realign buffer to committedOffset, compact and top up. + try { + buffer.realignTo(committedOffset); + } catch (Throwable e) { + failAttempt(attemptFuture, e); + return; + } + + // Determine the upload command for the realigned buffer. + // Preserve upload,finalize for a trailing partial after realignment. + ResumableUploadCommand realignedCommand; + if (buffer.isFinal()) { + realignedCommand = + buffer.isEmpty() + ? ResumableUploadCommand.FINALIZE + : ResumableUploadCommand.UPLOAD_FINALIZE; + } else { + realignedCommand = ResumableUploadCommand.UPLOAD; + } + + ChunkUploadRequest realignedRequest = + ChunkUploadRequest.newBuilder() + .setUploadUrl(uploadUrl) + .setPayload(buffer.getBuffer()) + .setPayloadLength(buffer.getPayloadLength()) + .setOffset(buffer.getBufferBaseOffset()) + .setFinal(buffer.isFinal()) + .build(); + + this.currentRequest = realignedRequest; + this.currentCommand = realignedCommand; + + dispatchChunkUpload(attemptFuture, attemptContext, currentRetryingFuture); + } + + private void dispatchChunkUpload( + SettableApiFuture> attemptFuture, + ApiCallContext attemptContext, + RetryingFuture> currentRetryingFuture) { + ApiFuture> chunkFuture = + uploadChunkCallable.futureCall(currentRequest, attemptContext); + this.inFlightFuture = chunkFuture; + + ApiFutures.addCallback( + chunkFuture, + new ApiFutureCallback>() { + @Override + public void onSuccess(ChunkUploadResponse response) { + lastFailure = null; + lastResponse = response; + attemptFuture.set(response); + } + + @Override + public void onFailure(Throwable t) { + failAttempt(attemptFuture, t); + } + }, + MoreExecutors.directExecutor()); + } + + @Override + public @Nullable ChunkUploadResponse call() { + RetryingFuture> currentRetryingFuture = + checkNotNull(retryingFuture, "retryingFuture must be set before call()"); + ApiCallContext attemptContext = originalCallContext; + + Duration rpcTimeout = currentRetryingFuture.getAttemptSettings().getRpcTimeoutDuration(); + if (!rpcTimeout.isZero() && attemptContext.getTimeoutDuration() == null) { + attemptContext = attemptContext.withTimeoutDuration(rpcTimeout); + } + + SettableApiFuture> attemptFuture = SettableApiFuture.create(); + currentRetryingFuture.setAttemptFuture(attemptFuture); + + if (currentRetryingFuture.isDone()) { + return null; + } + + currentRetryingFuture.addListener( + () -> { + if (currentRetryingFuture.isCancelled()) { + ApiFuture inFlight = inFlightFuture; + if (inFlight != null) { + inFlight.cancel(true); + } + attemptFuture.cancel(true); + } + }, + MoreExecutors.directExecutor()); + + try { + if (needsRecovery()) { + prepareAttempt(attemptFuture, attemptContext, currentRetryingFuture); + } else { + dispatchChunkUpload(attemptFuture, attemptContext, currentRetryingFuture); + } + } catch (Throwable t) { + failAttempt(attemptFuture, t); + } + + return null; + } +} diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java index dbb63b0184cf..8069f34bec13 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java @@ -35,9 +35,16 @@ import com.google.api.core.ApiFutures; import com.google.api.core.BetaApi; import com.google.api.core.InternalApi; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadClient; import com.google.api.gax.resumable.ResumableUploadSession; +import com.google.api.gax.retrying.ExponentialRetryAlgorithm; +import com.google.api.gax.retrying.RetryAlgorithm; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.retrying.ScheduledRetryingExecutor; import java.io.InputStream; +import java.time.Duration; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -54,9 +61,22 @@ public class ResumableUploadCallableImpl extends ResumableUploadCallable { + static final RetrySettings DEFAULT_QUERY_RETRY_SETTINGS = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(1.3) + .setMaxRetryDelayDuration(Duration.ofMinutes(1)) + .setInitialRpcTimeoutDuration(Duration.ofSeconds(30)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeoutDuration(Duration.ofSeconds(30)) + .setTotalTimeoutDuration(Duration.ofMinutes(5)) + .build(); + private final ResumableUploadClient client; private final ResumableUploadCallSettings defaultCallSettings; private final ClientContext clientContext; + private final UnaryCallable> + retryingQueryCallable; public ResumableUploadCallableImpl( ResumableUploadClient client, @@ -66,6 +86,17 @@ public ResumableUploadCallableImpl( this.defaultCallSettings = checkNotNull(defaultCallSettings, "defaultCallSettings must not be null"); this.clientContext = checkNotNull(clientContext, "clientContext must not be null"); + + RetryAlgorithm> queryRetryAlgorithm = + new RetryAlgorithm<>( + new ResumableUploadResultRetryAlgorithm<>(ResumableUploadCommand.QUERY), + new ExponentialRetryAlgorithm(DEFAULT_QUERY_RETRY_SETTINGS, clientContext.getClock())); + + this.retryingQueryCallable = + new RetryingCallable<>( + clientContext.getDefaultCallContext(), + checkNotNull(client.queryStatusCallable(), "queryStatusCallable must not be null"), + new ScheduledRetryingExecutor<>(queryRetryAlgorithm, clientContext.getExecutor())); } @Override @@ -89,6 +120,7 @@ public ResumableUploadFuture futureCall( return ResumableUploadFutureImpl.create( startFuture, client.uploadChunkCallable(), + retryingQueryCallable, payload, effectiveSettings, clientContext.getDefaultCallContext(), diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java index f34f6827bd08..a74b6964cfb7 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java @@ -38,6 +38,8 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.resumable.ChunkUploadRequest; import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetryAlgorithm; @@ -81,8 +83,11 @@ final class ResumableUploadChunkCoordinator { private final SettableApiFuture result; private final ApiFuture startFuture; - private final RetryingCallable> - retryingChunkCallable; + private final UnaryCallable> + uploadChunkCallable; + private final UnaryCallable> + queryStatusCallable; + private final ScheduledRetryingExecutor> retryingExecutor; private final InputStream payload; private final int chunkSize; private final ApiCallContext callContext; @@ -105,13 +110,17 @@ final class ResumableUploadChunkCoordinator { SettableApiFuture result, ApiFuture startFuture, UnaryCallable> uploadChunkCallable, + UnaryCallable> queryStatusCallable, InputStream payload, ResumableUploadCallSettings settings, ApiCallContext callContext, ClientContext clientContext) { this.result = checkNotNull(result, "result must not be null"); this.startFuture = checkNotNull(startFuture, "startFuture must not be null"); - checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null"); + this.uploadChunkCallable = + checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null"); + this.queryStatusCallable = + checkNotNull(queryStatusCallable, "queryStatusCallable must not be null"); this.payload = checkNotNull(payload, "payload must not be null"); checkNotNull(settings, "settings must not be null"); checkArgument(settings.getChunkSize() > 0, "chunkSize must be > 0"); @@ -124,11 +133,8 @@ final class ResumableUploadChunkCoordinator { new RetryAlgorithm<>( new ResumableUploadResultRetryAlgorithm<>(ResumableUploadCommand.UPLOAD), new ExponentialRetryAlgorithm(chunkRetrySettings, clientContext.getClock())); - ScheduledRetryingExecutor> retryingExecutor = + this.retryingExecutor = new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); - this.retryingChunkCallable = - new RetryingCallable<>( - clientContext.getDefaultCallContext(), uploadChunkCallable, retryingExecutor); synchronized (lock) { this.inFlightFuture = startFuture; @@ -273,6 +279,16 @@ private void transmitSingleChunk(long currentOffset) { return; } + ResumableUploadCommand command; + if (streamBuffer.isFinal()) { + command = + streamBuffer.isEmpty() + ? ResumableUploadCommand.FINALIZE + : ResumableUploadCommand.UPLOAD_FINALIZE; + } else { + command = ResumableUploadCommand.UPLOAD; + } + ChunkUploadRequest chunkRequest = ChunkUploadRequest.newBuilder() .setUploadUrl(url) @@ -282,12 +298,21 @@ private void transmitSingleChunk(long currentOffset) { .setFinal(streamBuffer.isFinal()) .build(); + ChunkAttemptCallable attemptCallable = + new ChunkAttemptCallable<>( + uploadChunkCallable, + queryStatusCallable, + streamBuffer, + url, + chunkRequest, + callContext, + command); + RetryingFuture> retryingFuture = - retryingChunkCallable.futureCall(chunkRequest, callContext); + retryingExecutor.createFuture(attemptCallable, callContext); + attemptCallable.setRetryingFuture(retryingFuture); setInFlightFuture(retryingFuture); - long chunkLength = chunkRequest.getPayloadLength(); - boolean isFinal = chunkRequest.isFinal(); ApiFutures.addCallback( retryingFuture, new ApiFutureCallback>() { @@ -298,10 +323,10 @@ public void onSuccess(ChunkUploadResponse response) { return; } } - long nextOffset = currentOffset + chunkLength; + long nextOffset = streamBuffer.getBufferBaseOffset() + streamBuffer.getPayloadLength(); if (response.isComplete()) { finish(response.getResponse(), null); - } else if (isFinal) { + } else if (streamBuffer.isFinal()) { finish( null, new IllegalStateException( @@ -322,5 +347,13 @@ public void onFailure(Throwable t) { } }, MoreExecutors.directExecutor()); + + try { + attemptCallable.call(); + } catch (Throwable t) { + if (!retryingFuture.isDone()) { + finish(null, t); + } + } } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java index 02435da45690..62454177e566 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java @@ -35,6 +35,8 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.resumable.ChunkUploadRequest; import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadSession; import java.io.InputStream; import java.util.concurrent.ExecutionException; @@ -58,21 +60,7 @@ final class ResumableUploadFutureImpl implements ResumableUploadFutur static ResumableUploadFutureImpl create( ApiFuture startFuture, UnaryCallable> uploadChunkCallable, - InputStream payload, - ResumableUploadCallSettings settings, - ApiCallContext callContext) { - return create( - startFuture, - uploadChunkCallable, - payload, - settings, - callContext, - ClientContext.newBuilder().setDefaultCallContext(callContext).build()); - } - - static ResumableUploadFutureImpl create( - ApiFuture startFuture, - UnaryCallable> uploadChunkCallable, + UnaryCallable> queryStatusCallable, InputStream payload, ResumableUploadCallSettings settings, ApiCallContext callContext, @@ -83,6 +71,7 @@ static ResumableUploadFutureImpl create( result, startFuture, uploadChunkCallable, + queryStatusCallable, payload, settings, callContext, diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithm.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithm.java index ab35c0e3a9f6..66e047b48eaf 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithm.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithm.java @@ -29,8 +29,10 @@ */ package com.google.api.gax.rpc; +import static com.google.api.gax.rpc.ResumableUploadErrorClassifier.Category.RECOVERABLE; import static com.google.api.gax.rpc.ResumableUploadErrorClassifier.Category.TRANSIENT; +import com.google.api.gax.resumable.ChunkUploadResponse; import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.rpc.ResumableUploadErrorClassifier.Category; import java.util.Objects; @@ -41,8 +43,9 @@ /** * An adapter that integrates {@link ResumableUploadErrorClassifier} into GAX retrying machinery. * - *

Only transient errors should retry with an identical request; other recoverable errors will - * need to query the upload server to determine the appropriate next request. + *

Retries transient errors with the identical request and recoverable errors via session query + * and buffer realignment. Responses lacking an upload status header also enter recovery per + * protocol specification. * * @param the response type of the upload attempt */ @@ -59,12 +62,18 @@ final class ResumableUploadResultRetryAlgorithm @Override public boolean shouldRetry( @Nullable Throwable previousThrowable, @Nullable ResponseT previousResponse) { - // Successful commands (null throwable) and cancellations should not retry. - if (previousThrowable == null || previousThrowable instanceof CancellationException) { + if (previousThrowable instanceof CancellationException) { return false; } - Category category = ResumableUploadErrorClassifier.classify(previousThrowable, command); - // Transient errors are retried directly with the identical request. - return category == TRANSIENT; + Category category = null; + if (previousThrowable != null) { + category = ResumableUploadErrorClassifier.classify(previousThrowable, command); + } else if (previousResponse instanceof ChunkUploadResponse) { + ChunkUploadResponse chunkResponse = (ChunkUploadResponse) previousResponse; + if (chunkResponse.getUploadStatus() == null) { + category = ResumableUploadErrorClassifier.classifyMissingStatusHeader(command); + } + } + return category == TRANSIENT || category == RECOVERABLE; } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/CallableTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/CallableTest.java index 745cb294bb7b..cc972b80adef 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/CallableTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/CallableTest.java @@ -214,6 +214,8 @@ void testWatched_usesJavaTimeMethods() { void testResumableUploadCallable() { ResumableUploadClient uploadClient = mock(ResumableUploadClient.class, Mockito.withSettings().withoutAnnotations()); + when(uploadClient.queryStatusCallable()) + .thenReturn(mock(UnaryCallable.class, Mockito.withSettings().withoutAnnotations())); ResumableUploadCallSettings settings = ResumableUploadCallSettings.newBuilder().setChunkSize(1024).build(); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ChunkAttemptCallableTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ChunkAttemptCallableTest.java new file mode 100644 index 000000000000..8ebbd20c172b --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ChunkAttemptCallableTest.java @@ -0,0 +1,310 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.core.SettableApiFuture; +import com.google.api.gax.resumable.ChunkUploadRequest; +import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.TimedAttemptSettings; +import com.google.api.gax.rpc.testing.FakeCallContext; +import java.io.ByteArrayInputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class ChunkAttemptCallableTest { + + private static final RetrySettings TEST_RETRY_SETTINGS = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(1.0) + .setMaxRetryDelayDuration(Duration.ofMillis(100)) + .setInitialRpcTimeoutDuration(Duration.ofSeconds(10)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeoutDuration(Duration.ofSeconds(10)) + .setTotalTimeoutDuration(Duration.ofMinutes(1)) + .setMaxAttempts(3) + .build(); + + private RetryingFuture> mockExternalFuture; + private TimedAttemptSettings attemptSettings; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + mockExternalFuture = mock(RetryingFuture.class, withSettings().withoutAnnotations()); + attemptSettings = + TimedAttemptSettings.newBuilder() + .setGlobalSettings(TEST_RETRY_SETTINGS) + .setAttemptCount(0) + .setOverallAttemptCount(0) + .setFirstAttemptStartTimeNanos(0) + .setRetryDelayDuration(Duration.ofSeconds(1)) + .setRandomizedRetryDelayDuration(Duration.ofSeconds(1)) + .setRpcTimeoutDuration(Duration.ZERO) + .build(); + when(mockExternalFuture.getAttemptSettings()).thenReturn(attemptSettings); + } + + @Test + @SuppressWarnings("unchecked") + void call_successfulChunk_setsAttemptFuture() throws Exception { + UnaryCallable> mockChunkCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable> mockQueryCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.url/test") + .setPayload(new byte[] {1, 2, 3}) + .setOffset(0L) + .setFinal(false) + .build(); + + ChunkUploadResponse expectedResponse = + ChunkUploadResponse.create(false, null, "active"); + ApiFuture> internalFuture = + ApiFutures.immediateFuture(expectedResponse); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn(internalFuture); + + ApiCallContext callContext = FakeCallContext.createDefault(); + ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(stream, 8, "https://upload.url/test"); + buffer.fill(0L); + + ChunkAttemptCallable callable = + new ChunkAttemptCallable<>( + mockChunkCallable, + mockQueryCallable, + buffer, + "https://upload.url/test", + request, + callContext, + ResumableUploadCommand.UPLOAD); + + callable.setRetryingFuture(mockExternalFuture); + ChunkUploadResponse callResult = callable.call(); + + // Call returns immediately without blocking + assertThat(callResult).isNull(); + verify(mockChunkCallable).futureCall(eq(request), any()); + ArgumentCaptor>> captor = + ArgumentCaptor.forClass(ApiFuture.class); + verify(mockExternalFuture).setAttemptFuture(captor.capture()); + assertThat(captor.getValue().get()).isEqualTo(expectedResponse); + } + + @Test + @SuppressWarnings("unchecked") + void call_returnsWithoutBlocking_andPropagatesCancellation() throws Exception { + UnaryCallable> mockChunkCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable> mockQueryCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + SettableApiFuture> internalFuture = SettableApiFuture.create(); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) + .thenReturn(internalFuture); + + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.url/test") + .setPayload(new byte[] {1, 2, 3}) + .setOffset(0L) + .setFinal(false) + .build(); + + ApiCallContext callContext = FakeCallContext.createDefault(); + ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(stream, 8, "https://upload.url/test"); + buffer.fill(0L); + + ChunkAttemptCallable callable = + new ChunkAttemptCallable<>( + mockChunkCallable, + mockQueryCallable, + buffer, + "https://upload.url/test", + request, + callContext, + ResumableUploadCommand.UPLOAD); + + List listeners = new ArrayList<>(); + doAnswer( + invocation -> { + listeners.add(invocation.getArgument(0)); + return null; + }) + .when(mockExternalFuture) + .addListener(any(), any()); + + callable.setRetryingFuture(mockExternalFuture); + + // Call returns immediately without blocking + assertThat(callable.call()).isNull(); + assertThat(internalFuture.isDone()).isFalse(); + + // Cancellation of external future propagates to internal future + when(mockExternalFuture.isCancelled()).thenReturn(true); + for (Runnable listener : listeners) { + listener.run(); + } + assertThat(internalFuture.isCancelled()).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + void call_perAttemptDeadline_appliesRpcTimeoutToCallContext() throws Exception { + UnaryCallable> mockChunkCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable> mockQueryCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + when(mockExternalFuture.getAttemptSettings()) + .thenReturn( + TimedAttemptSettings.newBuilder() + .setGlobalSettings(TEST_RETRY_SETTINGS) + .setAttemptCount(0) + .setOverallAttemptCount(0) + .setFirstAttemptStartTimeNanos(0) + .setRetryDelayDuration(Duration.ofSeconds(1)) + .setRandomizedRetryDelayDuration(Duration.ofSeconds(1)) + .setRpcTimeoutDuration(Duration.ofSeconds(15)) + .build()); + + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.url/test") + .setPayload(new byte[] {1, 2, 3}) + .setOffset(0L) + .setFinal(false) + .build(); + + ApiFuture> internalFuture = + ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, "active")); + ArgumentCaptor contextCaptor = ArgumentCaptor.forClass(ApiCallContext.class); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), contextCaptor.capture())) + .thenReturn(internalFuture); + + ApiCallContext callContext = FakeCallContext.createDefault(); + ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(stream, 8, "https://upload.url/test"); + buffer.fill(0L); + + ChunkAttemptCallable callable = + new ChunkAttemptCallable<>( + mockChunkCallable, + mockQueryCallable, + buffer, + "https://upload.url/test", + request, + callContext, + ResumableUploadCommand.UPLOAD); + + callable.setRetryingFuture(mockExternalFuture); + callable.call(); + + assertThat(contextCaptor.getValue().getTimeoutDuration()).isEqualTo(Duration.ofSeconds(15)); + } + + @Test + @SuppressWarnings("unchecked") + void call_nonBlockingExecution_callingThreadMakesImmediateProgress() throws Exception { + UnaryCallable> mockChunkCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable> mockQueryCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + SettableApiFuture> hungFuture = SettableApiFuture.create(); + when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())).thenReturn(hungFuture); + + ChunkUploadRequest request = + ChunkUploadRequest.newBuilder() + .setUploadUrl("https://upload.url/test") + .setPayload(new byte[] {1, 2, 3}) + .setOffset(0L) + .setFinal(false) + .build(); + + ApiCallContext callContext = FakeCallContext.createDefault(); + ByteArrayInputStream stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + RewindableStreamBuffer buffer = + new RewindableStreamBuffer(stream, 8, "https://upload.url/test"); + buffer.fill(0L); + + ChunkAttemptCallable callable = + new ChunkAttemptCallable<>( + mockChunkCallable, + mockQueryCallable, + buffer, + "https://upload.url/test", + request, + callContext, + ResumableUploadCommand.UPLOAD); + + callable.setRetryingFuture(mockExternalFuture); + + CountDownLatch threadCompleted = new CountDownLatch(1); + Thread callerThread = + new Thread( + () -> { + callable.call(); + threadCompleted.countDown(); + }); + callerThread.start(); + + assertThat(threadCompleted.await(1, TimeUnit.SECONDS)).isTrue(); + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java index bd23b302f158..17c420635553 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java @@ -44,6 +44,8 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.resumable.ChunkUploadRequest; import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadClient; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.api.gax.rpc.testing.FakeCallContext; @@ -56,7 +58,11 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -69,9 +75,11 @@ class ResumableUploadCallableImplTest { private ResumableUploadClient mockClient; private UnaryCallable mockStartCallable; private UnaryCallable> mockChunkCallable; + private UnaryCallable> mockQueryCallable; private ResumableUploadCallSettings defaultSettings; private FakeCallContext callContext; + private ScheduledExecutorService executor; private ResumableUploadCallableImpl callable; @BeforeEach @@ -80,17 +88,27 @@ void setUp() { mockClient = mock(ResumableUploadClient.class, withSettings().withoutAnnotations()); mockStartCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); mockChunkCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockQueryCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); lenient().when(mockClient.startUploadCallable()).thenReturn(mockStartCallable); lenient().when(mockClient.uploadChunkCallable()).thenReturn(mockChunkCallable); + lenient().when(mockClient.queryStatusCallable()).thenReturn(mockQueryCallable); defaultSettings = ResumableUploadCallSettings.newBuilder().setChunkSize(8).build(); callContext = FakeCallContext.createDefault(); + executor = Executors.newScheduledThreadPool(2); ClientContext clientContext = - ClientContext.newBuilder().setDefaultCallContext(callContext).build(); + ClientContext.newBuilder().setDefaultCallContext(callContext).setExecutor(executor).build(); callable = new ResumableUploadCallableImpl<>(mockClient, defaultSettings, clientContext); } + @AfterEach + void tearDown() { + if (executor != null) { + executor.shutdownNow(); + } + } + @Test void testUploadCallable_singleChunk_happyPath() throws Exception { stubStartSession("https://upload.url/single"); @@ -404,13 +422,13 @@ void testChunkRetry_streamNotAdvancedByRetry_sameBytesSent() throws Exception { } @Test - void testChunkRetry_cat2Failure_failsFastWithoutRetrying() { - stubStartSession("https://upload.url/chunk-cat2-fail"); - // HTTP 400 Bad Request is Category 2 (RECOVERABLE) + void testChunkRetry_cat3Failure_failsFastWithoutRetrying() { + stubStartSession("https://upload.url/chunk-cat3-fail"); + // HTTP 403 Forbidden is Category 3 (FATAL) when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) .thenReturn( ApiFutures.immediateFailedFuture( - createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + createApiException(403, StatusCode.Code.PERMISSION_DENIED))); ResumableUploadFuture future = callable.futureCall("resource-path", streamOf("hello"), null); @@ -418,9 +436,9 @@ void testChunkRetry_cat2Failure_failsFastWithoutRetrying() { ExecutionException exception = assertThrows(ExecutionException.class, future::get); assertThat(exception.getCause()).isInstanceOf(ApiException.class); assertThat(((ApiException) exception.getCause()).getStatusCode().getTransportCode()) - .isEqualTo(400); + .isEqualTo(403); - // In G4, Category 2 is not yet retryable / recoverable, so it fails after 1 attempt. + // Category 3 is fatal / non-retryable, so it fails after 1 attempt without retrying. verify(mockChunkCallable, times(1)).futureCall(any(), any()); } @@ -482,10 +500,6 @@ void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() { ClientContext clientContext = ClientContext.newBuilder().setDefaultCallContext(callContext).build(); - // Leave startFuture incomplete so sessionFuture does not automatically instantiate its - // coordinator - SettableApiFuture startFuture = SettableApiFuture.create(); - SettableApiFuture result = SettableApiFuture.create(); SettableApiFuture startSessionFuture = SettableApiFuture.create(); startSessionFuture.set( @@ -499,6 +513,7 @@ void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() { result, startSessionFuture, mockChunkCallable, + mockQueryCallable, streamOf("hello"), customSettings, callContext, @@ -520,6 +535,312 @@ void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() { verify(mockChunkCallable, times(1)).futureCall(any(), any()); } + private static QueryStatusResponse createQueryResponse( + boolean isComplete, + @Nullable Long committedOffset, + @Nullable String response, + @Nullable String uploadStatus) { + return QueryStatusResponse.newBuilder() + .setComplete(isComplete) + .setCommittedOffset(committedOffset) + .setResponse(response) + .setUploadStatus(uploadStatus) + .build(); + } + + @Test + void testRecovery_category2Error_recoversViaQueryAndSucceeds() throws Exception { + stubStartSession("https://upload.url/recovery-success"); + // Attempt 0 fails with 400 (Category 2) + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(true, "recovered-response", "final"))); + + // Query status returns active session with committed offset 0 + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 0L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + assertThat(future.get()).isEqualTo("recovered-response"); + assertThat(future.isDone()).isTrue(); + verify(mockQueryCallable, times(1)).futureCall(any(), any()); + verify(mockChunkCallable, times(2)).futureCall(any(), any()); + } + + @Test + void testRecovery_queryReturnsComplete_completesSessionWithoutResending() throws Exception { + stubStartSession("https://upload.url/recovery-already-complete"); + // Attempt 0 fails with 400 (Category 2) + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + + // Query status returns that the server already finalized the upload + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFuture(createQueryResponse(true, 5L, "server-finalized", "final"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + assertThat(future.get()).isEqualTo("server-finalized"); + assertThat(future.isDone()).isTrue(); + verify(mockQueryCallable, times(1)).futureCall(any(), any()); + // Only 1 chunk upload attempt happened; no resend occurred because the server was already + // complete + verify(mockChunkCallable, times(1)).futureCall(any(), any()); + } + + @Test + void testRecovery_queryReturnsIncompleteWithNullOffset_failsFatal() throws Exception { + stubStartSession("https://upload.url/recovery-null-offset"); + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + + // Incomplete response with null committed offset violates the protocol contract + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, null, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + ExecutionException exception = assertThrows(ExecutionException.class, future::get); + assertThat(exception.getCause()).isInstanceOf(FailedPreconditionException.class); + assertThat(exception.getCause()) + .hasMessageThat() + .contains("did not include a committed offset"); + verify(mockQueryCallable, times(1)).futureCall(any(), any()); + verify(mockChunkCallable, times(1)).futureCall(any(), any()); + } + + @Test + void testRecovery_serverCommittedAheadWithinBuffer_compactsAndTopsUp() throws Exception { + stubStartSession("https://upload.url/recovery-compact-topup"); + // 16-byte payload, chunkSize = 8 + // Attempt 0 for chunk 0 (bytes 0..7) fails with 400 + // Query returns committedOffset = 4 (server committed 4 bytes) + // Buffer realigns to offset 4: compacts [4..7] ("4567") and tops up from stream ("89ab") -> + // window [4..11] + // Attempt 1 for realigned chunk transmits 8 bytes (offset 4, len 8) and succeeds + // Chunk 2 (offset 12..15, "cdef") succeeds and finalizes + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, "active"))) + .thenReturn( + ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "all-done", "final"))); + + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 4L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("0123456789abcdef"), null); + + assertThat(future.get()).isEqualTo("all-done"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); + verify(mockChunkCallable, times(3)).futureCall(captor.capture(), any()); + List requests = captor.getAllValues(); + + // Request 0: initial chunk 0 (offset 0, length 8) + assertChunk(requests.get(0), 0, 8, false); + // Request 1: realigned chunk (offset 4, length 8) + assertChunk(requests.get(1), 4, 8, false); + // Request 2: final chunk (offset 12, length 4) + assertChunk(requests.get(2), 12, 4, true); + } + + @Test + void testRecovery_serverOffsetBehind_resendsFromOffset() throws Exception { + stubStartSession("https://upload.url/recovery-behind"); + // Chunk 0 (0..7) succeeds + // Chunk 1 (8..15) fails with 409 + // Query returns committed offset 10 (between base 8 and attempt 16) + // Buffer realigns to 10: compacts remaining 6 bytes, stream at EOF, isFinal remains true + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, "active"))) + .thenReturn( + ApiFutures.immediateFailedFuture(createApiException(409, StatusCode.Code.ABORTED))) + .thenReturn( + ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "resend-ok", "final"))); + + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 10L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("0123456789abcdef"), null); + + assertThat(future.get()).isEqualTo("resend-ok"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); + verify(mockChunkCallable, times(3)).futureCall(captor.capture(), any()); + List requests = captor.getAllValues(); + assertChunk(requests.get(0), 0, 8, false); + assertChunk(requests.get(1), 8, 8, false); + assertChunk(requests.get(2), 10, 6, true); + } + + @Test + void testRecovery_serverOffsetBelowBase_failsWithFatalFailedPrecondition() throws Exception { + stubStartSession("https://upload.url/recovery-below-base"); + // Chunk 0 (0..7) succeeds + // Chunk 1 (8..15) fails with 400 + // Query returns committed offset 4 (below buffer base of 8) + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, "active"))) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 4L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("0123456789abcdef"), null); + + ExecutionException exception = assertThrows(ExecutionException.class, future::get); + assertThat(exception.getCause()).isInstanceOf(FailedPreconditionException.class); + assertThat(exception.getCause()).hasMessageThat().contains("below buffer base offset"); + } + + @Test + void testRecovery_missingStatusHeaderOn200_triggersRecovery() throws Exception { + stubStartSession("https://upload.url/missing-status-200"); + // Attempt 0 succeeds with HTTP 200, but uploadStatus is null + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, null))) + .thenReturn( + ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "recovered-ok", "final"))); + + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 0L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + assertThat(future.get()).isEqualTo("recovered-ok"); + verify(mockQueryCallable, times(1)).futureCall(any(), any()); + verify(mockChunkCallable, times(2)).futureCall(any(), any()); + } + + @Test + void testRecovery_onFinalChunk_preservesUploadFinalize() throws Exception { + stubStartSession("https://upload.url/recovery-final-chunk"); + // 12 bytes with chunkSize = 8 -> chunk 0 is 8 bytes, chunk 1 is 4 bytes (trailing partial) + // Chunk 0 succeeds + // Chunk 1 attempt 0 (offset 8, len 4, isFinal true) fails with 400 + // Query returns committed offset 10 (mid-buffer within trailing partial) + // Buffer realigns to 10: remaining length 2 bytes, isFinal true + // Chunk 1 attempt 1 transmits offset 10, len 2, isFinal true and completes + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null, "active"))) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(true, "final-chunk-done", "final"))); + + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 10L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("0123456789ab"), null); + + assertThat(future.get()).isEqualTo("final-chunk-done"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); + verify(mockChunkCallable, times(3)).futureCall(captor.capture(), any()); + List requests = captor.getAllValues(); + assertChunk(requests.get(0), 0, 8, false); + assertChunk(requests.get(1), 8, 4, true); + assertChunk(requests.get(2), 10, 2, true); + } + + @Test + void testRecovery_queryReturnsMissingStatusHeader_failsFatalProtocolViolation() throws Exception { + stubStartSession("https://upload.url/recovery-missing-status-query"); + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + + // Query status response missing upload status header is a fatal protocol violation + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 0L, null, null))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + ExecutionException exception = assertThrows(ExecutionException.class, future::get); + assertThat(exception.getCause()).isInstanceOf(FailedPreconditionException.class); + assertThat(exception.getCause()) + .hasMessageThat() + .contains("missing X-Goog-Upload-Status header"); + verify(mockQueryCallable, times(1)).futureCall(any(), any()); + verify(mockChunkCallable, times(1)).futureCall(any(), any()); + } + + @Test + void testRecovery_category3ErrorOnQuery_isFatal() throws Exception { + stubStartSession("https://upload.url/recovery-query-cat3"); + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))); + + // Query status fails with 403 Forbidden (Category 3 / FATAL) + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(403, StatusCode.Code.PERMISSION_DENIED))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + ExecutionException exception = assertThrows(ExecutionException.class, future::get); + assertThat(exception.getCause()).isInstanceOf(ApiException.class); + assertThat(((ApiException) exception.getCause()).getStatusCode().getCode()) + .isEqualTo(StatusCode.Code.PERMISSION_DENIED); + // Query failed fatally, so no resend of chunk + verify(mockChunkCallable, times(1)).futureCall(any(), any()); + } + + @Test + void testRecovery_transientErrorOnQuery_isRetried() throws Exception { + stubStartSession("https://upload.url/recovery-query-transient"); + when(mockChunkCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture( + createApiException(400, StatusCode.Code.INVALID_ARGUMENT))) + .thenReturn( + ApiFutures.immediateFuture( + ChunkUploadResponse.create(true, "query-retry-ok", "final"))); + + // Query status fails first with 503 (transient), then succeeds + when(mockQueryCallable.futureCall(any(), any())) + .thenReturn( + ApiFutures.immediateFailedFuture(createApiException(503, StatusCode.Code.UNAVAILABLE))) + .thenReturn(ApiFutures.immediateFuture(createQueryResponse(false, 0L, null, "active"))); + + ResumableUploadFuture future = + callable.futureCall("resource-path", streamOf("hello"), null); + + assertThat(future.get()).isEqualTo("query-retry-ok"); + verify(mockQueryCallable, times(2)).futureCall(any(), any()); + verify(mockChunkCallable, times(2)).futureCall(any(), any()); + } + @Test void testBufferWindow_noArrayCopyForPartialChunk_backingArrayIdentityPreserved() throws Exception { @@ -540,29 +861,6 @@ void testBufferWindow_noArrayCopyForPartialChunk_backingArrayIdentityPreserved() assertThat(chunk.getPayload().length).isEqualTo(8); } - @Test - void testBufferWindow_reusesSingleArrayAcrossChunks() throws Exception { - stubStartSession("https://upload.url/reuse-array"); - // 20 bytes with chunkSize = 8 -> 3 chunks: [0..8), [8..16), [16..20) - when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) - .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null))) - .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(false, null))) - .thenReturn(ApiFutures.immediateFuture(ChunkUploadResponse.create(true, "done"))); - - ResumableUploadFuture future = - callable.futureCall("resource-path", streamOf("01234567890123456789"), null); - - assertThat(future.get()).isEqualTo("done"); - ArgumentCaptor captor = ArgumentCaptor.forClass(ChunkUploadRequest.class); - verify(mockChunkCallable, times(3)).futureCall(captor.capture(), any()); - List chunks = captor.getAllValues(); - - // All chunks must reference the exact same backing byte[] instance - byte[] backingArray = chunks.get(0).getPayload(); - assertThat(chunks.get(1).getPayload()).isSameInstanceAs(backingArray); - assertThat(chunks.get(2).getPayload()).isSameInstanceAs(backingArray); - } - private static class HttpStatusStatusCode implements StatusCode { private final int httpStatus; private final StatusCode.Code code; @@ -616,7 +914,10 @@ private void stubStartSession(String uploadUrl) { when(mockStartCallable.futureCall(any(), any())) .thenReturn( ApiFutures.immediateFuture( - ResumableUploadSession.newBuilder().setUploadUrl(uploadUrl).build())); + ResumableUploadSession.newBuilder() + .setUploadUrl(uploadUrl) + .setUploadStatus("active") + .build())); } private static InputStream streamOf(String content) { diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinatorTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinatorTest.java index bdc9c8f36f80..0ca00f757eb6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinatorTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinatorTest.java @@ -38,6 +38,8 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.resumable.ChunkUploadRequest; import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.api.gax.rpc.testing.FakeCallContext; import com.google.common.util.concurrent.MoreExecutors; @@ -58,6 +60,7 @@ class ResumableUploadChunkCoordinatorTest { private ClientContext clientContext; private ApiCallContext callContext; private UnaryCallable> mockChunkCallable; + private UnaryCallable> mockQueryCallable; private ResumableUploadCallSettings settings; @BeforeEach @@ -68,6 +71,7 @@ void setUp() { clientContext = ClientContext.newBuilder().setDefaultCallContext(callContext).setExecutor(executor).build(); mockChunkCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockQueryCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); settings = ResumableUploadCallSettings.newBuilder().setChunkSize(256).build(); } @@ -101,7 +105,14 @@ public void close() throws IOException { ResumableUploadChunkCoordinator coordinator = new ResumableUploadChunkCoordinator<>( - result, startFuture, mockChunkCallable, payload, settings, callContext, clientContext); + result, + startFuture, + mockChunkCallable, + mockQueryCallable, + payload, + settings, + callContext, + clientContext); AtomicInteger completionListenerCount = new AtomicInteger(0); result.addListener(completionListenerCount::incrementAndGet, MoreExecutors.directExecutor()); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadFutureImplTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadFutureImplTest.java index f5d7726a3d5c..71c3e04affe3 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadFutureImplTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadFutureImplTest.java @@ -39,6 +39,8 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.resumable.ChunkUploadRequest; import com.google.api.gax.resumable.ChunkUploadResponse; +import com.google.api.gax.resumable.QueryStatusRequest; +import com.google.api.gax.resumable.QueryStatusResponse; import com.google.api.gax.resumable.ResumableUploadSession; import com.google.api.gax.rpc.testing.FakeCallContext; import com.google.common.util.concurrent.MoreExecutors; @@ -59,11 +61,14 @@ private ResumableUploadChunkCoordinator createCoordinator( mock(UnaryCallable.class, withSettings().withoutAnnotations()); when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any())) .thenReturn(chunkFuture); + UnaryCallable> mockQueryCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); ResumableUploadCallSettings settings = ResumableUploadCallSettings.newBuilder().build(); return new ResumableUploadChunkCoordinator<>( result, startFuture, mockChunkCallable, + mockQueryCallable, new ByteArrayInputStream(new byte[] {1, 2, 3}), settings, clientContext.getDefaultCallContext(), diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithmTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithmTest.java index eb4f5edcf81d..c515306cbfc2 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithmTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadResultRetryAlgorithmTest.java @@ -29,10 +29,12 @@ */ package com.google.api.gax.rpc; +import static com.google.api.gax.rpc.ResumableUploadCommand.QUERY; import static com.google.api.gax.rpc.ResumableUploadCommand.UPLOAD; import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.google.api.gax.resumable.ChunkUploadResponse; import com.google.api.gax.rpc.StatusCode.Code; import java.io.IOException; import java.util.concurrent.CancellationException; @@ -91,14 +93,41 @@ void testShouldRetry_transientErrorReturnsTrue() { } @Test - void testShouldRetry_recoverableErrorReturnsFalse() { - // In G1/G4, only TRANSIENT is retried. RECOVERABLE errors fail the chunk attempt - // to trigger the query-status recovery loop rather than retrying blindly. + void testShouldRetry_recoverableErrorIsRetryable() { ResumableUploadResultRetryAlgorithm algorithm = new ResumableUploadResultRetryAlgorithm<>(UPLOAD); + ApiException recoverable400 = createApiException(400, Code.INVALID_ARGUMENT); + ApiException recoverable409 = createApiException(409, Code.ABORTED); ApiException recoverable412 = createApiException(412, Code.FAILED_PRECONDITION); - assertThat(algorithm.shouldRetry(recoverable412, null)).isFalse(); + + assertThat(algorithm.shouldRetry(recoverable400, null)).isTrue(); + assertThat(algorithm.shouldRetry(recoverable409, null)).isTrue(); + assertThat(algorithm.shouldRetry(recoverable412, null)).isTrue(); + } + + @Test + void testShouldRetry_missingStatusHeaderOnChunkResponse_returnsTrueForUpload() { + ResumableUploadResultRetryAlgorithm> algorithm = + new ResumableUploadResultRetryAlgorithm<>(UPLOAD); + + ChunkUploadResponse responseWithNullStatus = + ChunkUploadResponse.create(false, "payload", null); + assertThat(algorithm.shouldRetry(null, responseWithNullStatus)).isTrue(); + + ChunkUploadResponse responseWithActiveStatus = + ChunkUploadResponse.create(false, "payload", "active"); + assertThat(algorithm.shouldRetry(null, responseWithActiveStatus)).isFalse(); + } + + @Test + void testShouldRetry_missingStatusHeaderOnChunkResponse_returnsFalseForQuery() { + ResumableUploadResultRetryAlgorithm> algorithm = + new ResumableUploadResultRetryAlgorithm<>(QUERY); + + ChunkUploadResponse responseWithNullStatus = + ChunkUploadResponse.create(false, "payload", null); + assertThat(algorithm.shouldRetry(null, responseWithNullStatus)).isFalse(); } @Test